# common/management/commands/translation_tracking.py

from django.core.management.base import BaseCommand
from django.core.management import call_command
from common.translation_utils.update_status import update_status_run
from common.translation_utils.make_testing_po import make_testing_po_run
from common.translation_utils.make_pos import make_pos_run
import polib
import csv
import copy
import os

#This management function runs a series of functions related to the translation process, for detailed information see common/translation_utils/README

class Command(BaseCommand):
    help = 'Updates translation translation tracking sheets and PO files'

    def handle(self, *args, **options):
        #Extracts message content from codebase and stores in locale/xy/LC_MESSAGES
        call_command('makemessages', '-l' 'xy', '-d django', '--keep-pot')
        call_command('makemessages', '-l' 'xy', '-d djangojs', '--keep-pot')

        # Processes messages from locale/xy/LC_MESSAGES, looks through ‘translation_status.csv’ and ‘translations.csv’ 
        # and creates an up to date log of translation which are present and their status
        update_status_run()

        # Uses the excel files to create the necessary PO files, stores them in locale. 
        # Also creates a file ‘translatable_strings’ with translations that we need but don’t have
        make_testing_po_run()
        make_pos_run()

        os.remove('locale/django.pot')
        os.remove('locale/djangojs.pot')


# home/management/commands/check_multiple_translation_sources.py

from django.core.management import BaseCommand
from wagtail.core.models import Page
from wagtail_localize.models import TranslationSource


class Command(BaseCommand):
    def handle(self, *args, **options):
        for page in Page.objects.all():
            translation_sources = TranslationSource.objects.filter(
                object_id=page.translation_key,
                specific_content_type=page.content_type_id,
                translations__target_locale=page.locale,
            )

            if translation_sources.count() > 1:
                self.stdout.write(f"Multiple translation sources found for page: {page.id}, title: {page.title}")
                for translation_source in translation_sources:
                    self.stdout.write(f" - Translation source: {translation_source.id}, locale: {translation_source.locale}")


# home/management/commands/clear_svg_to_png_map.py

from django.core.management.base import BaseCommand

from home.models import SVGToPNGMap


class Command(BaseCommand):
    def handle(self, *args, **options):
        for map in SVGToPNGMap.objects.all():
            map.png_image_file.delete()
            map.delete()

        self.stdout.write(self.style.SUCCESS('SVG to PNG map cleared.'))


# home/management/commands/clear_user_data.py

from django.contrib.auth import get_user_model
from django.core.management import BaseCommand
from django_comments_xtd.models import XtdComment

from messaging.models import Thread
from questionnaires.models import UserSubmission


class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument(
            '--exclude-user-ids',
            nargs="+",
            required=False,
            type=int,
            help="IDs for users that wouldn't be cleared"
        )

    def handle(self, *args, **options):
        user_ids = options.get('exclude_user_ids') or []
        get_user_model().objects.exclude(pk__in=user_ids).delete()
        XtdComment.objects.all().delete()
        Thread.objects.all().delete()
        UserSubmission.objects.all().delete()
        self.stdout.write('User data cleared.')


# home/management/commands/create_default_site.py

from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site as DjangoSite
from django.core.management.base import BaseCommand
from wagtail.core.models import Site as WagtailSite

from home.models import HomePage

User = get_user_model()


class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            '--domain',
            help='The domain on which the website is deployed'
        )
        parser.add_argument(
            '--port',
            type=int,
            help='The port on which the website is deployed'
        )

    def handle(self, *args, **options):
        domain = options.get('domain') or 'localhost'
        port = options.get('port') or 8000

        __, created = DjangoSite.objects.update_or_create(pk=settings.SITE_ID, defaults={
            'domain': domain,
            'name': 'default'
        })
        self.stdout.write(self.style.SUCCESS(f'Django Site object {"created" if created else "updated"}'))
        __, created = WagtailSite.objects.update_or_create(is_default_site=True, defaults={
            'hostname': domain,
            'port': port,
            'site_name': 'default',
            'root_page': HomePage.objects.all().order_by('id').first()
        })
        self.stdout.write(self.style.SUCCESS(f'Wagtail Site object {"created" if created else "updated"}'))


# home/management/commands/create_index_pages.py

from django.contrib.contenttypes.models import ContentType
from django.core.management.base import BaseCommand

from home.models import HomePage, BannerIndexPage, FooterIndexPage, SectionIndexPage


class Command(BaseCommand):

    def handle(self, *args, **options):
        homepage_content_type, __ = ContentType.objects.get_or_create(
            model='homepage', app_label='home')
        try:
            homepage, __ = HomePage.objects.update_or_create(slug='home', defaults={
                'title': "Home",
                'draft_title': "Home",
                'content_type': homepage_content_type,
                'path': '00010001',
                'depth': 2,
                'numchild': 0,
                'url_path': '/home/',
                'show_in_menus': True,
            })
            banner_index_page = BannerIndexPage(title='Banners')
            footer_index_page = FooterIndexPage(title='Footers')
            section_index_page = SectionIndexPage(title='Sections')
            homepage.add_child(instance=banner_index_page)
            homepage.add_child(instance=footer_index_page)
            homepage.add_child(instance=section_index_page)
            self.stdout.write(self.style.SUCCESS('Index pages added.'))
        except:
            pass


# home/management/commands/create_initial_data.py

from datetime import datetime, timezone
from io import BytesIO

import requests
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group, Permission
from django.contrib.contenttypes.models import ContentType
from django.core import management
from django.core.files.images import ImageFile
from django.core.management.base import BaseCommand
from wagtail.core.models import Site, Page
from wagtail.core.rich_text import RichText
from wagtail.images.models import Image

import home.models as models
from comments.models import CommentStatus

User = get_user_model()


class Command(BaseCommand):

    def clear(self):
        models.MiscellaneousIndexPage.objects.all().delete()
        models.BannerIndexPage.objects.all().delete()
        models.Article.objects.all().delete()
        models.Section.objects.all().delete()
        models.SectionIndexPage.objects.all().delete()
        Image.objects.all().delete()
        Page.objects.filter(id=2).delete()

    def create_image(self):
        response = requests.get('https://via.placeholder.com/729x576.png?text=Youth')

        title = 'youth_banner.jpg'
        image_file = ImageFile(BytesIO(response.content), name=title)

        return Image.objects.create(title=title, file=image_file)

    def create_homepage(self):
        homepage_content_type, __ = ContentType.objects.get_or_create(
            model='homepage', app_label='home')

        # Create a new homepage
        homepage, __ = models.HomePage.objects.update_or_create(slug='home', defaults={
            'title': "Home",
            'draft_title': "Home",
            'content_type': homepage_content_type,
            'path': '00010001',
            'depth': 2,
            'numchild': 0,
            'url_path': '/home/',
            'show_in_menus': True,
        })

        # Create a site with the new homepage set as the root
        Site.objects.get_or_create(hostname='localhost', defaults={
            'root_page': homepage,
            'is_default_site': True,
        })

    def create(self, owner, home):
        article = models.Article(
            title='Do you know the person adding you?',
            body=[('paragraph',
                   RichText('Someone sent me a friend request - but I don’t know this person, what should I do?'))],
            owner=owner,
            first_published_at=datetime.now(timezone.utc),
            commenting_status=CommentStatus.OPEN
        )
        internet_safety = models.Section(
            title='Internet Safety',
            show_in_menus=True,
        )
        youth = models.Section(
            title='Youth',
            show_in_menus=True,
            font_color='1CABE2'
        )
        section_index_page = models.SectionIndexPage(title='Sections')

        home.add_child(instance=section_index_page)
        section_index_page.add_child(instance=youth)
        youth.add_child(instance=internet_safety)
        internet_safety.add_child(instance=article)

        models.FeaturedContent.objects.create(source=home, content=youth)

        banner_index_page = models.BannerIndexPage(title='Banners')
        home.add_child(instance=banner_index_page)

        image = self.create_image()

        banner_page = models.BannerPage(title='Youth', banner_image=image, banner_link_page=youth)
        banner_index_page.add_child(instance=banner_page)

        models.HomePageBanner.objects.create(source=home, banner_page=banner_page)

        footer_index_page = models.FooterIndexPage(title='Footers')
        home.add_child(instance=footer_index_page)

        footer = models.FooterPage(
            title='Footer1?',
            body=[('paragraph',
                   RichText('Footer1 paragraph1'))],
            owner=owner,
            first_published_at=datetime.now(timezone.utc),
            commenting_status=CommentStatus.OPEN,
        )
        footer_index_page.add_child(instance=footer)

        miscellaneous_index_page = models.MiscellaneousIndexPage(title='Miscellaneous')
        home.add_child(instance=miscellaneous_index_page)

    def populate_group_permissions(self):
        self.stdout.write('Adding group permissions')

        permissions = Permission.objects.filter(codename__in=['can_moderate', 'delete_xtdcomment'])

        group = Group.objects.get(name='Moderators')
        for permission in permissions:
            group.permissions.add(permission)

    def handle(self, *args, **options):
        self.clear()
        self.stdout.write('Existing site structure cleared')

        self.create_homepage()

        owner = User.objects.first()
        home = models.HomePage.objects.first()
        if home:
            self.stdout.write(f"Home page found, title={home.title}")
            self.create(owner, home)
        else:
            self.stdout.write('No home page found. Quitting.')

        self.populate_group_permissions()

        management.call_command('create_default_site')

        # Create RapidPro Bot User
        management.call_command('sync_rapidpro_bot_user')


# home/management/commands/create_initial_menu.py

from pathlib import Path

from django.conf import settings
from django.core.files import File
from django.core.management.base import BaseCommand
from wagtail.core.models import Site, Locale
from wagtailmenus.models import FlatMenu
from wagtailsvg.models import Svg

from home.models import IogtFlatMenuItem, HomePage


class Command(BaseCommand):
    def handle(self, *args, **options):
        site = Site.objects.filter(is_default_site=True).first()
        if not site:
            self.stdout.write(self.style.ERROR('Default site not found.'))
            return

        home_page = HomePage.objects.first()
        if not home_page:
            self.stdout.write(self.style.ERROR('Homepage not found.'))
            return

        locales = Locale.objects.all()
        file = File(open(Path(settings.BASE_DIR) / 'iogt/static/icons/burger.svg'), name='burger.svg')
        icon = Svg.objects.create(title='burger', file=file)
        for locale in locales:
            menu, __ = FlatMenu.objects.get_or_create(handle=f'{locale.language_code}_menu_live', defaults={
                'title': f'{locale.language_code} main menu',
                'site': site,
                'heading': 'Main Menu',
            })

            translated_home_page = home_page.get_translation_or_none(locale)
            if translated_home_page:
                IogtFlatMenuItem.objects.get_or_create(
                    link_page=translated_home_page,
                    menu=menu,
                    defaults={
                        'link_text': translated_home_page.title.split(' ')[0],
                    }
                )

            IogtFlatMenuItem.objects.get_or_create(
                link_url='#',
                menu=menu,
                url_append='top-level-sections',
                defaults={
                    'link_text': 'Sections',
                }
            )
            IogtFlatMenuItem.objects.get_or_create(
                link_url='#menu',
                menu=menu,
                defaults={
                    'link_text': 'Menu',
                    'icon': icon,
                    'display_only_in_single_column_view': True,
                }
            )

        self.stdout.write(self.style.SUCCESS('Menu items created.'))


# home/management/commands/create_sandbox_section.py

from django.core.management.base import BaseCommand

from home.models import HomePage, Section


class Command(BaseCommand):

    def handle(self, *args, **options):
        home_page = HomePage.objects.get(slug='home')
        sandbox_section = home_page.get_children().filter(slug='sandbox').first()
        if sandbox_section:
            self.stdout.write(self.style.SUCCESS('Sandbox Section already exists.'))
        else:
            section = Section(title='Sandbox', live=False)
            home_page.add_child(instance=section)
            self.stdout.write(self.style.SUCCESS('Sandbox Section added.'))


# home/management/commands/fix_misplaced_sections.py

from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site
from django.core.management.base import BaseCommand

from home.models import HomePage, Section, SectionIndexPage

User = get_user_model()


class Command(BaseCommand):
    """
    This command picks all the Sections that are direct children of HomePage
    and moves them as children of the SectionIndexPage.
    """

    def handle(self, *args, **options):
        home_pages = HomePage.objects.all()

        for home_page in home_pages:
            misplaced_sections = home_page.get_children().type(Section)
            section_index_page = home_page.get_children().type(SectionIndexPage).first()

            for section in misplaced_sections:
                section.move(section_index_page, pos='last-child')
                self.stdout.write(self.style.SUCCESS(f'Successfully moved {section} to {section_index_page}'))


# home/management/commands/load_po_files.py

from django.core.management.base import BaseCommand
from django.utils.translation import gettext_lazy as _

from home.translation_manager.manager import IogtTranslationManager as Manager


class Command(BaseCommand):
    def handle(self, *args, **options):
        self.stdout.write(_('Loading translations...'))
        Manager().load_data_from_po()
        self.stdout.write(_('The translations have been loaded successfully!'))


# home/management/commands/sync_translated_pages.py

from django.core.exceptions import ValidationError
from django.core.management.base import BaseCommand
from wagtail.core.models import Page
from wagtail_localize.models import TranslationSource


class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument(
            '--publish',
            action='store_true',
            help='This will publish immediately, before any new translations happen.'
        )

    def handle(self, *args, **options):
        publish = options.get('publish')

        for page in Page.objects.all().specific():
            source = TranslationSource.objects.get_for_instance_or_none(page)
            if source:
                source.update_from_db()
                if publish:
                    for translation in source.translations.filter(enabled=True).select_related("target_locale"):
                        try:
                            translation.save_target(publish=True)
                        except ValidationError:
                            pass

        self.stdout.write(self.style.SUCCESS('Translated pages synced.'))


# home/management/commands/update_show_in_menu.py

from django.core.management.base import BaseCommand

from wagtail.core.models import Page

from home.models import HomePage, Section, Article, PageLinkPage
from questionnaires.models import Poll, Quiz, Survey


class Command(BaseCommand):

    def handle(self, *args, **options):
        Page.objects.type(HomePage, Section, Article, PageLinkPage, Poll, Quiz, Survey).update(show_in_menus=True)


# home/tests/test_links_in_page/test_external_links.py

from bs4 import BeautifulSoup
from django.test import TestCase

from rest_framework import status
from wagtail.core.models import Site
from wagtail.core.rich_text import RichText

from home.factories import ArticleFactory, HomePageFactory
from wagtail_factories import SiteFactory


class TransitionPageForExtIntLinksInRichTextTests(TestCase):
    def setUp(self):
        Site.objects.all().delete()
        self.site = SiteFactory(site_name='IoGT', port=8000, is_default_site=True)
        self.home_page = HomePageFactory(parent=self.site.root_page)
        self.external_link_pattern = "/external-link/?next="

    def test_if_rich_text_field_external_links_redirect_to_transition_page(self):
        body = [
            (
                "paragraph",
                RichText(
                    '<p><a href="https://aynthing.com">external link at the beginning</a> dolor sit amet</p>'
                ),
            ),
            (
                "paragraph",
                RichText(
                    '<p>Lorem ipsum <a href="https://aynthing.com">external link in middle</a> dolor sit amet</p>'
                ),
            ),
            (
                "paragraph",
                RichText(
                    '<p>Lorem dolor sit amet <a href="https://aynthing.com">external link at the end</a></p>'
                ),
            ),
        ]

        article = ArticleFactory(parent=self.home_page, body=body)
        response = self.client.get(article.get_url())

        self.assertEqual(response.status_code, status.HTTP_200_OK)

        html_parser = BeautifulSoup(response.content, "html.parser")
        links = html_parser.select("section.article__content .block-paragraph a")

        self.assertEqual(len(links), len(body))

        for link in links:
            self.assertIn(self.external_link_pattern, link["href"])

    def test_if_rich_text_field_internal_links_redirect_directly(self):
        body = [
            (
                "paragraph",
                RichText(
                    f'<p><a id="{self.home_page.id}" linktype="page">internal link at the beginning</a> dolor sit amet</p>'
                ),
            ),
            (
                "paragraph",
                RichText(
                    f'<p>Lorem ipsum <a id="{self.home_page.id}" linktype="page">internal link in middle</a> dolor sit amet</p>'
                ),
            ),
            (
                "paragraph",
                RichText(
                    f'<p>Lorem dolor sit amet <a id="{self.home_page.id}" linktype="page">internal link at the end</a></p>'
                ),
            ),
        ]

        article = ArticleFactory(parent=self.home_page, body=body)
        response = self.client.get(article.url)

        self.assertEqual(response.status_code, status.HTTP_200_OK)

        html_parser = BeautifulSoup(response.content, "html.parser")
        links = html_parser.select("section.article__content .block-paragraph a")

        self.assertEqual(len(links), len(body))

        for link in links:
            self.assertNotIn(self.external_link_pattern, link["href"])


# messaging/management/commands/get_rapidpro_authentication_header_value.py

from django.contrib.auth import get_user_model
from django.core.management.base import BaseCommand

User = get_user_model()


class Command(BaseCommand):
    """
    This command prints the Authorization Header Value for RapidPro
    """

    def handle(self, *args, **options):
        tokens = User.get_rapidpro_bot_auth_tokens()
        for username, token in tokens.items():
            self.stdout.write(self.style.SUCCESS(f'{username}: {token}'))


# messaging/management/commands/sync_rapidpro_bot_user.py

from uuid import uuid4

from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group
from django.core.management.base import BaseCommand

User = get_user_model()


class Command(BaseCommand):
    """
    Creates or updates the RapidPro Bot user that is used for integration with RapidPro.
    """

    def add_arguments(self, parser):
        parser.add_argument(
            '--username',
            default=uuid4(),
            help='The username of the RapidPro chatbot user to create/update.',
        )

    def handle(self, *args, **options):
        username = options.get('username')
        user, created = User.objects.get_or_create(
            username=username,
            defaults={'first_name': 'RapidPro', 'last_name': 'Bot'})
        group, _ = Group.objects.get_or_create(name=settings.RAPIDPRO_BOT_GROUP_NAME)
        user.groups.add(group)

        if created:
            user.set_password(User.objects.make_random_password())
            user.save(update_fields=['password'])
            action = "created"
        else:
            action = "updated"

        self.stdout.write(
            self.style.SUCCESS(f"RapidPro Bot user {action} successfully."))


# questionnaires/api/v1/serializers.py

import json

from rest_framework import serializers
from wagtail.contrib.forms.models import AbstractFormField
from wagtail.core.models import Page

from questionnaires.models import (
    Survey,
    SurveyFormField,
    Poll,
    PollFormField,
    QuizFormField,
    Quiz,
    UserSubmission,
    QuestionnairePage,
)


class QuestionnairePageSerializer(serializers.ModelSerializer):
    type = serializers.CharField(source='content_type.model')

    class Meta:
        model = Page
        fields = ['id', 'title', 'type', 'last_published_at']


class BaseFormFieldSerializer(serializers.ModelSerializer):
    choices = serializers.SerializerMethodField()
    default_value = serializers.SerializerMethodField()

    def get_choices(self, instance):
        return instance.choices and instance.choices.split('|')

    def get_default_value(self, instance):
        return instance.default_value and instance.default_value.split('|')

    class Meta:
        model = AbstractFormField
        fields = [
            'id', 'sort_order', 'label', 'clean_name', 'help_text', 'required', 'field_type', 'choices',
            'default_value',
        ]
        abstract = True


class BaseQuestionnairePageDetailSerializer(serializers.ModelSerializer):
    description = serializers.JSONField(source='description.get_prep_value')
    thank_you_text = serializers.JSONField(source='thank_you_text.get_prep_value')
    terms_and_conditions = serializers.JSONField(source='terms_and_conditions.get_prep_value')
    url = serializers.CharField(source='full_url')

    class Meta:
        model = QuestionnairePage
        fields = [
            'id', 'title', 'last_published_at', 'live', 'url', 'last_published_at', 'allow_anonymous_submissions',
            'allow_multiple_submissions', 'submit_button_text', 'direct_display', 'index_page_description',
            'index_page_description_line_2', 'description', 'thank_you_text', 'terms_and_conditions',
        ]
        abstract = True


class PollFormFieldSerializer(BaseFormFieldSerializer):
    class Meta:
        model = PollFormField
        fields = BaseFormFieldSerializer.Meta.fields + ['admin_label']


class PollPageDetailSerializer(BaseQuestionnairePageDetailSerializer):
    questions = PollFormFieldSerializer(source='poll_form_fields', many=True)

    class Meta:
        model = Poll
        fields = BaseQuestionnairePageDetailSerializer.Meta.fields + [
            'show_results', 'result_as_percentage', 'randomise_options', 'show_results_with_no_votes', 'questions',
        ]


class SurveyFormFieldSerializer(BaseFormFieldSerializer):
    skip_logic = serializers.JSONField(source='skip_logic.get_prep_value')

    class Meta:
        model = SurveyFormField
        fields = BaseFormFieldSerializer.Meta.fields + ['skip_logic', 'admin_label', 'page_break']


class SurveyPageDetailSerializer(BaseQuestionnairePageDetailSerializer):
    questions = SurveyFormFieldSerializer(source='survey_form_fields', many=True)

    class Meta:
        model = Survey
        fields = BaseQuestionnairePageDetailSerializer.Meta.fields + ['multi_step', 'questions']


class QuizFormFieldSerializer(BaseFormFieldSerializer):
    correct_answer = serializers.SerializerMethodField()

    def get_correct_answer(self, instance):
        return instance.correct_answer and instance.correct_answer.split('|')

    class Meta:
        model = QuizFormField
        fields = BaseFormFieldSerializer.Meta.fields + ['correct_answer', 'feedback', 'admin_label', 'page_break']


class QuizPageDetailSerializer(BaseQuestionnairePageDetailSerializer):
    questions = QuizFormFieldSerializer(source='quiz_form_fields', many=True)

    class Meta:
        model = Quiz
        fields = BaseQuestionnairePageDetailSerializer.Meta.fields + ['multi_step', 'questions']


class QuestionnairePageDetailSerializer(serializers.Serializer):
    poll = PollPageDetailSerializer()
    survey = SurveyPageDetailSerializer()
    quiz = QuizPageDetailSerializer()


class UserSubmissionSerializer(serializers.ModelSerializer):
    submission = serializers.SerializerMethodField()
    user = serializers.CharField()
    page_url = serializers.CharField(source='page.full_url')

    def get_submission(self, instance):
        form_data = json.loads(instance.form_data)
        form_data_ = []
        for clean_name, answer in form_data.items():
            question = instance.page.specific.get_form_fields().filter(clean_name=clean_name).first()
            if question:
                form_data_.append({
                    "admin_label": question.admin_label,
                    "clean_name": clean_name,
                    "user_answer": answer,
                })
        return form_data_

    class Meta:
        model = UserSubmission
        fields = ['id', 'user', 'submit_time', 'page_url', 'submission']


# questionnaires/api/v1/tests.py

import datetime
import io
import json
from datetime import timedelta

import pytz
from django.contrib.auth.models import Permission
from django.test import TestCase
from django.urls import reverse
from django.utils import timezone
from openpyxl import load_workbook
from rest_framework import status
from rest_framework.test import APIClient
from wagtail.core.models import Site
from wagtail_factories import SiteFactory, PageFactory

from home.factories import HomePageFactory
from home.tests.tests import parse_html
from iogt_users.factories import (
    UserFactory,
    GroupFactory,
    GroupPagePermissionFactory,
)
from questionnaires.factories import (
    PollFactory,
    SurveyFactory,
    QuizFactory,
    PollFormFieldFactory,
    SurveyFormFieldFactory,
    QuizFormFieldFactory,
    UserSubmissionFactory
)
from questionnaires.utils import SkipLogicPaginator


class QuestionnairesListAPIViewTests(TestCase):
    def setUp(self):
        self.user = UserFactory()
        self.group = GroupFactory(name='questionnaires')
        self.user.groups.add(self.group)
        self.client = APIClient()
        self.client.force_authenticate(user=self.user)
        self.url = reverse('questionnaires_list')

        Site.objects.all().delete()
        self.site = SiteFactory(site_name='IoGT', port=8000, is_default_site=True)
        self.home_page = HomePageFactory(parent=self.site.root_page)

    def test_questionnaires_list(self):
        current_datetime = timezone.now()
        poll_01 = PollFactory(parent=self.home_page, title='Poll 01', last_published_at=current_datetime)
        poll_02 = PollFactory(
            parent=self.home_page, title='Poll 02', last_published_at=current_datetime - timedelta(days=1))
        poll_03 = PollFactory(
            parent=self.home_page, title='Poll 03', last_published_at=current_datetime - timedelta(days=2))
        survey_01 = SurveyFactory(
            parent=self.home_page, title='Survey 01', last_published_at=current_datetime - timedelta(days=3))
        survey_02 = SurveyFactory(
            parent=self.home_page, title='Survey 02', last_published_at=current_datetime - timedelta(days=4))
        survey_03 = SurveyFactory(
            parent=self.home_page, title='Survey 03', last_published_at=current_datetime - timedelta(days=5))
        quiz_01 = QuizFactory(
            parent=self.home_page, title='Quiz 01', last_published_at=current_datetime - timedelta(days=6))
        quiz_02 = QuizFactory(
            parent=self.home_page, title='Quiz 02', last_published_at=current_datetime - timedelta(days=7))
        quiz_03 = QuizFactory(
            parent=self.home_page, title='Quiz 03', last_published_at=current_datetime - timedelta(days=8))
        GroupPagePermissionFactory(group=self.group, page=poll_01)
        GroupPagePermissionFactory(group=self.group, page=poll_02)
        GroupPagePermissionFactory(group=self.group, page=poll_03)
        GroupPagePermissionFactory(group=self.group, page=survey_01)
        GroupPagePermissionFactory(group=self.group, page=survey_02)
        GroupPagePermissionFactory(group=self.group, page=survey_03)
        GroupPagePermissionFactory(group=self.group, page=quiz_01)
        GroupPagePermissionFactory(group=self.group, page=quiz_02)
        GroupPagePermissionFactory(group=self.group, page=quiz_03)

        response = self.client.get(self.url)

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 9)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['title'], poll_01.title)
        self.assertEqual(response.data['results'][1]['title'], poll_02.title)
        self.assertEqual(response.data['results'][2]['title'], poll_03.title)
        self.assertEqual(response.data['results'][3]['title'], survey_01.title)
        self.assertEqual(response.data['results'][4]['title'], survey_02.title)
        self.assertEqual(response.data['results'][5]['title'], survey_03.title)
        self.assertEqual(response.data['results'][6]['title'], quiz_01.title)
        self.assertEqual(response.data['results'][7]['title'], quiz_02.title)
        self.assertEqual(response.data['results'][8]['title'], quiz_03.title)

    def test_questionnaires_list_type_filter(self):
        poll_01 = PollFactory(parent=self.home_page, title='Poll 01')
        poll_02 = PollFactory(parent=self.home_page, title='Poll 02')
        poll_03 = PollFactory(parent=self.home_page, title='Poll 03')
        survey_01 = SurveyFactory(parent=self.home_page, title='Survey 01')
        survey_02 = SurveyFactory(parent=self.home_page, title='Survey 02')
        survey_03 = SurveyFactory(parent=self.home_page, title='Survey 03')
        quiz_01 = QuizFactory(parent=self.home_page, title='Quiz 01')
        quiz_02 = QuizFactory(parent=self.home_page, title='Quiz 02')
        quiz_03 = QuizFactory(parent=self.home_page, title='Quiz 03')
        GroupPagePermissionFactory(group=self.group, page=poll_01)
        GroupPagePermissionFactory(group=self.group, page=poll_02)
        GroupPagePermissionFactory(group=self.group, page=poll_03)
        GroupPagePermissionFactory(group=self.group, page=survey_01)
        GroupPagePermissionFactory(group=self.group, page=survey_02)
        GroupPagePermissionFactory(group=self.group, page=survey_03)
        GroupPagePermissionFactory(group=self.group, page=quiz_01)
        GroupPagePermissionFactory(group=self.group, page=quiz_02)
        GroupPagePermissionFactory(group=self.group, page=quiz_03)

        response = self.client.get(f'{self.url}?type=poll')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 3)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['title'], poll_01.title)
        self.assertEqual(response.data['results'][1]['title'], poll_02.title)
        self.assertEqual(response.data['results'][2]['title'], poll_03.title)

        response = self.client.get(f'{self.url}?type=survey')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 3)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['title'], survey_01.title)
        self.assertEqual(response.data['results'][1]['title'], survey_02.title)
        self.assertEqual(response.data['results'][2]['title'], survey_03.title)

        response = self.client.get(f'{self.url}?type=quiz')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 3)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['title'], quiz_01.title)
        self.assertEqual(response.data['results'][1]['title'], quiz_02.title)
        self.assertEqual(response.data['results'][2]['title'], quiz_03.title)

    def test_questionnaires_list_last_published_at_filter(self):
        current_datetime = timezone.now()
        poll_01 = PollFactory(parent=self.home_page, title='Poll 01', last_published_at=current_datetime)
        poll_02 = PollFactory(parent=self.home_page, title='Poll 02', last_published_at=current_datetime - timedelta(days=1))
        poll_03 = PollFactory(parent=self.home_page, title='Poll 03', last_published_at=current_datetime - timedelta(days=2))
        GroupPagePermissionFactory(group=self.group, page=poll_01)
        GroupPagePermissionFactory(group=self.group, page=poll_02)
        GroupPagePermissionFactory(group=self.group, page=poll_03)

        response = self.client.get(f'{self.url}?last_published_at_start={(current_datetime - timedelta(days=1)).date()}')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 2)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['title'], poll_01.title)
        self.assertEqual(response.data['results'][1]['title'], poll_02.title)

        response = self.client.get(f'{self.url}?last_published_at_end={(current_datetime - timedelta(days=1)).date()}')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 2)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['title'], poll_02.title)
        self.assertEqual(response.data['results'][1]['title'], poll_03.title)

    def test_questionnaires_list_page_size(self):
        current_datetime = timezone.now()
        poll_01 = PollFactory(parent=self.home_page, title='Poll 01', last_published_at=current_datetime)
        poll_02 = PollFactory(
            parent=self.home_page, title='Poll 02', last_published_at=current_datetime - timedelta(days=1))
        poll_03 = PollFactory(
            parent=self.home_page, title='Poll 03', last_published_at=current_datetime - timedelta(days=2))
        survey_01 = SurveyFactory(
            parent=self.home_page, title='Survey 01', last_published_at=current_datetime - timedelta(days=3))
        survey_02 = SurveyFactory(
            parent=self.home_page, title='Survey 02', last_published_at=current_datetime - timedelta(days=4))
        survey_03 = SurveyFactory(
            parent=self.home_page, title='Survey 03', last_published_at=current_datetime - timedelta(days=5))
        quiz_01 = QuizFactory(
            parent=self.home_page, title='Quiz 01', last_published_at=current_datetime - timedelta(days=6))
        quiz_02 = QuizFactory(
            parent=self.home_page, title='Quiz 02', last_published_at=current_datetime - timedelta(days=7))
        quiz_03 = QuizFactory(
            parent=self.home_page, title='Quiz 03', last_published_at=current_datetime - timedelta(days=8))
        GroupPagePermissionFactory(group=self.group, page=poll_01)
        GroupPagePermissionFactory(group=self.group, page=poll_02)
        GroupPagePermissionFactory(group=self.group, page=poll_03)
        GroupPagePermissionFactory(group=self.group, page=survey_01)
        GroupPagePermissionFactory(group=self.group, page=survey_02)
        GroupPagePermissionFactory(group=self.group, page=survey_03)
        GroupPagePermissionFactory(group=self.group, page=quiz_01)
        GroupPagePermissionFactory(group=self.group, page=quiz_02)
        GroupPagePermissionFactory(group=self.group, page=quiz_03)

        response = self.client.get(f'{self.url}?page_size=3')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 9)
        self.assertEqual(response.data['next'], 'http://testserver/api/v1/questionnaires/?page=2&page_size=3')
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['title'], poll_01.title)
        self.assertEqual(response.data['results'][1]['title'], poll_02.title)
        self.assertEqual(response.data['results'][2]['title'], poll_03.title)

        response = self.client.get(f'{self.url}?page=2&page_size=3')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 9)
        self.assertEqual(response.data['next'], 'http://testserver/api/v1/questionnaires/?page=3&page_size=3')
        self.assertEqual(response.data['previous'], 'http://testserver/api/v1/questionnaires/?page_size=3')
        self.assertEqual(response.data['results'][0]['title'], survey_01.title)
        self.assertEqual(response.data['results'][1]['title'], survey_02.title)
        self.assertEqual(response.data['results'][2]['title'], survey_03.title)

        response = self.client.get(f'{self.url}?page=3&page_size=3')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 9)
        self.assertIsNone(response.data['next'])
        self.assertEqual(response.data['previous'], 'http://testserver/api/v1/questionnaires/?page=2&page_size=3')
        self.assertEqual(response.data['results'][0]['title'], quiz_01.title)
        self.assertEqual(response.data['results'][1]['title'], quiz_02.title)
        self.assertEqual(response.data['results'][2]['title'], quiz_03.title)

    def test_questionnaires_list_page_permission(self):
        current_datetime = timezone.now()
        poll_01 = PollFactory(parent=self.home_page, title='Poll 01', last_published_at=current_datetime)
        poll_02 = PollFactory(
            parent=self.home_page, title='Poll 02', last_published_at=current_datetime - timedelta(days=1))
        poll_03 = PollFactory(
            parent=self.home_page, title='Poll 03', last_published_at=current_datetime - timedelta(days=2))
        GroupPagePermissionFactory(group=self.group, page=poll_01)
        GroupPagePermissionFactory(group=self.group, page=poll_02)

        response = self.client.get(self.url)

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 2)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['title'], poll_01.title)
        self.assertEqual(response.data['results'][1]['title'], poll_02.title)


class QuestionnaireDetailAPIViewTests(TestCase):
    def setUp(self):
        self.user = UserFactory()
        self.group = GroupFactory(name='questionnaires')
        self.user.groups.add(self.group)
        self.client = APIClient()
        self.client.force_authenticate(user=self.user)
        self.url = 'questionnaire_detail'

        Site.objects.all().delete()
        self.site = SiteFactory(site_name='IoGT', port=8000, is_default_site=True)
        self.home_page = HomePageFactory(parent=self.site.root_page)

    def test_poll_detail(self):
        poll = PollFactory(parent=self.home_page)
        GroupPagePermissionFactory(group=self.group, page=poll)
        poll_question = PollFormFieldFactory(page=poll, field_type='checkboxes', choices='c1|c2|c3', default_value='c2')

        response = self.client.get(reverse(self.url, kwargs={'pk': poll.id}))
        self.assertEqual(response.status_code, status.HTTP_200_OK)

        self.assertEqual(response.data['id'], poll.id)
        self.assertEqual(response.data['title'], poll.title)
        self.assertEqual(response.data['live'], poll.live)
        self.assertEqual(response.data['url'], poll.full_url)
        self.assertIsNotNone(response.data['last_published_at'])
        self.assertEqual(response.data['allow_anonymous_submissions'], poll.allow_anonymous_submissions)
        self.assertEqual(response.data['allow_multiple_submissions'], poll.allow_multiple_submissions)
        self.assertEqual(response.data['show_results'], poll.show_results)
        self.assertEqual(response.data['result_as_percentage'], poll.result_as_percentage)
        self.assertEqual(response.data['randomise_options'], poll.randomise_options)
        self.assertEqual(response.data['show_results_with_no_votes'], poll.show_results_with_no_votes)
        self.assertEqual(response.data['submit_button_text'], poll.submit_button_text)
        self.assertEqual(response.data['direct_display'], poll.direct_display)
        self.assertEqual(response.data['index_page_description'], poll.index_page_description)
        self.assertEqual(response.data['index_page_description_line_2'], poll.index_page_description_line_2)
        self.assertEqual(response.data['questions'][0]['id'], poll_question.id)
        self.assertEqual(response.data['questions'][0]['label'], poll_question.label)
        self.assertEqual(response.data['questions'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['questions'][0]['help_text'], poll_question.help_text)
        self.assertEqual(response.data['questions'][0]['required'], poll_question.required)
        self.assertEqual(response.data['questions'][0]['field_type'], poll_question.field_type)
        self.assertEqual(response.data['questions'][0]['choices'][0], 'c1')
        self.assertEqual(response.data['questions'][0]['choices'][1], 'c2')
        self.assertEqual(response.data['questions'][0]['choices'][2], 'c3')
        self.assertEqual(response.data['questions'][0]['default_value'], [poll_question.default_value])
        self.assertEqual(response.data['questions'][0]['admin_label'], poll_question.admin_label)

    def test_survey_detail(self):
        survey = SurveyFactory(parent=self.home_page)
        GroupPagePermissionFactory(group=self.group, page=survey)
        skip_logic = json.dumps(
            [
                {
                    "type": "skip_logic",
                    "value": {
                        "choice": "c1",
                        "skip_logic": "next",
                        "question": None
                    }
                },
                {
                    "type": "skip_logic",
                    "value": {
                        "choice": "c2",
                        "skip_logic": "next",
                        "question": None
                    }
                },
                {
                    "type": "skip_logic",
                    "value": {
                        "choice": "c3",
                        "skip_logic": "next",
                        "question": None
                    }
                }
            ]
        )
        survey_question = SurveyFormFieldFactory(
            page=survey, field_type='checkboxes', skip_logic=skip_logic, default_value='c2')

        response = self.client.get(reverse(self.url, kwargs={'pk': survey.id}))

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['id'], survey.id)
        self.assertEqual(response.data['title'], survey.title)
        self.assertEqual(response.data['live'], survey.live)
        self.assertEqual(response.data['url'], survey.full_url)
        self.assertIsNotNone(response.data['last_published_at'])
        self.assertEqual(response.data['allow_anonymous_submissions'], survey.allow_anonymous_submissions)
        self.assertEqual(response.data['allow_multiple_submissions'], survey.allow_multiple_submissions)
        self.assertEqual(response.data['submit_button_text'], survey.submit_button_text)
        self.assertEqual(response.data['multi_step'], survey.multi_step)
        self.assertEqual(response.data['direct_display'], survey.direct_display)
        self.assertEqual(response.data['index_page_description'], survey.index_page_description)
        self.assertEqual(response.data['index_page_description_line_2'], survey.index_page_description_line_2)
        self.assertEqual(response.data['questions'][0]['id'], survey_question.id)
        self.assertEqual(response.data['questions'][0]['label'], survey_question.label)
        self.assertEqual(response.data['questions'][0]['clean_name'], survey_question.clean_name)
        self.assertEqual(response.data['questions'][0]['help_text'], survey_question.help_text)
        self.assertEqual(response.data['questions'][0]['required'], survey_question.required)
        self.assertEqual(response.data['questions'][0]['field_type'], survey_question.field_type)
        self.assertEqual(response.data['questions'][0]['skip_logic'][0]['type'], 'skip_logic')
        self.assertEqual(response.data['questions'][0]['skip_logic'][0]['value']['choice'], 'c1')
        self.assertEqual(response.data['questions'][0]['skip_logic'][0]['value']['skip_logic'], 'next')
        self.assertEqual(response.data['questions'][0]['skip_logic'][0]['value']['question'], None)
        self.assertEqual(response.data['questions'][0]['skip_logic'][1]['type'], 'skip_logic')
        self.assertEqual(response.data['questions'][0]['skip_logic'][1]['value']['choice'], 'c2')
        self.assertEqual(response.data['questions'][0]['skip_logic'][1]['value']['skip_logic'], 'next')
        self.assertEqual(response.data['questions'][0]['skip_logic'][1]['value']['question'], None)
        self.assertEqual(response.data['questions'][0]['skip_logic'][2]['type'], 'skip_logic')
        self.assertEqual(response.data['questions'][0]['skip_logic'][2]['value']['choice'], 'c3')
        self.assertEqual(response.data['questions'][0]['skip_logic'][2]['value']['skip_logic'], 'next')
        self.assertEqual(response.data['questions'][0]['skip_logic'][2]['value']['question'], None)
        self.assertEqual(response.data['questions'][0]['default_value'], [survey_question.default_value])
        self.assertEqual(response.data['questions'][0]['admin_label'], survey_question.admin_label)

    def test_quiz_detail(self):
        quiz = QuizFactory(parent=self.home_page)
        GroupPagePermissionFactory(group=self.group, page=quiz)
        quiz_question = QuizFormFieldFactory(
            page=quiz, field_type='checkboxes', choices='c1|c2|c3', default_value='c2', correct_answer='c3')

        response = self.client.get(reverse(self.url, kwargs={'pk': quiz.id}))
        self.assertEqual(response.status_code, status.HTTP_200_OK)

        self.assertEqual(response.data['id'], quiz.id)
        self.assertEqual(response.data['title'], quiz.title)
        self.assertEqual(response.data['live'], quiz.live)
        self.assertEqual(response.data['url'], quiz.full_url)
        self.assertIsNotNone(response.data['last_published_at'])
        self.assertEqual(response.data['allow_anonymous_submissions'], quiz.allow_anonymous_submissions)
        self.assertEqual(response.data['allow_multiple_submissions'], quiz.allow_multiple_submissions)
        self.assertEqual(response.data['submit_button_text'], quiz.submit_button_text)
        self.assertEqual(response.data['multi_step'], quiz.multi_step)
        self.assertEqual(response.data['direct_display'], quiz.direct_display)
        self.assertEqual(response.data['index_page_description'], quiz.index_page_description)
        self.assertEqual(response.data['index_page_description_line_2'], quiz.index_page_description_line_2)
        self.assertEqual(response.data['questions'][0]['id'], quiz_question.id)
        self.assertEqual(response.data['questions'][0]['label'], quiz_question.label)
        self.assertEqual(response.data['questions'][0]['clean_name'], quiz_question.clean_name)
        self.assertEqual(response.data['questions'][0]['help_text'], quiz_question.help_text)
        self.assertEqual(response.data['questions'][0]['required'], quiz_question.required)
        self.assertEqual(response.data['questions'][0]['field_type'], quiz_question.field_type)
        self.assertEqual(response.data['questions'][0]['choices'][0], 'c1')
        self.assertEqual(response.data['questions'][0]['choices'][1], 'c2')
        self.assertEqual(response.data['questions'][0]['choices'][2], 'c3')
        self.assertEqual(response.data['questions'][0]['default_value'], [quiz_question.default_value])
        self.assertEqual(response.data['questions'][0]['correct_answer'], [quiz_question.correct_answer])
        self.assertEqual(response.data['questions'][0]['feedback'], quiz_question.feedback)
        self.assertEqual(response.data['questions'][0]['admin_label'], quiz_question.admin_label)

    def test_questionnaires_detail_page_permission(self):
        poll = PollFactory(parent=self.home_page)
        poll_question = PollFormFieldFactory(page=poll, field_type='checkboxes', choices='c1|c2|c3', default_value='c2')

        response = self.client.get(reverse(self.url, kwargs={'pk': poll.id}))
        self.assertEqual(response.status_code, status.HTTP_302_FOUND)


class QuestionnaireSubmissionsAPIViewTests(TestCase):
    def setUp(self):
        self.user = UserFactory()
        self.group = GroupFactory(name='questionnaires')
        self.user.groups.add(self.group)
        self.client = APIClient()
        self.client.force_authenticate(user=self.user)
        self.url = 'questionnaire_submissions'

        Site.objects.all().delete()
        self.site = SiteFactory(site_name='IoGT', port=8000, is_default_site=True)
        self.home_page = HomePageFactory(parent=self.site.root_page)

    def test_questionnaire_submission_list(self):
        current_datetime = timezone.now()
        poll = PollFactory(parent=self.home_page, last_published_at=current_datetime - timedelta(days=3))
        GroupPagePermissionFactory(group=self.group, page=poll)
        poll_question = PollFormFieldFactory(page=poll, field_type='checkboxes', choices='c1|c2|c3', default_value='c2')

        user_01 = UserFactory()
        form_data_01 = json.dumps({
            poll_question.clean_name: [
                'c1',
            ],
        })
        user_submission_01 = UserSubmissionFactory(page=poll, user=user_01, form_data=form_data_01)
        user_submission_01.submit_time = current_datetime
        user_submission_01.save()

        user_02 = UserFactory()
        form_data_02 = json.dumps({
            poll_question.clean_name: [
                'c2',
                'c3',
            ],
        })
        user_submission_02 = UserSubmissionFactory(page=poll, user=user_02, form_data=form_data_02)
        user_submission_02.submit_time = current_datetime - timedelta(days=1)
        user_submission_02.save()

        user_03 = UserFactory()
        form_data_03 = json.dumps({
            poll_question.clean_name: [
                'c3',
            ],
        })
        user_submission_03 = UserSubmissionFactory(page=poll, user=user_03, form_data=form_data_03)
        user_submission_03.submit_time = current_datetime - timedelta(days=2)
        user_submission_03.save()

        response = self.client.get(reverse(self.url, kwargs={'pk': poll.id}))

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 3)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['id'], user_submission_01.id)
        self.assertEqual(response.data['results'][0]['user'], user_01.username)
        self.assertIsNotNone(response.data['results'][0]['submit_time'])
        self.assertEqual(response.data['results'][0]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][0]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][0]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][0]['submission'][0]['user_answer'], ['c1'])
        self.assertEqual(response.data['results'][1]['id'], user_submission_02.id)
        self.assertEqual(response.data['results'][1]['user'], user_02.username)
        self.assertIsNotNone(response.data['results'][1]['submit_time'])
        self.assertEqual(response.data['results'][1]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][1]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][1]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][1]['submission'][0]['user_answer'], ['c2', 'c3'])
        self.assertEqual(response.data['results'][2]['id'], user_submission_03.id)
        self.assertEqual(response.data['results'][2]['id'], user_submission_03.id)
        self.assertEqual(response.data['results'][2]['user'], user_03.username)
        self.assertIsNotNone(response.data['results'][2]['submit_time'])
        self.assertEqual(response.data['results'][2]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][2]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][2]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][2]['submission'][0]['user_answer'], ['c3'])

    def test_questionnaire_submission_list_submit_time_filter(self):
        current_datetime = timezone.now()
        poll = PollFactory(parent=self.home_page, last_published_at=current_datetime - timedelta(days=3))
        GroupPagePermissionFactory(group=self.group, page=poll)
        poll_question = PollFormFieldFactory(page=poll, field_type='checkboxes', choices='c1|c2|c3', default_value='c2')

        user_01 = UserFactory()
        form_data_01 = json.dumps({
            poll_question.clean_name: [
                'c1',
            ],
        })
        user_submission_01 = UserSubmissionFactory(page=poll, user=user_01, form_data=form_data_01)
        user_submission_01.submit_time = current_datetime
        user_submission_01.save()

        user_02 = UserFactory()
        form_data_02 = json.dumps({
            poll_question.clean_name: [
                'c2',
                'c3',
            ],
        })
        user_submission_02 = UserSubmissionFactory(page=poll, user=user_02, form_data=form_data_02)
        user_submission_02.submit_time = current_datetime - timedelta(days=1)
        user_submission_02.save()

        user_03 = UserFactory()
        form_data_03 = json.dumps({
            poll_question.clean_name: [
                'c3',
            ],
        })
        user_submission_03 = UserSubmissionFactory(page=poll, user=user_03, form_data=form_data_03)
        user_submission_03.submit_time = current_datetime - timedelta(days=2)
        user_submission_03.save()

        response = self.client.get(
            f"{reverse(self.url, kwargs={'pk': poll.id})}?submit_time_start={(current_datetime - timedelta(days=1)).date()}")

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 2)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['id'], user_submission_01.id)
        self.assertEqual(response.data['results'][0]['user'], user_01.username)
        self.assertIsNotNone(response.data['results'][0]['submit_time'])
        self.assertEqual(response.data['results'][0]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][0]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][0]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][0]['submission'][0]['user_answer'], ['c1'])
        self.assertEqual(response.data['results'][1]['id'], user_submission_02.id)
        self.assertEqual(response.data['results'][1]['user'], user_02.username)
        self.assertIsNotNone(response.data['results'][1]['submit_time'])
        self.assertEqual(response.data['results'][1]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][1]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][1]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][1]['submission'][0]['user_answer'], ['c2', 'c3'])

        response = self.client.get(
            f"{reverse(self.url, kwargs={'pk': poll.id})}?submit_time_end={(current_datetime - timedelta(days=1)).date()}")

        self.assertEqual(response.data['count'], 2)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['id'], user_submission_02.id)
        self.assertEqual(response.data['results'][0]['user'], user_02.username)
        self.assertIsNotNone(response.data['results'][0]['submit_time'])
        self.assertEqual(response.data['results'][0]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][0]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][0]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][0]['submission'][0]['user_answer'], ['c2', 'c3'])
        self.assertEqual(response.data['results'][1]['id'], user_submission_03.id)
        self.assertEqual(response.data['results'][1]['id'], user_submission_03.id)
        self.assertEqual(response.data['results'][1]['user'], user_03.username)
        self.assertIsNotNone(response.data['results'][1]['submit_time'])
        self.assertEqual(response.data['results'][1]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][1]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][1]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][1]['submission'][0]['user_answer'], ['c3'])

    def test_questionnaire_submission_list_user_ids_filter(self):
        current_datetime = timezone.now()
        poll = PollFactory(parent=self.home_page, last_published_at=current_datetime - timedelta(days=3))
        GroupPagePermissionFactory(group=self.group, page=poll)
        poll_question = PollFormFieldFactory(
            page=poll, field_type='checkboxes', choices='c1|c2|c3', default_value='c2', admin_label='Q1')

        user_01 = UserFactory()
        form_data_01 = json.dumps({
            poll_question.clean_name: [
                'c1',
            ],
        })
        user_submission_01 = UserSubmissionFactory(page=poll, user=user_01, form_data=form_data_01)
        user_submission_01.submit_time = current_datetime
        user_submission_01.save()

        user_02 = UserFactory()
        form_data_02 = json.dumps({
            poll_question.clean_name: [
                'c2',
                'c3',
            ],
        })
        user_submission_02 = UserSubmissionFactory(page=poll, user=user_02, form_data=form_data_02)
        user_submission_02.submit_time = current_datetime - timedelta(days=1)
        user_submission_02.save()

        user_03 = UserFactory()
        form_data_03 = json.dumps({
            poll_question.clean_name: [
                'c3',
            ],
        })
        user_submission_03 = UserSubmissionFactory(page=poll, user=user_03, form_data=form_data_03)
        user_submission_03.submit_time = current_datetime - timedelta(days=2)
        user_submission_03.save()

        response = self.client.get(f"{reverse(self.url, kwargs={'pk': poll.id})}?user_ids={user_01.id}")

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 1)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['id'], user_submission_01.id)
        self.assertEqual(response.data['results'][0]['user'], user_01.username)
        self.assertIsNotNone(response.data['results'][0]['submit_time'])
        self.assertEqual(response.data['results'][0]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][0]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][0]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][0]['submission'][0]['user_answer'], ['c1'])

        response = self.client.get(f"{reverse(self.url, kwargs={'pk': poll.id})}?user_ids={user_02.id},{user_03.id}")

        self.assertEqual(response.data['count'], 2)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['id'], user_submission_02.id)
        self.assertEqual(response.data['results'][0]['user'], user_02.username)
        self.assertIsNotNone(response.data['results'][0]['submit_time'])
        self.assertEqual(response.data['results'][0]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][0]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][0]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][0]['submission'][0]['user_answer'], ['c2', 'c3'])
        self.assertEqual(response.data['results'][1]['id'], user_submission_03.id)
        self.assertEqual(response.data['results'][1]['id'], user_submission_03.id)
        self.assertEqual(response.data['results'][1]['user'], user_03.username)
        self.assertIsNotNone(response.data['results'][1]['submit_time'])
        self.assertEqual(response.data['results'][1]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][1]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][1]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][1]['submission'][0]['user_answer'], ['c3'])

    def test_questionnaire_submission_list_page_size(self):
        current_datetime = timezone.now()
        poll = PollFactory(parent=self.home_page, last_published_at=current_datetime - timedelta(days=3))
        GroupPagePermissionFactory(group=self.group, page=poll)
        poll_question = PollFormFieldFactory(page=poll, field_type='checkboxes', choices='c1|c2|c3', default_value='c2')

        user_01 = UserFactory()
        form_data_01 = json.dumps({
            poll_question.clean_name: [
                'c1',
            ],
        })
        user_submission_01 = UserSubmissionFactory(page=poll, user=user_01, form_data=form_data_01)
        user_submission_01.submit_time = current_datetime
        user_submission_01.save()

        user_02 = UserFactory()
        form_data_02 = json.dumps({
            poll_question.clean_name: [
                'c2',
                'c3',
            ],
        })
        user_submission_02 = UserSubmissionFactory(page=poll, user=user_02, form_data=form_data_02)
        user_submission_02.submit_time = current_datetime - timedelta(days=1)
        user_submission_02.save()

        user_03 = UserFactory()
        form_data_03 = json.dumps({
            poll_question.clean_name: [
                'c3',
            ],
        })
        user_submission_03 = UserSubmissionFactory(page=poll, user=user_03, form_data=form_data_03)
        user_submission_03.submit_time = current_datetime - timedelta(days=2)
        user_submission_03.save()

        response = self.client.get(f"{reverse(self.url, kwargs={'pk': poll.id})}?page_size=1")

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 3)
        self.assertEqual(
            response.data['next'], f'http://testserver/api/v1/questionnaires/{poll.id}/submissions/?page=2&page_size=1')
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'][0]['id'], user_submission_01.id)
        self.assertEqual(response.data['results'][0]['user'], user_01.username)
        self.assertIsNotNone(response.data['results'][0]['submit_time'])
        self.assertEqual(response.data['results'][0]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][0]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][0]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][0]['submission'][0]['user_answer'], ['c1'])

        response = self.client.get(f"{reverse(self.url, kwargs={'pk': poll.id})}?page=2&page_size=1")

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 3)
        self.assertEqual(
            response.data['next'], f'http://testserver/api/v1/questionnaires/{poll.id}/submissions/?page=3&page_size=1')
        self.assertEqual(
            response.data['previous'], f'http://testserver/api/v1/questionnaires/{poll.id}/submissions/?page_size=1')
        self.assertEqual(response.data['results'][0]['id'], user_submission_02.id)
        self.assertEqual(response.data['results'][0]['user'], user_02.username)
        self.assertIsNotNone(response.data['results'][0]['submit_time'])
        self.assertEqual(response.data['results'][0]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][0]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][0]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][0]['submission'][0]['user_answer'], ['c2', 'c3'])

        response = self.client.get(f"{reverse(self.url, kwargs={'pk': poll.id})}?page=3&page_size=1")

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 3)
        self.assertIsNone(response.data['next'])
        self.assertEqual(
            response.data['previous'], f'http://testserver/api/v1/questionnaires/{poll.id}/submissions/?page=2&page_size=1')
        self.assertEqual(response.data['results'][0]['id'], user_submission_03.id)
        self.assertEqual(response.data['results'][0]['user'], user_03.username)
        self.assertIsNotNone(response.data['results'][0]['submit_time'])
        self.assertEqual(response.data['results'][0]['page_url'], poll.full_url)
        self.assertEqual(response.data['results'][0]['submission'][0]['admin_label'], poll_question.admin_label)
        self.assertEqual(response.data['results'][0]['submission'][0]['clean_name'], poll_question.clean_name)
        self.assertEqual(response.data['results'][0]['submission'][0]['user_answer'], ['c3'])

    def test_questionnaires_detail_page_permission(self):
        current_datetime = timezone.now()
        poll = PollFactory(parent=self.home_page, last_published_at=current_datetime - timedelta(days=3))
        poll_question = PollFormFieldFactory(page=poll, field_type='checkboxes', choices='c1|c2|c3', default_value='c2')

        user_01 = UserFactory()
        form_data_01 = json.dumps({
            poll_question.clean_name: [
                'c1',
            ],
        })
        user_submission_01 = UserSubmissionFactory(page=poll, user=user_01, form_data=form_data_01)
        user_submission_01.submit_time = current_datetime
        user_submission_01.save()

        response = self.client.get(reverse(self.url, kwargs={'pk': poll.id}))

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['count'], 0)
        self.assertIsNone(response.data['next'])
        self.assertIsNone(response.data['previous'])
        self.assertEqual(response.data['results'], [])


class FormDataPerUserAdminTests(TestCase):
    def setUp(self):
        self.group = GroupFactory(name='questionnaires')
        self.admin_access_permission = Permission.objects.get(codename='access_admin')
        self.group.permissions.add(self.admin_access_permission)
        self.staff_user = UserFactory()
        self.staff_user.groups.add(self.group)
        self.client.force_login(self.staff_user)
        self.url = reverse('form_data_per_user')
        self.current_datetime = datetime.datetime(
            year=2022, month=9, day=1, hour=23, minute=0, second=0, microsecond=0, tzinfo=timezone.utc)

        Site.objects.all().delete()
        self.site = SiteFactory(site_name='IoGT', port=8000, is_default_site=True)
        self.home_page = HomePageFactory(parent=self.site.root_page)

        self.poll = PollFactory(parent=self.home_page, last_published_at=self.current_datetime)
        GroupPagePermissionFactory(group=self.group, page=self.poll)
        self.poll_question = PollFormFieldFactory(page=self.poll, field_type='checkboxes', choices='c1|c2|c3', default_value='c2')

        self.survey = SurveyFactory(parent=self.home_page, title='Survey 01', last_published_at=self.current_datetime - timedelta(days=1))
        GroupPagePermissionFactory(group=self.group, page=self.survey)
        self.skip_logic = json.dumps(
            [
                {
                    "type": "skip_logic",
                    "value": {
                        "choice": "c1",
                        "skip_logic": "next",
                        "question": None
                    }
                },
                {
                    "type": "skip_logic",
                    "value": {
                        "choice": "c2",
                        "skip_logic": "next",
                        "question": None
                    }
                },
                {
                    "type": "skip_logic",
                    "value": {
                        "choice": "c3",
                        "skip_logic": "next",
                        "question": None
                    }
                }
            ]
        )
        self.survey_question = SurveyFormFieldFactory(
            page=self.survey, label='Question 01', admin_label='Q 01', field_type='checkboxes',
            skip_logic=self.skip_logic, default_value='c2')

        self.quiz = QuizFactory(parent=self.home_page, last_published_at=self.current_datetime - timedelta(days=2))
        GroupPagePermissionFactory(group=self.group, page=self.quiz)
        self.quiz_question = QuizFormFieldFactory(
            page=self.quiz, field_type='checkboxes', choices='c1|c2|c3', default_value='c2', correct_answer='c3')

        self.user_01 = UserFactory(username='test')
        form_data_01 = json.dumps({
            self.poll_question.clean_name: [
                'c1',
            ],
        })
        self.user_submission_01 = UserSubmissionFactory(page=self.poll, user=self.user_01, form_data=form_data_01)
        self.user_submission_01.submit_time = self.current_datetime
        self.user_submission_01.save()

        form_data_02 = json.dumps({
            self.survey_question.clean_name: [
                'c2',
            ],
        })
        self.user_submission_02 = UserSubmissionFactory(page=self.survey, user=self.user_01, form_data=form_data_02)
        self.user_submission_02.submit_time = self.current_datetime - timedelta(days=1)
        self.user_submission_02.save()

        form_data_03 = json.dumps({
            self.quiz_question.clean_name: [
                'c3',
            ],
        })
        self.user_submission_03 = UserSubmissionFactory(page=self.quiz, user=self.user_01, form_data=form_data_03)
        self.user_submission_03.submit_time = self.current_datetime - timedelta(days=2)
        self.user_submission_03.save()

        self.user_02 = UserFactory()
        form_data_04 = json.dumps({
            self.quiz_question.clean_name: [
                'c3',
            ],
        })
        self.user_submission_04 = UserSubmissionFactory(page=self.quiz, user=self.user_02, form_data=form_data_04)
        self.user_submission_04.submit_time = self.current_datetime - timedelta(days=2)
        self.user_submission_04.save()

    def test_listing(self):
        response = self.client.get(f'{self.url}?user_id={self.user_01.id}')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.context['form_pages'].count(), 3)
        self.assertEqual(response.context['form_pages'][0].title, self.poll.title)
        self.assertEqual(response.context['form_pages'][1].title, self.survey.title)
        self.assertEqual(response.context['form_pages'][2].title, self.quiz.title)

        response = self.client.get(f'{self.url}?user_id={self.user_02.id}')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.context['form_pages'].count(), 1)
        self.assertEqual(response.context['form_pages'][0].title, self.quiz.title)

    def test_date_range_filter(self):
        response = self.client.get(f'{self.url}?user_id={self.user_01.id}&date_from={(self.current_datetime - timedelta(days=1)).date()}')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.context['form_pages'].count(), 2)
        self.assertEqual(response.context['form_pages'][0].title, self.poll.title)
        self.assertEqual(response.context['form_pages'][1].title, self.survey.title)

        response = self.client.get(f'{self.url}?user_id={self.user_01.id}&date_to={(self.current_datetime - timedelta(days=1)).date()}')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.context['form_pages'].count(), 2)
        self.assertEqual(response.context['form_pages'][0].title, self.survey.title)
        self.assertEqual(response.context['form_pages'][1].title, self.quiz.title)

        response = self.client.get(f'{self.url}?user_id={self.user_01.id}&date_from={(self.current_datetime - timedelta(days=1)).date()}&date_to={(self.current_datetime - timedelta(days=1)).date()}')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.context['form_pages'].count(), 1)
        self.assertEqual(response.context['form_pages'][0].title, self.survey.title)

    def test_export_page_ids_filter(self):
        response = self.client.get(f'{self.url}?user_id={self.user_01.id}&export=csv&page_ids={self.poll.id},{self.quiz.id}')

        byte_response = b''
        for stream in response.streaming_content:
            byte_response += stream
        expected_response = \
            f'ID,Name,Submission Date,Field,Value\r\n' \
            f'{self.user_submission_01.id},{self.poll.title},{self.user_submission_01.submit_time},User,{self.user_01.username}\r\n' \
            f'{self.user_submission_01.id},{self.poll.title},{self.user_submission_01.submit_time},URL,{self.poll.full_url}\r\n' \
            f'{self.user_submission_01.id},{self.poll.title},{self.user_submission_01.submit_time},{self.poll_question.clean_name},c1\r\n' \
            f'{self.user_submission_03.id},{self.quiz.title},{self.user_submission_03.submit_time},User,{self.user_01.username}\r\n' \
            f'{self.user_submission_03.id},{self.quiz.title},{self.user_submission_03.submit_time},URL,{self.quiz.full_url}\r\n' \
            f'{self.user_submission_03.id},{self.quiz.title},{self.user_submission_03.submit_time},{self.quiz_question.clean_name},c3\r\n'
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(byte_response.decode(), expected_response)

    def test_csv_export(self):
        response = self.client.get(f'{self.url}?user_id={self.user_01.id}&export=csv')

        byte_response = b''
        for stream in response.streaming_content:
            byte_response += stream
        expected_response = \
            f'ID,Name,Submission Date,Field,Value\r\n' \
            f'{self.user_submission_01.id},{self.poll.title},{self.user_submission_01.submit_time},User,{self.user_01.username}\r\n' \
            f'{self.user_submission_01.id},{self.poll.title},{self.user_submission_01.submit_time},URL,{self.poll.full_url}\r\n' \
            f'{self.user_submission_01.id},{self.poll.title},{self.user_submission_01.submit_time},{self.poll_question.admin_label},c1\r\n' \
            f'{self.user_submission_02.id},{self.survey.title},{self.user_submission_02.submit_time},User,{self.user_01.username}\r\n' \
            f'{self.user_submission_02.id},{self.survey.title},{self.user_submission_02.submit_time},URL,{self.survey.full_url}\r\n' \
            f'{self.user_submission_02.id},{self.survey.title},{self.user_submission_02.submit_time},{self.survey_question.admin_label},c2\r\n' \
            f'{self.user_submission_03.id},{self.quiz.title},{self.user_submission_03.submit_time},User,{self.user_01.username}\r\n' \
            f'{self.user_submission_03.id},{self.quiz.title},{self.user_submission_03.submit_time},URL,{self.quiz.full_url}\r\n' \
            f'{self.user_submission_03.id},{self.quiz.title},{self.user_submission_03.submit_time},{self.quiz_question.admin_label},c3\r\n'
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(byte_response.decode(), expected_response)

    def test_xlsx_export(self):
        response = self.client.get(f'{self.url}?user_id={self.user_01.id}&export=xlsx')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        sheet = load_workbook(io.BytesIO(response.content)).active
        expected_response = {
            'A': {
                1: 'ID',
                2: f'{self.user_submission_01.id}',
                3: f'{self.user_submission_01.id}',
                4: f'{self.user_submission_01.id}',
                5: f'{self.user_submission_02.id}',
                6: f'{self.user_submission_02.id}',
                7: f'{self.user_submission_02.id}',
                8: f'{self.user_submission_03.id}',
                9: f'{self.user_submission_03.id}',
                10: f'{self.user_submission_03.id}',
            },
            'B': {
                1: 'Name',
                2: self.poll.title,
                3: self.poll.title,
                4: self.poll.title,
                5: self.survey.title,
                6: self.survey.title,
                7: self.survey.title,
                8: self.quiz.title,
                9: self.quiz.title,
                10: self.quiz.title,
            },
            'C': {
                1: 'Submission Date',
                2: self.user_submission_01.submit_time,
                3: self.user_submission_01.submit_time,
                4: self.user_submission_01.submit_time,
                5: self.user_submission_02.submit_time,
                6: self.user_submission_02.submit_time,
                7: self.user_submission_02.submit_time,
                8: self.user_submission_03.submit_time,
                9: self.user_submission_03.submit_time,
                10: self.user_submission_03.submit_time,
            },
            'D': {
                1: 'Field',
                2: 'User',
                3: 'URL',
                4: self.poll_question.admin_label,
                5: 'User',
                6: 'URL',
                7: self.survey_question.admin_label,
                8: 'User',
                9: 'URL',
                10: self.quiz_question.admin_label,
            },
            'E': {
                1: 'Value',
                2: self.user_01.username,
                3: self.poll.full_url,
                4: 'c1',
                5: self.user_01.username,
                6: self.survey.full_url,
                7: 'c2',
                8: self.user_01.username,
                9: self.quiz.full_url,
                10: 'c3',
            },
        }

        for row in [1, 2, 3, 4, 5, 6, 7]:
            for column in ['A', 'B', 'C', 'D', 'E']:
                value = sheet[f'{column}{row}'].value
                if isinstance(value, datetime.datetime):
                    value = value.replace(tzinfo=pytz.UTC)
                self.assertEqual(value, expected_response[column][row])

    def test_page_permission(self):
        quiz = QuizFactory(parent=self.home_page, last_published_at=self.current_datetime - timedelta(days=2))
        quiz_question = QuizFormFieldFactory(
            page=quiz, field_type='checkboxes', choices='c1|c2|c3', default_value='c2', correct_answer='c3')

        form_data_04 = json.dumps({
            quiz_question.clean_name: [
                'c3',
            ],
        })
        UserSubmissionFactory(page=quiz, user=self.user_01, form_data=form_data_04)

        response = self.client.get(f'{self.url}?user_id={self.user_01.id}')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.context['form_pages'].count(), 3)
        self.assertEqual(response.context['form_pages'][0].title, self.poll.title)
        self.assertEqual(response.context['form_pages'][1].title, self.survey.title)
        self.assertEqual(response.context['form_pages'][2].title, self.quiz.title)

    def test_csv_export_with_modified_questions(self):
        self.survey_question.delete()
        survey_question = SurveyFormFieldFactory(
            page=self.survey, label='Question 02', admin_label='Q 02', field_type='checkboxes',
            skip_logic=self.skip_logic, default_value='c3')
        form_data = json.dumps({
            survey_question.clean_name: [
                'c3',
            ],
        })
        user_submission = UserSubmissionFactory(page=self.survey, user=self.user_01, form_data=form_data)
        user_submission.submit_time = self.current_datetime
        user_submission.save()
        response = self.client.get(f'{self.url}?user_id={self.user_01.id}&export=csv&page_ids={self.survey.id}')

        byte_response = b''
        for stream in response.streaming_content:
            byte_response += stream
        expected_response = \
            f'ID,Name,Submission Date,Field,Value\r\n' \
            f'{user_submission.id},Survey 01,2022-09-01 23:00:00+00:00,User,test\r\n' \
            f'{user_submission.id},Survey 01,2022-09-01 23:00:00+00:00,URL,{self.survey.full_url}\r\n' \
            f'{user_submission.id},Survey 01,2022-09-01 23:00:00+00:00,Q 02,c3\r\n' \
            f'{self.user_submission_02.id},Survey 01,2022-08-31 23:00:00+00:00,User,test\r\n' \
            f'{self.user_submission_02.id},Survey 01,2022-08-31 23:00:00+00:00,URL,{self.survey.full_url}\r\n' \
            f'{self.user_submission_02.id},Survey 01,2022-08-31 23:00:00+00:00,question_01,c2\r\n'
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(byte_response.decode(), expected_response)


class PollTest(TestCase):
    def setUp(self):
        root_page = PageFactory(parent=None)
        home_page = HomePageFactory(parent=root_page)
        SiteFactory(hostname='testserver', port=80, root_page=home_page)
        self.poll = PollFactory(parent=home_page)

    def test_drop_down_defaults_to_the_blank_option(self):
        PollFormFieldFactory(page=self.poll, field_type='dropdown', choices='c1|c2')

        response = self.client.get(self.poll.url)
        parsed_response = parse_html(response.content)
        default_drop_down_option = parsed_response.select_one('.quest-item select option').text

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(default_drop_down_option, '')

    def test_drop_down_blank_option_submission_for_required_field(self):
        poll_question = PollFormFieldFactory(page=self.poll, field_type='dropdown', choices='c1|c2')

        response = self.client.post(self.poll.url, {poll_question.clean_name: ''})
        parsed_response = parse_html(response.content)
        field_required_text = parsed_response.select_one('.quest-item .cust-input__error').text

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(field_required_text, 'This field is required.')

    def test_drop_down_blank_option_submission_for_non_required_field(self):
        poll_question = PollFormFieldFactory(page=self.poll, field_type='dropdown', choices='c1|c2', required=False)

        response = self.client.post(self.poll.url, {poll_question.clean_name: ''})
        parsed_response = parse_html(response.content)
        results = parsed_response.select('.cust-check__title')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(results), 2)
        self.assertEqual(results[0].select_one('.cust-check__title-left').text.strip(), 'c1')
        self.assertEqual(" ".join(results[0].select_one('.cust-check__title-right').text.split()), '0 %')
        self.assertEqual(results[1].select_one('.cust-check__title-left').text.strip(), 'c2')
        self.assertEqual(" ".join(results[1].select_one('.cust-check__title-right').text.split()), '0 %')

    def test_drop_down_non_blank_option_submission(self):
        poll_question = PollFormFieldFactory(page=self.poll, field_type='dropdown', choices='c1|c2')

        response = self.client.post(self.poll.url, {poll_question.clean_name: 'c1'})
        parsed_response = parse_html(response.content)
        results = parsed_response.select('.cust-check__title')

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(results), 2)
        self.assertEqual(results[0].select_one('.cust-check__title-left').text.strip(), 'c1\nYour answer')
        self.assertEqual(" ".join(results[0].select_one('.cust-check__title-right').text.split()), '100 %')
        self.assertEqual(results[1].select_one('.cust-check__title-left').text.strip(), 'c2')
        self.assertEqual(" ".join(results[1].select_one('.cust-check__title-right').text.split()), '0 %')


class TestQuestionnaireSubmitButtonText(TestCase):
    def setUp(self):
        root_page = PageFactory(parent=None)
        self.en_home_page = HomePageFactory(parent=root_page)

    def test_survey_submit_button_text_without_multi_step_enabled(self):
        survey = SurveyFactory(parent=self.en_home_page, multi_step=False)
        SurveyFormFieldFactory(page=survey, field_type='singleline')

        self.assertEqual(survey.get_submit_button_text(), "Submit")

    def test_survey_submit_button_text_with_multi_step_enabled(self):
        survey = SurveyFactory(parent=self.en_home_page, multi_step=True)
        SurveyFormFieldFactory(page=survey, field_type='multiline')
        SurveyFormFieldFactory(page=survey, field_type='multiline')

        # This page has 2 steps
        paginator = SkipLogicPaginator(survey.get_form_fields(), {}, {})
        step1 = paginator.page(1)
        step2 = paginator.page(2)

        self.assertEqual(survey.get_submit_button_text(step1), "Next")
        self.assertEqual(survey.get_submit_button_text(step2), "Submit")

    def test_poll_submit_button_text(self):
        poll = PollFactory(parent=self.en_home_page)
        PollFormFieldFactory(page=poll, field_type='checkboxes', choices='c1|c2|c3')

        self.assertEqual(poll.get_submit_button_text(), "Submit")


# questionnaires/api/v1/urls.py

from django.urls import path

from questionnaires.api.v1.views import (
    QuestionnairesListAPIView,
    QuestionnaireDetailAPIView,
    QuestionnaireSubmissionsAPIView,
)

urlpatterns = [
    path('', QuestionnairesListAPIView.as_view(), name='questionnaires_list'),
    path('<int:pk>/', QuestionnaireDetailAPIView.as_view(), name='questionnaire_detail'),
    path('<int:pk>/submissions/', QuestionnaireSubmissionsAPIView.as_view(), name='questionnaire_submissions'),
]


# questionnaires/api/v1/views.py

from django.utils.decorators import method_decorator
from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema
from rest_framework import status
from rest_framework.generics import ListAPIView, RetrieveAPIView
from rest_framework.permissions import IsAuthenticated
from wagtail.contrib.forms.utils import get_forms_for_user

from questionnaires.filters import QuestionnaireFilter, UserSubmissionFilter
from questionnaires.models import Survey, Poll, Quiz, UserSubmission
from questionnaires.paginators import IoGTPagination
from questionnaires.api.v1.serializers import (
    QuestionnairePageSerializer,
    SurveyPageDetailSerializer,
    PollPageDetailSerializer,
    QuizPageDetailSerializer,
    UserSubmissionSerializer,
    QuestionnairePageDetailSerializer,
)


@method_decorator(name='get', decorator=swagger_auto_schema(
    manual_parameters=[
        openapi.Parameter('last_published_at_start', openapi.IN_QUERY, format="YYYY-MM-DD", type=openapi.TYPE_STRING),
        openapi.Parameter('last_published_at_end', openapi.IN_QUERY, format="YYYY-MM-DD", type=openapi.TYPE_STRING),
        openapi.Parameter('type', openapi.IN_QUERY, enum=['poll', 'survey', 'quiz'], type=openapi.TYPE_STRING),
    ],
    responses={
        status.HTTP_200_OK: openapi.Response(
            description="Questionnaires List API",
            schema=QuestionnairePageSerializer,
        )
    }
))
class QuestionnairesListAPIView(ListAPIView):
    permission_classes = (IsAuthenticated,)
    serializer_class = QuestionnairePageSerializer
    filterset_class = QuestionnaireFilter
    pagination_class = IoGTPagination

    def get_queryset(self):
        return get_forms_for_user(self.request.user).order_by('-last_published_at')


@method_decorator(name='get', decorator=swagger_auto_schema(
    responses={
        status.HTTP_200_OK: openapi.Response(
            description="Questionnaire Page Detail API",
            schema=QuestionnairePageDetailSerializer,
        )
    }
))
class QuestionnaireDetailAPIView(RetrieveAPIView):
    permission_classes = (IsAuthenticated,)

    def get_queryset(self):
        return get_forms_for_user(self.request.user).specific().order_by('-last_published_at')

    def get_serializer_class(self):
        page = self.get_object()
        if isinstance(page, Poll):
            return PollPageDetailSerializer
        elif isinstance(page, Survey):
            return SurveyPageDetailSerializer
        elif isinstance(page, Quiz):
            return QuizPageDetailSerializer


@method_decorator(name='get', decorator=swagger_auto_schema(
    manual_parameters=[
        openapi.Parameter('submit_time_start', openapi.IN_QUERY, format="YYYY-MM-DD", type=openapi.TYPE_STRING),
        openapi.Parameter('submit_time_end', openapi.IN_QUERY, format="YYYY-MM-DD", type=openapi.TYPE_STRING),
        openapi.Parameter('user_ids', openapi.IN_QUERY, format='Comma (,) separated with no spaces', type=openapi.TYPE_STRING),
    ],
    responses={
        status.HTTP_200_OK: openapi.Response(
            description="Questionnaire Submissions List API",
            schema=UserSubmissionSerializer,
        )
    }
))
class QuestionnaireSubmissionsAPIView(ListAPIView):
    permission_classes = (IsAuthenticated,)
    serializer_class = UserSubmissionSerializer
    filterset_class = UserSubmissionFilter
    pagination_class = IoGTPagination

    def get_queryset(self):
        return UserSubmission.objects.filter(
            page_id=self.kwargs.get(self.lookup_field),
            page__in=get_forms_for_user(self.request.user)
        ).select_related(
            'user', 'page'
        ).order_by('-submit_time')


# questionnaires/management/commands/create_questionnaires_index_pages.py

from django.contrib.contenttypes.models import ContentType
from django.core.management.base import BaseCommand

from home.models import HomePage
from questionnaires.models import PollIndexPage, SurveyIndexPage, QuizIndexPage


class Command(BaseCommand):

    def handle(self, *args, **options):
        homepage = HomePage.objects.get(slug='home')
        poll_index_page = PollIndexPage(title='Polls')
        survey_index_page = SurveyIndexPage(title='Surveys')
        quiz_index_page = QuizIndexPage(title='Quizzes')

        try:
            homepage.add_child(instance=poll_index_page)
            homepage.add_child(instance=survey_index_page)
            homepage.add_child(instance=quiz_index_page)
            self.stdout.write(self.style.SUCCESS('Index pages added.'))
        except:
            pass


# questionnaires/management/commands/fix_questionnaires_choices.py

from django.contrib.contenttypes.models import ContentType
from django.core.management.base import BaseCommand

from home.models import HomePage
from questionnaires.models import PollIndexPage, SurveyIndexPage, QuizIndexPage, PollFormField, SurveyFormField, \
    QuizFormField


class Command(BaseCommand):

    def handle(self, *args, **options):
        poll_form_fields = PollFormField.objects.all()
        poll_form_field_list = []
        for poll_form_field in poll_form_fields:
            poll_form_field.choices = '|'.join(poll_form_field.choices.split(','))
            poll_form_field_list.append(poll_form_field)

        PollFormField.objects.bulk_update(poll_form_field_list, ['choices'], batch_size=1000)

        survey_form_fields = SurveyFormField.objects.all()
        survey_form_field_list = []
        for survey_form_field in survey_form_fields:
            survey_form_field.choices = '|'.join(survey_form_field.choices.split(','))
            survey_form_field_list.append(survey_form_field)

        SurveyFormField.objects.bulk_update(survey_form_field_list, ['choices'], batch_size=1000)

        quiz_form_fields = QuizFormField.objects.all()
        quiz_form_field_list = []
        for quiz_form_field in quiz_form_fields:
            quiz_form_field.choices = '|'.join(quiz_form_field.choices.split(','))
            quiz_form_field.correct_answer = '|'.join(quiz_form_field.correct_answer.split(','))
            quiz_form_field_list.append(quiz_form_field)

        QuizFormField.objects.bulk_update(quiz_form_field_list, ['choices', 'correct_answer'], batch_size=1000)

        self.stdout.write(self.style.SUCCESS('Fixed questionnaires choices.'))


# questionnaires/management/commands/fix_survey_skip_logic.py

from django.core.management.base import BaseCommand

from questionnaires.models import SurveyFormField


class Command(BaseCommand):

    def handle(self, *args, **options):
        survey_form_fields = SurveyFormField.objects.all()
        for survey_form_field in survey_form_fields:
            if survey_form_field.skip_logic:
                skip_logics = survey_form_field.skip_logic.raw_data
                for skip_logic in skip_logics:
                    skip_logic.get('value', {}).pop('survey', None)

        SurveyFormField.objects.bulk_update(survey_form_fields, fields=['skip_logic'], batch_size=1000)

        self.stdout.write(self.style.SUCCESS('Fixed surveys skip logic.'))


# comments/migrations/0001_initial.py

# Generated by Django 3.1.13 on 2021-07-07 07:02

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='CannedResponse',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('text', models.TextField(null=True)),
            ],
        ),
    ]


# comments/migrations/0002_cannedresponse_header.py

# Generated by Django 3.1.13 on 2021-09-20 12:10

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('comments', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='cannedresponse',
            name='header',
            field=models.TextField(null=True),
        ),
    ]


# comments/migrations/0003_commentmoderation.py

# Generated by Django 3.1.14 on 2022-09-30 14:52

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('django_comments_xtd', '0008_auto_20200920_2037'),
        ('comments', '0002_cannedresponse_header'),
    ]

    operations = [
        migrations.CreateModel(
            name='CommentModeration',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('state', models.CharField(choices=[('UNMODERATED', 'Unmoderated'), ('APPROVED', 'Approved'), ('REJECTED', 'Rejected'), ('UNSURE', 'Unsure')], default='UNMODERATED', max_length=255)),
                ('comment', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='comment_moderation', to='django_comments_xtd.xtdcomment')),
            ],
        ),
    ]


# comments/templatetags/comment_tags.py

from django import template
from django.conf import settings
from django_comments_xtd.models import LIKEDIT_FLAG, DISLIKEDIT_FLAG

register = template.Library()


@register.simple_tag
def user_can_report(comment, user):
    if user.is_anonymous:
        return False
    return (comment.user != user) and not comment.flags.filter(user=user).exists()


@register.simple_tag
def filter_removed_comments(comments):
    return [comment for comment in comments if not comment['comment'].is_removed]


@register.simple_tag
def get_comments_page(comments, request):
    return list(reversed(comments))[:get_current_num_records(request)]


@register.simple_tag
def has_more_comments(comments, request):
    return get_current_num_records(request) < len(comments)


@register.simple_tag
def get_current_num_records(request):
    return int(request.GET.get('num_records', 5))


@register.simple_tag
def get_next_num_records(request):
    current_num_records = int(request.GET.get('num_records', 5))
    return current_num_records + 5


@register.simple_tag
def get_comment_report_count(comment):
    return comment.flags.exclude(flag__in=[LIKEDIT_FLAG, DISLIKEDIT_FLAG]).count()


@register.inclusion_tag('comments/community_moderation_button.html', takes_context=True)
def comments_moderation_button(context):
    request = context['request']
    context.update({
        'show_comments_community_moderation_button': settings.COMMENTS_COMMUNITY_MODERATION and request.user.has_perm('django_comments_xtd.can_moderate')
    })
    return context


# comments/templatetags/wagtail_comments_xtd_tags.py

from django import template


register = template.Library()


@register.inclusion_tag('wagtail_comments_xtd/partials/comments_breadcrumb.html', takes_context=True)
def comments_breadcrumb(context):
    return context


# common/translation_utils/make_pos.py

import polib
import csv
import copy
import os
from common.translation_utils.po_template import get_po_metadata_template

class Translation(object):
    pass

def make_translation():

    po_metadata_template = get_po_metadata_template()

    translation = Translation()
    po = polib.POFile()
    po.metadata = copy.deepcopy(po_metadata_template)
    translation.pofile = po
    return translation

def make_pos_run():

    sheet0 = open("common/translation_utils/translation_status.csv", newline='')
    status_data = list(csv.reader(sheet0))
    js_phrases = set()
    used_phrases = set()
    for row in status_data:
        if row[3] == 'js':
            js_phrases.add(row[0])
        if row[2] == 'translate' and row[4] != 'unused':
            used_phrases.add(row[0])

    processed_phrases = set()


    sheet = open("common/translation_utils/translations.csv", newline='')
    reader = csv.reader(sheet)

    translations = []
    translationsjs = []
    # Compile PO files from official translation sheet
    for i, row in enumerate(reader):
        if i == 0:
            for lang in row[4:]:
                translations.append(make_translation())
                translationsjs.append(make_translation())
        if row[0] == 'll':
            for translation, phrase in zip(translations, row[4:]):
                translation.path = "locale/{}/LC_MESSAGES/".format(phrase)
            for translation, phrase in zip(translationsjs, row[4:]):
                translation.path = "locale/{}/LC_MESSAGES/".format(phrase)
        if row[0] == 'Language':
            for translation, phrase in zip(translations, row[4:]):
                translation.path = "locale/{}/LC_MESSAGES/".format(phrase)
            for translation, phrase in zip(translationsjs, row[4:]):
                translation.pofile.metadata["Language"] = phrase
                translation.path = "locale/{}/LC_MESSAGES/".format(phrase)

        phrase_eng = row[3]
        if phrase_eng not in used_phrases:
            continue

        translation_list = translations
        if phrase_eng in js_phrases:
            translation_list = translationsjs

        for translation, phrase in zip(translation_list, row[4:]):
            if row[0] == 'Section':
                continue
            if row[0] == '':
                entry = polib.POEntry(
                    msgid=phrase_eng,
                    msgstr=phrase,
                )
                translation.pofile.append(entry)
                processed_phrases.add(phrase_eng)
            if row[0] == 'sg':
                translation.sg_buffer = phrase
                translation.sg_buffer_eng = phrase_eng
            if row[0] == 'pl':
                if not hasattr(translation, "sg_buffer"):
                    print("Warning: Row {} has no matching singular.".format(i))
                    continue
                entry = polib.POEntry(
                    msgid=translation.sg_buffer_eng,
                    msgid_plural=phrase_eng,
                    msgstr_plural={0: translation.sg_buffer, 1: phrase}
                )
                translation.pofile.append(entry)
                processed_phrases.add(translation.sg_buffer_eng)
                processed_phrases.add(phrase_eng)

    sheet.close()

    translatable_strings = []
    # Update with translations that we need but don't have
    for i, row in enumerate(status_data):
        phrase = row[0]
        if i == 0:
            continue
        if row[2] == 'translate' and row[4] != 'unused':
            if phrase not in processed_phrases:
                assert not row[4].startswith('has translation')
                translation_list = translations
                if row[3] == 'js':
                    translation_list = translationsjs
                for translation in translation_list:
                    entry = polib.POEntry(
                        msgid=phrase,
                        msgstr='',
                    )
                    translation.pofile.append(entry)                
                processed_phrases.add(phrase)

            if row[4] == 'needs translation' or row[4] == 'has partial translation':
                translatable_strings.append(phrase)

    # Write output
    for translation in translations:
        os.makedirs(translation.path, exist_ok=True)
        filename = os.path.join(translation.path, "django.po")
        translation.pofile.save(filename)
    for translation in translationsjs:
        filename = os.path.join(translation.path, "djangojs.po")
        translation.pofile.save(filename)

    with open("home/translatable_strings.py", 'w') as f:
        content = ",\n    ".join(e.__repr__() for e in translatable_strings)
        f.write(f'translatable_strings = [\n    {content}\n]')


# common/translation_utils/make_testing_po.py

import polib
import csv
import copy
import os
from common.translation_utils.po_template import get_po_metadata_template

def make_testing_po_run():

    po_metadata_template = get_po_metadata_template()

    sheet = open("common/translation_utils/translation_status.csv", newline='')
    reader = csv.reader(sheet)

    sheet2 = open("common/translation_utils/translations.csv", newline='')
    data = list(csv.reader(sheet2))
    all_phrases = {r[3] : r for r in data}
    processed_phrases = set()

    entries = []
    entriesjs = []
    for i, row in enumerate(reader):
        if i == 0:
            po = polib.POFile()
            po.metadata = copy.deepcopy(po_metadata_template)
            po.metadata["Language"] = "Testing"
            pojs = polib.POFile()
            pojs.metadata = copy.deepcopy(po_metadata_template)
            pojs.metadata["Language"] = "Testing"
            continue
        plural = False
        phrase = row[0]
        if phrase in processed_phrases:
            continue
        processed_phrases.add(phrase)
        if phrase in all_phrases and all_phrases[phrase][0] == 'pl':
            plural = True
        tag = row[4].replace(' ', '-')
        trans = f"([{tag}]-{phrase})"

        if plural:
            entry = entries[-1]
            new_entry = polib.POEntry(
                msgid=entry.msgid,
                msgid_plural=phrase,
                msgstr_plural={0: entry.msgstr, 1: trans}
            )
            entries[-1] = new_entry
        else:
            entry = polib.POEntry(
                msgid=phrase,
                msgstr=trans,
            )
            if row[3] == 'js':
                entriesjs.append(entry)
            else:
                entries.append(entry)

    for entry in entries:
        po.append(entry)
    for entry in entriesjs:
        pojs.append(entry)

    sheet.close()

    os.makedirs("locale/xy/LC_MESSAGES/", exist_ok=True)
    po.save("locale/xy/LC_MESSAGES/django.po")
    pojs.save("locale/xy/LC_MESSAGES/djangojs.po")


# common/translation_utils/po_template.py

def get_po_metadata_template():
    po_metadata_template = {
        "Report-Msgid-Bugs-To": "",
        "POT-Creation-Date": "2021-07-31 09:38+0000",
        "PO-Revision-Date": "2021-07-31 09:38+0000",
        "Last-Translator": "FULL NAME <EMAIL@ADDRESS>",
        "Language-Team": "LANGUAGE <LL@li.org>",
        "Language": "",
        "MIME-Version": "1.0",
        "Content-Type": "text/plain; charset=UTF-8",
        "Content-Transfer-Encoding": "8bit",
        "Project-Id-Version": "PACKAGE VERSION",
        "Plural-Forms": "nplurals=2; plural=(n > 1);",
    }
    return po_metadata_template


# common/translation_utils/update_status.py

import csv
import polib

class Translation:
    def __init__(self, row):
        self.phrase = row[3]
        self.bad_locales = row[1].split(',')
        self.bad_source_phrase = row[2]
        self.tag = row[0]
        self.translations = row[3:]

    def is_complete(self):
        return all(bool(entry) for entry in self.translations)

    def is_phrase(row):
        return row[0] in ['', 'sg', 'pl']

def lookup_status(msg, translations, replacement_strings=[]):
    status = 'needs translation'
    if msg in translations:
        tr = translations[msg]
        if tr.is_complete():
            status = 'has translation'
        else:
            status = 'has partial translation'
    else:
        for replacement_string in replacement_strings:
            if replacement_string:
                if replacement_string in translations:
                    tr = translations[replacement_string]
                    status = 'TODO'
                else:
                    print(f'Warning: Replacement string "{replacement_string}" for message "{msg}" not in translations.')
    return status

def process_message(msg, po_msgs, logged_phrases, status_sheet, translations, is_js=False):
    po_msgs.add(msg)
    if msg not in logged_phrases:
        status = lookup_status(msg, translations)
        status_sheet.append([msg, '', '', 'js' if is_js else '', status, '', '', ''])
    else:
        status_row = status_sheet[logged_phrases[msg]]
        if is_js:
            status_row[3] = 'js'
        tag = status_row[2]
        replacement_strings = status_row[6].split('|')
        if tag == 'not needed':
            status_row[4] = 'not needed'
        elif tag == 'remove':
            status_row[4] = 'TODO'
        else:
            status_row[4] = lookup_status(msg, translations, replacement_strings)
            if status_row[4] == 'needs translation' and status_row[3] == 'django':
                status_row[4] = 'needs translation (partial)'

def process_po(po, po_msgs, logged_phrases, status_sheet, translations, is_js=False):
    for entry in po:
        if entry.obsolete:
            continue
        process_message(entry.msgid, po_msgs, logged_phrases, status_sheet, translations, is_js)
        if entry.msgid_plural:
            process_message(entry.msgid_plural, po_msgs, logged_phrases, status_sheet, translations, is_js)

def update_status_run():

    sheet = open("common/translation_utils/translation_status.csv", newline='')
    status_sheet = list(csv.reader(sheet))
    logged_phrases = {status_sheet[i][0] : i for i in range(1, len(status_sheet))}

    sheet2 = open("common/translation_utils/translations.csv", newline='')
    translations = {r[3] : Translation(r) for r in list(csv.reader(sheet2)) if Translation.is_phrase(r)}
    po_msgs = set()

    # process those rows that are in the PO file
    po = polib.pofile('locale/django.pot', encoding='UTF-8')
    process_po(po, po_msgs, logged_phrases, status_sheet, translations)
    pojs = polib.pofile('locale/djangojs.pot', encoding='UTF-8')
    process_po(pojs, po_msgs, logged_phrases, status_sheet, translations, is_js=True)

    # Process the remaining rows
    for status_row in status_sheet[1:]:
        if status_row[0] not in po_msgs:
            if status_row[3] == 'django':
                process_message(status_row[0], po_msgs, logged_phrases, status_sheet, translations)
            else:
                status_row[4] = 'unused'


    out_sheet = open("common/translation_utils/translation_status.csv", 'w', newline='')
    writer = csv.writer(out_sheet)
    writer.writerows(status_sheet)


# home/migrations/0002_auto_20210707_0742.py

# Generated by Django 3.1.13 on 2021-07-07 07:42

from django.db import migrations
from wagtailmenus.models import FlatMenu


def _create_homepage(apps, schema_editor):
    # Get models
    ContentType = apps.get_model('contenttypes.ContentType')
    Page = apps.get_model('wagtailcore.Page')
    Site = apps.get_model('wagtailcore.Site')
    HomePage = apps.get_model('home.HomePage')

    # Delete the default homepage
    # If migration is run multiple times, it may have already been deleted
    Page.objects.filter(id=2).delete()

    # Create content type for homepage model
    homepage_content_type, __ = ContentType.objects.get_or_create(
        model='homepage', app_label='home')

    # Create a new homepage
    homepage = HomePage.objects.create(
        title="Home",
        draft_title="Home",
        slug='home',
        content_type=homepage_content_type,
        path='00010001',
        depth=2,
        numchild=0,
        url_path='/home/',
        show_in_menus=True
    )

    # Create a site with the new homepage set as the root
    Site.objects.create(
        hostname='localhost', root_page=homepage, is_default_site=True)


def _delete_homepage(apps, schema_editor):
    # Get models
    ContentType = apps.get_model('contenttypes.ContentType')
    Site = apps.get_model('wagtailcore.Site')
    HomePage = apps.get_model('home.HomePage')

    # Delete the default homepage
    # Page and Site objects CASCADE
    HomePage.objects.filter(slug='home', depth=2).delete()

    # Delete content type for homepage model
    ContentType.objects.filter(model='homepage', app_label='home').delete()

    Site.objects.all().delete()


class Migration(migrations.Migration):

    run_before = [
        ('wagtailcore', '0053_locale_model'),
    ]

    dependencies = [
        ('home', '0001_initial'),
    ]

    operations = [
        migrations.RunPython(_create_homepage, _delete_homepage),
    ]


# home/migrations/0003_auto_20210707_0702.py

# Generated by Django 3.1.13 on 2021-07-07 07:02

from django.db import migrations, models
import django.db.models.deletion
import modelcluster.contrib.taggit
import modelcluster.fields


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ('taggit', '0003_taggeditem_add_unique_index'),
        ('wagtailmenus', '0023_remove_use_specific'),
        ('wagtailsvg', '0002_svg_edit_code'),
        ('home', '0002_auto_20210707_0742'),
        ('wagtailimages', '0022_uploadedimage'),
        ('questionnaires', '0001_initial'),
        ('wagtailcore', '0059_apply_collection_ordering'),
    ]

    operations = [
        migrations.AddField(
            model_name='sitesettings',
            name='registration_survey',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='questionnaires.survey'),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='site',
            field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.site'),
        ),
        migrations.AddField(
            model_name='sectiontaggeditem',
            name='content_object',
            field=modelcluster.fields.ParentalKey(on_delete=django.db.models.deletion.CASCADE, related_name='tagged_items', to='home.section'),
        ),
        migrations.AddField(
            model_name='sectiontaggeditem',
            name='tag',
            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='home_sectiontaggeditem_items', to='taggit.tag'),
        ),
        migrations.AddField(
            model_name='section',
            name='icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailsvg.svg'),
        ),
        migrations.AddField(
            model_name='section',
            name='lead_image',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='section',
            name='tags',
            field=modelcluster.contrib.taggit.ClusterTaggableManager(blank=True, help_text='A comma-separated list of tags.', through='home.SectionTaggedItem', to='taggit.Tag', verbose_name='Tags'),
        ),
        migrations.AddField(
            model_name='iogtflatmenuitem',
            name='icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailsvg.svg'),
        ),
        migrations.AddField(
            model_name='iogtflatmenuitem',
            name='link_page',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.page', verbose_name='link to an internal page'),
        ),
        migrations.AddField(
            model_name='iogtflatmenuitem',
            name='menu',
            field=modelcluster.fields.ParentalKey(on_delete=django.db.models.deletion.CASCADE, related_name='iogt_flat_menu_items', to='wagtailmenus.flatmenu'),
        ),
        migrations.AddField(
            model_name='homepagebanner',
            name='banner_page',
            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.bannerpage'),
        ),
        migrations.AddField(
            model_name='homepagebanner',
            name='source',
            field=modelcluster.fields.ParentalKey(blank=True, on_delete=django.db.models.deletion.CASCADE, related_name='home_page_banners', to='wagtailcore.page'),
        ),
        migrations.AddField(
            model_name='featuredcontent',
            name='content',
            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.page'),
        ),
        migrations.AddField(
            model_name='featuredcontent',
            name='source',
            field=modelcluster.fields.ParentalKey(blank=True, on_delete=django.db.models.deletion.CASCADE, related_name='featured_content', to='wagtailcore.page'),
        ),
        migrations.AddField(
            model_name='cachesettings',
            name='site',
            field=models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.site'),
        ),
        migrations.AddField(
            model_name='bannerpage',
            name='banner_background_image',
            field=models.ForeignKey(blank=True, help_text='Background image', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='bannerpage',
            name='banner_icon_button',
            field=models.ForeignKey(blank=True, help_text='Icon Button', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='bannerpage',
            name='banner_image',
            field=models.ForeignKey(blank=True, help_text='Image to display as the banner', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='bannerpage',
            name='banner_link_page',
            field=models.ForeignKey(blank=True, help_text='Optional page to which the banner will link to', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='banners', to='wagtailcore.page'),
        ),
        migrations.AddField(
            model_name='articletaggeditem',
            name='content_object',
            field=modelcluster.fields.ParentalKey(on_delete=django.db.models.deletion.CASCADE, related_name='tagged_items', to='home.article'),
        ),
        migrations.AddField(
            model_name='articletaggeditem',
            name='tag',
            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='home_articletaggeditem_items', to='taggit.tag'),
        ),
        migrations.AddField(
            model_name='articlerecommendation',
            name='article',
            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.article'),
        ),
        migrations.AddField(
            model_name='articlerecommendation',
            name='source',
            field=modelcluster.fields.ParentalKey(blank=True, on_delete=django.db.models.deletion.CASCADE, related_name='recommended_articles', to='home.article'),
        ),
        migrations.AddField(
            model_name='article',
            name='icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailsvg.svg'),
        ),
        migrations.AddField(
            model_name='article',
            name='lead_image',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='article',
            name='tags',
            field=modelcluster.contrib.taggit.ClusterTaggableManager(blank=True, help_text='A comma-separated list of tags.', through='home.ArticleTaggedItem', to='taggit.Tag', verbose_name='Tags'),
        ),
    ]


# home/migrations/0004_auto_20210707_1005.py

# Generated by Django 3.1.13 on 2021-07-07 10:05
from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0003_auto_20210707_0702'),
        ('wagtail_localize', '0012_localesynchronization'),
    ]

    operations = [
    ]


# home/migrations/0005_manifestsettings.py

# Generated by Django 3.1.13 on 2021-07-13 11:46

from django.db import migrations, models
import django.db.models.deletion
import home.models


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailimages', '0022_uploadedimage'),
        ('home', '0004_auto_20210707_1005'),
    ]

    operations = [
        migrations.CreateModel(
            name='ManifestSettings',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.CharField(help_text='Provide name', max_length=255, verbose_name='Name')),
                ('short_name', models.CharField(help_text='Provide short name', max_length=255, verbose_name='Short name')),
                ('scope', models.CharField(help_text='Provide scope', max_length=255, verbose_name='Scope')),
                ('start_url', models.CharField(help_text='Provide start URL', max_length=255, verbose_name='Start URL')),
                ('display', models.CharField(choices=[('FULLSCREEN', 'fullscreen'), ('STANDALONE', 'standalone'), ('MINIMAL_UI', 'minimal-ui'), ('BROWSER', 'browser')], help_text='Provide browser UI', max_length=255, verbose_name='Browser UI')),
                ('background_color', models.CharField(help_text='Provide background color (example: #FFF)', max_length=10, verbose_name='Background color')),
                ('theme_color', models.CharField(help_text='Provide theme color(example: #493174)', max_length=10, verbose_name='Theme color')),
                ('description', models.CharField(help_text='Provide description', max_length=500, verbose_name='Description')),
                ('language', models.CharField(choices=[('ar', 'Arabic'), ('ch', 'Chichewa'), ('en', 'English'), ('fr', 'French'), ('km', 'Khmer'), ('rw', 'Kinyarwanda'), ('rn', 'Kirundi'), ('ku', 'Kurdish'), ('mg', 'Malagasy'), ('ne', 'Nepali'), ('nr', 'Ndebele'), ('pt', 'Portuguese'), ('qu', 'Quechua'), ('ru', 'Russian'), ('sho', 'Shona'), ('es', 'Spanish'), ('sw', 'Swahili'), ('tg', 'Tajik'), ('ti', 'Tigrinya'), ('ur', 'Urdu'), ('uz', 'Uzbek'), ('zu', 'Zulu')], default='en', help_text='Choose language', max_length=3, verbose_name='Language')),
                ('icon_196_196', models.ForeignKey(help_text='Add PNG icon 196x196 px (maskable image can be created using https://maskable.app/)', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image', validators=[home.models.ImageValidator(height=196, width=196)], verbose_name='Icon 196x196 (maskable)')),
                ('icon_512_512', models.ForeignKey(help_text='Add PNG icon 512x512 px', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image', validators=[home.models.ImageValidator(height=512, width=512)], verbose_name='Icon 512x512')),
                ('icon_96_96', models.ForeignKey(help_text='Add PNG icon 96x96 px', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image', validators=[home.models.ImageValidator(height=96, width=96)], verbose_name='Icon 96x96')),
            ],
            options={
                'verbose_name': 'Manifest settings',
                'verbose_name_plural': 'Manifests settings',
                'unique_together': {('language', 'scope')},
            },
        ),
    ]


# home/migrations/0006_auto_20210719_1820.py

# Generated by Django 3.1.11 on 2021-07-19 18:20

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0005_manifestsettings'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='article',
            options={'verbose_name': 'article', 'verbose_name_plural': 'articles'},
        ),
        migrations.AlterModelOptions(
            name='section',
            options={'verbose_name': 'section', 'verbose_name_plural': 'sections'},
        ),
    ]


# home/migrations/0007_auto_20210728_1001.py

# Generated by Django 3.1.13 on 2021-07-28 10:01

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailimages', '0022_uploadedimage'),
        ('home', '0006_auto_20210719_1820'),
    ]

    operations = [
        migrations.AlterField(
            model_name='sitesettings',
            name='logo',
            field=models.ForeignKey(blank=True, help_text='Upload an image file (.jpg, .png, .svg). The ideal size is 100px x 40px', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image'),
        ),
    ]


# home/migrations/0008_auto_20210810_0803.py

# Generated by Django 3.1.13 on 2021-08-10 08:03

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailsvg', '0002_svg_edit_code'),
        ('home', '0007_auto_20210728_1001'),
    ]

    operations = [
        migrations.AlterField(
            model_name='iogtflatmenuitem',
            name='icon',
            field=models.ForeignKey(blank=True, help_text='If Page link is a section page and icon is blank then the section icon will be used. Specify an icon here to override this.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailsvg.svg'),
        ),
    ]


# home/migrations/0009_auto_20210812_0657.py

# Generated by Django 3.1.11 on 2021-08-12 06:57

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0008_auto_20210810_0803'),
    ]

    operations = [
        migrations.AlterField(
            model_name='manifestsettings',
            name='language',
            field=models.CharField(choices=[('ar', 'Arabic'), ('ny', 'Chichewa'), ('en', 'English'), ('fr', 'French'), ('id', 'Indonesian'), ('kaa', 'Karakalpak'), ('km', 'Khmer'), ('rw', 'Kinyarwanda'), ('rn', 'Kirundi'), ('ku', 'Kurdish'), ('mg', 'Malagasy'), ('ne', 'Nepali'), ('nr', 'Ndebele'), ('pt', 'Portuguese'), ('qu', 'Quechua'), ('ru', 'Russian'), ('sn', 'Shona'), ('es', 'Spanish'), ('sw', 'Swahili'), ('tg', 'Tajik'), ('ti', 'Tigrinya'), ('ur', 'Urdu'), ('uz', 'Uzbek'), ('zu', 'Zulu'), ('xy', 'Testing')], default='en', help_text='Choose language', max_length=3, verbose_name='Language'),
        ),
    ]


# home/migrations/0009_themesettings.py

# Generated by Django 3.1.13 on 2021-08-24 13:52

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailcore', '0059_apply_collection_ordering'),
        ('home', '0008_auto_20210810_0803'),
    ]

    operations = [
        migrations.CreateModel(
            name='ThemeSettings',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('global_background_color', models.CharField(blank=True, default='#FFFFFF', help_text='The background color of the website', max_length=8, null=True)),
                ('header_background_color', models.CharField(blank=True, default='#FFFFFF', help_text='The background color of the header background as a HEX code', max_length=8, null=True)),
                ('language_picker_background_color', models.CharField(blank=True, default='#FDD256', help_text='The background color of the language picker button as a HEX code', max_length=8, null=True)),
                ('language_picker_font_color', models.CharField(blank=True, default='#303030', help_text='The font color of the language picker button as a HEX code', max_length=8, null=True)),
                ('section_listing_questionnaire_background_color', models.CharField(blank=True, default='#f0f0f0', help_text='The background color of the Questionnaire in section listing as a HEX code', max_length=8, null=True)),
                ('section_listing_questionnaire_font_color', models.CharField(blank=True, default='#444', help_text='The font color of the Questionnaire in section listing as a HEX code', max_length=8, null=True)),
                ('article_card_font_color', models.CharField(blank=True, default='#444', help_text='The background color of the Quiz in section listing as a HEX code', max_length=8, null=True)),
                ('article_card_background_color', models.CharField(blank=True, default='#ffffff', help_text='The background color of the Quiz in section listing as a HEX code', max_length=8, null=True)),
                ('primary_button_font_color', models.CharField(blank=True, default='#444', help_text='The font/icon color of the primary button as a HEX code', max_length=8, null=True)),
                ('primary_button_background_color', models.CharField(blank=True, default='#f0f0f0', help_text='The background color of the primary button as a HEX code', max_length=8, null=True)),
                ('site', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.site')),
            ],
            options={
                'abstract': False,
            },
        ),
    ]


# home/migrations/0010_v1tov2objectmap.py

# Generated by Django 3.1.13 on 2021-09-07 04:56

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('contenttypes', '0002_remove_content_type_name'),
        ('home', '0009_auto_20210812_0657'),
    ]

    operations = [
        migrations.CreateModel(
            name='V1ToV2ObjectMap',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('v1_object_id', models.PositiveIntegerField()),
                ('object_id', models.PositiveIntegerField()),
                ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')),
            ],
        ),
    ]


# home/migrations/0011_sitesettings_favicon.py

# Generated by Django 3.1.13 on 2021-09-14 11:05

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailimages', '0022_uploadedimage'),
        ('home', '0010_v1tov2objectmap'),
    ]

    operations = [
        migrations.AddField(
            model_name='sitesettings',
            name='favicon',
            field=models.ForeignKey(blank=True, help_text='Upload an image file (.jpg, .png, .svg). The ideal size is 40px x 40px', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image'),
        ),
    ]


# home/migrations/0011_v1tov2objectmap_extra.py

# Generated by Django 3.1.13 on 2021-09-07 12:45

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0010_v1tov2objectmap'),
    ]

    operations = [
        migrations.AddField(
            model_name='v1tov2objectmap',
            name='extra',
            field=models.CharField(blank=True, max_length=100, null=True),
        ),
    ]


# home/migrations/0012_sitesettings_opt_in_to_google_web_light.py

# Generated by Django 3.1.11 on 2021-09-14 14:51

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0011_sitesettings_favicon'),
    ]

    operations = [
        migrations.AddField(
            model_name='sitesettings',
            name='opt_in_to_google_web_light',
            field=models.BooleanField(default=False),
        ),
    ]


# home/migrations/0013_merge_20210920_1202.py

# Generated by Django 3.1.13 on 2021-09-20 12:02

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0012_sitesettings_opt_in_to_google_web_light'),
        ('home', '0011_v1tov2objectmap_extra'),
    ]

    operations = [
    ]

# home/migrations/0013_merge_20210928_0741.py

# Generated by Django 3.1.8 on 2021-09-28 07:41

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0009_themesettings'),
        ('home', '0012_sitesettings_opt_in_to_google_web_light'),
    ]

    operations = [
    ]


# home/migrations/0014_article_index_page_description.py

# Generated by Django 3.1.13 on 2021-09-28 10:46

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0013_merge_20210920_1202'),
    ]

    operations = [
        migrations.AddField(
            model_name='article',
            name='index_page_description',
            field=models.TextField(blank=True, null=True),
        ),
    ]


# home/migrations/0014_themesettings_mobile_navbar_background_color.py

# Generated by Django 3.1.13 on 2021-09-30 12:36

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0013_merge_20210928_0741'),
    ]

    operations = [
        migrations.AddField(
            model_name='themesettings',
            name='mobile_navbar_background_color',
            field=models.CharField(blank=True, default='#0094F4', help_text='The background color of the mobile-only navbar as a HEX code', max_length=8, null=True),
        ),
    ]


# home/migrations/0015_auto_20210930_1308.py

# Generated by Django 3.1.13 on 2021-09-30 13:08

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0014_themesettings_mobile_navbar_background_color'),
    ]

    operations = [
        migrations.AlterField(
            model_name='themesettings',
            name='article_card_background_color',
            field=models.CharField(blank=True, default='#ffffff', help_text='The background color of the Embedded Article in Home > Featured Content as a HEX code', max_length=8, null=True),
        ),
        migrations.AlterField(
            model_name='themesettings',
            name='article_card_font_color',
            field=models.CharField(blank=True, default='#444', help_text='The background color of the Embedded Article in Home > Featured Content as a HEX code', max_length=8, null=True),
        ),
    ]


# home/migrations/0016_merge_20211003_0815.py

# Generated by Django 3.1.11 on 2021-10-03 08:15

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0013_merge_20210920_1202'),
        ('home', '0015_auto_20210930_1308'),
    ]

    operations = [
    ]


# home/migrations/0017_merge_20211004_1341.py

# Generated by Django 3.1.11 on 2021-10-04 13:41

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0016_merge_20211003_0815'),
        ('home', '0014_article_index_page_description'),
    ]

    operations = [
    ]


# home/migrations/0018_auto_20211007_1256.py

# Generated by Django 3.1.8 on 2021-10-07 12:56

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0017_merge_20211004_1341'),
    ]

    operations = [
        migrations.AlterField(
            model_name='iogtflatmenuitem',
            name='color',
            field=models.CharField(blank=True, max_length=255, null=True),
        ),
        migrations.AlterField(
            model_name='iogtflatmenuitem',
            name='color_text',
            field=models.CharField(blank=True, max_length=255, null=True),
        ),
    ]


# home/migrations/0018_auto_20211013_1158.py

# Generated by Django 3.1.13 on 2021-10-13 11:58

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailcore', '0059_apply_collection_ordering'),
        ('home', '0017_merge_20211004_1341'),
    ]

    operations = [
        migrations.AlterField(
            model_name='bannerpage',
            name='banner_link_page',
            field=models.ForeignKey(blank=True, help_text='Optional page to which the banner will link to', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='banners', to='wagtailcore.page'),
        ),
    ]


# home/migrations/0018_svgtopngmap.py

# Generated by Django 3.1.13 on 2021-10-12 10:37

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0017_merge_20211004_1341'),
    ]

    operations = [
        migrations.CreateModel(
            name='SVGToPNGMap',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('svg_path', models.TextField()),
                ('fill_color', models.TextField(null=True)),
                ('stroke_color', models.TextField(null=True)),
                ('png_image_file', models.ImageField(upload_to='svg-to-png-maps/')),
            ],
            options={
                'unique_together': {('svg_path', 'fill_color', 'stroke_color')},
            },
        ),
    ]


# home/migrations/0019_auto_20211015_1356.py

# Generated by Django 3.1.13 on 2021-10-15 13:56

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0018_auto_20211007_1256'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='article',
            name='allow_comments',
        ),
        migrations.AddField(
            model_name='article',
            name='commenting_ends_at',
            field=models.DateTimeField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='article',
            name='commenting_starts_at',
            field=models.DateTimeField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='article',
            name='commenting_status',
            field=models.CharField(choices=[('open', 'Open'), ('closed', 'Closed'), ('disabled', 'Disabled'), ('timestamped', 'Timestamped')], default='open', max_length=15),
            preserve_default=False,
        ),
    ]


# home/migrations/0019_merge_20211014_1723.py

# Generated by Django 3.1.11 on 2021-10-14 17:23

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0018_auto_20211007_1256'),
        ('home', '0018_auto_20211013_1158'),
    ]

    operations = [
    ]


# home/migrations/0020_pagelinkpage.py

# Generated by Django 3.1.13 on 2021-10-18 13:27

from django.db import migrations, models
import django.db.models.deletion
import home.mixins


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailcore', '0059_apply_collection_ordering'),
        ('wagtailsvg', '0002_svg_edit_code'),
        ('home', '0019_merge_20211014_1723'),
    ]

    operations = [
        migrations.CreateModel(
            name='PageLinkPage',
            fields=[
                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.page')),
                ('icon', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailsvg.svg')),
                ('page', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='page_link_pages', to='wagtailcore.page')),
            ],
            options={
                'abstract': False,
            },
            bases=('wagtailcore.page', home.mixins.TitleIconMixin),
        ),
    ]


# home/migrations/0021_merge_20211019_0747.py

# Generated by Django 3.1.11 on 2021-10-19 07:47

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0020_pagelinkpage'),
        ('home', '0018_svgtopngmap'),
    ]

    operations = [
    ]


# home/migrations/0022_merge_20211020_0850.py

# Generated by Django 3.1.11 on 2021-10-20 08:50

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0019_auto_20211015_1356'),
        ('home', '0021_merge_20211019_0747'),
    ]

    operations = [
    ]


# home/migrations/0023_auto_20211029_1521.py

# Generated by Django 3.1.13 on 2021-10-29 15:21

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0022_merge_20211020_0850'),
    ]

    operations = [
        migrations.RenameField(
            model_name='iogtflatmenuitem',
            old_name='color',
            new_name='background_color',
        ),
        migrations.RenameField(
            model_name='iogtflatmenuitem',
            old_name='color_text',
            new_name='font_color',
        ),
        migrations.RenameField(
            model_name='themesettings',
            old_name='mobile_navbar_background_color',
            new_name='navbar_background_color',
        ),
        migrations.AddField(
            model_name='themesettings',
            name='navbar_font_color',
            field=models.CharField(blank=True, default='#FFFFFF', help_text='The font color of the navbar as a HEX code', max_length=8, null=True),
        ),
    ]


# home/migrations/0024_auto_20211029_1522.py

# Generated by Django 3.1.13 on 2021-10-29 15:22

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0023_auto_20211029_1521'),
    ]

    operations = [
        migrations.AlterField(
            model_name='iogtflatmenuitem',
            name='background_color',
            field=models.CharField(blank=True, help_text='The background color of the flat menu item on Desktop + Mobile', max_length=255, null=True),
        ),
        migrations.AlterField(
            model_name='iogtflatmenuitem',
            name='font_color',
            field=models.CharField(blank=True, help_text='The font color of the flat menu item on Desktop + Mobile', max_length=255, null=True),
        ),
        migrations.AlterField(
            model_name='themesettings',
            name='navbar_background_color',
            field=models.CharField(blank=True, default='#0094F4', help_text='The background color of the navbar as a HEX code', max_length=8, null=True),
        ),

    ]


# home/migrations/0026_auto_20211029_1634.py

# Generated by Django 3.1.13 on 2021-10-29 16:34

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0025_auto_20211029_1634'),
    ]

    operations = [
    ]


# home/migrations/0027_auto_20211105_1237.py

# Generated by Django 3.1.13 on 2021-11-05 12:37

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0026_auto_20211029_1634'),
    ]

    operations = [
        migrations.AlterField(
            model_name='iogtflatmenuitem',
            name='link_url',
            field=models.CharField(blank=True, help_text='If you are linking back to a URL on your own IoGT site, be sure to remove the domain and everything before it. For example "http://sd.goodinternet.org/url/" should instead be "/url/".', max_length=255, null=True, verbose_name='link to a custom URL'),
        ),
    ]


# home/migrations/0028_auto_20211110_0645.py

# Generated by Django 3.1.13 on 2021-11-10 06:45

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailimages', '0022_uploadedimage'),
        ('home', '0027_auto_20211105_1237'),
    ]

    operations = [
        migrations.AddField(
            model_name='sitesettings',
            name='apple_touch_icon',
            field=models.ForeignKey(blank=True, help_text='Upload an image file (.jpg, .png) to be used as apple touch icon. The ideal size is 120px x 120px', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AlterField(
            model_name='sitesettings',
            name='favicon',
            field=models.ForeignKey(blank=True, help_text='Upload an image file (.jpg, .png). The ideal size is 40px x 40px', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AlterField(
            model_name='sitesettings',
            name='logo',
            field=models.ForeignKey(blank=True, help_text='Upload an image file (.jpg, .png). The ideal size is 100px x 40px', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image'),
        ),
    ]


# home/migrations/0028_auto_20211110_0848.py

# Generated by Django 3.1.11 on 2021-11-10 08:48

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0027_auto_20211105_1237'),
    ]

    operations = [
        migrations.AlterField(
            model_name='manifestsettings',
            name='background_color',
            field=models.CharField(help_text='Background color member defines a placeholder background color for the application page to display before its stylesheet is loaded. (example: #FFF)', max_length=10, verbose_name='Background color'),
        ),
        migrations.AlterField(
            model_name='manifestsettings',
            name='description',
            field=models.CharField(help_text='Provide description for application', max_length=500, verbose_name='Description'),
        ),
        migrations.AlterField(
            model_name='manifestsettings',
            name='display',
            field=models.CharField(choices=[('FULLSCREEN', 'fullscreen'), ('STANDALONE', 'standalone'), ('MINIMAL_UI', 'minimal-ui'), ('BROWSER', 'browser')], help_text="Determines the preferred display mode for the website. The possible values are: 'fullscreen', 'standalone', 'minimal-ui', 'browser'. A better choice would be to use standalone as it looks great on mobile as well. For further information refer to: https://developer.mozilla.org/en-US/docs/Web/Manifest/display#values", max_length=255, verbose_name='Browser UI'),
        ),
        migrations.AlterField(
            model_name='manifestsettings',
            name='name',
            field=models.CharField(help_text='Provide name that usually represents the name of the web application to user', max_length=255, verbose_name='Name'),
        ),
        migrations.AlterField(
            model_name='manifestsettings',
            name='scope',
            field=models.CharField(help_text="Provide navigation scope to limit what web pages can be viewed Example: 'https://www.iogt.com/<page-url>/' limits navigation to <page-url> of https://www.iogt.com:", max_length=255, verbose_name='Scope'),
        ),
        migrations.AlterField(
            model_name='manifestsettings',
            name='short_name',
            field=models.CharField(help_text='Provide short name to be displayed, if there is not enough space to display full name', max_length=255, verbose_name='Short name'),
        ),
        migrations.AlterField(
            model_name='manifestsettings',
            name='start_url',
            field=models.CharField(help_text='Start URL is the preferred URL that should be loaded when the user launches the web application', max_length=255, verbose_name='Start URL'),
        ),
        migrations.AlterField(
            model_name='manifestsettings',
            name='theme_color',
            field=models.CharField(help_text='Theme color defines the default theme color for the application (example: #493174)', max_length=10, verbose_name='Theme color'),
        ),
    ]


# home/migrations/0029_merge_20211111_1852.py

# Generated by Django 3.1.11 on 2021-11-11 18:52

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0028_auto_20211110_0848'),
        ('home', '0028_auto_20211110_0645'),
    ]

    operations = [
    ]


# home/migrations/0030_auto_20211116_0744.py

# Generated by Django 3.1.13 on 2021-11-16 07:44

from django.db import migrations, models
import django.db.models.deletion
import home.models


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailimages', '0022_uploadedimage'),
        ('home', '0029_merge_20211111_1852'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='manifestsettings',
            name='icon_196_196',
        ),
        migrations.AddField(
            model_name='manifestsettings',
            name='icon_192_192',
            field=models.ForeignKey(help_text='Add PNG icon 192x192 px (maskable image can be created using https://maskable.app/)', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image', validators=[home.models.ImageValidator(height=192, width=192)], verbose_name='Icon 192x192 (maskable)'),
        ),
    ]


# home/migrations/0031_merge_20211123_0857.py

# Generated by Django 3.1.11 on 2021-11-23 08:57

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0030_auto_20211123_0720'),
        ('home', '0030_auto_20211116_0744'),
    ]

    operations = [
    ]


# home/migrations/0032_auto_20211124_0858.py

# Generated by Django 3.1.13 on 2021-11-24 08:58

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0031_merge_20211123_0857'),
    ]

    operations = [
        migrations.AddField(
            model_name='themesettings',
            name='section_card_background_color',
            field=models.CharField(blank=True, default='#ffffff', help_text='The background color of the sub section as a HEX code', max_length=8, null=True),
        ),
        migrations.AddField(
            model_name='themesettings',
            name='section_card_font_color',
            field=models.CharField(blank=True, default='#444', help_text='The background color of the sub section as a HEX code', max_length=8, null=True),
        ),
    ]


# home/migrations/0032_auto_20211130_0936.py

# Generated by Django 3.1.13 on 2021-11-30 09:36

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0031_merge_20211123_0857'),
    ]

    operations = [
        migrations.AddField(
            model_name='section',
            name='commenting_ends_at',
            field=models.DateTimeField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='section',
            name='commenting_starts_at',
            field=models.DateTimeField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='section',
            name='commenting_status',
            field=models.CharField(choices=[('open', 'Open'), ('closed', 'Closed'), ('disabled', 'Disabled'), ('timestamped', 'Timestamped'), ('inherited', 'Inherited')], default='inherited', max_length=15),
        ),
        migrations.AlterField(
            model_name='article',
            name='commenting_status',
            field=models.CharField(choices=[('open', 'Open'), ('closed', 'Closed'), ('disabled', 'Disabled'), ('timestamped', 'Timestamped'), ('inherited', 'Inherited')], default='inherited', max_length=15),
        ),
        migrations.AlterField(
            model_name='offlineapppage',
            name='commenting_status',
            field=models.CharField(choices=[('open', 'Open'), ('closed', 'Closed'), ('disabled', 'Disabled'), ('timestamped', 'Timestamped'), ('inherited', 'Inherited')], default='inherited', max_length=15),
        ),
    ]


# home/migrations/0033_merge_20211201_1715.py

# Generated by Django 3.1.11 on 2021-12-01 17:15

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0032_auto_20211124_0858'),
        ('home', '0032_auto_20211130_0936'),
    ]

    operations = [
    ]


# home/migrations/0034_v1pageurltov2pagemap.py

# Generated by Django 3.1.13 on 2021-12-02 12:46

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailcore', '0059_apply_collection_ordering'),
        ('home', '0033_merge_20211201_1715'),
    ]

    operations = [
        migrations.CreateModel(
            name='V1PageURLToV2PageMap',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('v1_page_url', models.CharField(max_length=255)),
                ('v2_page', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.page')),
            ],
        ),
    ]


# home/migrations/0035_auto_20211221_0822.py

# Generated by Django 3.1.14 on 2021-12-21 08:22

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailcore', '0059_apply_collection_ordering'),
        ('wagtailimages', '0022_uploadedimage'),
        ('home', '0034_v1pageurltov2pagemap'),
    ]

    operations = [
        migrations.AddField(
            model_name='article',
            name='image_icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='iogtflatmenuitem',
            name='image_icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='offlineapppage',
            name='image_icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='pagelinkpage',
            name='external_link',
            field=models.URLField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='pagelinkpage',
            name='image_icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='section',
            name='image_icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AlterField(
            model_name='pagelinkpage',
            name='page',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='page_link_pages', to='wagtailcore.page'),
        ),
    ]


# home/migrations/0035_delete_cachesettings.py

# Generated by Django 3.1.14 on 2021-12-24 06:19

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0034_v1pageurltov2pagemap'),
    ]

    operations = [
        migrations.DeleteModel(
            name='CacheSettings',
        ),
    ]


# home/migrations/0036_merge_20211226_1127.py

# Generated by Django 3.1.13 on 2021-12-26 11:27

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0035_delete_cachesettings'),
        ('home', '0035_auto_20211221_0822'),
    ]

    operations = [
    ]


# home/migrations/0037_localedetail.py

# Generated by Django 3.1.14 on 2021-12-27 06:54

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailcore', '0059_apply_collection_ordering'),
        ('home', '0036_merge_20211226_1127'),
    ]

    operations = [
        migrations.CreateModel(
            name='LocaleDetail',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('is_active', models.BooleanField(default=True, help_text='When active locale will be shown in the language selector')),
                ('is_main_language', models.BooleanField(default=False, help_text='When active locale will be used as default language for the site')),
                ('locale', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='locale_detail', to='wagtailcore.locale')),
            ],
        ),
    ]


# home/migrations/0039_iogtflatmenuitem_display_only_in_single_column_view.py

# Generated by Django 3.1.14 on 2022-01-21 13:02

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0038_auto_20220106_0908'),
    ]

    operations = [
        migrations.AddField(
            model_name='iogtflatmenuitem',
            name='display_only_in_single_column_view',
            field=models.BooleanField(default=False),
        ),
    ]


# home/migrations/0041_auto_20220127_1109.py

# Generated by Django 3.1.14 on 2022-01-27 11:09

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0040_auto_20220125_1128'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='bannerpage',
            name='align_center',
        ),
        migrations.RemoveField(
            model_name='bannerpage',
            name='banner_background_image',
        ),
        migrations.RemoveField(
            model_name='bannerpage',
            name='banner_button_text',
        ),
        migrations.RemoveField(
            model_name='bannerpage',
            name='banner_description',
        ),
        migrations.RemoveField(
            model_name='bannerpage',
            name='banner_icon_button',
        ),
    ]


# home/migrations/0042_auto_20220131_1453.py

# Generated by Django 3.1.13 on 2022-01-31 14:53

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailimages', '0022_uploadedimage'),
        ('home', '0041_auto_20220127_1109'),
    ]

    operations = [
        migrations.AlterField(
            model_name='sitesettings',
            name='logo',
            field=models.ForeignKey(blank=True, help_text='Upload an image file (.jpg, .png). The ideal size is 120px x 48px', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailimages.image'),
        ),
    ]


# home/migrations/0043_merge_20220209_1327.py

# Generated by Django 3.1.13 on 2022-02-09 13:27

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0042_auto_20220208_1153'),
        ('home', '0042_auto_20220131_1453'),
    ]

    operations = [
    ]


# home/migrations/0044_auto_20220223_0819.py

# Generated by Django 3.1.14 on 2022-02-23 08:19

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0043_merge_20220209_1327'),
    ]

    operations = [
        migrations.AlterField(
            model_name='v1pageurltov2pagemap',
            name='v1_page_url',
            field=models.TextField(),
        ),
    ]


# home/migrations/0045_auto_20220405_0956.py

# Generated by Django 3.1.14 on 2022-04-05 09:56

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0044_auto_20220223_0819'),
    ]

    operations = [
        migrations.AlterField(
            model_name='manifestsettings',
            name='language',
            field=models.CharField(choices=[('ar', 'Arabic'), ('bn', 'Bengali'), ('ny', 'Chichewa'), ('en', 'English'), ('fr', 'French'), ('id', 'Indonesian'), ('kaa', 'Karakalpak'), ('km', 'Khmer'), ('rw', 'Kinyarwanda'), ('rn', 'Kirundi'), ('ku', 'Kurdish'), ('mg', 'Malagasy'), ('ne', 'Nepali'), ('nr', 'Ndebele'), ('pt', 'Portuguese'), ('qu', 'Quechua'), ('ru', 'Russian'), ('sn', 'Shona'), ('es', 'Spanish'), ('sw', 'Swahili'), ('tg', 'Tajik'), ('ti', 'Tigrinya'), ('ur', 'Urdu'), ('uz', 'Uzbek'), ('zu', 'Zulu'), ('xy', 'Testing')], default='en', help_text='Choose language', max_length=3, verbose_name='Language'),
        ),
    ]


# home/migrations/0046_auto_20220422_1825.py

# Generated by Django 3.1.13 on 2022-04-22 18:25

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0045_auto_20220405_0956'),
    ]

    operations = [
        migrations.AlterField(
            model_name='manifestsettings',
            name='language',
            field=models.CharField(choices=[('ar', 'Arabic'), ('bn', 'Bengali'), ('ny', 'Chichewa'), ('prs', 'Dari'), ('en', 'English'), ('fr', 'French'), ('hi', 'Hindi'), ('id', 'Indonesian'), ('kaa', 'Karakalpak'), ('km', 'Khmer'), ('rw', 'Kinyarwanda'), ('rn', 'Kirundi'), ('ku', 'Kurdish'), ('mg', 'Malagasy'), ('ne', 'Nepali'), ('nr', 'Ndebele'), ('ps', 'Pashto'), ('pt', 'Portuguese'), ('qu', 'Quechua'), ('ru', 'Russian'), ('sn', 'Shona'), ('si', 'Sinhala'), ('es', 'Spanish'), ('sw', 'Swahili'), ('tg', 'Tajik'), ('ta', 'Tamil'), ('ti', 'Tigrinya'), ('ur', 'Urdu'), ('uz', 'Uzbek'), ('zu', 'Zulu'), ('xy', 'Testing')], default='en', help_text='Choose language', max_length=3, verbose_name='Language'),
        ),
    ]


# home/migrations/0049_auto_20221018_0943.py

# Generated by Django 3.1.14 on 2022-10-18 09:43

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0048_auto_delete_offline_app_page_and_create_offline_content_index_page'),
    ]

    operations = [
        migrations.AlterField(
            model_name='manifestsettings',
            name='language',
            field=models.CharField(choices=[('ar', 'Arabic'), ('bn', 'Bengali'), ('ny', 'Chichewa'), ('prs', 'Dari'), ('en', 'English'), ('fr', 'French'), ('hi', 'Hindi'), ('id', 'Indonesian'), ('kaa', 'Karakalpak'), ('km', 'Khmer'), ('rw', 'Kinyarwanda'), ('rn', 'Kirundi'), ('ku', 'Kurdish'), ('mg', 'Malagasy'), ('ne', 'Nepali'), ('nr', 'Ndebele'), ('ps', 'Pashto'), ('pt', 'Portuguese'), ('qu', 'Quechua'), ('ru', 'Russian'), ('sn', 'Shona'), ('si', 'Sinhala'), ('es', 'Spanish'), ('sw', 'Swahili'), ('tg', 'Tajik'), ('ta', 'Tamil'), ('ti', 'Tigrinya'), ('uk', 'Ukraine'), ('ur', 'Urdu'), ('uz', 'Uzbek'), ('zu', 'Zulu'), ('xy', 'Testing')], default='en', help_text='Choose language', max_length=3, verbose_name='Language'),
        ),
    ]


# home/migrations/0050_auto_20230213_1214.py

# Generated by Django 3.1.14 on 2023-02-13 12:14

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0049_auto_20221018_0943'),
    ]

    operations = [
        migrations.AlterField(
            model_name='manifestsettings',
            name='language',
            field=models.CharField(choices=[('ar', 'Arabic'), ('bn', 'Bengali'), ('ny', 'Chichewa'), ('prs', 'Dari'), ('en', 'English'), ('fr', 'French'), ('hi', 'Hindi'), ('id', 'Indonesian'), ('kaa', 'Karakalpak'), ('km', 'Khmer'), ('rw', 'Kinyarwanda'), ('rn', 'Kirundi'), ('ku', 'Kurdish'), ('mg', 'Malagasy'), ('ne', 'Nepali'), ('nr', 'Ndebele'), ('ps', 'Pashto'), ('pt', 'Portuguese'), ('qu', 'Quechua'), ('ru', 'Russian'), ('sn', 'Shona'), ('si', 'Sinhala'), ('es', 'Spanish'), ('sw', 'Swahili'), ('tg', 'Tajik'), ('ta', 'Tamil'), ('ti', 'Tigrinya'), ('tu', 'Turkish'), ('uk', 'Ukraine'), ('ur', 'Urdu'), ('uz', 'Uzbek'), ('zu', 'Zulu'), ('xy', 'Testing')], default='en', help_text='Choose language', max_length=3, verbose_name='Language'),
        ),
    ]


# home/migrations/0051_auto_20230330_1105.py

# Generated by Django 3.1.14 on 2023-03-30 11:05

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0050_auto_20230213_1214'),
    ]

    operations = [
        migrations.AlterField(
            model_name='manifestsettings',
            name='language',
            field=models.CharField(choices=[('ar', 'Arabic'), ('bn', 'Bengali'), ('ny', 'Chichewa'), ('prs', 'Dari'), ('en', 'English'), ('fa', 'Farsi'), ('fr', 'French'), ('hi', 'Hindi'), ('id', 'Indonesian'), ('kaa', 'Karakalpak'), ('km', 'Khmer'), ('rw', 'Kinyarwanda'), ('rn', 'Kirundi'), ('ku', 'Kurdish'), ('mg', 'Malagasy'), ('ne', 'Nepali'), ('nr', 'Ndebele'), ('ps', 'Pashto'), ('pt', 'Portuguese'), ('qu', 'Quechua'), ('ru', 'Russian'), ('sn', 'Shona'), ('si', 'Sinhala'), ('es', 'Spanish'), ('sw', 'Swahili'), ('tg', 'Tajik'), ('ta', 'Tamil'), ('ti', 'Tigrinya'), ('tr', 'Turkish'), ('uk', 'Ukraine'), ('ur', 'Urdu'), ('uz', 'Uzbek'), ('zu', 'Zulu'), ('xy', 'Testing')], default='en', help_text='Choose language', max_length=3, verbose_name='Language'),
        ),
    ]


# home/templatetags/generic_components.py

from urllib.parse import urlencode, parse_qs, urlparse

from django import template
from django.conf import settings
from django.urls import reverse
from google_analytics import CAMPAIGN_TRACKING_PARAMS

import iogt.iogt_globals as globals_


register = template.Library()


@register.inclusion_tag('generic_components/primary_button.html')
def primary_button(title, extra_classnames='', href=None, icon_path=None, icon=None, font_color=None,
                   background_color=None):
    theme_settings = globals_.theme_settings

    font_color = font_color or theme_settings.primary_button_font_color
    background_color = background_color or theme_settings.primary_button_background_color

    return {
        'title': title,
        'href': href,
        'icon_path': icon_path,
        'icon': icon,
        'font_color': font_color,
        'background_color': background_color,
        'extra_classnames': extra_classnames,
    }


@register.inclusion_tag('generic_components/article_card.html', takes_context=True)
def render_article_card(context, page, is_first_content=False, display_section_title=False, background_color=None, font_color=None):
    theme_settings = globals_.theme_settings

    font_color = font_color or theme_settings.article_card_font_color
    background_color = background_color or theme_settings.article_card_background_color

    context.update({
        'article': page,
        'is_first_content': is_first_content,
        'display_section_title': display_section_title,
        'background_color': background_color,
        'font_color': font_color
    })
    return context


@register.inclusion_tag('generic_components/section_card.html', takes_context=True)
def render_section_card(context, page, is_first_content=False, background_color=None, font_color=None):
    theme_settings = globals_.theme_settings

    font_color = font_color or theme_settings.section_card_font_color
    background_color = background_color or theme_settings.section_card_background_color

    context.update({
        'section': page,
        'is_first_content': is_first_content,
        'background_color': background_color,
        'font_color': font_color
    })
    return context


@register.inclusion_tag('generic_components/questionnaire_card.html', takes_context=True)
def render_questionnaire_card(context, page, background_color=None, font_color=None):
    theme_settings = globals_.theme_settings

    font_color = font_color or theme_settings.section_listing_questionnaire_font_color
    background_color = background_color or theme_settings.section_listing_questionnaire_background_color

    context.update({
        'questionnaire': page,
        'background_color': background_color,
        'font_color': font_color
    })
    return context


@register.simple_tag
def language_picker_style():
    theme_settings = globals_.theme_settings
    return f"color:{theme_settings.language_picker_font_color};background-color:" \
           f"{theme_settings.language_picker_background_color}"


@register.simple_tag
def site_version():
    return settings.SITE_VERSION


@register.simple_tag(takes_context=True)
def google_analytics(context, tracking_code=None, debug=False):
    if not tracking_code:
        try:
            assert settings.GOOGLE_ANALYTICS['google_analytics_id']
        except KeyError:
            return ''
    # attempt get the request from the context
    request = context.get('request', None)
    if request is None:
        raise RuntimeError("Request context required")
    # intialise the parameters collection
    params = {}
    # collect the campaign tracking parameters from the request
    for param in CAMPAIGN_TRACKING_PARAMS.values():
        value = request.GET.get(param, None)
        if value:
            params[param] = value
    # pass on the referer if present
    referer = request.META.get('HTTP_REFERER', None)
    if referer:
        params['r'] = referer
    # remove collected parameters from the path and pass it on
    path = request.get_full_path()
    parsed_url = urlparse(path)
    query = parse_qs(parsed_url.query, keep_blank_values=True)
    for param in params:
        if param in query:
            del query[param]
    query = urlencode(query, doseq=True)
    new_url = parsed_url._replace(query=query)
    params['p'] = new_url.geturl()
    params['tracking_code'] = tracking_code or settings.GOOGLE_ANALYTICS[
        'google_analytics_id']
    # append the debug parameter if requested
    if debug:
        params['utmdebug'] = 1
    # build and return the url
    url = reverse('google-analytics')
    if params:
        url += '?&' + urlencode(params)
    return url


# home/templatetags/home_tags.py

from django import template
import django.utils.translation as translation
from django.core.exceptions import ObjectDoesNotExist
from django.shortcuts import get_object_or_404
from django.urls import translate_url, reverse, resolve, Resolver404
from django.conf import settings
from wagtail.core.models import Locale, Site, Page

from home.models import SectionIndexPage, Section, Article, FooterIndexPage, PageLinkPage, LocaleDetail, HomePage, SiteSettings
from iogt.settings.base import LANGUAGES

register = template.Library()


@register.inclusion_tag('home/tags/language_switcher.html', takes_context=True)
def language_switcher(context, page):
    """
    The current requirement for this is really tricky.
    See https://github.com/unicef/iogt/pull/955#issuecomment-1008277982 for more
    context on why this logic is complicated.
    """

    language_options = list()
    locales = Locale.objects.select_related('locale_detail').all()
    current_language = translation.get_language_info(translation.get_language())

    try:
        if resolve(context.request.path_info).url_name == 'translation-not-found':
            page = get_object_or_404(Page, pk=context.request.GET.get('page'), live=True)
    except Resolver404:
        pass

    for locale in locales:
        option = {}
        try:
            language = translation.get_language_info(locale.language_code)
            option['language'] = language
            option['selected'] = locale.language_code == current_language.get('code')
        except:
            continue

        try:
            should_append = locale.locale_detail.is_active
        except LocaleDetail.DoesNotExist:
            should_append = True

        if should_append:
            if page:  # If the current URL belongs to a wagtail page
                translated_page = page and page.get_translation_or_none(locale)
                if translated_page and translated_page.live:
                    url = translated_page.url
                else:
                    translated_url = translate_url(reverse('translation-not-found'), locale.language_code)
                    url = f'{translated_url}?page={page.id}'
            else:  # If the current URL belongs to a django view
                url = translate_url(context.request.path_info, locale.language_code)

            option['url'] = url
            language_options.append(option)

    context.update({
        'language_options': language_options,
        'current_language': current_language,
    })

    return context


@register.inclusion_tag('home/tags/previous-next-buttons.html')
def render_previous_next_buttons(page):
    return {
        'next_sibling': page.get_next_siblings().not_type(PageLinkPage).live().first(),
        'previous_sibling': page.get_prev_siblings().not_type(PageLinkPage).live().first()
    }


@register.inclusion_tag('home/tags/footer.html', takes_context=True)
def render_footer(context):
    return {
        'footers': FooterIndexPage.get_active_footers(),
        'request': context['request'],
    }


@register.inclusion_tag('home/tags/top_level_sections.html', takes_context=True)
def render_top_level_sections(context):
    return {
        'top_level_sections': SectionIndexPage.get_top_level_sections(),
        'request': context['request'],
    }


@register.inclusion_tag('home/tags/section_progress.html')
def render_user_progress(user_progress, show_count=True):
    return {
        **user_progress,
        'show_count': show_count,
    }


@register.inclusion_tag('home/tags/is_complete.html', takes_context=True)
def render_is_complete(context, page):
    if isinstance(page, (Section, Article)):
        context.update({
            'is_complete': page.is_complete(context['request'])
        })
    return context


@register.simple_tag
def locale_set(locale, url):
    for item in LANGUAGES:
        code = item[0]
        url = url.replace(f"/{code}/", "")
    return f'/{locale}/{url}'


@register.simple_tag
def translated_home_page_url(language_code):
    locale = Locale.objects.get(language_code=language_code)
    default_home_page = Site.objects.filter(is_default_site=True).first().root_page
    home_page = default_home_page.get_translation_or_none(locale)
    page = home_page or default_home_page
    return page.url


@register.simple_tag(takes_context=True)
def change_lang(context, lang=None, *args, **kwargs):
    path = context['request'].path
    return translate_url(path, lang)


@register.inclusion_tag('wagtailadmin/shared/field_as_li.html')
def render_external_link_with_help_text(field):
    field.help_text = f'If you are linking back to a URL on your own IoGT site, be sure to remove the domain and ' \
                      f'everything before it. For example "http://sd.goodinternet.org/url/" should instead be "/url/".'

    return {'field': field, 'red_help_text': True}


@register.inclusion_tag('wagtailadmin/shared/field_as_li.html')
def render_redirect_from_with_help_text(field):
    field.help_text = f'A relative path to redirect from e.g. /en/youth. ' \
                      f'See "https://docs.wagtail.io/en/stable/editor_manual/managing_redirects.html" for more details.'

    return {'field': field, 'red_help_text': True}


@register.inclusion_tag('home/tags/image.html', takes_context=True)
def render_image(context, image, half_width=False, img_class=None):
    width = settings.IMAGE_SIZE_PRESET
    if half_width:
        width //= 2

    context.update({
        'image': image,
        'width': width,
        'class': img_class,
    })
    return context


@register.inclusion_tag('home/tags/social_meta_tags.html', takes_context=True)
def social_meta_tags(context):
    width = settings.IMAGE_SIZE_PRESET
    page = context.get('page')
    if hasattr(page, 'lead_image') and page.lead_image:
        lead_image = page.lead_image.get_rendition(f'width-{width}')
        context['lead_image'] = lead_image

    if page:
        context['site_name'] = page.get_site().site_name

    return context


# home/templatetags/image_tags.py

from pathlib import Path
from django import template
from django.conf import settings

from home.models import SVGToPNGMap

register = template.Library()


@register.simple_tag
def svg_to_png_url(svg_path, fill_color=None, stroke_color=None,):
    image = SVGToPNGMap.get_png_image(svg_path, fill_color=fill_color, stroke_color=stroke_color)
    return image.url if image else ''


@register.inclusion_tag('home/tags/render_png.html')
def render_icon(icon_path=None, icon=None, height=None, width=None, fill_color=None, stroke_color=None, attrs=None):
    image_url = ''
    if icon_path:
        image_url = svg_to_png_url(
            svg_path=Path(settings.STATIC_ROOT) / icon_path,
            fill_color=fill_color,
            stroke_color=stroke_color
        )
    elif icon:
        if icon.is_svg_icon:
            image_url = svg_to_png_url(
                svg_path=Path(settings.MEDIA_ROOT) / icon.path,
                fill_color=fill_color,
                stroke_color=stroke_color
            )
        else:
            image_url = icon.url

    return {
        'height': height,
        'width': width,
        'attrs': attrs,
        'image_url': image_url,
    }


# home/tests/faker.py

from faker import Factory as FakerFactory

faker = FakerFactory.create()


# home/tests/test_translation_manager.py

from pathlib import Path

from django.test import TestCase, override_settings
from translation_manager.models import TranslationEntry

from home.translation_manager.manager import IogtTranslationManager

class LoadTranslationsFromFileTests(TestCase):

    LANGUAGES = [ ('es', 'Spanish') ]
    LOCALE_PATHS = [ Path(__file__).parent / 'locale_test' ]

    def setUp(self):
        self.manager = IogtTranslationManager()


    @override_settings(LOCALE_PATHS=LOCALE_PATHS, LANGUAGES=LANGUAGES)
    def test_existing_translated_entry_is_preserved(self):
        self.create_entry('test_case_1', 'test_case_1_translation_existing')

        self.manager.load_data_from_po()

        entry = TranslationEntry.objects.get(original='test_case_1')
        self.assertEqual(entry.translation, 'test_case_1_translation_existing')
        self.assertTrue(entry.is_published)


    @override_settings(LOCALE_PATHS=LOCALE_PATHS, LANGUAGES=LANGUAGES)
    def test_untranslated_entry_is_replaced_with_translated_entry(self):
        self.create_entry('test_case_2', '')

        self.manager.load_data_from_po()

        entry = TranslationEntry.objects.get(original='test_case_2')
        self.assertEqual(entry.translation, 'test_case_2_translation_update')
        self.assertTrue(entry.is_published)


    @override_settings(LOCALE_PATHS=LOCALE_PATHS, LANGUAGES=LANGUAGES)
    def test_new_entry_created_from_file(self):
        self.manager.load_data_from_po()

        entry = TranslationEntry.objects.get(original='test_case_3.1')
        self.assertEqual(entry.translation, 'test_case_3.1_translation')
        self.assertEqual(entry.language, 'es')
        self.assertEqual(entry.domain, 'django')
        self.assertEqual(entry.locale_path, 'home/tests/locale_test')
        self.assertEqual(entry.locale_parent_dir, 'tests')
        self.assertTrue(entry.is_published)

        entry = TranslationEntry.objects.get(original='test_case_3.2')
        self.assertEqual(entry.translation, '')
        self.assertTrue(entry.is_published)


    @override_settings(LOCALE_PATHS=LOCALE_PATHS, LANGUAGES=LANGUAGES)
    def test_existing_untranslated_entry_is_preserved_if_no_translation_in_file(self):
        self.create_entry('test_case_4', '')

        self.manager.load_data_from_po()

        entry = TranslationEntry.objects.get(original='test_case_4')
        self.assertEqual(entry.translation, '')
        self.assertTrue(entry.is_published)


    def create_entry(self, original, translation):
        TranslationEntry(
            original=original,
            language='es',
            translation=translation,
            domain='django',
            locale_path='home/tests/locale_test',
            locale_parent_dir='tests',
            is_published=True
        ).save()


# home/tests/tests.py

from bs4 import BeautifulSoup
from django.test import TestCase, override_settings
from django.http import HttpRequest
from translation_manager.models import TranslationEntry
from wagtail.core.models import Site
from wagtail_factories import SiteFactory, PageFactory
from wagtail_localize.operations import TranslationCreator

from home.wagtail_hooks import limit_page_chooser
from home.factories import (
    SectionFactory,
    ArticleFactory,
    HomePageFactory,
    MediaFactory,
    LocaleFactory,
    SiteSettingsFactory
)


class LimitPageChooserHookTests(TestCase):
    def setUp(self):
        Site.objects.all().delete()
        self.site = SiteFactory(site_name='IoGT', port=8000, is_default_site=True)
        self.home_page = HomePageFactory(parent=self.site.root_page)

        self.article01 = ArticleFactory(parent=self.home_page)
        self.section01 = SectionFactory(parent=self.home_page)
        self.section02 = SectionFactory(parent=self.section01)
        self.article02 = ArticleFactory(parent=self.section01)

    def test_start_from_section_when_current_page_is_section(self):
        request = HttpRequest()
        request.path = '/admin/choose-page/'
        request.META['HTTP_REFERER'] = f'https://example.com/admin/pages/{self.section01.id}/edit/'
        pages = self.home_page.get_children()

        pages = limit_page_chooser(pages, request)

        self.assertEqual([i for i in pages.values_list('id', flat=True)], [self.section02.id, self.article02.id])

    def test_start_from_section_when_parent_page_is_section(self):
        request = HttpRequest()
        request.path = f'/admin/choose-page/{self.section01.id}/'
        pages = self.home_page.get_children()

        pages = limit_page_chooser(pages, request)

        self.assertEqual([i for i in pages.values_list('id', flat=True)], [self.section02.id, self.article02.id])

    def test_do_not_change_queryset_when_current_page_is_not_a_section(self):
        request = HttpRequest()
        request.path = '/admin/choose-page/'
        request.META['HTTP_REFERER'] = f'https://example.com/admin/pages/{self.home_page.id}/edit/'
        pages_before = self.home_page.get_children()

        pages_after = limit_page_chooser(pages_before, request)

        self.assertEqual(pages_after, pages_before)

    def test_do_not_change_queryset_when_parent_page_is_not_a_section(self):
        request = HttpRequest()
        request.path = f'/admin/choose-page/{self.home_page.id}/'
        pages_before = self.home_page.get_children()

        pages_after = limit_page_chooser(pages_before, request)

        self.assertEqual(pages_after, pages_before)


class MediaTranslationTest(TestCase):
    def setUp(self):
        root_page = PageFactory(parent=None)
        en_home_page = HomePageFactory(parent=root_page)
        SiteFactory(hostname='testserver', port=80, root_page=en_home_page)
        self.en_article = ArticleFactory(
            parent=en_home_page,
            body__0__media=MediaFactory(type='video'),
            body__1__media=MediaFactory(type='audio'),
        )

    def test_media_block_translation_of_english_language(self):
        response = self.client.get(self.en_article.url)

        self.assertEqual(response.status_code, 200)
        self.assertContains(response, f"If you cannot view the above video, you can instead <a href=\"{self.en_article.body[0].value.url}\" download>download it</a>", count=1)
        self.assertContains(response, f"If you cannot listen to the above audio, you can instead <a href=\"{self.en_article.body[1].value.url}\" download>download it</a>", count=1)

    def test_media_block_translation_of_bengali_language(self):
        bn_locale = LocaleFactory(language_code='bn')
        bn_translation_creator = TranslationCreator(user=None, target_locales=[bn_locale])
        bn_translation_creator.create_translations(self.en_article)
        bn_article = self.en_article.get_translation(bn_locale)

        TranslationEntry.objects.create(
            original="If you cannot view the above video, you can instead %(start_link)sdownload it%(end_link)s.",
            translation="উপরের ভিডিও দেখা না গেলে %(start_link)s এর পরিবর্তে এটা %(end_link)s ডাউনলোড করুন",
            language=bn_locale.language_code)
        TranslationEntry.objects.create(
            original="If you cannot listen to the above audio, you can instead %(start_link)sdownload it%(end_link)s.",
            translation="উপরের অডিও শুনতে না পেলে %(start_link)s এর পরিবর্তে এটা %(end_link)s ডাউনলোড করুন",
            language=bn_locale.language_code)

        response = self.client.get(bn_article.url)

        self.assertEqual(response.status_code, 200)
        self.assertContains(response, f"উপরের ভিডিও দেখা না গেলে <a href=\"{self.en_article.body[0].value.url}\" download> এর পরিবর্তে এটা </a> ডাউনলোড করুন", count=1)
        self.assertContains(response, f"উপরের অডিও শুনতে না পেলে <a href=\"{self.en_article.body[1].value.url}\" download> এর পরিবর্তে এটা </a> ডাউনলোড করুন", count=1)


class ImageResizeTest(TestCase):
    def setUp(self):
        self.site = SiteFactory(site_name='IoGT', port=8000, is_default_site=True)
        self.site_settings = SiteSettingsFactory(site=self.site)
        self.section = SectionFactory(parent=self.site_settings.site.root_page)
        self.article = ArticleFactory(parent=self.section)

    def test_default_image_within_preset(self):
        response = self.client.get(self.article.url)
        parsed_response = parse_html(response.content)
        rendered_image = parsed_response.find("img", {"class": "article__lead-img-featured"})
        image_rendition = self.article.lead_image.get_rendition('width-360')

        self.assertEqual(response.status_code, 200)
        self.assertEqual(rendered_image.get('alt'), image_rendition.alt)
        self.assertEqual(int(rendered_image.get('width')), image_rendition.width)
        self.assertEqual(int(rendered_image.get('height')), image_rendition.height)
        self.assertEqual(rendered_image.get('src'), image_rendition.url)


    def test_half_default_image_within_preset(self):
        response = self.client.get(self.section.url)
        parsed_response = parse_html(response.content)
        rendered_image = parsed_response.find("div", {"class": "article-card"}).find("img")
        image_rendition = self.article.lead_image.get_rendition('width-180')

        self.assertEqual(response.status_code, 200)
        self.assertEqual(rendered_image.get('alt'), image_rendition.alt)
        self.assertEqual(int(rendered_image.get('width')), image_rendition.width)
        self.assertEqual(int(rendered_image.get('height')), image_rendition.height)
        self.assertEqual(rendered_image.get('src'), image_rendition.url)

    @override_settings(IMAGE_SIZE_PRESET=750)
    def test_custom_image_within_preset(self):
        response = self.client.get(self.article.url)
        parsed_response = parse_html(response.content)
        rendered_image = parsed_response.find("img", {"class": "article__lead-img-featured"})
        image_rendition = self.article.lead_image.get_rendition('width-750')

        self.assertEqual(response.status_code, 200)
        self.assertEqual(rendered_image.get('alt'), image_rendition.alt)
        self.assertEqual(int(rendered_image.get('width')), image_rendition.width)
        self.assertEqual(int(rendered_image.get('height')), image_rendition.height)
        self.assertEqual(rendered_image.get('src'), image_rendition.url)

    @override_settings(IMAGE_SIZE_PRESET=750)
    def test_half_custom_image_within_preset(self):
        response = self.client.get(self.section.url)
        parsed_response = parse_html(response.content)
        rendered_image = parsed_response.find("div", {"class": "article-card"}).find("img")
        image_rendition = self.article.lead_image.get_rendition('width-375')

        self.assertEqual(response.status_code, 200)
        self.assertEqual(rendered_image.get('alt'), image_rendition.alt)
        self.assertEqual(int(rendered_image.get('width')), image_rendition.width)
        self.assertEqual(int(rendered_image.get('height')), image_rendition.height)
        self.assertEqual(rendered_image.get('src'), image_rendition.url)


def parse_html(html: str) -> BeautifulSoup:
    return BeautifulSoup(html, 'lxml')


# home/translation_manager/manager.py

import os
import polib

import translation_manager.manager
from translation_manager.models import TranslationEntry
from translation_manager.utils import (
    get_lang_from_dirname,
    get_locale_parent_dirname,
    get_relative_locale_path,
)


class IogtTranslationManager(translation_manager.manager.Manager):

    def store_to_db(self, pofile, locale, store_translations=False):
        language = get_lang_from_dirname(locale)
        domain = os.path.splitext(os.path.basename(pofile))[0]
        messages = polib.pofile(pofile)
        translations = TranslationEntry.objects.filter(language=language)

        tdict = {
            (t.original, t.language, t.domain): t
            for t in translations
        }

        to_create = []
        to_delete = []
        for m in messages:
            occs = []
            for occ in m.occurrences:
                path = ":".join(occ)
                occs.append(path)

            if store_translations:
                translation = m.msgstr
            else:
                translation = ""

            locale_path = get_relative_locale_path(pofile)

            if os.path.split(pofile)[-1] == 'angularjs.po':
                locale_dir_name = ''
            else:
                locale_dir_name = get_locale_parent_dirname(pofile)

            entry = tdict.get((m.msgid, language, domain))

            t = TranslationEntry(
                original=m.msgid,
                language=language,
                domain=domain,
                occurrences="\n".join(occs),
                translation=translation,
                locale_parent_dir=locale_dir_name,
                is_published=True,
                locale_path=locale_path,
            )

            if not entry:
                to_create.append(t)
            elif entry.translation == '' and entry.translation != translation:
                to_delete.append(entry)
                to_create.append(t)

            if locale_path not in self.tors:
                self.tors[locale_path] = {}
            if language not in self.tors[locale_path]:
                self.tors[locale_path][language] = {}
            if domain not in self.tors[locale_path][language]:
                self.tors[locale_path][language][domain] = []
            self.tors[locale_path][language][domain].append(t.original)

        TranslationEntry.objects.filter(
            id__in=[t.id for t in to_delete]
        ).delete()
        TranslationEntry.objects.bulk_create(to_create)



# home/utils/__init__.py

from bs4 import BeautifulSoup


def get_all_renditions_urls(image):
    return [
        rendition.url
        for rendition
        in image.get_rendition_model().objects.filter(image_id=image.id)
    ]


def extract_urls_from_rich_text(text):
    return [
        img['src']
        for img
        in BeautifulSoup(str(text), 'lxml').find_all('img')
    ]


def collect_urls_from_streamfield(field):
    urls = []
    for block in field:
        if block.block_type == 'image':
            urls += get_all_renditions_urls(block.value)
        if block.block_type == 'paragraph':
            urls += extract_urls_from_rich_text(block.value)
        if block.block_type == 'download':
            urls += extract_urls_from_rich_text(
                block.value.get('description', '')
            )
        if block.block_type == 'media':
            urls.append(block.value.url)
    return urls


# home/utils/image.py

import codecs
import cairosvg
import re

from bs4 import BeautifulSoup
from django.core.files.base import ContentFile


def convert_svg_to_png_bytes(svg_file_path, fill_color=None, stroke_color=None, width=32):
    with open(svg_file_path, 'r', encoding='utf-8') as f:
        soup = BeautifulSoup(f, 'lxml')
        elements_to_fill = soup.find_all('path')
        for el in elements_to_fill:
            if fill_color:
                el['fill'] = fill_color
            if stroke_color:
                el['stroke'] = stroke_color

    parsed_soup = str(soup.body.next)
    parsed_soup = parsed_soup.replace('viewbox', 'viewBox')

    file_bytes = cairosvg.svg2png(bytestring=parsed_soup, output_width=width)
    return ContentFile(file_bytes, 'svg-to-png.png')


# home/utils/progress_manager.py

class ProgressManager:

    def __init__(self, request):
        self.request = request

    def _get_read_articles_ids(self):
        if self.request.user.is_anonymous:
            read_article_ids = set(self.request.session.get('read_articles', []))
        else:
            read_article_ids = set(self.request.user.read_articles.values_list('pk', flat=True))

        return read_article_ids

    def _get_progress(self, section):
        read_article_ids = self._get_read_articles_ids()
        section_articles = section.get_descendant_articles()
        section_article_ids = set(section_articles.values_list('pk', flat=True))

        read_section_articles = read_article_ids.intersection(section_article_ids)

        return len(read_section_articles), len(section_article_ids)

    def get_progress(self, section):
        progress_enabled_ancestor = section.get_progress_bar_enabled_ancestor()
        if progress_enabled_ancestor:
            return self._get_progress(progress_enabled_ancestor)

        return None, None

    def is_section_complete(self, section):
        progress_enabled_ancestor = section.get_progress_bar_enabled_ancestor()
        if progress_enabled_ancestor:
            read_article_count, total_article_count = self._get_progress(section)

            return read_article_count == total_article_count

        return False

    def is_article_complete(self, article):
        progress_enabled_ancestor = article.get_progress_enabled_section()
        if progress_enabled_ancestor:
            return article.id in self._get_read_articles_ids()

        return False


# iogt/settings/base.py

"""
Django settings for iogt project.

Generated by 'django-admin startproject' using Django 3.1.7.

For more information on this file, see
https://docs.djangoproject.com/en/3.1/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.1/ref/settings/
"""
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import re
from datetime import timedelta
from uuid import uuid4

import django.conf.locale
import django.conf.global_settings
from django.core.exceptions import ImproperlyConfigured
from django.utils.translation import gettext_lazy as _

from iogt.settings.profanity_settings import (  # noqa: F401
    COMMENTS_ALLOW_PROFANITIES,
    PROFANITIES_LIST
)


PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BASE_DIR = os.path.dirname(PROJECT_DIR)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/


# Application definition

INSTALLED_APPS = [
    'home',
    'search',
    'iogt_users',
    'comments',
    'iogt_content_migration',
    'questionnaires',
    'messaging',
    'common',
    'notifications',
    'django.contrib.humanize',
    'wagtail_localize',
    'wagtail_localize.locales',
    'wagtail.contrib.forms',
    'wagtail.contrib.redirects',
    'wagtail.embeds',
    'wagtail.sites',
    'wagtail.users',
    'wagtail.snippets',
    'wagtail.documents',
    'wagtail.images',
    'wagtail.search',
    'wagtail.admin',
    'wagtail.core',
    'wagtail.contrib.modeladmin',
    'wagtailcache',
    'wagtailmenus',
    'wagtailmedia',
    'wagtailmarkdown',
    'wagtail_transfer',
    'wagtailsvg',
    'wagtail.contrib.settings',

    'django_comments_xtd',
    'django_comments',
    'modelcluster',
    'taggit',
    'allauth',
    'allauth.account',
    'allauth.socialaccount',
    'rest_framework',
    'sass_processor',
    'translation_manager',
    'health_check',
    'health_check.db',
    'health_check.cache',
    'health_check.storage',
    'health_check.contrib.migrations',
    'rest_framework_simplejwt',
    'google_analytics',
    'django_filters',
    'drf_yasg',
    'webpush',
    'matomo',

    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.sites',
]

MIDDLEWARE = [
    'wagtailcache.cache.UpdateCacheMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'whitenoise.middleware.WhiteNoiseMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    "iogt.middleware.LocaleMiddleware",
    "iogt.middleware.AdminLocaleMiddleware",
    'iogt.middleware.CustomRedirectMiddleware',
    'iogt_users.middlewares.RegistrationSurveyRedirectMiddleware',
    'external_links.middleware.RewriteExternalLinksMiddleware',
    'iogt.middleware.CacheControlMiddleware',
    'iogt.middleware.GlobalDataMiddleware',
    'wagtailcache.cache.FetchFromCacheMiddleware',
]

# Prevent Wagtail's built in menu from showing in Admin > Settings
WAGTAILMENUS_MAIN_MENUS_EDITABLE_IN_WAGTAILADMIN = False

ROOT_URLCONF = 'iogt.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [
            os.path.join(PROJECT_DIR, 'templates'),
        ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                "wagtail.contrib.settings.context_processors.settings",
                'wagtailmenus.context_processors.wagtailmenus',
                'wagtail.contrib.settings.context_processors.settings',
                'django.template.context_processors.i18n',
                'home.processors.commit_hash',
                'home.processors.show_footers',
                'messaging.processors.add_vapid_public_key',
                'notifications.processors.push_notification',
                'home.processors.jquery',
            ],
        },
    },
]

WSGI_APPLICATION = 'iogt.wsgi.application'

# Database
# https://docs.djangoproject.com/en/3.1/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Authentication
AUTHENTICATION_BACKENDS = [
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend'
]

AUTH_USER_MODEL = 'iogt_users.User'

# Password validation
# https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
        'OPTIONS': {
            'min_length': 4
        }
    }
]

# Internationalization
# https://docs.djangoproject.com/en/3.1/topics/i18n/

LANGUAGE_CODE = 'en'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.1/howto/static-files/

STATICFILES_FINDERS = [
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'sass_processor.finders.CssFinder',
]

STATICFILES_DIRS = [
    os.path.join(PROJECT_DIR, 'static'),
]

STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL = '/static/'

MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = '/media/'

SASS_PROCESSOR_ROOT = os.path.join(BASE_DIR, 'static')

# Allauth settings (https://django-allauth.readthedocs.io/en/latest/configuration.html)
# ACCOUNT_SIGNUP_FORM_CLASS = 'iogt_users.forms.AccountSignUpAdditionalFieldsForm'

# Control the forms that django-allauth uses
ACCOUNT_FORMS = {
    "login": "allauth.account.forms.LoginForm",
    "add_email": "allauth.account.forms.AddEmailForm",
    "change_password": "iogt_users.forms.ChangePasswordForm",
    "set_password": "allauth.account.forms.SetPasswordForm",
    "reset_password": "allauth.account.forms.ResetPasswordForm",
    "reset_password_from_key": "allauth.account.forms.ResetPasswordKeyForm",
    "disconnect": "allauth.socialaccount.forms.DisconnectForm",
    # Use our custom signup form
    "signup": "iogt_users.forms.AccountSignupForm",
}
# ACCOUNT_SIGNUP_FORM_CLASS = 'iogt_users.extra_forms.AccountSignUpAdditionalFieldsForm'

# Wagtail settings

WAGTAIL_SITE_NAME = "IoGT"
ACCOUNT_ADAPTER = 'iogt_users.adapters.AccountAdapter'

WAGTAIL_USER_EDIT_FORM = 'iogt_users.forms.WagtailAdminUserEditForm'
WAGTAIL_USER_CREATION_FORM = 'iogt_users.forms.WagtailAdminUserCreateForm'
WAGTAIL_USER_CUSTOM_FIELDS = ['display_name', 'first_name', 'last_name', 'email', 'terms_accepted']

# Base URL to use when referring to full URLs within the Wagtail admin backend -
# e.g. in notification emails. Don't include '/admin' or a trailing slash
BASE_URL = os.getenv('BASE_URL', '')

# SITE ID
SITE_ID = 1

# Comments
COMMENTS_APP = 'django_comments_xtd'
COMMENTS_XTD_MAX_THREAD_LEVEL = 1

# Miscellaneous
LOGIN_REDIRECT_URL = "user_profile"
ACCOUNT_LOGOUT_REDIRECT_URL = "logout_redirect"
LOGIN_URL = 'account_login'
WAGTAIL_FRONTEND_LOGIN_URL = LOGIN_URL

#  To help obfuscating comments before they are sent for confirmation.
COMMENTS_XTD_SALT = (b"Timendi causa est nescire. "
                     b"Aequam memento rebus in arduis servare mentem.")

# Source mail address used for notifications.
COMMENTS_XTD_FROM_EMAIL = "noreply@example.com"

# Contact mail address to show in messages.
COMMENTS_XTD_CONTACT_EMAIL = "helpdesk@example.com"

COMMENTS_XTD_CONFIRM_EMAIL = False

COMMENTS_XTD_FORM_CLASS = 'comments.forms.CommentForm'

COMMENTS_XTD_APP_MODEL_OPTIONS = {
    'default': {
        'allow_flagging': True,
        'allow_feedback': True,
        'show_feedback': True,
        'who_can_post': 'users'
    }
}

WAGTAIL_I18N_ENABLED = True

WAGTAIL_CONTENT_LANGUAGES = LANGUAGES = [
    ('ar', _('Arabic')),
    ('bn', _('Bengali')),
    ('ny', _('Chichewa')), # previously 'ch'
    ('prs', _('Dari')),
    ('en', _('English')),
    ('fa', _('Farsi')),
    ('fr', _('French')),
    ('hi', _('Hindi')),
    ('id', _('Indonesian')),
    ('kaa', _('Karakalpak')),
    ('km', _('Khmer')),
    ('rw', _('Kinyarwanda')),
    ('rn', _('Kirundi')),
    ('ku', _('Kurdish')),
    ('mg', _('Malagasy')),
    ('ne', _('Nepali')),
    ('nr', _('Ndebele')),
    ('ps', _('Pashto')),
    ('pt', _('Portuguese')),
    ('qu', _('Quechua')),
    ('ru', _('Russian')),
    ('sn', _('Shona')), # previously 'sho'
    ('si', _('Sinhala')),
    ('es', _('Spanish')),
    ('sw', _('Swahili')),
    ('tg', _('Tajik')),
    ('ta', _('Tamil')),
    ('ti', _('Tigrinya')),
    ('tr', _('Turkish')),
    ('uk', _('Ukraine')),
    ('ur', _('Urdu')),
    ('uz', _('Uzbek')),
    ('zu', _('Zulu')),
    ('xy', _('Testing')),
]

EXTRA_LANG_INFO = {
    'bn': {
        'bidi': False,
        'code': 'bn',
        'name': 'Bengali',
        'name_local': 'Bengali'
    },
    'ku': {
        'bidi': False,
        'code': 'ku',
        'name': 'Kurdish',
        'name_local': 'Kurdish'
    },
    'kaa': {
        'bidi': False,
        'code': 'kaa',
        'name': 'Karakalpak',
        'name_local': 'Karakalpak'
    },
    'mg': {
        'bidi': False,
        'code': 'mg',
        'name': 'Malagasy',
        'name_local': 'Malagasy',
    },
    'nr': {
        'bidi': False,
        'code': 'nr',
        'name': 'Ndebele',
        'name_local': 'Ndebele',
    },
    'ny': {
        'bidi': False,
        'code': 'ny',
        'name': 'Chichewa',
        'name_local': 'Chichewa',
    },
    'qu': {
        'bidi': False,
        'code': 'qu',
        'name': 'Quechua',
        'name_local': 'Quechua',
    },
    'prs': {
        'bidi': True,
        'code': 'prs',
        'name': 'Dari',
        'name_local': 'Dari',
    },
    'ps': {
        'bidi': True,
        'code': 'ps',
        'name': 'Pashto',
        'name_local': 'Pashto',
    },
    'rn': {
        'bidi': False,
        'code': 'rn',
        'name': 'Kirundi',
        'name_local': 'Ikirundi',
    },
    'rw': {
        'bidi': False,
        'code': 'rw',
        'name': 'Kinyarwanda',
        'name_local': 'Kinyarwanda',
    },
    'sn': {
        'bidi': False,
        'code': 'sn',
        'name': 'Shona',
        'name_local': 'Shona',
    },
    'si': {
        'bidi': False,
        'code': 'si',
        'name': 'Sinhala',
        'name_local': 'Sinhala',
    },
    'ti': {
        'bidi': False,
        'code': 'ti',
        'name': 'Tigrinya',
        'name_local': 'Tigrinya',
    },
    'zu': {
        'bidi': False,
        'code': 'zu',
        'name': 'Zulu',
        'name_local': 'Zulu',
    },
    'xy': {
        'bidi': False,
        'code': 'xy',
        'name': 'Testing',
        'name_local': 'Testing',
    },
}

django.conf.locale.LANG_INFO.update(EXTRA_LANG_INFO)

LANGUAGES_BIDI = django.conf.global_settings.LANGUAGES_BIDI + ["ps", "prs"]

LOCALE_PATHS = [
    os.path.join(BASE_DIR, "locale"),
]

TRANSLATIONS_BASE_DIR = BASE_DIR

# ========= Rapid Pro =================
RAPIDPRO_BOT_GROUP_NAME = os.getenv('RAPIDPRO_BOT_GROUP_NAME', 'rapidpro_chatbot')

# Wagtail transfer default values. Override these in local.py
WAGTAILTRANSFER_SECRET_KEY = os.getenv('WAGTAILTRANSFER_SECRET_KEY')
WAGTAILTRANSFER_SOURCES = {}

WAGTAILMENUS_FLAT_MENU_ITEMS_RELATED_NAME = 'iogt_flat_menu_items'

WAGTAIL_RICH_TEXT_FIELD_FEATURES = [
    'h2', 'h3', 'h4',
    'bold', 'italic',
    'ol', 'ul',
    'hr',
    'link',
    'document-link',
    'image',
]

# Search results
SEARCH_RESULTS_PER_PAGE = 10

COMMIT_HASH = os.getenv('COMMIT_HASH')

EXPORT_FILENAME_TIMESTAMP_FORMAT = '%Y-%m-%dT%H%M%S'

WAGTAILMARKDOWN = {
    'allowed_tags': ['i', 'b'],
}

TRANSLATIONS_PROJECT_BASE_DIR = BASE_DIR

from iogt.patch import *

WAGTAILTRANSFER_UPDATE_RELATED_MODELS = ['wagtailimages.image', 'wagtailsvg.svg',]
WAGTAILTRANSFER_SHOW_ERROR_FOR_REFERENCED_PAGES = True
WAGTAILTRANSFER_LOOKUP_FIELDS = {
    'taggit.tag': ['slug'],
    'wagtailcore.locale': ['language_code'],
    'iogt_users.user': ['username'],
}

REST_FRAMEWORK = {
    'DATETIME_FORMAT': '%Y-%m-%dT%H:%M:%S.%fZ',
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.BasicAuthentication',
        'rest_framework_simplejwt.authentication.JWTAuthentication',
    ),
    'DEFAULT_FILTER_BACKENDS': (
        'django_filters.rest_framework.DjangoFilterBackend',
    ),
}

SIMPLE_JWT = {
    'ACCESS_TOKEN_LIFETIME': timedelta(days=365),
}

CACHE = os.getenv('CACHE', '') == 'enable'
if CACHE:
    CACHE_LOCATION = os.getenv('CACHE_LOCATION')
    if not CACHE_LOCATION:
        raise ImproperlyConfigured(
            "CACHE_LOCATION must be set if CACHE is set to 'enable'")
    CACHE_BACKEND = os.getenv(
        'CACHE_BACKEND',
        'wagtailcache.compat_backends.django_redis.RedisCache')
    DJANGO_REDIS_IGNORE_EXCEPTIONS = True
    SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'
    WAGTAIL_CACHE = True
    WAGTAIL_CACHE_BACKEND = 'pagecache'
    CACHE_TIMEOUT = int(os.getenv('CACHE_TIMEOUT', '300'))
    KEY_PREFIX = os.getenv('CACHE_KEY_PREFIX', str(uuid4()))
    CACHES = {
        'default': {
            'BACKEND': CACHE_BACKEND,
            'LOCATION': CACHE_LOCATION,
            'TIMEOUT': CACHE_TIMEOUT,
            'KEY_PREFIX': f'{KEY_PREFIX}_default',
        },
        # https://docs.wagtail.org/en/v2.15.6/advanced_topics/performance.html#caching-image-renditions
        'renditions': {
            'BACKEND': CACHE_BACKEND,
            'LOCATION': CACHE_LOCATION,
            'TIMEOUT': CACHE_TIMEOUT,
            'KEY_PREFIX': f'{KEY_PREFIX}_renditions',
        },
        'pagecache': {
            'BACKEND': CACHE_BACKEND,
            'LOCATION': CACHE_LOCATION,
            'TIMEOUT': CACHE_TIMEOUT,
            'KEY_PREFIX': f'{KEY_PREFIX}_pagecache',
        },
    }
else:
    WAGTAIL_CACHE = False
    SESSION_ENGINE = 'django.contrib.sessions.backends.db'

SITE_VERSION = os.getenv('SITE_VERSION', 'unknown')

HAS_MD5_HASH_REGEX = re.compile(r"\.[a-f0-9]{12}\..*$")

WEBPUSH_SETTINGS = {
    'VAPID_PUBLIC_KEY': os.getenv('VAPID_PUBLIC_KEY'),
    'VAPID_PRIVATE_KEY': os.getenv('VAPID_PRIVATE_KEY'),
    'VAPID_ADMIN_EMAIL': os.getenv('VAPID_ADMIN_EMAIL'),
}

COMMENTS_COMMUNITY_MODERATION = os.getenv('COMMENTS_COMMUNITY_MODERATION') == 'enable'
COMMENT_MODERATION_CLASS = os.getenv('COMMENT_MODERATION_CLASS', 'comments.clients.AlwaysApproveModerator')
BLACKLISTED_WORDS = os.getenv('BLACKLISTED_WORDS', '').split(',')

SUPERSET_BASE_URL = os.getenv('SUPERSET_BASE_URL')
SUPERSET_DATABASE_NAME = os.getenv('SUPERSET_DATABASE_NAME')
SUPERSET_USERNAME = os.getenv('SUPERSET_USERNAME')
SUPERSET_PASSWORD = os.getenv('SUPERSET_PASSWORD')

PUSH_NOTIFICATION = os.getenv('PUSH_NOTIFICATION', 'disable') == 'enable'
JQUERY = os.getenv('JQUERY', 'enable') == 'enable'

DATA_UPLOAD_MAX_NUMBER_FIELDS = int(os.getenv('DATA_UPLOAD_MAX_NUMBER_FIELDS', '') or '1000')

# Matomo tracking server and site information

MATOMO_SERVER_URL = os.getenv('MATOMO_SERVER_URL', '')
MATOMO_SITE_ID = int(os.getenv('MATOMO_SITE_ID', '') or '0')
MATOMO_TRACKING = os.getenv('MATOMO_TRACKING', 'disable') == 'enable'

# Width size options are 360, 750
IMAGE_SIZE_PRESET = int(os.getenv('IMAGE_SIZE_PRESET', '') or '360')

# Default primary key field type introduced in Django 3.2 or later versions
# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field

DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'


# iogt/settings/dev.py

from .base import *

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
DEBUG_TOOLBAR_ENABLE = False

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'jjfayj6d=90@@@(rop$98ryt36vuyf3!chtneyoku3_f)*z^h_'

# SECURITY WARNING: define the correct hosts in production!
ALLOWED_HOSTS = ['*']

EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

if DEBUG and DEBUG_TOOLBAR_ENABLE:
    INSTALLED_APPS += ("debug_toolbar",)
    MIDDLEWARE += ("debug_toolbar.middleware.DebugToolbarMiddleware",)

    INTERNAL_IPS = ("127.0.0.1",)
    DEBUG_TOOLBAR_CONFIG = {
        "INTERCEPT_REDIRECTS": False,
        "SHOW_TOOLBAR_CALLBACK": lambda *x: True,
    }

INSTALLED_APPS += ("django_extensions",)

try:
    from .local import *
except ImportError:
    pass


# iogt/settings/docker_compose_dev.py

import os
from .dev import *

WAGTAILSEARCH_BACKENDS = {
    'default': {
        'BACKEND': 'wagtail.search.backends.elasticsearch7',
        'URLS': ['http://elasticsearch:9200'],
        'INDEX': 'iogt',
        'TIMEOUT': 5,
        'OPTIONS': {},
        'INDEX_SETTINGS': {},
        'AUTO_UPDATE': True,
        'ATOMIC_REBUILD': True
    }
}


# Uncomment if you want to run Postgres
# DATABASES = {
#     'default': {
#         'ENGINE': 'django.db.backends.postgresql',
#         'NAME': os.environ.get('DB_NAME'),
#         'USER': os.environ.get('DB_USER'),
#         'PASSWORD': os.environ.get('DB_PASSWORD'),
#         'HOST': os.environ.get('DB_HOST'),
#         'PORT': os.environ.get('DB_PORT'),
#     }
# }


# iogt/settings/production.py

import os
from .base import *

DEBUG = os.getenv('DEBUG') == 'enable'
SECRET_KEY = os.environ.get('SECRET_KEY')
ALLOWED_HOSTS = ['*']
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql',
        'NAME': os.environ.get('DB_NAME'),
        'USER': os.environ.get('DB_USER'),
        'PASSWORD': os.environ.get('DB_PASSWORD'),
        'HOST': os.environ.get('DB_HOST'),
        'PORT': os.environ.get('DB_PORT'),
        'CONN_MAX_AGE': int(os.getenv('DB_CONN_MAX_AGE', '0')),
        'OPTIONS': {
            'sslmode': os.environ.get('DB_SSL_MODE', 'require')
        }
    }
}

WAGTAILTRANSFER_SECRET_KEY = os.environ.get('WAGTAILTRANSFER_SECRET_KEY')
WAGTAILTRANSFER_SOURCES = {
   os.environ.get('WAGTAILTRANSFER_SOURCE_NAME', 'default'): {
      'BASE_URL': os.environ.get('WAGTAILTRANSFER_SOURCE_BASE_URL'),
      'SECRET_KEY': os.environ.get('WAGTAILTRANSFER_SOURCE_SECRET_KEY'),
   },
}

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
        },
    },
    'root': {
        'handlers': ['console'],
        'level': os.getenv('LOG_LEVEL', 'INFO')
    },
}

SITE_VERSION = '2.3.7'

try:
    from .local import *
except ImportError:
    pass


# iogt/settings/profanity_settings.py

COMMENTS_ALLOW_PROFANITIES = True
PROFANITIES_LIST = ('crap', 'hell')


# iogt/settings/test.py

from os import getenv

from .base import *


ALLOWED_HOSTS = ['*']
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql',
        'NAME': getenv('DB_NAME', 'postgres'),
        'USER': getenv('DB_USER', 'postgres'),
        'PASSWORD': getenv('DB_PASSWORD', 'iogt'),
        'HOST': getenv('DB_HOST', 'database'),
        'PORT': getenv('DB_PORT', '5432'),
    }
}
SE_APP_HOST = "django"
SE_HUB_HOST = "selenium-hub"
SE_HUB_PORT = "4444"
SECRET_KEY = "##secret_key_for_testing##"
STATICFILES_STORAGE = "django.contrib.staticfiles.storage.StaticFilesStorage"
WAGTAILSEARCH_BACKENDS = {
    'default': {
        'BACKEND': 'wagtail.search.backends.database',
    }
}


# iogt_users/migrations/0001_initial.py

# Generated by Django 3.1.13 on 2021-07-07 08:03

from django.conf import settings
import django.contrib.auth.models
import django.contrib.auth.validators
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ('auth', '0012_alter_user_first_name_max_length'),
    ]

    operations = [
        migrations.CreateModel(
            name='User',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('password', models.CharField(max_length=128, verbose_name='password')),
                ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')),
                ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')),
                ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')),
                ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')),
                ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')),
                ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')),
                ('first_name', models.CharField(blank=True, max_length=150, null=True, verbose_name='first name')),
                ('last_name', models.CharField(blank=True, max_length=150, null=True, verbose_name='last name')),
                ('email', models.EmailField(blank=True, max_length=254, null=True, verbose_name='email address')),
                ('terms_accepted', models.BooleanField(default=False)),
                ('has_filled_registration_survey', models.BooleanField(default=False)),
                ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')),
                ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')),
            ],
            options={
                'ordering': ('id',),
            },
            managers=[
                ('objects', django.contrib.auth.models.UserManager()),
            ],
        ),
        migrations.CreateModel(
            name='Profile',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
            ],
        ),
    ]


# iogt_users/migrations/0002_user_read_articles.py

# Generated by Django 3.1.13 on 2021-07-07 08:03

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('home', '0003_auto_20210707_0702'),
        ('iogt_users', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='user',
            name='read_articles',
            field=models.ManyToManyField(to='home.Article'),
        ),
    ]


# iogt_users/migrations/0003_user_display_name.py

# Generated by Django 3.1.13 on 2021-11-17 09:30

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('iogt_users', '0002_user_read_articles'),
    ]

    operations = [
        migrations.AddField(
            model_name='user',
            name='display_name',
            field=models.CharField(blank=True, max_length=255, null=True, verbose_name='display name'),
        ),
    ]


# iogt_users/migrations/0004_user_has_viewed_registration_survey.py

# Generated by Django 3.1.14 on 2022-01-20 07:41

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('iogt_users', '0003_user_display_name'),
    ]

    operations = [
        migrations.AddField(
            model_name='user',
            name='has_viewed_registration_survey',
            field=models.BooleanField(default=False),
        ),
    ]


# iogt_users/migrations/0005_auto_20220120_0813.py

# Generated by Django 3.1.14 on 2022-01-20 08:13

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('iogt_users', '0004_user_has_viewed_registration_survey'),
    ]

    operations = [
    ]


# matomo/templatetags/matomo_tags.py

from django import template
from django.conf import settings


register = template.Library()


@register.inclusion_tag('matomo_tracking_tags.html', takes_context=True)
def matomo_tracking_tags(context):
    context.update({
        'tracking_enabled': (
                settings.MATOMO_TRACKING
                and settings.MATOMO_SERVER_URL
                and settings.MATOMO_SITE_ID
        ),
        'matomo_site_id': settings.MATOMO_SITE_ID,
        'matomo_server_url': settings.MATOMO_SERVER_URL
    })
    return context


# messaging/api/permissions.py

from django.conf import settings
from rest_framework import permissions


class IsRapidProGroupUser(permissions.BasePermission):
    message = 'User is not allowed to access the webhook'

    def has_permission(self, request, view):
        return request.user.groups.filter(name=settings.RAPIDPRO_BOT_GROUP_NAME).exists()


# messaging/api/serializers.py

from rest_framework import serializers


class RapidProMessageSerializer(serializers.Serializer):
    channel = serializers.UUIDField()
    from_ = serializers.CharField(required=False)
    id = serializers.CharField()
    quick_replies = serializers.JSONField()
    text = serializers.CharField()
    to = serializers.UUIDField()

    def get_fields(self):
        fields = super().get_fields()
        fields['from'] = fields.pop('from_')
        return fields


# messaging/api/urls.py

from django.urls import path

from messaging.api.views import RapidProWebhook

app_name = 'api'

urlpatterns = [
    path('rapidpro-webhook/', RapidProWebhook.as_view(), name='rapidpro_webhook')
]


# messaging/api/views.py

import json

from django.contrib.auth import get_user_model
from rest_framework import status
from rest_framework.generics import get_object_or_404
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.permissions import IsAuthenticated

from .permissions import IsRapidProGroupUser
from .serializers import RapidProMessageSerializer
from ..chat import ChatManager
from ..models import Thread

User = get_user_model()


class RapidProWebhook(APIView):
    permission_classes = [IsAuthenticated, IsRapidProGroupUser]

    def post(self, request):
        serializer = RapidProMessageSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)

        rapidpro_message_id = serializer.validated_data['id']
        thread_uuid = serializer.validated_data['to']
        text = serializer.validated_data['text']
        quick_replies = serializer.validated_data['quick_replies']

        thread = get_object_or_404(Thread, uuid=thread_uuid)

        chat_manager = ChatManager(thread)
        chat_manager.record_reply(
            sender=request.user, text=text, quick_replies=quick_replies,
            rapidpro_message_id=rapidpro_message_id)

        return Response(data='ok', status=status.HTTP_200_OK)


# messaging/migrations/0001_initial.py

# Generated by Django 3.1.13 on 2021-07-07 07:02

from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import uuid


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='ChatbotChannel',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('display_name', models.CharField(max_length=80)),
                ('request_url', models.URLField()),
            ],
        ),
        migrations.CreateModel(
            name='Thread',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('last_message_at', models.DateTimeField(default=None, editable=False, null=True)),
                ('subject', models.CharField(max_length=150)),
                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False)),
                ('chatbot', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='messaging.chatbotchannel')),
            ],
        ),
        migrations.CreateModel(
            name='UserThread',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('is_active', models.BooleanField(default=True)),
                ('is_read', models.BooleanField(default=False)),
                ('thread', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='user_threads', to='messaging.thread')),
                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
            ],
        ),
        migrations.AddField(
            model_name='thread',
            name='users',
            field=models.ManyToManyField(through='messaging.UserThread', to=settings.AUTH_USER_MODEL),
        ),
        migrations.CreateModel(
            name='Message',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('quick_replies', models.JSONField(default=list)),
                ('rapidpro_message_id', models.IntegerField(null=True)),
                ('sent_at', models.DateTimeField(auto_now_add=True)),
                ('text', models.TextField()),
                ('sender', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='sent_messages', to=settings.AUTH_USER_MODEL)),
                ('thread', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='messages', to='messaging.thread')),
            ],
            options={
                'ordering': ('sent_at',),
            },
        ),
    ]


# messaging/migrations/0002_auto_20210719_1332.py

# Generated by Django 3.1.13 on 2021-07-19 13:32

from django.db import migrations, models
import django.db.models.deletion
import django_extensions.db.fields


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailimages', '0022_uploadedimage'),
        ('messaging', '0001_initial'),
    ]

    operations = [
        migrations.CreateModel(
            name='Attachment',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
                ('external_link', models.URLField()),
                ('file', models.FileField(blank=True, null=True, upload_to='')),
                ('image', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='wagtailimages.image')),
            ],
            options={
                'get_latest_by': 'modified',
                'abstract': False,
            },
        ),
        migrations.AddField(
            model_name='message',
            name='attachments',
            field=models.ManyToManyField(blank=True, to='messaging.Attachment'),
        ),
    ]


# messaging/migrations/0003_message_is_post_processed.py

# Generated by Django 3.1.14 on 2021-12-08 13:15

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('messaging', '0002_auto_20210719_1332'),
    ]

    operations = [
        migrations.AddField(
            model_name='message',
            name='is_post_processed',
            field=models.BooleanField(default=False),
        ),
    ]


# messaging/migrations/0004_auto_20211217_1319.py

# Generated by Django 3.1.14 on 2021-12-17 13:19

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('messaging', '0003_message_is_post_processed'),
    ]

    operations = [
        migrations.AlterField(
            model_name='chatbotchannel',
            name='display_name',
            field=models.CharField(help_text='Name for the bot that the user will see when interacting with it', max_length=80),
        ),
        migrations.AlterField(
            model_name='chatbotchannel',
            name='request_url',
            field=models.URLField(help_text='To set up a chatbot channel on your RapidPro server and get a request URL, follow the steps outline in the Section "Setting up a Chatbot channel" here: https://github.com/unicef/iogt/blob/develop/messaging/README.md'),
        ),
    ]


# messaging/migrations/0005_auto_20230315_0446.py

# Generated by Django 3.1.14 on 2023-03-15 04:46

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('messaging', '0004_auto_20211217_1319'),
    ]

    operations = [
        migrations.AlterField(
            model_name='message',
            name='rapidpro_message_id',
            field=models.BigIntegerField(null=True),
        ),
    ]


# messaging/templatetags/messaging_tags.py

from django import template
from django.contrib.auth import get_user_model

register = template.Library()
User = get_user_model()

@register.filter
def unread(thread, user):
    """
    Check whether there are any unread messages for a particular thread for a user.
    """
    return thread.user_threads.filter(user=user, is_read=False).exists()


@register.inclusion_tag('messaging/tags/quick_reply_form.html')
def render_quick_reply_form(thread, user, text):
    return {
        'thread': thread,
        'user': user,
        'text': text,
    }


@register.inclusion_tag('messaging/tags/chatbot_auth_tokens.html')
def render_chatbot_auth_tokens():
    return {
        'tokens': User.get_rapidpro_bot_auth_tokens(),
    }


# messaging/tests/webhook_tests.py

from uuid import uuid4

from django.core import management
from django.urls import reverse
from rest_framework.test import APITestCase, APIClient
from rest_framework_simplejwt.tokens import RefreshToken

from home.models import User
from messaging.factories import ThreadFactory
from messaging.models import Message, Attachment


class RapidProWebhookTest(APITestCase):
    def setUp(self) -> None:
        username = uuid4()
        management.call_command('sync_rapidpro_bot_user', username=username)
        self.bot_user = User.objects.get(username=username)
        self.client = APIClient()
        self.client.credentials(HTTP_AUTHORIZATION=f'Bearer {RefreshToken.for_user(self.bot_user).access_token}')

    def test_webhook_stitches_messages(self):
        thread = ThreadFactory()

        rapidpro_data_list = [
            {
                "id": "1",
                "text": "Some message with the first part of text. First part ends at the exclamation mark!",
                "to": str(thread.uuid),
                "from": "abcd",
                "channel": "bd3577c6-65b1-4bb7-9611-306c11b1dcc5",
                "quick_replies": [
                    "Baby (0 to 23 months old)", "Young child (2 to 9 years)", "Teenager (10 to 17 years)"
                ]
            },
            {
                "id": "1",
                "text": "The second part starts here",
                "to": str(thread.uuid),
                "from": "abcd",
                "channel": "bd3577c6-65b1-4bb7-9611-306c11b1dcc5",
                "quick_replies": [
                    "Baby (0 to 23 months old)", "Young child (2 to 9 years)", "Teenager (10 to 17 years)"
                ]
            },

        ]

        for data in rapidpro_data_list:
            response = self.client.post(path=reverse('messaging:api:rapidpro_webhook'), data=data, format='json')

        message = Message.objects.first()

        self.assertEqual(response.status_code, 200)

        self.assertEqual(Message.objects.count(), 1)
        self.assertEqual(message.text, 'Some message with the first part of text.'
                                       ' First part ends at the exclamation mark!The second part starts here')
        self.assertEqual(len(message.quick_replies), 3)

    def test_webhook_parses_attachments(self):
        thread = ThreadFactory()

        rapidpro_data = {
            "id": "1",
            "text": "Some message with attachment.\nhttps://rapidpro.idems.international/media/attachments/43/"
                    "15890/steps/3de4f80a-1eab-42db-8b7e-d7c35edecd06.bin",
            "to": str(thread.uuid),
            "from": "abcd",
            "channel": "bd3577c6-65b1-4bb7-9611-306c11b1dcc5",
            "quick_replies": [
                "Baby (0 to 23 months old)", "Young child (2 to 9 years)", "Teenager (10 to 17 years)"]}

        response = self.client.post(path=reverse('messaging:api:rapidpro_webhook'), data=rapidpro_data, format='json')

        message = thread.get_renderable_messages().first()
        attachment = message.attachments.all()[0]

        self.assertEqual(response.status_code, 200)

        self.assertEqual(Message.objects.count(), 1)
        self.assertEqual(message.attachments.all().count(), 1)
        self.assertEqual(message.text, 'Some message with attachment.')
        self.assertEqual(attachment.external_link,
                         'https://rapidpro.idems.international/media/attachments/43/15890/steps/'
                         '3de4f80a-1eab-42db-8b7e-d7c35edecd06.bin')
        self.assertIsNotNone(attachment.file)

    def test_stitched_attachment_parsing(self):
        thread = ThreadFactory()

        rapidpro_data_list = [
            {
                "id": "1",
                "text": "Some message with a stitched url. \nhttps://via.placeholder.com/200",
                "to": str(thread.uuid),
                "from": "abcd",
                "channel": "bd3577c6-65b1-4bb7-9611-306c11b1dcc5",
                "quick_replies": [
                    "Baby (0 to 23 months old)", "Young child (2 to 9 years)", "Teenager (10 to 17 years)"
                ]
            },
            {
                "id": "1",
                "text": "678910.jpg",
                "to": str(thread.uuid),
                "from": "abcd",
                "channel": "bd3577c6-65b1-4bb7-9611-306c11b1dcc5",
                "quick_replies": [
                    "Baby (0 to 23 months old)", "Young child (2 to 9 years)", "Teenager (10 to 17 years)"
                ]
            },

        ]

        for data in rapidpro_data_list:
            response = self.client.post(path=reverse('messaging:api:rapidpro_webhook'), data=data, format='json')

        message = thread.get_renderable_messages().first()
        attachment = message.attachments.first()

        self.assertEqual(response.status_code, 200)

        self.assertEqual(Message.objects.count(), 1)
        self.assertEqual(message.text, 'Some message with a stitched url. ')
        self.assertEqual(message.attachments.count(), 1)
        self.assertEqual(attachment.external_link, 'https://via.placeholder.com/200678910.jpg')
        self.assertEqual(len(message.quick_replies), 3)

    def test_single_download(self):
        thread = ThreadFactory()

        rapidpro_data_list = [
            {
                "id": "1",
                "text": "Some message with attachment \nhttps://via.placeholder.com/200",
                "to": str(thread.uuid),
                "from": "abcd",
                "channel": "bd3577c6-65b1-4bb7-9611-306c11b1dcc5",
                "quick_replies": [
                    "Baby (0 to 23 months old)", "Young child (2 to 9 years)", "Teenager (10 to 17 years)"
                ]
            },
            {
                "id": "2",
                "text": "Another message with the same attachment \nhttps://via.placeholder.com/200",
                "to": str(thread.uuid),
                "from": "abcd",
                "channel": "bd3577c6-65b1-4bb7-9611-306c11b1dcc5",
                "quick_replies": [
                    "Baby (0 to 23 months old)", "Young child (2 to 9 years)", "Teenager (10 to 17 years)"
                ]
            },

        ]

        for data in rapidpro_data_list:
            response = self.client.post(path=reverse('messaging:api:rapidpro_webhook'), data=data, format='json')

        messages = thread.get_renderable_messages()

        self.assertEqual(response.status_code, 200)

        self.assertEqual(len(messages), 2)
        self.assertEqual(messages[0].text, 'Another message with the same attachment ')
        self.assertEqual(messages[1].text, 'Some message with attachment ')
        self.assertEqual(Attachment.objects.count(), 1)

        attachment = Attachment.objects.first()
        self.assertEqual('https://via.placeholder.com/200', attachment.external_link)

        self.assertIsNotNone(attachment.image)


# notifications/migrations/0001_initial.py

# Generated by Django 3.1.14 on 2022-07-29 13:17

from django.db import migrations, models


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ('auth', '0012_alter_user_first_name_max_length'),
    ]

    operations = [
        migrations.CreateModel(
            name='Notification',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('head', models.CharField(max_length=255)),
                ('body', models.TextField()),
                ('url', models.URLField(blank=True, null=True)),
                ('groups', models.ManyToManyField(related_name='notifications', to='auth.Group')),
            ],
        ),
    ]


# questionnaires/migrations/0001_initial.py

# Generated by Django 3.1.13 on 2021-07-07 07:02

from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import home.blocks
import home.mixins
import modelcluster.fields
import questionnaires.blocks
import wagtail.core.blocks
import wagtail.core.fields
import wagtail.images.blocks


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('wagtailcore', '0059_apply_collection_ordering'),
    ]

    operations = [
        migrations.CreateModel(
            name='Poll',
            fields=[
                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.page')),
                ('description', wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock()), ('image', wagtail.images.blocks.ImageChooserBlock())], blank=True, null=True)),
                ('thank_you_text', wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock()), ('media', home.blocks.MediaBlock(icon='media')), ('image', wagtail.images.blocks.ImageChooserBlock())], blank=True, null=True)),
                ('allow_anonymous_submissions', models.BooleanField(default=True, help_text='Check this to allow users who are NOT logged in to complete surveys.')),
                ('allow_multiple_submissions', models.BooleanField(default=True, help_text='Check this to allow multiple form submissions for users')),
                ('submit_button_text', models.CharField(default='Submit', help_text='Submit button text', max_length=40, null=True)),
                ('direct_display', models.BooleanField(default=False)),
                ('show_results', models.BooleanField(default=True, help_text='This option allows the users to see the results.')),
                ('result_as_percentage', models.BooleanField(default=True, help_text='If not checked, the results will be shown as a total instead of a percentage.')),
            ],
            options={
                'verbose_name': 'Poll',
                'verbose_name_plural': 'Polls',
            },
            bases=('wagtailcore.page', home.mixins.PageUtilsMixin),
        ),
        migrations.CreateModel(
            name='Quiz',
            fields=[
                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.page')),
                ('description', wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock()), ('image', wagtail.images.blocks.ImageChooserBlock())], blank=True, null=True)),
                ('thank_you_text', wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock()), ('media', home.blocks.MediaBlock(icon='media')), ('image', wagtail.images.blocks.ImageChooserBlock())], blank=True, null=True)),
                ('allow_anonymous_submissions', models.BooleanField(default=True, help_text='Check this to allow users who are NOT logged in to complete surveys.')),
                ('allow_multiple_submissions', models.BooleanField(default=True, help_text='Check this to allow multiple form submissions for users')),
                ('submit_button_text', models.CharField(default='Submit', help_text='Submit button text', max_length=40, null=True)),
                ('direct_display', models.BooleanField(default=False)),
                ('multi_step', models.BooleanField(default=False, help_text='Whether to display the survey questions to the user one at a time, instead of all at once.', verbose_name='Multi-step')),
            ],
            options={
                'verbose_name': 'quiz',
                'verbose_name_plural': 'quizzes',
            },
            bases=('wagtailcore.page', home.mixins.PageUtilsMixin),
        ),
        migrations.CreateModel(
            name='Survey',
            fields=[
                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.page')),
                ('description', wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock()), ('image', wagtail.images.blocks.ImageChooserBlock())], blank=True, null=True)),
                ('thank_you_text', wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock()), ('media', home.blocks.MediaBlock(icon='media')), ('image', wagtail.images.blocks.ImageChooserBlock())], blank=True, null=True)),
                ('allow_anonymous_submissions', models.BooleanField(default=True, help_text='Check this to allow users who are NOT logged in to complete surveys.')),
                ('allow_multiple_submissions', models.BooleanField(default=True, help_text='Check this to allow multiple form submissions for users')),
                ('submit_button_text', models.CharField(default='Submit', help_text='Submit button text', max_length=40, null=True)),
                ('direct_display', models.BooleanField(default=False)),
                ('multi_step', models.BooleanField(default=False, help_text='Whether to display the survey questions to the user one at a time, instead of all at once.', verbose_name='Multi-step')),
            ],
            options={
                'verbose_name': 'survey',
                'verbose_name_plural': 'surveys',
            },
            bases=('wagtailcore.page', home.mixins.PageUtilsMixin),
        ),
        migrations.CreateModel(
            name='UserSubmission',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('form_data', models.TextField()),
                ('submit_time', models.DateTimeField(auto_now_add=True, verbose_name='submit time')),
                ('page', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='wagtailcore.page')),
                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
            ],
            options={
                'verbose_name': 'form submission',
                'verbose_name_plural': 'form submissions',
                'abstract': False,
            },
        ),
        migrations.CreateModel(
            name='SurveyFormField',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('sort_order', models.IntegerField(blank=True, editable=False, null=True)),
                ('clean_name', models.CharField(blank=True, default='', help_text='Safe name of the form field, the label converted to ascii_snake_case', max_length=255, verbose_name='name')),
                ('label', models.CharField(help_text='The label of the form field', max_length=255, verbose_name='label')),
                ('field_type', models.CharField(choices=[('singleline', 'Single line text'), ('multiline', 'Multi-line text'), ('email', 'Email'), ('number', 'Number'), ('url', 'URL'), ('checkbox', 'Checkbox'), ('checkboxes', 'Checkboxes'), ('dropdown', 'Drop down'), ('multiselect', 'Multiple select'), ('radio', 'Radio buttons'), ('date', 'Date'), ('datetime', 'Date/time'), ('hidden', 'Hidden field')], max_length=16, verbose_name='field type')),
                ('choices', models.TextField(blank=True, help_text='Comma separated list of choices. Only applicable in checkboxes, radio and dropdown.', verbose_name='choices')),
                ('default_value', models.CharField(blank=True, help_text='Default value. Comma separated values supported for checkboxes.', max_length=255, verbose_name='default value')),
                ('help_text', models.CharField(blank=True, max_length=255, verbose_name='help text')),
                ('required', models.BooleanField(default=False, verbose_name='required')),
                ('admin_label', models.CharField(help_text='Column header used during CSV export of survey responses.', max_length=256, verbose_name='admin_label')),
                ('skip_logic', questionnaires.blocks.SkipLogicField([('skip_logic', wagtail.core.blocks.StructBlock([('choice', wagtail.core.blocks.CharBlock(required=False)), ('skip_logic', wagtail.core.blocks.ChoiceBlock(choices=[('next', 'Next default question'), ('end', 'End of survey'), ('question', 'Another question')], required=False)), ('question', wagtail.core.blocks.IntegerBlock(required=False))]))], blank=True, verbose_name='Answer options')),
                ('page_break', models.BooleanField(default=False, help_text='Inserts a page break which puts the next question onto a new page')),
                ('page', modelcluster.fields.ParentalKey(on_delete=django.db.models.deletion.CASCADE, related_name='survey_form_fields', to='questionnaires.survey')),
            ],
            options={
                'ordering': ['sort_order'],
                'abstract': False,
            },
        ),
        migrations.CreateModel(
            name='QuizFormField',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('sort_order', models.IntegerField(blank=True, editable=False, null=True)),
                ('clean_name', models.CharField(blank=True, default='', help_text='Safe name of the form field, the label converted to ascii_snake_case', max_length=255, verbose_name='name')),
                ('label', models.CharField(help_text='The label of the form field', max_length=255, verbose_name='label')),
                ('field_type', models.CharField(choices=[('singleline', 'Single line text'), ('multiline', 'Multi-line text'), ('email', 'Email'), ('number', 'Number'), ('url', 'URL'), ('checkbox', 'Checkbox'), ('checkboxes', 'Checkboxes'), ('dropdown', 'Drop down'), ('multiselect', 'Multiple select'), ('radio', 'Radio buttons'), ('date', 'Date'), ('datetime', 'Date/time'), ('hidden', 'Hidden field')], max_length=16, verbose_name='field type')),
                ('choices', models.TextField(blank=True, help_text='Comma separated list of choices. Only applicable in checkboxes, radio and dropdown.', verbose_name='choices')),
                ('default_value', models.CharField(blank=True, help_text='Default value. Comma separated values supported for checkboxes.', max_length=255, verbose_name='default value')),
                ('help_text', models.CharField(blank=True, max_length=255, verbose_name='help text')),
                ('required', models.BooleanField(default=True, verbose_name='required')),
                ('admin_label', models.CharField(help_text='Column header used during CSV export of survey responses.', max_length=256, verbose_name='admin_label')),
                ('skip_logic', questionnaires.blocks.SkipLogicField([('skip_logic', wagtail.core.blocks.StructBlock([('choice', wagtail.core.blocks.CharBlock(required=False)), ('skip_logic', wagtail.core.blocks.ChoiceBlock(choices=[('next', 'Next default question'), ('end', 'End of survey'), ('question', 'Another question')], required=False)), ('question', wagtail.core.blocks.IntegerBlock(required=False))]))], blank=True, verbose_name='Answer options')),
                ('page_break', models.BooleanField(default=False, help_text='Inserts a page break which puts the next question onto a new page')),
                ('correct_answer', models.CharField(help_text='Please provide correct answer for this question', max_length=256, verbose_name='correct_answer')),
                ('feedback', models.CharField(blank=True, help_text='Feedback message for user answer.', max_length=255, null=True, verbose_name='Feedback')),
                ('page', modelcluster.fields.ParentalKey(on_delete=django.db.models.deletion.CASCADE, related_name='quiz_form_fields', to='questionnaires.quiz')),
            ],
            options={
                'ordering': ['sort_order'],
                'abstract': False,
            },
        ),
        migrations.CreateModel(
            name='PollFormField',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('sort_order', models.IntegerField(blank=True, editable=False, null=True)),
                ('clean_name', models.CharField(blank=True, default='', help_text='Safe name of the form field, the label converted to ascii_snake_case', max_length=255, verbose_name='name')),
                ('label', models.CharField(help_text='The label of the form field', max_length=255, verbose_name='label')),
                ('required', models.BooleanField(default=True, verbose_name='required')),
                ('choices', models.TextField(blank=True, help_text='Comma separated list of choices. Only applicable in checkboxes, radio and dropdown.', verbose_name='choices')),
                ('default_value', models.CharField(blank=True, help_text='Default value. Comma separated values supported for checkboxes.', max_length=255, verbose_name='default value')),
                ('help_text', models.CharField(blank=True, max_length=255, verbose_name='help text')),
                ('field_type', models.CharField(choices=[('checkbox', 'Checkbox'), ('checkboxes', 'Checkboxes'), ('multiselect', 'Multiple select'), ('radio', 'Radio buttons')], max_length=16, verbose_name='field type')),
                ('page', modelcluster.fields.ParentalKey(on_delete=django.db.models.deletion.CASCADE, related_name='poll_form_fields', to='questionnaires.poll')),
            ],
            options={
                'ordering': ['sort_order'],
                'abstract': False,
            },
        ),
    ]


# questionnaires/migrations/0002_auto_20210719_1820.py

# Generated by Django 3.1.11 on 2021-07-19 18:20

from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0001_initial'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='poll',
            options={'verbose_name': 'poll', 'verbose_name_plural': 'polls'},
        ),
    ]


# questionnaires/migrations/0003_auto_20210720_1347.py

# Generated by Django 3.1.13 on 2021-07-20 13:47

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0002_auto_20210719_1820'),
    ]

    operations = [
        migrations.AlterField(
            model_name='quizformfield',
            name='correct_answer',
            field=models.CharField(help_text='Comma separated list of choices. Only applicable in checkboxes, radio, dropdown and multiselect.', max_length=256, verbose_name='correct_answer'),
        ),
    ]


# questionnaires/migrations/0004_auto_20210723_0906.py

# Generated by Django 3.1.13 on 2021-07-23 09:06

from django.db import migrations
import wagtail.core.blocks
import wagtail.core.fields


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0003_auto_20210720_1347'),
    ]

    operations = [
        migrations.AlterField(
            model_name='quizformfield',
            name='skip_logic',
            field=wagtail.core.fields.StreamField([('skip_logic', wagtail.core.blocks.StructBlock([('choice', wagtail.core.blocks.CharBlock(required=False)), ('skip_logic', wagtail.core.blocks.ChoiceBlock(choices=[('next', 'Next default question'), ('end', 'End of survey'), ('question', 'Another question')], required=False)), ('question', wagtail.core.blocks.IntegerBlock(required=False))]))], blank=True),
        ),
        migrations.AlterField(
            model_name='surveyformfield',
            name='skip_logic',
            field=wagtail.core.fields.StreamField([('skip_logic', wagtail.core.blocks.StructBlock([('choice', wagtail.core.blocks.CharBlock(required=False)), ('skip_logic', wagtail.core.blocks.ChoiceBlock(choices=[('next', 'Next default question'), ('end', 'End of survey'), ('question', 'Another question')], required=False)), ('question', wagtail.core.blocks.IntegerBlock(required=False))]))], blank=True),
        ),
    ]


# questionnaires/migrations/0005_auto_20210723_1321.py

# Generated by Django 3.1.13 on 2021-07-23 13:21

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0004_auto_20210723_0906'),
    ]

    operations = [
        migrations.AlterField(
            model_name='quizformfield',
            name='field_type',
            field=models.CharField(choices=[('checkbox', 'Checkbox'), ('checkboxes', 'Checkboxes'), ('date', 'Date'), ('datetime', 'Date/time'), ('dropdown', 'Drop down'), ('email', 'Email'), ('hidden', 'Hidden field'), ('singleline', 'Single line text'), ('multiline', 'Multi-line text'), ('multiselect', 'Multiple select'), ('number', 'Number'), ('positivenumber', 'Positive number'), ('radio', 'Radio buttons'), ('url', 'URL')], max_length=16, verbose_name='field type'),
        ),
        migrations.AlterField(
            model_name='surveyformfield',
            name='field_type',
            field=models.CharField(choices=[('checkbox', 'Checkbox'), ('checkboxes', 'Checkboxes'), ('date', 'Date'), ('datetime', 'Date/time'), ('dropdown', 'Drop down'), ('email', 'Email'), ('hidden', 'Hidden field'), ('singleline', 'Single line text'), ('multiline', 'Multi-line text'), ('multiselect', 'Multiple select'), ('number', 'Number'), ('positivenumber', 'Positive number'), ('radio', 'Radio buttons'), ('url', 'URL')], max_length=16, verbose_name='field type'),
        ),
    ]


# questionnaires/migrations/0006_auto_20210728_1112.py

# Generated by Django 3.1.13 on 2021-07-28 11:12

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0005_auto_20210723_1321'),
    ]

    operations = [
        migrations.AlterField(
            model_name='pollformfield',
            name='field_type',
            field=models.CharField(choices=[('checkbox', 'Checkbox'), ('checkboxes', 'Checkboxes'), ('radio', 'Radio buttons')], max_length=16, verbose_name='field type'),
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='field_type',
            field=models.CharField(choices=[('checkbox', 'Checkbox'), ('checkboxes', 'Checkboxes'), ('date', 'Date'), ('datetime', 'Date/time'), ('dropdown', 'Drop down'), ('email', 'Email'), ('hidden', 'Hidden field'), ('singleline', 'Single line text'), ('multiline', 'Multi-line text'), ('number', 'Number'), ('positivenumber', 'Positive number'), ('radio', 'Radio buttons'), ('url', 'URL')], max_length=16, verbose_name='field type'),
        ),
        migrations.AlterField(
            model_name='surveyformfield',
            name='field_type',
            field=models.CharField(choices=[('checkbox', 'Checkbox'), ('checkboxes', 'Checkboxes'), ('date', 'Date'), ('datetime', 'Date/time'), ('dropdown', 'Drop down'), ('email', 'Email'), ('hidden', 'Hidden field'), ('singleline', 'Single line text'), ('multiline', 'Multi-line text'), ('number', 'Number'), ('positivenumber', 'Positive number'), ('radio', 'Radio buttons'), ('url', 'URL')], max_length=16, verbose_name='field type'),
        ),
    ]


# questionnaires/migrations/0007_auto_20210728_1220.py

# Generated by Django 3.1.11 on 2021-07-28 12:20

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0006_auto_20210728_1112'),
    ]

    operations = [
        migrations.AlterField(
            model_name='pollformfield',
            name='field_type',
            field=models.CharField(choices=[('checkboxes', 'Checkboxes'), ('dropdown', 'Dropdown'), ('radio', 'Radio buttons')], max_length=16, verbose_name='field type'),
        ),
    ]


# questionnaires/migrations/0008_auto_20210729_2133.py

# Generated by Django 3.1.11 on 2021-07-29 21:33

from django.db import migrations, models
import wagtail.core.blocks
import wagtail.core.fields


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0007_auto_20210728_1220'),
    ]

    operations = [
        migrations.AlterField(
            model_name='quizformfield',
            name='correct_answer',
            field=models.CharField(help_text='The correct answer/choice(s). For checkboxes: a comma separated list of choices. For checkbox: Either "on" or "off".', max_length=256, verbose_name='correct_answer'),
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='skip_logic',
            field=wagtail.core.fields.StreamField([('skip_logic', wagtail.core.blocks.StructBlock([('choice', wagtail.core.blocks.CharBlock(required=False)), ('skip_logic', wagtail.core.blocks.ChoiceBlock(choices=[('next', 'Next default question'), ('end', 'End of survey'), ('question', 'Another question')], required=False)), ('question', wagtail.core.blocks.IntegerBlock(required=False))]))], blank=True, help_text='This is used to add choices for field type radio, checkbox, checkboxes, and dropdown only. This can be used to skip questions and skipping is only allowed for radio and dropdown.'),
        ),
        migrations.AlterField(
            model_name='surveyformfield',
            name='skip_logic',
            field=wagtail.core.fields.StreamField([('skip_logic', wagtail.core.blocks.StructBlock([('choice', wagtail.core.blocks.CharBlock(required=False)), ('skip_logic', wagtail.core.blocks.ChoiceBlock(choices=[('next', 'Next default question'), ('end', 'End of survey'), ('question', 'Another question')], required=False)), ('question', wagtail.core.blocks.IntegerBlock(required=False))]))], blank=True, help_text='This is used to add choices for field type radio, checkbox, checkboxes, and dropdown only. This can be used to skip questions and skipping is only allowed for radio and dropdown.'),
        ),
    ]


# questionnaires/migrations/0009_pollindexpage_quizindexpage_surveyindexpage.py

# Generated by Django 3.1.13 on 2021-08-02 13:47
from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailcore', '0059_apply_collection_ordering'),
        ('questionnaires', '0008_auto_20210729_2133'),
    ]

    operations = [
        migrations.CreateModel(
            name='PollIndexPage',
            fields=[
                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.page')),
            ],
            options={
                'abstract': False,
            },
            bases=('wagtailcore.page',),
        ),
        migrations.CreateModel(
            name='QuizIndexPage',
            fields=[
                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.page')),
            ],
            options={
                'abstract': False,
            },
            bases=('wagtailcore.page',),
        ),
        migrations.CreateModel(
            name='SurveyIndexPage',
            fields=[
                ('page_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='wagtailcore.page')),
            ],
            options={
                'abstract': False,
            },
            bases=('wagtailcore.page',),
        ),
    ]


# questionnaires/migrations/0010_auto_20210802_1406.py

# Generated by Django 3.1.13 on 2021-08-02 14:06
from django.db import migrations


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0009_pollindexpage_quizindexpage_surveyindexpage'),
    ]

    operations = [
    ]


# questionnaires/migrations/0011_usersubmission_session_key.py

# Generated by Django 3.1.13 on 2021-08-12 06:58

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0010_auto_20210802_1406'),
    ]

    operations = [
        migrations.AddField(
            model_name='usersubmission',
            name='session_key',
            field=models.CharField(blank=True, max_length=255, null=True),
        ),
    ]


# questionnaires/migrations/0012_auto_20210827_1150.py

# Generated by Django 3.1.13 on 2021-08-27 11:50

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0011_usersubmission_session_key'),
    ]

    operations = [
        migrations.AlterField(
            model_name='pollformfield',
            name='choices',
            field=models.TextField(blank=True, help_text='Pipe (|) separated list of choices.', verbose_name='choices'),
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='correct_answer',
            field=models.TextField(help_text='If multiple choices are correct, separate choices with a pipe (|) symbol.', verbose_name='correct_answer'),
        ),
    ]


# questionnaires/migrations/0013_auto_20210825_0508.py

# Generated by Django 3.1.13 on 2021-08-25 05:08

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0012_auto_20210827_1150'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='quizformfield',
            name='skip_logic',
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='choices',
            field=models.TextField(blank=True, help_text='Pipe (|) separated list of choices.', verbose_name='choices'),
        ),
    ]


# questionnaires/migrations/0014_auto_20210827_1033.py

# Generated by Django 3.1.13 on 2021-08-27 10:33

from django.db import migrations
import questionnaires.blocks
import wagtail.core.blocks


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0013_auto_20210825_0508'),
    ]

    operations = [
        migrations.AlterField(
            model_name='surveyformfield',
            name='skip_logic',
            field=questionnaires.blocks.SkipLogicField([('skip_logic', wagtail.core.blocks.StructBlock([('choice', wagtail.core.blocks.CharBlock(required=False)), ('skip_logic', wagtail.core.blocks.ChoiceBlock(choices=[('next', 'Next default question'), ('end', 'End of survey'), ('question', 'Another question')], required=False)), ('question', questionnaires.blocks.QuestionSelectBlock(help_text='Please save the survey as a draft to populate or update the list of questions.', required=False))]))], blank=True, help_text='<strong>Checkbox must include only 2 Answer Options. true and false in that order.</strong>', null=True, verbose_name='Answer options'),
        ),
    ]


# questionnaires/migrations/0015_auto_20210827_1806.py

# Generated by Django 3.1.11 on 2021-08-27 18:06

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0014_auto_20210827_1033'),
    ]

    operations = [
        migrations.AlterField(
            model_name='quizformfield',
            name='choices',
            field=models.TextField(blank=True, help_text='For checkboxes, dropdown and radio: Pipe (|) separated list of choices.', verbose_name='choices'),
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='correct_answer',
            field=models.TextField(help_text='The correct answer/choice(s). If multiple choices are correct, separate choices with a pipe (|) symbol. For checkbox: Either "true" or "false".', verbose_name='correct_answer'),
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='field_type',
            field=models.CharField(choices=[('checkbox', 'Checkbox'), ('checkboxes', 'Checkboxes'), ('date', 'Date'), ('datetime', 'Date/time'), ('dropdown', 'Drop down'), ('email', 'Email'), ('singleline', 'Single line text'), ('multiline', 'Multi-line text'), ('number', 'Number'), ('positivenumber', 'Positive number'), ('radio', 'Radio buttons'), ('url', 'URL')], max_length=16, verbose_name='field type'),
        ),
        migrations.AlterField(
            model_name='surveyformfield',
            name='field_type',
            field=models.CharField(choices=[('checkbox', 'Checkbox'), ('checkboxes', 'Checkboxes'), ('date', 'Date'), ('datetime', 'Date/time'), ('dropdown', 'Drop down'), ('email', 'Email'), ('singleline', 'Single line text'), ('multiline', 'Multi-line text'), ('number', 'Number'), ('positivenumber', 'Positive number'), ('radio', 'Radio buttons'), ('url', 'URL')], max_length=16, verbose_name='field type'),
        ),
    ]


# questionnaires/migrations/0016_auto_20210828_1301.py

# Generated by Django 3.1.13 on 2021-08-28 13:01

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0015_auto_20210827_1806'),
    ]

    operations = [
        migrations.AlterField(
            model_name='pollformfield',
            name='default_value',
            field=models.TextField(blank=True, help_text='Default value. Pipe (|) separated values supported for checkboxes.', verbose_name='default value'),
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='default_value',
            field=models.TextField(blank=True, help_text='Default value. Pipe (|) separated values supported for checkboxes.', verbose_name='default value'),
        ),
        migrations.AlterField(
            model_name='surveyformfield',
            name='default_value',
            field=models.TextField(blank=True, help_text='Default value. Pipe (|) separated values supported for checkboxes.', verbose_name='default value'),
        ),
    ]


# questionnaires/migrations/0017_auto_20210901_1844.py

# Generated by Django 3.1.11 on 2021-09-01 18:44

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0016_auto_20210828_1301'),
    ]

    operations = [
        migrations.AlterField(
            model_name='pollformfield',
            name='choices',
            field=models.TextField(blank=True, help_text='For checkboxes, dropdown and radio: Pipe (|) separated list of choices.', null=True, verbose_name='choices'),
        ),
        migrations.AlterField(
            model_name='pollformfield',
            name='field_type',
            field=models.CharField(choices=[('checkboxes', 'Checkboxes'), ('dropdown', 'Dropdown'), ('multiline', 'Multi-line text'), ('radio', 'Radio buttons')], max_length=16, verbose_name='field type'),
        ),
    ]


# questionnaires/migrations/0018_auto_20210929_1126.py

# Generated by Django 3.1.13 on 2021-09-29 11:26

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0017_auto_20210901_1844'),
    ]

    operations = [
        migrations.AlterField(
            model_name='pollformfield',
            name='clean_name',
            field=models.TextField(blank=True, default='', help_text='Safe name of the form field, the label converted to ascii_snake_case', verbose_name='name'),
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='clean_name',
            field=models.TextField(blank=True, default='', help_text='Safe name of the form field, the label converted to ascii_snake_case', verbose_name='name'),
        ),
        migrations.AlterField(
            model_name='surveyformfield',
            name='clean_name',
            field=models.TextField(blank=True, default='', help_text='Safe name of the form field, the label converted to ascii_snake_case', verbose_name='name'),
        ),
    ]


# questionnaires/migrations/0019_auto_20211004_1113.py

# Generated by Django 3.1.13 on 2021-10-04 11:13

from django.db import migrations, models
import wagtail.core.blocks
import wagtail.core.fields


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0018_auto_20210929_1126'),
    ]

    operations = [
        migrations.AddField(
            model_name='poll',
            name='index_page_description',
            field=models.TextField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='poll',
            name='index_page_description_line_2',
            field=models.TextField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='poll',
            name='randomise_options',
            field=models.BooleanField(default=False, help_text='Randomising the options allows the options to be shown in a different order each time the page is displayed.'),
        ),
        migrations.AddField(
            model_name='poll',
            name='terms_and_conditions',
            field=wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock()), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AddField(
            model_name='pollformfield',
            name='admin_label',
            field=models.CharField(help_text='Column header used during CSV export of poll responses.', max_length=256, null=True, verbose_name='admin_label'),
        ),
        migrations.AddField(
            model_name='quiz',
            name='index_page_description',
            field=models.TextField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='quiz',
            name='index_page_description_line_2',
            field=models.TextField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='quiz',
            name='terms_and_conditions',
            field=wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock()), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AddField(
            model_name='survey',
            name='index_page_description',
            field=models.TextField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='survey',
            name='index_page_description_line_2',
            field=models.TextField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='survey',
            name='terms_and_conditions',
            field=wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock()), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
    ]


# questionnaires/migrations/0020_auto_20211013_0953.py

# Generated by Django 3.1.13 on 2021-10-13 09:53

from django.db import migrations
import wagtail.core.blocks
import wagtail.core.fields
import wagtail.images.blocks


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0019_auto_20211004_1113'),
    ]

    operations = [
        migrations.AlterField(
            model_name='poll',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('image', wagtail.images.blocks.ImageChooserBlock()), ('list', wagtail.core.blocks.ListBlock(wagtail.core.blocks.CharBlock(label='Item'))), ('numbered_list', wagtail.core.blocks.ListBlock(wagtail.core.blocks.CharBlock(label='Item'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='quiz',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('image', wagtail.images.blocks.ImageChooserBlock()), ('list', wagtail.core.blocks.ListBlock(wagtail.core.blocks.CharBlock(label='Item'))), ('numbered_list', wagtail.core.blocks.ListBlock(wagtail.core.blocks.CharBlock(label='Item'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='survey',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('image', wagtail.images.blocks.ImageChooserBlock()), ('list', wagtail.core.blocks.ListBlock(wagtail.core.blocks.CharBlock(label='Item'))), ('numbered_list', wagtail.core.blocks.ListBlock(wagtail.core.blocks.CharBlock(label='Item'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
    ]


# questionnaires/migrations/0021_auto_20211015_1301.py

# Generated by Django 3.1.13 on 2021-10-15 13:01

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailsvg', '0002_svg_edit_code'),
        ('questionnaires', '0020_auto_20211013_0953'),
    ]

    operations = [
        migrations.AddField(
            model_name='poll',
            name='icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailsvg.svg'),
        ),
        migrations.AddField(
            model_name='quiz',
            name='icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailsvg.svg'),
        ),
        migrations.AddField(
            model_name='survey',
            name='icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='wagtailsvg.svg'),
        ),
    ]


# questionnaires/migrations/0022_auto_20211029_1533.py

# Generated by Django 3.1.13 on 2021-10-29 15:33

from django.db import migrations
import home.blocks
import wagtail.core.blocks
import wagtail.core.fields
import wagtail.images.blocks
import wagtailmarkdown.blocks


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0021_auto_20211015_1301'),
    ]

    operations = [
        migrations.AlterField(
            model_name='poll',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('paragraph_v1_legacy', home.blocks.RawHTMLBlock(icon='code')), ('image', wagtail.images.blocks.ImageChooserBlock()), ('list', wagtailmarkdown.blocks.MarkdownBlock(icon='code')), ('numbered_list', home.blocks.NumberedListBlock(wagtailmarkdown.blocks.MarkdownBlock(icon='code'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='quiz',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('paragraph_v1_legacy', home.blocks.RawHTMLBlock(icon='code')), ('image', wagtail.images.blocks.ImageChooserBlock()), ('list', wagtailmarkdown.blocks.MarkdownBlock(icon='code')), ('numbered_list', home.blocks.NumberedListBlock(wagtailmarkdown.blocks.MarkdownBlock(icon='code'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='survey',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('paragraph_v1_legacy', home.blocks.RawHTMLBlock(icon='code')), ('image', wagtail.images.blocks.ImageChooserBlock()), ('list', wagtailmarkdown.blocks.MarkdownBlock(icon='code')), ('numbered_list', home.blocks.NumberedListBlock(wagtailmarkdown.blocks.MarkdownBlock(icon='code'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
    ]


# questionnaires/migrations/0023_poll_show_results_with_no_votes.py

# Generated by Django 3.1.13 on 2021-11-08 17:00

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0022_auto_20211029_1533'),
    ]

    operations = [
        migrations.AddField(
            model_name='poll',
            name='show_results_with_no_votes',
            field=models.BooleanField(default=True, help_text='Display options with 0 votes in results.'),
        ),
    ]


# questionnaires/migrations/0024_auto_20211221_0604.py

# Generated by Django 3.1.14 on 2021-12-21 06:04

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

    dependencies = [
        ('wagtailimages', '0022_uploadedimage'),
        ('questionnaires', '0023_poll_show_results_with_no_votes'),
    ]

    operations = [
        migrations.AddField(
            model_name='poll',
            name='image_icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='quiz',
            name='image_icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
        migrations.AddField(
            model_name='survey',
            name='image_icon',
            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='wagtailimages.image'),
        ),
    ]


# questionnaires/migrations/0025_auto_20220106_0908.py

# Generated by Django 3.1.14 on 2022-01-06 09:08

from django.db import migrations
import home.blocks
import wagtail.core.blocks
import wagtail.core.fields
import wagtail.images.blocks
import wagtailmarkdown.blocks


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0024_auto_20211221_0604'),
    ]

    operations = [
        migrations.AlterField(
            model_name='poll',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title', template='blocks/heading.html')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('paragraph_v1_legacy', home.blocks.RawHTMLBlock(icon='code')), ('image', wagtail.images.blocks.ImageChooserBlock()), ('list', wagtailmarkdown.blocks.MarkdownBlock(icon='code')), ('numbered_list', home.blocks.NumberedListBlock(wagtailmarkdown.blocks.MarkdownBlock(icon='code'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='quiz',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title', template='blocks/heading.html')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('paragraph_v1_legacy', home.blocks.RawHTMLBlock(icon='code')), ('image', wagtail.images.blocks.ImageChooserBlock()), ('list', wagtailmarkdown.blocks.MarkdownBlock(icon='code')), ('numbered_list', home.blocks.NumberedListBlock(wagtailmarkdown.blocks.MarkdownBlock(icon='code'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='survey',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title', template='blocks/heading.html')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('paragraph_v1_legacy', home.blocks.RawHTMLBlock(icon='code')), ('image', wagtail.images.blocks.ImageChooserBlock()), ('list', wagtailmarkdown.blocks.MarkdownBlock(icon='code')), ('numbered_list', home.blocks.NumberedListBlock(wagtailmarkdown.blocks.MarkdownBlock(icon='code'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
    ]


# questionnaires/migrations/0026_auto_20220125_1149.py

# Generated by Django 3.1.14 on 2022-01-25 11:49

from django.db import migrations
import home.blocks
import wagtail.core.blocks
import wagtail.core.fields
import wagtail.images.blocks
import wagtailmarkdown.blocks


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0025_auto_20220106_0908'),
    ]

    operations = [
        migrations.AlterField(
            model_name='poll',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title', template='blocks/heading.html')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('paragraph_v1_legacy', home.blocks.RawHTMLBlock(icon='code')), ('image', wagtail.images.blocks.ImageChooserBlock(template='blocks/image.html')), ('list', wagtailmarkdown.blocks.MarkdownBlock(icon='code')), ('numbered_list', home.blocks.NumberedListBlock(wagtailmarkdown.blocks.MarkdownBlock(icon='code'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='poll',
            name='terms_and_conditions',
            field=wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='poll',
            name='thank_you_text',
            field=wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('media', home.blocks.MediaBlock(icon='media')), ('image', wagtail.images.blocks.ImageChooserBlock(template='blocks/image.html'))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='quiz',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title', template='blocks/heading.html')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('paragraph_v1_legacy', home.blocks.RawHTMLBlock(icon='code')), ('image', wagtail.images.blocks.ImageChooserBlock(template='blocks/image.html')), ('list', wagtailmarkdown.blocks.MarkdownBlock(icon='code')), ('numbered_list', home.blocks.NumberedListBlock(wagtailmarkdown.blocks.MarkdownBlock(icon='code'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='quiz',
            name='terms_and_conditions',
            field=wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='quiz',
            name='thank_you_text',
            field=wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('media', home.blocks.MediaBlock(icon='media')), ('image', wagtail.images.blocks.ImageChooserBlock(template='blocks/image.html'))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='survey',
            name='description',
            field=wagtail.core.fields.StreamField([('heading', wagtail.core.blocks.CharBlock(form_classname='full title', template='blocks/heading.html')), ('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('paragraph_v1_legacy', home.blocks.RawHTMLBlock(icon='code')), ('image', wagtail.images.blocks.ImageChooserBlock(template='blocks/image.html')), ('list', wagtailmarkdown.blocks.MarkdownBlock(icon='code')), ('numbered_list', home.blocks.NumberedListBlock(wagtailmarkdown.blocks.MarkdownBlock(icon='code'))), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='survey',
            name='terms_and_conditions',
            field=wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('page_button', wagtail.core.blocks.StructBlock([('page', wagtail.core.blocks.PageChooserBlock()), ('text', wagtail.core.blocks.CharBlock(max_length=255, required=False))]))], blank=True, null=True),
        ),
        migrations.AlterField(
            model_name='survey',
            name='thank_you_text',
            field=wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock(features=['h2', 'h3', 'h4', 'bold', 'italic', 'ol', 'ul', 'hr', 'link', 'document-link', 'image'])), ('media', home.blocks.MediaBlock(icon='media')), ('image', wagtail.images.blocks.ImageChooserBlock(template='blocks/image.html'))], blank=True, null=True),
        ),
    ]


# questionnaires/migrations/0027_auto_20220128_1047.py

# Generated by Django 3.1.14 on 2022-01-28 10:47

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0026_auto_20220125_1149'),
    ]

    operations = [
        migrations.AlterField(
            model_name='pollformfield',
            name='label',
            field=models.TextField(help_text='The label of the form field', verbose_name='label'),
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='feedback',
            field=models.TextField(blank=True, help_text='Feedback message for user answer.', null=True, verbose_name='Feedback'),
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='label',
            field=models.TextField(help_text='The label of the form field', verbose_name='label'),
        ),
        migrations.AlterField(
            model_name='surveyformfield',
            name='label',
            field=models.TextField(help_text='The label of the form field', verbose_name='label'),
        ),
    ]


# questionnaires/migrations/0028_auto_20220405_0956.py

# Generated by Django 3.1.14 on 2022-04-05 09:56

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0027_auto_20220128_1047'),
    ]

    operations = [
        migrations.AlterField(
            model_name='surveyformfield',
            name='required',
            field=models.BooleanField(default=True, verbose_name='required'),
        ),
    ]


# questionnaires/migrations/0029_auto_20220512_1350.py

# Generated by Django 3.1.14 on 2022-05-12 13:50

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('questionnaires', '0028_auto_20220405_0956'),
    ]

    operations = [
        migrations.AlterField(
            model_name='pollformfield',
            name='clean_name',
            field=models.TextField(blank=True, default='', help_text='Safe name of the form field, the label converted to ascii_snake_case. This will appear in the HTML but will not be visible to the user', verbose_name='name'),
        ),
        migrations.AlterField(
            model_name='quizformfield',
            name='clean_name',
            field=models.TextField(blank=True, default='', help_text='Safe name of the form field, the label converted to ascii_snake_case. This will appear in the HTML but will not be visible to the user', verbose_name='name'),
        ),
        migrations.AlterField(
            model_name='surveyformfield',
            name='clean_name',
            field=models.TextField(blank=True, default='', help_text='Safe name of the form field, the label converted to ascii_snake_case. This will appear in the HTML but will not be visible to the user', verbose_name='name'),
        ),
    ]


# questionnaires/superset/__init__.py

ALLOWED_COLUMNS = ['id', 'form_data', 'submit_time', 'page_id', 'user_id']


# questionnaires/superset/charts.py

import json


class Chart:
    viz_type = None

    def __init__(self, dashboard_id, dataset_id, name, clean_name=None):
        self.dashboard_id = dashboard_id
        self.dataset_id = dataset_id
        self.name = name
        self.clean_name = clean_name

    def post_body(self):
        return {
            'dashboards': [self.dashboard_id],
            'datasource_id': self.dataset_id,
            'datasource_type': 'table',
            'slice_name': self.name,
            'viz_type': self.viz_type,
            'params': self.params(),
        }

    def params(self):
        return json.dumps({})


class PieChart(Chart):
    viz_type = 'pie'

    def params(self):
        return json.dumps({
            "groupby": [self.name],
            "metric": "response_count",
            "show_legend": False,
            "label_type": "key_percent",
        })


class BarChart(Chart):
    viz_type = 'dist_bar'

    def params(self):
        return json.dumps({
            "groupby": [self.name],
            "metrics": ["response_count"],
            "show_legend": False,
            "y_axis_label": "Responses",
        })


class TableChart(Chart):
    viz_type = 'table'

    def params(self):
        return json.dumps({
            "adhoc_filters": [
                {
                    "clause": "WHERE",
                    "expressionType": "SQL",
                    "sqlExpression": f"((form_data::json)->'{self.clean_name}')::text <> '\"\"'",
                }
            ],
            "groupby": [self.name],
            "metrics": ["response_count"],
        })


class BigNumberTotalChart(Chart):
    viz_type = 'big_number_total'

    def params(self):
        return json.dumps({
            "metric": "response_count",
            "subheader": "Responses"
        })


class BigNumberTotalOpenEndedQuestionChart(BigNumberTotalChart):
    viz_type = 'big_number_total'

    def params(self):
        return json.dumps({
            "adhoc_filters": [
                {
                    "clause": "WHERE",
                    "expressionType": "SQL",
                    "sqlExpression": f"((form_data::json)->'{self.clean_name}')::text <> '\"\"'",
                }
            ],
            "metric": "response_count",
            "subheader": "Responses"
        })


class BigNumberTotalMeanChart(BigNumberTotalChart):
    def params(self):
        return json.dumps({
            "metric": {
                "expressionType": "SQL",
                "sqlExpression": f"AVG((form_data::json->>'{self.clean_name}')::DECIMAL)",
            },
            "subheader": "Mean of responses",
        })


# questionnaires/superset/client.py

from django.conf import settings
from requests import Session, Request
from rest_framework import status


class SupersetClient:
    api_url = f'{settings.SUPERSET_BASE_URL}/api/v1'
    security_url = f'{api_url}/security'
    login_url = f'{security_url}/login'
    refresh_url = f'{security_url}/refresh'
    csrf_token_url = f'{security_url}/csrf_token/'
    database_url = f'{api_url}/database/'
    dashboard_url = f'{api_url}/dashboard/'
    dataset_url = f'{api_url}/dataset/'
    chart_url = f'{api_url}/chart/'

    def __init__(self):
        self.session = Session()

    def _get_csrf_token(self):
        return {
            'X_CSRFToken': self.csrf_token,
        }

    def _get_auth_headers(self):
        return {
            'Authorization': f"Bearer {self.access_token}",
        }

    def _get_headers(self):
        headers = self._get_auth_headers()
        headers.update(self._get_csrf_token())
        return headers

    def _get_params(self):
        return {
            'override_columns': 'true'
        }

    def _get_validated_response(self, response):
        if response.ok:
            return response.json()
        elif response.status_code == status.HTTP_401_UNAUTHORIZED:
            raise Exception('Unauthorized: Invalid username or password.')
        else:
            raise Exception(f'Something went wrong. Status: {response.status_code}')

    def _api_caller(self, request):
        request = self.session.prepare_request(request)
        response = self.session.send(request)
        return self._get_validated_response(response)

    def authenticate(self, username, password):
        data = {
            'username': username,
            'password': password,
            'refresh': True,
            'provider': 'db',
        }
        request = Request(method='POST', url=self.login_url, json=data)
        response = self._api_caller(request)
        self.access_token = response.get('access_token')
        self.refresh_token = response.get('refresh_token')

        request = Request(method='GET', url=self.csrf_token_url, headers=self._get_auth_headers())
        response = self._api_caller(request)
        self.csrf_token = response.get('result')

    def get_databases(self):
        request = Request(method='GET', url=self.database_url, headers=self._get_headers())
        return self._api_caller(request)

    def create_dashboard(self, data):
        request = Request(method='POST', url=self.dashboard_url, headers=self._get_headers(), json=data)
        return self._api_caller(request)

    def get_dashboard(self, id):
        request = Request(method='GET', url=f'{self.dashboard_url}{id}', headers=self._get_auth_headers())
        return self._api_caller(request)

    def create_dataset(self, data):
        request = Request(method='POST', url=self.dataset_url, headers=self._get_headers(), json=data)
        return self._api_caller(request)

    def get_dataset(self, id):
        request = Request(method='GET', url=f'{self.dataset_url}{id}', headers=self._get_headers())
        return self._api_caller(request)

    def update_dataset(self, id, data):
        request = Request(method='PUT', url=f'{self.dataset_url}{id}', params=self._get_params(), headers=self._get_headers(), json=data)
        return self._api_caller(request)

    def create_chart(self, data):
        request = Request(method='POST', url=f'{self.chart_url}', headers=self._get_headers(), json=data)
        return self._api_caller(request)


# questionnaires/superset/dashboard.py

class Dashboard:
    def __init__(self, dashboard_title):
        self.dashboard_title = dashboard_title

    def post_body(self):
        return {
            'dashboard_title': self.dashboard_title,
            'published': True,
        }


# questionnaires/superset/datasets.py

from questionnaires.superset import ALLOWED_COLUMNS


class Dataset:
    def __init__(self, database_id, owner_id, table_name, dataset_name, page_id):
        self.database_id = database_id
        self.owner_id = owner_id
        self.table_name = table_name
        self.dataset_name = dataset_name
        self.page_id = page_id

    def post_body(self):
        return {
            'database': self.database_id,
            'table_name': self.table_name,
            'owners': [
                self.owner_id,
            ],
        }

    def put_body(self, columns, metrics):
        sql = f"SELECT {', '.join(ALLOWED_COLUMNS)} " \
              f"FROM {self.table_name} " \
              f"WHERE page_id = {self.page_id}"

        return {
            'sql': sql,
            'table_name': self.dataset_name,
            'columns': columns,
            'metrics': metrics,
            'owners': [
                self.owner_id,
            ],
        }


# questionnaires/superset/utils.py

from copy import copy

from django.conf import settings
from django.utils import timezone

from home.models import SiteSettings
from questionnaires.models import UserSubmission
from questionnaires.superset.charts import (
    PieChart,
    BarChart,
    TableChart,
    BigNumberTotalChart,
    BigNumberTotalMeanChart,
    BigNumberTotalOpenEndedQuestionChart,
)
from questionnaires.superset.client import SupersetClient
from questionnaires.superset.dashboard import Dashboard
from questionnaires.superset.datasets import Dataset
from questionnaires.superset import ALLOWED_COLUMNS

CHART_TYPE_MAP = {
    'checkbox': PieChart,
    'checkboxes': BarChart,
    'dropdown': BarChart,
    'email': TableChart,
    'singleline': BigNumberTotalOpenEndedQuestionChart,
    'multiline': BigNumberTotalOpenEndedQuestionChart,
    'number': BigNumberTotalMeanChart,
    'positivenumber': BigNumberTotalMeanChart,
    'radio': BarChart,
    'url': TableChart,
}

CALCULATED_COLUMN_EXPRESSION_MAP = {
    'checkbox': "((form_data::json)->'{}')::text",
    'checkboxes': "trim(both '\"' from json_array_elements((form_data::json)->'{}')::TEXT)",
    'dropdown': "trim(both '\"' from ((form_data::json)->'{}')::text)",
    'email': "trim(both '\"' from ((form_data::json)->'{}')::text)",
    'singleline': "((form_data::json)->'{}')::text",
    'multiline': "((form_data::json)->'{}')::text",
    'number': "(form_data::json->>'{}')::DECIMAL",
    'positivenumber': "(form_data::json->>'{}')::DECIMAL",
    'radio': "trim(both '\"' from ((form_data::json)->'{}')::text)",
    'url': "trim(both '\"' from ((form_data::json)->'{}')::text)",
}


class DashboardGenerator:
    def __init__(self, user, questionnaire, superset_username, superset_password):
        self.user = user
        self.questionnaire = questionnaire
        self.questions = questionnaire.get_form_fields().order_by('sort_order')
        self.client = SupersetClient()
        self.client.authenticate(superset_username, superset_password)
        self.admin_client = SupersetClient()
        self.admin_client.authenticate(settings.SUPERSET_USERNAME, settings.SUPERSET_PASSWORD)
        self.table_name = UserSubmission._meta.db_table
        self.current_datetime = timezone.now().strftime("%Y-%m-%d %H:%M:%S")

    def _get_database_id(self):
        resp = self.client.get_databases()
        for database in resp.get('result'):
            if database.get('database_name') == settings.SUPERSET_DATABASE_NAME:
                return database.get('id')

        raise Exception('No database found.')

    def _create_dashboard(self):
        dashboard = Dashboard(dashboard_title=f'{self.questionnaire.title} - {self.current_datetime}')
        resp = self.client.create_dashboard(data=dashboard.post_body())
        dashboard_id = resp.get('id')
        resp = self.client.get_dashboard(dashboard_id)
        result = resp.get('result', {})
        dashboard_url = result.get('url')
        owner_id = result.get('owners', [{}])[0].get('id')
        return dashboard_id, f'{settings.SUPERSET_BASE_URL}{dashboard_url}', owner_id

    def _create_dataset(self, database_id, owner_id):
        dataset_name = f'{SiteSettings.get_for_default_site()}_autodashboard_' \
                       f'{self.questionnaire.__class__.__name__.lower()}_{self.questionnaire.id}_' \
                       f'{self.questionnaire.title}_{self.current_datetime}'
        dataset = Dataset(
            database_id=database_id, owner_id=owner_id, table_name=self.table_name, dataset_name=dataset_name,
            page_id=self.questionnaire.id)
        resp = self.admin_client.create_dataset(data=dataset.post_body())
        dataset_id = resp.get('id')

        dataset_detail = self.client.get_dataset(dataset_id)

        columns = [
            {'column_name': column.get('column_name')}
            for column in dataset_detail.get('result', {}).get('columns')
            if column.get('column_name') in ALLOWED_COLUMNS
        ]

        for question in self.questions:
            calculated_column_expression = CALCULATED_COLUMN_EXPRESSION_MAP.get(question.field_type)
            if calculated_column_expression:
                columns.append({
                    "column_name": question.label,
                    "expression": calculated_column_expression.format(question.clean_name),
                })

        metrics = copy(dataset_detail.get('result', {}).get('metrics'))
        for metric in metrics:
            metric.pop('changed_on', None)
            metric.pop('created_on', None)
            metric.pop('uuid', None)

        metrics.append({
            "expression": "COUNT(*)",
            "metric_name": "response_count",
            "metric_type": "count",
            "verbose_name": "Responses",
        })
        self.admin_client.update_dataset(id=dataset_id, data=dataset.put_body(columns, metrics))

        return dataset_id

    def _create_charts(self, dashboard_id, dataset_id):
        chart = BigNumberTotalChart(dashboard_id=dashboard_id, dataset_id=dataset_id, name='Total Submissions')
        self.client.create_chart(data=chart.post_body())

        for question in self.questions:
            chart_class = CHART_TYPE_MAP.get(question.field_type)
            if chart_class:
                chart = chart_class(
                    dashboard_id=dashboard_id, dataset_id=dataset_id, name=question.label,
                    clean_name=question.clean_name)
                self.client.create_chart(data=chart.post_body())

    def generate(self):
        database_id = self._get_database_id()
        dashboard_id, dashboard_url, owner_id = self._create_dashboard()
        dataset_id = self._create_dataset(database_id, owner_id)
        self._create_charts(dashboard_id, dataset_id)

        return dashboard_url


# questionnaires/templatetags/questionnaires_tags.py

from django import template
from django.db.models import Q

from questionnaires.models import Poll
from questionnaires.utils import SkipLogicPaginator
import iogt.iogt_globals as globals_

register = template.Library()


@register.inclusion_tag('questionnaires/tags/checkbox.html')
def render_checkbox(field):
    return {'field': field}


@register.inclusion_tag('questionnaires/tags/checkboxes.html')
def render_checkboxes(field):
    return {'field': field}


@register.inclusion_tag("questionnaires/tags/select.html")
def render_select(field, is_multiselect=False):
    return {"field": field, "is_multiselect": is_multiselect}


@register.inclusion_tag('questionnaires/tags/textarea.html')
def render_textarea(field):
    return {'field': field}


@register.inclusion_tag('questionnaires/tags/radios.html')
def render_radios(field):
    return {'field': field}


@register.inclusion_tag('questionnaires/tags/text_field.html')
def render_text_field(field):
    return {'field': field}


@register.inclusion_tag('questionnaires/tags/field_description.html')
def field_description(field):
    return {"field": field}


@register.inclusion_tag('questionnaires/tags/render_fields.html')
def render_field(field):
    return {'field': field}


@register.inclusion_tag('questionnaires/tags/field_counter.html')
def field_counter(form, forloop, form_length, fields_step, questionnaire):
    if form_length != None:
        if form.errors:
            counter = forloop.get("counter")
        else:
            counter = forloop.get("counter") + int(form_length or 0)
    else:
        counter = forloop.get("counter")

    if hasattr(questionnaire, "multi_step") and (questionnaire.multi_step or questionnaire.has_page_breaks):
        total = fields_step.paginator.count
    else:
        total = len(form.fields)

    return {"counter": counter, "total": total}


@register.inclusion_tag('questionnaires/tags/submit_button.html')
def render_submit_button(page, fields_step=None):
    return {'submit_button_text': page.get_submit_button_text(fields_step)}


@register.inclusion_tag('questionnaires/tags/action_url.html')
def get_action_url(page, self, fields_step, request, form):
    return {"page": page, "self": self, "fields_step": fields_step,
            "request": request, "form": form}


@register.inclusion_tag('questionnaires/tags/questionnaire_template_wrapper.html', takes_context=True)
def render_questionnaire_form(context, page, background_color=None, font_color=None):
    theme_settings = globals_.theme_settings

    font_color = font_color or theme_settings.section_listing_questionnaire_font_color
    background_color = background_color or theme_settings.section_listing_questionnaire_background_color

    request = context['request']

    form_class = page.get_form_class()

    if isinstance(page, Poll):
        template = 'questionnaires/tags/embedded_poll.html'
        context.update({
            'results': page.get_results(),
            'result_as_percentage': page.result_as_percentage,
        })
    else:
        template = 'questionnaires/tags/embedded_questionnaire.html'
        if hasattr(page, 'multi_step') and page.multi_step and page.get_form_fields():
            paginator = SkipLogicPaginator(page.get_form_fields(), {}, {})
            step = paginator.page(1)
            form_class = page.get_form_class_for_step(step)
            context.update({
                'fields_step': step,
            })

    context.update({
        'template': template,
        'font_color': font_color,
        'background_color': background_color,
        'questionnaire': page,
    })

    multiple_submission_filter = (
        Q(session_key=request.session.session_key) if request.user.is_anonymous else Q(user__pk=request.user.pk)
    )
    multiple_submission_check = (
            not page.allow_multiple_submissions
            and page.get_submission_class().objects.filter(multiple_submission_filter,
                                                           page=page).exists()
    )
    anonymous_user_submission_check = request.user.is_anonymous and not page.allow_anonymous_submissions
    if multiple_submission_check or anonymous_user_submission_check:
        context.update({
            'form': None,
        })
        return context

    form = form_class(page=page, user=context['request'].user)

    context.update({
        'form': form,
    })

    return context


@register.filter
def get_item(dictionary, key):
    return dictionary.get(key)


@register.simple_tag
def subtract(value, arg):
    return int(value) - int(arg)


@register.inclusion_tag('questionnaires/tags/questionnaire_wrapper.html', takes_context=True)
def render_questionnaire_wrapper(context, page, direct_display=False, background_color=None, font_color=None):
    context.update({
        'questionnaire': page,
        'direct_display': direct_display,
        'background_color': background_color,
        'font_color': font_color,
    })
    return context


@register.simple_tag
def get_answer_options(field, field_option, fields_info):
    label = field_option.choice_label
    correct_answers = fields_info.get(field.name, {}).get('correct_answer_list', [])
    is_selected = field_option.data.get('selected', False)
    rv = ''
    if is_selected and label in correct_answers:
        rv = {
            'class': 'success',
            'aria_label': 'Checkbox with tick, indicating correct and selected',
        }
    elif is_selected and label not in correct_answers:
        rv = {
            'class': 'error',
            'aria_label': 'Checkbox with X, indicating incorrect and selected',
        }
    elif not is_selected and label in correct_answers:
        rv = {
            'class': 'clear-tick',
            'aria_label': 'Checkbox with tick, indicating correct but not selected',
        }
    elif not is_selected and label not in correct_answers:
        rv = {
            'class': 'clear-cross',
            'aria_label': 'Checkbox with X, indicating incorrect and not selected',
        }

    return rv


# selenium_tests/admin/test_skip_logic.py

import json
from django.urls import reverse

from iogt_users.factories import AdminUserFactory
from questionnaires.factories import SurveyFactory, SurveyFormFieldFactory
from selenium_tests.base import BaseSeleniumTests
from selenium_tests.pages import WagtailAdminPage


class AdminPanelSeleniumTests(BaseSeleniumTests):
    def setUp(self):
        self.setup_blank_site()
        admin_user = AdminUserFactory()
        self.visit_login_page().login_user(admin_user)
        self.survey01 = SurveyFactory(
            parent=self.home,
            owner=admin_user,
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]"
            )

    def test_skip_logic_validation_errors_on_admin_panel(self):
        SurveyFormFieldFactory(label='Q1', page=self.survey01, required=True, field_type='checkbox')
        SurveyFormFieldFactory(label='Q2', page=self.survey01, required=True, field_type='checkbox')
        SurveyFormFieldFactory(label='Q3', page=self.survey01, required=True, field_type='checkbox')

        self.visit_url(reverse('wagtailadmin_pages:edit', args=(self.survey01.id,)))

        admin_page = WagtailAdminPage(self.selenium)
        admin_page.skip_to_question(question=0, skip_logic=0, skip_to=3)
        admin_page.submit_response()

        self.assertIn(
            "Skip to question Q3 with in-between required questions isn't allowed.",
            self.selenium.page_source
        )

    def test_skip_logic_requires_multi_step_enabled_on_admin_panel(self):
        skip_logic = json.dumps(
            [
                {
                    "type": "skip_logic",
                    "value": {
                        "choice": "true",
                        "skip_logic": "end",
                        "question": None
                    }
                },
                {
                    "type": "skip_logic",
                    "value": {
                        "choice": "false",
                        "skip_logic": "next",
                        "question": None
                    }
                }
            ]
        )
        SurveyFormFieldFactory(label='Q1', page=self.survey01, required=True, field_type='checkbox',
                               skip_logic=skip_logic)
        SurveyFormFieldFactory(label='Q2', page=self.survey01, required=True, field_type='checkbox',
                               skip_logic=skip_logic)

        self.visit_url(reverse('wagtailadmin_pages:edit', args=(self.survey01.id,)))
        admin_page = WagtailAdminPage(self.selenium)
        admin_page.submit_response()

        self.assertIn(
            "Multi-step is required to properly display questionnaires that include Skip Logic.",
            self.selenium.page_source
        )


# selenium_tests/articles/test_comments.py

from unittest import skip

from comments.models import CommentStatus
from home.factories import ArticleFactory, SectionFactory
from iogt_users.factories import AdminUserFactory, UserFactory
from selenium_tests.base import BaseSeleniumTests
from selenium_tests.pages import ArticlePage


@skip("Unstable in CI environment")
class ArticleCommentsSeleniumTests(BaseSeleniumTests):
    def setUp(self):
        self.setup_blank_site()
        self.user = AdminUserFactory()
        self.user2 = UserFactory()
        self.user3 = UserFactory()
        self.section = SectionFactory(parent=self.site.root_page)
        self.article01 = ArticleFactory(parent=self.section, title="article01")
        self.article02 = ArticleFactory(
            parent=self.section,
            title="article02",
            commenting_status=CommentStatus.CLOSED,
        )

    def test_basic_article_comment(self):
        # login as an admin so we can leave a comment
        login_page = self.visit_login_page()
        login_page.login_user(self.user)

        # visit an article and leave a comment
        test_comment = "test comment here"
        self.visit_page(self.article01)
        article_page = ArticlePage(self.selenium)
        article_page.comments_section.submit_comment(test_comment)
        self.visit_page(self.article01)
        self.assertIn(test_comment, article_page.comments_section.retrieve_comments())

    def test_remove_article_comment(self):
        # login as an admin
        login_page = self.visit_login_page()
        login_page.login_user(self.user)

        # visit an article and leave a comment
        test_comment = "test comment here"
        self.visit_page(self.article01)
        article_page = ArticlePage(self.selenium)
        article_page.comments_section.submit_comment(test_comment)
        self.visit_page(self.article01)

        # delete comment
        article_page.comments_section.delete_last_comment()
        self.assertIn(
            "The comment has been removed successfully!",
            article_page.get_messages_text(),
        )

    def test_user_flagging_comments(self):
        # login as a normal user
        login_page = self.visit_login_page()
        login_page.login_user(self.user2)
        # leave a comment
        test_comment = "test comment here"
        self.visit_page(self.article01)
        article_page = ArticlePage(self.selenium)
        article_page.comments_section.submit_comment(test_comment)
        # logout
        logout_page = self.visit_logout_page()
        logout_page.logout_user()
        # log in as a different user
        login_page = self.visit_login_page()
        login_page.login_user(self.user3)
        # Go to article and report comment
        self.visit_page(self.article01)
        article_page = ArticlePage(self.selenium)
        article_page.comments_section.report_last_comment()
        self.visit_page(self.article01)
        self.assertIn(
            "This comment has been reported.", article_page.comments_section.html.text
        )

    def test_moderator_removing_others_comments(self):
        # login as a normal user
        login_page = self.visit_login_page()
        login_page.login_user(self.user2)
        # leave a comment
        test_comment = "test comment here"
        self.visit_page(self.article01)
        article_page = ArticlePage(self.selenium)
        article_page.comments_section.submit_comment(test_comment)
        # logout
        logout_page = self.visit_logout_page()
        logout_page.logout_user()
        # log in as an admin user
        login_page = self.visit_login_page()
        login_page.login_user(self.user)
        # Go to article and remove comment
        self.visit_page(self.article01)
        article_page = ArticlePage(self.selenium)
        article_page.comments_section.delete_last_comment()
        self.assertIn(
            "The comment has been removed successfully!",
            article_page.get_messages_text(),
        )

    def test_commenting_restrictions(self):
        # login as a normal user
        login_page = self.visit_login_page()
        login_page.login_user(self.user)
        self.visit_page(self.article02)
        article_page = ArticlePage(self.selenium)
        self.assertIn(
            "New comments have been disabled for this page",
            article_page.comments_section.html.text,
        )

    def test_reply_article_comment(self):
        # login as an admin so we can leave a comment
        login_page = self.visit_login_page()
        login_page.login_user(self.user)

        # visit an article and leave a comment
        test_comment = "test comment here"
        self.visit_page(self.article01)
        article_page = ArticlePage(self.selenium)
        article_page.comments_section.submit_comment(test_comment)

        # leave a reply to the first comment
        test_reply = "test reply here"
        self.visit_page(self.article01)
        article_page.comments_section.reply_last_comment(test_reply)
        self.visit_page(self.article01)
        self.assertIn(test_reply, article_page.comments_section.retrieve_comments())


# selenium_tests/articles/test_components.py

from selenium_tests.base import BaseSeleniumTests
from home.factories import (
    SectionFactory,
    ArticleFactory
)
from selenium_tests.pages import ArticlePage

class ArticleNavigationSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        self.setup_blank_site()
        self.section = SectionFactory(parent=self.site.root_page)
        self.article01 = ArticleFactory(parent=self.section, title = 'article01')

    def test_title(self):

        self.visit_page(self.article01)
        article_page = ArticlePage(self.selenium)
        self.assertIn(self.article01.title, article_page.title)
        
    def test_image(self):

        self.visit_page(self.article01)
        article_page = ArticlePage(self.selenium)
        self.assertTrue(article_page.has_lead_image(), 'Image should be present and visible')
        
    
        
        


# selenium_tests/articles/test_embedded_elements.py

from selenium_tests.base import BaseSeleniumTests
from wagtail.core.models import Site
from iogt_users.factories import AdminUserFactory
from home.factories import SectionFactory, ArticleFactory
from questionnaires.factories import SurveyFactory, SurveyFormFieldFactory
from selenium_tests.pages import QuestionnairePage, QuestionnaireResultsPage

class ArticleEmbeddedSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        Site.objects.all().delete()
        self.setup_blank_site()
        self.user = AdminUserFactory()
        self.article01 = ArticleFactory(parent=self.home, owner=self.user)
        self.section01 = SectionFactory(parent=self.home, owner=self.user)

    def test_simple_embedded(self):
        survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user, 
            direct_display = True,
            allow_multiple_submissions = False,
            description="[{\"type\": \"paragraph\", \"value\": \"<p>This is a simple survey description</p>\"}]",
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]" 
            )
        SurveyFormFieldFactory(
            page=survey01, 
            required = True,
            default_value='',  
            field_type='singleline', 
        )
        self.article01.body.append((
            'embedded_survey', {
                'direct_display': survey01.specific.direct_display,
                'survey': survey01,
            }
        ))
        self.visit_page(self.section01)
        #We are looking at an article here, but use the Questionnaire page as we want to test the functionality of the embedded questionnaire
        survey_page = QuestionnairePage(self.selenium)
        self.assertIn("This is a simple survey description", survey_page.get_content_text())
        survey_page.enter_text("Testing some singleline text")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())
        revisit_page = self.visit_page(self.section01)
        self.assertIn("You have already completed this survey.", revisit_page.get_content_text())

    def test_multiple_page_embedded(self):
        survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user,
            direct_display = True,
            multi_step = True,
            description="[{\"type\": \"paragraph\", \"value\": \"<p>This is a simple survey description</p>\"}]",
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]"
            )
        SurveyFormFieldFactory(
            page=survey01, 
            required = False,  
            default_value='',  
            field_type='singleline',
            page_break = True 
        )
        SurveyFormFieldFactory(
            page=survey01, 
            required = False,
            choices = "A|B|C",  
            default_value='',  
            field_type='checkboxes', 
        )
        self.article01.body.append((
            'embedded_survey', {
                'direct_display': survey01.specific.direct_display,
                'survey': survey01,
            }
        ))

        self.visit_page(self.section01)
        #We are looking at an article here, but use the Questionnaire page as we want to test the functionality of the embedded questionnaire
        survey_page = QuestionnairePage(self.selenium)
        self.assertIn("This is a simple survey description", survey_page.get_content_text())
        survey_page.enter_text("Testing some singleline text")
        survey_page.submit_response()
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())
        
        
        
        

        
        


# selenium_tests/articles/test_navigation.py

from selenium_tests.base import BaseSeleniumTests
from home.factories import (
    SectionFactory,
    ArticleFactory
)

from selenium_tests.pages import ArticlePage

class ArticleNavigationSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        self.setup_blank_site()
        self.section = SectionFactory(parent=self.site.root_page)
        self.article01 = ArticleFactory(parent=self.section, title = 'article01')
        self.article02 = ArticleFactory(parent=self.section, title = 'article02')

    def test_navigation_next(self):
        self.visit_page(self.article01)
        article_page = ArticlePage(self.selenium)
        article_page.navigate_next()
        article_page = ArticlePage(self.selenium)
        self.assertIn(self.article02.title, article_page.title)
        

    def test_navigation_previous(self):

        self.visit_page(self.article02)
        article_page = ArticlePage(self.selenium)
        article_page.navigate_previous()
        article_page = ArticlePage(self.selenium)
        self.assertIn(self.article01.title, article_page.title)
    
        
        


# selenium_tests/homepage/test_banners.py

import time

import home.models as models
from selenium_tests.base import BaseSeleniumTests
from home.factories import (
    BannerIndexPageFactory,
    BannerFactory,
    ArticleFactory
)
from selenium_tests.pages import (
    ArticlePage,
    HomePage)

class BannerSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        self.setup_blank_site()
        self.banner_index = BannerIndexPageFactory(parent=self.site.root_page)
        self.article01 = ArticleFactory(parent = self.site.root_page, title = 'article01')
        self.banner01 = BannerFactory(parent = self.banner_index, title = 'banner01', banner_link_page = self.article01)
        models.HomePageBanner.objects.create(source = self.home, banner_page = self.banner01)


    def test_banner_is_present(self):
        
        self.visit_page(self.home)
        home_page = HomePage(self.selenium)
        self.assertTrue(home_page.has_banner, 'banner should be visible')


    def test_banner_navigation(self):
        
        self.visit_page(self.home)
        home_page = HomePage(self.selenium)  
        home_page.click_banner()
        banner_article = ArticlePage(self.selenium)
        self.assertTrue(
            banner_article.url_matches(self.article01.url),
            'Clicking on banner navigates to correct article'
        )

        

        



# selenium_tests/homepage/test_footer.py

from selenium_tests.base import BaseSeleniumTests
from home.factories import (
    FooterIndexPageFactory,
    SectionFactory,
    SectionIndexFactory,
)

class FooterSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        self.setup_blank_site()
        self.footer_index = FooterIndexPageFactory(parent=self.site.root_page)
        self.sections_index = SectionIndexFactory(parent=self.site.root_page)

    def test_footer_is_present(self):
        section = SectionFactory(
            parent=self.footer_index,
            title='Footer Section'
        )

        home_page = self.visit_page(self.home)
        self.assertTrue(home_page.footer.is_displayed, 'Footer must be visible')
        section_item = home_page.footer.items[0]
        self.assertTrue(section_item.has_icon, 'Icon must be visible')
        self.assertEqual(
            section_item.title,
            section.title,
            'Section title is the link text'
        )
        section_page = section_item.click()
        self.assertTrue(
            section_page.url_matches(section.url),
            'Clicking on footer link navigates to section page'
        )

    def test_navbar_is_present(self):
        section = SectionFactory(
            parent=self.sections_index,
            background_color='#0A0',
            font_color='#DDD',
        )

        home_page = self.visit_page(self.home)
        self.assertTrue(home_page.navbar.is_displayed, 'Navbar must be visible')
        item = home_page.navbar.items[0]
        self.assertTrue(item.has_icon, 'Icon must be visible')
        self.assertEqual(
            item.title,
            section.title,
            'Section title is the link text'
        )
        self.assertEqual(
            item.background_color,
            'rgba(0, 170, 0, 1)',
            'Navbar item background color must match section background color'
        )
        self.assertEqual(
            item.font_color,
            'rgba(221, 221, 221, 1)',
            'Navbar item font color must match section font color'
        )
        section_page = item.click()
        self.assertTrue(
            section_page.url_matches(section.url),
            'Clicking on footer link navigates to section page'
        )


# selenium_tests/homepage/test_layout.py

from selenium_tests.base import BaseSeleniumTests
from home.factories import (
    FooterIndexPageFactory,
    SectionFactory,
    SectionIndexFactory,
)

class LayoutSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        self.setup_blank_site()
        self.footer_index = FooterIndexPageFactory(parent=self.site.root_page)
        self.sections_index = SectionIndexFactory(parent=self.site.root_page)
        SectionFactory(
            parent=self.footer_index,
            title='Footers'
        )
        SectionFactory(
            parent=self.sections_index,
            title = 'Sections'
        )                

    def test_page_layout_compact(self):

        home_page = self.visit_page(self.home)
        self.assertTrue(home_page.footer.is_displayed, 'Footer must be visible')
        self.assertTrue(home_page.navbar.is_displayed, 'Navbar must be visible')

        self.assertTrue(home_page.footer_below_navbar_below_content(), 'footer_below_navbar_below_content when screen is compressed')

    def test_page_layout_large(self):
        
        self.selenium.set_window_size(1500, 1000)

        home_page = self.visit_page(self.home)

        self.assertTrue(home_page.footer_rightof_content_rightof_navbar(), 'footer_rightof_content_rightof_navbar when screen is full size')

    
        

        


# selenium_tests/homepage/test_login.py

from selenium_tests.base import BaseSeleniumTests
from iogt_users.factories import AdminUserFactory
from selenium_tests.pages import LoginPage
from selenium.webdriver.common.by import By

class LoginSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        
        self.setup_blank_site()
        self.user = AdminUserFactory()
        
    def test_login(self):

        login_page = self.visit_login_page()        
        login_page.login_user(self.user)
    
        self.assertIn(self.user.username, login_page.get_content_text())
        
        


# selenium_tests/homepage/test_search.py

import time

from selenium_tests.base import BaseSeleniumTests
from home.factories import (
    SectionIndexFactory,
    SectionFactory,
    ArticleFactory
)

from selenium_tests.pages import SearchPage

class SearchSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        self.setup_blank_site()
        self.sections_index = SectionIndexFactory(parent=self.site.root_page)
        self.section01 = SectionFactory(parent = self.sections_index)
        self.article01 = ArticleFactory(parent=self.section01, title = 'Test article for search - asdf;lkj')

    #Test temporarily turned off as search is not showing results, error in test environment not in site itself
    def skip_test_search_bar(self):
        home_page = self.visit_page(self.home)
        home_page.small_search_button_select()
        search_page = SearchPage(self.selenium)
        search_page.search('asdf;lkj')

        


# selenium_tests/polls_specific/test_poll_functionality.py

import time

from selenium_tests.base import BaseSeleniumTests
from selenium.webdriver.support.ui import Select
from wagtail.core.models import Site
from iogt_users.factories import AdminUserFactory
from home.factories import SectionFactory
from questionnaires.factories import PollFactory, PollFormFieldFactory
from selenium_tests.pages import QuestionnairePage, QuestionnaireResultsPage

class PollSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        Site.objects.all().delete()
        self.setup_blank_site()
        self.user = AdminUserFactory()
        self.section01 = SectionFactory(parent=self.home, owner=self.user)
        

    def test_result_totals(self):
        poll01 = PollFactory(
            parent=self.section01, 
            owner=self.user,
            result_as_percentage = False
            )
        PollFormFieldFactory(
            page=poll01, 
            required = True,
            choices = "A|B|C",  
            default_value='',  
            field_type='checkboxes', 
        )

        self.visit_page(poll01)
        poll_page = QuestionnairePage(self.selenium)
        poll_page.select_checkboxes("A")
        poll_page.submit_response()
        self.visit_page(poll01)
        poll_page.select_checkboxes("A")
        poll_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        poll_result = results_page.get_poll_results_text()
        self.assertIn("2", poll_result)

    def test_result_percentage(self):
        poll01 = PollFactory(
            parent=self.section01, 
            owner=self.user,
            result_as_percentage = True
            )
        PollFormFieldFactory(
            page=poll01, 
            required = True,
            choices = "A|B|C",  
            default_value='',  
            field_type='checkboxes', 
        )

        self.visit_page(poll01)
        poll_page = QuestionnairePage(self.selenium)
        poll_page.select_checkboxes("A")
        poll_page.submit_response()
        self.visit_page(poll01)
        poll_page.select_checkboxes("A")
        poll_page.submit_response()
        self.visit_page(poll01)
        poll_page.select_checkboxes("B")
        poll_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        poll_result = results_page.get_poll_results_text()
        self.assertIn("67 %", poll_result)
        self.assertIn("33 %", poll_result)

    def test_multi_select_option_results_for_checkboxes(self):
        poll01 = PollFactory(
            parent=self.section01,
            owner=self.user,
            result_as_percentage=True
            )
        PollFormFieldFactory(
            page=poll01,
            required=True,
            choices="A|B|C",
            default_value='',
            field_type='checkboxes',
        )

        self.visit_page(poll01)
        poll_page = QuestionnairePage(self.selenium)
        poll_page.select_checkboxes("A")
        poll_page.select_checkboxes("B")
        poll_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        poll_result = results_page.get_poll_results_text()

        self.assertIn("A\nYour answer\n100 %", poll_result)
        self.assertIn("B\nYour answer\n100 %", poll_result)
        self.assertIn("C\n0 %", poll_result)


# selenium_tests/questionnaires_all/test_questionnaire_components.py

import time

from selenium_tests.base import BaseSeleniumTests
from selenium.webdriver.support.ui import Select
from wagtail.core.models import Site
from iogt_users.factories import AdminUserFactory
from home.factories import SectionFactory
from questionnaires.factories import SurveyFactory, SurveyFormFieldFactory
from selenium_tests.pages import QuestionnairePage, QuestionnaireResultsPage

class QuestionnaireComponentsSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        Site.objects.all().delete()
        self.setup_blank_site()
        self.user = AdminUserFactory()
        self.section01 = SectionFactory(parent=self.home, owner=self.user)
        self.survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user, 
            direct_display=True, 
            description="[{\"type\": \"paragraph\", \"value\": \"<p>This is a simple survey description</p>\"}]",
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]"
            )
        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,  
            default_value='',  
            field_type='singleline', 
        )

    def test_direct_display(self):
        self.visit_page(self.section01)
        #We are looking at a section here, but use the Questionnaire page as we want to test the functionality of the direct display questionnaire
        survey_page = QuestionnairePage(self.selenium)
        self.assertIn("This is a simple survey description", survey_page.get_content_text())
        survey_page.enter_text("Testing some singleline text")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_title(self):
        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        self.assertIn(self.survey01.title, survey_page.title)

    def test_description(self):
        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        self.assertIn("This is a simple survey description", survey_page.get_content_text())

    def test_basic_result(self):
        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_text("Testing some singleline text")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())



        
        
        


# selenium_tests/questionnaires_all/test_questionnaire_functionality.py

import time
import json

from selenium_tests.base import BaseSeleniumTests
from selenium.webdriver.support.ui import Select
from wagtail.core.models import Site
from iogt_users.factories import AdminUserFactory
from home.factories import SectionFactory
from questionnaires.factories import SurveyFactory, SurveyFormFieldFactory
from selenium_tests.pages import QuestionnairePage, QuestionnaireResultsPage, LoginPage

class QuestionnaireFunctionalitySeleniumTests(BaseSeleniumTests):

    def setUp(self):
        Site.objects.all().delete()
        self.setup_blank_site()
        self.user = AdminUserFactory()
        self.section01 = SectionFactory(parent=self.home, owner=self.user)
        

    def test_single_submission(self):
        survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user,
            allow_multiple_submissions = False
            )
        SurveyFormFieldFactory(
            page=survey01, 
            required = True,  
            default_value='',  
            field_type='singleline', 
        )
        self.visit_page(survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_text("Testing some singleline text")
        survey_page.submit_response()
        revisit_page = self.visit_page(survey01)
        self.assertIn("You have already completed this survey.", revisit_page.get_content_text())

    def test_multiple_submission(self):
        survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user,
            allow_multiple_submissions = True,
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]"
            )
        SurveyFormFieldFactory(
            page=survey01, 
            required = True,  
            default_value='',  
            field_type='singleline', 
        )
        self.visit_page(survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_text("Testing some singleline text")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())
        self.visit_page(survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_text("Testing some singleline text")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_required_optional_submission(self):
        survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user,
            allow_multiple_submissions = True,
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]"
            )
        SurveyFormFieldFactory(
            page=survey01, 
            required = False,  
            default_value='',  
            field_type='singleline', 
        )
        SurveyFormFieldFactory(
            page=survey01, 
            required = True,
            choices = "A|B|C",  
            default_value='',  
            field_type='checkboxes', 
        )
        self.visit_page(survey01)
        survey_page = QuestionnairePage(self.selenium)
        self.assertIn("OPTIONAL", survey_page.get_content_text())
        survey_page.submit_response()
        self.assertIn("This field is required.", survey_page.get_content_text())
        survey_page.select_checkboxes("B")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_multiple_page_survey(self):
        survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user,
            multi_step = True,
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]"
            )
        SurveyFormFieldFactory(
            page=survey01, 
            required = False,  
            default_value='',  
            field_type='singleline',
            page_break = True 
        )
        SurveyFormFieldFactory(
            page=survey01, 
            required = False,
            choices = "A|B|C",  
            default_value='',  
            field_type='checkboxes', 
        )
        self.visit_page(survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.submit_response()
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_survey_require_login(self):
        survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user,
            allow_anonymous_submissions = False,
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]"
            )
        SurveyFormFieldFactory(
            page=survey01, 
            required = True,  
            default_value='',  
            field_type='singleline',
            page_break = True 
        )
        
        self.visit_page(survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.login_button_submit()
        login_page = LoginPage(self.selenium)
        login_page.login_user(self.user)
        self.visit_page(survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_text("Testing some singleline text")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_survey_back(self):
        survey01 = SurveyFactory(parent=self.section01, owner=self.user)
        SurveyFormFieldFactory(
            page=survey01, 
            required = True,  
            default_value='',  
            field_type='singleline',
            page_break = True 
        )
        
        self.visit_page(survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_text("Testing some singleline text")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        results_page.go_back()
        current_page = QuestionnairePage(self.selenium)
        self.assertIn(survey01.title, current_page.title)

    def test_skip_question(self):
        survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user,
            allow_multiple_submissions = True,
            multi_step = True,
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]"
            )

        skip_logic = [
            {
                'type': 'skip_logic',
                'value': {
                    'choice': 'A',
                    'skip_logic': 'next'
                    },
            }, 
            {
                'type': 'skip_logic', 
                'value': {
                    'choice': 'B', 
                    'skip_logic': 'question', 
                    'question': 3
                    },
            }, 
            {
                'type': 'skip_logic', 
                'value': {
                    'choice': 'C', 
                    'skip_logic': 'next', 
                    }, 
            }
        ]
        json_skip_logic = json.dumps(skip_logic)

        SurveyFormFieldFactory(
            page=survey01,
            sort_order= 0,
            required=True,
            choices= "A|B|C",
            clean_name= "question_1",
            label= "XXXX",
            field_type= "radio",
            admin_label= "q1",
            skip_logic= json_skip_logic,
            default_value= "",
            page_break=True
        )

        SurveyFormFieldFactory(
            page=survey01,
            sort_order= 1,
            required=False,
            choices= "",
            help_text= "",
            clean_name = "question_2",
            label= "YYYY",
            field_type= "singleline",
            admin_label= "q2",
            skip_logic= "[]",
            default_value= "",
            page_break=True
        )

        SurveyFormFieldFactory(
            page=survey01,
            sort_order= 2,
            required=False,
            choices= "",
            help_text= "",
            clean_name= "question_3",
            label= "ZZZZ",
            field_type= "singleline",
            admin_label= "q3",
            skip_logic= "[]",
            default_value= "",
            page_break=True
        )
        
        self.visit_page(survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.select_checkboxes("B")
        survey_page.submit_response()
        current_question = QuestionnairePage(self.selenium)
        self.assertIn("ZZZZ", current_question.get_content_text())
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_skip_end_survey(self):
        survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user,
            allow_multiple_submissions = True,
            multi_step = True,
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]"
            )
        
        skip_logic = [
            {
                'type': 'skip_logic',
                'value': {
                    'choice': 'A',
                    'skip_logic': 'next',
                    },
            }, 
            {
                'type': 'skip_logic', 
                'value': {
                    'choice': 'C', 
                    'skip_logic': 'end', 
                    }, 
            }
        ]
        json_skip_logic = json.dumps(skip_logic)

        SurveyFormFieldFactory(
            page=survey01,
            sort_order= 0,
            required=True,
            choices= "A|C",
            clean_name= "question_1",
            label= "XXXX",
            field_type= "radio",
            admin_label= "q1",
            skip_logic= json_skip_logic,
            default_value= "",
            page_break=True
        )

        SurveyFormFieldFactory(
            page=survey01,
            sort_order= 1,
            required=False,
            choices= "",
            help_text= "",
            clean_name = "question_2",
            label= "YYYY",
            field_type= "singleline",
            admin_label= "q2",
            skip_logic= "[]",
            default_value= "",
            page_break=True
        )
        
        self.visit_page(survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.select_checkboxes("C")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

# selenium_tests/questionnaires_all/test_questionnaire_inputs.py

from selenium_tests.base import BaseSeleniumTests
from wagtail.core.models import Site
from iogt_users.factories import AdminUserFactory
from home.factories import SectionFactory
from questionnaires.factories import SurveyFactory, SurveyFormFieldFactory
from selenium_tests.pages import QuestionnairePage, QuestionnaireResultsPage

class QuestionnaireInputsSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        Site.objects.all().delete()
        self.setup_blank_site()
        self.user = AdminUserFactory()
        self.section01 = SectionFactory(parent=self.home, owner=self.user)
        self.survey01 = SurveyFactory(
            parent=self.section01, 
            owner=self.user,
            thank_you_text="[{\"type\": \"paragraph\", \"value\": \"<p>Thankyou for completing the survey</p>\"}]"
            )

    def test_checkbox(self): 

        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,
            choices = "true|false",  
            default_value='',  
            field_type='checkbox', 
        )       

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.select_checkbox()
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())
        
    def test_checkboxes(self):    

        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,
            choices = "A|B|C",  
            default_value='',  
            field_type='checkboxes', 
        )

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.select_checkboxes("B")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_date(self):

        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,  
            default_value='',  
            field_type='date', 
        )

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_date("11111111")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_date_time(self):

        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,  
            default_value='',  
            field_type='datetime', 
        )

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_date_time("12122000", "1111AM")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        results_page_content = results_page.get_content_text()
        self.assertIn("Thankyou for completing the survey", results_page_content)

    def test_dropdown(self):

        question = SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,
            choices = "A|B|C",  
            default_value='',  
            field_type='dropdown', 
        )

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        selection = survey_page.use_dropdown(question.label, "C")
        self.assertIn("C", selection)
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_email(self):

        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,  
            default_value='',  
            field_type='email', 
        )

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_email("tester@gmail.com")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_singleline(self):

        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,  
            default_value='',  
            field_type='singleline', 
        )

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_text("Testing some singleline text")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_multiline(self):

        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,  
            default_value='',  
            field_type='multiline', 
        )

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_multiline_text("""Testing some multiline text
        Checking that everying
        Looks ok
        when there are multiple lines""")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_number(self):

        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,  
            default_value='',  
            field_type='number', 
        )

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_number("5")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_radio(self):

        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,
            choices = "A|B|C",  
            default_value='',  
            field_type='radio', 
        )

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.select_checkboxes("B")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())

    def test_url(self):

        SurveyFormFieldFactory(
            page=self.survey01, 
            required = True,  
            default_value='',  
            field_type='url', 
        )

        self.visit_page(self.survey01)
        survey_page = QuestionnairePage(self.selenium)
        survey_page.enter_url("https://www.google.com/")
        survey_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        self.assertIn("Thankyou for completing the survey", results_page.get_content_text())
        
        


# selenium_tests/quiz_specific/test_quiz_functionality.py

from selenium_tests.base import BaseSeleniumTests
from wagtail.core.models import Site
from iogt_users.factories import AdminUserFactory
from home.factories import SectionFactory
from questionnaires.factories import QuizFactory, QuizFormFieldFactory
from selenium_tests.pages import QuestionnairePage, QuestionnaireResultsPage

class QuizSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        Site.objects.all().delete()
        self.setup_blank_site()
        self.user = AdminUserFactory()
        self.section01 = SectionFactory(parent=self.home, owner=self.user)

    def test_correct_quiz(self):
        quiz01 = QuizFactory(
            parent=self.section01, 
            owner=self.user,
            )
        QuizFormFieldFactory(
            page=quiz01, 
            required = True,
            choices = "A|B|C",  
            default_value='',
            correct_answer = "B",  
            field_type='checkboxes', 
        )

        self.visit_page(quiz01)
        quiz_page = QuestionnairePage(self.selenium)
        quiz_page.select_checkboxes("B")
        quiz_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        quiz_result = results_page.get_quiz_results_text()
        self.assertIn("1 Correct", quiz_result)

    def test_incorrect_quiz(self):
        quiz01 = QuizFactory(
            parent=self.section01, 
            owner=self.user,
            )
        QuizFormFieldFactory(
            page=quiz01, 
            required = True,
            choices = "A|B|C",  
            default_value='',
            correct_answer = "B",  
            field_type='checkboxes', 
        )

        self.visit_page(quiz01)
        quiz_page = QuestionnairePage(self.selenium)
        quiz_page.select_checkboxes("A")
        quiz_page.submit_response()
        results_page = QuestionnaireResultsPage(self.selenium)
        quiz_result = results_page.get_quiz_results_text()
        self.assertIn("1 Incorrect", quiz_result)
        
        

        
        


# selenium_tests/sections/test_components.py

from selenium_tests.base import BaseSeleniumTests
from wagtail.core.models import Site
from iogt_users.factories import AdminUserFactory
from home.factories import SectionFactory
from questionnaires.factories import PollFactory, PollFormFieldFactory
from selenium_tests.pages import QuestionnairePage, SectionPage
import home.models as models

class SectionComponentSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        Site.objects.all().delete()
        self.setup_blank_site()
        self.user = AdminUserFactory()
        self.section01 = SectionFactory(parent=self.home, owner=self.user)

    def test_featured(self):
        poll01 = PollFactory(parent=self.home, owner=self.user)
        PollFormFieldFactory(page=poll01, field_type='checkboxes', choices='c1|c2|c3', default_value='c2')
        models.FeaturedContent.objects.create(source = self.section01, content = poll01)
        self.visit_page(self.section01)
        section_page = SectionPage(self.selenium)
        section_page.navigate_featured_content(poll01.title)
        poll_page = QuestionnairePage(self.selenium)
        self.assertIn(poll01.title, poll_page.title)
       
    def test_title(self):
        self.visit_page(self.section01)
        section_page = SectionPage(self.selenium)
        self.assertIn(self.section01.title, section_page.title)        
        
    def test_image(self):
        self.visit_page(self.section01)
        article_page = SectionPage(self.selenium)
        self.assertTrue(article_page.has_lead_image(), 'Image should be present and visible')
       

        
    

# selenium_tests/sections/test_content.py

from selenium_tests.base import BaseSeleniumTests
from wagtail.core.models import Site
from iogt_users.factories import AdminUserFactory
from home.factories import SectionFactory, ArticleFactory
from questionnaires.factories import PollFactory, PollFormFieldFactory
from selenium_tests.pages import QuestionnairePage, SectionPage, ArticlePage

class SectionContentSeleniumTests(BaseSeleniumTests):

    def setUp(self):
        Site.objects.all().delete()
        self.setup_blank_site()
        self.user = AdminUserFactory()
        self.section01 = SectionFactory(parent=self.home, owner=self.user)

    def test_questionnaire(self):
        poll01 = PollFactory(parent=self.section01, owner=self.user)
        PollFormFieldFactory(page=poll01, field_type='checkboxes', choices='c1|c2|c3', default_value='c2')
        self.visit_page(self.section01)
        section_page = SectionPage(self.selenium)
        section_page.navigate_featured_content(poll01.title)
        poll_page = QuestionnairePage(self.selenium)
        self.assertIn(poll01.title, poll_page.title)

    def test_article(self):
        article01 = ArticleFactory(parent=self.section01, owner=self.user)
        self.visit_page(self.section01)
        section_page = SectionPage(self.selenium)
        section_page.navigate_featured_content(article01.title)
        article_page = ArticlePage(self.selenium)
        self.assertIn(article01.title, article_page.title)

    def test_sub_section(self):
        section02 = SectionFactory(parent=self.section01, owner=self.user)
        self.visit_page(self.section01)
        section_page = SectionPage(self.selenium)
        section_page.navigate_featured_content(section02.title)
        section_page2 = SectionPage(self.selenium)
        self.assertIn(section02.title, section_page2.title)
       
    
       

        
    

# comments/__init__.py

import importlib

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured


def get_comments_moderation_class():
    try:
        pkg, attr = settings.COMMENT_MODERATION_CLASS.rsplit('.', 1)
        return getattr(importlib.import_module(pkg), attr)
    except (AttributeError, ValueError):
        raise ImproperlyConfigured(
            f"No comments community moderator class found at this path '{settings.COMMENT_MODERATION_CLASS}'")


# comments/apps.py

from django.apps import AppConfig


class CommentsConfig(AppConfig):
    name = 'comments'


# comments/button_helpers.py

from django.urls import reverse
from wagtail.contrib.modeladmin.helpers import ButtonHelper


class XtdCommentAdminButtonHelper(ButtonHelper):
    def hide_show_toggle_button(self, comment):

        if comment.is_removed:
            action = 'show'
            label = 'Show'
        else:
            action = 'hide'
            label = 'Hide'

        return {
            'url': reverse('wagtail_comments_xtd_publication', kwargs={
                'comment_pk': comment.pk,
                'action': action
            }),
            'label': label,
            'classname': 'button button-small button-secondary',
            'title': f'{label} this {self.verbose_name}'
        }

    def publish_unpublish_toggle_button(self, comment):
        if comment.is_public:
            action = 'unpublish'
            label = 'Unpublish'
            cn = 'button no button-small button-secondary'
        else:
            action = 'publish'
            label = 'Publish'
            cn = 'button button-small button-secondary'
        return {
            'url': reverse('wagtail_comments_xtd_publication', kwargs={
                'comment_pk': comment.pk,
                'action': action
            }),
            'label': label,
            'classname': cn,
            'title': f'{label} this {self.verbose_name}'
        }

    def clear_flags_button(self, comment):
        if comment.flags.count():
            return {
                'url': reverse('wagtail_comments_xtd_publication', kwargs={
                    'comment_pk': comment.pk,
                    'action': 'clear_flags'
                }),
                'label': 'Clear Flags',
                'classname': 'button button-small button-secondary',
                'title': f'Clear flags for {self.verbose_name}'
            }

    def comment_reply_button(self, comment):
        return {
            'url': reverse('comment_reply_view', kwargs={
                'comment_pk': comment.pk
            }),
            'label': 'Reply',
            'classname': 'button button-small button-secondary',
            'title': f'Add Reply for {self.verbose_name}'
        }

    def get_buttons_for_obj(self, obj, *args, **kwargs):
        buttons = super().get_buttons_for_obj(obj, exclude='delete', *args, **kwargs)
        buttons.append(self.publish_unpublish_toggle_button(comment=obj))
        buttons.append(self.clear_flags_button(comment=obj))
        buttons.append(self.comment_reply_button(comment=obj))
        return buttons


# comments/clients.py

from abc import ABC

from django.conf import settings


class BaseModerator(ABC):
    @classmethod
    def moderate(cls, comment):
        raise NotImplementedError


class BlacklistedWordsModerator(BaseModerator):
    @classmethod
    def moderate(cls, comment):
        for blacklisted_word in settings.BLACKLISTED_WORDS:
            if blacklisted_word in comment.comment:
                return False

        return True


class AlwaysApproveModerator(BaseModerator):
    @classmethod
    def moderate(cls, comment):
        return True


# comments/filters.py

from django.contrib.admin import SimpleListFilter
from django.db.models import Count

from comments.models import CommentModeration


class FlaggedFilter(SimpleListFilter):
    title = "Flagged?"
    parameter_name = "flagged"

    def lookups(self, request, model_admin):
        return [
            (True, 'Yes'),
            (False, 'No')
        ]

    def queryset(self, request, queryset):
        if self.value():
            return queryset.annotate(num_flags=Count('flags')).filter(num_flags__gt=0)
        return queryset


class PublishedFilter(SimpleListFilter):
    title = "Published?"
    parameter_name = "is_published"

    def lookups(self, request, model_admin):
        return [
            (True, 'Yes'),
            (False, 'No')
        ]

    def queryset(self, request, queryset):
        if self.value():
            return queryset.filter(is_public=self.value())
        return queryset


class ModerationFilter(SimpleListFilter):
    title = "Moderation?"
    parameter_name = "status"

    def lookups(self, request, model_admin):
        return [
            (CommentModeration.CommentModerationState.UNMODERATED, 'Unmoderated'),
            (CommentModeration.CommentModerationState.APPROVED, 'Approved'),
            (CommentModeration.CommentModerationState.REJECTED, 'Rejected'),
            (CommentModeration.CommentModerationState.UNSURE, 'Unsure'),
        ]

    def queryset(self, request, queryset):
        if self.value():
            return queryset.filter(comment_moderation__state=self.value())
        return queryset


# comments/forms.py

from django import forms
from django_comments_xtd.forms import XtdCommentForm as BaseCommentForm
from django.utils.translation import gettext as _

from comments.models import CannedResponse, CommentModeration


class CommentForm(BaseCommentForm):

    def __init__(self, *args, **kwargs):
        super(CommentForm, self).__init__(*args, **kwargs)
        self.fields['email'].initial = 'noreply@example.com'
        self.fields['email'].widget = forms.HiddenInput()

        canned_responses_choices = [(None, 'Select Canned Response')]
        for canned_response in CannedResponse.objects.all():
            text = f"{canned_response.text[:50]}..." if len(canned_response.text) > 50 else canned_response.text
            canned_responses_choices.append((canned_response.id, f'{canned_response.header} - {text}'))

        self.fields['canned_responses'] = forms.ChoiceField(choices=canned_responses_choices, required=False)
        self.fields['canned_responses'].widget.attrs['class'] = 'canned-response-select'

        self.fields['name'].widget = forms.HiddenInput()

        self.fields['followup'].initial = False
        self.fields['followup'].widget = forms.HiddenInput()

        self.fields['post_anonymously'] = forms.BooleanField(
            label=_('Don\'t display my username next to my comment'), required=False)

    def get_comment_create_data(self, site_id=None):
        data = super().get_comment_create_data(site_id=site_id)

        if self.cleaned_data['canned_responses']:
            data['comment'] = f'{data["comment"]} {self.cleaned_data["canned_responses"]}'

        if self.cleaned_data['post_anonymously']:
            data['user_email'] = ''
            data['user_name'] = 'Anonymous'
        return data


class AdminCommentForm(CommentForm):

    def __init__(self, *args, **kwargs):
        super(AdminCommentForm, self).__init__(*args, **kwargs)
        self.is_admin = True


class CommentFilterForm(forms.Form):
    state = forms.ChoiceField(label='Moderation',
                              choices=(('ALL', 'All'),
                                       (CommentModeration.CommentModerationState.UNMODERATED, 'Unmoderated'),
                                       (CommentModeration.CommentModerationState.APPROVED, 'Approved'),
                                       (CommentModeration.CommentModerationState.REJECTED, 'Rejected'),
                                       (CommentModeration.CommentModerationState.UNSURE, 'Unsure')),
                              required=False)
    from_date = forms.DateField(label='From', required=False, widget=forms.DateInput(attrs={'type': 'date'}))
    to_date = forms.DateField(label='To', required=False, widget=forms.DateInput(attrs={'type': 'date'}))

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields['state'].initial = 'UNMODERATED'


# comments/models.py

from django.contrib.auth import get_user_model
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.utils import timezone
from django_comments.signals import comment_was_flagged
from django_comments_xtd.models import XtdComment
from wagtail.admin.edit_handlers import FieldPanel
from wagtail.core.models import Page

from comments import get_comments_moderation_class

User = get_user_model()
Moderator = get_comments_moderation_class()


class CommentStatus:
    OPEN = 'open'
    CLOSED = 'closed'
    DISABLED = 'disabled'
    TIMESTAMPED = 'timestamped'
    INHERITED = 'inherited'

    Choices = (
        (OPEN, 'Open'),
        (CLOSED, 'Closed'),
        (DISABLED, 'Disabled'),
        (TIMESTAMPED, 'Timestamped'),
        (INHERITED, 'Inherited'),
    )


class CommentableMixin(models.Model):
    """
    Add this mixin to any Model which wants to allow/disallow comments.
    Make sure you update get_absolute_url if this it hasn't already been
    included.

    Use comments_panels to modify commenting_status from the admin.
    """
    commenting_status = models.CharField(max_length=15, choices=CommentStatus.Choices, default=CommentStatus.INHERITED)
    commenting_starts_at = models.DateTimeField(null=True, blank=True)
    commenting_ends_at = models.DateTimeField(null=True, blank=True)

    comments_panels = [
        FieldPanel('commenting_status', heading='Status'),
        FieldPanel('commenting_starts_at', heading='Commenting Opens At'),
        FieldPanel('commenting_ends_at', heading='Commenting Ends At')

    ]

    def _get_valid_ancestor_for_commenting(self):
        from home.models import Section

        ancestors = self.get_ancestors().type(Section).specific().order_by('-path')
        for ancestor in ancestors:
            if ancestor.commenting_status in [
                CommentStatus.OPEN, CommentStatus.CLOSED, CommentStatus.DISABLED, CommentStatus.TIMESTAMPED]:
                return ancestor

    def should_show_comments_list(self):
        from home.models import Section

        commenting_status = self.commenting_status
        if commenting_status == CommentStatus.INHERITED:
            ancestor = self._get_valid_ancestor_for_commenting()
            commenting_status = ancestor.commenting_status if ancestor else CommentStatus.DISABLED

        return commenting_status in [CommentStatus.OPEN, CommentStatus.CLOSED, CommentStatus.TIMESTAMPED]

    def should_show_new_comment_box(self):
        from home.models import Section

        commenting_still_valid = True
        commenting_status = self.commenting_status
        commenting_starts_at = self.commenting_starts_at
        commenting_ends_at = self.commenting_ends_at

        if commenting_status == CommentStatus.INHERITED:
            ancestor = self._get_valid_ancestor_for_commenting()
            if ancestor:
                commenting_status = ancestor.commenting_status
                commenting_starts_at = ancestor.commenting_starts_at
                commenting_ends_at = ancestor.commenting_ends_at
            else:
                commenting_status = CommentStatus.DISABLED
                commenting_starts_at = None
                commenting_ends_at = None

        if commenting_starts_at:
            commenting_still_valid = commenting_starts_at < timezone.now()
        if commenting_ends_at:
            commenting_still_valid = commenting_still_valid and commenting_ends_at > timezone.now()

        return commenting_status == CommentStatus.OPEN or \
               (commenting_status == CommentStatus.TIMESTAMPED and commenting_still_valid)

    def get_absolute_url(self):
        if isinstance(self, Page):
            return self.get_url()
        raise NotImplementedError('Implement get_absolute_url for your Model')

    class Meta:
        abstract = True


class CannedResponse(models.Model):
    header = models.TextField(null=True)
    text = models.TextField(null=True)

    def __str__(self):
        return self.text


class CommentModeration(models.Model):
    class CommentModerationState(models.TextChoices):
        UNMODERATED = "UNMODERATED", "Unmoderated"
        APPROVED = "APPROVED", "Approved"
        REJECTED = "REJECTED", "Rejected"
        UNSURE = "UNSURE", "Unsure"

    state = models.CharField(max_length=255, choices=CommentModerationState.choices, default=CommentModerationState.UNMODERATED)
    comment = models.OneToOneField(
        to='django_comments_xtd.XtdComment', related_name='comment_moderation', on_delete=models.CASCADE)

    def __str__(self):
        return f'{self.comment.id} | {self.state}'


@receiver(post_save, sender=XtdComment)
def comment_moderation_handler(sender, instance, created, **kwargs):
    if created:
        moderator = Moderator()
        instance.is_public = moderator.moderate(instance)
        instance.save(update_fields=['is_public'])
        CommentModeration.objects.create(comment_id=instance.id)


@receiver(comment_was_flagged, sender=XtdComment)
def comment_flagged(sender, comment, **kwargs):
    if hasattr(comment, 'comment_moderation'):
        comment_moderation = comment.comment_moderation
        comment_moderation.state = CommentModeration.CommentModerationState.UNMODERATED
        comment_moderation.save(update_fields=['state'])


# comments/urls.py

from __future__ import absolute_import, unicode_literals
from django.conf.urls import url
from django.urls import path

from comments import views
from comments.views import ProcessCannedResponseView, CommentsCommunityModerationView

urlpatterns = [
    url(r'^comment/(?P<comment_pk>\d+)/update/(?P<action>publish|unpublish|hide|show|clear_flags|approve|reject|unsure)/$',
        views.update, name='wagtail_comments_xtd_publication'),
    path('comment/<int:comment_pk>/reply',
        views.CommentReplyView.as_view(), name='comment_reply_view'),
    path('comment/new',
            views.post_admin_comment, name='comment_post_admin_comment'),
    path('comment/process-canned-response', ProcessCannedResponseView.as_view(), name='process_canned_response'),
    path('community-moderation/', CommentsCommunityModerationView.as_view(), name='comments_community_moderation'),
]


# comments/views.py

from urllib.parse import urlparse, urlunparse, urlencode, parse_qsl

from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django.db.models import Q, Count
from django.contrib import messages
from django.shortcuts import redirect, get_object_or_404
from django.utils.decorators import method_decorator
from django.views import View
from django.views.decorators.csrf import csrf_protect
from django.views.decorators.http import require_POST
from django.views.generic import TemplateView, ListView
from django_comments.views.comments import post_comment
from django_comments_xtd.models import XtdComment
from django.utils.translation import ugettext as _

from comments.forms import AdminCommentForm, CommentFilterForm
from comments.models import CannedResponse, CommentModeration


def update(request, comment_pk, action):
    get_comment_with_children_filter = Q(parent_id=comment_pk) | Q(pk=comment_pk)
    comments = XtdComment.objects.filter(get_comment_with_children_filter)
    comment_moderations = []
    verb = ''
    if action == 'approve':
        for comment in comments:
            comment.is_public = True
            comment.flags.all().delete()
            comment_moderation = comment.comment_moderation
            comment_moderation.state = CommentModeration.CommentModerationState.APPROVED
            comment_moderations.append(comment_moderation)
        verb = 'approved'
    elif action == 'reject':
        for comment in comments:
            comment.is_public = False
            comment_moderation = comment.comment_moderation
            comment_moderation.state = CommentModeration.CommentModerationState.REJECTED
            comment_moderations.append(comment_moderation)
        verb = 'rejected'
    elif action == 'unsure':
        for comment in comments:
            comment_moderation = comment.comment_moderation
            comment_moderation.state = CommentModeration.CommentModerationState.UNSURE
            comment_moderations.append(comment_moderation)
        verb = 'unsure'
    elif action == 'unpublish':
        for comment in comments:
            comment.is_public = False
        verb = 'unpublished'
    elif action == 'publish':
        for comment in comments:
            comment.is_public = True
        verb = 'published'
    elif action == 'hide':
        for comment in comments:
            comment.is_removed = True
        verb = 'removed'
    elif action == 'show':
        for comment in comments:
            comment.is_removed = False
        verb = 'shown'
    elif action == 'clear_flags':
        comment = XtdComment.objects.get(pk=comment_pk)
        comment.flags.all().delete()
        verb = 'cleared'
    XtdComment.objects.bulk_update(comments, ['is_public', 'is_removed'])
    CommentModeration.objects.bulk_update(comment_moderations, ['state'])

    messages.success(request, _(f'The comment has been {verb} successfully!'))

    return redirect(request.META.get('HTTP_REFERER'))


class CommentReplyView(TemplateView):
    template_name = 'comment_reply.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        comment = XtdComment.objects.get(pk=kwargs['comment_pk'])
        context.update({
            'form': AdminCommentForm(comment.content_object, comment=comment),
            'comment': comment,
            'canned_responses': CannedResponse.objects.all()
        })
        return context


@csrf_protect
@require_POST
def post_admin_comment(request):
    """
    This is a hack to make the post comment view send a message. There
    was no better way to do this than use a decorator pattern. Suggestions/Improvements
    are welcome
    :param request:
    :return:
    """
    response = post_comment(request, next='/')
    messages.success(request, _("Sent Reply successfully"))
    return response


class ProcessCannedResponseView(View):
    def post(self, request):
        referer = request.META.get('HTTP_REFERER')
        canned_response_id = request.POST.get('canned_responses')
        parsed_url = urlparse(referer)

        if not request.user.has_perm('django_comments_xtd.can_moderate'):
            messages.error(request, _("You do not have the permission to perform this action."))
        else:
            query_dict = dict(parse_qsl(parsed_url.query))
            canned_response_text = get_object_or_404(CannedResponse,
                                                     pk=canned_response_id).text if canned_response_id else ''
            comment_text = request.POST.get('comment')
            query_dict.update({'canned_response': f'{comment_text} {canned_response_text}'})

            parsed_url = parsed_url._replace(query=urlencode(query_dict))

        referer_url = urlunparse(parsed_url)
        return redirect(to=referer_url)


class CommentsCommunityModerationView(ListView):
    model = XtdComment
    template_name = 'comments/community_moderation.html'
    context_object_name = 'comments'
    paginate_by = 10

    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        if not settings.COMMENTS_COMMUNITY_MODERATION or not request.user.has_perm('django_comments_xtd.can_moderate'):
            raise PermissionDenied(
                "You do not have permission."
            )
        return super().dispatch(request, *args, **kwargs)

    def _get_form(self):
        return CommentFilterForm(self.request.GET or {'state': CommentModeration.CommentModerationState.UNMODERATED})

    def get_queryset(self):
        queryset = super().get_queryset().filter(comment_moderation__isnull=False).order_by('-submit_date')
        form = self._get_form()
        if form.is_valid():
            data = form.cleaned_data
            state = data['state']
            from_date = data['from_date']
            to_date = data['to_date']

            if state != 'ALL':
                queryset = queryset.filter(comment_moderation__state=state)

            if to_date:
                if from_date:
                    queryset = queryset.filter(submit_date__date__range=[from_date, to_date])
                else:
                    queryset = queryset.filter(submit_date__date__lte=to_date)
            elif from_date:
                queryset = queryset.filter(submit_date__date__gte=from_date)
        return queryset

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        form = self._get_form()
        if form.is_valid():
            data = form.cleaned_data
            context.update({
                'form': form,
                'params': urlencode(data)
            })
        return context


# comments/wagtail_hooks.py

from django.conf import settings
from django.utils import timezone
from django.utils.html import format_html
from django_comments_xtd.models import XtdComment
from wagtail.contrib.modeladmin.options import ModelAdminGroup, ModelAdmin, modeladmin_register

from .button_helpers import XtdCommentAdminButtonHelper
from .filters import FlaggedFilter, ModerationFilter, PublishedFilter
from .models import CannedResponse


class XtdCommentAdmin(ModelAdmin):
    model = XtdComment
    menu_label = 'All Comments'
    menu_icon = 'edit'
    list_display = ('comment', 'user', 'published', 'moderation', 'num_flags', 'num_replies', 'submit_date', 'view_live')
    list_filter = (FlaggedFilter, PublishedFilter, ModerationFilter, 'submit_date',)
    form_fields_exclude = ('thread_id', 'parent_id', 'level', 'order', 'followup', 'nested_count',
                           'content_type', 'object_id', 'user_email', 'user_url')
    search_fields = ('comment',)
    list_export = (
        'comment', 'user', 'published', 'moderation', 'num_flags', 'num_replies', 'submit_date', 'article',
        'article_url', 'article_language_code',
    )
    button_helper_class = XtdCommentAdminButtonHelper
    menu_order = 601

    def published(self, obj):
        rv = 'No'
        if obj.is_public:
            rv = 'Yes'
        return format_html(rv)

    def moderation(self, obj):
        return format_html(obj.comment_moderation.state)

    def get_queryset(self, request):
        return super().get_queryset(request).prefetch_related('flags')

    def num_replies(self, obj):
        return obj.nested_count

    def num_flags(self, obj):
        return obj.flags.count()

    def article(self, obj):
        return getattr(obj.content_object, 'title', 'N/A')

    def article_url(self, obj):
        return getattr(obj.content_object, 'url', 'N/A')

    def article_language_code(self, obj):
        locale = getattr(obj.content_object, 'locale', object)
        return getattr(locale, 'language_code', 'N/A')

    def view_live(self, obj):
        content_object = obj.content_object
        url = getattr(content_object, 'url', None)
        if url:
            return f'<a href="{url}" target="_blank">{content_object.title}</a>'

        return 'N/A'

    view_live.allow_tags = True

    @property
    def export_filename(self):
        return f'comments_{timezone.now().strftime(settings.EXPORT_FILENAME_TIMESTAMP_FORMAT)}'


class CannedResponseAdmin(ModelAdmin):
    model = CannedResponse
    menu_label = 'Canned Responses'
    menu_icon = 'placeholder'
    list_display = ('header', 'text')
    search_fields = ('header', 'text')
    menu_order = 602


class CommentsGroup(ModelAdminGroup):
    menu_label = 'Comments'
    menu_icon = 'openquote'
    menu_order = 600
    items = (XtdCommentAdmin, CannedResponseAdmin)


modeladmin_register(CommentsGroup)


# external_links/middleware.py

import html
import re

from django.conf import settings
from django.template.defaultfilters import urlencode
from django.urls import reverse


SAFE_EXTERNAL_LINK_PATTERNS = getattr(settings, 'SAFE_EXTERNAL_LINK_PATTERNS', ())
safe_urls = ''
if SAFE_EXTERNAL_LINK_PATTERNS:
    safe_urls = '(?!(' + '|'.join(SAFE_EXTERNAL_LINK_PATTERNS) + '))'


class RewriteExternalLinksMiddleware:
    """
    Rewrite all external links to go via a message page.
    Rewrite:
        <a href="http://www.example.com">
    To:
        <a href="/external-link/?next=http://www.example.com">
    """
    def __init__(self, get_response):
        self.get_response = get_response
        self.extlinks = re.compile(r'''
            (?P<before><a[^>]*href=['"]?)  # content from `<a` to `href='`
            (?P<link>https?://{}[^'">]*)  # href link
            (?P<after>[^>]*)  # content after href to closing bracket `>`
        '''.format(safe_urls), re.VERBOSE)

    def __call__(self, request):
        response = self.get_response(request)
        external_link_root = reverse('external-link')

        # neither external_link_root nor request.path include hostname
        if response.streaming or request.path.startswith('/admin'):
            return response

        html_content_type = 'text/html' in response.get('content-type', '')
        start_link = request.path.startswith(external_link_root)

        if (response.content and html_content_type and not start_link):
            next = request.path

            def linkrepl(m):
                return '{before}{root}?next={link}&from={next}{after}'.format(
                    root=external_link_root,
                    next=next,
                    before=m.group('before'),
                    # unescape the link before encoding it to ensure entities
                    # such as '&' don't get double escaped
                    link=urlencode(html.unescape(m.group('link')), safe=''),
                    after=m.group('after'),
                )
            response.content = self.extlinks.sub(
                linkrepl,
                response.content.decode('utf-8')
            )
            response['Content-Length'] = len(response.content)

        return response


# home/admin.py

from django.contrib import admin
from wagtail.contrib.modeladmin.options import ModelAdmin, modeladmin_register

from home.models import ManifestSettings, SVGToPNGMap


class ManifestSettingsAdmin(ModelAdmin):
    model = ManifestSettings
    menu_label = "Manifest Settings"
    menu_icon = "cog"
    add_to_settings_menu = True
    list_display = ("language", "name")
    exclude_from_explorer = False


modeladmin_register(ManifestSettingsAdmin)


@admin.register(SVGToPNGMap)
class SVGToPNGMapAdmin(admin.ModelAdmin):
    list_display = ('id', 'svg_path', 'fill_color', 'stroke_color', 'png_image_file')




# home/blocks.py

from django.conf import settings
from django.forms.utils import flatatt
from django.utils.html import format_html, format_html_join
from django.utils.safestring import mark_safe
from django.utils.translation import gettext as _

from wagtail.core import blocks
from wagtail.core.blocks import PageChooserBlock
from wagtail.images.blocks import ImageChooserBlock
from wagtailmarkdown.utils import render_markdown
from wagtailmedia.blocks import AbstractMediaChooserBlock


class MediaBlock(AbstractMediaChooserBlock):
    def get_context(self, value, parent_context=None):
        context = super().get_context(value, parent_context)
        context.update({
            'start_link': mark_safe(f'<a href="{value.url}" download>'),
            'end_link': mark_safe('</a>'),
            'template': 'blocks/media-video.html' if value.type == 'video' else 'blocks/media-audio.html'
        })

        return context

    class Meta:
        template = 'blocks/media.html'


class SocialMediaLinkBlock(blocks.StructBlock):
    title = blocks.CharBlock(max_length=255)
    link = blocks.URLBlock()
    image = ImageChooserBlock(template='blocks/image.html')

    class Meta:
        icon = 'site'


class SocialMediaShareButtonBlock(blocks.StructBlock):
    platform = blocks.CharBlock(max_length=255)
    is_active = blocks.BooleanBlock(required=False)
    image = ImageChooserBlock(template='blocks/image.html', required=False)

    class Meta:
        icon = 'site'


class EmbeddedQuestionnaireChooserBlock(blocks.PageChooserBlock):
    class Meta:
        icon = 'form'


class EmbeddedQuestionnaireBlock(blocks.StructBlock):
    direct_display = blocks.BooleanBlock(required=False)


class EmbeddedPollBlock(EmbeddedQuestionnaireBlock):
    poll = EmbeddedQuestionnaireChooserBlock(target_model='questionnaires.Poll')

    def get_context(self, value, parent_context=None):
        context = super().get_context(value, parent_context)
        poll = value.get('poll')
        if poll and poll.live:
            context.update({
                'direct_display': value['direct_display'],
                'questionnaire': poll.specific,
            })
        return context

    class Meta:
        template = 'questionnaires/tags/questionnaire_wrapper.html'


class EmbeddedSurveyBlock(EmbeddedQuestionnaireBlock):
    survey = EmbeddedQuestionnaireChooserBlock(target_model='questionnaires.Survey')

    def get_context(self, value, parent_context=None):
        context = super().get_context(value, parent_context)
        survey = value.get('survey')
        if survey and survey.live:
            context.update({
                'direct_display': value['direct_display'],
                'questionnaire': survey.specific,
            })
        return context

    class Meta:
        template = 'questionnaires/tags/questionnaire_wrapper.html'


class EmbeddedQuizBlock(EmbeddedQuestionnaireBlock):
    quiz = EmbeddedQuestionnaireChooserBlock(target_model='questionnaires.Quiz')

    def get_context(self, value, parent_context=None):
        context = super().get_context(value, parent_context)
        quiz = value.get('quiz')
        if quiz and quiz.live:
            context.update({
                'direct_display': value['direct_display'],
                'questionnaire': quiz.specific,
            })
        return context

    class Meta:
        template = 'questionnaires/tags/questionnaire_wrapper.html'


class PageButtonBlock(blocks.StructBlock):
    page = blocks.PageChooserBlock()
    text = blocks.CharBlock(required=False, max_length=255)

    def get_context(self, value, parent_context=None):
        context = super().get_context(value, parent_context)
        button_page = value.get('page')
        if button_page and button_page.live:
            context.update({
                'button_page': button_page.specific,
                'text': value.get('text') or button_page.title
            })
        return context

    class Meta:
        template = 'blocks/page_button.html'


class ArticleBlock(blocks.StructBlock):
    display_section_title = blocks.BooleanBlock(required=False)
    article = PageChooserBlock(target_model='home.Article')

    def get_context(self, value, parent_context=None):
        context = super().get_context(value, parent_context)
        article = value.get('article')
        if article and article.live:
            context.update({
                'display_section_title': value['display_section_title'],
                'article': article.specific,
            })
        return context

    class Meta:
        template = 'blocks/article.html'


class NumberedListBlock(blocks.ListBlock):

    def render_basic(self, value, context=None):
        children = format_html_join(
            '\n', '<li>{0}</li>',
            [
                (self.child_block.render(child_value, context=context),)
                for child_value in value
            ]
        )
        return format_html("<ol>{0}</ol>", children)


class RawHTMLBlock(blocks.RawHTMLBlock):
    def render_basic(self, value, context=None):
        result = super(RawHTMLBlock, self).render_basic(value, context)

        return render_markdown(result)


class DownloadButtonBlock(blocks.StructBlock):
    available_text = blocks.CharBlock(
        help_text=_('This text appears when it is possible for the user to install the app on their phone.'))
    unavailable_text = blocks.CharBlock(
        required=False,
        help_text=_(
            'This text appears when the user is using a feature phone and thus cannot install the app '
            '(the button will be disabled in this case). [Currently not implemented]'),
        form_classname='red-help-text')
    offline_text = blocks.CharBlock(
        required=False, help_text=_(
            'This text appears when the user is navigating the site via the offline app and '
            'thus it doesn\'t make sense to install the offline app again '
            '(the button will be disabled in this case).'))
    page = PageChooserBlock(target_model='wagtailcore.Page')
    description = blocks.RichTextBlock(features=settings.WAGTAIL_RICH_TEXT_FIELD_FEATURES)

    class Meta:
        template = 'blocks/download_button.html'


# home/factories.py

import factory
from django.core.files.base import ContentFile
from factory.django import DjangoModelFactory, ImageField
from wagtail.core.models import Locale
from wagtail_factories import (
    ImageChooserBlockFactory,
    ImageFactory,
    PageFactory,
    StreamFieldFactory,
)
from wagtail_factories.blocks import ChooserBlockFactory, BlockFactory
from wagtail_factories.factories import CollectionMemberFactory
from wagtailmedia.models import get_media_model

from comments.models import CommentStatus
from home.blocks import MediaBlock
from home.models import (
    Article,
    FooterIndexPage,
    BannerIndexPage,
    BannerPage,
    HomePage,
    MiscellaneousIndexPage,
    OfflineContentIndexPage,
    SVGToPNGMap,
    Section,
    SectionIndexPage,
    SiteSettings,
)


class HomePageFactory(PageFactory):
    title = factory.Sequence(lambda n: f'homepage{n}')

    class Meta:
        model = HomePage

class SectionIndexFactory(PageFactory):
    title = 'Sections'

    class Meta:
        model = SectionIndexPage

class SectionFactory(PageFactory):
    title = factory.Sequence(lambda n: f'section{n}')
    lead_image = factory.SubFactory(ImageFactory)
    image_icon = factory.SubFactory(ImageFactory)

    class Meta:
        model = Section


class MediaFactory(CollectionMemberFactory):
    class Meta:
        model = get_media_model()

    title = factory.Sequence(lambda n: f'media{n}')
    file = factory.django.FileField()


class MediaBlockFactory(BlockFactory):
    media = factory.SubFactory(MediaFactory)

    class Meta:
        model = MediaBlock


class ArticleFactory(PageFactory):
    title = factory.Sequence(lambda n: f'article{n}')
    lead_image = factory.SubFactory(ImageFactory)
    commenting_status = CommentStatus.OPEN
    body = StreamFieldFactory(
        {
            "image": factory.SubFactory(ImageChooserBlockFactory),
            "media": factory.SubFactory(MediaBlockFactory),
        }
    )

    class Meta:
        model = Article


class MiscellaneousIndexPageFactory(PageFactory):
    title = factory.Sequence(lambda n: f'miscellaneous{n}')

    class Meta:
        model = MiscellaneousIndexPage


class OfflineContentIndexPageFactory(ArticleFactory):
    title = factory.Sequence(lambda n: f'offline-content-index{n}')

    class Meta:
        model = OfflineContentIndexPage


class SVGToPNGMapFactory(DjangoModelFactory):
    png_image_file = factory.LazyAttribute(
            lambda _: ContentFile(
                ImageField()._make_data(
                    {'width': 1024, 'height': 768}
                ), 'example.jpg'
            )
        )

    class Meta:
        model = SVGToPNGMap

class SiteSettingsFactory(DjangoModelFactory):
    class Meta:
        model = SiteSettings

class LocaleFactory(DjangoModelFactory):
    class Meta:
        model = Locale
        django_get_or_create = ('language_code',)

class FooterIndexPageFactory(PageFactory):
    title = 'Footers'

    class Meta:
        model = FooterIndexPage

class BannerIndexPageFactory(PageFactory):
    title = 'Banners'

    class Meta:
        model = BannerIndexPage

class BannerFactory(PageFactory):

    title = factory.Sequence(lambda n: f'banner{n}')
    banner_image = factory.SubFactory(ImageFactory)
    class Meta:
        model = BannerPage



# home/forms.py

from django.apps import apps
from wagtail.admin.forms import WagtailAdminPageForm

class SectionPageForm(WagtailAdminPageForm):

    def clean(self):
        cleaned_data = super().clean()

        if cleaned_data['show_progress_bar']:
            Section = apps.get_model('home', 'Section')
            if self.instance not in Section.get_progress_bar_eligible_sections():
                progress_bar_enabled_ancestor_title = self.instance.get_progress_bar_enabled_ancestor().title
                self.add_error(
                    'show_progress_bar',
                    f'This section is not eligible for showing the progress bar. '
                    f'Disable "Show Progress Bar" on "{progress_bar_enabled_ancestor_title}" section first.')

        return cleaned_data


# home/image_formats.py

from django.utils.translation import gettext_lazy as _
from django.conf import settings
from wagtail.images.formats import Format, register_image_format, unregister_image_format, get_image_formats


image_formats = get_image_formats()
for image_format in image_formats:
    unregister_image_format(image_format.name)


register_image_format(Format('fullwidth', _('Full width'), 'richtext-image full-width', f'width-{settings.IMAGE_SIZE_PRESET}'))
register_image_format(Format('left', _('Left-aligned'), 'richtext-image left', f'width-{settings.IMAGE_SIZE_PRESET}'))
register_image_format(Format('right', _('Right-aligned'), 'richtext-image right', f'width-{settings.IMAGE_SIZE_PRESET}'))
register_image_format(Format('original',
                             _('Original Size (Can use LARGE amounts of user data package, not recommended)'),
                             'richtext-image original', 'original'))


# home/mixins.py

from django.utils.functional import cached_property


class PageUtilsMixin:
    """
    This mixin contains the common properties/utilities shared across most children
    of wagtail.core.models.Page
    """

    @cached_property
    def parent_section(self):
        from .models import Section
        return Section.objects.parent_of(self).type(Section).first()

    @cached_property
    def is_first_content(self):
        from .models import Article, Section, PageLinkPage

        rv = False
        if isinstance(self, (Section, Article, PageLinkPage)):
            parent = self.get_parent().specific
            children = list(parent.get_children().live().specific().order_by('path'))
            index = children.index(self)
            if index == 0 and parent.larger_image_for_top_page_in_list_as_in_v1:
                rv = True

        return rv

    @cached_property
    def get_type(self):
        return self.__class__.__name__.lower()


class TitleIconMixin:
    """
    This mixin is used for duck-typing
    """

    def get_page(self):
        return self

    def get_icon(self):
        class Icon(object):
            url = ''
            is_svg_icon = False
            path = ''

            def __init__(self, url='', is_svg_icon=False):
                self.url = url
                self.is_svg_icon = is_svg_icon

            def __str__(self):
                return f'{self.url}'

        icon = Icon()
        if hasattr(self, 'icon') and self.icon:
            icon = Icon(self.icon.url, True)
            icon.path = self.icon.file.name
        elif hasattr(self, 'image_icon') and self.image_icon:
            icon = Icon(self.image_icon.get_rendition('fill-32x32').url, False)

        return icon


# home/models.py

import logging
import os

from django.conf import settings
from django.contrib.admin.utils import flatten
from django.contrib.auth import get_user_model
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError, ObjectDoesNotExist
from django.core.files.images import get_image_dimensions
from django.core.cache import cache
from django.db import models
from django.utils.deconstruct import deconstructible
from django.utils.encoding import force_str
from django.utils.translation import gettext_lazy as _
from modelcluster.contrib.taggit import ClusterTaggableManager
from iogt.settings.base import WAGTAIL_CONTENT_LANGUAGES
from modelcluster.fields import ParentalKey
from rest_framework import status
from taggit.models import TaggedItemBase
from wagtail.admin.edit_handlers import (
    FieldPanel,
    InlinePanel,
    MultiFieldPanel,
    ObjectList,
    PageChooserPanel,
    StreamFieldPanel,
    TabbedInterface
)
from wagtail.contrib.settings.models import BaseSetting
from wagtail.contrib.settings.registry import register_setting
from wagtail.core import blocks
from wagtail.core.fields import StreamField
from wagtail.core.models import Orderable, Page, Site, Locale
from wagtail.core.rich_text import get_text_for_indexing
from wagtail.images.blocks import ImageChooserBlock
from wagtail.images.edit_handlers import ImageChooserPanel
from wagtail.images.models import Image
from wagtail.search import index
from wagtailmarkdown.blocks import MarkdownBlock
from wagtailmenus.models import AbstractFlatMenuItem
from wagtailsvg.models import Svg
from wagtailsvg.edit_handlers import SvgChooserPanel

from messaging.blocks import ChatBotButtonBlock
from comments.models import CommentableMixin
from home.blocks import (
    MediaBlock, SocialMediaLinkBlock, SocialMediaShareButtonBlock, EmbeddedPollBlock, EmbeddedSurveyBlock,
    EmbeddedQuizBlock, PageButtonBlock, NumberedListBlock, RawHTMLBlock, ArticleBlock, DownloadButtonBlock,
)
from .forms import SectionPageForm
from .mixins import PageUtilsMixin, TitleIconMixin
from .utils.image import convert_svg_to_png_bytes
from .utils.progress_manager import ProgressManager
from home.utils import (
    collect_urls_from_streamfield,
    get_all_renditions_urls,
)
import iogt.iogt_globals as globals_

User = get_user_model()
logger = logging.getLogger(__name__)


class HomePage(Page, PageUtilsMixin, TitleIconMixin):
    parent_page_types = ['wagtailcore.page']
    template = 'home/home_page.html'
    show_in_menus_default = True

    home_featured_content = StreamField([
        ('page_button', PageButtonBlock()),
        ('embedded_poll', EmbeddedPollBlock()),
        ('embedded_survey', EmbeddedSurveyBlock()),
        ('embedded_quiz', EmbeddedQuizBlock()),
        ('article', ArticleBlock()),
        ('download', DownloadButtonBlock()),
    ], null=True, blank=True)

    content_panels = Page.content_panels + [
        MultiFieldPanel([
            InlinePanel('home_page_banners', label=_("Home Page Banner")),
        ], heading=_('Home Page Banners')),
        StreamFieldPanel('home_featured_content')
    ]

    def get_context(self, request):
        context = super().get_context(request)
        banners = []
        for home_page_banner in self.home_page_banners.select_related('banner_page', 'banner_page__banner_link_page').all():
            banner_page = home_page_banner.banner_page
            if banner_page.live and ((banner_page.banner_link_page and banner_page.banner_link_page.live) or
                                     banner_page.banner_link_page == None):
                banners.append(banner_page.specific)
        context['banners'] = banners
        return context

    @property
    def offline_urls(self):
        return [self.url] + collect_urls_from_streamfield(self.home_featured_content)


class FeaturedContent(Orderable):
    source = ParentalKey(Page, related_name='featured_content',
                         on_delete=models.CASCADE, blank=True)
    content = models.ForeignKey(Page, on_delete=models.CASCADE)

    panels = [
        PageChooserPanel('content'),
    ]


class HomePageBanner(Orderable):
    source = ParentalKey(Page, related_name='home_page_banners',
                         on_delete=models.CASCADE, blank=True)
    banner_page = models.ForeignKey('home.BannerPage', on_delete=models.CASCADE)

    panels = [
        PageChooserPanel('banner_page'),
    ]


class SectionTaggedItem(TaggedItemBase):
    """The through model between Section and Tag"""
    content_object = ParentalKey('Section', related_name='tagged_items',
                                 on_delete=models.CASCADE)


class ArticleTaggedItem(TaggedItemBase):
    """The through model between Article and Tag"""
    content_object = ParentalKey('Article', related_name='tagged_items',
                                 on_delete=models.CASCADE)


class SectionIndexPage(Page):
    parent_page_types = ['home.HomePage']
    subpage_types = ['home.Section']

    @classmethod
    def get_top_level_sections(cls):
        section_index_page = cls.objects.filter(locale=Locale.get_active()).first()
        if section_index_page:
            return section_index_page.get_children().live().specific()
        return cls.objects.none()


class Section(Page, PageUtilsMixin, CommentableMixin, TitleIconMixin):
    lead_image = models.ForeignKey(
        'wagtailimages.Image',
        on_delete=models.PROTECT,
        related_name='+',
        blank=True,
        null=True
    )
    icon = models.ForeignKey(
        Svg,
        related_name='+',
        null=True,
        blank=True,
        on_delete=models.PROTECT,
    )
    image_icon = models.ForeignKey(
        'wagtailimages.Image',
        on_delete=models.PROTECT,
        related_name='+',
        blank=True,
        null=True
    )
    background_color = models.CharField(
        max_length=8,
        blank=True,
        null=True,
    )
    font_color = models.CharField(
        max_length=8,
        blank=True,
        null=True,
    )
    body = StreamField([
        ('download', DownloadButtonBlock()),
    ], null=True, blank=True)

    tags = ClusterTaggableManager(through='SectionTaggedItem', blank=True)
    show_progress_bar = models.BooleanField(default=False)
    larger_image_for_top_page_in_list_as_in_v1 = models.BooleanField(default=False)

    show_in_menus_default = True

    promote_panels = Page.promote_panels + [
        MultiFieldPanel([FieldPanel("tags"), ], heading='Metadata'),
    ]

    content_panels = Page.content_panels + [
        ImageChooserPanel('lead_image'),
        SvgChooserPanel('icon'),
        ImageChooserPanel('image_icon'),
        FieldPanel('background_color'),
        FieldPanel('font_color'),
        FieldPanel('larger_image_for_top_page_in_list_as_in_v1'),
        MultiFieldPanel([
            InlinePanel('featured_content', max_num=1,
                        label=_("Featured Content")),
        ], heading=_('Featured Content')),
        StreamFieldPanel('body'),
    ]

    settings_panels = Page.settings_panels + [
        FieldPanel('show_progress_bar')
    ]

    edit_handler = TabbedInterface([
        ObjectList(content_panels, heading='Content'),
        ObjectList(promote_panels, heading='Promote'),
        ObjectList(settings_panels, heading='Settings'),
        ObjectList(CommentableMixin.comments_panels, heading='Comments')
    ])

    base_form_class = SectionPageForm

    def get_descendant_articles(self):
        return Article.objects.descendant_of(self).live().exact_type(Article)

    def get_progress_bar_enabled_ancestor(self):
        return Section.objects.ancestor_of(self, inclusive=True).exact_type(
            Section).live().filter(
            show_progress_bar=True).first()

    def get_user_progress_dict(self, request):
        progress_manager = ProgressManager(request)
        read_article_count, total_article_count = progress_manager.get_progress(self)
        return {
            'read': read_article_count,
            'total': total_article_count,
            'range_': list(range(total_article_count)) if total_article_count else 0,
            'width_': 100 / total_article_count if total_article_count else 0,
        }

    def is_complete(self, request):
        progress_manager = ProgressManager(request)
        return progress_manager.is_section_complete(self)

    def get_context(self, request):
        context = super().get_context(request)
        featured_content = self.featured_content.all().first()
        context['featured_content'] = featured_content.content.specific if featured_content and featured_content.content.live else None
        context['children'] = self.get_children().live().specific()
        context['user_progress'] = self.get_user_progress_dict(request)

        return context

    @staticmethod
    def get_progress_bar_eligible_sections():
        """
        Eligibility criteria:
        Sections whose ancestors don't have show_progress_bar=True are eligible to
        show progress bars.
        :return:e
        """
        progress_bar_sections = Section.objects.filter(show_progress_bar=True)
        all_descendants = [list(
            Section.objects.type(Section).descendant_of(section).values_list(
                'pk', flat=True)) for
            section in
            progress_bar_sections]
        all_descendants = set(flatten(all_descendants))

        return Section.objects.exclude(pk__in=all_descendants)

    @property
    def offline_urls(self):
        urls = [self.url] + collect_urls_from_streamfield(self.body)

        if self.lead_image:
            urls += get_all_renditions_urls(self.lead_image)

        if self.image_icon:
            urls += get_all_renditions_urls(self.image_icon)

        return urls

    class Meta:
        verbose_name = _("section")
        verbose_name_plural = _("sections")


class ArticleRecommendation(Orderable):
    source = ParentalKey('Article', related_name='recommended_articles',
                         on_delete=models.CASCADE, blank=True)
    article = models.ForeignKey('Article', on_delete=models.CASCADE)

    panels = [
        PageChooserPanel('article')
    ]


class AbstractArticle(Page, PageUtilsMixin, CommentableMixin, TitleIconMixin):
    lead_image = models.ForeignKey(
        'wagtailimages.Image',
        on_delete=models.PROTECT,
        related_name='+',
        blank=True,
        null=True
    )
    icon = models.ForeignKey(
        Svg,
        related_name='+',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
    )
    image_icon = models.ForeignKey(
        'wagtailimages.Image',
        on_delete=models.PROTECT,
        related_name='+',
        blank=True,
        null=True
    )
    index_page_description = models.TextField(null=True, blank=True)

    body = StreamField([
        ('heading', blocks.CharBlock(form_classname="full title", template='blocks/heading.html')),
        ('paragraph', blocks.RichTextBlock(features=settings.WAGTAIL_RICH_TEXT_FIELD_FEATURES)),
        ('markdown', MarkdownBlock(icon='code')),
        ('paragraph_v1_legacy', RawHTMLBlock(icon='code')),
        ('image', ImageChooserBlock(template='blocks/image.html')),
        ('list', blocks.ListBlock(MarkdownBlock(icon='code'))),
        ('numbered_list', NumberedListBlock(MarkdownBlock(icon='code'))),
        ('page_button', PageButtonBlock()),
        ('embedded_poll', EmbeddedPollBlock()),
        ('embedded_survey', EmbeddedSurveyBlock()),
        ('embedded_quiz', EmbeddedQuizBlock()),
        ('media', MediaBlock(icon='media')),
        ('chat_bot', ChatBotButtonBlock()),
        ('download', DownloadButtonBlock()),
    ])
    show_in_menus_default = True

    content_panels = Page.content_panels + [
        ImageChooserPanel('lead_image'),
        SvgChooserPanel('icon'),
        ImageChooserPanel('image_icon'),
        StreamFieldPanel('body'),
        FieldPanel('index_page_description'),
    ]

    edit_handler_list = [
        ObjectList(content_panels, heading='Content'),
        ObjectList(Page.settings_panels, heading='Settings'),
        ObjectList(CommentableMixin.comments_panels, heading='Comments')
    ]

    edit_handler = TabbedInterface(edit_handler_list)

    search_fields = Page.search_fields + [
        index.SearchField('get_heading_values', partial_match=True, boost=1),
        index.SearchField('get_paragraph_values', partial_match=True),
    ]

    def _get_child_block_values(self, block_type):
        searchable_content = []
        for block in self.body:
            if block.block_type == block_type:
                value = force_str(block.value)
                searchable_content.append(get_text_for_indexing(value))
        return searchable_content

    def get_heading_values(self):
        heading_values = self._get_child_block_values('heading')
        return '\n'.join(heading_values)

    def get_paragraph_values(self):
        paragraph_values = self._get_child_block_values('paragraph')
        return '\n'.join(paragraph_values)

    def get_progress_enabled_section(self):
        """
        Returning .first() will bypass any discrepancies in settings show_progress_bar=True
        for sections
        :return:
        """
        return Section.objects.ancestor_of(self).type(Section).filter(
            show_progress_bar=True).first()

    def get_context(self, request):
        context = super().get_context(request)

        progress_enabled_section = self.get_progress_enabled_section()
        if progress_enabled_section:
            context.update({
                'user_progress': progress_enabled_section.get_user_progress_dict(
                    request)
            })

        return context

    def description(self):
        for block in self.body:
            if block.block_type == 'paragraph':
                return block
        return ''

    def is_complete(self, request):
        progress_manager = ProgressManager(request)
        return progress_manager.is_article_complete(self)

    @property
    def top_level_section(self):
        return self.get_ancestors().filter(depth=4).first().specific

    @property
    def offline_urls(self):
        urls = [self.url] + collect_urls_from_streamfield(self.body)

        if self.lead_image:
            urls += get_all_renditions_urls(self.lead_image)

        if self.image_icon:
            urls += get_all_renditions_urls(self.image_icon)

        return urls

    class Meta:
        abstract = True
        verbose_name = _("article")
        verbose_name_plural = _("articles")


class Article(AbstractArticle):
    tags = ClusterTaggableManager(through='ArticleTaggedItem', blank=True)

    content_panels = AbstractArticle.content_panels + [
        MultiFieldPanel([
            InlinePanel('recommended_articles',
                        label=_("Recommended Articles")),
        ],
            heading='Recommended Content')
    ]

    promote_panels = AbstractArticle.promote_panels + [
        MultiFieldPanel([FieldPanel("tags"), ], heading='Metadata'),
    ]

    edit_handler_list = [
        ObjectList(content_panels, heading='Content'),
        ObjectList(Page.settings_panels, heading='Settings'),
        ObjectList(CommentableMixin.comments_panels, heading='Comments'),
        ObjectList(promote_panels, heading='Promote'),
    ]

    edit_handler = TabbedInterface(edit_handler_list)

    def get_context(self, request):
        context = super().get_context(request)
        context['recommended_articles'] = [
            recommended_article.article.specific
            for recommended_article in self.recommended_articles.all() if recommended_article.article.live
        ]

        return context

    def serve(self, request):
        response = super().serve(request)
        if response.status_code == status.HTTP_200_OK:
            User.record_article_read(request=request, article=self)
        return response


class MiscellaneousIndexPage(Page):
    parent_page_types = ['home.HomePage']
    subpage_types = ['home.OfflineContentIndexPage']


class OfflineContentIndexPage(AbstractArticle):
    template = 'home/article.html'
    parent_page_types = ['home.MiscellaneousIndexPage']
    subpage_types = []

    class Meta:
        verbose_name = 'Offline Content Index Page'


class BannerIndexPage(Page):
    parent_page_types = ['home.HomePage']
    subpage_types = ['home.BannerPage']


class BannerPage(Page, PageUtilsMixin):
    parent_page_types = ['home.BannerIndexPage']
    subpage_types = []

    banner_image = models.ForeignKey(
        'wagtailimages.Image',
        related_name='+',
        on_delete=models.PROTECT,
        null=True, blank=True,
        help_text=_('Image to display as the banner')
    )
    banner_link_page = models.ForeignKey(
        Page, null=True, blank=True, related_name='banners',
        on_delete=models.SET_NULL,
        help_text=_('Optional page to which the banner will link to'))

    content_panels = Page.content_panels + [
        ImageChooserPanel('banner_image'),
        PageChooserPanel('banner_link_page'),
    ]

    @property
    def offline_urls(self):
        return get_all_renditions_urls(self.banner_image)


class FooterIndexPage(Page):
    parent_page_types = ['home.HomePage']
    subpage_types = [
        'home.Section', 'home.Article', 'home.PageLinkPage', 'questionnaires.Poll',
        'questionnaires.Survey', 'questionnaires.Quiz',
    ]

    @classmethod
    def get_active_footers(cls):
        footer_index_page = cls.objects.filter(locale=Locale.get_active()).first()
        if footer_index_page:
            footers = footer_index_page.get_children().live().specific()
            return [footer for footer in footers if footer.get_page()]
        return cls.objects.none()

    def __str__(self):
        return self.title


class FooterPage(Article, TitleIconMixin):
    parent_page_types = []
    subpage_types = []
    template = 'home/article.html'

    class Meta:
        verbose_name = _("footer")
        verbose_name_plural = _("footers")


class PageLinkPage(Page, PageUtilsMixin, TitleIconMixin):
    parent_page_types = ['home.FooterIndexPage', 'home.Section']
    subpage_types = []
    show_in_menus_default = True

    icon = models.ForeignKey(
        Svg,
        related_name='+',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
    )
    image_icon = models.ForeignKey(
        'wagtailimages.Image',
        on_delete=models.PROTECT,
        related_name='+',
        blank=True,
        null=True
    )

    page = models.ForeignKey(Page, null=True, blank=True, related_name='page_link_pages', on_delete=models.PROTECT)
    external_link = models.URLField(null=True, blank=True)

    content_panels = Page.content_panels + [
        SvgChooserPanel('icon'),
        ImageChooserPanel('image_icon'),
        PageChooserPanel('page'),
        FieldPanel('external_link'),
    ]

    def get_page(self):
        return self.page.specific if self.page and self.page.live else self

    def get_icon(self):
        icon = super().get_icon()
        if not icon.url and self.page and self.page.live:
            icon = self.page.specific.get_icon()

        return icon

    def get_url(self, request=None, current_site=None):
        url = ''
        if self.page and self.page.live:
            url = self.page.specific.url
        elif self.external_link:
            url = self.external_link

        return url

    url = property(get_url)


@register_setting
class SiteSettings(BaseSetting):
    logo = models.ForeignKey(
        'wagtailimages.Image',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
        help_text="Upload an image file (.jpg, .png). The ideal size is 120px x 48px"
    )
    favicon = models.ForeignKey(
        'wagtailimages.Image',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
        help_text="Upload an image file (.jpg, .png). The ideal size is 40px x 40px"
    )
    apple_touch_icon = models.ForeignKey(
        'wagtailimages.Image',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
        help_text="Upload an image file (.jpg, .png) to be used as apple touch icon. "
                  "The ideal size is 120px x 120px"
    )
    show_only_translated_pages = models.BooleanField(
        default=False,
        help_text=_('When selecting this option, untranslated pages'
                    ' will not be visible to the front end user'
                    ' when viewing a child language of the site'))
    # TODO: GA, FB analytics should be global.
    fb_analytics_app_id = models.CharField(
        verbose_name=_('Facebook Analytics App ID'),
        max_length=25,
        null=True,
        blank=True,
        help_text=_(
            "The tracking ID to be used to view Facebook Analytics")
    )
    local_ga_tag_manager = models.CharField(
        verbose_name=_('Local GA Tag Manager'),
        max_length=255,
        null=True,
        blank=True,
        help_text=_(
            "Local GA Tag Manager tracking code (e.g GTM-XXX) to be used to "
            "view analytics on this site only")
    )
    global_ga_tag_manager = models.CharField(
        verbose_name=_('Global GA Tag Manager'),
        max_length=255,
        null=True,
        blank=True,
        help_text=_(
            "Global GA Tag Manager tracking code (e.g GTM-XXX) to be used"
            " to view analytics on more than one site globally")
    )
    local_ga_tracking_code = models.CharField(
        verbose_name=_('Local GA Tracking Code'),
        max_length=255,
        null=True,
        blank=True,
        help_text=_(
            "Local GA tracking code to be used to "
            "view analytics on this site only")
    )
    global_ga_tracking_code = models.CharField(
        verbose_name=_('Global GA Tracking Code'),
        max_length=255,
        null=True,
        blank=True,
        help_text=_(
            "Global GA tracking code to be used"
            " to view analytics on more than one site globally")
    )
    social_media_link = StreamField([
        ('social_media_link', SocialMediaLinkBlock()),
    ], null=True, blank=True)
    social_media_content_sharing_button = StreamField([
        ('social_media_content_sharing_button', SocialMediaShareButtonBlock()),
    ], null=True, blank=True)
    media_file_size_threshold = models.IntegerField(
        default=9437184,
        help_text=_('Show warning if uploaded media file size is greater than this in bytes. Default is 9 MB'))
    allow_anonymous_comment = models.BooleanField(default=False)
    registration_survey = models.ForeignKey('questionnaires.Survey', null=True,
                                            blank=True,
                                            on_delete=models.SET_NULL)
    opt_in_to_google_web_light = models.BooleanField(default=False)

    panels = [
        ImageChooserPanel('logo'),
        ImageChooserPanel('favicon'),
        ImageChooserPanel('apple_touch_icon'),
        MultiFieldPanel(
            [
                FieldPanel('show_only_translated_pages'),
            ],
            heading="Multi Language",
        ),
        MultiFieldPanel(
            [
                FieldPanel('fb_analytics_app_id'),
            ],
            heading="Facebook Analytics Settings",
        ),
        MultiFieldPanel(
            [
                FieldPanel('local_ga_tag_manager'),
                FieldPanel('global_ga_tag_manager'),
            ],
            heading="GA Tag Manager Settings",
        ),
        MultiFieldPanel(
            [
                FieldPanel('local_ga_tracking_code'),
                FieldPanel('global_ga_tracking_code'),
            ],
            heading="GA Tracking Code Settings",
        ),
        MultiFieldPanel(
            [
                MultiFieldPanel(
                    [
                        StreamFieldPanel('social_media_link'),
                    ],
                    heading="Social Media Footer Page", ),
            ],
            heading="Social Media Page Links", ),
        MultiFieldPanel(
            [
                MultiFieldPanel(
                    [
                        StreamFieldPanel('social_media_content_sharing_button'),
                    ],
                    heading="Social Media Content Sharing Buttons", ),
            ],
            heading="Social Media Content Sharing Buttons", ),
        MultiFieldPanel(
            [
                FieldPanel('media_file_size_threshold'),
            ],
            heading="Media File Size Threshold",
        ),
        MultiFieldPanel(
            [
                FieldPanel('allow_anonymous_comment'),
            ],
            heading="Allow Anonymous Comment",
        ),
        MultiFieldPanel(
            [
                PageChooserPanel('registration_survey'),
            ],
            heading="Registration Settings",
        ),
        MultiFieldPanel(
            [
                FieldPanel('opt_in_to_google_web_light'),
            ],
            heading="Opt in to Google web light",
        ),
    ]

    @classmethod
    def get_for_default_site(cls):
        default_site = Site.objects.filter(is_default_site=True).first()
        return cls.for_site(default_site)

    def __str__(self):
        return self.site.site_name

    class Meta:
        verbose_name = _('Site Settings')
        verbose_name_plural = _('Site Settings')


class IogtFlatMenuItem(AbstractFlatMenuItem, TitleIconMixin):
    menu = ParentalKey(
        'wagtailmenus.FlatMenu',
        on_delete=models.CASCADE,
        related_name="iogt_flat_menu_items",
    )
    link_url = models.CharField(
        verbose_name=_('link to a custom URL'),
        max_length=255,
        blank=True,
        null=True,
        help_text=_(
            'If you are linking back to a URL on your own IoGT site, be sure to remove the domain and everything '
            'before it. For example "http://sd.goodinternet.org/url/" should instead be "/url/".'
        ),
    )
    icon = models.ForeignKey(
        Svg,
        related_name='+',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        help_text=_('If Page link is a section page and icon is blank then the section icon will be used. '
                    'Specify an icon here to override this.')
    )
    image_icon = models.ForeignKey(
        'wagtailimages.Image',
        on_delete=models.PROTECT,
        related_name='+',
        blank=True,
        null=True
    )

    background_color = models.CharField(
        max_length=255,
        blank=True,
        null=True,
        help_text=_('The background color of the flat menu item on Desktop + Mobile')
    )

    font_color = models.CharField(
        max_length=255,
        blank=True,
        null=True,
        help_text=_('The font color of the flat menu item on Desktop + Mobile')
    )
    display_only_in_single_column_view = models.BooleanField(default=False)

    panels = [
        PageChooserPanel('link_page'),
        FieldPanel('link_url', classname='red-help-text'),
        FieldPanel('url_append'),
        FieldPanel('link_text'),
        FieldPanel('handle'),
        FieldPanel('allow_subnav'),
        SvgChooserPanel('icon'),
        ImageChooserPanel('image_icon'),
        FieldPanel('background_color'),
        FieldPanel('font_color'),
        FieldPanel('display_only_in_single_column_view'),
    ]

    def get_icon(self):
        icon = super().get_icon()
        if not icon.url and self.link_page and hasattr(self.link_page, 'get_icon'):
            icon = self.link_page.get_icon()

        return icon

    def get_background_color(self):
        theme_settings = globals_.theme_settings
        return self.background_color or theme_settings.navbar_background_color

    def get_font_color(self):
        theme_settings = globals_.theme_settings
        return self.font_color or theme_settings.navbar_font_color

    def get_single_column_view(self):
        return 'single-column-view' if self.display_only_in_single_column_view else ''


@deconstructible
class ImageValidator:
    def __init__(self, width=None, height=None):
        self.width = width
        self.height = height

    def __call__(self, image):
        img = Image.objects.get(id=image)

        w, h = get_image_dimensions(img.file)
        ext = os.path.splitext(img.filename)[1]
        valid_extensions = [".png"]

        if not ext.lower() in valid_extensions:
            raise ValidationError("Only .png images can be used")

        if self.width is not None and w != self.width:
            raise ValidationError(
                f"({img.filename} - Width: {w}px, Height: {h}px) - The width image must be {self.width}px"
            )

        if self.height is not None and h != self.height:
            raise ValidationError(
                f"({img.filename} - Height: {h}px, Width: {w}px) - The height image must be {self.height}px "
            )


class ManifestSettings(models.Model):
    name = models.CharField(
        max_length=255,
        verbose_name=_("Name"),
        help_text=_("Provide name that usually represents the name of the web application to user"),
    )
    short_name = models.CharField(
        max_length=255,
        verbose_name=_("Short name"),
        help_text=_("Provide short name to be displayed, if there is not enough space to display full name"),
    )
    scope = models.CharField(
        max_length=255,
        verbose_name=_("Scope"),
        help_text=_(
            "Provide navigation scope to limit what web pages can be viewed "
            "Example: 'https://www.iogt.com/<page-url>/' limits navigation "
            "to <page-url> of https://www.iogt.com:"
        ),
    )
    start_url = models.CharField(
        max_length=255,
        verbose_name=_("Start URL"),
        help_text=_(
            "Start URL is the preferred URL that should be loaded "
            "when the user launches the web application"
        ),
    )
    display = models.CharField(
        max_length=255,
        choices=[
            ('FULLSCREEN', 'fullscreen'),
            ('STANDALONE', 'standalone'),
            ('MINIMAL_UI', 'minimal-ui'),
            ('BROWSER', 'browser')
        ],
        verbose_name=_("Browser UI"),
        help_text=_(
            "Determines the preferred display mode for the website. The possible values are: "
            "'fullscreen', 'standalone', 'minimal-ui', 'browser'. A better choice would be to use standalone "
            "as it looks great on mobile as well. For further information refer to: "
            "https://developer.mozilla.org/en-US/docs/Web/Manifest/display#values"
        ),
    )
    background_color = models.CharField(
        max_length=10,
        verbose_name=_("Background color"),
        help_text=_(
            "Background color member defines a placeholder background color "
            "for the application page to display before its stylesheet is loaded. (example: #FFF)"
        ),
    )
    theme_color = models.CharField(
        max_length=10,
        verbose_name=_("Theme color"),
        help_text=_("Theme color defines the default theme color for the application (example: #493174)"),
    )
    description = models.CharField(
        max_length=500,
        verbose_name=_("Description"),
        help_text=_("Provide description for application"),
    )
    language = models.CharField(
        max_length=3,
        choices=WAGTAIL_CONTENT_LANGUAGES,
        default="en",
        verbose_name=_("Language"),
        help_text=_("Choose language"),
    )
    icon_96_96 = models.ForeignKey(
        "wagtailimages.Image",
        on_delete=models.SET_NULL,
        null=True,
        related_name="+",
        verbose_name=_("Icon 96x96"),
        help_text=_("Add PNG icon 96x96 px"),
        validators=[ImageValidator(width=96, height=96)],
    )
    icon_512_512 = models.ForeignKey(
        "wagtailimages.Image",
        on_delete=models.SET_NULL,
        null=True,
        related_name="+",
        verbose_name=_("Icon 512x512"),
        help_text=_("Add PNG icon 512x512 px"),
        validators=[ImageValidator(width=512, height=512)],
    )
    icon_192_192 = models.ForeignKey(
        "wagtailimages.Image",
        on_delete=models.SET_NULL,
        null=True,
        related_name="+",
        verbose_name=_("Icon 192x192 (maskable)"),
        help_text=_(
            "Add PNG icon 192x192 px (maskable image can be created using https://maskable.app/)"
        ),
        validators=[ImageValidator(width=192, height=192)],
    )

    panels = [
        MultiFieldPanel(
            [
                FieldPanel("language"),
            ],
            heading="Language",
        ),
        MultiFieldPanel(
            [
                FieldPanel("name"),
                FieldPanel("short_name"),
                FieldPanel("description"),
                FieldPanel("scope"),
                FieldPanel("start_url"),
                FieldPanel("display"),
            ],
            heading="Info",
        ),
        MultiFieldPanel(
            [
                FieldPanel("theme_color"),
                FieldPanel("background_color"),
            ],
            heading="Colors",
        ),
        MultiFieldPanel(
            [
                ImageChooserPanel("icon_96_96"),
                ImageChooserPanel("icon_512_512"),
                ImageChooserPanel("icon_192_192"),
            ],
            heading="Icons",
        ),
    ]

    def __str__(self):
        return f"Manifest for {self.name} - {self.language}"

    class Meta:
        unique_together = (
            "language",
            "scope",
        )
        verbose_name = "Manifest settings"
        verbose_name_plural = "Manifests settings"


@register_setting
class ThemeSettings(BaseSetting):
    global_background_color = models.CharField(
        null=True, blank=True, help_text='The background color of the website',
        max_length=8, default='#FFFFFF')

    header_background_color = models.CharField(
        null=True, blank=True, help_text='The background color of the header background as a HEX code', max_length=8,
        default='#FFFFFF')

    language_picker_background_color = models.CharField(
        null=True, blank=True, help_text='The background color of the language picker button as a HEX code',
        max_length=8, default='#FDD256')
    language_picker_font_color = models.CharField(
        null=True, blank=True, help_text='The font color of the language picker button as a HEX code',
        max_length=8, default='#303030')

    section_listing_questionnaire_background_color = models.CharField(
        null=True, blank=True, help_text='The background color of the Questionnaire in section listing as a HEX code',
        max_length=8, default='#f0f0f0')
    section_listing_questionnaire_font_color = models.CharField(
        null=True, blank=True, help_text='The font color of the Questionnaire in section listing as a HEX code',
        max_length=8, default='#444')

    section_card_font_color = models.CharField(
        null=True, blank=True, help_text='The background color of the sub section as a HEX code',
        max_length=8, default='#444')
    section_card_background_color = models.CharField(
        null=True, blank=True, help_text='The background color of the sub section as a HEX code',
        max_length=8, default='#ffffff')

    article_card_font_color = models.CharField(
        null=True, blank=True, help_text='The background color of the Embedded Article in Home > Featured Content'
                                         ' as a HEX code', max_length=8, default='#444')
    article_card_background_color = models.CharField(
        null=True, blank=True, help_text='The background color of the Embedded Article in Home > Featured Content'
                                         ' as a HEX code', max_length=8, default='#ffffff')

    primary_button_font_color = models.CharField(
        null=True, blank=True, help_text='The font/icon color of the primary button as a HEX code', max_length=8,
        default='#444')
    primary_button_background_color = models.CharField(
        null=True, blank=True, help_text='The background color of the primary button as a HEX code', max_length=8,
        default='#f0f0f0')

    navbar_background_color = models.CharField(
        null=True, blank=True, help_text='The background color of the navbar as a HEX code', max_length=8,
        default='#0094F4')
    navbar_font_color = models.CharField(
        null=True, blank=True, help_text='The font color of the navbar as a HEX code', max_length=8,
        default='#FFFFFF')


class V1ToV2ObjectMap(models.Model):
    v1_object_id = models.PositiveIntegerField()
    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = GenericForeignKey('content_type', 'object_id')
    extra = models.CharField(max_length=100, null=True, blank=True)

    def __str__(self):
        return f'{self.v1_object_id} -> {self.object_id}'

    @classmethod
    def get_v1_id(cls, klass, object_id, extra=None):
        content_type = ContentType.objects.get_for_model(klass)

        try:
            return cls.objects.get(content_type=content_type, object_id=object_id, extra=extra).v1_object_id
        except ObjectDoesNotExist:
            return None

    @classmethod
    def get_v2_obj(cls, klass, v1_object_id, extra=None):
        content_type = ContentType.objects.get_for_model(klass)

        try:
            return cls.objects.get(content_type=content_type, v1_object_id=v1_object_id, extra=extra).content_object
        except ObjectDoesNotExist:
            return None

    @classmethod
    def create_map(cls, content_object, v1_object_id, extra=None):
        content_type = ContentType.objects.get_for_model(type(content_object))
        obj, __ = cls.objects.get_or_create(content_type=content_type, object_id=content_object.pk,
                                            v1_object_id=v1_object_id, extra=extra)
        return obj


class SVGToPNGMap(models.Model):
    svg_path = models.TextField()
    fill_color = models.TextField(null=True)
    stroke_color = models.TextField(null=True)
    png_image_file = models.ImageField(upload_to='svg-to-png-maps/')

    @property
    def url(self):
        return self.png_image_file.url

    @classmethod
    def get_png_image(cls, svg_path, fill_color=None, stroke_color=None):
        try:
            cache_key = (
                svg_path,
                cls._db_color(fill_color),
                cls._db_color(stroke_color)
            )
            return cache.get('svg_to_png_map')[cache_key].png_image_file
        except (KeyError, TypeError):
            try:
                tmp = cls.objects.get(
                    svg_path=svg_path,
                    fill_color=cls._db_color(fill_color),
                    stroke_color=cls._db_color(stroke_color)
                )
                return tmp.png_image_file
            except Exception as e:
                logger.info(f"PNG not found, file={svg_path}, exception: {e}")
                try:
                    return cls.create(
                        svg_path,
                        fill_color,
                        stroke_color
                    ).png_image_file
                except Exception as e:
                    logger.error(f"Failed to create SVG to PNG, file={svg_path}, exception: {e}")
                    return None

    @classmethod
    def create(cls, svg_path, fill_color=None, stroke_color=None):
        png_image = convert_svg_to_png_bytes(
            svg_path,
            fill_color=fill_color,
            stroke_color=stroke_color,
            width=32
        )
        return cls.objects.create(
            svg_path=svg_path,
            fill_color=cls._db_color(fill_color),
            stroke_color=cls._db_color(stroke_color),
            png_image_file=png_image
        )

    @classmethod
    def _db_color(cls, color):
        return color if color else ''

    def __str__(self):
        return f'(svg={self.svg_path}, fill={self.fill_color}, stroke={self.stroke_color}, png={self.png_image_file})'

    class Meta:
        unique_together = ('svg_path', 'fill_color', 'stroke_color')


class V1PageURLToV2PageMap(models.Model):
    v1_page_url = models.TextField()
    v2_page = models.ForeignKey(Page, on_delete=models.CASCADE)

    def __str__(self):
        return f'{self.v1_page_url} -> {self.v2_page.id}'

    @classmethod
    def create_map(cls, url, page):
        obj, __ = cls.objects.get_or_create(v2_page=page, defaults={'v1_page_url': url})

        return obj

    @classmethod
    def get_page_or_none(cls, v1_page_url):
        # See https://github.com/unicef/iogt/issues/850 for more details on why /home/ is prepended
        urls_to_match = [
            f'/home{v1_page_url}',
            f'/home{v1_page_url}/'
        ]
        obj = cls.objects.filter(v1_page_url__in=urls_to_match).first()
        return obj.v2_page if obj else None


class LocaleDetail(models.Model):
    is_active = models.BooleanField(
        default=True,
        help_text=_('When active locale will be shown in the language selector'))
    is_main_language = models.BooleanField(
        default=False,
        help_text=_('When active locale will be used as default language for the site'))

    locale = models.OneToOneField('wagtailcore.Locale', related_name='locale_detail', on_delete=models.CASCADE)

    def clean(self):
        if self.is_main_language and LocaleDetail.objects.filter(is_main_language=True).exclude(id=self.id).exists():
            raise ValidationError(_('There is already a main language for this site'))

    def __str__(self):
        return f'{self.locale}'


# home/processors.py

from django.conf import settings
from django.urls import reverse


def commit_hash(request):
    return {'commit_hash': settings.COMMIT_HASH}


def show_footers(request):
    external_link_root = reverse('external-link')
    start_link = request.path.startswith(external_link_root)
    show_footers_ = True
    if start_link:
        show_footers_ = False
    return {'show_footers': show_footers_}


def jquery(request):
    return {
        'jquery': settings.JQUERY,
    }


# home/test.py

from urllib.parse import parse_qs

from django.conf import settings
from django.db.utils import IntegrityError
from django.template import Context
from django.test import TestCase, RequestFactory
from django.urls import reverse
from rest_framework import status
from wagtail.core.models import PageViewRestriction, Site
from unittest.mock import patch

from home.models import SVGToPNGMap
from home.templatetags.generic_components import google_analytics
from iogt_users.factories import UserFactory, GroupFactory
from home.factories import ArticleFactory, HomePageFactory
from wagtail_factories import SiteFactory


class PageViewGroupPermissionTests(TestCase):
    def setUp(self):
        self.user = UserFactory()

        Site.objects.all().delete()
        self.site = SiteFactory(site_name='IoGT', port=8000, is_default_site=True)
        self.home_page = HomePageFactory(parent=self.site.root_page)

        self.group_restricted_article = ArticleFactory(parent=self.home_page)
        view_restriction = PageViewRestriction.objects.create(
            page=self.group_restricted_article, restriction_type=PageViewRestriction.GROUPS)

        self.allowed_group = GroupFactory(name='Allowed group')
        view_restriction.groups.add(self.allowed_group)

    def test_group_limited_article_without_login_redirects_to_login_page(self):
        response = self.client.get(self.group_restricted_article.url)
        self.assertEqual(response.status_code, status.HTTP_302_FOUND)
        self.assertEqual(
            f'{reverse("account_login")}?next={self.group_restricted_article.url}', response.url)

    def test_group_limited_article_without_group_returns_403(self):
        self.client.login(username=self.user.username, password='test@123')
        response = self.client.get(self.group_restricted_article.url)
        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)

    def test_group_limited_article_with_group_user_returns_200(self):
        self.user.groups.add(self.allowed_group)
        self.client.login(username=self.user.username, password='test@123')
        response = self.client.get(self.group_restricted_article.url)
        self.assertEqual(response.status_code, status.HTTP_200_OK)


class SVGToPNGMapTests(TestCase):

    def setUp(self) -> None:
        self.svg_path = 'static/icons/search.svg'

    def test_create_png_if_not_found(self):
        png = SVGToPNGMap.get_png_image(self.svg_path)
        expected_path_regex = ''.join([
            settings.MEDIA_ROOT,
            r"/svg-to-png-maps/svg-to-png.*\.png"
        ])
        self.assertRegex(png.path, expected_path_regex)
        self.assertGreater(png.size, 0)

        png_2 = SVGToPNGMap.get_png_image(self.svg_path)
        self.assertEquals(png, png_2)

    def test_ignore_duplicates(self):
        png = SVGToPNGMap.get_png_image(self.svg_path)
        duplicate = {
            'svg_path': self.svg_path,
            'fill_color': None,
            'stroke_color': None,
            'png_image_file': png
        }
        SVGToPNGMap.objects.create(**duplicate)
        SVGToPNGMap.objects.create(**duplicate)
        count = SVGToPNGMap.objects.filter(
            svg_path=self.svg_path,
            fill_color=None,
            stroke_color=None
        ).count()
        self.assertEquals(count, 2)
        png_2 = SVGToPNGMap.get_png_image(self.svg_path, None, None)
        self.assertEquals(png, png_2)

    @patch.object(SVGToPNGMap, 'create')
    def test_get_png_must_not_fail(self, create):
            create.side_effect = Exception('boom')
            png = SVGToPNGMap.get_png_image(self.svg_path)
            self.assertIsNone(png)

    def test_uniqueness_unspecified_stroke_and_fill(self):
        SVGToPNGMap.create(self.svg_path)
        with self.assertRaises(IntegrityError):
            SVGToPNGMap.create(self.svg_path)

    def test_uniqueness_no_stroke_and_fill(self):
        SVGToPNGMap.create(self.svg_path, None, None)
        with self.assertRaises(IntegrityError):
            SVGToPNGMap.create(self.svg_path, None, None)

    def test_uniqueness_fill_no_stroke(self):
        SVGToPNGMap.create(
            self.svg_path,
            fill_color='#a1b2c3',
            stroke_color=None
        )
        with self.assertRaises(IntegrityError):
            SVGToPNGMap.create(
                self.svg_path,
                fill_color='#a1b2c3',
                stroke_color=None
            )

    def test_uniqueness_stroke_no_fill(self):
        SVGToPNGMap.create(self.svg_path, fill_color=None, stroke_color='#fff')
        with self.assertRaises(IntegrityError):
            SVGToPNGMap.create(
                self.svg_path,
                fill_color=None,
                stroke_color='#fff'
            )

    def test_uniqueness_stroke_and_fill(self):
        SVGToPNGMap.create(
            self.svg_path,
            fill_color='#555',
            stroke_color='#666'
        )
        with self.assertRaises(IntegrityError):
            SVGToPNGMap.create(
                self.svg_path,
                fill_color='#555',
                stroke_color='#666'
            )


class GoogleAnalyticsTagsTestCase(TestCase):
    def setUp(self):
        self.request_factory = RequestFactory()

    def test_query_param_without_value(self):
        request = self.request_factory.get('/en/?test')
        context = Context({'request': request})

        rendered_template = google_analytics(context, tracking_code='my-code')

        parsed_qs = parse_qs(rendered_template)
        self.assertEqual(parsed_qs['tracking_code'][0], "my-code")
        self.assertEqual(parsed_qs['p'][0], "/en/?test=")

    def test_query_param_with_value(self):
        request = self.request_factory.get('/en/?test=abc')
        context = Context({'request': request})

        rendered_template = google_analytics(context, tracking_code='my-code')

        parsed_qs = parse_qs(rendered_template)
        self.assertEqual(parsed_qs['tracking_code'][0], "my-code")
        self.assertEqual(parsed_qs['p'][0], "/en/?test=abc")

    def test_query_param_with_multiple_values_with_same_key(self):
        request = self.request_factory.get('/en/?test=abc&test=xyz')
        context = Context({'request': request})

        rendered_template = google_analytics(context, tracking_code='my-code')

        parsed_qs = parse_qs(rendered_template)
        self.assertEqual(parsed_qs['tracking_code'][0], "my-code")
        self.assertEqual(parsed_qs['p'][0], "/en/?test=abc&test=xyz")

    def test_query_param_with_utm(self):
        request = self.request_factory.get(
            '/en/?utm_content=content&utm_term=term&utm_source=source&utm_medium=medium&utm_campaign=campaign')
        context = Context({'request': request})

        rendered_template = google_analytics(context, tracking_code='my-code')

        parsed_qs = parse_qs(rendered_template)
        self.assertEqual(parsed_qs['tracking_code'][0], "my-code")
        self.assertEqual(parsed_qs['p'][0], "/en/")
        self.assertEqual(parsed_qs['utm_content'][0], "content")
        self.assertEqual(parsed_qs['utm_term'][0], "term")
        self.assertEqual(parsed_qs['utm_source'][0], "source")
        self.assertEqual(parsed_qs['utm_medium'][0], "medium")
        self.assertEqual(parsed_qs['utm_campaign'][0], "campaign")


# home/translatable_strings.py

translatable_strings = [
    'Username',
    'This field is required.',
    'survey',
    'poll',
    'Chat',
    'Cancel',
    'Send',
    'Reply',
    'Remove this comment?',
    'Remove',
    'Leave comment',
    'This comment has been removed.',
    'moderator',
    'Report',
    'sections',
    'articles',
    '%(counter)s of %(total)s questions',
    'Optional',
    'Select one',
    'Check all that apply',
    'Check if applies',
    'Comments',
    'Please log in or create your account',
    'New comments have been disabled for this page.',
    'Related Articles',
    'Log in',
    'Previous',
    'Page not found',
    'Sorry, this page could not be found.',
    'Internal server error',
    'Sorry, there seems to be an error. Please try again soon.',
    'Create your account',
    'quiz',
    'quizzes',
    'Your answer',
    'Next',
    'No limit',
    'Go to homepage',
    'Your results!',
    'Correct',
    'Incorrect',
    'Replay Quiz',
    'Correct answer',
    '1 result',
    'No search results were found for',
    'BACK',
    '© The Internet of Good Things',
    'Download',
    'Search the site...',
    'Log in / Create account',
    'You are leaving the <b>Internet of Good Things</b> to visit an external website and standard data charges by your network provider might apply',
    'Continue to external site',
    '4-digit PIN',
    'Old 4-digit PIN',
    'New 4-digit PIN',
    'Confirm new 4-digit PIN',
    'Hey, %(username)s',
    'Change Digital Pin',
    'Log out',
    'Personal Details',
    'Save changes',
    'Back',
    'section',
    'article',
    'Remember me',
    'You have signed out.',
    'Enter a valid username. This value may contain only letters, numbers, and @/./+/-/_ characters.',
    'You must type the same password each time.',
    'This password is too short. It must contain at least 4 characters.',
    'Please type your current password.',
    'Password successfully changed.',
    'Enter a valid date/time.',
    'Enter a valid date.',
    'Enter a valid URL.',
    'Enter a valid email address.',
    'Enter a valid integer.',
    'Ensure this value is greater than or equal to %(limit_value)s.',
    'Your comment',
    'Reset PIN',
    'Forgot PIN',
    'Your PIN has been changed successfully.',
    'I accept the Terms and Conditions.',
    '%(counter)s results',
    'Are you sure you want to log out?',
    'You have already completed this survey.',
    'Log in to participate',
    'You have read %(read)s out of %(total)s',
    'Profile',
    'This comment has been reported.',
    'Your browser does not support video playback.',
    'Your browser does not support audio playback.',
    "Don't display my username next to my comment",
    'Anonymous',
    'More',
    'Report this comment?',
    'It will be reviewed by the team and may be removed if it breaks our Platform Rules',
    'Display name',
    'Reported by %(comment_report_count)s user',
    'Reported by %(comment_report_count)s users',
    'Sorry, %(page_title)s is not available in %(language)s. If you want to keep browsing in %(language)s you can click below to return to the homepage.',
    'Username not available.',
    'Display name not available.',
    'Please complete the questions marked as required to continue',
    'You have already completed this quiz.',
    'You have already completed this poll.',
    'If you cannot view the above video, you can instead %(start_link)sdownload it%(end_link)s.',
    'If you cannot listen to the above audio, you can instead %(start_link)sdownload it%(end_link)s.',
    'You cannot submit when offline',
    'Search results for:',
    'Date must be in this (YYYY-MM-DD) format',
    'Datetime must be in this YYYY-MM-DDTHH:SS format',
    'Date must be in this (YYYY-MM-DD) format',
    'Datetime must be in this YYYY-MM-DDTHH:SS format',
    'The content you are trying to reach is not available. To see offline content click <a href="%(offline_content_index_page_url)s">here</a>',
    'Submit'
]

# home/views.py

from django.http import JsonResponse
from django.shortcuts import get_object_or_404, redirect
from django.utils import translation
from django.utils.translation import gettext_lazy as _
from django.views import View
from django.views.generic import TemplateView

from .models import ManifestSettings


class ServiceWorkerView(TemplateView):
    template_name = "sw.js"
    content_type = "application/javascript"
    name = "sw.js"


def get_manifest(request):
    language = translation.get_language()
    manifest = get_object_or_404(ManifestSettings, language=language)
    response = {
        "name": manifest.name,
        "short_name": manifest.short_name,
        "scope": manifest.scope,
        "background_color": manifest.background_color,
        "theme_color": manifest.theme_color,
        "description": manifest.description,
        "lang": manifest.language,
        "start_url": manifest.start_url,
        "display": manifest.display,
        "icons": [
            {
                "src": f"{manifest.icon_96_96.file.url}",
                "type": f"image/{manifest.icon_96_96.title.split('.')[1]}",
                "sizes": f"{manifest.icon_96_96.height}x{manifest.icon_96_96.width}",
            },
            {
                "src": f"{manifest.icon_512_512.file.url}",
                "type": f"image/{manifest.icon_512_512.title.split('.')[1]}",
                "sizes": f"{manifest.icon_512_512.height}x{manifest.icon_512_512.width}",
            },
            {
                "src": f"{manifest.icon_192_192.file.url}",
                "type": f"image/{manifest.icon_192_192.title.split('.')[1]}",
                "sizes": f"{manifest.icon_192_192.height}x{manifest.icon_192_192.width}",
                "purpose": "any maskable",
            },
        ],
    }

    http_response = JsonResponse(response)
    http_response['Content-Disposition'] = 'attachment; filename="manifest.json"'
    return http_response


class LogoutRedirectHackView(View):
    def get(self, request):
        return redirect(f'/{request.LANGUAGE_CODE}/')


# home/wagtail_hooks.py

from urllib.parse import urlparse

from django.contrib.admin import SimpleListFilter
from django.core.exceptions import PermissionDenied
from django.db.models import Q
from django.urls import resolve
from django.utils.translation import gettext_lazy as _
from translation_manager.models import TranslationEntry
from wagtail.contrib.modeladmin.options import ModelAdmin, modeladmin_register
from django.templatetags.static import static
from django.utils.html import format_html
from wagtail.core import hooks
from wagtail.core.models import Page
from wagtail.core.models import PageViewRestriction
from wagtail.admin import widgets as wagtailadmin_widgets


from home.models import FooterIndexPage, BannerIndexPage, Section, \
    SectionIndexPage, LocaleDetail
from home.translatable_strings import translatable_strings


@hooks.register('before_serve_page', order=-1)
def check_group(page, request, serve_args, serve_kwargs):
    if request.user.is_authenticated:
        for restriction in page.get_view_restrictions():
            if not restriction.accept_request(request):
                if restriction.restriction_type == PageViewRestriction.GROUPS:
                    current_user_groups = request.user.groups.all()
                    if not any(group in current_user_groups for group in
                               restriction.groups.all()):
                        raise PermissionDenied


@hooks.register('construct_explorer_page_queryset')
def sort_page_listing_by_path(parent_page, pages, request):
    if isinstance(parent_page, (
            BannerIndexPage, FooterIndexPage, Section, SectionIndexPage)):
        pages = pages.order_by('path')

    return pages


@hooks.register('construct_main_menu')
def rename_forms_menu_item(request, menu_items):
    for item in menu_items:
        if item.name == "forms":
            item.label = _("Form Data")
        if item.name == 'translations':
            item.url = f'{TranslationEntryAdmin().url_helper.get_action_url("index")}?limited=yes'


@hooks.register('construct_page_chooser_queryset')
def limit_page_chooser(pages, request):
    """
    For featured content page chooser panel in section start from current section
    otherwise don't change wagtail page queryset
    :param pages:
    :param request:
    :return: wagtail page queryset
    """
    current_path_match = resolve(request.path)
    if current_path_match.kwargs:
        # This is for the child pages access in page chooser panel
        page_id = current_path_match.kwargs.get('parent_page_id')
    else:
        # This is for the initial loading of children of current section
        parsed_referer_url = urlparse(request.META.get('HTTP_REFERER'))
        referer_path_match = resolve(parsed_referer_url.path)
        page_id = referer_path_match.kwargs.get('page_id')

    # If parent page is a section then return its children otherwise skip filtering
    if Section.objects.filter(id=page_id).exists():
        pages = Page.objects.get(id=page_id).get_children()

    return pages


@hooks.register('insert_global_admin_css', order=100)
def global_admin_css():
    return format_html('<link rel="stylesheet" href="{}">', static('css/global/admin.css'))


@hooks.register('insert_global_admin_js', order=100)
def global_admin_js():
    return format_html(
        '<script src="{}"></script>',
        static("js/admin.js")
    )


@hooks.register('register_page_listing_buttons')
def page_listing_buttons(page, page_perms, is_parent=False, next_url=None):
    # Using more menu's "Sort menu order" button from wagtail
    if is_parent:
        yield wagtailadmin_widgets.PageListingButton(
            _('Sort child pages'),
            '?ordering=ord',
            attrs={'title': _("Change ordering of child pages of '%(title)s'") % {'title': page.get_admin_display_title()}},
            priority=60
        )


class LimitedTranslatableStringsFilter(SimpleListFilter):
    title = _('limited translatable strings')
    parameter_name = 'limited'

    def lookups(self, request, model_admin):
        return (
            ('yes', 'Yes'),
            ('no', 'No'),
        )

    def queryset(self, request, queryset):
        if self.value() in ['yes']:
            translatable_string_filter = Q()
            for translatable_string in translatable_strings:
                translatable_string_filter |= Q(original__iexact=translatable_string)
            queryset = queryset.filter(translatable_string_filter)

        return queryset


class MissingTranslationsFilter(SimpleListFilter):
    title = _('missing translations')
    parameter_name = 'missing'

    def lookups(self, request, model_admin):
        return (
            ('yes', 'Yes'),
            ('no', 'No'),
        )

    def queryset(self, request, queryset):
        if self.value() == 'yes':
            queryset = queryset.filter(translation='')
        if self.value() == 'no':
            queryset = queryset.exclude(translation='')

        return queryset


class TranslationEntryAdmin(ModelAdmin):
    model = TranslationEntry
    menu_label = 'Translations'
    menu_icon = 'edit'
    list_display = ('original', 'language', 'translation',)
    list_filter = ('language', LimitedTranslatableStringsFilter, MissingTranslationsFilter)
    search_fields = ('original', 'translation',)
    index_template_name = 'modeladmin/translation_manager/translationentry/index.html'
    menu_order = 601


class LocaleDetailAdmin(ModelAdmin):
    model = LocaleDetail
    menu_label = 'Locale Detail'
    menu_icon = 'site'
    list_display = ('locale', 'is_active', 'is_main_language',)
    list_filter = ('is_active', 'is_main_language',)
    add_to_settings_menu = True
    menu_order = 700


modeladmin_register(TranslationEntryAdmin)
modeladmin_register(LocaleDetailAdmin)


# iogt/iogt_globals.py

site = None
site_settings = None
theme_settings = None
locale = None


# iogt/middleware.py

from urllib.parse import unquote

from django.conf import settings
from django.conf.urls.i18n import is_language_prefix_patterns_used
from django.core.cache import cache
from django.http.response import HttpResponsePermanentRedirect
from django.middleware.locale import LocaleMiddleware as DjangoLocaleMiddleware
from django.utils import translation
from django.utils.deprecation import MiddlewareMixin
from django.utils.translation import get_language_from_path, check_for_language, get_supported_language_variant
from django.utils.translation.trans_real import get_languages, parse_accept_lang_header, language_code_re
from translation_manager.models import TranslationEntry
from wagtail.contrib.redirects.middleware import RedirectMiddleware
from wagtail.core.models import Locale, Site

from home.models import SiteSettings, V1PageURLToV2PageMap, ThemeSettings, SVGToPNGMap
import iogt.iogt_globals as globals_


class CacheControlMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        response = self.get_response(request)

        if SiteSettings.for_request(request).opt_in_to_google_web_light:
            response['Cache-Control'] = 'no-transform'

        return response


class LocaleMiddleware(DjangoLocaleMiddleware):
    def _get_language_from_request(self, request, check_path=False):
        if check_path:
            lang_code = get_language_from_path(request.path_info)
            if lang_code is not None:
                return lang_code

        lang_code = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME)
        if lang_code is not None and lang_code in get_languages() and check_for_language(lang_code):
            return lang_code

        try:
            return get_supported_language_variant(lang_code)
        except LookupError:
            pass

        accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '')
        for accept_lang, unused in parse_accept_lang_header(accept):
            if accept_lang == '*':
                break

            if not language_code_re.search(accept_lang):
                continue

            try:
                lang_code = get_supported_language_variant(accept_lang)
            except LookupError:
                continue

            if Locale.objects.filter(language_code=lang_code, locale_detail__is_active=True).exists():
                return lang_code

        locale = Locale.objects.filter(locale_detail__is_active=True, locale_detail__is_main_language=True).first()
        return locale.language_code if locale else settings.LANGUAGE_CODE

    def process_request(self, request):
        urlconf = getattr(request, 'urlconf', settings.ROOT_URLCONF)
        i18n_patterns_used, __ = is_language_prefix_patterns_used(urlconf)
        language = self._get_language_from_request(request, check_path=i18n_patterns_used)
        translation.activate(language)
        request.LANGUAGE_CODE = translation.get_language()


class AdminLocaleMiddleware(MiddlewareMixin):
    """Ensures that the admin locale doesn't change with user selection"""

    def process_request(self, request):
        if request.path.startswith('/admin/') or request.path.startswith('/django-admin/'):
            translation.activate(settings.LANGUAGE_CODE)


class CustomRedirectMiddleware(RedirectMiddleware):
    def process_response(self, request, response):
        """
        This custom middleware is written to mitigate broken links from IOGT v1.
        See https://github.com/unicef/iogt/issues/850 for more details.
        """
        return_value = super().process_response(request, response)

        # If the page is not found by wagtail RedirectMiddleware, look for the page in V1PageURLToV2PageMap.
        # If you find the page, redirect the user to the new page.
        if return_value.status_code == 404:
            url = unquote(request.get_full_path())
            page = V1PageURLToV2PageMap.get_page_or_none(url)

            if page:
                return HttpResponsePermanentRedirect(page.url)

        return return_value


class GlobalDataMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        site = Site.objects.filter(is_default_site=True).first()
        locale = Locale.get_active()
        language_code = locale.language_code
        globals_.site = site
        globals_.site_settings = SiteSettings.for_request(request)
        globals_.theme_settings = ThemeSettings.for_site(site)
        globals_.locale = locale
        if not cache.get('svg_to_png_map'):
            map = {}
            for svg_to_png_map in SVGToPNGMap.objects.all():
                map.update({
                    (svg_to_png_map.svg_path, svg_to_png_map.fill_color, svg_to_png_map.stroke_color): svg_to_png_map,
                })
            cache.set('svg_to_png_map', map)
        if not cache.get(f'{language_code}_translation_map'):
            map = {}
            for translation_entry in TranslationEntry.objects.filter(language=language_code):
                map.update({
                    (translation_entry.original, language_code): translation_entry
                })
            cache.set(f'{language_code}_translation_map', map)

        response = self.get_response(request)

        return response


# iogt/patch.py

from django.templatetags import i18n
import iogt.iogt_globals as globals_


def _translate_node_render(self, context):
    from django.template.base import render_value_in_context
    from django.utils.safestring import SafeData
    from django.utils.safestring import mark_safe
    from django.core.cache import cache

    lookup = (self.filter_expression.var.literal or 
                self.filter_expression.var._resolve_lookup(context))

    try:
        translation_entry = cache.get(f'{globals_.locale.language_code}_translation_map')[
            (lookup, globals_.locale.language_code)]
    except (KeyError, TypeError):
        translation_entry = None

    if translation_entry and translation_entry.translation:
        return translation_entry.translation

    self.filter_expression.var.translate = not self.noop
    if self.message_context:
        self.filter_expression.var.message_context = (
            self.message_context.resolve(context))
    output = self.filter_expression.resolve(context)


    value = render_value_in_context(output, context)
    # Restore percent signs. Percent signs in template text are doubled
    # so they are not interpreted as string format flags.
    is_safe = isinstance(value, SafeData)
    value = value.replace('%%', '%')
    value = mark_safe(value) if is_safe else value
    if self.asvar:
        context[self.asvar] = value
        return ''
    else:
        return value


def _translate_block_node_render(self, context, nested=False):
    from django.core.cache import cache
    from django.template import TemplateSyntaxError
    from django.template.base import render_value_in_context
    from django.utils import translation

    if self.message_context:
        message_context = self.message_context.resolve(context)
    else:
        message_context = None
    # Update() works like a push(), so corresponding context.pop() is at
    # the end of function
    context.update({var: val.resolve(context) for var, val in self.extra_context.items()})
    singular, vars = self.render_token_list(self.singular)
    if self.plural and self.countervar and self.counter:
        count = self.counter.resolve(context)
        context[self.countervar] = count
        plural, plural_vars = self.render_token_list(self.plural)
        if message_context:
            result = translation.npgettext(message_context, singular,
                                           plural, count)
        else:
            result = translation.ngettext(singular, plural, count)
        vars.extend(plural_vars)
    else:
        if message_context:
            result = translation.pgettext(message_context, singular)
        else:
            result = translation.gettext(singular)
    default_value = context.template.engine.string_if_invalid

    try:
        translation_entry = cache.get(f'{globals_.locale.language_code}_translation_map')[
            (singular, globals_.locale.language_code)]
    except (KeyError, TypeError):
        translation_entry = None

    if translation_entry and translation_entry.translation:
        result = translation_entry.translation

    def render_value(key):
        if key in context:
            val = context[key]
        else:
            val = default_value % key if '%s' in default_value else default_value
        return render_value_in_context(val, context)

    data = {v: render_value(v) for v in vars}
    context.pop()
    try:
        result = result % data
    except (KeyError, ValueError):
        if nested:
            # Either string is malformed, or it's a bug
            raise TemplateSyntaxError(
                '%r is unable to format string returned by gettext: %r '
                'using %r' % (self.tag_name, result, data)
            )
        with translation.override(None):
            result = self.render(context, nested=True)
    if self.asvar:
        context[self.asvar] = result
        return ''
    else:
        return result


i18n.TranslateNode.render = _translate_node_render
i18n.BlockTranslateNode.render = _translate_block_node_render


# iogt/signals.py

from translation_manager.signals import post_save as translation_post_save

translation_post_save.connect(restart_server, sender=None)


# iogt/tests.py

import unittest

from django.test import TestCase
from django.urls import reverse
from rest_framework import status
from wagtail.core.models import Site
from wagtail.core.rich_text import RichText
from wagtail_factories import ImageFactory, SiteFactory

from home.factories import (
    ArticleFactory,
    HomePageFactory,
    LocaleFactory,
    MediaFactory,
    MiscellaneousIndexPageFactory,
    OfflineContentIndexPageFactory,
    SectionFactory,
    SVGToPNGMapFactory,
)
from iogt.utils import has_md5_hash


class PageTreeAPIViewTests(TestCase):
    def setUp(self):
        self.url_name = 'page_tree'
        Site.objects.all().delete()
        self.site = SiteFactory(site_name='IoGT', port=8000, is_default_site=True)
        self.en_locale = LocaleFactory(language_code='en')
        self.ar_locale = LocaleFactory(language_code='ar')
        self.en_home_page = HomePageFactory(parent=self.site.root_page, locale=self.en_locale)
        self.ar_home_page = HomePageFactory(parent=self.site.root_page, locale=self.ar_locale)
        self.en_miscellaneous_index_page = MiscellaneousIndexPageFactory(parent=self.en_home_page)
        self.ar_miscellaneous_index_page = MiscellaneousIndexPageFactory(parent=self.ar_home_page)
        self.en_offline_content_index_page = OfflineContentIndexPageFactory(parent=self.en_miscellaneous_index_page)
        self.ar_offline_content_index_page = OfflineContentIndexPageFactory(parent=self.ar_miscellaneous_index_page)
        self.section = SectionFactory(parent=self.en_home_page)
        self.article = ArticleFactory(
            parent=self.en_home_page,
            body=[("image", ImageFactory()),
                  ("media", MediaFactory(type='video')),
                  ("media", MediaFactory(type='audio'))])

    def test_root_page_is_returned(self):
        response = self.client.get(reverse(self.url_name, kwargs={'page_id': self.en_home_page.id}))

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertIn(self.en_home_page.url, response.data)

    def test_descendants_are_returned(self):
        response = self.client.get(reverse(self.url_name, kwargs={'page_id': self.en_home_page.id}))

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertIn(self.section.url, response.data)
        self.assertIn(self.article.url, response.data)

    def test_images_are_returned(self):
        svg_to_png_map = SVGToPNGMapFactory()
        section_lead_image = get_rendition(self.section.lead_image)
        article_lead_image = get_rendition(self.article.lead_image)
        body_image = get_rendition(self.article.body[0].value)

        response = self.client.get(
            reverse(self.url_name, kwargs={'page_id': self.en_home_page.id})
        )

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertIn(section_lead_image.url, response.data)
        self.assertIn(article_lead_image.url, response.data)
        self.assertIn(body_image.url, response.data)
        self.assertIn(svg_to_png_map.url, response.data)

    def test_media_are_returned(self):
        response = self.client.get(
            reverse(self.url_name, kwargs={'page_id': self.en_home_page.id})
        )

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertIn(self.article.body[1].value.url, response.data)
        self.assertIn(self.article.body[2].value.url, response.data)

    def test_image_urls_extracted_from_rich_text(self):
        image = ImageFactory()
        source = f'<embed embedtype="image" format="left" id="{image.id}"/>'
        ArticleFactory(
            parent=self.en_home_page,
            body=[('paragraph', RichText(source))]
        )
        response = self.client.get(
            reverse(self.url_name, kwargs={'page_id': self.en_home_page.id})
        )
        self.assertEqual(response.status_code, status.HTTP_200_OK)

        renditions = image.get_rendition_model().objects.filter(image_id=image.id)
        self.assertEquals(len(renditions), 1)
        self.assertIn(renditions[0].url, response.data)

    def test_static_assets_are_returned(self):
        response = self.client.get(reverse(self.url_name, kwargs={'page_id': self.en_home_page.id}))

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertTrue([item for item in response.data if item.startswith('/static/css/iogt')])
        self.assertTrue([item for item in response.data if item.startswith('/static/js/iogt')])
        self.assertTrue([item for item in response.data if item.startswith('/static/icons/search')])
        self.assertTrue([item for item in response.data if item.startswith('/static/fonts/memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgsg-1x4gaVQUwaEQbjA')])
        self.assertIn('/en/jsi18n/', response.data)
        self.assertIn('/ar/jsi18n/', response.data)

    def test_offline_content_not_found_pages_are_returned(self):
        response = self.client.get(reverse(self.url_name, kwargs={'page_id': self.en_home_page.id}))

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertIn('/en/offline-content-not-found/', response.data)
        self.assertIn('/ar/offline-content-not-found/', response.data)

    def test_offline_content_index_pages_are_returned(self):
        response = self.client.get(reverse(self.url_name, kwargs={'page_id': self.en_home_page.id}))

        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertIn(self.en_offline_content_index_page.url, response.data)
        self.assertIn(self.ar_offline_content_index_page.url, response.data)


class TestUtils(unittest.TestCase):
    def test_has_md5_hash_with_hash_values(self):
        with_md5_hash = [
            'iogt.c288d51a7c1f.css',
            'iogt.9bbc5bdef847.js',
            'search.6f2883e8e48b.svg',
            'memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgsg-1x4gaVQUwaEQbjA.186a2a0afcb7.woff',
        ]

        for value in with_md5_hash:
            self.assertTrue(has_md5_hash(value))

    def test_has_md5_hash_without_has_values(self):
        without_md5_hash = [
            'iogt.css',
            'iogt.js',
            'search.svg',
            'memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgsg-1x4gaVQUwaEQbjA.woff',
        ]

        for value in without_md5_hash:
            self.assertFalse(has_md5_hash(value))


def get_rendition(image):
    return image.get_rendition('fill-360x360')


# iogt/urls.py

from django.conf import settings
from django.conf.urls.i18n import i18n_patterns
from django.contrib import admin
from django.urls import include, path, re_path
from django.views.i18n import JavaScriptCatalog
from drf_yasg import openapi
from drf_yasg.views import get_schema_view
from rest_framework.authentication import SessionAuthentication
from rest_framework.permissions import IsAuthenticated
from wagtail.images.views.serve import ServeView
from webpush.views import save_info

from home.views import get_manifest, LogoutRedirectHackView
from iogt_users import urls as users_urls
from search import views as search_views
from wagtail.admin import urls as wagtailadmin_urls
from wagtail.core import urls as wagtail_urls
from wagtail.documents import urls as wagtaildocs_urls
from home import views as pwa_views
from wagtail_transfer import urls as wagtailtransfer_urls
from iogt.views import (
    TransitionPageView,
    SitemapAPIView,
    TranslationNotFoundPage,
    PageTreeAPIView,
    OfflineContentNotFoundPageView,
)


api_url_patterns = [
    path('api/v1/questionnaires/', include('questionnaires.api.v1.urls')),
]

schema_view = get_schema_view(
    openapi.Info(
        title="IoGT API",
        default_version='v1',
        description="IoGT APIs",
    ),
    url=settings.BASE_URL,
    public=True,
    patterns=api_url_patterns
)

urlpatterns = api_url_patterns + [
    path('django-admin/', admin.site.urls),
    path('admin/', include(wagtailadmin_urls)),
    path('documents/', include(wagtaildocs_urls)),
    *i18n_patterns(path('logout_hack_view', LogoutRedirectHackView.as_view(), name='logout_redirect')),
    *i18n_patterns(path('search/', search_views.search, name='search')),
    *i18n_patterns(path('users/', include(users_urls), name='users_urls')),
    *i18n_patterns(path('accounts/', include('allauth.urls'), name='allauth-urls')),
    *i18n_patterns(path('comments/', include('django_comments_xtd.urls'))),
    path(
        'sw.js',
        pwa_views.ServiceWorkerView.as_view(),
        name=pwa_views.ServiceWorkerView.name,
    ),
    *i18n_patterns(path("external-link/", TransitionPageView.as_view(), name="external-link")),
    *i18n_patterns(path("translation-not-found/", TranslationNotFoundPage.as_view(), name="translation-not-found")),
    *i18n_patterns(path("offline-content-not-found/", OfflineContentNotFoundPageView.as_view(), name="offline_content_not_found")),

    path('messaging/', include('messaging.urls'), name='messaging-urls'),
    path('wagtail-transfer/', include(wagtailtransfer_urls)),
    path('sitemap/', SitemapAPIView.as_view(), name='sitemap'),
    path("manifest.webmanifest", get_manifest, name="manifest"),
    *i18n_patterns(path('comments/', include('comments.urls'))),
    *i18n_patterns(path('jsi18n/', JavaScriptCatalog.as_view(), name='javascript-catalog')),
    path('health-check/', include('health_check.urls')),
    path('page-tree/<int:page_id>/', PageTreeAPIView.as_view(), name='page_tree'),
    path('api/docs/', schema_view.with_ui('swagger'), name='swagger'),
    path('webpush/subscribe/', save_info, name='save_webpush_info'),
]

if settings.DEBUG:
    import debug_toolbar
    from django.conf.urls.static import static
    from django.contrib.staticfiles.urls import staticfiles_urlpatterns

    # Serve static and media files from development server
    urlpatterns += staticfiles_urlpatterns()
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

    # add django debug toolbar links
    urlpatterns = urlpatterns + [path(r"__debug__/", include(debug_toolbar.urls))]

urlpatterns = urlpatterns + i18n_patterns(
    re_path(r'^images/([^/]*)/(\d*)/([^/]*)/[^/]*$', ServeView.as_view(), name='wagtailimages_serve'),
    re_path('djga/', include('google_analytics.urls')),
    path("", include(wagtail_urls)),
)


# iogt/utils.py

from django.conf import settings


def has_md5_hash(name):
    return bool(settings.HAS_MD5_HASH_REGEX.search(name))


# iogt/views.py

import logging
import os
from pathlib import Path, PurePosixPath

from django.conf import settings
from django.contrib.admin.utils import flatten
from django.shortcuts import get_object_or_404
from django.templatetags.static import static
from django.urls import reverse
from django.utils import translation
from django.views.generic import TemplateView
from rest_framework.response import Response
from rest_framework.views import APIView
from wagtail.core.models import Page, Locale
from wagtail.images.models import Rendition
from wagtailmedia.models import Media

from home.models import HomePage, Section, Article, SVGToPNGMap, FooterPage, OfflineContentIndexPage
from iogt.utils import has_md5_hash
from questionnaires.models import Poll, Survey, Quiz

logger = logging.getLogger(__name__)


class TransitionPageView(TemplateView):
    template_name = "transition-page.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data()
        context["next"] = self.request.GET.get("next", "/")
        context["prev"] = self.request.META.get("HTTP_REFERER", "/")
        return context


class TranslationNotFoundPage(TemplateView):
    template_name = "translation_not_found_page.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data()
        from home.models import HomePage
        origin_page = get_object_or_404(Page, pk=self.request.GET.get('page'))

        context['home_page'] = HomePage.objects.first().localized
        context['prev'] = origin_page.url

        context['page_title'] = origin_page.title
        return context


class SitemapAPIView(APIView):
    def get(self, request):
        home_page_urls = [p.url for p in HomePage.objects.live()]
        section_urls = [p.url for p in Section.objects.live()]
        article_urls = [p.url for p in Article.objects.live()]
        footer_urls = [p.url for p in FooterPage.objects.live()]
        poll_urls = [p.url for p in Poll.objects.live()]
        survey_urls = [p.url for p in Survey.objects.live()]
        quiz_urls = [p.url for p in Quiz.objects.live()]

        jsi18n_urls = []
        for locale in Locale.objects.all():
            jsi18n_urls.append(f'/{locale.language_code}/jsi18n/')

        image_urls = []
        for image in Rendition.objects.all():
            image_urls.append(f'{settings.MEDIA_URL}{image.file.name}')

        for image in SVGToPNGMap.objects.all():
            image_urls.append(f'{settings.MEDIA_URL}{image.png_image_file.name}')

        media_urls = []
        for media in Media.objects.all():
            media_urls.append(f'{settings.MEDIA_URL}{media.file.name}')
            if media.thumbnail:
                image_urls.append(f'{settings.MEDIA_URL}{media.thumbnail.name}')

        static_urls = []
        static_dirs = [
            {'name': 'css', 'extensions': ('.css',)},
            {'name': 'js', 'extensions': ('.js',)},
            {'name': 'fonts', 'extensions': ('.woff', '.woff2',)},
            {'name': 'icons', 'extensions': ('.svg',)},
        ]
        for static_dir in static_dirs:
            for root, dirs, files in os.walk(Path(settings.STATIC_ROOT) / static_dir['name']):
                for file in files:
                    if file.endswith(static_dir['extensions']):
                        static_urls.append(f'{settings.STATIC_URL}{root.split("/static/")[-1]}/{file}')

        sitemap = set(flatten(
            home_page_urls +
            section_urls +
            article_urls +
            footer_urls +
            poll_urls +
            survey_urls +
            quiz_urls +
            static_urls +
            image_urls +
            media_urls +
            jsi18n_urls
        ))
        return Response(sitemap)


class PageTreeAPIView(APIView):
    authentication_classes = ()
    permission_classes = ()

    def get(self, request, page_id):
        page = get_object_or_404(Page, id=page_id)
        pages = page.get_descendants(inclusive=True).live().specific()
        page_urls = []
        active_locale = Locale.get_active()
        for locale in Locale.objects.all():
            translation.activate(locale.language_code)
            page_urls.append(reverse('offline_content_not_found'))
            page_urls.append(reverse('javascript-catalog'))
            page = OfflineContentIndexPage.objects.filter(locale=locale).first()
            if page:
                page_urls.append(page.url)
        translation.activate(active_locale.language_code)

        no_impl = set()
        for page in pages:
            try:
                page_urls += page.offline_urls
            except AttributeError:
                no_impl.add(type(page).__name__)

        if no_impl:
            types = ", ".join(sorted(no_impl))
            logger.warn(
                f"Offline URLs could not be obtained for the following Page types: "
                f"{types}"
            )

        image_urls = []
        for svg_to_png_map in SVGToPNGMap.objects.all():
            image_urls.append(svg_to_png_map.url)

        static_urls = []
        static_dirs = [
            {'name': 'css', 'extensions': ('.css',)},
            {'name': 'js', 'extensions': ('.js',)},
            {'name': 'fonts', 'extensions': ('.woff', '.woff2',)},
            {'name': 'icons', 'extensions': ('.svg',)},
        ]
        for static_dir in static_dirs:
            for root, dirs, files in os.walk(Path(settings.STATIC_ROOT).joinpath(static_dir['name'])):
                for file in files:
                    if file.endswith(static_dir['extensions']):
                        try:
                            if not has_md5_hash(file):
                                static_urls.append(
                                    static(f'{PurePosixPath(root).relative_to(settings.STATIC_ROOT).joinpath(file)}'))
                        except Exception as e:
                            logger.exception(e)

        urls = set(flatten(
            page_urls +
            image_urls +
            static_urls
        ))
        return Response(urls)


class OfflineContentNotFoundPageView(TemplateView):
    template_name = "offline_content_not_found.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data()
        page = OfflineContentIndexPage.objects.filter(locale=Locale.get_active()).first()
        context["offline_content_index_page_url"] = (page and page.url) or ''
        return context


# iogt/wsgi.py

"""
WSGI config for iogt project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/3.1/howto/deployment/wsgi/
"""

import os

from django.core.wsgi import get_wsgi_application

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "iogt.settings.dev")

application = get_wsgi_application()


# iogt_users/adapters.py

from allauth.account.adapter import DefaultAccountAdapter


class AccountAdapter(DefaultAccountAdapter):
    def save_user(self, request, user, form, commit=True):
        user = super(AccountAdapter, self).save_user(request, user, form, commit)
        data = form.cleaned_data
        user.display_name = data.get('display_name') or data.get('username')
        user.first_name = data.get('first_name')
        user.last_name = data.get('last_name')
        user.terms_accepted = data.get('terms_accepted', False)
        user.save(update_fields=['display_name', 'first_name', 'last_name', 'terms_accepted'])
        return user


# iogt_users/admin.py

from django.contrib import admin
from django.contrib.auth.admin import UserAdmin

from .forms import CustomUserCreationForm, CustomUserChangeForm
from .models import User


class CustomUserAdmin(UserAdmin):
    add_form = CustomUserCreationForm
    form = CustomUserChangeForm
    model = User
    list_display = ('username', 'first_name', 'last_name', 'is_staff', 'is_active',)
    list_filter = ('is_staff', 'is_active',)
    fieldsets = (
        (None, {'fields': ('username', 'password', 'first_name', 'last_name')}),
        ('Permissions', {'fields': ('is_staff', 'is_active', 'groups', 'user_permissions')}),
        ('Other', {'fields': ('terms_accepted',
                              'has_filled_registration_survey')}),
    )
    add_fieldsets = (
        (None, {
            'classes': ('wide',),
            'fields': ('username', 'password1', 'password2', 'is_staff', 'is_active')}
        ),
    )
    search_fields = ('username',)
    ordering = ('username',)


admin.site.register(User, CustomUserAdmin)

# iogt_users/apps.py

from django.apps import AppConfig


class UsersConfig(AppConfig):
    name = 'iogt_users'


# iogt_users/factories.py

from django.contrib.auth import get_user_model
import factory
from django.contrib.auth.hashers import make_password
from django.contrib.auth.models import Group
from factory.django import DjangoModelFactory
from wagtail.core.models import GroupPagePermission


class UserFactory(DjangoModelFactory):
    first_name = 'John'
    last_name = 'Doe'
    username = factory.Sequence(lambda n: f'user{n}@example.com')
    password = factory.LazyFunction(lambda: make_password('test@123'))
    has_filled_registration_survey = True

    class Meta:
        model = get_user_model()


class AdminUserFactory(UserFactory):
    is_superuser = True

class GroupFactory(DjangoModelFactory):
    class Meta:
        model = Group


class GroupPagePermissionFactory(DjangoModelFactory):
    permission_type = 'edit'

    class Meta:
        model = GroupPagePermission


# iogt_users/fields.py

from allauth.account.forms import PasswordField
from django.forms import TextInput


class IogtPasswordField(PasswordField):

    def __init__(self, label, autocomplete, **kwargs):
        super().__init__(label=label, max_length=4, autocomplete=autocomplete,
                         widget=TextInput(attrs={'type': 'number'}), **kwargs)


# iogt_users/filters.py

from django.contrib.admin import SimpleListFilter
from django.contrib.auth.models import Group


class GroupsFilter(SimpleListFilter):
    title = "Groups"
    parameter_name = "groups"

    def lookups(self, request, model_admin):
        return [(group.id, group.name) for group in Group.objects.all()]

    def queryset(self, request, queryset):
        if self.value():
            return queryset.filter(groups__in=[self.value()])
        return queryset


# iogt_users/forms.py

from allauth.account.forms import SignupForm, ChangePasswordForm as BaseChangePasswordForm
from allauth.utils import set_form_field_order
from django import forms
from django.contrib.auth.forms import UserCreationForm, UserChangeForm
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
from wagtail.users.forms import UserEditForm as WagtailUserEditForm, \
    UserCreationForm as WagtailUserCreationForm

from .fields import IogtPasswordField
from .models import User


class AccountSignupForm(SignupForm):
    display_name = forms.CharField(
        label=_("Display name"),
        widget=forms.TextInput(
            attrs={"placeholder": _("Display name"),}
        ),
        required=False,
    )
    terms_accepted = forms.BooleanField(label=_('I accept the Terms and Conditions.'))

    field_order = [
        "username",
        "display_name",
        "password1",
        "password2",
        "terms_accepted",
    ]

    def __init__(self, *args, **kwargs):
        super(AccountSignupForm, self).__init__(*args, **kwargs)
        self.fields.pop('email')
        self.fields["password1"] = IogtPasswordField(label=_("4-digit PIN"), autocomplete="new-password")

        if 'password2' in self.fields:
            self.fields["password2"] = IogtPasswordField(label=_("4-digit PIN"), autocomplete="new-password")

        if hasattr(self, "field_order"):
            set_form_field_order(self, self.field_order)

    def clean_username(self):
        username = self.cleaned_data.get('username')
        if User.objects.filter(username__iexact=username):
            raise ValidationError(_('Username not available.'))
        return username

    def clean_displayname(self):
        display_name = self.cleaned_data.get('display_name')
        if User.objects.filter(display_name__iexact=display_name):
            raise ValidationError(_('Display name not available.'))
        return display_name


class ChangePasswordForm(BaseChangePasswordForm):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields["oldpassword"] = IogtPasswordField(label=_("Old 4-digit PIN"), autocomplete='old-password')
        self.fields["password1"] = IogtPasswordField(label=_("New 4-digit PIN"), autocomplete='old-password')
        self.fields["password2"] = IogtPasswordField(label=_("Confirm new 4-digit PIN"), autocomplete='old-password')


class CustomUserCreationForm(UserCreationForm):
    class Meta:
        model = User
        fields = ('username', 'display_name',)


class CustomUserChangeForm(UserChangeForm):
    class Meta:
        model = User
        fields = ('username', 'display_name', 'groups', 'user_permissions')


class WagtailAdminUserCreateForm(WagtailUserCreationForm):
    email = forms.EmailField(required=False, label='Email')
    display_name = forms.CharField(required=False, label='Display Name')
    first_name = forms.CharField(required=False, label='First Name')
    last_name = forms.CharField(required=False, label='Last Name')
    terms_accepted = forms.BooleanField(label=_('I accept the Terms and Conditions.'))

    def clean_username(self):
        username = self.cleaned_data['username']
        if User.objects.filter(username__iexact=username):
            raise ValidationError(_('A user with that username already exists.'))
        return username

    def clean_displayname(self):
        display_name = self.cleaned_data.get('display_name')
        if User.objects.filter(display_name__iexact=display_name):
            raise ValidationError(_('Display name not available.'))
        return display_name


class WagtailAdminUserEditForm(WagtailUserEditForm):
    email = forms.EmailField(required=False, label='Email')
    display_name = forms.CharField(required=False, label='Display Name')
    first_name = forms.CharField(required=False, label='First Name')
    last_name = forms.CharField(required=False, label='Last Name')

    terms_accepted = forms.BooleanField(label=_('I accept the Terms and Conditions.'))


# iogt_users/middlewares.py

from django.contrib import messages
from django.shortcuts import redirect
from django.urls import resolve, Resolver404, translate_url
from django.utils import translation
from django.utils.translation import gettext as _

from home.models import SiteSettings


class RegistrationSurveyRedirectMiddleware:
    """
    The purpose of this middleware is to make the registration survey form
    mandatory. See https://github.com/unicef/iogt/issues/113 for details
    """

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        if request.user.has_perm('wagtailadmin.access_admin'):
            return self.get_response(request)

        request_path = request.path_info
        registration_survey = SiteSettings.for_request(request).registration_survey
        if registration_survey:
            registration_survey = registration_survey.localized
            is_registration_survey_url = request_path == registration_survey.url
        else:
            is_registration_survey_url = False

        allowed_url_names = ['account_logout']
        language = translation.get_language()
        try:
            current_url = resolve(request_path).url_name
        except Resolver404:
            current_url = translate_url(request_path, language)
        is_url_allowed = current_url in allowed_url_names or is_registration_survey_url

        user = request.user
        is_registered_user = not user.is_anonymous

        language = translation.get_language()
        if (request_path.startswith(('/media/', f'/{language}/images/', f'/{language}/jsi18n/'))
                or is_url_allowed
                or not is_registered_user):
            return self.get_response(request)

        should_redirect_to_registration_survey = False
        if (not user.has_filled_registration_survey
                and registration_survey
                and registration_survey.has_required_fields()):
            should_redirect_to_registration_survey = True
            if user.has_viewed_registration_survey and not user.has_filled_registration_survey:
                messages.add_message(
                    request, messages.ERROR, _('Please complete the questions marked as required to continue'))

        elif (not user.has_filled_registration_survey
                and not user.has_viewed_registration_survey
                and registration_survey):
            should_redirect_to_registration_survey = True

        if should_redirect_to_registration_survey:
            user.has_viewed_registration_survey = True
            user.save(update_fields=['has_viewed_registration_survey'])
            return redirect(registration_survey.localized.url)

        return self.get_response(request)


# iogt_users/models.py

import base64

from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.auth.models import AbstractUser
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from rest_framework_simplejwt.tokens import RefreshToken


class User(AbstractUser):
    first_name = models.CharField('first name', max_length=150, null=True,
                                  blank=True)
    last_name = models.CharField('last name', max_length=150, null=True,
                                 blank=True)
    display_name = models.CharField('display name', max_length=255, null=True, blank=True)
    email = models.EmailField('email address', null=True, blank=True)
    terms_accepted = models.BooleanField(default=False)

    has_filled_registration_survey = models.BooleanField(default=False)
    has_viewed_registration_survey = models.BooleanField(default=False)

    @property
    def is_rapidpro_bot_user(self):
        return self.groups.filter(name=settings.RAPIDPRO_BOT_GROUP_NAME).exists()

    @classmethod
    def get_rapidpro_bot_auth_tokens(cls):
        users = cls.objects.filter(groups__name=settings.RAPIDPRO_BOT_GROUP_NAME)

        tokens = {}
        for user in users:
            tokens[user.username] = f'Bearer {RefreshToken.for_user(user).access_token}'

        return tokens

    read_articles = models.ManyToManyField(to='home.Article')

    @classmethod
    def record_article_read(cls, request, article):
        user = request.user
        if user.is_anonymous:
            read_articles = request.session.get('read_articles', [])
            if read_articles:
                if article.pk not in read_articles:
                    # https://code.djangoproject.com/wiki/NewbieMistakes#Appendingtoalistinsessiondoesntwork
                    read_articles = request.session['read_articles']
                    read_articles.append(article.pk)
                    request.session['read_articles'] = read_articles
            else:
                request.session['read_articles'] = [article.pk]
        else:
            if article.id and not user.read_articles.filter(id=article.id).exists():
                user.read_articles.add(article)

    def get_display_name(self):
        return self.display_name or self.username

    class Meta:
        ordering = ('id',)


class Profile(models.Model):
    user = models.OneToOneField(get_user_model(), on_delete=models.CASCADE)

    def __str__(self):
        return f'{self.user}\'s profile'


@receiver(post_save, sender=get_user_model())
def create_user_profile(sender, instance, created, **kwargs):
    if created:
        Profile.objects.create(user=instance)


# iogt_users/tests.py

from django.test import TestCase
from rest_framework import status
from wagtail.core.models import Site

from home.factories import SiteSettingsFactory, HomePageFactory
from iogt_users.factories import UserFactory, AdminUserFactory, GroupFactory
from wagtail_factories import SiteFactory, PageFactory
from questionnaires.factories import SurveyFactory
from django.contrib.auth.models import Permission


class PostRegistrationRedirectTests(TestCase):
    def setUp(self):
        self.user = UserFactory(has_filled_registration_survey=False)

        root_page = PageFactory(parent=None)
        self.home_page = HomePageFactory(parent=root_page)
        self.site = SiteFactory(hostname='testserver', port=80, root_page=self.home_page)

        self.registration_survey = SurveyFactory(parent=self.home_page)
        SiteSettingsFactory.create(registration_survey=self.registration_survey, site=self.site)

    def test_user_locked_out_without_filling_registration_survey_form(self):
        self.client.force_login(self.user)
        response = self.client.get(self.home_page.url)
        self.assertEqual(response.status_code, status.HTTP_302_FOUND)
        self.assertEqual(response.url, self.registration_survey.url)

    def test_anonymous_user_can_browse_public_urls(self):
        response = self.client.get(self.home_page.url)
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_superuser_should_not_complete_registration_survey(self):
        admin_user = AdminUserFactory(has_filled_registration_survey=False)

        self.client.force_login(admin_user)
        response = self.client.get(self.home_page.url)

        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_user_with_admin_access_should_not_complete_registration_survey(self):
        admin_access_permission = Permission.objects.get(codename='access_admin')
        group = GroupFactory()
        group.permissions.add(admin_access_permission)

        self.user.groups.add(group)
        self.client.force_login(self.user)
        response = self.client.get(self.home_page.url)

        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_user_should_not_redirect_after_filling_registration_survey(self):
        self.user.has_filled_registration_survey = True
        self.user.save()
        self.client.force_login(self.user)
        response = self.client.get(self.home_page.url)

        self.assertEqual(response.status_code, status.HTTP_200_OK)


# iogt_users/urls.py

from django.urls import path
from iogt_users.views import UserDetailEditView, UserDetailView

urlpatterns = [
    path('profile/', UserDetailView.as_view(), name='user_profile'),
    path('profile/edit', UserDetailEditView.as_view(), name='user_profile_edit'),
]

# iogt_users/utils.py

from wagtail.users.forms import standard_fields


def has_completed_registration_survey(user):
    # TODO: Check if survey exists when surveys go live. This method should always return
    #  True if there is no post-registration survey
    return user.has_filled_registration_survey



# iogt_users/views.py

from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.shortcuts import redirect
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.views.generic import UpdateView, TemplateView


@method_decorator(login_required, name='dispatch')
class UserDetailView(TemplateView):
    template_name = 'profile.html'

    def get_context_data(self, **kwargs):
        return {'user': self.request.user}


@method_decorator(login_required, name='dispatch')
class UserDetailEditView(UpdateView):
    model = User
    fields = ('first_name', 'last_name', 'email')
    template_name = 'profile_edit.html'

    def get_success_url(self):
        return reverse('user_profile_edit')

    def get_object(self, queryset=None):
        return self.request.user


# iogt_users/wagtail_hooks.py

from functools import cached_property

from django.conf import settings
from django.contrib.auth import get_user_model
from django.db.models import Prefetch
from django.urls import reverse
from django.utils import timezone
from wagtail.contrib.modeladmin.helpers import ButtonHelper
from wagtail.contrib.modeladmin.options import ModelAdmin, modeladmin_register

from home.models import SiteSettings
from iogt_users.filters import GroupsFilter
from questionnaires.models import UserSubmission


class UserButtonHelper(ButtonHelper):
    view_button_classnames = ["button-small", "icon", "icon-form"]

    def form_data_button(self, obj):
        text = "Form Data"
        return {
            "url": f'{reverse("form_data_per_user")}?user_id={obj.id}',
            "label": text,
            "classname": self.finalise_classname(self.view_button_classnames),
            "title": text,
        }

    def get_buttons_for_obj(self, obj, exclude=None, classnames_add=None, classnames_exclude=None):
        btns = super().get_buttons_for_obj(obj, exclude, classnames_add, classnames_exclude)
        btns.append(self.form_data_button(obj))
        return btns


class UsersExportAdmin(ModelAdmin):
    model = get_user_model()
    menu_label = 'Export/Import Users'
    menu_icon = 'user'
    list_display = ('username', 'display_name', 'date_joined', 'is_staff', 'is_active')
    list_filter = (GroupsFilter, 'date_joined', 'is_staff', 'is_active')
    form_fields_exclude = ('password', 'last_login', 'is_superuser', 'groups', 'user_permissions')
    search_fields = ('username',)
    list_export = ('username', 'display_name', 'first_name', 'last_name', 'email', 'is_staff', 'is_active',
                   'date_joined', 'terms_accepted', 'has_filled_registration_survey', 'registration_survey_response',)
    add_to_settings_menu = True
    list_per_page = 20
    menu_order = 601
    button_helper_class = UserButtonHelper

    def registration_survey_response(self, obj):
        user_submissions = obj.user_submissions

        return user_submissions[0].form_data if user_submissions else ''

    @cached_property
    def get_registration_survey_ids(self):
        ids = []
        registration_survey = SiteSettings.get_for_default_site().registration_survey
        if registration_survey:
            ids = registration_survey.get_translations(inclusive=True).values_list('id', flat=True)

        return ids

    @property
    def export_filename(self):
        return f'users_{timezone.now().strftime(settings.EXPORT_FILENAME_TIMESTAMP_FORMAT)}'

    def get_queryset(self, request):
        qs = super().get_queryset(request)

        ids = self.get_registration_survey_ids
        return qs.prefetch_related(
            Prefetch(
                'usersubmission_set',
                queryset=UserSubmission.objects.filter(page__pk__in=ids),
                to_attr='user_submissions'
            )
        )


modeladmin_register(UsersExportAdmin)


# matomo/apps.py

from django.apps import AppConfig


class MatomoConfig(AppConfig):
    name = 'matomo'


# messaging/admin.py

# -*- coding: utf-8 -*-
from django.contrib import admin

from .models import ChatbotChannel, Thread, UserThread, Message, Attachment


@admin.register(ChatbotChannel)
class ChatbotChannelAdmin(admin.ModelAdmin):
    list_display = ('id', 'display_name', 'request_url')


@admin.register(Thread)
class ThreadAdmin(admin.ModelAdmin):
    list_display = ('id', 'last_message_at', 'subject', 'uuid', 'chatbot')
    list_filter = ('last_message_at', 'chatbot')
    raw_id_fields = ('users',)


@admin.register(UserThread)
class UserThreadAdmin(admin.ModelAdmin):
    list_display = ('id', 'is_active', 'is_read', 'thread', 'user')
    list_filter = ('is_active', 'is_read')


class AttachmentInlineAdmin(admin.TabularInline):
    model = Message.attachments.through


@admin.register(Message)
class MessageAdmin(admin.ModelAdmin):
    list_display = (
        'id',
        'quick_replies',
        'rapidpro_message_id',
        'sent_at',
        'text',
        'thread',
        'sender',
    )
    list_filter = ('sent_at',)
    inlines = (AttachmentInlineAdmin,)
    exclude = ('attachments', )


@admin.register(Attachment)
class AttachmentAdmin(admin.ModelAdmin):
    list_display = (
        'id',
        'created',
        'modified',
        'external_link',
        'file',
    )
    list_filter = ('created', 'modified')


# messaging/apps.py

from django.apps import AppConfig


class MessagingConfig(AppConfig):
    name = 'messaging'



# messaging/blocks.py

from django import forms
from wagtail.core import blocks

from .models import ChatbotChannel


class ChatBotChannelChooserBlock(blocks.ChooserBlock):
    target_model = ChatbotChannel
    widget = forms.Select


class ChatBotButtonBlock(blocks.StructBlock):
    subject = blocks.CharBlock()
    button_text = blocks.CharBlock()
    trigger_string = blocks.CharBlock()
    channel = ChatBotChannelChooserBlock()

    class Meta:
        icon = 'code'
        template = 'messaging/blocks/chatbot_button.html'


# messaging/chat.py

import logging

from django.contrib.auth import get_user_model
from django.core.exceptions import ValidationError
from django.core.validators import URLValidator
from django.utils import timezone
from webpush import send_user_notification

from .models import Message, Thread, UserThread
from messaging.rapidpro_client import RapidProClient

User = get_user_model()
logger = logging.getLogger(__name__)


class ChatManager:
    def __init__(self, thread):
        if thread is None:
            raise Exception('No thread found.')
        self.thread = thread

    def _record_message_in_database(self, sender, rapidpro_message_id, text, quick_replies):
        # Messages sent from User to RapidPro server don't have rapidpro_message_id
        from_rapidpro_server = bool(rapidpro_message_id)

        if from_rapidpro_server:
            message, created = Message.objects.get_or_create(rapidpro_message_id=rapidpro_message_id, defaults={
                'sender': sender,
                'text': text,
                'quick_replies': quick_replies,
                'thread': self.thread,
            })

            if not created:
                # If the message already exists, we concatenate the newly received
                # messages with the existing message. An assumption here is that
                # messages will always be received in the correct order. This should
                # be confirmed with RapidPro
                message.text = f'{message.text}{text}'
                message.save(update_fields=['text'])
            else:
                payload = {
                    'head': 'New Message!',
                    'body': 'Click here to view it.',
                    'url': self.thread.get_absolute_url(),
                }
                for user in self.thread.users.all():
                    try:
                        send_user_notification(user=user, payload=payload, ttl=1000)
                    except Exception as e:
                        logger.exception(e)

        else:
            Message.objects.create(
                rapidpro_message_id=rapidpro_message_id, sender=sender, text=text, thread=self.thread,
                quick_replies=quick_replies)

        self.thread.last_message_at = timezone.now()
        self.thread.save(update_fields=['last_message_at'])

    def _handle_attachments(self, message):
        cleaned_text, attachment_links = ChatManager._parse_rapidpro_message(message.text)
        message.text = cleaned_text
        message.update_or_create_attachments(attachment_links)
        message.save()

    @staticmethod
    def _parse_rapidpro_message(message_text):
        message_parts = message_text.split('\n')
        attachments = []
        cleaned_message = ""
        for message in reversed(message_parts):
            validator = URLValidator()
            try:
                validator(message)
                attachments.append(message)
            except ValidationError:
                cleaned_message = f'{message}{cleaned_message}'
        return cleaned_message, attachments

    def record_reply(self, text, sender, rapidpro_message_id=None, quick_replies=None, mark_unread=True):
        if quick_replies is None:
            quick_replies = []
        if not sender.is_rapidpro_bot_user:
            client = RapidProClient(self.thread)
            client.send_reply(text)

        self._record_message_in_database(
            sender=sender, rapidpro_message_id=rapidpro_message_id, text=text, quick_replies=quick_replies)

        if mark_unread:
            self.thread.mark_unread(sender)

    @staticmethod
    def initiate_thread(sender, recipients, chatbot, subject, text):
        sender_thread = UserThread.objects.filter(user=sender, thread__chatbot=chatbot).first()

        if sender_thread:
            thread = sender_thread.thread
        else:
            thread = Thread.objects.create(subject=subject, chatbot=chatbot)

            user_threads = []
            for user in recipients + [sender]:
                user_threads.append(UserThread(user=user, thread=thread))
            UserThread.objects.bulk_create(user_threads)

        chat_manager = ChatManager(thread)
        chat_manager.record_reply(sender=sender, text=text)

        thread.mark_unread(sender)
        return thread


# messaging/context_processors.py

from .models import Thread


def user_messages(request):
    context = {}
    if request.user.is_authenticated:
        context["inbox_threads"] = Thread.thread_objects.of_user(request.user).inbox()
        context["unread_threads"] = Thread.thread_objects.of_user(request.user).unread()
    return context


# messaging/factories.py

import factory
from factory.django import DjangoModelFactory


class ChatbotFactory(DjangoModelFactory):
    class Meta:
        model = 'messaging.ChatbotChannel'


class ThreadFactory(DjangoModelFactory):
    chatbot = factory.SubFactory(ChatbotFactory)
    subject = factory.Sequence(lambda n: f'Subject:0{n}')


    class Meta:
        model = 'messaging.Thread'


# messaging/forms.py

from django import forms
from django.contrib.auth import get_user_model

from .hooks import hookset
from .models import Message, ChatbotChannel, Thread

User = get_user_model()


class UserModelChoiceField(forms.ModelChoiceField):

    def label_from_instance(self, obj):
        return hookset.display_name(obj)


class UserModelMultipleChoiceField(forms.ModelMultipleChoiceField):

    def label_from_instance(self, obj):
        return hookset.display_name(obj)


class ChatbotChannelModelChoiceField(forms.ModelChoiceField):

    def label_from_instance(self, obj):
        return obj.display_name


class ChatbotChannelModelMultipleChoiceField(forms.ModelMultipleChoiceField):

    def label_from_instance(self, obj):
        return obj.display_name


class NewMessageForm(forms.Form):
    subject = forms.CharField()
    chatbot = ChatbotChannelModelChoiceField(queryset=ChatbotChannel.objects.all())
    text = forms.CharField(widget=forms.Textarea)


class NewMessageFormMultiple(forms.ModelForm):
    subject = forms.CharField()
    to_user = UserModelMultipleChoiceField(get_user_model().objects.none())
    chatbot = ChatbotChannelModelChoiceField(ChatbotChannel.objects.none())
    content = forms.CharField(widget=forms.Textarea)

    def __init__(self, *args, **kwargs):
        self.user = kwargs.pop("user")
        super().__init__(*args, **kwargs)
        self.fields["to_user"].queryset = hookset.get_user_choices(self.user)
        self.fields["chatbot"].queryset = ChatbotChannel.objects.all()
        if self.initial.get("to_user") is not None:
            qs = self.fields["to_user"].queryset.filter(pk__in=self.initial["to_user"])
            self.fields["to_user"].queryset = qs

    def save(self, commit=True):
        data = self.cleaned_data
        return Message.new_message(
            self.user, data["to_user"], data["chatbot"], data["subject"], data["content"]
        )

    class Meta:
        model = Message
        fields = ["to_user", "chatbot", "subject", "content"]


class MessageReplyForm(forms.Form):
    text = forms.CharField(widget=forms.Textarea)
    thread = forms.ModelChoiceField(queryset=Thread.objects.all(), widget=forms.HiddenInput)
    user = forms.ModelChoiceField(queryset=User.objects.all(), widget=forms.HiddenInput)


# messaging/hooks.py

from django.contrib.auth import get_user_model


class DefaultHookSet:

    def display_name(self, user):
        return str(user)

    def get_user_choices(self, user):
        return get_user_model().objects.exclude(id=user.id)


hookset = DefaultHookSet()


# messaging/models.py

import logging
import uuid
from io import BytesIO

import requests
from PIL import Image as PILImage, UnidentifiedImageError
from django.contrib.auth import get_user_model
from django.core.files import File
from django.core.validators import URLValidator
from django.db import models
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
from django_extensions.db.models import TimeStampedModel
from rest_framework import status
from django.core.exceptions import ValidationError
from wagtail.images.models import Image

from .querysets import ThreadQuerySet

logger = logging.getLogger(__name__)


class ChatbotChannel(models.Model):
    display_name = models.CharField(
        max_length=80, help_text=_('Name for the bot that the user will see when interacting with it'))
    request_url = models.URLField(
        max_length=200, help_text=_('To set up a chatbot channel on your RapidPro server and get a request URL, '
                                    'follow the steps outline in the Section "Setting up a Chatbot channel" '
                                    'here: https://github.com/unicef/iogt/blob/develop/messaging/README.md'))

    def __str__(self):
        return f"{self.display_name}, {self.request_url}"


class Thread(models.Model):
    last_message_at = models.DateTimeField(null=True, editable=False, default=None)
    subject = models.CharField(max_length=150)
    uuid = models.UUIDField(default=uuid.uuid4, editable=False)

    chatbot = models.ForeignKey('ChatbotChannel', on_delete=models.PROTECT)
    users = models.ManyToManyField(get_user_model(), through="UserThread")

    objects = models.Manager()
    thread_objects = ThreadQuerySet.as_manager()

    def get_renderable_messages(self):
        for message in self.messages.filter(is_post_processed=False):
            message.post_process()

        return self.messages.order_by('-sent_at')


    @property
    def latest_message(self):
        return self.messages.order_by('-sent_at').first()

    def mark_read(self, user):
        self.user_threads.filter(user=user).update(is_read=True)

    def mark_unread(self, sender=None):
        """
        Mark all related UserThread(s) unread
        """
        if sender:
            self.user_threads.exclude(user=sender).update(is_read=False)
            self.user_threads.filter(user=sender).update(is_read=True)
        else:
            self.user_threads.update(is_read=False)

    def __str__(self):
        return f"{self.subject}: {self.chatbot.display_name} {', '.join([str(user) for user in self.users.all()])}"

    def get_absolute_url(self):
        return reverse("messaging:thread", args=[self.pk])


class UserThread(models.Model):
    is_active = models.BooleanField(default=True)
    is_read = models.BooleanField(default=False)

    thread = models.ForeignKey('Thread', related_name='user_threads', on_delete=models.CASCADE)
    user = models.ForeignKey(get_user_model(), on_delete=models.CASCADE)

    @classmethod
    def get_user_inbox(cls, user):
        return cls.objects.filter(user=user, is_active=True).order_by(
            '-thread__last_message_at')


class Message(models.Model):
    # Quick replies are encoded as a json string
    quick_replies = models.JSONField(default=list)
    # If sent from RapidPro, the ID the message has in RapidPro.
    rapidpro_message_id = models.BigIntegerField(null=True)
    sent_at = models.DateTimeField(auto_now_add=True)
    text = models.TextField()

    thread = models.ForeignKey('Thread', related_name="messages", on_delete=models.CASCADE)
    sender = models.ForeignKey(get_user_model(), null=True, related_name="sent_messages", on_delete=models.CASCADE)

    is_post_processed = models.BooleanField(default=False)

    attachments = models.ManyToManyField('Attachment', blank=True)

    def _parse_attachments(self):
        message_parts = self.text.split('\n')
        attachments = []
        cleaned_message = ""
        for message in reversed(message_parts):
            validator = URLValidator()
            try:
                validator(message)
                attachments.append(message)
            except ValidationError:
                cleaned_message = f'{message}{cleaned_message}'
        return cleaned_message, attachments

    def process_attachments(self, attachment_links):
        for link in attachment_links:
            if not self.attachments.filter(external_link=link).exists():
                attachment, created = Attachment.objects.get_or_create(external_link=link)
                if not attachment.file:
                    attachment.download_external_file()
                self.attachments.add(attachment)

    def post_process(self):
        if self.is_post_processed:
            return
        text, attachments = self._parse_attachments()
        self.text = text
        self.process_attachments(attachments)
        self.is_post_processed = True
        self.save()

    def update_or_create_attachments(self, attachment_links):
        for link in attachment_links:
            if not self.attachments.filter(external_link=link).exists():
                attachment, created = Attachment.objects.get_or_create(external_link=link)
                if not attachment.file:
                    attachment.download_external_file()
                self.attachments.add(attachment)

    class Meta:
        ordering = ("sent_at",)

    def get_absolute_url(self):
        return self.thread.get_absolute_url()


class Attachment(TimeStampedModel):
    external_link = models.URLField()
    file = models.FileField(null=True, blank=True)
    image = models.ForeignKey(to=Image, on_delete=models.SET_NULL, null=True, blank=True)

    @staticmethod
    def _verify_image(content):
        try:
            image = PILImage.open(BytesIO(content))
            image.verify()
            image.close()
            return True
        except UnidentifiedImageError:
            return False

    def download_external_file(self):
        response = requests.get(self.external_link, allow_redirects=True)

        if response.status_code == status.HTTP_200_OK:
            filename = self.external_link.split('/')[-1]
            file = File(BytesIO(response.content), name=filename)

            if Attachment._verify_image(response.content):
                self.image = Image.objects.create(file=file)
            else:
                self.file = file
            self.save()

    def __str__(self):
        return f'Attachment #{self.pk}'


# messaging/processors.py

from django.conf import settings


def add_vapid_public_key(request):
    return {
        'vapid_public_key': settings.WEBPUSH_SETTINGS.get('VAPID_PUBLIC_KEY'),
    }


# messaging/querysets.py

from django.db.models import QuerySet


class ThreadQuerySet(QuerySet):
    def of_user(self, user):
        return self.filter(user_threads__user=user)

    def inbox(self):
        return self.filter(user_threads__is_active=True)

    def deleted(self):
        return self.filter(user_threads__is_active=False)

    def unread(self):
        return self.filter(
            user_threads__is_active=True,
            user_threads__is_read=False
        )

    def order_by_latest(self):
        return self.filter(last_message_at__isnull=False).order_by('-last_message_at')


# messaging/rapidpro_client.py

import requests


class RapidProClient:

    def __init__(self, thread):
        self.thread = thread

    def send_reply(self, text):
        response = requests.get(url=self.thread.chatbot.request_url, params={
            'from': self.thread.uuid,
            'text': text,
        }, timeout=10)
        return response


# messaging/urls.py

from django.urls import path, include

from . import views

app_name = "messaging"

urlpatterns = [
    path('api/', include('messaging.api.urls')),
    path('inbox/', views.InboxView.as_view(), name='inbox'),
    path('message/<int:thread_id>', views.ThreadDetailView.as_view(), name='thread'),
    path('message/create', views.MessageCreateView.as_view(), name='message_create'),
    path('thread/<int:pk>/delete/', views.ThreadDeleteView.as_view(), name="thread_delete"),
]



# messaging/views.py

from django.contrib import messages
from django.contrib.auth import get_user_model
from django.http import HttpResponseRedirect
from django.shortcuts import render, redirect, get_object_or_404
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.views import View
from django.views.generic import (DeleteView, TemplateView, )

from .chat import ChatManager
from .forms import MessageReplyForm, NewMessageForm
from .models import Thread, UserThread

from django.contrib.auth.decorators import login_required

User = get_user_model()


@method_decorator(login_required, name='dispatch')
class InboxView(TemplateView):
    template_name = 'messaging/inbox.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        folder = self.kwargs.get('deleted', 'inbox')
        if folder == 'deleted':
            threads = Thread.thread_objects.of_user(self.request.user).deleted().order_by_latest()
        else:
            threads = Thread.thread_objects.of_user(self.request.user).inbox().order_by_latest()

        context.update({
            "folder": folder,
            "threads": threads.prefetch_related('messages'),
            "user_threads": UserThread.get_user_inbox(self.request.user),
            "unread_threads": Thread.thread_objects.of_user(self.request.user).unread().order_by_latest(),
        })
        return context


@method_decorator(login_required, name='dispatch')
class ThreadDetailView(View):

    def get_context(self, thread):
        num_results = int(self.request.GET.get('num_results', 20))
        thread_messages = thread.get_renderable_messages()[:num_results]
        most_recent_message = thread_messages.first()

        return {
            'thread': thread,
            'most_recent_message_id': most_recent_message.id if most_recent_message else None,
            'user': self.request.user,
            'thread_messages': reversed(thread_messages),
            'load_more_num_results': num_results + 20,
        }

    def get(self, request, thread_id):
        thread = get_object_or_404(Thread, pk=thread_id, users__in=[request.user])
        thread.mark_read(request.user)
        return render(request, 'messaging/thread_detail.html', context=self.get_context(thread))

    def post(self, request, thread_id):
        thread = get_object_or_404(Thread, pk=thread_id)
        form = MessageReplyForm(data=request.POST)
        if form.is_valid():
            text = form.cleaned_data['text']

            chat_manager = ChatManager(thread)
            chat_manager.record_reply(text=text, sender=request.user, mark_unread=False)

            return redirect(reverse('messaging:thread', kwargs={'thread_id': thread.pk}))
        else:
            return render(request, 'messaging/thread_detail.html', context=self.get_context(thread))


@method_decorator(login_required, name='dispatch')
class MessageCreateView(View):
    """
    Create a new thread message.
    """

    def post(self, request):
        form = NewMessageForm(data=request.POST)
        if form.is_valid():
            user = request.user
            data = form.cleaned_data
            thread = ChatManager.initiate_thread(
                sender=user, recipients=[], chatbot=data['chatbot'], subject=data['subject'], text=data['text'])

            return redirect(reverse('messaging:thread', kwargs={'thread_id': thread.pk}))

        messages.add_message(request, messages.ERROR, 'Can\'t connect with bot.')
        return redirect(request.META.get('HTTP_REFERER'))


@method_decorator(login_required, name='dispatch')
class ThreadDeleteView(DeleteView):
    """
    Delete a thread.
    """
    model = Thread
    template_name = "messaging/thread_confirm_delete.html"

    def delete(self, request, *args, **kwargs):
        self.get_object().user_threads.filter(user=request.user).update(is_active=False)
        return HttpResponseRedirect(reverse("messaging:inbox"))


# messaging/wagtail_hooks.py

from wagtail.contrib.modeladmin.options import (
    ModelAdminGroup, ModelAdmin, modeladmin_register)
from .models import ChatbotChannel


class ChatbotChannelAdmin(ModelAdmin):
    model = ChatbotChannel
    menu_label = 'Chatbot Channels'
    menu_icon = 'code'
    index_template_name = 'messaging/chatbotchannel/index.html'


class ChatbotGroup(ModelAdminGroup):
    menu_label = 'Chatbot'
    menu_icon = 'code'
    items = (ChatbotChannelAdmin,)


modeladmin_register(ChatbotGroup)


# notifications/apps.py

from django.apps import AppConfig


class NotificationsConfig(AppConfig):
    name = 'notifications'


# notifications/models.py

from django.db import models


class Notification(models.Model):
    head = models.CharField(max_length=255)
    body = models.TextField()
    url = models.URLField(null=True, blank=True)

    groups = models.ManyToManyField(to='auth.Group', related_name='notifications')

    def __str__(self):
        return self.head


# notifications/processors.py

from django.conf import settings


def push_notification(request):
    return {
        'push_notification': settings.PUSH_NOTIFICATION,
    }


# notifications/views.py

from wagtail.contrib.modeladmin.views import CreateView
from webpush import send_user_notification


class CreateNotificationView(CreateView):
    def form_valid(self, form):
        payload = form.cleaned_data.copy()
        groups = payload.pop('groups')
        for group in groups:
            for user in group.user_set.all():
                send_user_notification(user=user, payload=payload, ttl=1000)
        return super().form_valid(form)


# notifications/wagtail_hooks.py

from django.conf import settings
from wagtail.contrib.modeladmin.options import ModelAdmin, modeladmin_register

from notifications.models import Notification
from notifications.views import CreateNotificationView


class NotificationModelAdmin(ModelAdmin):
    model = Notification
    menu_label = 'Notifications'
    menu_icon = 'mail'
    list_display = ('head', 'body', 'url',)
    list_filter = ('groups',)
    search_fields = ('head', 'body', 'url',)
    menu_order = 601
    create_view_class = CreateNotificationView


if settings.PUSH_NOTIFICATION:
    modeladmin_register(NotificationModelAdmin)


# questionnaires/adapters.py

from functools import cached_property

from django import forms
from wagtail.admin.staticfiles import versioned_static
from wagtail.core.blocks.stream_block import StreamBlockAdapter
from wagtail.core.blocks.struct_block import StructBlockAdapter
from wagtail.core.widget_adapters import WidgetAdapter


class SkipLogicStreamBlockAdapter(StreamBlockAdapter):
    js_constructor = 'questionnaires.blocks.SkipLogicStreamBlock'

    @cached_property
    def media(self):
        media = super().media
        return forms.Media(
            js=media._js + [versioned_static('js/blocks/skip_logic_stream_block.js')],
            css=media._css
        )


class SkipLogicBlockAdapter(StructBlockAdapter):
    js_constructor = 'questionnaires.blocks.SkipLogicBlock'

    @cached_property
    def media(self):
        media = super().media
        return forms.Media(
            js=media._js + [versioned_static('js/blocks/skip_logic_block.js')],
            css=media._css
        )


class SelectAndHiddenWidgetAdapter(WidgetAdapter):
    js_constructor = 'questionnaires.widgets.SelectAndHiddenWidget'

    class Media:
        js = [versioned_static('js/widgets/select_and_hidden_widget.js')]


# questionnaires/admin.py

from wagtail.contrib.modeladmin.helpers import ButtonHelper
from wagtail.contrib.modeladmin.options import ModelAdmin, modeladmin_register

from .models import Survey, Poll, Quiz


class HideCreateButtonHelper(ButtonHelper):

    def add_button(self, classnames_add=None, classnames_exclude=None):
        return None


class SurveyAdmin(ModelAdmin):
    model = Survey
    menu_label = "Surveys"
    menu_icon = "doc-full"
    add_to_settings_menu = False
    exclude_from_explorer = False
    button_helper_class = HideCreateButtonHelper


class PollAdmin(ModelAdmin):
    model = Poll
    menu_label = "Polls"
    menu_icon = "doc-full"
    add_to_settings_menu = False
    exclude_from_explorer = False
    button_helper_class = HideCreateButtonHelper


class QuizAdmin(ModelAdmin):
    model = Quiz
    menu_label = "Quizzes"
    menu_icon = "doc-full"
    add_to_settings_menu = False
    exclude_from_explorer = False
    button_helper_class = HideCreateButtonHelper


modeladmin_register(SurveyAdmin)
modeladmin_register(PollAdmin)
modeladmin_register(QuizAdmin)


# questionnaires/apps.py

from django.apps import AppConfig


class QuestionnairesConfig(AppConfig):
    name = 'questionnaires'


# questionnaires/blocks.py

from django.core.exceptions import ValidationError
from django.utils.safestring import mark_safe
from django.utils.translation import gettext_lazy as _

from wagtail.core import blocks
from wagtail.core.fields import StreamField
from wagtail.core.telepath import register

from questionnaires.adapters import SkipLogicStreamBlockAdapter, SkipLogicBlockAdapter
from questionnaires.widgets import SelectAndHiddenWidget


class SkipState:
    NEXT = "next"
    END = "end"
    QUESTION = "question"


VALID_SKIP_SELECTORS = [
    'checkbox', 'checkboxes', 'date', 'datetime', 'dropdown', 'email', 'singleline', 'multiline', 'number',
    'positivenumber', 'radio', 'url',
]
VALID_SKIP_LOGIC = VALID_SKIP_SELECTORS


class SkipLogicField(StreamField):
    def __init__(self, *args, **kwargs):
        args = [SkipLogicStreamBlock([('skip_logic', SkipLogicBlock())])]
        kwargs.update({
            'verbose_name': _('Answer options'),
            # Help text is used to display a message for a specific field type.
            # If different help text is required each type might need to be
            # wrapped in a <div id="<field-type>-helptext"> for the frontend
            'help_text': mark_safe(
                '<strong>{}</strong>'.format(
                    _('Checkbox must include exactly 2 Skip Logic Options: '
                      'true and false, in that order.')
                )
            )
        })
        super(SkipLogicField, self).__init__(*args, **kwargs)


class SkipLogicStreamBlock(blocks.StreamBlock):
    class Meta:
        form_classname = 'skip-logic-stream-block'


class QuestionSelectBlock(blocks.IntegerBlock):
    def __init__(self, *args, **kwargs):
        super(QuestionSelectBlock, self).__init__(*args, **kwargs)
        self.field.widget = SelectAndHiddenWidget()


class SkipLogicBlock(blocks.StructBlock):
    choice = blocks.CharBlock(required=False)
    skip_logic = blocks.ChoiceBlock(
        choices=[
            (SkipState.NEXT, _("Next default question")),
            (SkipState.END, _("End of survey")),
            (SkipState.QUESTION, _("Another question")),
        ],
        default=SkipState.NEXT,
        required=False,
    )
    question = QuestionSelectBlock(
        required=False,
        help_text=_('Please save the survey as a draft to populate or update the list of questions.'),
    )

    def clean(self, value):
        cleaned_data = super(SkipLogicBlock, self).clean(value)
        logic = cleaned_data['skip_logic']
        if logic == SkipState.QUESTION:
            if not cleaned_data['question']:
                raise ValidationError(
                    'A Question must be selected to progress to.',
                    params={'question': [_('Please select a question.')]}
                )

        if logic in [SkipState.END, SkipState.NEXT]:
            cleaned_data['question'] = None

        return cleaned_data

    class Meta:
        form_classname = 'skip-logic-block struct-block'


register(SkipLogicStreamBlockAdapter(), SkipLogicStreamBlock)
register(SkipLogicBlockAdapter(), SkipLogicBlock)


# questionnaires/edit_handlers.py

from wagtail.contrib.forms.edit_handlers import FormSubmissionsPanel as WagtailFormSubmissionsPanel


class FormSubmissionsPanel(WagtailFormSubmissionsPanel):
    def render(self):
        from home.models import SiteSettings

        registration_survey = SiteSettings.get_for_default_site().registration_survey
        if registration_survey:
            if self.instance in registration_survey.get_translations(inclusive=True):
                return ''

        return super().render()


# questionnaires/factories.py

import json

import factory
from django.utils import timezone
from factory.django import DjangoModelFactory
from wagtail_factories import PageFactory

from questionnaires.models import (
    Poll, 
    PollFormField, 
    PollIndexPage,
    Quiz,
    QuizFormField, 
    QuizIndexPage,
    Survey,
    SurveyFormField,
    SurveyIndexPage,  
    UserSubmission
)


class PollFactory(PageFactory):
    title = factory.Sequence(lambda n: f'poll{n}')
    last_published_at = timezone.now()
    index_page_description = factory.Sequence(lambda n: f'index page description{n}')
    index_page_description_line_2 = factory.Sequence(lambda n: f'index page description line 2{n}')

    class Meta:
        model = Poll

class PollFormFieldFactory(DjangoModelFactory):
    label = factory.Sequence(lambda n: f'question{n}')
    admin_label = factory.Sequence(lambda n: f'question{n}')
    help_text = factory.Sequence(lambda n: f'help text{n}')

    class Meta:
        model = PollFormField

class PollIndexPageFactory(PageFactory):
    title = 'Banners'

    class Meta:
        model = PollIndexPage

class QuizFactory(PageFactory):
    title = factory.Sequence(lambda n: f'quiz{n}')
    last_published_at = timezone.now()
    index_page_description = factory.Sequence(lambda n: f'index page description{n}')
    index_page_description_line_2 = factory.Sequence(lambda n: f'index page description line 2{n}')

    class Meta:
        model = Quiz

class QuizFormFieldFactory(DjangoModelFactory):
    label = factory.Sequence(lambda n: f'question{n}')
    admin_label = factory.Sequence(lambda n: f'question{n}')
    help_text = factory.Sequence(lambda n: f'help text{n}')
    feedback = factory.Sequence(lambda n: f'feedback{n}')

    class Meta:
        model = QuizFormField

class QuizIndexPageFactory(PageFactory):
    title = 'Quizzes'

    class Meta:
        model = QuizIndexPage


class SurveyFactory(PageFactory):
    title = factory.Sequence(lambda n: f'survey{n}')
    last_published_at = timezone.now()
    index_page_description = factory.Sequence(lambda n: f'index page description{n}')
    index_page_description_line_2 = factory.Sequence(lambda n: f'index page description line 2{n}')

    class Meta:
        model = Survey


class SurveyFormFieldFactory(DjangoModelFactory):
    label = factory.Sequence(lambda n: f'question{n}')
    admin_label = factory.Sequence(lambda n: f'question{n}')
    help_text = factory.Sequence(lambda n: f'help text{n}')

    class Meta:
        model = SurveyFormField

    @factory.post_generation
    def skip_logic(self, create, extracted, **kwargs):
        if extracted:
            self.skip_logic = extracted
        elif self.field_type == 'checkbox':
            self.skip_logic = json.dumps(
                [
                    {
                        "type": "skip_logic",
                        "value": {
                            "choice": "true",
                            "skip_logic": "next",
                            "question": None
                        }
                    },
                    {
                        "type": "skip_logic",
                        "value": {
                            "choice": "false",
                            "skip_logic": "next",
                            "question": None
                        }
                    }
                ]
            )


class SurveyIndexPageFactory(PageFactory):
    title = 'Surveys'

    class Meta:
        model = SurveyIndexPage

class UserSubmissionFactory(DjangoModelFactory):
    class Meta:
        model = UserSubmission


# questionnaires/filters.py

from django_filters import rest_framework as filters
from wagtail.core.models import Page

from questionnaires.models import UserSubmission


class ListIdFilter(filters.Filter):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.lookup_expr = 'in'

    def filter(self, queryset, value):
        list_values = value.split(',') if value else []
        return super().filter(queryset, list_values)


class QuestionnaireFilter(filters.FilterSet):
    last_published_at_start = filters.DateFilter(field_name='last_published_at__date', lookup_expr='gte')
    last_published_at_end = filters.DateFilter(field_name="last_published_at__date", lookup_expr='lte')
    type = filters.ChoiceFilter(field_name='content_type__model', lookup_expr='exact',
                                choices=(('poll', 'Poll'), ('survey', 'Survey'), ('quiz', 'Quiz')))

    class Meta:
        model = Page
        fields = ['last_published_at_start', 'last_published_at_end', 'type']


class UserSubmissionFilter(filters.FilterSet):
    submit_time_start = filters.DateFilter(field_name='submit_time__date', lookup_expr='gte')
    submit_time_end = filters.DateFilter(field_name="submit_time__date", lookup_expr='lte')
    user_ids = ListIdFilter(field_name='user_id')

    class Meta:
        model = UserSubmission
        fields = ['submit_time_start', 'submit_time_end', 'user_ids']


# questionnaires/forms.py

import random
from collections import defaultdict

from django.core.exceptions import ValidationError
from django import forms
from django.forms.utils import ErrorList, ErrorDict
from django.utils.translation import gettext_lazy as _

from wagtail.admin.forms import WagtailAdminPageForm
from wagtail.contrib.forms.forms import FormBuilder
from wagtail.core.blocks import StreamBlockValidationError
from wagtail.core.blocks.struct_block import StructBlockValidationError

from questionnaires.blocks import VALID_SKIP_SELECTORS, SkipState, VALID_SKIP_LOGIC


class CustomFormBuilder(FormBuilder):
    def create_date_field(self, field, options):
        options.update({
            'widget': forms.DateInput(attrs={'type': 'date'}),
        })

        return forms.DateField(**options)

    def create_datetime_field(self, field, options):
        options.update({
            'widget': forms.DateInput(attrs={'type': 'datetime-local'}),
        })

        return forms.DateTimeField(**options)

    def create_positivenumber_field(self, field, options):
        options.update({
            'min_value': 0,
        })

        return forms.IntegerField(**options)

    def create_dropdown_field(self, field, options):
        options['choices'] = [(x.strip(), x.strip()) for x in field.choices.split('|')]
        if getattr(field.page, 'randomise_options', False):
            random.shuffle(options['choices'])
        return forms.ChoiceField(**options)

    def create_multiselect_field(self, field, options):
        options['choices'] = [(x.strip(), x.strip()) for x in field.choices.split('|')]
        if getattr(field.page, 'randomise_options', False):
            random.shuffle(options['choices'])
        return forms.MultipleChoiceField(**options)

    def create_radio_field(self, field, options):
        options['choices'] = [(x.strip(), x.strip()) for x in field.choices.split('|')]
        if getattr(field.page, 'randomise_options', False):
            random.shuffle(options['choices'])
        return forms.ChoiceField(widget=forms.RadioSelect, **options)

    def create_checkboxes_field(self, field, options):
        options['choices'] = [(x.strip(), x.strip()) for x in field.choices.split('|')]
        options['initial'] = [x.strip() for x in field.default_value.split('|')]
        if getattr(field.page, 'randomise_options', False):
            random.shuffle(options['choices'])
        return forms.MultipleChoiceField(
            widget=forms.CheckboxSelectMultiple, **options
        )


class SurveyForm(WagtailAdminPageForm):
    form_field_name = 'survey_form_fields'
    _clean_errors = None

    def clean(self):
        cleaned_data = super().clean()

        question_data = {}
        for form in self.formsets[self.form_field_name]:
            form.is_valid()
            question_data[form.cleaned_data['ORDER']] = form

        for i, form in question_data.items():
            self._clean_errors = {}
            if form.is_valid():
                data = form.cleaned_data
                if not cleaned_data['multi_step'] and data['page_break']:
                    self.add_form_field_error(
                        'page_break',
                        _('Page break is only allowed with multi-step enabled.'),
                    )
                if data['field_type'] in VALID_SKIP_LOGIC and not data['required']:
                    for logic in data['skip_logic']:
                        if logic.value['skip_logic'] in [SkipState.QUESTION, SkipState.END]:
                            self.add_form_field_error(
                                'required',
                                _('Questions with non-trivial skip logic must be required.'),
                            )
                            break
                if data['field_type'] == 'checkbox':
                    if (len(data['skip_logic']) != 2 or
                            [logic.value['choice'] for logic in data['skip_logic']] != ['true', 'false']):
                        self.add_form_field_error(
                            'field_type',
                            _('Checkbox must include exactly 2 Skip Logic Options: true and false, in that order.'),
                        )
                elif data['field_type'] in ['checkboxes', 'dropdown', 'radio']:
                    if len(data['skip_logic']) < 2:
                        self.add_form_field_error(
                            'field_type',
                            _(f'{data["field_type"]} must include at least 2 Answer Options.'),
                        )

                for j, logic in enumerate(data['skip_logic']):
                    if not logic.value['choice']:
                        self.add_stream_field_error(
                            j,
                            'choice',
                            _('This field is required.'),
                        )
                    if '|' in logic.value['choice']:
                        self.add_stream_field_error(
                            j,
                            'choice',
                            _('Pipe (|) symbol not allowed.'),
                        )
                    if logic.value['skip_logic'] == SkipState.QUESTION and logic.value['question']:
                        last_question_number = logic.value['question']
                        msg = _(f'Skip to question {question_data[last_question_number].cleaned_data["label"]} '
                                f'with in-between required questions isn\'t allowed.')
                    elif logic.value['skip_logic'] == SkipState.END:
                        last_question_number = len(question_data) + 1
                        msg = _(f'Skip to end of survey with in-between required questions isn\'t allowed.')
                    else:
                        continue
                    for k in range(i + 1, last_question_number):
                        skip_to_question = question_data[k].cleaned_data
                        if skip_to_question['required']:
                            self.add_stream_field_error(j, 'skip_logic', msg)
                            break

                if data['field_type'] == "checkboxes":
                    for i, logic in enumerate(data['skip_logic']):
                        if logic.value['skip_logic'] != SkipState.NEXT:
                            self.add_stream_field_error(
                                i,
                                'skip_logic',
                                _(f'Skipping to {logic.value["skip_logic"]} not allowed for checkboxes.'),
                            )
                if self.clean_errors:
                    form._errors = self.clean_errors

        if self.should_be_multi_step(cleaned_data, question_data):
            self.add_error(
                "multi_step",
                "Multi-step is required to properly display questionnaires that include Skip Logic."
            )

        return cleaned_data

    def save(self, commit):
        # Tidy up the skip logic when field cant have skip logic
        for form in self.formsets[self.form_field_name]:
            field_type = form.instance.field_type
            # Copy choices values from skip logic to main choices property
            if field_type in VALID_SKIP_SELECTORS:
                choices_values = []
                for skip_logic in form.instance.skip_logic:
                    choices_values.append(skip_logic.value['choice'])
                form.instance.choices = "|".join(choices_values)
            else:
                form.instance.skip_logic = []

        return super().save(commit)

    def add_stream_field_error(self, position, field, message):
        if position not in self._clean_errors:
            self._clean_errors[position] = defaultdict(list)
        self._clean_errors[position][field].append(message)

    @property
    def clean_errors(self):
        if self._clean_errors.keys():
            errors = {
                key: ValidationError(value)
                for key, value in self._clean_errors.items()
                if isinstance(key, str)
            }

            blocks_errors = {}
            for block_index, errors_dict in self._clean_errors.items():
                if isinstance(block_index, int):
                    block_errors = {}
                    for field, errors_list in errors_dict.items():
                        block_errors[field] = ErrorList([ValidationError(message) for message in errors_list])
                    else:
                        blocks_errors[block_index] = ErrorList([StructBlockValidationError(block_errors=block_errors)])
            else:
                stream_block_errors = StreamBlockValidationError(block_errors=blocks_errors, non_block_errors=ErrorList())
                errors.update({
                    "skip_logic": ErrorList([stream_block_errors]),
                })

            return ErrorDict(errors)

    def add_form_field_error(self, field, message):
        if field not in self._clean_errors:
            self._clean_errors[field] = list()
        self._clean_errors[field].append(message)

    @classmethod
    def should_be_multi_step(cls, cleaned_data, question_data):
        for form in question_data.values():
            if form.is_valid():
                data = form.cleaned_data
                for logic in data['skip_logic']:
                    if (logic.value['skip_logic'] in [SkipState.QUESTION, SkipState.END]
                            and not cleaned_data['multi_step']):
                        return True

        return False


class QuizForm(WagtailAdminPageForm):
    form_field_name = 'quiz_form_fields'
    _clean_errors = None

    def clean(self):
        cleaned_data = super().clean()

        question_data = {}
        for form in self.formsets[self.form_field_name]:
            form.is_valid()
            question_data[form.cleaned_data['ORDER']] = form

        for i, form in question_data.items():
            self._clean_errors = {}
            if form.is_valid():
                data = form.cleaned_data
                if not cleaned_data['multi_step'] and data['page_break']:
                    self.add_form_field_error(
                        'page_break',
                        _('Page break is only allowed with multi-step enabled.'),
                    )
                if self.clean_errors:
                    form._errors = self.clean_errors

        return cleaned_data

    @property
    def clean_errors(self):
        if self._clean_errors.keys():
            params = {
                key: ErrorList(
                    [ValidationError('Error in form', params=value)]
                )
                for key, value in self._clean_errors.items()
                if isinstance(key, int)
            }
            errors = {
                key: ValidationError(value)
                for key, value in self._clean_errors.items()
                if isinstance(key, str)
            }
            errors.update({
                'skip_logic': ErrorList([ValidationError(
                    'Skip Logic Error',
                    params=params,
                )])
            })
            return errors

    def add_form_field_error(self, field, message):
        if field not in self._clean_errors:
            self._clean_errors[field] = list()
        self._clean_errors[field].append(message)


class GenerateDashboardForm(forms.Form):
    superset_username = forms.CharField()
    superset_password = forms.CharField(widget=forms.PasswordInput)


# questionnaires/models.py

import json

from django.conf import settings
from django.contrib.auth import get_user_model
from django.db.models import Q
from django.utils import timezone
from django.utils.functional import cached_property

from django.core.paginator import EmptyPage, PageNotAnInteger
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models
from django.shortcuts import render
from django.utils.translation import gettext_lazy as _
from wagtail.images.edit_handlers import ImageChooserPanel
from wagtail_localize.fields import TranslatableField
from wagtailmarkdown.blocks import MarkdownBlock
from wagtailsvg.edit_handlers import SvgChooserPanel
from wagtailsvg.models import Svg

from home.blocks import (
    MediaBlock,
    NumberedListBlock,
    PageButtonBlock,
    RawHTMLBlock,
)
from home.mixins import PageUtilsMixin, TitleIconMixin
from home.utils import (
    collect_urls_from_streamfield,
    get_all_renditions_urls,
)
from modelcluster.fields import ParentalKey
from wagtail.admin.edit_handlers import (FieldPanel, InlinePanel,
                                         MultiFieldPanel, StreamFieldPanel)
from wagtail.contrib.forms.models import (AbstractForm, AbstractFormField,
                                          AbstractFormSubmission)
from wagtail.core import blocks
from wagtail.core.fields import StreamField
from wagtail.core.models import Page
from wagtail.images.blocks import ImageChooserBlock

from questionnaires.blocks import SkipState, SkipLogicField
from questionnaires.edit_handlers import FormSubmissionsPanel
from questionnaires.forms import CustomFormBuilder, SurveyForm, QuizForm
from questionnaires.utils import SkipLogicPaginator, FormHelper


FORM_FIELD_CHOICES = (
    ('checkbox', _('Checkbox')),
    ('checkboxes', _('Checkboxes')),
    ('date', _('Date')),
    ('datetime', _('Date/time')),
    ('dropdown', _('Drop down')),
    ('email', _('Email')),
    ('singleline', _('Single line text')),
    ('multiline', _('Multi-line text')),
    ('number', _('Number')),
    ('positivenumber', _('Positive number')),
    ('radio', _('Radio buttons')),
    ('url', _('URL')),
)


class QuestionnairePage(Page, PageUtilsMixin, TitleIconMixin):
    template = None
    parent_page_types = []
    subpage_types = []
    show_in_menus_default = True

    description = StreamField(
        [
            ('heading', blocks.CharBlock(form_classname="full title", template='blocks/heading.html')),
            ('paragraph', blocks.RichTextBlock(features=settings.WAGTAIL_RICH_TEXT_FIELD_FEATURES)),
            ('paragraph_v1_legacy', RawHTMLBlock(icon='code')),
            ("image", ImageChooserBlock(template='blocks/image.html')),
            ('list', MarkdownBlock(icon='code')),
            ('numbered_list', NumberedListBlock(MarkdownBlock(icon='code'))),
            ('page_button', PageButtonBlock()),
        ],
        null=True,
        blank=True,
    )
    thank_you_text = StreamField(
        [
            ("paragraph", blocks.RichTextBlock(features=settings.WAGTAIL_RICH_TEXT_FIELD_FEATURES)),
            ("media", MediaBlock(icon="media")),
            ("image", ImageChooserBlock(template='blocks/image.html')),
        ],
        null=True,
        blank=True,
    )
    allow_anonymous_submissions = models.BooleanField(
        default=True,
        help_text=_(
            "Check this to allow users who are NOT logged in to complete surveys."
        ),
    )

    allow_multiple_submissions = models.BooleanField(
        default=True,
        help_text=_("Check this to allow multiple form submissions for users"),
    )
    submit_button_text = models.CharField(
        max_length=40, null=True, default=_("Submit"), help_text=_("Submit button text")
    )

    direct_display = models.BooleanField(default=False)

    index_page_description = models.TextField(null=True, blank=True)
    index_page_description_line_2 = models.TextField(null=True, blank=True)

    terms_and_conditions = StreamField(
        [
            ("paragraph", blocks.RichTextBlock(features=settings.WAGTAIL_RICH_TEXT_FIELD_FEATURES)),
            ('page_button', PageButtonBlock()),
        ],
        null=True,
        blank=True,
    )

    icon = models.ForeignKey(
        Svg,
        related_name='+',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
    )
    image_icon = models.ForeignKey(
        'wagtailimages.Image',
        on_delete=models.PROTECT,
        related_name='+',
        blank=True,
        null=True
    )

    settings_panels = Page.settings_panels + [
        FieldPanel('direct_display')
    ]

    def __str__(self):
        return self.title

    def serve(self, request, *args, **kwargs):
        if request.session.session_key is None:
            request.session.save()
        self.session = request.session

        multiple_submission_filter = (
            Q(session_key=request.session.session_key) if request.user.is_anonymous else Q(user__pk=request.user.pk)
        )
        multiple_submission_check = (
            not self.allow_multiple_submissions
            and self.get_submission_class().objects.filter(multiple_submission_filter, page=self).exists()
        )
        anonymous_user_submission_check = request.user.is_anonymous and not self.allow_anonymous_submissions
        if multiple_submission_check or anonymous_user_submission_check:
            return render(request, self.template, self.get_context(request))

        if hasattr(self, "multi_step") and self.multi_step and self.get_form_fields():
            return self.serve_questions_separately(request)

        return super().serve(request, *args, **kwargs)

    def serve_questions_separately(self, request, *args, **kwargs):
        form_helper = FormHelper(pk=self.pk, request=request)
        is_last_step = False
        step_number = request.GET.get("p", 1)

        form_data = form_helper.get_form_data()

        paginator = SkipLogicPaginator(
            self.get_form_fields(),
            request.POST,
            form_data,
        )

        try:
            step = paginator.page(step_number)
        except PageNotAnInteger:
            step = paginator.page(1)
        except EmptyPage:
            step = paginator.page(paginator.num_pages)
            is_last_step = True

        if request.method == "POST":
            # The first step will be submitted with step_number == 2,
            # so we need to get a form from previous step
            # Edge case - submission of the last step
            try:
                prev_step = (
                    step if is_last_step else paginator.page(int(step_number) - 1)
                )
            except EmptyPage:
                prev_step = step
                step = paginator.page(prev_step.next_page_number())

            # Create a form only for submitted step
            prev_form_class = self.get_form_class_for_step(prev_step)
            prev_form = prev_form_class(
                paginator.new_answers,
                page=self,
                user=request.user
            )
            if prev_form.is_valid():
                # If data for step is valid, update the session
                form_data = form_helper.get_form_data()
                form_data.update(prev_form.cleaned_data)
                form_helper.set_form_data(form_data)

                if prev_step.has_next():
                    # Create a new form for a following step, if the following step is present
                    form_class = self.get_form_class_for_step(step)
                    form = form_class(page=self, user=request.user)
                else:
                    # If there is no next step, create form for all fields
                    form = self.get_form(
                        form_helper.get_form_data(),
                        page=self, user=request.user
                    )

                    if form.is_valid():
                        # Perform validation again for whole form.
                        # After successful validation, save data into DB,
                        # and remove from the session.
                        form_submission = self.process_form_submission(form)
                        form_helper.set_full_form_data()
                        # render the landing page
                        return self.render_landing_page(
                            request, form_submission, *args, **kwargs
                        )
            else:
                # If data for step is invalid
                # we will need to display form again with errors,
                # so restore previous state.
                form = prev_form
                step = prev_step
        else:
            # Create empty form for non-POST requests
            form_class = self.get_form_class_for_step(step)
            form = form_class(page=self, user=request.user)

        context = self.get_context(request)
        context["form"] = form
        context["fields_step"] = step
        return render(request, self.template, context)

    def process_form_submission(self, form):
        user = form.user
        self.get_submission_class().objects.create(
            form_data=json.dumps(form.cleaned_data, cls=DjangoJSONEncoder),
            page=self,
            user=None if user.is_anonymous else user,
            session_key=self.session.session_key,
        )

    # Required by Wagtail Forms
    def get_submissions_list_view_class(self):
        from questionnaires.views import CustomSubmissionsListView
        return CustomSubmissionsListView

    def get_export_filename(self):
        object_type = self.__class__.__name__.lower()
        title = self.title
        timestamp = timezone.now().strftime(settings.EXPORT_FILENAME_TIMESTAMP_FORMAT)

        return f'{object_type}-{title}_{timestamp}'

    def get_data_fields(self):
        data_fields = [
            ('user', _('User')),
            ('submit_time', _('Submission Date')),
            ('page_url', _('Page URL')),
        ]
        data_fields += [
            (field.clean_name, field.admin_label or field.label)
            for field in self.get_form_fields()
        ]
        return data_fields

    @property
    def offline_urls(self):
        urls = (
            [self.url]
            + collect_urls_from_streamfield(self.description)
            + collect_urls_from_streamfield(self.thank_you_text)
        )

        if self.image_icon:
            urls += get_all_renditions_urls(self.image_icon)

        return urls

    def get_submit_button_text(self, fields_step=None):
        submit_button_text = self.submit_button_text
        if fields_step and fields_step.paginator.num_pages != fields_step.number:
            submit_button_text = _("Next")

        return submit_button_text

    class Meta:
        abstract = True


class SurveyFormField(AbstractFormField):
    page = ParentalKey("Survey", on_delete=models.CASCADE, related_name="survey_form_fields")
    clean_name = models.TextField(
        verbose_name=_('name'),
        blank=True,
        default='',
        help_text=_('Safe name of the form field, the label converted to ascii_snake_case. '
                    'This will appear in the HTML but will not be visible to the user')
    )
    label = models.TextField(
        verbose_name=_('label'),
        help_text=_('The label of the form field')
    )
    field_type = models.CharField(
        verbose_name=_('field type'),
        max_length=16,
        choices=FORM_FIELD_CHOICES
    )
    admin_label = models.CharField(
        verbose_name=_('admin_label'),
        max_length=256,
        help_text=_('Column header used during CSV export of survey '
                    'responses.'),
    )
    skip_logic = SkipLogicField(null=True, blank=True)
    default_value = models.TextField(
        verbose_name=_('default value'),
        blank=True,
        help_text=_('Default value. Pipe (|) separated values supported for checkboxes.')
    )
    page_break = models.BooleanField(
        default=False,
        help_text=_(
            'Inserts a page break which puts the next question onto a new page'
        )
    )
    panels = [
        FieldPanel('label'),
        FieldPanel('clean_name', classname='disabled-clean-name'),
        FieldPanel('help_text'),
        FieldPanel('required'),
        FieldPanel('field_type', classname="formbuilder-type"),
        StreamFieldPanel('skip_logic', classname='skip-logic'),
        FieldPanel('default_value', classname="formbuilder-default"),
        FieldPanel('admin_label'),
        FieldPanel('page_break'),
    ]

    @property
    def has_skipping(self):
        return any(
            logic.value['skip_logic'] != SkipState.NEXT
            for logic in self.skip_logic
        )

    def choice_index(self, choice):
        if self.field_type == 'checkbox':
            choice = 'true' if choice == 'on' else 'false'
        try:
            return self.choices.split('|').index(choice)
        except ValueError:
            pass

        return None

    def next_action(self, choice):
        choice_index = self.choice_index(choice)
        if choice_index is None:
            return SkipState.NEXT
        return self.skip_logic[choice_index].value['skip_logic']

    def is_next_action(self, choice, *actions):
        if self.has_skipping:
            return self.next_action(choice) in actions
        return False

    def next_page(self, choice):
        logic = self.skip_logic[self.choice_index(choice)]
        return logic.value[self.next_action(choice)]


class Survey(QuestionnairePage, AbstractForm):
    base_form_class = SurveyForm
    form_builder = CustomFormBuilder

    parent_page_types = [
        "home.HomePage", "home.Section", "home.Article", "questionnaires.SurveyIndexPage", 'home.FooterIndexPage',
    ]
    template = "survey/survey.html"
    multi_step = models.BooleanField(
        default=False,
        verbose_name=_("Multi-step"),
        help_text=_("Whether to display the survey questions to the user one at"
        " a time, instead of all at once."),
    )

    content_panels = Page.content_panels + [
        FormSubmissionsPanel(),
        MultiFieldPanel(
            [
                FieldPanel("allow_anonymous_submissions"),
                FieldPanel("allow_multiple_submissions"),
                FieldPanel("submit_button_text"),
                FieldPanel("multi_step"),
            ],
            heading=_(
                "General settings for survey",
            ),
        ),
        MultiFieldPanel(
            [
                StreamFieldPanel("description"),
            ],
            heading=_(
                "Description at survey page",
            ),
        ),
        MultiFieldPanel(
            [
                StreamFieldPanel("thank_you_text"),
            ],
            heading="Description at thank you page",
        ),
        FieldPanel('index_page_description'),
        FieldPanel('index_page_description_line_2'),
        MultiFieldPanel(
            [
                StreamFieldPanel("terms_and_conditions"),
            ],
            heading="Terms and conditions",
        ),
        SvgChooserPanel('icon'),
        ImageChooserPanel('image_icon'),
        InlinePanel("survey_form_fields", label="Form fields"),
    ]

    translatable_fields = [
        TranslatableField('title'),
        TranslatableField('description'),
        TranslatableField('submit_button_text'),
        TranslatableField('survey_form_fields'),
        TranslatableField('thank_you_text')
    ]

    @cached_property
    def has_page_breaks(self):
        return any(
            field.page_break
            for field in self.get_form_fields()
        )

    def get_form_class_for_step(self, step):
        return self.form_builder(step.object_list).get_form_class()

    def get_form_fields(self):
        return self.survey_form_fields.all()

    def get_submission_class(self):
        return UserSubmission

    def process_form_submission(self, form):
        from home.models import SiteSettings

        super().process_form_submission(form)

        site_settings = SiteSettings.get_for_default_site()
        if site_settings.registration_survey and site_settings.registration_survey.localized.pk == self.pk:
            user = form.user
            user.has_filled_registration_survey = True
            user.save(update_fields=['has_filled_registration_survey'])

    def get_context(self, request, *args, **kwargs):
        context = super().get_context(request, *args, **kwargs)
        context.update({'back_url': request.GET.get('back_url')})
        context.update({'form_length': request.GET.get('form_length')})
        return context

    def has_required_fields(self):
        return self.survey_form_fields.filter(required=True).exists()

    class Meta:
        verbose_name = _("survey")
        verbose_name_plural = _("surveys")


class UserSubmission(AbstractFormSubmission):
    user = models.ForeignKey(
        get_user_model(), on_delete=models.CASCADE, blank=True, null=True
    )
    session_key = models.CharField(max_length=255, null=True, blank=True)

    def get_data(self):
        form_data = super().get_data()
        form_data.update(
            {
                "user": self.user if self.user else None,
                "page_url": self.page.full_url,
            }
        )
        return form_data


class PollFormField(AbstractFormField):
    page = ParentalKey("Poll", on_delete=models.CASCADE, related_name="poll_form_fields")
    CHOICES = (
        ('checkboxes', _('Checkboxes')),
        ('dropdown', _('Dropdown')),
        ('multiline', _('Multi-line text')),
        ('radio', _('Radio buttons')),
    )
    clean_name = models.TextField(
        verbose_name=_('name'),
        blank=True,
        default='',
        help_text=_('Safe name of the form field, the label converted to ascii_snake_case. '
                    'This will appear in the HTML but will not be visible to the user')
    )
    label = models.TextField(
        verbose_name=_('label'),
        help_text=_('The label of the form field')
    )
    field_type = models.CharField(
        verbose_name=_('field type'),
        max_length=16,
        choices=CHOICES
    )
    choices = models.TextField(
        verbose_name=_('choices'),
        null=True,
        blank=True,
        help_text=_('For checkboxes, dropdown and radio: Pipe (|) separated list of choices.')
    )
    default_value = models.TextField(
        verbose_name=_('default value'),
        blank=True,
        help_text=_('Default value. Pipe (|) separated values supported for checkboxes.')
    )
    admin_label = models.CharField(
        verbose_name=_('admin_label'),
        max_length=256,
        help_text=_('Column header used during CSV export of poll responses.'),
        null=True
    )

    panels = [
        FieldPanel('label'),
        FieldPanel('clean_name', classname='disabled-clean-name'),
        FieldPanel('help_text'),
        FieldPanel('required'),
        FieldPanel('field_type', classname="formbuilder-type"),
        FieldPanel('choices', classname="formbuilder-choices"),
        FieldPanel('default_value', classname="formbuilder-default"),
        FieldPanel('admin_label'),
    ]


class Poll(QuestionnairePage, AbstractForm):
    form_builder = CustomFormBuilder
    template = "poll/poll.html"
    parent_page_types = [
        "home.HomePage", "home.Section", "home.Article", "questionnaires.PollIndexPage", 'home.FooterIndexPage',
    ]

    show_results = models.BooleanField(
        default=True, help_text=_("This option allows the users to see the results.")
    )
    result_as_percentage = models.BooleanField(
        default=True,
        help_text=_(
            "If not checked, the results will be shown as a total instead of a percentage."
        ),
    )
    randomise_options = models.BooleanField(
        default=False,
        help_text=_(
            "Randomising the options allows the options to be shown in a different "
            "order each time the page is displayed."
        ),
    )
    show_results_with_no_votes = models.BooleanField(
        default=True,
        help_text=_("Display options with 0 votes in results.")
    )

    content_panels = Page.content_panels + [
        FormSubmissionsPanel(),
        MultiFieldPanel(
            [
                FieldPanel("allow_anonymous_submissions"),
                FieldPanel("show_results"),
                FieldPanel("result_as_percentage"),
                FieldPanel("allow_multiple_submissions"),
                FieldPanel("randomise_options"),
                FieldPanel("show_results_with_no_votes"),
                FieldPanel("submit_button_text"),
            ],
            heading=_(
                "General settings for poll",
            ),
        ),
        MultiFieldPanel(
            [
                StreamFieldPanel("description"),
            ],
            heading=_(
                "Description at poll page",
            ),
        ),
        MultiFieldPanel(
            [
                StreamFieldPanel("thank_you_text"),
            ],
            heading="Description at thank you page",
        ),
        FieldPanel('index_page_description'),
        FieldPanel('index_page_description_line_2'),
        MultiFieldPanel(
            [
                StreamFieldPanel("terms_and_conditions"),
            ],
            heading="Terms and conditions",
        ),
        SvgChooserPanel('icon'),
        ImageChooserPanel('image_icon'),
        InlinePanel("poll_form_fields", label="Poll Form fields", min_num=1, max_num=1),
    ]

    class Meta:
        verbose_name = _("poll")
        verbose_name_plural = _("polls")

    def get_form_fields(self):
        return self.poll_form_fields.all()

    def get_submission_class(self):
        return UserSubmission

    def get_results(self, data=None):
        results = dict()
        results_list = []
        field = self.get_form_fields().first()
        name, choices = field.clean_name, field.choices
        submissions = self.get_submission_class().objects.filter(page=self)
        submissions = [submission for submission in submissions if submission.get_data().get(name) != '']

        # Default result counts to zero so choices with no votes are included
        if self.show_results_with_no_votes:
            results[name] = {
                choice: 0 for choice in choices.split('|') if len(choice) > 0
            }

        for submission in submissions:
            data = submission.get_data()
            answer = data.get(name)
            question_stats = results.get(name, {})
            if type(answer) != list:
                answer = [answer]

            for choice in answer:
                question_stats[choice] = question_stats.get(choice, 0) + 1

            results[name] = question_stats

        if submissions and self.result_as_percentage:
            total_submissions = len(submissions)
            for key in results:
                for k, v in results[key].items():
                    results[key][k] = round(v / total_submissions, 4) * 100

        for question, answers in results.items():
            for answer, count in answers.items():
                is_selected = data and (answer in data.get(question))
                results_list.append((answer, round(count), is_selected))

        return results_list

    def get_context(self, request, *args, **kwargs):
        context = super().get_context(request, *args, **kwargs)
        results = self.get_results(dict(request.POST))

        context.update({
            'results': results,
            'result_as_percentage': self.result_as_percentage,
            'back_url': request.GET.get('back_url'),
        })
        return context

    def get_submit_button_text(self, fields_step=None):
        return self.submit_button_text


class QuizFormField(AbstractFormField):
    page = ParentalKey("Quiz", on_delete=models.CASCADE, related_name="quiz_form_fields")
    clean_name = models.TextField(
        verbose_name=_('name'),
        blank=True,
        default='',
        help_text=_('Safe name of the form field, the label converted to ascii_snake_case. '
                    'This will appear in the HTML but will not be visible to the user')
    )
    label = models.TextField(
        verbose_name=_('label'),
        help_text=_('The label of the form field')
    )
    field_type = models.CharField(
        verbose_name=_('field type'),
        max_length=16,
        choices=FORM_FIELD_CHOICES
    )
    choices = models.TextField(
        verbose_name=_('choices'),
        blank=True,
        help_text=_('For checkboxes, dropdown and radio: Pipe (|) separated list of choices.')
    )
    admin_label = models.CharField(
        verbose_name=_('admin_label'),
        max_length=256,
        help_text=_('Column header used during CSV export of survey '
                    'responses.'),
    )
    default_value = models.TextField(
        verbose_name=_('default value'),
        blank=True,
        help_text=_('Default value. Pipe (|) separated values supported for checkboxes.')
    )
    page_break = models.BooleanField(
        default=False,
        help_text=_(
            'Inserts a page break which puts the next question onto a new page'
        )
    )
    correct_answer = models.TextField(
        verbose_name=_('correct_answer'),
        help_text=_('The correct answer/choice(s). '
                    'If multiple choices are correct, separate choices with a pipe (|) symbol. '
                    'For checkbox: Either "true" or "false".'))
    feedback = models.TextField(
        verbose_name=_('Feedback'),
        null=True,
        blank=True,
        help_text=_('Feedback message for user answer.')
    )

    panels = [
        FieldPanel('label'),
        FieldPanel('clean_name', classname='disabled-clean-name'),
        FieldPanel('help_text'),
        FieldPanel('required'),
        FieldPanel('field_type', classname="formbuilder-type"),
        FieldPanel('choices', classname="formbuilder-choices"),
        FieldPanel('default_value', classname="formbuilder-default"),
        FieldPanel('correct_answer'),
        FieldPanel('feedback'),
        FieldPanel('admin_label'),
        FieldPanel('page_break'),
    ]

    @property
    def has_skipping(self):
        return None

    def choice_index(self, choice):
        if choice:
            if self.field_type == 'checkbox':
                # clean checkboxes have True/False
                try:
                    return ['true', 'false'].index(choice)
                except ValueError:
                    return [True, False].index(choice)
            try:
                return self.choices.split('|').index(choice)
            except ValueError:
                pass

        return None

    def next_action(self, choice):
        choice_index = self.choice_index(choice)
        if choice_index is None:
            return SkipState.NEXT
        return self.skip_logic[choice_index].value['skip_logic']

    def is_next_action(self, choice, *actions):
        if self.has_skipping:
            return self.next_action(choice) in actions
        return False

    def next_page(self, choice):
        logic = self.skip_logic[self.choice_index(choice)]
        return logic.value[self.next_action(choice)]


class Quiz(QuestionnairePage, AbstractForm):
    base_form_class = QuizForm
    form_builder = CustomFormBuilder

    parent_page_types = [
        "home.HomePage", "home.Section", "home.Article", "questionnaires.QuizIndexPage", 'home.FooterIndexPage',
    ]
    template = "quizzes/quiz.html"

    multi_step = models.BooleanField(
        default=False,
        verbose_name=_("Multi-step"),
        help_text="Whether to display the survey questions to the user one at"
                  " a time, instead of all at once.",
    )

    content_panels = Page.content_panels + [
        FormSubmissionsPanel(),
        MultiFieldPanel(
            [
                FieldPanel("allow_anonymous_submissions"),
                FieldPanel("allow_multiple_submissions"),
                FieldPanel("submit_button_text"),
                FieldPanel("multi_step"),
            ],
            heading=_(
                "General settings for quiz",
            ),
        ),
        MultiFieldPanel(
            [
                StreamFieldPanel("description"),
            ],
            heading=_(
                "Description at quiz page",
            ),
        ),
        MultiFieldPanel(
            [
                StreamFieldPanel("thank_you_text"),
            ],
            heading="Description at thank you page",
        ),
        FieldPanel('index_page_description'),
        FieldPanel('index_page_description_line_2'),
        MultiFieldPanel(
            [
                StreamFieldPanel("terms_and_conditions"),
            ],
            heading="Terms and conditions",
        ),
        SvgChooserPanel('icon'),
        ImageChooserPanel('image_icon'),
        InlinePanel("quiz_form_fields", label="Form fields"),
    ]

    @cached_property
    def has_page_breaks(self):
        return any(
            field.page_break
            for field in self.get_form_fields()
        )

    def get_form_class_for_step(self, step):
        return self.form_builder(step.object_list).get_form_class()

    def get_form_fields(self):
        return self.quiz_form_fields.all()

    def get_submission_class(self):
        return UserSubmission

    def get_context(self, request, *args, **kwargs):
        context = super().get_context(request, *args, **kwargs)
        context.update({'back_url': request.GET.get('back_url')})
        context.update({'form_length': request.GET.get('form_length')})

        form_helper = FormHelper(pk=self.pk, request=request)
        if self.multi_step:
            form_data = form_helper.get_full_form_data()
        else:
            form_data = request.POST
        if request.method == 'POST' and form_data:
            form = self.get_form(
                form_data,
                page=self, user=request.user
            )

            fields_info = {}

            total = 0
            total_correct = 0
            form_data = dict(form.data)
            for field in self.get_form_fields():
                correct_answer = field.correct_answer.split('|')

                if field.field_type == 'checkbox':
                    answer = 'true' if form_data.get(field.clean_name) else 'false'
                else:
                    answer = form_data.get(field.clean_name)

                if type(answer) != list:
                    answer = [str(answer)]

                if field.field_type in ['radio', 'dropdown']:
                    is_correct = set(answer).issubset(set(correct_answer))
                else:
                    is_correct = set(answer) == set(correct_answer)

                if is_correct:
                    total_correct += 1
                total += 1
                fields_info[field.clean_name] = {
                    'feedback': field.feedback,
                    'correct_answer': field.correct_answer,
                    'correct_answer_list': correct_answer,
                    'is_correct': is_correct,
                }

            context['form'] = form
            context['fields_info'] = fields_info
            context['result'] = {
                'total': total,
                'total_correct': total_correct,
            }

            if self.multi_step:
                form_helper.remove_session_data()

        return context

    class Meta:
        verbose_name = _("quiz")
        verbose_name_plural = _("quizzes")


class PollIndexPage(Page):
    parent_page_types = ['home.HomePage']
    subpage_types = ['questionnaires.Poll']


class SurveyIndexPage(Page):
    parent_page_types = ['home.HomePage']
    subpage_types = ['questionnaires.Survey']


class QuizIndexPage(Page):
    parent_page_types = ['home.HomePage']
    subpage_types = ['questionnaires.Quiz']


# questionnaires/paginators.py

from rest_framework import pagination


class IoGTPagination(pagination.PageNumberPagination):
    page_size = 15
    max_page_size = 100
    page_size_query_param = 'page_size'


# questionnaires/utils.py

from __future__ import unicode_literals

import json

from django.core.paginator import Page, Paginator
from django.core.serializers.json import DjangoJSONEncoder

from .blocks import SkipState


class SkipLogicPaginator(Paginator):
    def __init__(self, object_list, new_answers=dict, previous_answers=dict):
        self.new_answers = new_answers.copy()
        self.previous_answers = previous_answers

        super().__init__(object_list, per_page=1)

        self.question_labels = [
            question.clean_name for question in self.object_list
        ]

        self.page_breaks = [
            i + 1 for i, field in enumerate(self.object_list)
            if field.has_skipping or field.page_break
        ]

        num_questions = self.object_list.count()

        if self.page_breaks:
            # Always have a break at start to create first page
            self.page_breaks.insert(0, 0)
            if self.page_breaks[-1] != num_questions:
                # Must break for last page
                self.page_breaks.append(num_questions)
        else:
            # display one question per page
            self.page_breaks = list(range(num_questions + 1))

    def _get_page(self, *args, **kwargs):
        return SkipLogicPage(*args, **kwargs)

    @property
    def num_pages(self):
        return len(self.page_breaks) - 1

    @property
    def last_question_index(self):
        return self.page_breaks[self.current_page] - 1

    @property
    def current_page(self):
        # search backwards to ensure we find correct lower bound
        reversed_breaks = reversed(self.page_breaks)
        page_break = next(
            index for index in reversed_breaks
            if index <= self.first_question_index
        )
        return self.page_breaks.index(page_break) + 1

    @property
    def first_question_index(self):
        last_answer = self.last_question_previous_page
        if last_answer >= 0:
            return self.next_question_from_previous_index(
                last_answer, self.previous_answers)
        return 0

    @property
    def last_question_previous_page(self):
        previous_answers_indexes = self.index_of_questions(
            self.previous_answers)
        try:
            return max(previous_answers_indexes)
        except ValueError:
            # There have been no previous questions, its the first page
            return -1

    def next_question_from_previous_index(self, index, data):
        last_question = self.object_list[index]
        last_answer = data.get(last_question.clean_name)
        if last_question.is_next_action(last_answer, SkipState.QUESTION):
            # Sorted or is 0 based in the backend and 1 on the front
            next_question_id = last_question.next_page(last_answer) - 1
            question_ids = [
                question.sort_order for question in self.object_list
            ]
            return question_ids.index(next_question_id)

        return index + 1

    @property
    def next_question_index(self):
        if self.new_answers:
            return self.next_question_from_previous_index(
                self.last_question_index,
                self.new_answers,
            )
        return 0

    @property
    def next_page(self):
        try:
            return next(
                page for page, break_index in enumerate(self.page_breaks)
                if break_index > self.next_question_index
            )
        except StopIteration:
            return self.num_pages

    @property
    def previous_page(self):
        # Prevent returning 0 if on the first page
        return max(1, next(
            page for page, break_index in enumerate(self.page_breaks)
            if break_index > self.last_question_previous_page
        ))

    def index_of_questions(self, data):
        return [
            self.question_labels.index(question) for question in data
            if question in self.question_labels
        ]

    @property
    def missing_checkboxes(self):
        return [
            question
            for question in self.object_list[
                # Correct for the slice
                self.first_question_index:self.last_question_index + 1
            ]
            if question.field_type == 'checkbox' and
            question.clean_name not in self.new_answers
        ]

    def page(self, number):
        number = self.validate_number(number)
        index = number - 1
        if not self.new_answers:
            top_index = index + self.per_page
            bottom = self.page_breaks[index]
            top = self.page_breaks[top_index]
        elif self.previous_page == number or self.current_page == number:
            # We are rebuilding the page with the data just submitted
            bottom = self.first_question_index
            # Correct for the slice
            top = self.last_question_index + 1
        else:
            index = self.next_page - 1
            bottom = self.next_question_index
            top_index = index + self.per_page
            top = self.page_breaks[top_index]

        return self._get_page(self.object_list[bottom:top], index + 1, self)


class SkipLogicPage(Page):
    def has_next(self):
        return super(SkipLogicPage, self).has_next() and not self.is_end()

    @property
    def last_question(self):
        return self.object_list[-1]

    @property
    def last_response(self):
        return self.paginator.new_answers[self.last_question.clean_name]

    def is_next_action(self, *actions):
        try:
            question_response = self.last_response
        except KeyError:
            return False
        return self.last_question.is_next_action(question_response, *actions)

    def is_end(self):
        return self.is_next_action(SkipState.END)

    def next_page_number(self):
        return self.paginator.next_page

    def previous_page_number(self):
        return self.paginator.previous_page


class FormHelper:
    def __init__(self, pk, request):
        self.request = request
        self.session_data_key = f'form_data-{pk}'
        self.complete_session_data_key = f'form_data-{pk}-completed'
        step_number = request.GET.get('p')

        if step_number is None:
            self.remove_session_data()

    def remove_session_data(self):
        self.request.session.pop(self.session_data_key, None)
        self.request.session.pop(self.complete_session_data_key, None)

    def get_form_data(self):
        return json.loads(self.request.session.get(self.session_data_key, '{}'))

    def set_form_data(self, form_data):
        self.request.session[self.session_data_key] = json.dumps(form_data, cls=DjangoJSONEncoder)

    def set_full_form_data(self):
        self.request.session[self.complete_session_data_key] = self.request.session.pop(self.session_data_key, None)

    def get_full_form_data(self):
        return json.loads(self.request.session.get(self.complete_session_data_key, '{}'))


# questionnaires/views.py

import csv
import json
from collections import defaultdict

from django.conf import settings
from django.contrib import messages
from django.contrib.auth import get_user_model
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
from django.urls import reverse
from django.utils import timezone
from wagtail.admin.views.mixins import SpreadsheetExportMixin, Echo
from wagtail.contrib.forms.forms import SelectDateForm
from wagtail.contrib.forms.utils import get_forms_for_user
from wagtail.contrib.forms.views import (
    SubmissionsListView,
    FormPagesListView as WagtailFormPagesListView,
    SafePaginateListView,
)
from wagtail.core.models import Page
from xlsxwriter.workbook import Workbook

from questionnaires.forms import GenerateDashboardForm
from questionnaires.models import UserSubmission, SurveyFormField, PollFormField, QuizFormField
from questionnaires.superset.utils import DashboardGenerator

User = get_user_model()


class FormPagesListView(WagtailFormPagesListView):
    def get_queryset(self):
        from home.models import SiteSettings

        queryset = super().get_queryset()
        registration_survey = SiteSettings.get_for_default_site().registration_survey
        if registration_survey:
            ids = registration_survey.get_translations(inclusive=True).values_list('id', flat=True)
            if ids:
                queryset = queryset.exclude(id__in=ids)
        return queryset


class CustomSubmissionsListView(SubmissionsListView):
    def get_filename(self):
        return self.form_page.get_export_filename()

    def get_queryset(self):
        return super().get_queryset().select_related('page', 'user')


class FormDataPerUserView(SpreadsheetExportMixin, SafePaginateListView):
    """
    This view is taking inspiration from `wagtail.contrib.forms.views.FormPagesListView` and
    `wagtail.contrib.forms.views.SubmissionsListView`
    """
    template_name = "questionnaires/form_data.html"
    context_object_name = 'form_pages'
    list_export = ['ID', 'Name', 'Submission Date', 'Field', 'Value']
    select_date_form = SelectDateForm
    page_ids = []
    user = None

    def dispatch(self, request, *args, **kwargs):
        self.is_export = (self.request.GET.get('export') in self.FORMATS)
        user_id = self.request.GET.get('user_id')
        self.user = get_object_or_404(User, pk=user_id)
        if self.is_export:
            page_ids = self.request.GET.get('page_ids')
            self.page_ids = page_ids.split(',') if page_ids else []
            self.paginate_by = None
        return super().dispatch(request, *args, **kwargs)

    def get_queryset(self):
        queryset = get_forms_for_user(self.request.user)
        queryset = queryset.filter(
            usersubmission__user_id=self.user.id
        ).distinct().order_by('-last_published_at')

        filtering = self.get_filtering()
        if filtering and isinstance(filtering, dict):
            queryset = queryset.filter(**filtering)

        return queryset

    def get_filtering(self, for_form_pages=True):
        filter_name = 'usersubmission__submit_time__date' if for_form_pages else 'submit_time__date'
        self.select_date_form = SelectDateForm(self.request.GET)
        result = dict()
        if self.select_date_form.is_valid():
            date_from = self.select_date_form.cleaned_data.get('date_from')
            date_to = self.select_date_form.cleaned_data.get('date_to')
            if date_to:
                if date_from:
                    result[f'{filter_name}__range'] = [date_from, date_to]
                else:
                    result[f'{filter_name}__lte'] = date_to
            elif date_from:
                result[f'{filter_name}__gte'] = date_from
        return result

    def render_to_response(self, context, **response_kwargs):
        if self.is_export:
            return self.as_spreadsheet(context['submissions'], self.request.GET.get('export'))
        return super().render_to_response(context, **response_kwargs)

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        form_pages = context[self.context_object_name]
        if self.is_export:
            if self.page_ids:
                form_pages = form_pages.filter(id__in=self.page_ids)
            context['submissions'] = UserSubmission.objects.select_related('page', 'user').filter(
                user_id=self.user.id, page__in=form_pages, **self.get_filtering(for_form_pages=False)
            ).order_by('-submit_time')
        context.update({
            'select_date_form': self.select_date_form,
            'user_id': self.user.id,
        })
        return context

    def get_form_fields_dict(self):
        context = self.get_context_data()
        form_fields_dict = defaultdict(dict)
        for form_field in [PollFormField, SurveyFormField, QuizFormField]:
            for (page_id, clean_name, admin_label) in form_field.objects.filter(
                    page__in=context[self.context_object_name]).values_list('page_id', 'clean_name', 'admin_label'):
                form_fields_dict[page_id].update({
                    clean_name: admin_label or clean_name
                })
        return form_fields_dict

    def get_rows(self, item, form_fields_dict):
        data = {
            'User': item.user.username,
            'URL': item.page.full_url,
        }
        for clean_name, answer in json.loads(item.form_data).items():
            data.update({
                form_fields_dict.get(item.page_id, {}).get(clean_name, clean_name): answer,
            })
        return [dict(zip(self.list_export, [item.id, item.page.title, item.submit_time, field, value]))
                for field, value in data.items()]

    def stream_csv(self, queryset):
        writer = csv.DictWriter(Echo(), fieldnames=self.list_export)
        yield writer.writerow(dict(zip(self.list_export, self.list_export)))

        form_fields_dict = self.get_form_fields_dict()

        for item in queryset:
            rows = self.get_rows(item, form_fields_dict)
            for row in rows:
                yield self.write_csv_row(writer, row)

    def write_xlsx(self, queryset, output):
        workbook = Workbook(
            output,
            {
                "in_memory": True,
                "constant_memory": True,
                "remove_timezone": True,
                "default_date_format": "dd/mm/yy hh:mm:ss",
            },
        )
        worksheet = workbook.add_worksheet()
        row_number = 0
        for col_number, field in enumerate(self.list_export):
            worksheet.write(row_number, col_number, self.get_heading(queryset, field))

        form_fields_dict = self.get_form_fields_dict()

        row_number += 1
        for item in queryset:
            rows = self.get_rows(item, form_fields_dict)
            for row in rows:
                self.write_xlsx_row(worksheet, row, row_number)
                row_number += 1

        workbook.close()

    def get_filename(self):
        timestamp = timezone.now().strftime(settings.EXPORT_FILENAME_TIMESTAMP_FORMAT)
        return f'{self.user.username}-submission_{timestamp}'


def generate_dashboard(request, pk):
    if request.method == 'POST':
        form = GenerateDashboardForm(request.POST)
        if form.is_valid():
            questionnaire = get_object_or_404(Page, pk=pk).specific
            data = form.cleaned_data
            dashboard_generator = DashboardGenerator(
                request.user, questionnaire, data.get('superset_username'), data.get('superset_password'))
            dashboard_url = dashboard_generator.generate()
            messages.add_message(
                request,
                messages.SUCCESS,
                f'Dashboard generated successfully. Click <a href="{dashboard_url}" '
                f'style="color: #FFFFFF; text-decoration: underline;" target="_blank">here</a> to view.')
            return HttpResponseRedirect(reverse('wagtailforms:index'))
    else:
        form = GenerateDashboardForm()

    return render(request, 'questionnaires/generate_dashboard.html', {'form': form})


# questionnaires/wagtail_hooks.py

from django.urls import path
from django.utils.html import format_html_join
from django.templatetags.static import static

from wagtail.core import hooks
from wagtail.core.models import Locale

from questionnaires.views import FormPagesListView, FormDataPerUserView, generate_dashboard


@hooks.register('insert_editor_js', order=0)
def editor_js():
    language_code = Locale.get_active().language_code
    js_files = [
        'js/surveys.js',
    ]
    js_includes = format_html_join('\n', '<script src="{0}"></script>',
        ((static(filename),) for filename in js_files)
    )
    js_includes = f'{js_includes}\n<script src="/{language_code}/jsi18n/"></script>'
    return js_includes


@hooks.register('register_admin_urls')
def register_custom_form_pages_list_view():
    return [
      path('forms/', FormPagesListView.as_view(), name='index'),
      path('form-data/', FormDataPerUserView.as_view(), name='form_data_per_user'),
      path('generate-dashboard/<int:pk>/', generate_dashboard, name='generate_dashboard'),
  ]


# questionnaires/widgets.py

from django import forms
from wagtail.core.telepath import register

from questionnaires.adapters import SelectAndHiddenWidgetAdapter


class SelectAndHiddenWidget(forms.MultiWidget):
    def __init__(self, *args, **kwargs):
        widgets = [forms.HiddenInput, forms.Select]
        super(SelectAndHiddenWidget, self).__init__(
            widgets=widgets,
            *args,
            **kwargs
        )

    def decompress(self, value):
        return [value, None]

    def value_from_datadict(self, *args):
        value = super(SelectAndHiddenWidget, self).value_from_datadict(*args)
        return value[1]


register(SelectAndHiddenWidgetAdapter(), SelectAndHiddenWidget)


# search/views.py

from django.core.paginator import EmptyPage, Paginator
from django.template.response import TemplateResponse
from wagtail.core.models import Locale

from home.models import Article, Section
from iogt.settings.base import SEARCH_RESULTS_PER_PAGE
from wagtail.search.models import Query
from questionnaires.models import Poll, Quiz, Survey


def search(request):
    search_groups = (Article, Poll, Quiz, Section, Survey)

    search_query = request.GET.get("query")
    page = int(request.GET.get("page", 1))

    results = {"search_query": search_query, "search_groups": {}}

    # Search
    sandbox_section = Section.objects.filter(slug='sandbox').first()
    sandbox_page_ids = []
    if sandbox_section:
        sandbox_page_ids = sandbox_section.get_descendants(inclusive=True).values_list('id', flat=True)

    for search_group in search_groups:
        search_results_count = 0
        if search_query:
            search_results = search_group.objects.exclude(id__in=sandbox_page_ids).live().\
                filter(locale=Locale.objects.get(language_code=request.LANGUAGE_CODE)).specific().search(search_query)
            search_results_count = search_results.count()
            query = Query.get(search_query)

            # Record hit
            query.add_hit()
        else:
            search_results = search_group.objects.none()

        # Pagination
        paginator = Paginator(search_results, SEARCH_RESULTS_PER_PAGE)
        try:
            search_results = paginator.page(page)
        except EmptyPage:
            search_results = paginator.page(paginator.num_pages)

        if search_results:
            results["search_groups"][search_group._meta.verbose_name_plural] = {
                "search_results": search_results,
                "search_results_count": search_results_count,
            }

    return TemplateResponse(request, "search/search.html", results)


# selenium_tests/base.py

from django.db import connections
from django.conf import settings
from django.core.management import call_command
from django.test import LiveServerTestCase
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from wagtail.core.models import Page, Site
from wagtail_factories import SiteFactory

from home.factories import HomePageFactory
from selenium_tests.pages import BasePage, LoginPage, LogoutPage


class BaseSeleniumTests(LiveServerTestCase):

    fixtures = ['selenium_tests/locales.json']

    host = settings.SE_APP_HOST

    @classmethod
    def setUpClass(cls):
        cls.selenium = create_remote_webdriver()
        super(BaseSeleniumTests, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        cls.selenium.quit()
        super(BaseSeleniumTests, cls).tearDownClass()

    def visit_page(self, page: Page) -> BasePage:
        self.visit_url(page.url)
        return BasePage(self.selenium)

    def visit_login_page(self) -> LoginPage:
        self.visit_url('/accounts/login/')
        return LoginPage(self.selenium)

    def visit_logout_page(self) -> LogoutPage:
        self.visit_url('/accounts/logout/')
        return LogoutPage(self.selenium)

    def visit_url(self, url: str) -> None:
        url = '%s%s' % (self.live_server_url, url)
        self.selenium.get(url)

    def setup_blank_site(self):
        Site.objects.all().delete()
        self.home = HomePageFactory()
        self.site = SiteFactory(
            site_name='IoGT',
            hostname=self.host,
            port=self.port,
            is_default_site=True,
            root_page=self.home
        )

    # https://github.com/wagtail/wagtail/issues/1824#issuecomment-450575883
    # Identical to TransactionTestCase._fixture_teardown except that 'allow_cascade' is
    # forced.
    def _fixture_teardown(self):
        # Allow TRUNCATE ... CASCADE and don't emit the post_migrate signal
        # when flushing only a subset of the apps
        for db_name in self._databases_names(include_mirrors=False):
            # Flush the database
            inhibit_post_migrate = (
                self.available_apps is not None or
                (   # Inhibit the post_migrate signal when using serialized
                    # rollback to avoid trying to recreate the serialized data.
                    self.serialized_rollback and
                    hasattr(connections[db_name], '_test_serialized_contents')
                )
            )
            call_command('flush', verbosity=0, interactive=False,
                         database=db_name, reset_sequences=False,
                         allow_cascade=True,
                         inhibit_post_migrate=inhibit_post_migrate)


def create_remote_webdriver(preset: str = "chrome") -> webdriver.Remote:
    if preset == "chrome":
        options = webdriver.ChromeOptions()
        options.add_argument('--ignore-ssl-errors=yes')
        options.add_argument('--ignore-certificate-errors')
        options.add_argument("--window-size=480,720")
        options.add_argument("--start-maximized")
        driver = webdriver.Remote(
            command_executor=get_hub_url(),
            desired_capabilities=DesiredCapabilities.CHROME,
            options=options,
        )
    elif preset == "firefox":
        options = webdriver.FirefoxOptions()
        driver = webdriver.Remote(
            command_executor=get_hub_url(),
            desired_capabilities=DesiredCapabilities.FIREFOX,
            options=options,
        )
    else:
        raise Exception(
            f"Invalid webdriver preset ('{preset}'); must be 'chrome' or 'firefox'"
        )
    driver.set_page_load_timeout(60)
    driver.implicitly_wait(10)
    return driver


def get_hub_url():
    return f"http://{settings.SE_HUB_HOST}:{settings.SE_HUB_PORT}/wd/hub"


# selenium_tests/pages.py

from typing import List
from urllib.parse import urlparse

from selenium.webdriver.common.by import By
from selenium.webdriver.support.relative_locator import locate_with
from selenium.webdriver.remote.webdriver import WebDriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import WebDriverException
from wagtail.core.models import Page


def safe_click(driver, button):
    try:
        button.click()
    except WebDriverException:
        driver.execute_script("arguments[0].click();", button)

def visible_with_size(item):
    return (
        item.size['width'] > 0
        and item.size['height'] > 0
        and item.is_displayed()
    )
class BasePage(object):

    body_text_locator = (By.TAG_NAME, 'body')
    content_text_locator = (By.CLASS_NAME, 'content')
    message_text_locator = (By.CLASS_NAME, 'messages')
    search_button_locator = (By.CLASS_NAME, 'xs-home-header__search')
    navbar_locator = (By.CSS_SELECTOR, '.top-level')
    footer_locator = (By.CSS_SELECTOR, '.bottom-level')

    def __init__(self, driver: WebDriver) -> None:
        self.driver = driver        

    def url_matches(self, path: str) -> bool:
        return urlparse(self.driver.current_url).path == path

    def get_body_text(self):
        return self.driver.find_element(*self.body_text_locator).text

    def get_content_text(self):
        return self.driver.find_element(*self.content_text_locator).text

    def get_messages_text(self):
        return self.driver.find_element(*self.message_text_locator).text

    def small_search_button_select(self):
        return self.driver.find_element(*self.search_button_locator).click()
    
    def footer_below_navbar_below_content(self):
        content = self.driver.find_element(*self.content_text_locator)
        navbar = self.driver.find_element(locate_with(*self.navbar_locator).below(content))
        footer = self.driver.find_element(locate_with(*self.footer_locator).below(navbar))
        if navbar.is_displayed() and footer.is_displayed():
            return True
        else:
            return False
    
    def footer_rightof_content_rightof_navbar(self):
        navbar = self.driver.find_element(*self.navbar_locator)
        content = self.driver.find_element(locate_with(*self.content_text_locator).to_right_of(navbar))        
        footer = self.driver.find_element(locate_with(*self.footer_locator).to_right_of(content))
        if content.is_displayed() and footer.is_displayed():
            return True
        else:
            return False

    @property
    def footer(self) -> 'FooterElement':
        return FooterElement(self.driver)

    @property
    def navbar(self) -> 'NavbarElement':
        return NavbarElement(self.driver)

class HomePage(BasePage):

    banner_area_locator = (By.CSS_SELECTOR, "section[class='banner-holder']")
    banner_image_locator = (By.CSS_SELECTOR, "img[alt='An image']")

    def __init__(self, driver: WebDriver) -> None:
        self.driver = driver

    def has_banner(self):
        banner = self.driver.find_element(*self.banner_area_locator)
        print()
        return visible_with_size(banner)            
    
    def click_banner(self):
        banner_image = self.driver.find_element(*self.banner_image_locator)
        banner_image.click()


class LoginPage(BasePage):    

    user_name_locator = (By.NAME, 'login')
    password_locator = (By.NAME, 'password')
    submit_locator = (By.CSS_SELECTOR, "button[type='Submit']")

    def __init__(self, driver: WebDriver) -> None:
        self.driver = driver
        
        self.login_user_name = driver.find_element(*self.user_name_locator)
        self.login_password = driver.find_element(*self.password_locator)
        self.login_submit = driver.find_element(*self.submit_locator)

    def get_username(self):
        return self.login_user_name
 
    def get_password(self):
        return self.login_password
 
    def get_login_submit(self):
        return self.login_submit

    def login_user(self, user):
        self.login_user_name.send_keys(user.username)
        self.login_password.send_keys('test@123')
        self.login_submit.click()

class LogoutPage(BasePage):    

    logout_button_locator = (By.CSS_SELECTOR, "button[type='Submit']")

    def __init__(self, driver: WebDriver) -> None:
        self.driver = driver
        
        self.logout_submit = driver.find_element(*self.logout_button_locator)

    def logout_user(self):
        self.logout_submit.click()

class SearchPage(BasePage):
      
    search_area_locator = (By.CLASS_NAME, "profile-form__input")
    search_button_locator = (By.CSS_SELECTOR, "button[type='submit']")

    def __init__(self, driver: WebDriver) -> None:
        self.driver = driver      
        self.search_area = self.driver.find_element(*self.search_area_locator)
        self.search_submit = driver.find_element(*self.search_button_locator)

    def search(self, searchtext):
       
        self.search_area.send_keys(searchtext)
        self.search_submit.click()

class ArticlePage(BasePage):

    heading_locator = (By.TAG_NAME, 'h1')
    lead_image_locator = (By.CLASS_NAME, 'article__lead-img-featured')
    navigate_next_locator = (By.CLASS_NAME, 'article__navigation--next')
    navigate_previous_locator = (By.CLASS_NAME, 'article__navigation--previous')

    def __init__(self, driver: WebDriver) -> None:
        self.driver = driver

    @property
    def comments_section(self) -> 'CommentsSectionElement':
        return CommentsSectionElement(self.driver)
        
    @property
    def title(self) -> str:
        return self.driver.find_element(*self.heading_locator).text

    def has_lead_image(self):
        lead_image = self.driver.find_element(*self.lead_image_locator)
        return visible_with_size(lead_image)
         
    def navigate_next(self):

        safe_click(self.driver, self.driver.find_element(*self.navigate_next_locator))
        return BasePage(self.driver)  

    def navigate_previous(self):
        safe_click(self.driver, self.driver.find_element(*self.navigate_previous_locator))
        return BasePage(self.driver)

class SectionPage(BasePage):

    heading_locator = (By.CLASS_NAME, 'image-overlay__text')
    lead_image_locator = (By.CLASS_NAME, 'lead-img')
    

    def __init__(self, driver: WebDriver) -> None:
        self.driver = driver
        
    @property
    def title(self) -> str:
        return self.driver.find_element(*self.heading_locator).text

    def has_lead_image(self):
        lead_image = self.driver.find_element(*self.lead_image_locator)
        return visible_with_size(lead_image)
         
    def navigate_featured_content(self, LinkText):
        safe_click(self.driver, self.driver.find_element(By.PARTIAL_LINK_TEXT, LinkText))

class QuestionnairePage(BasePage):

    heading_locator = (By.TAG_NAME, 'h1')
    checkbox_locator = (By.CSS_SELECTOR, "input[type='checkbox']")
    date_locator = (By.CSS_SELECTOR, "input[type='date']")
    date_time_locator = (By.CSS_SELECTOR, "input[type='datetime-local']")
    dropdown_locator = (By.CLASS_NAME, 'quest-item__input')
    email_locator = (By.CSS_SELECTOR, "input[type='email']")
    text_locator = (By.CLASS_NAME, 'cust-input')
    number_locator = (By.CSS_SELECTOR, "input[type='number']")
    radio_locator = (By.CSS_SELECTOR, "input[type='radio']")
    url_locator = (By.CSS_SELECTOR, "input[type='url']")
    submit_locator = (By.CSS_SELECTOR, "button[type='submit']")
    login_submit_locator = (By.PARTIAL_LINK_TEXT, "Log in to participate")

    def __init__(self, driver: WebDriver) -> None:
        self.driver = driver
        
    @property
    def title(self) -> str:
        return self.driver.find_element(*self.heading_locator).text

    def submit_response(self):
        safe_click(self.driver,self.driver.find_element(*self.submit_locator))

    def login_button_submit(self):
        safe_click(self.driver,self.driver.find_element(*self.login_submit_locator))

    def select_checkbox(self):
        self.driver.find_element(*self.checkbox_locator).click()

    def select_checkboxes(self, option):
        safe_click(self.driver, self.driver.find_element(By.CSS_SELECTOR, "input[value='" + option + "']"))

    def enter_date(self, date):
        date_input = self.driver.find_element(*self.date_locator)
        date_input.click()
        date_input.send_keys(date)

    def enter_date_time(self, date, time):
        date_input = self.driver.find_element(*self.date_time_locator)
        safe_click(self.driver, date_input)
        date_input.send_keys(date)
        date_input.send_keys(Keys.TAB)
        date_input.send_keys(time)

    def use_dropdown(self, question, selection):
        select = Select(self.driver.find_element(By.NAME,question))
        select.select_by_visible_text(selection)
        return select.first_selected_option.text

    def enter_email(self, email):
        input = self.driver.find_element(*self.email_locator)
        input.send_keys(email)

    def enter_text(self, text):
        input = self.driver.find_element(*self.text_locator)
        detailinput = input.find_element(By.CSS_SELECTOR, "input[type='text']")
        detailinput.send_keys(text)

    def enter_multiline_text(self, text):
        input = self.driver.find_element(*self.text_locator)
        detailinput = input.find_element(By.CSS_SELECTOR, "textarea")
        detailinput.send_keys(text)

    def enter_number(self, number):
        input = self.driver.find_element(*self.number_locator)
        input.send_keys(number)

    def select_radio(self, option):
        self.driver.find_element(By.CSS_SELECTOR, "input[value='" + option + "']").click()

    def enter_url(self, url):
        input = self.driver.find_element(*self.url_locator)
        input.send_keys(url)

class QuestionnaireResultsPage(BasePage):

    poll_result_locator = (By.CLASS_NAME, 'polls-widget__form')
    quiz_result_locator = (By.CLASS_NAME, 'quiz-answer-banner__answers')
    back_button_locator = (By.PARTIAL_LINK_TEXT, 'Back')

    def __init__(self, driver: WebDriver) -> None:
        self.driver = driver

    def get_poll_results_text(self):
        return self.driver.find_element(*self.poll_result_locator).text

    def get_quiz_results_text(self):
        return self.driver.find_element(*self.quiz_result_locator).text

    def go_back(self):
        safe_click(self.driver, self.driver.find_element(*self.back_button_locator))

class BaseElement():
    locator = (By.TAG_NAME, 'html')

    def __init__(self, driver: WebDriver) -> None:
        self.driver = driver
        self.html = self.driver.find_element(*self.locator)

    @property
    def is_displayed(self) -> bool:
        return self.html.is_displayed()

class CommentsSectionElement(BaseElement):
    locator =  (By.CLASS_NAME, 'comments')

    comment_area_locator = (By.NAME, "comment")
    reply_comment_area_locator = (By.NAME, "comment")
    comment_holder_locator = (By.CLASS_NAME, 'comments-holder')
    leave_comment_locator = (By.CSS_SELECTOR, "input[value='Leave comment']")
    delete_comment_locator = (By.LINK_TEXT, 'Remove')
    reply_comment_locator = (By.LINK_TEXT,'Reply')
    report_comment_locator = (By.LINK_TEXT,'Report') 
    report_button_locator = (By.CSS_SELECTOR, "input[value='Report']")

    def submit_comment(self, text):
        self.comment_area = self.driver.find_element(*self.comment_area_locator)
        self.comment_area.send_keys(text)
        safe_click(self.driver, self.driver.find_element(*self.leave_comment_locator))

    def retrieve_comments(self):
        self.comment_holder = self.driver.find_element(*self.comment_holder_locator)
        return self.comment_holder.text

    def delete_last_comment(self):
        safe_click(self.driver, self.driver.find_element(*self.delete_comment_locator))

    def reply_last_comment(self, reply):
        safe_click(self.driver, self.driver.find_element(*self.reply_comment_locator))
        self.submit_comment(reply)

    def report_last_comment(self):
        safe_click(self.driver, self.driver.find_element(*self.report_comment_locator))
        self.driver.find_element(*self.report_button_locator).click()
    

class FooterElement(BaseElement):
    locator = (By.CSS_SELECTOR, '.footer-main .bottom-level')

    @property
    def items(self) -> List['FooterItemElement']:
        return [
            FooterItemElement(self.driver, el)
            for el in self.html.find_elements(By.CSS_SELECTOR, 'nav a')
        ]

class FooterItemElement():
    def __init__(self, driver, el) -> None:
        self.driver = driver
        self.html = el

    @property
    def title(self) -> str:
        return self.html.text

    @property
    def has_icon(self) -> bool:
        icon = self.html.find_element(By.TAG_NAME, 'img')
        return visible_with_size(icon)

    @property
    def background_color(self):
        return self.html.value_of_css_property('background-color')

    @property
    def font_color(self):
        return self.html.value_of_css_property('color')

    def click(self) -> 'BasePage':
        self.html.click()
        return BasePage(self.driver)


class NavbarElement(FooterElement):
    locator = (By.CSS_SELECTOR, '.footer-main .top-level')


class WagtailAdminPage(object):
    submit_locator = (By.CSS_SELECTOR, "button[type='submit']")

    def __init__(self, driver: WebDriver) -> None:
        super().__init__()
        self.driver = driver

    def _select_skip_to_question(self, question_id):
        selected_skip_logic = Select(self.driver.find_element(By.ID, question_id))
        selected_skip_logic.select_by_value("question")

    def _select_skip_to_answer(self, answer_id, skip_to):
        selected_answer = Select(self.driver.find_element(By.ID, answer_id))
        selected_answer.select_by_value(f'{skip_to}')

    def _get_page_type(self):
        page_id = int(self.driver.current_url.split('/')[-3])
        return Page.objects.get(id=page_id).specific.get_type

    def skip_to_question(self, question, skip_logic, skip_to):
        page_type = self._get_page_type()
        skip_logic_question_id = f'{page_type}_form_fields-{question}-skip_logic-{skip_logic}-value-skip_logic'
        skip_logic_answer_id = f'{page_type}_form_fields-{question}-skip_logic-{skip_logic}-value-question_1'

        self._select_skip_to_question(skip_logic_question_id)
        self._select_skip_to_answer(skip_logic_answer_id, skip_to)

    def submit_response(self):
        safe_click(self.driver, self.driver.find_elements(*self.submit_locator)[1])


# manage.py

#!/usr/bin/env python
import os
import sys

if __name__ == "__main__":
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "iogt.settings.dev")

    from django.core.management import execute_from_command_line

    execute_from_command_line(sys.argv)


# iogt_content_migration/management/commands/load_logo_and_favicon.py

from pathlib import Path

from django.conf import settings
from django.core.files.images import ImageFile
from django.core.management.base import BaseCommand
from wagtail.images.models import Image

from home.models import SiteSettings


class Command(BaseCommand):
    def handle(self, *args, **options):
        self._load_logo_and_favicon()

        self.stdout.write(self.style.SUCCESS('Loaded successfully.'))

    def _open_file(self, path):
        try:
            return open(path, 'rb')
        except:
            self.stdout.write(self.style.WARNING(f'File not found: {path}.'))

    def _load_logo_and_favicon(self):
        file = ImageFile(self._open_file(Path(settings.BASE_DIR) / 'iogt/static/images/logo.png'), name='logo.png')
        logo = Image.objects.create(title='Logo', file=file)

        file = ImageFile(self._open_file(Path(settings.BASE_DIR) / 'iogt/static/images/favicon.png'), name='favicon.png')
        favicon = Image.objects.create(title='Favicon', file=file)

        site_settings = SiteSettings.objects.all()
        for site_setting in site_settings:
            site_setting.logo = logo
            site_setting.favicon = favicon

        SiteSettings.objects.bulk_update(site_settings, ['logo', 'favicon'])


# iogt_content_migration/management/commands/load_svg.py

import os

from django.core.files import File
from django.core.management.base import BaseCommand
from wagtailsvg.models import Svg


class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            '--svg-dir',
            required=True,
            help='Path to IoGT SVG directory'
        )
        parser.add_argument(
            '--clean',
            required=False,
            action='store_true',
            help='Remove existing SVG'
        )

    def handle(self, *args, **options):
        self.svg_dir = options.get('svg_dir')
        clean = options.get('clean')
        if clean:
            self._clean()
        self._load_svg()

        self.stdout.write(self.style.SUCCESS('SVG loaded successfully.'))

    def _clean(self):
        Svg.objects.all().delete()
        self.stdout.write(self.style.ERROR('Existing SVG removed.'))

    def _open_svg_file(self, svg_path):
        try:
            return open(svg_path, 'rb')
        except:
            self.stdout.write(self.style.WARNING(f'SVG file not found: {svg_path}.'))

    def _load_svg(self):
        for dir in os.listdir(self.svg_dir):
            dir_path = os.path.join(self.svg_dir, dir)
            if os.path.isdir(dir_path):
                for svg_file in os.listdir(dir_path):
                    svg_path = os.path.join(dir_path, svg_file)
                    file = self._open_svg_file(svg_path)
                    if file:
                        svg = Svg.objects.create(title=svg_file.strip('.svg'), file=File(file, name=svg_file))
                        svg.tags.add(dir)


# iogt_content_migration/management/commands/load_v1_db.py

from collections import defaultdict
from pathlib import Path

import csv
from bs4 import BeautifulSoup
from django.conf import settings
from django.core.files import File
from django.core.management.base import BaseCommand
from django.urls import reverse
from wagtail.core.models import Page, Site, Locale, Collection, PageRevision
from django.core.files.images import ImageFile
from wagtail.documents.models import Document
from wagtail.images.models import Image
from wagtail_localize.models import Translation
from wagtail_localize.operations import TranslationCreator
from wagtailmarkdown.utils import _get_bleach_kwargs
from wagtailmedia.models import Media
from wagtailsvg.models import Svg

import home.models as models
from comments.models import CommentStatus
from home.models import V1ToV2ObjectMap, V1PageURLToV2PageMap
from questionnaires.models import Poll, PollFormField, Survey, SurveyFormField, Quiz, QuizFormField
import psycopg2
import psycopg2.extras
import json

from questionnaires.models import PollIndexPage, SurveyIndexPage, QuizIndexPage


class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            '--host',
            default='0.0.0.0',
            help='IoGT V1 database host'
        )
        parser.add_argument(
            '--port',
            default='5432',
            help='IoGT V1 database port'
        )
        parser.add_argument(
            '--name',
            default='postgres',
            help='IoGT V1 database name'
        )
        parser.add_argument(
            '--user',
            default='postgres',
            help='IoGT V1 database user'
        )
        parser.add_argument(
            '--password',
            default='',
            help='IoGT V1 database password'
        )
        parser.add_argument(
            '--media-dir',
            required=True,
            help='**RELATIVE Path** to IoGT v1 media directory'
        )
        parser.add_argument(
            '--delete-users',
            action='store_true',
            help='Delete existing Users and their associated data. Use carefully'
        )
        parser.add_argument(
            '--v1-domains',
            nargs="+",
            required=True,
            help="IoGT V1 domains for manually inserted internal links, --v1-domains domain1 domain2 ..."
        )
        parser.add_argument(
            '--v2-domain',
            required=False,
            help="IoGT V2 domains to be used for URLs in the migration report"
        )
        parser.add_argument(
            '--sort',
            required=True,
            help='Sort page by "type1" or "type2"'
        )

    def handle(self, *args, **options):
        self.db_connect(options)
        self.media_dir = options.get('media_dir')
        self.v1_domains_list = options.get('v1_domains')
        self.sort = options.get('sort')
        self.v2_domain = options.get('v2_domain')

        self.collection_map = {}
        self.document_map = {}
        self.media_map = {}
        self.image_map = {}
        self.page_translation_map = {}
        self.v1_to_v2_page_map = {}
        self.post_migration_report_messages = defaultdict(list)
        self.registration_survey_translations = defaultdict()
        self.home_page_seo_title = ''

        self.clear()
        self.stdout.write('Existing site structure cleared')

        root = Page.get_first_root_node()
        self.migrate(root)
        self.print_post_migration_report()

    def clear(self):
        PageRevision.objects.all().delete()
        models.MiscellaneousIndexPage.objects.all().delete()
        models.PageLinkPage.objects.all().delete()
        PollFormField.objects.all().delete()
        Poll.objects.all().delete()
        SurveyFormField.objects.all().delete()
        Survey.objects.all().delete()
        QuizFormField.objects.all().delete()
        Quiz.objects.all().delete()
        models.FeaturedContent.objects.all().delete()
        models.ArticleRecommendation.objects.all().delete()
        models.FooterPage.objects.all().delete()
        models.FooterIndexPage.objects.all().delete()
        models.BannerPage.objects.all().delete()
        models.BannerIndexPage.objects.all().delete()
        models.Article.objects.all().delete()
        models.Section.objects.all().delete()
        models.SectionIndexPage.objects.all().delete()
        models.HomePage.objects.all().delete()
        Site.objects.all().delete()
        Image.objects.all().delete()
        Document.objects.all().delete()
        Media.objects.all().delete()
        V1ToV2ObjectMap.objects.all().delete()

    def db_connect(self, options):
        connection_string = self.create_connection_string(options)
        self.stdout.write(f'DB connection string created, string={connection_string}')
        self.v1_conn = psycopg2.connect(connection_string)
        self.stdout.write('Connected to v1 DB')

    def __del__(self):
        try:
            self.v1_conn.close()
            self.stdout.write('Closed connection to v1 DB')
        except AttributeError:
            pass

    def create_connection_string(self, options):
        host = options.get('host', '0.0.0.0')
        port = options.get('port', '5432')
        name = options.get('name', 'postgres')
        user = options.get('user', 'postgres')
        password = options.get('password', '')
        return f"host={host} port={port} dbname={name} user={user} password={password}"

    def db_query(self, q):
        cur = self.v1_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
        cur.execute(q)
        return cur

    def migrate(self, root):
        self.migrate_collections()
        self.migrate_documents()
        self.migrate_media()
        self.migrate_images()
        self.migrate_locales()
        self.load_page_translation_map()
        self.home_page = self.create_home_page(root)
        self.translate_home_pages()
        self.create_index_pages()
        self.translate_index_pages()
        self.migrate_sections()
        self.migrate_articles()
        self.migrate_footers()
        self.migrate_polls()
        self.migrate_surveys()
        self.migrate_banners()
        self.mark_pages_which_are_not_translated_in_v1_as_draft()
        Page.fix_tree(fix_paths=True)
        self.mark_empty_sections_as_draft()
        self.fix_articles_body()
        self.fix_footers_body()
        self.fix_survey_description()
        self.fix_banner_link_page()
        self.attach_banners_to_home_page()
        self.migrate_recommended_articles_for_article()
        self.migrate_featured_articles_for_homepage()
        self.add_surveys_from_surveys_index_page_to_footer_index_page_as_page_link_page()
        self.add_polls_from_polls_index_page_to_footer_index_page_as_page_link_page()
        self.add_polls_from_polls_index_page_to_home_page_featured_content()
        self.add_surveys_from_surveys_index_page_to_home_page_featured_content()
        self.move_footers_to_end_of_footer_index_page()
        self.migrate_article_related_sections()
        self.migrate_social_media_links()
        self.sort_pages()
        self.populate_registration_survey_translations()
        self.translate_default_survey_submit_button_text()
        self.migrate_post_registration_survey()
        self.migrate_page_revisions()
        self.stop_translations()

    def create_home_page(self, root):
        sql = 'select * ' \
              'from wagtailcore_site wcs, core_sitesettings css, core_main cm, wagtailcore_page wcp ' \
              'where wcs.id = css.site_id ' \
              'and wcs.root_page_id = cm.page_ptr_id ' \
              'and cm.page_ptr_id = wcp.id ' \
              'and wcs.is_default_site = true'
        cur = self.db_query(sql)
        main = cur.fetchone()
        cur.close()
        if not main:
            raise Exception('Could not find a main page in v1 DB')

        sql = 'select * ' \
              'from core_sitelanguage ' \
              'where is_main_language = true'
        cur = self.db_query(sql)
        language = cur.fetchone()
        cur.close()
        if not language:
            raise Exception('Could not find a main language in v1 DB')

        locale = Locale.objects.get(language_code=self._get_iso_locale(language['locale']))
        models.LocaleDetail.objects.update_or_create(
            locale=locale,
            defaults={
                'is_active': True,
                'is_main_language': True,
            }
        )

        self.home_page_seo_title = main['seo_title']

        home = models.HomePage(
            title=main['title'],
            draft_title=main['draft_title'],
            slug=main['slug'],
            live=main['live'],
            locked=main['locked'],
            go_live_at=main['go_live_at'],
            expire_at=main['expire_at'],
            first_published_at=main['first_published_at'],
            last_published_at=main['last_published_at'],
            search_description=main['search_description'],
            seo_title=main['seo_title'],
            locale=locale
        )
        root.add_child(instance=home)
        V1ToV2ObjectMap.create_map(content_object=home, v1_object_id=main['page_ptr_id'])

        Site.objects.create(
            hostname=self.v2_domain or self.v1_domains_list[0],
            port=443,
            root_page=home,
            is_default_site=True,
            site_name=main['site_name'] if main['site_name'] else 'Internet of Good Things',
        )
        logo = self.image_map.get(main['logo_id'])
        if logo:
            site_settings = models.SiteSettings.get_for_default_site()
            site_settings.logo_id = logo.id
            site_settings.save()
        else:
            self.post_migration_report_messages['other'].append(
                'Not site logo found. Using default site logo.'
            )

        sql = f'select * ' \
              f'from core_sitesettings css, wagtailcore_site wcs ' \
              f'where css.site_id = wcs.id ' \
              f'and wcs.is_default_site = true'
        cur = self.db_query(sql)
        for row in cur:
            social_media_links = json.loads(row['social_media_links_on_footer_page'])
            if social_media_links:
                links = []
                for social_media_link in social_media_links:
                    value = social_media_link.get('value')
                    if value:
                        links.append({
                            'title': value.get('title'),
                            'link': value.get('link'),
                        })

                self.post_migration_report_messages['social_media_links'].append(
                    f'site: {row["site_name"]}, hostname: {row["hostname"]} has following social media links '
                    f'{[(link["title"], link["link"]) for link in links]}.')

        cur.close()

        self.post_migration_report_messages['other'].append(
            'A default favicon has been chosen for the site.'
        )

        return home

    def create_index_pages(self):
        self.section_index_page = models.SectionIndexPage(title='Sections')
        self.home_page.add_child(instance=self.section_index_page)

        self.banner_index_page = models.BannerIndexPage(title='Banners')
        self.home_page.add_child(instance=self.banner_index_page)

        self.footer_index_page = models.FooterIndexPage(title='Footers')
        self.home_page.add_child(instance=self.footer_index_page)

        self.poll_index_page = PollIndexPage(title='Polls')
        self.home_page.add_child(instance=self.poll_index_page)

        self.survey_index_page = SurveyIndexPage(title='Surveys')
        self.home_page.add_child(instance=self.survey_index_page)

        self.quiz_index_page = QuizIndexPage(title='Quizzes')
        self.home_page.add_child(instance=self.quiz_index_page)

        self.miscellaneous_index_page = models.MiscellaneousIndexPage(title='Miscellaneous')
        self.home_page.add_child(instance=self.miscellaneous_index_page)

    def migrate_collections(self):
        cur = self.db_query('select * from wagtailcore_collection')
        for row in cur:
            collection, _ = Collection.objects.get_or_create(
                name=row['name'],
                defaults={
                    'path': row['path'],
                    'depth': row['depth'],
                    'numchild': row['numchild'],
                }
            )
            collection.save()
            self.collection_map.update({row['id']: collection})
            V1ToV2ObjectMap.create_map(content_object=collection, v1_object_id=row['id'])

        cur.close()
        self.stdout.write('Collections migrated')

    def migrate_documents(self):
        cur = self.db_query('select * from wagtaildocs_document')
        content_type = self.find_content_type_id('wagtaildocs', 'document')
        for row in cur:
            if not row['file']:
                self.post_migration_report_messages['document_file_not_found'].append(
                    f'Document file path not found, id={row["id"]}'
                )
                continue

            file = self.open_file(row['file'])
            if file:
                document = Document.objects.create(
                    title=row['title'],
                    file=File(file, name=row['file'].split('/')[-1]),
                    created_at=row['created_at'],
                    collection=self.collection_map.get(row['collection_id']),
                )
                V1ToV2ObjectMap.create_map(content_object=document, v1_object_id=row['id'])
                tags = self.find_tags(content_type, row['id'])
                if tags:
                    document.tags.add(*tags)
                self.document_map.update({row['id']: document})
        cur.close()
        self.stdout.write('Documents migrated')

    def migrate_media(self):
        cur = self.db_query('select * from core_molomedia')
        content_type = self.find_content_type_id('core', 'molomedia')
        for row in cur:
            if not row['file']:
                self.post_migration_report_messages['media_file_not_found'].append(
                    f'Media file path not found, id={row["id"]}'
                )
                continue

            file = self.open_file(row['file'])
            if file:
                thumbnail = self.open_file(row['thumbnail'])
                media = Media.objects.create(
                    title=row['title'],
                    file=File(file, name=row['file'].split('/')[-1]),
                    type=row['type'],
                    duration=row['duration'],
                    thumbnail=File(thumbnail, name=row['thumbnail'].split('/')[-1]) if thumbnail else None,
                    created_at=row['created_at'],
                    collection=self.collection_map.get(row['collection_id']),
                )
                V1ToV2ObjectMap.create_map(content_object=media, v1_object_id=row['id'])
                tags = self.find_tags(content_type, row['id'])
                if tags:
                    media.tags.add(*tags)
                self.media_map.update({row['id']: media})
        cur.close()
        self.stdout.write('Media migrated')

    def migrate_images(self):
        cur = self.db_query('select * from wagtailimages_image')
        content_type = self.find_content_type_id('wagtailimages', 'image')
        for row in cur:
            if not row['file']:
                self.post_migration_report_messages['image_file_not_found'].append(
                    f'Image file path not found, id={row["id"]}'
                )
                continue

            image_file = self.open_file(row['file'])
            if image_file:
                self.stdout.write(f"Creating image, file={row['file']}")
                image = Image.objects.create(
                    title=row['title'],
                    file=ImageFile(image_file, name=row['file'].split('/')[-1]),
                    focal_point_x=row['focal_point_x'],
                    focal_point_y=row['focal_point_y'],
                    focal_point_width=row['focal_point_width'],
                    focal_point_height=row['focal_point_height'],
                    created_at=row['created_at'],
                    collection=self.collection_map.get(row['collection_id']),
                )
                V1ToV2ObjectMap.create_map(content_object=image, v1_object_id=row['id'])
                image.get_file_size()
                image.get_file_hash()
                tags = self.find_tags(content_type, row['id'])
                if tags:
                    image.tags.add(*tags)
                self.image_map.update({row['id']: image})
        cur.close()
        self.stdout.write('Images migrated')

    def migrate_locales(self):
        sql = f'select * ' \
              f'from core_sitelanguage'
        cur = self.db_query(sql)
        should_deactivate_en_locale = True
        for row in cur:
            language_code = self._get_iso_locale(row['locale'])
            if language_code == 'en':
                should_deactivate_en_locale = False
            locale, __ = Locale.objects.get_or_create(language_code=language_code)
            models.LocaleDetail.objects.get_or_create(
                locale=locale,
                defaults={
                    'is_active': True,
                    'is_main_language': False,
                }
            )

        if should_deactivate_en_locale:
            locale, __ = Locale.objects.get_or_create(language_code='en')
            models.LocaleDetail.objects.update_or_create(
                locale=locale,
                defaults={
                    'is_active': False,
                    'is_main_language': False,
                }
            )
        cur.close()

    def find_content_type_id(self, app_label, model):
        cur = self.db_query(f"select id from django_content_type where app_label = '{app_label}' and model = '{model}'")
        content_type = cur.fetchone()
        cur.close()
        return content_type.get('id')

    def open_file(self, file):
        file_path = Path(self.media_dir) / file
        try:
            return open(file_path, 'rb')
        except:
            self.post_migration_report_messages['file_not_found'].append(
                f"File not found: {file_path}"
            )

    def find_tags(self, content_type, object_id):
        tags_query = 'select t.name from taggit_tag t join taggit_taggeditem ti on t.id = ti.tag_id where ti.content_type_id = {} and ti.object_id = {}'
        cur = self.db_query(tags_query.format(content_type, object_id))
        tags = [tag['name'] for tag in cur]
        cur.close()
        return tags

    def migrate_sections(self):
        sql = f"select * " \
              f"from core_sectionpage csp, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where csp.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        section_page_translations = []
        for row in cur:
            if row['page_ptr_id'] in self.page_translation_map:
                section_page_translations.append(row)
            else:
                self.create_section(row)
        else:
            for row in section_page_translations:
                section = self.v1_to_v2_page_map.get(self.page_translation_map[row['page_ptr_id']])
                locale, __ = Locale.objects.get_or_create(language_code=self._get_iso_locale(row['locale']))

                try:
                    self.translate_page(locale=locale, page=section)
                except:
                    self.post_migration_report_messages['untranslated_sections'].append(
                        f"Unable to translate section, title={row['title']}"
                    )
                    continue

                translated_section = section.get_translation_or_none(locale)
                if translated_section:
                    commenting_status, commenting_open_time, commenting_close_time = self._get_commenting_fields(row)

                    translated_section.lead_image = self.image_map.get(row['image_id'])
                    translated_section.title = row['title']
                    translated_section.slug = row['slug']
                    translated_section.draft_title = row['draft_title']
                    translated_section.live = row['live']
                    translated_section.locked = row['locked']
                    translated_section.go_live_at = row['go_live_at']
                    translated_section.expire_at = row['expire_at']
                    translated_section.first_published_at = row['first_published_at']
                    translated_section.last_published_at = row['last_published_at']
                    translated_section.search_description = row['search_description']
                    translated_section.seo_title = row['seo_title']
                    translated_section.font_color = self.get_color_hex(row['extra_style_hints']) or section.font_color
                    translated_section.larger_image_for_top_page_in_list_as_in_v1 = False
                    translated_section.commenting_status = CommentStatus.DISABLED
                    translated_section.commenting_starts_at = commenting_open_time
                    translated_section.commenting_ends_at = commenting_close_time
                    translated_section.latest_revision_created_at = row['latest_revision_created_at']
                    translated_section.save()
                    self.add_warning_for_sections_with_description(row, section)
                    content_type = self.find_content_type_id('core', 'sectionpage')
                    tags = self.find_tags(content_type, row['page_ptr_id'])
                    if tags:
                        translated_section.tags.add(*tags)

                    V1ToV2ObjectMap.create_map(content_object=translated_section, v1_object_id=row['page_ptr_id'])
                    V1PageURLToV2PageMap.create_map(url=row['url_path'], page=translated_section)

                    self.v1_to_v2_page_map.update({
                        row['page_ptr_id']: translated_section
                    })
                    if row['description'] is None:
                        self.post_migration_report_messages['sections_with_null_description'].append(
                            f'title: {translated_section.title}. URL: {translated_section.full_url}. '
                            f'Admin URL: {self.get_admin_url(translated_section.id)}.'
                        )
                    self.post_migration_report_messages['disabled_larger_image_for_top_page_in_list_as_in_v1'].append(
                        f'title: {translated_section.title}. URL: {translated_section.full_url}. '
                        f'Admin URL: {self.get_admin_url(translated_section.id)}.'
                    )
                    if commenting_status != CommentStatus.DISABLED:
                        self.post_migration_report_messages['disabled_comments_for_pages_which_has_other_settings'].append(
                            f'title: {translated_section.title}. URL: {translated_section.full_url}. '
                            f'Admin URL: {self.get_admin_url(translated_section.id)}. Original Setting: {commenting_status}.'
                        )

                self.stdout.write(f"Translated section, title={row['title']}")
        cur.close()

    def mark_empty_sections_as_draft(self):
        for section in models.Section.objects.all():
            if section.get_children().filter(live=True).count() == 0:
                section.live = False
                section.save(update_fields=['live'])

    def create_section(self, row):
        commenting_status, commenting_open_time, commenting_close_time = self._get_commenting_fields(row)

        section = models.Section(
            lead_image=self.image_map.get(row['image_id']),
            title=row['title'],
            draft_title=row['draft_title'],
            show_in_menus=True,
            slug=row['slug'],
            path=self.section_index_page.path + row['path'][12:],
            depth=row['depth'],
            numchild=row['numchild'],
            live=row['live'],
            locked=row['locked'],
            go_live_at=row['go_live_at'],
            expire_at=row['expire_at'],
            first_published_at=row['first_published_at'],
            last_published_at=row['last_published_at'],
            commenting_status=CommentStatus.DISABLED,
            commenting_starts_at=commenting_open_time,
            commenting_ends_at=commenting_close_time,
            search_description=row['search_description'],
            seo_title=row['seo_title'],
            font_color=self.get_color_hex(row['extra_style_hints']),
            larger_image_for_top_page_in_list_as_in_v1=False,
            latest_revision_created_at=row['latest_revision_created_at'],
        )
        section.save()
        self.add_warning_for_sections_with_description(row, section)
        content_type = self.find_content_type_id('core', 'sectionpage')
        tags = self.find_tags(content_type, row['page_ptr_id'])
        if tags:
            section.tags.add(*tags)

        V1ToV2ObjectMap.create_map(content_object=section, v1_object_id=row['page_ptr_id'])
        V1PageURLToV2PageMap.create_map(url=row['url_path'], page=section)

        self.v1_to_v2_page_map.update({
            row['page_ptr_id']: section
        })
        if row['description'] is None:
            self.post_migration_report_messages['sections_with_null_description'].append(
                f'title: {section.title}. URL: {section.full_url}. '
                f'Admin URL: {self.get_admin_url(section.id)}.'
            )
        self.post_migration_report_messages['disabled_larger_image_for_top_page_in_list_as_in_v1'].append(
            f'title: {section.title}. URL: {section.full_url}. '
            f'Admin URL: {self.get_admin_url(section.id)}.'
        )
        if commenting_status != CommentStatus.DISABLED:
            self.post_migration_report_messages['disabled_comments_for_pages_which_has_other_settings'].append(
                f'title: {section.title}. URL: {section.full_url}. '
                f'Admin URL: {self.get_admin_url(section.id)}. Original Setting: {commenting_status}.'
            )
        self.stdout.write(f"saved section, title={section.title}")

    def add_warning_for_sections_with_description(self, row, section):
        if row['description']:
            self.post_migration_report_messages['sections_with_description'].append(
                f'title: {section.title}. URL: {section.full_url}. '
                f'Admin URL: {self.get_admin_url(section.id)}. '
                f'Description (not migrated): {row["description"]}.'
            )

    def add_warning_for_articles_with_subtitle(self, row, article):
        if row['subtitle']:
            self.post_migration_report_messages['articles_with_subtitle'].append(
                f'title: {article.title}. URL: {article.full_url}. '
                f'Admin URL: {self.get_admin_url(article.id)}. '
                f'Subtitle (not migrated): {row["subtitle"]}.'
            )

    def migrate_articles(self):
        sql = f"select * " \
              f"from core_articlepage cap, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where cap.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id " \
              f"and wcp.path like '000100010002%' " \
              f"order by wcp.path"
        cur = self.db_query(sql)

        article_page_translations = []
        for row in cur:
            if row['page_ptr_id'] in self.page_translation_map:
                article_page_translations.append(row)
            else:
                self.create_article(row)
        else:
            for row in article_page_translations:
                article = self.v1_to_v2_page_map.get(self.page_translation_map[row['page_ptr_id']])
                locale, __ = Locale.objects.get_or_create(language_code=self._get_iso_locale(row['locale']))

                try:
                    self.translate_page(locale=locale, page=article)
                except:
                    self.post_migration_report_messages['untranslated_articles'].append(
                        f"Unable to translate article, title={row['title']}"
                    )
                    continue

                translated_article = article.get_translation_or_none(locale)

                if translated_article:
                    commenting_status, commenting_open_time, commenting_close_time = self._get_commenting_fields(row)

                    translated_article.lead_image = self.image_map.get(row['image_id'])
                    translated_article.title = row['title']
                    translated_article.slug = row['slug']
                    translated_article.draft_title = row['draft_title']
                    translated_article.live = row['live']
                    translated_article.locked = row['locked']
                    translated_article.go_live_at = row['go_live_at']
                    translated_article.expire_at = row['expire_at']
                    translated_article.first_published_at = row['first_published_at']
                    translated_article.last_published_at = row['last_published_at']
                    translated_article.search_description = row['search_description']
                    translated_article.seo_title = row['seo_title']
                    # translated_article.index_page_description = row['subtitle']
                    translated_article.commenting_status = CommentStatus.DISABLED
                    translated_article.commenting_starts_at = commenting_open_time
                    translated_article.commenting_ends_at = commenting_close_time
                    translated_article.latest_revision_created_at = row['latest_revision_created_at']
                    translated_article.save()
                    self.add_warning_for_articles_with_subtitle(row, translated_article)
                    if row['featured_in_latest']:
                        self.post_migration_report_messages['articles_featured_in_latest'].append(
                            f'title: {article.title}. URL: {article.full_url}. '
                            f'Admin URL: {self.get_admin_url(article.id)}.'
                        )

                    content_type = self.find_content_type_id('core', 'articlepage')
                    tags = self.find_tags(content_type, row['page_ptr_id'])
                    if tags:
                        translated_article.tags.add(*tags)

                    V1ToV2ObjectMap.create_map(content_object=translated_article, v1_object_id=row['page_ptr_id'])
                    V1PageURLToV2PageMap.create_map(url=row['url_path'], page=translated_article)

                    self.v1_to_v2_page_map.update({
                        row['page_ptr_id']: translated_article
                    })

                    if commenting_status != CommentStatus.DISABLED:
                        self.post_migration_report_messages['disabled_comments_for_pages_which_has_other_settings'].append(
                            f'title: {translated_article.title}. URL: {translated_article.full_url}. '
                            f'Admin URL: {self.get_admin_url(translated_article.id)}. Original Setting: {commenting_status}.'
                        )

                self.stdout.write(f"Translated article, title={row['title']}")
        cur.close()

    def _get_commenting_fields(self, row):
        comments_map = {
            'O': CommentStatus.OPEN,
            'C': CommentStatus.CLOSED,
            'D': CommentStatus.DISABLED,
            'T': CommentStatus.TIMESTAMPED
        }

        commenting_status = comments_map[row['commenting_state']] if row['commenting_state'] else CommentStatus.INHERITED
        return commenting_status, row['commenting_open_time'], row['commenting_close_time']

    def create_article(self, row):
        commenting_status, commenting_open_time, commenting_close_time = self._get_commenting_fields(row)

        article = models.Article(
            lead_image=self.image_map.get(row['image_id']),
            title=row['title'],
            draft_title=row['draft_title'],
            slug=row['slug'],
            path=self.section_index_page.path + row['path'][12:],
            depth=row['depth'],
            numchild=row['numchild'],
            live=row['live'],
            locked=row['locked'],
            go_live_at=row['go_live_at'],
            expire_at=row['expire_at'],
            first_published_at=row['first_published_at'],
            last_published_at=row['last_published_at'],
            commenting_status=CommentStatus.DISABLED,
            commenting_starts_at=commenting_open_time,
            commenting_ends_at=commenting_close_time,
            search_description=row['search_description'],
            seo_title=row['seo_title'],
            # index_page_description=row['subtitle'],
            latest_revision_created_at=row['latest_revision_created_at'],
        )
        try:
            article.save()
            self.add_warning_for_articles_with_subtitle(row, article)
            if row['featured_in_latest']:
                self.post_migration_report_messages['articles_featured_in_latest'].append(
                    f'title: {article.title}. URL: {article.full_url}. '
                    f'Admin URL: {self.get_admin_url(article.id)}.'
                )

            content_type = self.find_content_type_id('core', 'articlepage')
            tags = self.find_tags(content_type, row['page_ptr_id'])
            if tags:
                article.tags.add(*tags)

            V1ToV2ObjectMap.create_map(content_object=article, v1_object_id=row['page_ptr_id'])
            V1PageURLToV2PageMap.create_map(url=row['url_path'], page=article)

            self.v1_to_v2_page_map.update({
                row['page_ptr_id']: article
            })
            if commenting_status != CommentStatus.DISABLED:
                self.post_migration_report_messages['disabled_comments_for_pages_which_has_other_settings'].append(
                    f'title: {article.title}. URL: {article.full_url}. '
                    f'Admin URL: {self.get_admin_url(article.id)}. Original Setting: {commenting_status}.'
                )
        except Page.DoesNotExist:
            self.post_migration_report_messages['articles'].append(
                f"Skipping article with missing parent: title={row['title']}"
            )
            return
        self.stdout.write(f"saved article, title={article.title}")

    def get_unsupported_html_tags(self, value):
        bleach_kwargs = _get_bleach_kwargs()

        unsupported_html_tags = []
        tags = BeautifulSoup(value, "html.parser").find_all()
        for tag in tags:
            if tag.name not in bleach_kwargs['tags']:
                unsupported_html_tags.append(tag.name)

        return unsupported_html_tags

    def _map_body(self, type_, row, v2_body):
        for block in v2_body:
            if block['type'] == 'paragraph':
                unsupported_html_tags = self.get_unsupported_html_tags(block['value'])
                if unsupported_html_tags:
                    block['type'] = 'paragraph_v1_legacy'
                    page = self.v1_to_v2_page_map.get(row['page_ptr_id'])
                    if page:
                        self.post_migration_report_messages['page_with_unsupported_tags'].append(
                            f'title: {page.title}. URL: {page.full_url}. '
                            f'Admin URL: {self.get_admin_url(page.id)}. '
                            f'Tags: {unsupported_html_tags}.'
                        )
                else:
                    block['type'] = 'markdown'

                if bool([domain for domain in self.v1_domains_list if domain in block['value']]):
                    page = self.v1_to_v2_page_map.get(row['page_id'])
                    self.post_migration_report_messages['sections_with_internal_links'].append(
                        f"title: {page.title}. URL: {page.full_url}. "
                        f"Admin URL: {self.get_admin_url(page.id)}.")

            elif block['type'] == 'richtext':
                block['type'] = 'paragraph'

                if bool([domain for domain in self.v1_domains_list if domain in block['value']]):
                    page = self.v1_to_v2_page_map.get(row['page_id'])
                    self.post_migration_report_messages['sections_with_internal_links'].append(
                        f"title: {page.title}. URL: {page.full_url}. "
                        f"Admin URL: {self.get_admin_url(page.id)}.")

            elif block['type'] == 'image':
                image = self.image_map.get(block['value'])
                if image:
                    block['value'] = image.id
                else:
                    page = self.v1_to_v2_page_map.get(row['page_ptr_id'])
                    if page:
                        self.post_migration_report_messages['page_with_empty_image'].append(
                            f'title: {page.title}. URL: {page.full_url}. '
                            f'Admin URL: {self.get_admin_url(page.id)}. '
                            f'Image ID: {block["value"]}'
                        )
                    else:
                        self.post_migration_report_messages['invalid_image_id'].append(
                            f"title={row['title']} has image with invalid id {block['value']}"
                        )
                    block['value'] = None
            elif block['type'] == 'media':
                media = self.media_map.get(block['value'])
                if media:
                    block['value'] = media.id
                else:
                    self.post_migration_report_messages['invalid_media_id'].append(
                        f"title={row['title']} has media with invalid id {block['value']}"
                    )
                    block['value'] = None
            elif block['type'] == 'page':
                block['type'] = 'page_button'
                page = self.v1_to_v2_page_map.get(block['value'])
                if page:
                    block['value'] = {'page': page.id, 'text': ''}
                else:
                    block['value'] = {'page': None, 'text': ''}
                    self.post_migration_report_messages['invalid_page_id'].append(
                        f'Unable to attach v2 page for {type_[:-1]}, title={row["title"]}'
                    )
            elif block['type'] == 'list':
                value = ''
                for li in block['value']:
                    value += f'* {li}\r\n'
                block['type'] = 'markdown'
                block['value'] = value
            elif block['type'] == 'numbered_list':
                value = ''
                for i,li in enumerate(block['value']):
                    value += f'{i+1}. {li}\r\n'
                block['type'] = 'markdown'
                block['value'] = value

        return v2_body

    def map_article_body(self, row):
        v1_body = json.loads(row['body'])

        v2_body = self._map_body('articles', row, v1_body)

        if row['subtitle']:
            v2_body = [{
                'type': 'paragraph',
                'value': row['subtitle'],
            }] + v2_body
        return json.dumps(v2_body)

    def migrate_banners(self):
        sql = f"select * " \
              f"from core_bannerpage cbp, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where cbp.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        banner_page_translations = []
        for row in cur:
            if row['page_ptr_id'] in self.page_translation_map:
                banner_page_translations.append(row)
            else:
                self.create_banner(row)
        else:
            for row in banner_page_translations:
                banner = self.v1_to_v2_page_map.get(self.page_translation_map[row['page_ptr_id']])
                locale, __ = Locale.objects.get_or_create(language_code=self._get_iso_locale(row['locale']))

                try:
                    self.translate_page(locale=locale, page=banner)
                except:
                    self.post_migration_report_messages['untranslated_banners'].append(
                        f"Unable to translate banner, title={row['title']}"
                    )
                    continue

                translated_banner = banner.get_translation_or_none(locale)
                if translated_banner:
                    translated_banner.banner_image = self.image_map.get(row['banner_id'])
                    translated_banner.title = row['title']
                    translated_banner.slug = row['slug']
                    translated_banner.draft_title = row['draft_title']
                    translated_banner.live = row['live']
                    translated_banner.locked = row['locked']
                    translated_banner.go_live_at = row['go_live_at']
                    translated_banner.expire_at = row['expire_at']
                    translated_banner.first_published_at = row['first_published_at']
                    translated_banner.last_published_at = row['last_published_at']
                    translated_banner.search_description = row['search_description']
                    translated_banner.seo_title = row['seo_title']
                    translated_banner.latest_revision_created_at = row['latest_revision_created_at']
                    translated_banner.save()

                    if row['external_link']:
                        self.post_migration_report_messages['banners_with_external_link'].append(
                            f'title: {translated_banner.title}. URL: {translated_banner.full_url}. '
                            f'Admin URL: {self.get_admin_url(translated_banner.id)}. '
                            f'External link: {row["external_link"]}.'
                        )

                    V1ToV2ObjectMap.create_map(content_object=translated_banner, v1_object_id=row['page_ptr_id'])
                    V1PageURLToV2PageMap.create_map(url=row['url_path'], page=translated_banner)

                    self.v1_to_v2_page_map.update({
                        row['page_ptr_id']: translated_banner
                    })

                self.stdout.write(f"Translated banner, title={row['title']}")
        cur.close()

    def create_banner(self, row):
        banner = models.BannerPage(
            banner_image=self.image_map.get(row['banner_id']),
            title=row['title'],
            draft_title=row['draft_title'],
            slug=row['slug'],
            path=self.banner_index_page.path + row['path'][12:],
            depth=row['depth'],
            numchild=row['numchild'],
            live=row['live'],
            locked=row['locked'],
            go_live_at=row['go_live_at'],
            expire_at=row['expire_at'],
            first_published_at=row['first_published_at'],
            last_published_at=row['last_published_at'],
            search_description=row['search_description'],
            seo_title=row['seo_title'],
            latest_revision_created_at=row['latest_revision_created_at'],
        )
        banner.save()

        if row['external_link']:
            self.post_migration_report_messages['banners_with_external_link'].append(
                f'title: {banner.title}. URL: {banner.full_url}. '
                f'Admin URL: {self.get_admin_url(banner.id)}. '
                f'External link: {row["external_link"]}.'
            )

        V1ToV2ObjectMap.create_map(content_object=banner, v1_object_id=row['page_ptr_id'])
        V1PageURLToV2PageMap.create_map(url=row['url_path'], page=banner)

        self.v1_to_v2_page_map.update({
            row['page_ptr_id']: banner
        })
        self.stdout.write(f"saved banner, title={banner.title}")

    def map_banner_page(self, row):
        v2_page = None
        v1_banner_link_page_id = row['banner_link_page_id']
        if v1_banner_link_page_id:
            v2_page = self.v1_to_v2_page_map.get(v1_banner_link_page_id)
            if not v2_page:
                self.post_migration_report_messages['banner_page_link'].append(
                    f'Unable to attach v2 page for banner, title={row["title"]}'
                )

        return v2_page

    def migrate_footers(self):
        sql = f"select * " \
              f"from core_footerpage cfp, core_articlepage cap, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where cfp.articlepage_ptr_id = cap.page_ptr_id " \
              f"and cap.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        footer_page_translations = []
        for row in cur:
            if row['page_ptr_id'] in self.page_translation_map:
                footer_page_translations.append(row)
            else:
                self.create_footer(row)
        else:
            for row in footer_page_translations:
                footer = self.v1_to_v2_page_map.get(self.page_translation_map[row['page_ptr_id']])
                locale, __ = Locale.objects.get_or_create(language_code=self._get_iso_locale(row['locale']))

                try:
                    self.translate_page(locale=locale, page=footer)
                except:
                    self.post_migration_report_messages['untranslated_footers'].append(
                        f"Unable to translate footer, title={row['title']}"
                    )
                    continue

                translated_footer = footer.get_translation_or_none(locale)
                if translated_footer:
                    commenting_status, commenting_open_time, commenting_close_time = self._get_commenting_fields(row)

                    image = self.image_map.get(row['image_id'])
                    translated_footer.image_icon = image
                    translated_footer.title = row['title']
                    translated_footer.slug = row['slug']
                    translated_footer.draft_title = row['draft_title']
                    translated_footer.live = row['live']
                    translated_footer.locked = row['locked']
                    translated_footer.go_live_at = row['go_live_at']
                    translated_footer.expire_at = row['expire_at']
                    translated_footer.first_published_at = row['first_published_at']
                    translated_footer.last_published_at = row['last_published_at']
                    translated_footer.search_description = row['search_description']
                    translated_footer.seo_title = row['seo_title']
                    translated_footer.commenting_status = CommentStatus.DISABLED
                    translated_footer.commenting_starts_at = commenting_open_time
                    translated_footer.commenting_ends_at = commenting_close_time
                    translated_footer.latest_revision_created_at = row['latest_revision_created_at']
                    translated_footer.save()
                    if row['featured_in_latest']:
                        self.post_migration_report_messages['footers_featured_in_latest'].append(
                            f'title: {translated_footer.title}. URL: {translated_footer.full_url}. '
                            f'Admin URL: {self.get_admin_url(translated_footer.id)}.'
                        )

                    if image:
                        self.post_migration_report_messages['footers_with_image'].append(
                            f'title: {translated_footer.title}. URL: {translated_footer.full_url}. '
                            f'Admin URL: {self.get_admin_url(translated_footer.id)}.'
                        )

                    V1ToV2ObjectMap.create_map(content_object=translated_footer, v1_object_id=row['page_ptr_id'])
                    V1PageURLToV2PageMap.create_map(url=row['url_path'], page=translated_footer)

                    self.v1_to_v2_page_map.update({
                        row['page_ptr_id']: translated_footer
                    })

                    if commenting_status != CommentStatus.DISABLED:
                        self.post_migration_report_messages['disabled_comments_for_pages_which_has_other_settings'].append(
                            f'title: {translated_footer.title}. URL: {translated_footer.full_url}. '
                            f'Admin URL: {self.get_admin_url(translated_footer.id)}. Original Setting: {commenting_status}.'
                        )

                self.stdout.write(f"Translated footer, title={row['title']}")
        cur.close()

    def create_footer(self, row):
        commenting_status, commenting_open_time, commenting_close_time = self._get_commenting_fields(row)

        image = self.image_map.get(row['image_id'])
        footer = models.Article(
            image_icon=image,
            title=row['title'],
            draft_title=row['draft_title'],
            slug=row['slug'],
            path=self.footer_index_page.path + row['path'][12:],
            depth=row['depth'],
            numchild=row['numchild'],
            live=row['live'],
            locked=row['locked'],
            go_live_at=row['go_live_at'],
            expire_at=row['expire_at'],
            first_published_at=row['first_published_at'],
            last_published_at=row['last_published_at'],
            search_description=row['search_description'],
            seo_title=row['seo_title'],
            commenting_status=CommentStatus.DISABLED,
            commenting_starts_at=commenting_open_time,
            commenting_ends_at=commenting_close_time,
            latest_revision_created_at=row['latest_revision_created_at'],
        )
        footer.save()
        if row['featured_in_latest']:
            self.post_migration_report_messages['footers_featured_in_latest'].append(
                f'title: {footer.title}. URL: {footer.full_url}. '
                f'Admin URL: {self.get_admin_url(footer.id)}.'
            )

        if image:
            self.post_migration_report_messages['footers_with_image'].append(
                f'title: {footer.title}. URL: {footer.full_url}. Admin URL: {self.get_admin_url(footer.id)}.'
            )

        V1ToV2ObjectMap.create_map(content_object=footer, v1_object_id=row['page_ptr_id'])
        V1PageURLToV2PageMap.create_map(url=row['url_path'], page=footer)

        self.v1_to_v2_page_map.update({
            row['page_ptr_id']: footer
        })

        if commenting_status != CommentStatus.DISABLED:
            self.post_migration_report_messages['disabled_comments_for_pages_which_has_other_settings'].append(
                f'title: {footer.title}. URL: {footer.full_url}. '
                f'Admin URL: {self.get_admin_url(footer.id)}. Original Setting: {commenting_status}.'
            )

        self.stdout.write(f"saved footer, title={footer.title}")

    def load_page_translation_map(self):
        sql = "select * " \
              "from core_pagetranslation"
        cur = self.db_query(sql)
        for row in cur:
            if row['translated_page_id'] != row['page_id']:
                self.page_translation_map.update({
                    row['translated_page_id']: row['page_id'],
                })
        cur.close()
        self.stdout.write('Page translation map loaded.')

    def translate_page(self, locale, page):
        translator = TranslationCreator(user=None, target_locales=[locale])
        translator.create_translations(page)

    def stop_translations(self):
        Translation.objects.update(enabled=False)

        self.stdout.write('Translations stopped.')

    def migrate_polls(self):
        sql = f"select * " \
              f"from polls_pollsindexpage ppip, wagtailcore_page wcp " \
              f"where ppip.page_ptr_id = wcp.id " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        v1_poll_index_page = cur.fetchone()
        cur.close()

        self._migrate_polls(v1_poll_index_page, self.poll_index_page)

        sql = f"select * " \
              f"from core_sectionindexpage csip, wagtailcore_page wcp " \
              f"where csip.page_ptr_id = wcp.id " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        v1_section_index_page = cur.fetchone()
        cur.close()

        self._migrate_polls(v1_section_index_page, self.section_index_page)

    def _migrate_polls(self, v1_index_page, v2_index_page):
        sql = f"select * " \
              f"from polls_question pq, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where pq.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id " \
              f"and wcp.path like '{v1_index_page['path']}%' " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        poll_page_translations = []
        for row in cur:
            if row['page_ptr_id'] in self.page_translation_map:
                poll_page_translations.append(row)
            else:
                self.create_poll(v2_index_page, row)
        else:
            for row in poll_page_translations:
                poll = self.v1_to_v2_page_map.get(self.page_translation_map[row['page_ptr_id']])
                locale, __ = Locale.objects.get_or_create(language_code=self._get_iso_locale(row['locale']))

                try:
                    self.translate_page(locale=locale, page=poll)
                except Exception as e:
                    self.post_migration_report_messages['untranslated_polls'].append(
                        f"Unable to translate poll, title={row['title']}"
                    )
                    continue

                translated_poll = poll.get_translation_or_none(locale)
                if translated_poll:
                    translated_poll.title = row['title']
                    translated_poll.slug = row['slug']
                    translated_poll.draft_title = row['draft_title']
                    translated_poll.live = row['live']
                    translated_poll.result_as_percentage = row['result_as_percentage']
                    translated_poll.show_results = row['show_results']
                    translated_poll.locked = row['locked']
                    translated_poll.go_live_at = row['go_live_at']
                    translated_poll.expire_at = row['expire_at']
                    translated_poll.first_published_at = row['first_published_at']
                    translated_poll.last_published_at = row['last_published_at']
                    translated_poll.search_description = row['search_description']
                    translated_poll.seo_title = row['seo_title']
                    translated_poll.randomise_options = row['randomise_options']
                    translated_poll.allow_anonymous_submissions = False
                    translated_poll.allow_multiple_submissions = False
                    translated_poll.latest_revision_created_at = row['latest_revision_created_at']
                    translated_poll.save()

                    V1ToV2ObjectMap.create_map(content_object=translated_poll, v1_object_id=row['page_ptr_id'])
                    V1PageURLToV2PageMap.create_map(url=row['url_path'], page=translated_poll)

                    self.v1_to_v2_page_map.update({
                        row['page_ptr_id']: translated_poll
                    })

                    row['path'] = row['path'][:-4]
                    self.migrate_poll_questions(translated_poll, row)

                self.stdout.write(f"Translated poll, title={row['title']}")
        cur.close()

    def create_poll(self, v2_index_page, row):
        poll = Poll(
            title=row['title'],
            draft_title=row['draft_title'],
            show_in_menus=True,
            slug=row['slug'],
            path=v2_index_page.path + row['path'][12:],
            depth=row['depth'],
            numchild=row['numchild'],
            live=row['live'],
            show_results=row['show_results'],
            result_as_percentage=row['result_as_percentage'],
            locked=row['locked'],
            go_live_at=row['go_live_at'],
            expire_at=row['expire_at'],
            first_published_at=row['first_published_at'],
            last_published_at=row['last_published_at'],
            search_description=row['search_description'],
            seo_title=row['seo_title'],
            randomise_options=row['randomise_options'],
            allow_anonymous_submissions=False,
            allow_multiple_submissions=False,
            latest_revision_created_at=row['latest_revision_created_at'],
        )
        try:
            poll.save()
            V1ToV2ObjectMap.create_map(content_object=poll, v1_object_id=row['page_ptr_id'])
            V1PageURLToV2PageMap.create_map(url=row['url_path'], page=poll)
        except Exception as e:
            self.post_migration_report_messages['polls'].append(
                f"Unable to save poll, title={row['title']}"
            )
            return

        self.migrate_poll_questions(poll, row)

        self.v1_to_v2_page_map.update({
            row['page_ptr_id']: poll
        })
        self.stdout.write(f"saved poll, title={poll.title}")

    def migrate_poll_questions(self, poll, poll_row):
        sql = f'select * ' \
              f'from polls_choice pc, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl ' \
              f'where pc.page_ptr_id = wcp.id ' \
              f'and wcp.path like \'{poll_row["path"]}%\' ' \
              f'and wcp.id = clr.page_id ' \
              f'and clr.language_id = csl.id ' \
              f'and csl.locale = \'{poll_row["locale"]}\' ' \
              f'order by wcp.path'
        cur = self.db_query(sql)
        self.create_poll_question(poll, poll_row, cur)
        cur.close()

    def create_poll_question(self, poll, poll_row, cur):
        PollFormField.objects.filter(page=poll).delete()
        choices = []
        for row in cur:
            choices.append(row['title'])

        choices_length = len(choices)

        if choices_length == 0:
            field_type = 'multiline'
        elif choices_length > 1:
            if poll_row['allow_multiple_choice']:
                field_type = 'checkboxes'
            else:
                field_type = 'radio'
        else:
            self.post_migration_report_messages['poll_questions'].append(
                f'Unable to determine field type for poll={poll_row["title"]}.'
            )
            return

        choices = '|'.join(choices)

        poll_form_field = PollFormField.objects.create(
            page=poll, label=poll.title, field_type=field_type, choices=choices,
            admin_label=poll_row['short_name'] or poll.title)
        if choices:
            cur.scroll(0, 'absolute')
        for row in cur:
            V1ToV2ObjectMap.create_map(content_object=poll_form_field, v1_object_id=row['page_ptr_id'])
        self.stdout.write(f"saved poll question, label={poll.title}")

    def migrate_surveys(self):
        sql = f"select * " \
              f"from surveys_surveysindexpage ssip, wagtailcore_page wcp " \
              f"where ssip.page_ptr_id = wcp.id " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        v1_survey_index_page = cur.fetchone()
        cur.close()

        self._migrate_surveys(v1_survey_index_page, self.survey_index_page)

        sql = f"select * " \
              f"from core_sectionindexpage csip, wagtailcore_page wcp " \
              f"where csip.page_ptr_id = wcp.id " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        v1_section_index_page = cur.fetchone()
        cur.close()

        self._migrate_surveys(v1_section_index_page, self.section_index_page)

    def _migrate_surveys(self, v1_index_page, v2_index_page):
        sql = f"select * " \
              f"from surveys_molosurveypage smsp, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where smsp.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id " \
              f"and wcp.path like '{v1_index_page['path']}%' " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        survey_page_translations = []
        for row in cur:
            if row['page_ptr_id'] in self.page_translation_map:
                survey_page_translations.append(row)
            else:
                self.create_survey(v2_index_page, row)
        else:
            for row in survey_page_translations:
                survey = self.v1_to_v2_page_map.get(self.page_translation_map[row['page_ptr_id']])
                locale, __ = Locale.objects.get_or_create(language_code=self._get_iso_locale(row['locale']))

                try:
                    self.translate_page(locale=locale, page=survey)
                except Exception as e:
                    self.post_migration_report_messages['untranslated_surveys'].append(
                        f"Unable to translate survey, title={row['title']}"
                    )

                    continue

                translated_survey = survey.get_translation_or_none(locale)
                if translated_survey:
                    translated_survey.title = row['title']
                    translated_survey.slug = row['slug']
                    translated_survey.draft_title = row['draft_title']
                    translated_survey.live = row['live']
                    translated_survey.thank_you_text = self.map_survey_thank_you_text(row)
                    translated_survey.allow_anonymous_submissions = row['allow_anonymous_submissions']
                    translated_survey.allow_multiple_submissions = row['allow_multiple_submissions_per_user']
                    translated_survey.submit_button_text = row['submit_text'][:40] if row['submit_text'] else 'Submit'
                    translated_survey.direct_display = row['display_survey_directly']
                    translated_survey.multi_step = row['multi_step']
                    translated_survey.locked = row['locked']
                    translated_survey.go_live_at = row['go_live_at']
                    translated_survey.expire_at = row['expire_at']
                    translated_survey.first_published_at = row['first_published_at']
                    translated_survey.last_published_at = row['last_published_at']
                    translated_survey.search_description = row['search_description']
                    translated_survey.seo_title = row['seo_title']
                    translated_survey.index_page_description = row['homepage_introduction']
                    translated_survey.index_page_description_line_2 = row['homepage_button_text']
                    translated_survey.terms_and_conditions = self.map_survey_terms_and_conditions(row)
                    translated_survey.latest_revision_created_at = row['latest_revision_created_at']
                    translated_survey.save()

                    if row['submit_text'] and len(row['submit_text']) > 40:
                        self.post_migration_report_messages['truncated_submit_button_text'].append(
                            f'title: {translated_survey.title}. URL: {translated_survey.full_url}. '
                            f'Admin URL: {self.get_admin_url(translated_survey.id)}. '
                            f'Full text: {row["submit_text"]}.'
                        )

                    V1ToV2ObjectMap.create_map(content_object=translated_survey, v1_object_id=row['page_ptr_id'])
                    V1PageURLToV2PageMap.create_map(url=row['url_path'], page=translated_survey)

                    self.v1_to_v2_page_map.update({
                        row['page_ptr_id']: translated_survey
                    })

                    self.migrate_survey_questions(translated_survey, row)

                self.stdout.write(f"Translated survey, title={row['title']}")
        cur.close()

    def create_survey(self, v2_index_page, row):
        survey = Survey(
            title=row['title'],
            draft_title=row['draft_title'],
            show_in_menus=True,
            slug=row['slug'],
            path=v2_index_page.path + row['path'][12:],
            depth=row['depth'],
            numchild=row['numchild'],
            live=row['live'],
            thank_you_text=self.map_survey_thank_you_text(row),
            allow_anonymous_submissions=row['allow_anonymous_submissions'],
            allow_multiple_submissions=row['allow_multiple_submissions_per_user'],
            submit_button_text=row['submit_text'][:40] if row['submit_text'] else 'Submit',
            direct_display=row['display_survey_directly'],
            multi_step=row['multi_step'],
            locked=row['locked'],
            go_live_at=row['go_live_at'],
            expire_at=row['expire_at'],
            first_published_at=row['first_published_at'],
            last_published_at=row['last_published_at'],
            search_description=row['search_description'],
            seo_title=row['seo_title'],
            index_page_description=row['homepage_introduction'],
            index_page_description_line_2=row['homepage_button_text'],
            terms_and_conditions=self.map_survey_terms_and_conditions(row),
            latest_revision_created_at=row['latest_revision_created_at'],
        )

        try:
            survey.save()
            if row['submit_text'] and len(row['submit_text']) > 40:
                self.post_migration_report_messages['truncated_submit_button_text'].append(
                    f'title: {survey.title}. URL: {survey.full_url}. '
                    f'Admin URL: {self.get_admin_url(survey.id)}. '
                    f'Full text: {row["submit_text"]}.'
                )
            V1ToV2ObjectMap.create_map(content_object=survey, v1_object_id=row['page_ptr_id'])
            V1PageURLToV2PageMap.create_map(url=row['url_path'], page=survey)
        except Exception as e:
            self.post_migration_report_messages['surveys'].append(
                f"Unable to save survey, title={row['title']}"
            )
            return

        self.migrate_survey_questions(survey, row)

        self.v1_to_v2_page_map.update({
            row['page_ptr_id']: survey
        })
        self.stdout.write(f"saved survey, title={survey.title}")

    def map_survey_description(self, row):
        v1_survey_description = json.loads(row['description'])
        v2_survey_description = self._map_body('surveys', row, v1_survey_description)

        if row['introduction']:
            v2_survey_description = [{
                'type': 'paragraph',
                'value': row['introduction'],
            }] + v2_survey_description

        return json.dumps(v2_survey_description)

    def map_survey_thank_you_text(self, row):
        v2_thank_you_text = []
        if row['thank_you_text']:
            v2_thank_you_text.append({'type': 'paragraph', 'value': row['thank_you_text']})

        return json.dumps(v2_thank_you_text)

    def map_survey_terms_and_conditions(self, row):
        sql = f'select * ' \
              f'from surveys_surveytermsconditions stc, surveys_molosurveypage msp, wagtailcore_page wcp ' \
              f'where stc.page_id = msp.page_ptr_id ' \
              f'and stc.terms_and_conditions_id = wcp.id ' \
              f'and stc.page_id = {row["page_ptr_id"]} ' \
              f'order by wcp.path'

        cur = self.db_query(sql)
        v1_term_and_condition = cur.fetchone()
        cur.close()
        if v1_term_and_condition:
            return json.dumps([
                {
                    "type": "page_button",
                    "value": {
                        "page": self.v1_to_v2_page_map[v1_term_and_condition["terms_and_conditions_id"]].id,
                    },
                },
            ])

    def migrate_survey_questions(self, survey, survey_row):
        self._migrate_survey_questions(survey, survey_row, 'surveys_molosurveyformfield')
        self._migrate_survey_questions(survey, survey_row, 'surveys_personalisablesurveyformfield')

    def _migrate_survey_questions(self, survey, survey_row, formfield_table):
        sql = f'select *, smsff.id as smsffid ' \
              f'from {formfield_table} smsff, surveys_molosurveypage smsp, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl ' \
              f'where smsff.page_id = smsp.page_ptr_id ' \
              f'and smsp.page_ptr_id = wcp.id ' \
              f'and wcp.id = clr.page_id ' \
              f'and clr.language_id = csl.id ' \
              f'and wcp.id = {survey_row["page_ptr_id"]} ' \
              f'order by wcp.path'
        cur = self.db_query(sql)
        self.create_survey_question(survey, survey_row, cur)
        cur.close()

    def create_survey_question(self, survey, survey_row, cur):
        for row in cur:
            field_type = 'positivenumber' if row['field_type'] == 'positive_number' else row['field_type']
            if row['skip_logic']:
                skip_logics = json.loads(row['skip_logic'])
                for skip_logic in skip_logics:
                    skip_logic.get('value', {}).pop('survey')
                row['skip_logic'] = json.dumps(skip_logics)

            survey_form_field = SurveyFormField.objects.create(
                page=survey, sort_order=row['sort_order'], label=row['label'], required=row['required'],
                default_value=row['default_value'], help_text=row['help_text'], field_type=field_type,
                admin_label=row['admin_label'], page_break=row['page_break'],
                choices='|'.join(row['choices'].split(',')), skip_logic=row['skip_logic']
            )
            V1ToV2ObjectMap.create_map(content_object=survey_form_field, v1_object_id=row['smsffid'])
            skip_logic_next_actions = [logic['value']['skip_logic'] for logic in json.loads(row['skip_logic'])]
            if not survey_row['multi_step'] and (
                    'end' in skip_logic_next_actions or 'question' in skip_logic_next_actions):
                self.post_migration_report_messages['survey_multistep'].append(
                    f'skip logic without multi step'
                )
            self.stdout.write(f"saved survey question, label={row['label']}")

    def _get_iso_locale(self, locale):
        iso_locales_map = {
            'sho': 'sn',
            'ch': 'ny',
        }

        return iso_locales_map.get(locale, locale)

    def translate_home_pages(self):
        self.post_migration_report_messages['homepage'].append(
            f'A language indicator was added to your Homepages Title so that it\'s easier to identify '
            f'which Homepage is for which language. For example "IoGT" becomes "IoGT [French]". '
            f'This Title is shown to users in their browser tab\'s title, unless the "Page Title" is set, '
            f'which can be found in the Promote tab when editing the Homepage.')

        locales = Locale.objects.all()
        for locale in locales:
            self.translate_page(locale=locale, page=self.home_page)
            translated_home_page = self.home_page.get_translation_or_none(locale)
            if translated_home_page:
                translated_home_page.seo_title = self.home_page_seo_title or translated_home_page.title
                translated_home_page.title = f"{translated_home_page.title} [{str(locale)}]"
                translated_home_page.draft_title = f"{translated_home_page.draft_title} [{str(locale)}]"
                translated_home_page.save()

                if not self.home_page_seo_title:
                    self.post_migration_report_messages['homepage'].append(
                        f'title: {translated_home_page.title}. URL: {translated_home_page.full_url}. '
                        f'Admin URL: {self.get_admin_url(translated_home_page.id)}.'
                    )
        self.post_migration_report_messages['homepage'].append(
            'Copied the Homepage\'s original Title from IoGT V1 to the Page Title in IoGT V2, so that the Homepage '
            'appears without eg "[English]" in users\' browser tab titles. To change the way your Homepage appears in '
            'users\' browser tab titles you can change the Page Title in the Promote tab when editing the Homepage.')

    def translate_index_pages(self):
        index_pages = [
            self.section_index_page, self.banner_index_page, self.footer_index_page, self.poll_index_page,
            self.survey_index_page, self.quiz_index_page, self.miscellaneous_index_page,
        ]
        locales = Locale.objects.all()
        for page in index_pages:
            for locale in locales:
                self.translate_page(locale=locale, page=page)

    def migrate_recommended_articles_for_article(self):
        article_cur = self.db_query(f'select DISTINCT page_id from core_articlepagerecommendedsections')
        for article_row in article_cur:
            v1_article_id = article_row['page_id']
            v2_article = self.v1_to_v2_page_map.get(v1_article_id)
            if v2_article:
                cur = self.db_query(
                    f'select * from core_articlepagerecommendedsections where page_id = {v1_article_id} and recommended_article_id is not null')
                for row in cur:
                    v2_recommended_article = self.v1_to_v2_page_map.get(row['recommended_article_id'])
                    if v2_recommended_article:
                        models.ArticleRecommendation.objects.create(
                            sort_order=row['sort_order'],
                            article=v2_recommended_article,
                            source=v2_article
                        )
                cur.close()
        article_cur.close()
        self.stdout.write('Recommended articles migrated')

    def migrate_featured_articles_for_homepage(self):
        locale_cur = self.db_query(f"select * from core_sitelanguage")
        for locale_row in locale_cur:
            articles_cur = self.db_query(
                f"select * "
                f"from core_articlepage cap, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl "
                f"where cap.page_ptr_id = wcp.id "
                f"and wcp.id = clr.page_id "
                f"and clr.language_id = csl.id "
                f"and wcp.live = true "
                f"and csl.locale = '{locale_row['locale']}' "
                f"order by left(wcp.path, 16) "
            )
            articles_list = []
            for article_row in articles_cur:
                translated_from_page_id = self.page_translation_map.get(article_row['page_ptr_id'])
                featured_in_homepage_start_date = article_row['featured_in_homepage_start_date']
                if translated_from_page_id:
                    translated_from_article_cur = self.db_query(
                        f'select * from core_articlepage where page_ptr_id = {translated_from_page_id}')
                    translated_from_article_row = translated_from_article_cur.fetchone()
                    translated_from_article_cur.close()
                    # For translated articles, only the date of the translated from matters
                    featured_in_homepage_start_date = translated_from_article_row['featured_in_homepage_start_date']

                if featured_in_homepage_start_date:
                    article = self.v1_to_v2_page_map.get(article_row['page_ptr_id'])
                    if article:
                        article.featured_in_homepage_start_date = featured_in_homepage_start_date
                        articles_list.append(article)

            articles_cur.close()

            articles_list = sorted(articles_list, key=lambda x: x.featured_in_homepage_start_date, reverse=True)
            articles_list = sorted(articles_list, key=lambda x: x.path[:16])

            article_groups = defaultdict(list)
            for article in articles_list:
                article_groups[article.path[:16]].append(article)

            for k, v in article_groups.items():
                for i, article in enumerate(v):
                    if i < 5:
                        self.add_article_as_featured_content_in_home_page(article)
                    else:
                        self.post_migration_report_messages['ommitted_old_featured_article'].append(
                            f'title: {article.title}. URL: {article.full_url}. '
                            f'Admin URL: {self.get_admin_url(article.id)}. '
                            f'featured since: {article.featured_in_homepage_start_date}.'
                        )

                section = models.Section.objects.filter(path=k).first()
                if section:
                    self.add_section_as_featured_content_in_home_page(section)

        locale_cur.close()

    def add_article_as_featured_content_in_home_page(self, article):
        home_page = self.home_page.get_translation_or_none(article.locale)
        if home_page:
            home_page.home_featured_content.append((
                'article', {
                    'article': article,
                    'display_section_title': True,
                }
            ))
            home_page.save()

    def add_section_as_featured_content_in_home_page(self, section):
        home_page = self.home_page.get_translation_or_none(section.locale)
        if home_page:
            home_page.home_featured_content.append((
                'page_button', {
                    'page': section,
                    'text': '',
                }
            ))
            home_page.save()

    def attach_banners_to_home_page(self):
        sql = f"select * " \
              f"from core_bannerpage cbp, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where cbp.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        for row in cur:
            v2_banner = self.v1_to_v2_page_map.get(row['page_ptr_id'])
            if v2_banner:
                home_page = v2_banner.get_ancestors().exact_type(models.HomePage).first().specific
                models.HomePageBanner.objects.create(source=home_page, banner_page=v2_banner)
        cur.close()

    def get_color_hex(self, color_name):
        return {
            '--tiber': '#07292F',
            '--mecury': '#eae9e9',
            '--light_scampi': '#685FA1',
            '--dove_gray': '#737373',
            '--mineral_gray': '#dedede',
            '--washed_gray': '#f1f1f1',
            '--brown': '#a03321',
            '--medium_red_violet': '#B62A99',
            '--dark_medium_red_violet': '#b43393',
            '--violet_blue': '#a54f9e',
            '--mandy': '#E24256',
            '--plum': '#7e2268',
            '--wisteria': '#8e68ad',
            '--grape': '#541c56',
            '--paris_m': '#202855',
            '--east_bay': '#4E4682',
            '--victoria': '#4D4391',
            '--scampi': '#685FA1',
            '--sandybrown': '#EF9955',
            '--jaffa': '#ee8c39',
            '--saffron': '#F2B438',
            '--saffron_light': '#f2b437',
            '--cinnabar': '#EC3B3A',
            '--cinnabar_dark': '#ee5523',
            '--cardinal': '#bf2026',
            '--pomegranate': '#ed3330',
            '--roman': '#DF6859',
            '--mauvelous': '#F38AA5',
            '--beed_blush': '#e764a0',
            '--maxican_red': '#a21d2e',
            '--kobi': '#d481b5',
            '--illusion': '#ee97ac',
            '--celery': '#A4CE55',
            '--de_york': '#6EC17F',
            '--eucalyptus': '#2A9B58',
            '--tradewind': '#4bab99',
            '--moss_green': '#b3d9a1',
            '--danube': '#6093CD',
            '--light_danube': '#627abc',
            '--indigo': '#5F7AC9',
            '--mariner': '#4759a6',
            '--robin_egg_blue': '#00BFC6',
            '--pelorous': '#37BFBE',
            '--iris_blue': '#03acc3',
            '--red_berry': '#711e29',
            '--bay_of_may': '#2b378c',
            '--viking': '#3bbfbd',
            '--denim': '#127f99',
            '--tory_blue': '#134b90',
        }.get(color_name)

    def fix_articles_body(self):
        sql = f"select * " \
              f"from core_articlepage cap, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where cap.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id " \
              f"and wcp.path like '000100010002%' " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        for row in cur:
            v2_article = self.v1_to_v2_page_map.get(row['page_ptr_id'])
            if v2_article:
                v2_article = models.Article.objects.get(id=v2_article.id)
                v2_article.body = self.map_article_body(row)
                v2_article.save()
            else:
                self.post_migration_report_messages['articles'].append(
                    f'Unable to add article body, title={row["title"]}'
                )

        cur.close()

    def fix_footers_body(self):
        sql = f"select * " \
              f"from core_footerpage cfp, core_articlepage cap, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where cfp.articlepage_ptr_id = cap.page_ptr_id " \
              f"and cap.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        for row in cur:
            v2_footer = self.v1_to_v2_page_map.get(row['page_ptr_id'])
            if v2_footer:
                v2_footer.refresh_from_db()
                v2_footer.body = self.map_article_body(row)
                v2_footer.save()
        cur.close()

    def fix_survey_description(self):
        sql = f"select * " \
              f"from surveys_molosurveypage smsp, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where smsp.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id  " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        for row in cur:
            v2_survey = self.v1_to_v2_page_map.get(row['page_ptr_id'])
            if v2_survey:
                v2_survey.refresh_from_db()
                v2_survey.description = self.map_survey_description(row)
                v2_survey.save()
        cur.close()

    def fix_banner_link_page(self):
        sql = f"select * " \
              f"from core_bannerpage cbp, wagtailcore_page wcp, core_languagerelation clr, core_sitelanguage csl " \
              f"where cbp.page_ptr_id = wcp.id " \
              f"and wcp.id = clr.page_id " \
              f"and clr.language_id = csl.id " \
              f"order by wcp.path"
        cur = self.db_query(sql)
        for row in cur:
            v2_banner = self.v1_to_v2_page_map.get(row['page_ptr_id'])
            if v2_banner:
                v2_banner.refresh_from_db()
                v2_banner.banner_link_page = self.map_banner_page(row)
                v2_banner.save()
        cur.close()

    def add_polls_from_polls_index_page_to_footer_index_page_as_page_link_page(self):
        self.poll_index_page.refresh_from_db()
        self.footer_index_page.refresh_from_db()
        file = File(open(Path(settings.BASE_DIR) / 'iogt/static/icons/clip_board_pen.svg'), name='clip_board_pen.svg')
        icon = Svg.objects.create(title='clip board pen', file=file)
        poll_index_pages = self.poll_index_page.get_translations(inclusive=True)
        for poll_index_page in poll_index_pages:
            polls = poll_index_page.get_children()
            for poll in polls:
                page_link_page = models.PageLinkPage(title=poll.title, page=poll, icon=icon, live=poll.live)
                footer_index_page = self.footer_index_page.get_translation_or_none(poll.locale)
                footer_index_page.add_child(instance=page_link_page)

        self.stdout.write('Added polls from poll index page to footer index page as page link page.')

    def add_surveys_from_surveys_index_page_to_footer_index_page_as_page_link_page(self):
        self.survey_index_page.refresh_from_db()
        self.footer_index_page.refresh_from_db()
        file = File(open(Path(settings.BASE_DIR) / 'iogt/static/icons/loud_speaker.svg'), name='loud_speaker.svg')
        icon = Svg.objects.create(title='loud speaker', file=file)
        survey_index_page = self.survey_index_page.get_translations(inclusive=True)
        for survey_index_page in survey_index_page:
            surveys = survey_index_page.get_children()
            for survey in surveys:
                page_link_page = models.PageLinkPage(title=survey.title, page=survey, icon=icon, live=survey.live)
                footer_index_page = self.footer_index_page.get_translation_or_none(survey.locale)
                footer_index_page.add_child(instance=page_link_page)

        self.stdout.write('Added surveys from survey index page to footer index page as page link page.')

    def mark_pages_which_are_not_translated_in_v1_as_draft(self):
        self.section_index_page.refresh_from_db()
        self.banner_index_page.refresh_from_db()
        self.footer_index_page.refresh_from_db()
        self.poll_index_page.refresh_from_db()
        self.survey_index_page.refresh_from_db()
        self.quiz_index_page.refresh_from_db()

        page_ids_to_exclude = []
        page_ids_to_exclude += self.section_index_page.get_translations(inclusive=True).values_list('id', flat=True)
        page_ids_to_exclude += self.banner_index_page.get_translations(inclusive=True).values_list('id', flat=True)
        page_ids_to_exclude += self.footer_index_page.get_translations(inclusive=True).values_list('id', flat=True)
        page_ids_to_exclude += self.poll_index_page.get_translations(inclusive=True).values_list('id', flat=True)
        page_ids_to_exclude += self.survey_index_page.get_translations(inclusive=True).values_list('id', flat=True)
        page_ids_to_exclude += self.quiz_index_page.get_translations(inclusive=True).values_list('id', flat=True)

        Page.objects.filter(alias_of__isnull=False).exclude(id__in=page_ids_to_exclude).update(live=False)

    def migrate_social_media_links(self):
        self.footer_index_page.refresh_from_db()
        sql = f'select * from core_sitesettings'
        cur = self.db_query(sql)
        for row in cur:
            social_media_links = json.loads(row['social_media_links_on_footer_page'])
            for social_media_link in social_media_links:
                block_value = social_media_link.get('value')
                if block_value:
                    page_link_page_data = {
                        'title': block_value.get('title'),
                        'external_link': block_value.get('link'),
                    }
                    v2_image = self.image_map.get(block_value.get('image'))
                    if v2_image:
                        page_link_page_data.update({'image_icon_id': v2_image.id})

                    page_link_page = models.PageLinkPage(**page_link_page_data)
                    self.footer_index_page.add_child(instance=page_link_page)

    def migrate_page_revisions(self):
        PageRevision.objects.all().delete()
        sql = f"select * " \
              f"from wagtailcore_pagerevision wcpr"
        cur = self.db_query(sql)
        for row in cur:
            v2_page = self.v1_to_v2_page_map.get(row['page_id'])
            if v2_page:
                page_revision = PageRevision.objects.create(
                    page=v2_page,
                    submitted_for_moderation=row['submitted_for_moderation'],
                    created_at=row['created_at'],
                    content_json=row['content_json'],
                    approved_go_live_at=row['approved_go_live_at'],
                )
                V1ToV2ObjectMap.create_map(page_revision, row['id'])
        cur.close()

    def add_polls_from_polls_index_page_to_home_page_featured_content(self):
        self.poll_index_page.refresh_from_db()
        self.home_page.refresh_from_db()
        poll_index_pages = self.poll_index_page.get_translations(inclusive=True)
        for poll_index_page in poll_index_pages:
            home_page = self.home_page.get_translation_or_none(poll_index_page.locale)
            polls = poll_index_page.get_children().live()
            for poll in polls:
                home_page.home_featured_content.append((
                    'embedded_poll', {
                        'direct_display': True,
                        'poll': poll,
                    }
                ))
            home_page.save()

        self.stdout.write('Added polls from poll index page to home page featured content.')

    def add_surveys_from_surveys_index_page_to_home_page_featured_content(self):
        self.survey_index_page.refresh_from_db()
        self.home_page.refresh_from_db()
        survey_index_pages = self.survey_index_page.get_translations(inclusive=True)
        for survey_index_page in survey_index_pages:
            home_page = self.home_page.get_translation_or_none(survey_index_page.locale)
            surveys = survey_index_page.get_children().live()
            for survey in surveys:
                home_page.home_featured_content.append((
                    'embedded_survey', {
                        'direct_display': survey.specific.direct_display,
                        'survey': survey,
                    }
                ))
            home_page.save()

        self.stdout.write('Added surveys from survey index page to home page featured content.')

    def migrate_article_related_sections(self):
        cur = self.db_query('select * from core_articlepagerelatedsections caprs')
        sections = defaultdict(list)
        for row in cur:
            section = self.v1_to_v2_page_map.get(row['section_id'])
            article = self.v1_to_v2_page_map.get(row['page_id'])
            if (not section) or (not article):
                self.post_migration_report_messages['articles_in_related_sections'].append(
                    f"Couldn't find v2 page for v1 section: {row['section_id']} and article: {row['page_id']}"
                )
                continue
            section.refresh_from_db()
            article.refresh_from_db()
            page_link_page = models.PageLinkPage(title=article.title, page=article, live=article.live)
            section.add_child(instance=page_link_page)
            page = Page.objects.get(id=page_link_page.id)
            self.move_page(page_to_move=page, position=0)
            sections[section.id].append(article.title)

        for k, v in sections.items():
            page = Page.objects.get(id=k)
            self.post_migration_report_messages['unordered_related_articles_in_section'].append(
                f"title: {page.title}. URL: {page.full_url}. Admin URL: {self.get_admin_url(page.id)}. "
                f"articles: {', '.join(v)}"
            )

    def move_footers_to_end_of_footer_index_page(self):
        footer_index_pages = self.footer_index_page.get_translations(inclusive=True)
        for footer_index_page in footer_index_pages:
            footer_index_page_children = footer_index_page.get_children()
            articles = footer_index_page_children.exact_type(models.Article)
            for article in articles:
                self.move_page(page_to_move=article, position=footer_index_page_children.count())

    def move_page(self, page_to_move, position):
        parent_page = page_to_move.get_parent()

        # Find page that is already in this position
        position_page = None
        if position is not None:
            try:
                position_page = parent_page.get_children()[int(position)]
            except IndexError:
                pass  # No page in this position

        # Move page

        # any invalid moves *should* be caught by the permission check above,
        # so don't bother to catch InvalidMoveToDescendant

        if position_page:
            # If the page has been moved to the right, insert it to the
            # right. If left, then left.
            old_position = list(parent_page.get_children()).index(page_to_move)
            if int(position) < old_position:
                page_to_move.move(position_page, pos='left')
            elif int(position) > old_position:
                page_to_move.move(position_page, pos='right')
        else:
            # Move page to end
            page_to_move.move(parent_page, pos='last-child')

    def _sort_articles(self):
        pages = models.Section.objects.all().order_by('path')
        for page in pages:
            page.refresh_from_db()
            articles = page.get_children().type(models.Article)
            children_list = []
            for article in articles:
                try:
                    v1_id = V1ToV2ObjectMap.get_v1_id(article.specific, article.id)
                except:
                    continue
                if v1_id:
                    translated_from_page_id = self.page_translation_map.get(v1_id)
                    if translated_from_page_id:
                        v1_id = translated_from_page_id
                    cur = self.db_query(f'select * from wagtailcore_page wcp where id = {v1_id}')
                    v1_row = cur.fetchone()
                    cur.close()
                    setattr(article, 'creation_date', v1_row['first_published_at'])
                else:
                    setattr(article, 'creation_date', None)

                children_list.append(article)

            children_list = sorted(
                children_list, key=lambda x: (x.creation_date is not None, x.creation_date))
            for article in children_list:
                article.refresh_from_db()
                article.move(page, pos='first-child')

    def _sort_sections(self):
        locales = Locale.objects.all()
        for locale in locales:
            pages = models.Section.objects.filter(locale=locale).order_by('path')
            for page in pages:
                page.refresh_from_db()
                try:
                    v1_id = V1ToV2ObjectMap.get_v1_id(page.specific, page.id)
                except:
                    continue

                translated_from_page_id = self.page_translation_map.get(v1_id)
                if not translated_from_page_id:
                    continue

                translated_from_page = self.v1_to_v2_page_map.get(translated_from_page_id)
                if not translated_from_page:
                    continue

                translated_from_page.refresh_from_db()

                translated_from_sub_sections = translated_from_page.get_children().type(models.Section)
                translated_sub_sections = page.get_children().type(models.Section)

                if translated_sub_sections:
                    index_to_move = list(page.get_children()).index(translated_sub_sections.first())
                    for child in translated_from_sub_sections:
                        child.refresh_from_db()
                        translated_sub_section = child.get_translation_or_none(locale)
                        if translated_sub_section:
                            self.move_page(page_to_move=translated_sub_section, position=index_to_move)
                            index_to_move += 1

    def sort_pages(self):
        if self.sort != 'type1':
            return

        self._sort_sections()
        self._sort_articles()

        self.stdout.write('Pages sorted.')

    def populate_registration_survey_translations(self):
        with open(f'{settings.BASE_DIR}/iogt_content_migration/files/registration_survey_translations.csv',
                  newline='') as csvfile:
            reader = csv.DictReader(csvfile)
            for row in reader:
                str_key = self._get_iso_locale(row.pop('str'))
                self.registration_survey_translations[str_key] = row

    def translate_default_survey_submit_button_text(self):
        surveys = Survey.objects.all()
        for survey in surveys:
            if survey.submit_button_text == 'Submit':
                # Technically, someone could have manually put 'Submit' on a non-English button,
                # which we would now translate even though we shouldn't.
                # This is quite unlikely though.
                submit_button_text = self.registration_survey_translations['submit_button_text'].get(survey.locale.language_code)
                if not submit_button_text:
                    self.post_migration_report_messages['untranslated_survey_button'].append(
                            f'title: {survey.title}. URL: {survey.full_url}. '
                            f'Admin URL: {self.get_admin_url(survey.id)}.'
                        )
                if submit_button_text and len(submit_button_text) > 40:
                    # This should never happen in practice as we provide submit_button_text
                    self.stdout.write(f"Truncated default submit button text, title={survey.title}")

                survey.submit_button_text = submit_button_text[:40] if submit_button_text else 'Submit'
                survey.save()

    def migrate_post_registration_survey(self):
        sql = 'select * from profiles_userprofilessettings pups ' \
              'inner join wagtailcore_site ws on pups.site_id = ws.id ' \
              'where is_default_site = true'
        cur = self.db_query(sql)
        row = cur.fetchone()

        survey = Survey(
            title='Registration Survey', live=True, allow_multiple_submissions=True,
            allow_anonymous_submissions=False, submit_button_text='Register',
            thank_you_text=json.dumps([{
                'type': 'paragraph',
                'value': self.registration_survey_translations['thank_you_text']['en'],
            }]))

        self.survey_index_page.add_child(instance=survey)

        for (should_add_field_key, translation_key, is_required_key, field_type, admin_label) in [
            ('activate_dob', 'dob', 'dob_required', 'date', 'date_of_birth'),
            ('activate_gender', 'gender', 'gender_required', 'singleline', 'gender'),
            ('activate_location', 'location', 'location_required', 'singleline', 'location'),
            ('activate_education_level', 'education_level', 'activate_education_level_required', 'singleline',
             'education_level'),
            ('show_mobile_number_field', 'mobile_number', 'mobile_number_required', 'singleline', 'mobile_number'),
            ('show_email_field', 'email_address', 'email_required', 'email', 'email'),
        ]:
            if row[should_add_field_key]:
                SurveyFormField.objects.create(
                    page=survey,
                    label=self.registration_survey_translations[translation_key]['en'],
                    required=bool(row[is_required_key]),
                    field_type=field_type,
                    admin_label=admin_label,
                    help_text=self.registration_survey_translations[f'{translation_key}_helptext']['en']
                )

        self.stdout.write('Successfully migrated post registration survey')

        default_site_settings = models.SiteSettings.get_for_default_site()
        default_site_settings.registration_survey = survey
        default_site_settings.save()

        for locale in Locale.objects.all():
            try:
                self.translate_page(locale=locale, page=survey)
                translated_survey = survey.get_translation_or_none(locale)
            except Exception as e:
                self.post_migration_report_messages['registration_survey'].append(
                    f"Unable to translate survey, title={survey.title} to locale={locale}"
                )
                continue

            translation_is_incomplete = False
            submit_button_text = self.registration_survey_translations['register_button_text'].get(locale.language_code)
            if not submit_button_text:
                translation_is_incomplete = True
            if submit_button_text and len(submit_button_text) > 40:
                # This should never happen in practice as we provide submit_button_text
                self.stdout.write(f"Truncated survey submit button text, title={translated_survey.title}")
            translated_survey.submit_button_text = submit_button_text[:40] if submit_button_text else 'Register'

            translated_thank_you_text = self.registration_survey_translations['thank_you_text'].get(locale.language_code)
            if translated_thank_you_text:
                translated_survey.thank_you_text = json.dumps([{
                        'type': 'paragraph',
                        'value': translated_thank_you_text,
                    }])
            else:
                translation_is_incomplete = True

            translated_survey.save()
            if translated_survey:
                for (admin_label, label_identifier) in [
                    ('date_of_birth', 'dob'),
                    ('gender', 'gender'),
                    ('location', 'location'),
                    ('mobile_number', 'mobile_number'),
                    ('education_level', 'education_level'),
                    ('email', 'email_address')
                ]:
                    try:
                        field = SurveyFormField.objects.get(page=translated_survey, admin_label=admin_label)
                    except SurveyFormField.DoesNotExist:
                        # This field is not marked as required in the registration survey
                        continue
                    label = self.registration_survey_translations[label_identifier].get(locale.language_code)
                    help_text = self.registration_survey_translations[
                        f'{label_identifier}_helptext'].get(locale.language_code)
                    if not label or not help_text:
                        translation_is_incomplete = True
                    field.label = label or self.registration_survey_translations[label_identifier]['en']
                    field.help_text = help_text or self.registration_survey_translations[
                        f'{label_identifier}_helptext']['en']
                    field.save()
            if translation_is_incomplete:
                self.post_migration_report_messages['registration_survey_translation_not_found'].append(
                    f'Incomplete translation for registration survey to locale: {locale}')

        self.post_migration_report_messages['other'].append(
            'Title of registration survey (Pages > Internet of Good Things [Language] > Surveys > Registration Survey) '
            'has not been translated for any language.'
        )

    def get_admin_url(self, id):
        site = Site.objects.filter(is_default_site=True).first()
        return f"{site.root_url}{reverse('wagtailadmin_pages:edit', args=(id,))}"

    def print_post_migration_report(self):
        self.stdout.write(self.style.ERROR('====================='))
        self.stdout.write(self.style.ERROR('POST MIGRATION REPORT'))
        self.stdout.write(self.style.ERROR('====================='))

        for k, v in self.post_migration_report_messages.items():
            self.stdout.write(self.style.ERROR(f"===> {k.replace('_', ' ').upper()}"))
            self.stdout.write(self.style.ERROR('\n'.join(v)))


# iogt_content_migration/management/commands/load_v1_users.py

import json
from collections import defaultdict
from time import sleep

import csv
import psycopg2
from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group
from django.contrib.contenttypes.models import ContentType
from django.core.management import BaseCommand
from django.core.serializers.json import DjangoJSONEncoder
from django.db.models import Q
from django_comments.models import CommentFlag
from django_comments_xtd.models import XtdComment
from pip._vendor.distlib.compat import raw_input
from wagtail.contrib.forms.utils import get_field_clean_name

from tqdm import tqdm
from wagtail.core.models import Page, PageViewRestriction, PageRevision

from comments.models import CannedResponse
from home.models import Article, V1ToV2ObjectMap, SiteSettings
from questionnaires.models import Survey, UserSubmission, Poll, SurveyFormField


class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            '--host',
            default='0.0.0.0',
            help='IoGT V1 database host'
        )
        parser.add_argument(
            '--port',
            default='5432',
            help='IoGT V1 database port'
        )
        parser.add_argument(
            '--name',
            default='postgres',
            help='IoGT V1 database name'
        )
        parser.add_argument(
            '--user',
            default='postgres',
            help='IoGT V1 database user'
        )
        parser.add_argument(
            '--password',
            default='',
            help='IoGT V1 database password'
        )

        parser.add_argument(
            '--skip-locales',
            action='store_true',
            help='Skip data of locales other than default language'
        )

        parser.add_argument(
            '--delete-users',
            action='store_true',
            help='Delete existing Users and their associated data. Use carefully'
        )

        parser.add_argument(
            '--group-ids',
            nargs='+',
            type=int,
            help='Groups IDs to mark registration survey mandatory for'
        )

    def handle(self, *args, **options):
        self.db_connect(options)

        mandatory_survey_group_ids = options.get('group_ids')

        self.registration_survey_mandatory_group_ids = mandatory_survey_group_ids or \
                                                       self.request_registration_survey_mandatory_groups()
        self.content_type_map = dict()
        self.registration_survey_translations = defaultdict()
        self.delete_users = options.get('delete_users')
        self.post_migration_report_messages = defaultdict(list)

        self.clear()

        self.migrate()

    def clear(self):
        if self.delete_users:
            self.stdout.write('Deleted Existing Users')
            get_user_model().objects.all().delete()

    def db_connect(self, options):
        connection_string = self.create_connection_string(options)
        self.stdout.write(f'DB connection string created, string={connection_string}')
        self.v1_conn = psycopg2.connect(connection_string)
        self.stdout.write('Connected to v1 DB')

    def __del__(self):
        try:
            self.v1_conn.close()
            self.stdout.write('Closed connection to v1 DB')
        except AttributeError:
            pass

    def get_query_results_count(self, sql):
        sql = f'select count(*) from ({sql}) as count_table'
        cur = self.db_query(sql).fetchone()
        return cur['count']

    def with_progress(self, sql, iterable, title):
        self.stdout.write(title)
        return tqdm(iterable, total=self.get_query_results_count(sql))

    def request_registration_survey_mandatory_groups(self):
        sql = f'select * from auth_group'
        cur = self.db_query(sql)

        self.stdout.write('==============================')
        self.stdout.write('User Groups in v1:')
        [self.stdout.write(f'{group["id"]} - {group["name"]}') for group in cur]

        self.stdout.write('\n Please mention the groups for which you want to mark the registration survey'
                          'mandatory?')
        group_ids = raw_input('(Use comma separated ids, leave blank to make it optional)')

        if group_ids:
            group_ids = group_ids.split(',')
            self.stdout.write(f'\n The script will mark registration survey mandatory for all'
                              f' users of group_ids: {group_ids}')
        else:
            self.stdout.write(f'\n The script will mark registration survey optional for all users')
            group_ids = []

        sleep(5)
        return group_ids

    def create_connection_string(self, options):
        host = options.get('host', '0.0.0.0')
        port = options.get('port', '5432')
        name = options.get('name', 'postgres')
        user = options.get('user', 'postgres')
        password = options.get('password', '')
        return f"host={host} port={port} dbname={name} user={user} password={password}"

    def db_query(self, q):
        cur = self.v1_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
        cur.execute(q)
        return cur

    def migrate(self):
        self.populate_content_type_map()

        self.migrate_user_groups()
        self.migrate_user_accounts()
        self.mark_user_registration_survey_required()

        self.migrate_user_comments()
        self.migrate_comment_flags()
        self.migrate_canned_responses()

        self.migrate_user_survey_submissions()
        self.migrate_user_poll_submissions()
        self.migrate_user_freetext_poll_submissions()

        self.migrate_page_view_restrictions()

        self.migrate_registration_survey_submissions()

        self.migrate_page_revision_users()

        self.print_post_migration_report()

    def populate_content_type_map(self):
        sql = f'select * from django_content_type'
        cur = self.db_query(sql)

        for row in cur:
            new_content_type = ContentType.objects.filter(model=row["model"]).first()
            if not new_content_type:
                self.stdout.write(self.style.ERROR(f'Content Type missing for {row["model"]}'))
            self.content_type_map.update({
                row["model"]: new_content_type
            })

        self.content_type_map.update({
            'articlepage': ContentType.objects.filter(app_label='home', model='article').first()
        })

        cur.close()

    def migrate_user_accounts(self):
        sql = f'select lower(username), count(*) from auth_user group by lower(username) having count(*) > 1'
        cur = self.db_query(sql)

        colliding_usernames = [row['lower'] for row in cur]
        self.post_migration_report_messages['colliding_users_in_v1'].append(','.join(colliding_usernames))

        sql = f"select lower(alias), count(*) " \
              f"from profiles_userprofile " \
              f"where (alias is not null or alias != '') " \
              f"group by lower(alias) " \
              f"having count(*) > 1"
        cur = self.db_query(sql)
        colliding_display_names = [row['lower'] for row in cur]
        cur.close()
        self.post_migration_report_messages['colliding_display_names_in_v1'].append(','.join(colliding_display_names))


        sql = f'select * ' \
              f'from auth_user au, profiles_userprofile pup ' \
              f'where au.id = pup.user_id ' \
              f'order by au.date_joined'
        cur = self.db_query(sql)

        renamed_users = []
        for row in self.with_progress(sql, cur, 'User Migration in progress'):
            v1_user_id = row.pop('id')

            user_data = {
                'password': row['password'],
                'last_login': row['last_login'],
                'is_superuser': row['is_superuser'],
                'username': row['username'],
                'first_name': row['first_name'],
                'last_name': row['last_name'],
                'email': row['email'],
                'is_staff': row['is_staff'],
                'is_active': row['is_active'],
                'date_joined': row['date_joined'],
                'display_name': row['alias'],
                'has_filled_registration_survey': True,
                'has_viewed_registration_survey': True,
            }

            migrated_user = V1ToV2ObjectMap.get_v2_obj(get_user_model(), v1_user_id)

            if not migrated_user:
                if get_user_model().objects.filter(username__iexact=row['username']).exists():
                    existing_user = get_user_model().objects.get(username__iexact=row['username'])
                    modified_username = f'{existing_user}_v2'
                    renamed_users.append(f'{existing_user} -> {modified_username}')
                    existing_user.username = modified_username
                    existing_user.save()

                is_user_display_name_colliding = False

                display_name_matched = get_user_model().objects.filter(
                    Q(Q(display_name__iexact=row['alias']) | Q(username__iexact=row['alias']))
                ).exists()
                username_matched = get_user_model().objects.filter(
                    Q(Q(display_name__iexact=row['username']) | Q(username__iexact=row['username']))
                ).exists()

                if (row['alias'] and display_name_matched) or (not row['alias'] and username_matched):
                    is_user_display_name_colliding = True

                user = get_user_model().objects.create(**user_data)
                V1ToV2ObjectMap.create_map(user, v1_user_id)

                user_groups_sql = f'select * from auth_user_groups aug ' \
                                  f'inner join auth_group ag on aug.group_id = ag.id ' \
                                  f'where user_id={v1_user_id}'

                user_groups_cursor = self.db_query(user_groups_sql)

                user.groups.clear()
                for row_ in user_groups_cursor:
                    group = Group.objects.get(name=row_['name'])
                    group.user_set.add(user)

                if is_user_display_name_colliding:
                    group = Group.objects.get(name='Colliding Display Names')
                    group.user_set.add(user)

                user_groups_cursor.close()

        if renamed_users:
            self.post_migration_report_messages['renamed_users'].append(','.join(renamed_users))

    def migrate_user_groups(self):
        self.stdout.write(self.style.SUCCESS('Starting User Groups Migration'))

        sql = f'select * from auth_group'
        cur = self.db_query(sql)

        for row in cur:
            group, __ = Group.objects.get_or_create(name=row['name'])
            V1ToV2ObjectMap.create_map(group, row['id'])

        Group.objects.get_or_create(name='Colliding Display Names')

        self.stdout.write(self.style.SUCCESS('Completed User Groups Migration'))

    def mark_user_registration_survey_required(self):
        users = get_user_model().objects.filter(groups__id__in=self.registration_survey_mandatory_group_ids)
        users.update(has_filled_registration_survey=False, has_viewed_registration_survey=False)

    def migrate_root_level_user_comments(self):
        sql = f'select dc.id as comment_id, wcp.id as wagtailpage_id, wcp.title, comment, * ' \
              f'from django_comments dc ' \
              f'inner join commenting_molocomment mc on dc.id = mc.comment_ptr_id ' \
              f'inner join wagtailcore_page wcp on CAST (dc.object_pk as int) = wcp.id ' \
              f'inner join django_content_type dct on dc.content_type_id = dct.id where parent_id is null ' \
              f'order by submit_date'
        cur = self.db_query(sql)

        for row in self.with_progress(sql, cur, 'User [root] comments migration in progress'):
            comment_id = row.pop('comment_id')
            content_type = self.content_type_map[row['model']]

            if not content_type:
                self.stdout.write(self.style.ERROR(f'Content Type for {row["model"]} not found.'))
                continue

            new_article = V1ToV2ObjectMap.get_v2_obj(Article, row['object_pk'])

            if not new_article:
                self.stdout.write(self.style.ERROR(f'New Article for object_pk:{row["object_pk"]} not found.'))
                continue

            max_thread_id_comment = XtdComment.objects.filter(
                content_type_id=content_type.id, object_pk=new_article.pk).order_by('-thread_id').first()

            max_thread_id = 0
            if max_thread_id_comment:
                max_thread_id = max_thread_id_comment.id

            migrated_comment = V1ToV2ObjectMap.get_v2_obj(XtdComment, comment_id)

            if not migrated_comment:
                comment = XtdComment.objects.create(
                    content_type_id=content_type.id, object_pk=new_article.pk, user_name=row['user_name'],
                    user_email=row['user_email'], submit_date=row['submit_date'],
                    comment=row['comment'], is_public=row['is_public'], is_removed=row['is_removed'],
                    thread_id=max_thread_id + 1, user=V1ToV2ObjectMap.get_v2_obj(get_user_model(), row['user_id']),
                    order=1, followup=0, nested_count=0, ip_address=row['ip_address'], site_id=1)
                V1ToV2ObjectMap.create_map(comment, comment_id)

    def migrate_nested_user_comments(self):
        sql = f'select dc.id as comment_id, wcp.id as wagtailpage_id, wcp.title, comment, * ' \
              f'from django_comments dc ' \
              f'inner join commenting_molocomment mc on dc.id = mc.comment_ptr_id ' \
              f'inner join wagtailcore_page wcp on CAST (dc.object_pk as int) = wcp.id ' \
              f'inner join django_content_type dct on dc.content_type_id = dct.id where parent_id is not null ' \
              f'order by submit_date'
        cur = self.db_query(sql)

        for row in self.with_progress(sql, cur, 'User [nested] comments migration in progress...'):
            comment_id = row.pop('comment_id')
            content_type = self.content_type_map[row['model']]

            if not content_type:
                self.stdout.write(self.style.ERROR(f'Content Type for {row["model"]} not found.'))
                continue

            parent_comment = V1ToV2ObjectMap.get_v2_obj(XtdComment, row['parent_id'])

            if not parent_comment:
                self.stdout.write(self.style.ERROR(f'Parent comment for Comment ID:{comment_id} not found in V2'))

            migrated_comment = V1ToV2ObjectMap.get_v2_obj(XtdComment, comment_id)

            if not migrated_comment:
                max_order_value = XtdComment.objects.filter(thread_id=parent_comment.thread_id) \
                    .values_list('order', flat=True).first()
                comment = XtdComment.objects.create(
                    content_type_id=content_type.id, object_pk=parent_comment.object_pk, user_name=row['user_name'],
                    user_email=row['user_email'], submit_date=row['submit_date'],
                    comment=row['comment'], is_public=row['is_public'], is_removed=row['is_removed'],
                    thread_id=parent_comment.thread_id,
                    user=V1ToV2ObjectMap.get_v2_obj(get_user_model(), row['user_id']),
                    level=parent_comment.level + 1, order=max_order_value + 1, followup=0, nested_count=0,
                    ip_address=row['ip_address'],
                    parent_id=parent_comment.pk,
                    site_id=1)
                V1ToV2ObjectMap.create_map(comment, comment_id)

    def migrate_user_comments(self):
        self.migrate_root_level_user_comments()
        self.migrate_nested_user_comments()

    def migrate_comment_flags(self):
        sql = f'select * from django_comment_flags'
        cur = self.db_query(sql)

        for row in cur:
            migrated_comment = V1ToV2ObjectMap.get_v2_obj(XtdComment, row['comment_id'])
            migrated_user = V1ToV2ObjectMap.get_v2_obj(get_user_model(), row['user_id'])

            migrated_comment_flag = V1ToV2ObjectMap.get_v2_obj(CommentFlag, row['id'])

            if not migrated_comment:
                self.stdout.write(f"Comment with ID {row['comment_id']} was flagged but doesn't exist.")
            elif not migrated_comment_flag:
                comment_flag = CommentFlag.objects.create(
                    flag=row['flag'], flag_date=row['flag_date'], comment_id=migrated_comment.id,
                    user_id=migrated_user.id)
                V1ToV2ObjectMap.create_map(comment_flag, row['id'])

    def migrate_canned_responses(self):
        sql = f'select * from commenting_cannedresponse'
        cur = self.db_query(sql)

        for row in cur:
            migrated_canned_response = V1ToV2ObjectMap.get_v2_obj(CannedResponse, row['id'])
            if not migrated_canned_response:
                canned_response = CannedResponse.objects.create(header=row['response_header'], text=row['response'])
                V1ToV2ObjectMap.create_map(canned_response, row['id'])

    def migrate_user_survey_submissions(self):
        sql = 'select * from surveys_molosurveysubmission mss ' \
              'inner join surveys_molosurveypage msp on mss.page_id = msp.page_ptr_id'
        cur = self.db_query(sql)

        for row in self.with_progress(sql, cur, 'User Survey migration in progress...'):
            try:
                new_survey = V1ToV2ObjectMap.get_v2_obj(Survey, row['page_id'])
                new_survey_page = Page.objects.get(pk=new_survey.id)
            except (KeyError, AttributeError):
                self.stdout.write(self.style.ERROR(f'Skipping Page: {row["page_id"]}'))
                continue

            form_data = json.loads(row['form_data'])
            altered_form_data = dict()

            for key, value in form_data.items():
                new_key = key.replace('-', '_')
                altered_form_data.update({
                    new_key: value
                })

            migrated_submission = V1ToV2ObjectMap.get_v2_obj(UserSubmission, row['id'], extra='survey')

            if not migrated_submission:
                submission = UserSubmission.objects.create(
                    form_data=json.dumps(altered_form_data, cls=DjangoJSONEncoder),
                    page=new_survey_page,
                    user=V1ToV2ObjectMap.get_v2_obj(get_user_model(), row['user_id']) if row['user_id'] else None,
                )
                submission.submit_time = row['created_at']
                submission.save()
                V1ToV2ObjectMap.create_map(submission, row['id'], extra='survey')

    def migrate_user_poll_submissions(self):
        sql = 'select  pcv.user_id, pcv.question_id, wcp_question.title, wcp_question.id, wcp_question.path \
                from polls_choice pc \
                    inner join wagtailcore_page wcp on pc.page_ptr_id = wcp.id \
                inner join polls_choice_choice_votes pccv on pc.page_ptr_id = pccv.choice_id \
                inner join polls_choicevote pcv on pccv.choicevote_id = pcv.id \
                right outer join wagtailcore_page wcp_question on pcv.question_id = wcp_question.id \
                where pcv.question_id is not null \
                group by pcv.user_id, pcv.question_id, wcp_question.title, wcp_question.id,  wcp_question.path'
        cur = self.db_query(sql)

        for unique_submission in self.with_progress(sql, cur, 'User poll submissions migration in progress'):
            question_id = unique_submission['question_id']
            user_id = unique_submission['user_id']
            title = unique_submission['title']
            page_id = unique_submission['id']

            user_submissions_sql = f'select wcp.id as id, wcp.title as answer_title, pccv.choicevote_id, pcv.*, wcp_question.title \
                                    from polls_choice pc \
                                        inner join wagtailcore_page wcp on pc.page_ptr_id = wcp.id \
                                    inner join polls_choice_choice_votes pccv on pc.page_ptr_id = pccv.choice_id \
                                    inner join polls_choicevote pcv on pccv.choicevote_id = pcv.id \
                                    right outer join wagtailcore_page wcp_question on pcv.question_id = wcp_question.id \
                                    where pcv.question_id is not null and pcv.question_id={question_id} and ' \
                                   f'pcv.user_id={user_id}'
            cursor = self.db_query(user_submissions_sql)

            v2_poll = V1ToV2ObjectMap.get_v2_obj(Poll, page_id)
            v2_poll_page = Page.objects.get(pk=v2_poll.page_ptr_id)

            answers = []
            for row in cursor:
                answers.append(row['answer_title'])

            form_title = get_field_clean_name(title)
            form_data = {
                form_title: answers
            }

            migrated_submission = V1ToV2ObjectMap.get_v2_obj(UserSubmission, row['id'], extra='poll')

            if not migrated_submission:
                submission = UserSubmission.objects.create(
                    form_data=json.dumps(form_data, cls=DjangoJSONEncoder),
                    page=v2_poll_page,
                    user=V1ToV2ObjectMap.get_v2_obj(get_user_model(), row['user_id']) if row['user_id'] else None,
                )
                submission.submit_time = row['submission_date']
                submission.save()
                V1ToV2ObjectMap.create_map(submission, row['id'], extra='poll')

    def migrate_user_freetext_poll_submissions(self):
        sql = f'select wcp.id, wcp.title, pftv.answer, pftv.id as submission_id, pftv.user_id, pftv.submission_date ' \
              f'from polls_freetextquestion pftq ' \
              f'inner join wagtailcore_page wcp on pftq.question_ptr_id = wcp.id ' \
              f'inner join polls_freetextvote pftv on pftv.question_id = wcp.id'

        cur = self.db_query(sql)

        for freetext_submission in self.with_progress(sql, cur, 'User freetext poll submissions migration in progress'):
            title = freetext_submission['title']
            answer = freetext_submission['answer']
            submission_id = freetext_submission['submission_id']

            form_title = get_field_clean_name(title)
            form_data = {
                form_title: answer
            }

            migrated_submission = V1ToV2ObjectMap.get_v2_obj(UserSubmission, freetext_submission['id'],
                                                             extra='freetext_poll')

            v2_poll = V1ToV2ObjectMap.get_v2_obj(Poll, freetext_submission['id'])
            v2_poll_page = Page.objects.get(pk=v2_poll.id)

            if not migrated_submission:
                submission = UserSubmission.objects.create(
                    form_data=json.dumps(form_data, cls=DjangoJSONEncoder),
                    page=v2_poll_page,
                    user=V1ToV2ObjectMap.get_v2_obj(get_user_model(), freetext_submission['user_id']) if
                    freetext_submission['user_id'] else None
                )
                submission.submit_time = freetext_submission['submission_date']
                submission.save()
                V1ToV2ObjectMap.create_map(submission, submission_id, extra='freetext_poll')

    def migrate_page_view_restrictions(self):
        sql = f'select * from wagtailcore_pageviewrestriction'
        cur = self.db_query(sql)

        for row in self.with_progress(sql, cur, 'User Page View Restrictions migration in progress'):
            if not V1ToV2ObjectMap.get_v2_obj(PageViewRestriction, row['id']):
                for klass in [Page, Article, Survey, Poll]:
                    migrated_page = V1ToV2ObjectMap.get_v2_obj(klass, row['page_id'])
                    if migrated_page:
                        break
                else:
                    continue

                pvr = PageViewRestriction.objects.create(
                    page=migrated_page, restriction_type=row['restriction_type'], password=row['password'])
                V1ToV2ObjectMap.create_map(pvr, row['id'])

                pvr_groups_sql = f'select * from wagtailcore_pageviewrestriction_groups ' \
                                 f'where pageviewrestriction_id={row["id"]}'
                pvr_groups_cur = self.db_query(pvr_groups_sql)

                for pvr_group in pvr_groups_cur:
                    migrated_group = V1ToV2ObjectMap.get_v2_obj(Group, pvr_group['group_id'])
                    PageViewRestriction.groups.add(migrated_group)

    def migrate_registration_survey_submissions(self):
        sql = f'select * from profiles_userprofile pup inner join auth_user au on pup.user_id = au.id'
        cur = self.db_query(sql)

        site_settings = SiteSettings.get_for_default_site()
        registration_survey = site_settings.registration_survey

        if registration_survey:
            for row in self.with_progress(sql, cur, 'User Registration Survey migration in progress'):

                form_data = {
                    'date_of_birth': row['date_of_birth'],
                    'gender': row['gender'],
                    'location': row['location'],
                    'education_level': row['education_level'],
                    'mobile_number': row['mobile_number'],
                    'email': row['email'],
                }

                if not V1ToV2ObjectMap.get_v2_obj(UserSubmission, row['id'], extra='registration'):
                    submission = UserSubmission.objects.create(
                        form_data=json.dumps(form_data, cls=DjangoJSONEncoder),
                        page=registration_survey,
                        user=V1ToV2ObjectMap.get_v2_obj(get_user_model(), row['user_id'])
                    )

                    V1ToV2ObjectMap.create_map(submission, row['id'], extra='registration')

    def migrate_page_revision_users(self):
        sql = f"select * " \
              f"from wagtailcore_pagerevision wcpr"
        cur = self.db_query(sql)
        for row in self.with_progress(sql, cur, 'Page revision migration in progress'):
            v2_page_revision = V1ToV2ObjectMap.get_v2_obj(PageRevision, row['id'])
            v2_user = V1ToV2ObjectMap.get_v2_obj(get_user_model(), row['user_id'])
            if v2_page_revision and v2_user:
                v2_page_revision.user = v2_user
                v2_page_revision.save()

    def print_post_migration_report(self):
        self.stdout.write(self.style.ERROR('====================='))
        self.stdout.write(self.style.ERROR('POST MIGRATION REPORT'))
        self.stdout.write(self.style.ERROR('====================='))

        for k, v in self.post_migration_report_messages.items():
            self.stdout.write(self.style.ERROR(f"===> {k.replace('_', ' ').upper()}"))
            self.stdout.write(self.style.ERROR('\n'.join(v)))


# iogt_content_migration/management/commands/fix_surveys.py
import json

from django.core.management.base import BaseCommand
from questionnaires.models import Survey, SurveyFormField
from wagtail.core.models import PageRevision


class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument(
            "--fix",
            action="store_true",
            help="Fix problems identified by the report",
        )

    def handle(self, *args, **options):
        if problems_report := report():
            for entry in problems_report:
                self.stdout.write(str(entry))
        else:
            self.stdout.write("No problems found")

        if options.get("fix"):
            self.stdout.write("Fix application started")
            fix(problems_report)
            self.stdout.write("Fix application completed")


class ReportEntry:
    def __init__(self, survey):
        self.survey = survey
        self.problems = report_on_revision(survey)

    @property
    def id(self):
        return self.survey.id

    @property
    def has_problems(self):
        return len(self.problems) > 0

    def __str__(self):
        status = "live " if self.survey.live else "draft"
        return f'{status}, {self.survey.id}, "{self.survey.title}", {self.problems}'


class SurveyRevision:
    def __init__(self, revision):
        self.revision = revision
        self.content = json.loads(revision.content_json)

    @property
    def pk(self):
        return self.content.get("pk")

    @property
    def fields(self):
        return [
            SurveyRevisionField(field)
            for field in self.content.get("survey_form_fields", [])
        ]


class SurveyRevisionField:
    def __init__(self, field):
        self._raw = field

    @property
    def pk(self):
        return self._raw.get("pk")

    @property
    def label(self):
        return self._raw.get("label")


def report():
    return [
        entry
        for survey in Survey.objects.all()
        if (entry := ReportEntry(survey)).has_problems
    ]


def report_on_revision(survey):
    try:
        return identify_problems(survey, get_latest_revision(survey))
    except PageRevision.DoesNotExist:
        return {}


def get_latest_revision(page):
    return SurveyRevision(PageRevision.objects.filter(page=page).latest("created_at"))


def identify_problems(survey, revision):
    return {
        problem
        for p in [
            field_ids_mismatch,
            id_mismatch,
            labels_mismatch,
            no_questions,
            no_revision_questions,
        ]
        if (problem := p(survey, revision))
    }


def no_questions(survey, revision):
    return "no_qs" if len(survey.get_form_fields()) < 1 else None


def no_revision_questions(survey, revision):
    return "no_rev_qs" if len(revision.fields) < 1 else None


def id_mismatch(survey, revision):
    return "id" if revision.pk != survey.id else None


def field_ids_mismatch(survey, revision):
    survey_field_ids = {field.id for field in survey.get_form_fields()}
    revision_field_ids = {field.pk for field in revision.fields if field.pk}

    return "field_ids" if survey_field_ids != revision_field_ids else None


def labels_mismatch(survey, revision):
    survey_field_labels = {field.label for field in survey.get_form_fields()}
    revision_field_labels = {field.label for field in revision.fields if field.label}

    return "labels" if survey_field_labels != revision_field_labels else None


def fix(problems_report):
    for entry in problems_report:
        if {"id", "field_ids"}.issubset(entry.problems):
            print(f"Revision update required, survey={entry.survey}")
            entry.survey.save_revision(log_action=True)
        elif "no_qs" in entry.problems:
            print(
                f"Restore fields from previous revision required, survey={entry.survey}"
            )
            for field in find_first_restorable_revision(entry.survey).fields:
                create_field(entry.survey, field._raw).save()
            latest_revision = entry.survey.save_revision(log_action=True)
            latest_revision.publish()
        else:
            print(f"No action taken, survey={entry.survey}")


def find_first_restorable_revision(page):
    return next(
        sr
        for revision in PageRevision.objects.filter(page=page).order_by("-created_at")
        if is_restorable_v1(((sr := SurveyRevision(revision))), page)
    )


def is_restorable_v1(revision, page):
    """Identifies a v1 PageRevision that can be used to restore a v2 Survey.
    PageRevisions from v1 reference primary keys that do not match the v2 database
    because revisions were copied verbatim from v1. It is possible, though unlikely
    that the primary keys might be the same across v1 and v2. The alternative would be
    to read the surveys directly from the v1 database. This method was chosen for the
    sake of convenience.
    """
    return revision.pk != page.id and len(revision.fields) > 0


def create_field(survey, data):
    return SurveyFormField(
        admin_label=data.get("admin_label"),
        choices="|".join(
            choice.strip() for choice in data.get("choices", "").split(",")
        ),
        default_value=data.get("default_value"),
        field_type=(
            "positivenumber"
            if (ftype := data.get("field_type")) == "positive_number"
            else ftype
        ),
        help_text=data.get("help_text"),
        label=data.get("label"),
        page=survey,
        page_break=data.get("page_break"),
        required=data.get("required"),
        skip_logic=[
            create_answer_option(item)
            for item in json.loads(data.get("skip_logic", "[]"))
        ],
        sort_order=data.get("sort_order"),
    )


def create_answer_option(item):
    value = item.get("value", {})

    return (
        "skip_logic",
        {
            "choice": value.get("choice"),
            "skip_logic": value.get("skip_logic"),
            "question": value.get("question"),
        },
    )
