_dict['categories'] = zipped_metadata.get('categories') if zipped_metadata.get('categories') else []
        metadata_dict['tags'] = zipped_metadata.get('tags') if zipped_metadata.get('tags') else []
        metadata_dict['useCases'] = zipped_metadata.get('useCases') if zipped_metadata.get('useCases') else []
        metadata_dict['keywords'] = zipped_metadata.get('keywords') if zipped_metadata.get('keywords') else []
        metadata_dict['githubUser'] = [self.gh_user] if self.gh_user else []
        metadata_dict['marketplaces'] = zipped_metadata.get('marketplaces') or MARKETPLACES
        metadata_dict = ContributionConverter.create_pack_metadata(data=metadata_dict)
        metadata_path = os.path.join(self.pack_dir_path, 'pack_metadata.json')
        with open(metadata_path, 'w') as pack_metadata_file:
            json.dump(metadata_dict, pack_metadata_file, indent=4)

    @staticmethod
    def create_pack_metadata(data: Dict = None) -> Dict:
        """Builds pack metadata JSON content.

        Args:
            data (dict): Dictionary keys and value to insert into the pack metadata.

        Returns:
            Dict. Pack metadata JSON content.
        """
        pack_metadata = {
            'name': '## FILL MANDATORY FIELD ##',
            'description': '## FILL MANDATORY FIELD ##',
            'support': XSOAR_SUPPORT,
            'currentVersion': PACK_INITIAL_VERSION,
            'author': XSOAR_AUTHOR,
            'url': XSOAR_SUPPORT_URL,
            'email': '',
            'created': datetime.utcnow().strftime(ContributionConverter.DATE_FORMAT),
            'categories': [],
            'tags': [],
            'useCases': [],
            'keywords': [],
            'marketplaces': [],
        }

        if data:
            pack_metadata.update(data)

        return pack_metadata

    def execute_update_rn(self):
        """
        Bump the pack version in the pack metadata according to the update type
        and create a release-note file using the release-notes text.

        """
        rn_mng = UpdateReleaseNotesManager(user_input=self.dir_name, update_type=self.update_type, )
        rn_mng.manage_rn_update()
        self.replace_RN_template_with_value(rn_mng.rn_path[0])

    def format_user_input(self) -> Dict[str, str]:
        """
        Replace the content item name with the content item display name if exists
        to match the template that being generated by the UpdateRN class by calling
        the function get_display_name(file_path)

        Build a dictionary with the release notes text per content item detected.

        Returns:
            Dict: Key is content item name, value is release note entry
        """
        entity_identifier = '##### '
        content_item_type_identifier = '#### '
        rn_per_content_item: dict = defaultdict(str)
        entity_name = 'NonEntityRelated'

        items_path = {content_item.get('source_id'): content_item.get('source_file_name')
                      for content_item in self.detected_content_items}

        for line in filter(None, self.release_notes.splitlines()):
            if line.startswith(entity_identifier):
                entity_name = line.lstrip(entity_identifier)
                if items_path.get(entity_name):
                    entity_name = get_display_name(items_path.get(entity_name))
            elif not line.startswith(content_item_type_identifier):
                rn_per_content_item[entity_name] = rn_per_content_item[entity_name] + line + '\n'
        return rn_per_content_item

    def replace_RN_template_with_value(self, rn_path: str):
        """
        Replace the release notes template for a given release-note file path
        with the contributor's text.
        Will only affect the detected content items template text.
        New items rn entries and updated docker image entries for the detected items won't get changed.

        Args:
            rn_path: path to the rn file created.
        """
        entity_identifier = '##### '
        new_entity_identifier = "##### New: "
        template_text = '%%UPDATE_RN%%'

        rn_per_content_item = self.format_user_input()

        with open(rn_path, 'r+') as rn_file:
            lines = rn_file.readlines()
            for index in range(len(lines)):
                previous_line = lines[index - 1] if index > 0 else ""
                if template_text in lines[index] or previous_line.startswith(new_entity_identifier):
                    # when contributing a new entity to existing pack, the release notes will look something like that:
                    # "##### New: entity name". The following code will extract the entity name in each case.
                    if previous_line.startswith(new_entity_identifier):
                        template_entity = previous_line.lstrip(new_entity_identifier).rstrip('\n')
                    else:
                        template_entity = previous_line.lstrip(entity_identifier).rstrip('\n')
                    curr_content_items = rn_per_content_item.get(template_entity)
                    if curr_content_items:
                        lines[index] = curr_content_items

            rn_file.seek(0)
            rn_file.writelines(lines)
            rn_file.truncate()


# demisto_sdk/commands/init/initiator.py

import glob
import os
import shutil
from distutils.dir_util import copy_tree
from distutils.version import LooseVersion
from typing import Dict, List

import click

from demisto_sdk.commands.common import tools
from demisto_sdk.commands.common.configuration import Configuration
from demisto_sdk.commands.common.constants import (CLASSIFIERS_DIR, CONNECTIONS_DIR, DASHBOARDS_DIR,
                                                   DEFAULT_CONTENT_ITEM_FROM_VERSION, DOC_FILES_DIR,
                                                   GENERIC_DEFINITIONS_DIR, GENERIC_FIELDS_DIR, GENERIC_MODULES_DIR,
                                                   GENERIC_TYPES_DIR, INCIDENT_FIELDS_DIR, INCIDENT_TYPES_DIR,
                                                   INDICATOR_FIELDS_DIR, INDICATOR_TYPES_DIR, INTEGRATION_CATEGORIES,
                                                   INTEGRATIONS_DIR, JOBS_DIR, LAYOUTS_DIR,
                                                   MARKETPLACE_LIVE_DISCUSSIONS, MARKETPLACES, PACK_INITIAL_VERSION,
                                                   PACK_SUPPORT_OPTIONS, PLAYBOOKS_DIR, REPORTS_DIR, SCRIPTS_DIR,
                                                   TEST_PLAYBOOKS_DIR, WIDGETS_DIR, WIZARDS_DIR, XSOAR_AUTHOR,
                                                   XSOAR_SUPPORT, XSOAR_SUPPORT_URL)
from demisto_sdk.commands.common.git_content_config import GitContentConfig
from demisto_sdk.commands.common.handlers import JSON_Handler, YAML_Handler
from demisto_sdk.commands.common.tools import (LOG_COLORS, get_common_server_path, get_pack_name, print_error, print_v,
                                               print_warning)
from demisto_sdk.commands.secrets.secrets import SecretsValidator

json = JSON_Handler()
yaml = YAML_Handler()


def extract_values_from_nested_dict_to_a_set(given_dictionary: dict, return_set: set):
    """Recursively extracts values from a nested dictionary to a set.

    Args:
        given_dictionary: The nested dictionary to extract the values from.
        return_set: the set with the extracted values.

    """

    for value in given_dictionary.values():
        if isinstance(value, dict):  # value can be a dictionary
            extract_values_from_nested_dict_to_a_set(value, return_set)
        else:
            for secret in value:  # value is a list
                return_set.add(secret)


class Initiator:
    """Initiator creates a new pack/integration/script.

       Attributes:
           output (str): The directory in which init will create the new pack/integration/script
           name (str): The name for the new pack/integration/script directory.
           id (str): The id for the created script/integration.
           integration (bool): Indicates whether to create an integration.
           template (str): If an integration is initialized, specifies the integration template.
           script (bool): Indicates whether to create a script.
           full_output_path (str): The full path to the newly created pack/integration/script
    """

    TEST_DATA_DIR = 'test_data'

    ''' INTEGRATION TEMPLATES CONSTANTS '''
    DEFAULT_INTEGRATION_TEMPLATE = 'BaseIntegration'
    HELLO_WORLD_INTEGRATION = 'HelloWorld'
    HELLO_IAM_WORLD_INTEGRATION = 'HelloIAMWorld'
    HELLO_WORLD_FEED_INTEGRATION = 'FeedHelloWorld'
    HELLO_WORLD_SLIM_INTEGRATION = 'HelloWorldSlim'

    INTEGRATION_TEMPLATE_OPTIONS = [HELLO_WORLD_INTEGRATION, HELLO_IAM_WORLD_INTEGRATION, HELLO_WORLD_FEED_INTEGRATION,
                                    DEFAULT_INTEGRATION_TEMPLATE, HELLO_WORLD_SLIM_INTEGRATION]

    TEMPLATE_INTEGRATION_NAME = '%%TEMPLATE_NAME%%'
    TEMPLATE_INTEGRATION_FILES = {f'{TEMPLATE_INTEGRATION_NAME}.py',
                                  f'{TEMPLATE_INTEGRATION_NAME}.yml',
                                  f'{TEMPLATE_INTEGRATION_NAME}_description.md',
                                  f'{TEMPLATE_INTEGRATION_NAME}_image.png',
                                  f'{TEMPLATE_INTEGRATION_NAME}_test.py',
                                  'Pipfile', 'Pipfile.lock', 'README.md', 'command_examples'}

    DEFAULT_INTEGRATION_TEST_DATA_FILES = {os.path.join(TEST_DATA_DIR, 'baseintegration-dummy.json')}

    HELLO_WORLD_TEST_DATA_FILES = {os.path.join(TEST_DATA_DIR, 'domain_reputation.json'),
                                   os.path.join(TEST_DATA_DIR, 'get_alert.json'),
                                   os.path.join(TEST_DATA_DIR, 'ip_reputation.json'),
                                   os.path.join(TEST_DATA_DIR, 'scan_results.json'),
                                   os.path.join(TEST_DATA_DIR, 'search_alerts.json'),
                                   os.path.join(TEST_DATA_DIR, 'update_alert_status.json'),
                                   os.path.join(TEST_DATA_DIR, 'domain_reputation.json')}

    HELLO_WORLD_FEED_TEST_DATA_FILES = {os.path.join(TEST_DATA_DIR, 'build_iterator_results.json'),
                                        os.path.join(TEST_DATA_DIR, 'get_indicators_command_results.json'),
                                        os.path.join(TEST_DATA_DIR, 'FeedHelloWorld_mock.txt')}

    HELLO_WORLD_SLIM_TEST_DATA_FILES = {os.path.join(TEST_DATA_DIR, 'get_alert.json'),
                                        os.path.join(TEST_DATA_DIR, 'update_alert_status.json')}

    ''' SCRIPT TEMPLATES CONSTANTS '''
    DEFAULT_SCRIPT_TEMPLATE = 'BaseScript'
    HELLO_WORLD_SCRIPT = 'HelloWorldScript'

    SCRIPT_TEMPLATE_OPTIONS = [HELLO_WORLD_SCRIPT, DEFAULT_SCRIPT_TEMPLATE]

    TEMPLATE_SCRIPT_NAME = '%%TEMPLATE_NAME%%'
    TEMPLATE_SCRIPT_FILES = {f'{TEMPLATE_SCRIPT_NAME}.py',
                             f'{TEMPLATE_SCRIPT_NAME}.yml',
                             f'{TEMPLATE_SCRIPT_NAME}_test.py',
                             'README.md'}

    DEFAULT_SCRIPT_TEST_DATA_FILES = {os.path.join(TEST_DATA_DIR, 'basescript-dummy.json')}

    ''' TEMPLATES PACKS CONSTANTS '''
    DEFAULT_TEMPLATE_PACK_NAME = 'StarterPack'
    HELLO_WORLD_PACK_NAME = 'HelloWorld'
    DEFAULT_TEMPLATES = [DEFAULT_INTEGRATION_TEMPLATE, DEFAULT_SCRIPT_TEMPLATE]
    HELLO_WORLD_BASE_TEMPLATES = [HELLO_WORLD_SCRIPT, HELLO_WORLD_INTEGRATION]

    DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
    PACK_INITIAL_VERSION = "1.0.0"
    SUPPORTED_FROM_VERSION = "5.5.0"

    DIR_LIST = [INTEGRATIONS_DIR, SCRIPTS_DIR, INCIDENT_FIELDS_DIR, INCIDENT_TYPES_DIR, INDICATOR_FIELDS_DIR,
                PLAYBOOKS_DIR, LAYOUTS_DIR, TEST_PLAYBOOKS_DIR, CLASSIFIERS_DIR, CONNECTIONS_DIR, DASHBOARDS_DIR,
                INDICATOR_TYPES_DIR, REPORTS_DIR, WIDGETS_DIR, DOC_FILES_DIR, GENERIC_MODULES_DIR,
                GENERIC_DEFINITIONS_DIR, GENERIC_FIELDS_DIR, GENERIC_TYPES_DIR, JOBS_DIR, WIZARDS_DIR]

    def __init__(self, output: str, name: str = '', id: str = '', integration: bool = False, template: str = '',
                 category: str = '', script: bool = False, pack: bool = False, author_image: str = '',
                 demisto_mock: bool = False,
                 common_server: bool = False):
        self.output = output if output else ''
        self.id = id

        self.is_integration = integration
        self.is_script = script
        self.is_pack = pack
        self.author_image = author_image
        self.demisto_mock = demisto_mock
        self.common_server = common_server
        self.category = category
        self.configuration = Configuration()

        # if no flag given automatically create a pack.
        if not integration and not script and not pack:
            self.is_pack = True

        self.template = self.get_selected_template(template)

        self.full_output_path = ''
        if name:
            while ' ' in name:
                name = str(input("The directory and file name cannot have spaces in it, Enter a different name: "))

        self.dir_name = name

        self.is_pack_creation = not all([self.is_script, self.is_integration])

    def init(self):
        """Starts the init command process.

        """
        if self.is_integration:
            self.get_created_dir_name(created_object="integration")
            self.get_object_id(created_object="integration")
            return self.integration_init()

        elif self.is_script:
            self.get_created_dir_name(created_object="script")
            self.get_object_id(created_object="script")
            return self.script_init()

        elif self.is_pack:
            self.get_created_dir_name(created_object="pack")
            return self.pack_init()

    def get_selected_template(self, template: str = '') -> str:
        """ Makes sure a valid template is selected

        Args:
            template (str): the given template name (empty string if not given)

        Returns:
            str. A valid template name. If no template was specified, returns the default template name.
        """
        if self.is_integration:
            while template and template not in self.INTEGRATION_TEMPLATE_OPTIONS:
                options_str = ', '.join(self.INTEGRATION_TEMPLATE_OPTIONS)
                template = str(input(f"Enter a valid template name, or press enter to choose the default template"
                                     f" ({self.DEFAULT_INTEGRATION_TEMPLATE}).\nValid options: {options_str}\n"))
            return template if template else self.DEFAULT_INTEGRATION_TEMPLATE

        elif self.is_script:
            while template and template not in self.SCRIPT_TEMPLATE_OPTIONS:
                options_str = ', '.join(self.SCRIPT_TEMPLATE_OPTIONS)
                template = str(input(f"Enter a valid template name, or press enter to choose the default template"
                                     f" ({self.DEFAULT_SCRIPT_TEMPLATE}).\nValid options: {options_str}\n"))
            return template if template else self.DEFAULT_SCRIPT_TEMPLATE

        # if reached here it is a pack init - will be used again if user decides to create an integration
        return template

    def get_created_dir_name(self, created_object: str):
        """Makes sure a name is given for the created object

        Args:
            created_object (str): the type of the created object (integration/script/pack)
        """
        while not self.dir_name or len(self.dir_name) == 0:
            self.dir_name = str(input(f"Please input the name of the initialized {created_object}: "))
            while ' ' in self.dir_name:
                self.dir_name = str(input("The directory name cannot have spaces in it, Enter a different name: "))

    def get_object_id(self, created_object: str):
        if not self.id:
            if self.is_pack_creation:  # There was no option to enter the ID in this process.
                use_dir_name = str(input(f"Do you want to use the directory name as an "
                                         f"ID for the {created_object}? Y/N "))
            else:
                use_dir_name = str(input(f"No ID given for the {created_object}'s yml file. "
                                         f"Do you want to use the directory name? Y/N "))

            if use_dir_name and use_dir_name.lower() in ['y', 'yes']:
                self.id = self.dir_name
            else:
                while not self.id:
                    self.id = str(input(f"Please enter the id name for the {created_object}: "))

    def pack_init(self) -> bool:
        """Creates a pack directory tree.

        Returns:
            bool. Returns True if pack was created successfully and False otherwise
        """
        # if an output directory given create the pack there
        if self.output:
            self.full_output_path = os.path.join(self.output, self.dir_name)

        # content-descriptor file indicates we are in "content" repository
        # thus we will create the pack under Packs directory
        elif os.path.isfile('content-descriptor.json'):
            self.full_output_path = os.path.join("Packs", self.dir_name)

        # if in an external repo check for the existence of Packs directory
        # if it does not exist create it
        elif tools.is_external_repository():
            if not os.path.isdir("Packs"):
                print("Creating 'Packs' directory")
                os.mkdir('Packs')
            self.full_output_path = os.path.join("Packs", self.dir_name)

        # if non of the above conditions apply - create the pack in current directory
        else:
            self.full_output_path = self.dir_name

        if not self.create_new_directory():
            return False
        for directory in self.DIR_LIST:
            path = os.path.join(self.full_output_path, directory)
            os.mkdir(path=path)

        self.create_pack_base_files()
        click.echo(
            f"Successfully created the pack {self.dir_name} in: {self.full_output_path}",
            color=LOG_COLORS.GREEN
        )

        metadata_path = os.path.join(self.full_output_path, 'pack_metadata.json')
        with open(metadata_path, 'a') as fp:
            user_response = str(input("\nWould you like to fill pack's metadata file? Y/N ")).lower()
            fill_manually = user_response in ['y', 'yes']

            pack_metadata = Initiator.create_metadata(fill_manually)
            self.category = pack_metadata['categories'][0] if pack_metadata['categories'] else 'Utilities'
            json.dump(pack_metadata, fp, indent=4)

            click.echo(f"Created pack metadata at path : {metadata_path}", color=LOG_COLORS.GREEN)

        create_integration = str(input("\nDo you want to create an integration in the pack? Y/N ")).lower()
        if create_integration in ['y', 'yes']:
            is_same_category = str(input("\nDo you want to set the integration category as you defined in the pack "
                                         "metadata? Y/N ")).lower()

            integration_category = self.category if is_same_category in ['y', 'yes'] else ''
            integration_init = Initiator(output=os.path.join(self.full_output_path, 'Integrations'),
                                         integration=True, common_server=self.common_server,
                                         demisto_mock=self.demisto_mock, template=self.template,
                                         category=integration_category)
            return integration_init.init()

        return True

    def create_pack_base_files(self):
        """
        Create empty 'README.md', '.secrets-ignore', '.pack-ignore' and 'Author_image.png' files that are expected
        to be in the base directory of a pack
        """
        click.echo('Creating pack base files', color=LOG_COLORS.NATIVE)
        fp = open(os.path.join(self.full_output_path, 'README.md'), 'a')
        fp.close()

        fp = open(os.path.join(self.full_output_path, '.secrets-ignore'), 'a')
        fp.close()

        fp = open(os.path.join(self.full_output_path, '.pack-ignore'), 'a')
        fp.close()

        # if an `Author_image.png` file was given - replace the default file with it
        author_image_path = os.path.join(self.full_output_path, 'Author_image.png')
        if self.author_image:
            shutil.copyfile(self.author_image, author_image_path)
        else:
            fp = open(author_image_path, 'a')
            fp.close()

    @staticmethod
    def create_metadata(fill_manually: bool, data: Dict = None) -> Dict:
        """Builds pack metadata JSON content.

        Args:
            fill_manually (bool): Whether to interact with the user to fill in metadata details or not.
            data (dict): Dictionary keys and value to insert into the pack metadata.

        Returns:
            Dict. Pack metadata JSON content.
        """
        pack_metadata = {
            'name': '## FILL MANDATORY FIELD ##',
            'description': '## FILL MANDATORY FIELD ##',
            'support': XSOAR_SUPPORT,
            'currentVersion': PACK_INITIAL_VERSION,
            'author': XSOAR_AUTHOR,
            'url': XSOAR_SUPPORT_URL,
            'email': '',
            'categories': [],
            'tags': [],
            'useCases': [],
            'keywords': [],
            'marketplaces': MARKETPLACES,
        }

        if data:
            pack_metadata.update(data)

        if not fill_manually:
            return pack_metadata  # return xsoar template

        pack_metadata['name'] = input("\nDisplay name of the pack: ")
        if not pack_metadata.get('name'):
            pack_metadata['name'] = '## FILL MANDATORY FIELD ##'

        pack_metadata['description'] = input("\nDescription of the pack: ")
        if not pack_metadata.get('description'):
            pack_metadata['description'] = '## FILL MANDATORY FIELD ##'

        pack_metadata['support'] = Initiator.get_valid_user_input(options_list=PACK_SUPPORT_OPTIONS,
                                                                  option_message="\nSupport type of the pack: \n")
        pack_metadata['categories'] = [Initiator.get_valid_user_input(options_list=INTEGRATION_CATEGORIES,
                                                                      option_message="\nPack category options: \n")]

        marketplaces = input("\nSupported marketplaces for this pack, comma separated values.\n"
                             "Possible options are: xsoar, marketplacev2. default value is 'xsoar,marketplacev2'.\n")
        mp_list = [m.strip() for m in marketplaces.split(',') if m]
        if mp_list:
            pack_metadata['marketplaces'] = mp_list

        if pack_metadata.get('support') == XSOAR_SUPPORT:
            pack_metadata['author'] = XSOAR_AUTHOR
            pack_metadata['url'] = XSOAR_SUPPORT_URL

            return pack_metadata

        pack_metadata['author'] = input("\nAuthor of the pack: ")

        if pack_metadata.get('support') != 'community':  # get support details from the user for non community packs
            support_url = input("\nThe url of support, should be a valid support/info URL (optional): ")
            while support_url and "http" not in support_url:
                support_url = input("\nIncorrect input. Please enter full valid url: ")
            pack_metadata['url'] = support_url
            pack_metadata['email'] = input("\nThe email in which users can reach out for support (optional): ")
        else:  # community pack url should refer to the marketplace live discussions
            pack_metadata['url'] = MARKETPLACE_LIVE_DISCUSSIONS

        dev_email = input("\nThe email will be used to inform you for any changes made to your pack (optional): ")
        if dev_email:
            pack_metadata['devEmail'] = [e.strip() for e in dev_email.split(',') if e]

        tags = input("\nTags of the pack, comma separated values: ")
        tags_list = [t.strip() for t in tags.split(',') if t]
        pack_metadata['tags'] = tags_list

        github_users = input("\nPack default reviewers, comma separated github username: ")
        github_users_list = [u.strip() for u in github_users.split(',') if u]
        pack_metadata['githubUser'] = github_users_list

        return pack_metadata

    @staticmethod
    def get_valid_user_input(options_list: List[str], option_message: str) -> str:
        """Gets user input from a list of options, by integer represents the choice.

        Args:
            options_list (List[str]): List of options for the user to choose from.
            option_message (str): The message to show the user along with the list of options.

        Returns:
            str. The chosen option.
        """
        for index, option in enumerate(options_list, start=1):
            option_message += f"[{index}] {option}\n"
        option_message += "\nEnter option: "

        user_input = input(option_message)

        while True:
            try:
                user_choice = int(user_input)
                if user_choice not in range(1, len(options_list) + 1):
                    user_input = input(f"\nInvalid option {user_input}, please enter valid choice: ")
                else:
                    return options_list[user_choice - 1]
            except ValueError:
                user_input = input("\nThe option must be number, please enter valid choice: ")

    def find_secrets(self):
        files_and_directories = glob.glob(f'{self.full_output_path}/**/*', recursive=True)

        sv = SecretsValidator(white_list_path='./Tests/secrets_white_list.json', ignore_entropy=True)
        # remove directories and irrelevant files
        files = [file for file in files_and_directories if os.path.isfile(file) and sv.is_text_file(file)]
        # The search_potential_secrets method returns a nested dict with values of type list. The values are the secrets
        # {'a': {'b': ['secret1', 'secret2'], 'e': ['secret1']}, 'g': ['secret3']}
        nested_dict_of_secrets = sv.search_potential_secrets(files)
        set_of_secrets: set = set()

        extract_values_from_nested_dict_to_a_set(nested_dict_of_secrets, set_of_secrets)

        return set_of_secrets

    def ignore_secrets(self, secrets):
        pack_dir = get_pack_name(self.full_output_path)
        try:
            with open(f'Packs/{pack_dir}/.secrets-ignore', 'a') as f:
                for secret in secrets:
                    f.write(secret)
                    f.write('\n')
        except FileNotFoundError:
            print_warning("Could not find the .secrets-ignore file - make sure your path is correct")

    def integration_init(self) -> bool:
        """Creates a new integration according to a template.

        Returns:
            bool. True if the integration was created successfully, False otherwise.
        """
        # if output directory given create the integration there
        if self.output:
            self.full_output_path = os.path.join(self.output, self.dir_name)

        # will create the integration under the Integrations directory of the pack
        elif os.path.isdir(INTEGRATIONS_DIR):
            self.full_output_path = os.path.join(INTEGRATIONS_DIR, self.dir_name)

        # if non of the conditions above apply - create the integration in the local directory
        else:
            self.full_output_path = self.dir_name

        if not self.create_new_directory():
            return False

        integration_template_files = self.get_template_files()
        if not self.get_remote_templates(integration_template_files, dir=INTEGRATIONS_DIR):
            local_template_path = os.path.normpath(os.path.join(__file__, "..", 'templates', self.template))
            copy_tree(str(local_template_path), self.full_output_path)

        if self.id != self.template:
            # note rename does not work on the yml file - that is done in the yml_reformatting function.
            self.rename(current_suffix=self.template)
            self.yml_reformatting(current_suffix=self.template, integration=True)
            self.fix_test_file_import(name_to_change=self.template)

        self.copy_common_server_python()
        self.copy_demistotmock()

        if self.template != self.DEFAULT_INTEGRATION_TEMPLATE:  # DEFAULT_INTEGRATION_TEMPLATE there are no secrets
            secrets = self.find_secrets()
            if secrets:
                new_line = '\n'
                click.echo(f"\nThe following secrets were detected:\n"
                           f"{new_line.join(secret for secret in secrets)}", color=LOG_COLORS.GREEN)

                ignore_secrets = input("\nWould you like ignore them automatically? Y/N ").lower()
                if ignore_secrets in ['y', 'yes']:
                    self.ignore_secrets(secrets)

        click.echo(f"Finished creating integration: {self.full_output_path}.", color=LOG_COLORS.GREEN)

        return True

    def script_init(self) -> bool:
        """Creates a new script according to a template.

        Returns:
            bool. True if the script was created successfully, False otherwise.
        """
        # if output directory given create the script there
        if self.output:
            self.full_output_path = os.path.join(self.output, self.dir_name)

        # will create the script under the Scripts directory of the pack
        elif os.path.isdir(SCRIPTS_DIR):
            self.full_output_path = os.path.join('Scripts', self.dir_name)

        # if non of the conditions above apply - create the integration in the local directory
        else:
            self.full_output_path = self.dir_name

        if not self.create_new_directory():
            return False

        used_template = self.template
        script_template_files = self.get_template_files()
        if not self.get_remote_templates(script_template_files, dir=SCRIPTS_DIR):
            local_template_path = os.path.normpath(os.path.join(__file__, "..", 'templates', self.template))
            copy_tree(str(local_template_path), self.full_output_path)

        if self.id != self.template:
            # note rename does not work on the yml file - that is done in the yml_reformatting function.
            self.change_template_name_script_py(current_suffix=self.template, current_template=used_template)
            self.rename(current_suffix=self.template)
            self.yml_reformatting(current_suffix=self.template)
            self.fix_test_file_import(name_to_change=self.template)

        self.copy_common_server_python()
        self.copy_demistotmock()

        secrets = self.find_secrets()
        if secrets:
            new_line = '\n'
            click.echo(f"\nThe following secrets were detected in the pack:\n"
                       f"{new_line.join(secret for secret in secrets)}", color=LOG_COLORS.GREEN)

            ignore_secrets = input("\nWould you like ignore them automatically? Y/N ").lower()
            if ignore_secrets in ['y', 'yes']:
                self.ignore_secrets(secrets)

        click.echo(f"Finished creating script: {self.full_output_path}", color=LOG_COLORS.GREEN)

        return True

    def yml_reformatting(self, current_suffix: str, integration: bool = False):
        """Formats the given yml to fit the newly created integration/script

        Args:
            current_suffix (str): The yml file name (HelloWorld or HelloWorldScript)
            integration (bool): Indicates if integration yml is being reformatted.
        """
        with open(os.path.join(self.full_output_path, f"{current_suffix}.yml")) as f:
            yml_dict = yaml.load(f)
        yml_dict["commonfields"]["id"] = self.id
        yml_dict['name'] = self.id

        from_version = input("\nThe fromversion value that will be used (optional): ")
        if from_version:
            yml_dict['fromversion'] = from_version

        if LooseVersion(yml_dict.get('fromversion', DEFAULT_CONTENT_ITEM_FROM_VERSION)) < LooseVersion(
                self.SUPPORTED_FROM_VERSION):
            yml_dict['fromversion'] = self.SUPPORTED_FROM_VERSION

        if integration:
            yml_dict["display"] = self.id
            yml_dict["category"] = self.category if self.category else Initiator.get_valid_user_input(
                options_list=INTEGRATION_CATEGORIES, option_message="\nIntegration category options: \n")

        with open(os.path.join(self.full_output_path, f"{self.dir_name}.yml"), 'w') as f:
            yaml.dump(yml_dict, f)

        os.remove(os.path.join(self.full_output_path, f"{current_suffix}.yml"))

    def change_template_name_script_py(self, current_suffix: str, current_template: str):
        """Change all script template name appearances with the real script name in the script python file.

        Args:
            current_suffix (str): The py file name
            current_template (str): The script template being used.
        """
        with open(os.path.join(self.full_output_path, f"{current_suffix}.py"), "r+") as f:
            py_file_data = f.read()
            py_file_data = py_file_data.replace(current_template, self.id)
            f.seek(0)
            f.write(py_file_data)
            f.truncate()

    def rename(self, current_suffix: str):
        """Renames the python, description, test and image file in the path to fit the newly created integration/script

        Args:
            current_suffix (str): The yml file name (HelloWorld or HelloWorldScript)
        """
        os.rename(os.path.join(self.full_output_path, f"{current_suffix}.py"),
                  os.path.join(self.full_output_path, f"{self.dir_name}.py"))
        os.rename(os.path.join(self.full_output_path, f"{current_suffix}_test.py"),
                  os.path.join(self.full_output_path, f"{self.dir_name}_test.py"))
        if self.is_integration:
            os.rename(os.path.join(self.full_output_path, f"{current_suffix}_image.png"),
                      os.path.join(self.full_output_path, f"{self.dir_name}_image.png"))
            os.rename(os.path.join(self.full_output_path, f"{current_suffix}_description.md"),
                      os.path.join(self.full_output_path, f"{self.dir_name}_description.md"))

    def create_new_directory(self, ) -> bool:
        """Creates a new directory for the integration/script/pack.

        Returns:
            bool. True if directory was successfully created, False otherwise.
        """
        try:
            os.mkdir(self.full_output_path)

        except FileExistsError:
            to_delete = str(input(f"The directory {self.full_output_path} "
                                  f"already exists.\nDo you want to overwrite it? Y/N ")).lower()
            while to_delete != 'y' and to_delete != 'n':
                to_delete = str(input("Your response was invalid.\nDo you want to delete it? Y/N ").lower())

            if to_delete in ['y', 'yes']:
                shutil.rmtree(path=self.full_output_path, ignore_errors=True)
                os.mkdir(self.full_output_path)

            else:
                print_error(f"Pack not created in {self.full_output_path}")
                return False

        return True

    def fix_test_file_import(self, name_to_change: str):
        """Fixes the import statement in the _test.py file in the newly created initegration/script

        Args:
            name_to_change (str): The name of the former integration/script to replace in the import.
        """
        with open(os.path.join(self.full_output_path, f"{self.dir_name}_test.py"), 'r') as fp:
            file_contents = fp.read()

        file_contents = file_contents.replace(f'.{name_to_change}', self.dir_name)

        with open(os.path.join(self.full_output_path, f"{self.dir_name}_test.py"), 'w') as fp:
            fp.write(file_contents)

    def copy_common_server_python(self):
        """copy commonserverpython from the base pack"""
        if self.common_server:
            try:
                common_server_path = get_common_server_path(self.configuration.env_dir)
                shutil.copy(common_server_path, self.full_output_path)
            except Exception as err:
                print_v(f'Could not copy CommonServerPython: {str(err)}')

    def copy_demistotmock(self):
        """copy demistomock from content"""
        if self.demisto_mock:
            try:
                shutil.copy(f'{self.configuration.env_dir}/Tests/demistomock/demistomock.py', self.full_output_path)
            except Exception as err:
                print_v(f'Could not copy demistomock: {str(err)}')

    def get_template_files(self):
        """
        Gets the list of the integration/script file names to create according to the selected template.
        Returns:
            set. The names of integration/script files to create.
        """
        if self.is_integration:
            template_files = {filename.replace(self.TEMPLATE_INTEGRATION_NAME, self.template)
                              for filename in self.TEMPLATE_INTEGRATION_FILES}

            if self.template == self.HELLO_WORLD_INTEGRATION:
                template_files = template_files.union(self.HELLO_WORLD_TEST_DATA_FILES)

            elif self.template == self.HELLO_WORLD_FEED_INTEGRATION:
                template_files = template_files.union(self.HELLO_WORLD_FEED_TEST_DATA_FILES)

            elif self.template == self.HELLO_WORLD_SLIM_INTEGRATION:
                template_files = template_files.union(self.HELLO_WORLD_SLIM_TEST_DATA_FILES)

            elif self.template == self.DEFAULT_INTEGRATION_TEMPLATE:
                template_files = template_files.union(self.DEFAULT_INTEGRATION_TEST_DATA_FILES)
        else:
            template_files = {filename.replace(self.TEMPLATE_SCRIPT_NAME, self.template)
                              for filename in self.TEMPLATE_SCRIPT_FILES}

            if self.template == self.DEFAULT_SCRIPT_TEMPLATE:
                template_files = template_files.union(self.DEFAULT_SCRIPT_TEST_DATA_FILES)
        return template_files

    def get_remote_templates(self, files_list, dir):
        """
        Downloading the object related template-files and saving them in the output path.
        Args:
            files_list: List of files to download.
            dir: The name of the relevant directory (e.g. "Integrations", "Scripts").
        Returns:
            bool. True if the files were downloaded and saved successfully, False otherwise.
        """
        # create test_data dir
        if self.template in [self.HELLO_WORLD_INTEGRATION] + self.DEFAULT_TEMPLATES \
                + [self.HELLO_WORLD_FEED_INTEGRATION]:
            os.mkdir(os.path.join(self.full_output_path, self.TEST_DATA_DIR))

        if self.template in self.DEFAULT_TEMPLATES:
            pack_name = self.DEFAULT_TEMPLATE_PACK_NAME

        elif self.template in self.HELLO_WORLD_BASE_TEMPLATES + [self.HELLO_WORLD_FEED_INTEGRATION]:
            pack_name = self.HELLO_WORLD_PACK_NAME

        else:
            pack_name = self.template

        path = os.path.join('Packs', pack_name, dir, self.template)

        for file in files_list:
            try:
                filename = file
                if 'README.md' in file and self.template not in self.HELLO_WORLD_BASE_TEMPLATES:
                    # This is for the cases when the actual readme file name in content repo
                    # is `README_example.md` - which happens when we do not want the readme
                    # files to appear in https://xsoar.pan.dev/docs/reference/index.
                    filename = file.replace('README.md', 'README_example.md')
                file_content = tools.get_remote_file(
                    os.path.join(path, filename),
                    return_content=True,
                    # Templates available only in the official repo
                    git_content_config=GitContentConfig(repo_name=GitContentConfig.OFFICIAL_CONTENT_REPO_NAME)
                )
                with open(os.path.join(self.full_output_path, file), 'wb') as f:
                    f.write(file_content)
            except Exception:
                print_warning(f"Could not fetch remote template - {file}. Using local templates instead.")
                return False

        return True


# demisto_sdk/commands/integration_diff/integration_diff_detector.py

import os

import click

from demisto_sdk.commands.common.constants import (ARGUMENT_FIELDS_TO_CHECK, INTEGRATION_ARGUMENT_TYPES,
                                                   PARAM_FIELDS_TO_CHECK)
from demisto_sdk.commands.common.tools import get_yaml


class IntegrationDiffDetector:

    def __init__(self, new: str = '', old: str = '', docs_format: bool = False):

        if not os.path.exists(new):
            click.secho('No such file or directory for the new integration version.', fg='bright_red')

        if not os.path.exists(old):
            click.secho('No such file or directory for the old integration version.', fg='bright_red')

        self.new = new
        self.old = old
        self.docs_format_output = docs_format

        self.old_yaml_data = get_yaml(self.old)
        self.new_yaml_data = get_yaml(self.new)

        self.fount_missing = False
        self.missing_items_report: dict = {}

    def check_different(self) -> bool:
        """
        Checks differences between two integration yaml files.

        Return:
            bool. return true if the new integration contains everything in the old integration.
        """

        self.missing_items_report = self.get_differences()

        if self.print_items():
            return False

        return True

    def get_differences(self) -> dict:
        """
            Gets the different elements between the two integrations.

            Return:
                A dict contains all the found different elements between the integrations.
        """

        differences_result = {}

        old_commands_data = self.old_yaml_data['script']['commands']
        new_commands_data = self.new_yaml_data['script']['commands']

        commands, arguments, outputs = self.get_different_commands(old_commands_data, new_commands_data)

        parameters = self.get_different_params(self.old_yaml_data['configuration'], self.new_yaml_data['configuration'])

        if parameters:
            differences_result['parameters'] = parameters

        if commands:
            differences_result['commands'] = commands

        if arguments:
            differences_result['arguments'] = arguments

        if outputs:
            differences_result['outputs'] = outputs

        return differences_result

    def get_different_commands(self, old_commands, new_commands) -> tuple:
        """
        Checks differences between two list of the integration commands.

        Args:
            old_commands: List of the old integration commands.
            new_commands: List of the new integration commands.

        Returns:
            Three lists. Each one is a list of the found different commands/arguments/outputs.
        """
        commands = []
        arguments = []
        outputs = []

        # for each old integration command check if exist in the new, if not, check what is missing
        for old_command in old_commands:
            if old_command not in new_commands:
                new_command = self.check_if_element_exist(old_command, new_commands, 'name')

                if not new_command:
                    commands.append({
                        'type': 'commands',
                        'name': old_command['name'],
                        'message': f'Missing the command \'{old_command["name"]}\'.',
                        'description': old_command.get('description', '')
                    })

                else:
                    # Gets all the fields that are different between the two commands
                    changed_fields = [field for field in new_command if field in old_command and
                                      new_command[field] != old_command[field]]

                    if 'arguments' in changed_fields:
                        arguments.extend(self.get_different_arguments(new_command, old_command))

                    if 'outputs' in changed_fields:
                        outputs.extend(self.get_different_outputs(new_command, old_command))

        return commands, arguments, outputs

    def get_different_arguments(self, new_command, old_command) -> list:
        """
        Checks for different arguments between two commands.

        Args:
            new_command: The new integration command.
            old_command: The old integration command.

        Return:
            List of all the found different arguments.
        """
        arguments = []

        new_command_arguments = new_command['arguments']
        old_command_arguments = old_command['arguments']

        for argument in old_command_arguments:
            if argument not in new_command_arguments:

                new_command_argument = IntegrationDiffDetector.check_if_element_exist(argument, new_command_arguments,
                                                                                      'name')

                if not new_command_argument:
                    arguments.append({
                        'type': 'arguments',
                        'name': argument['name'],
                        'command_name': new_command['name'],
                        'message': f'Missing the argument \'{argument["name"]}\' in the command \'{new_command["name"]}\'.'
                    })

                else:
                    # Gets all the fields that are different between the two arguments
                    changed_fields = [field for field in new_command_argument if
                                      (field in argument and new_command_argument[field] != argument[field]) or
                                      (field not in argument and new_command_argument[field])]

                    arguments.extend(self.check_changed_fields(element=new_command_argument, element_type='arguments',
                                                               fields_to_check=ARGUMENT_FIELDS_TO_CHECK,
                                                               changed_fields=changed_fields,
                                                               command=new_command))

        return arguments

    def check_changed_fields(self, element, element_type, fields_to_check, changed_fields, command=None) -> list:
        """
        Checks for important changed fields in a given element.

        Args:
            element: The element to check.
            element_type: The element type.
            fields_to_check: List of the important fields to check if changed.
            changed_fields: List of changed fields.
            command: The command to check his argument.

        Return:
            A list contain all the elements with important changed fields.
        """
        result = []

        for field in fields_to_check:
            if field in changed_fields:

                # We want to return the element only if his field was False and changed to be True.
                if (field == 'required' or field == 'isArray') and not element[field]:
                    continue

                if element_type == 'arguments':
                    result.append({
                        'type': element_type,
                        'name': element['name'],
                        'command_name': command["name"],
                        'message': f'The argument `{element["name"]}` in the command `{command["name"]}` '
                                   f'- {self.get_change_description(field, element[field])}.',
                        'changed_field': field,
                        'changed_value': element[field]
                    })

                elif element_type == 'parameters':
                    result.append({
                        'type': element_type,
                        'name': element['display'],
                        'message': f'The parameter `{element["display"]}` '
                                   f'- {self.get_change_description(field, element[field])}.',
                        'changed_field': field,
                        'changed_value': element[field]
                    })

        return result

    @staticmethod
    def get_change_description(field, value) -> str:
        """
        Gets an element change description.

        Args:
             field: The field that has changed.
             value: The new value in the field.

        Return:
            A description of the change.
        """

        if field in ['defaultValue', 'defaultvalue']:
            return f'The default value changed to \'{value}\''

        elif field == 'required':
            return 'Is now required'

        elif field == 'isArray':
            return 'Now supports comma separated values'

        elif field == 'type':
            return f'The type changed to \'{INTEGRATION_ARGUMENT_TYPES[str(value)]}\''

        return ''

    @staticmethod
    def get_different_outputs(new_command, old_command) -> list:
        """
        Checks for different outputs between two commands.

        Args:
            new_command: The new integration command.
            old_command: The old integration command.

        Return:
            List of all the found different outputs.
        """
        outputs = []

        new_command_outputs = new_command['outputs']
        old_command_outputs = old_command['outputs']

        for output in old_command_outputs:
            if output not in new_command_outputs:

                new_command_output = IntegrationDiffDetector.check_if_element_exist(output, new_command_outputs,
                                                                                    'contextPath')

                if not new_command_output:
                    outputs.append({
                        'type': 'outputs',
                        'name': output['contextPath'],
                        'command_name': new_command["name"],
                        'message': f'Missing the output \'{output["contextPath"]}\' in the command '
                                   f'\'{new_command["name"]}\'.'
                    })

                else:
                    # Gets all the fields that are different between the two outputs
                    changed_fields = [field for field in new_command_output if
                                      field in output and new_command_output[field] != output[field]]

                    if 'type' in changed_fields:
                        outputs.append({
                            'type': 'outputs',
                            'name': output['contextPath'],
                            'command_name': new_command["name"],
                            'message': f'The output \'{output["contextPath"]}\' in the command '
                                       f'\'{new_command["name"]}\' was changed in field \'type\'.',
                            'changed_field': 'type'
                        })

        return outputs

    def get_different_params(self, old_params, new_params) -> list:
        """
        Checks differences between two list of the integration parameters.

        Args:
            old_params: The old integration parameters.
            new_params: The new integration parameters.

        Return:
            List of all the found different parameters.
        """
        parameters = []

        for old_param in old_params:
            if old_param not in new_params:
                param = self.check_if_element_exist(old_param, new_params, 'display')

                if not param:
                    parameters.append({
                        'type': 'parameters',
                        'name': old_param['display'],
                        'message': f'Missing the parameter \'{old_param["display"]}\'.'
                    })

                else:
                    # Gets all the fields that are different between the two params
                    changed_fields = [field for field in param if
                                      (field in old_param and param[field] != old_param[field]) or
                                      (field not in old_param and param[field])]

                    parameters.extend(self.check_changed_fields(element=param, element_type='parameters',
                                                                fields_to_check=PARAM_FIELDS_TO_CHECK,
                                                                changed_fields=changed_fields))
        return parameters

    @staticmethod
    def check_if_element_exist(element_to_check, list_of_elements, field_to_check) -> dict:
        """
        Check if a given element exists in a given list.

        Args:
            element_to_check: The element to check.
            list_of_elements: The list of elements.
            field_to_check: The field being checked.

        Return:
            The element if exist and an empty dict if not.
        """

        for element in list_of_elements:

            if element[field_to_check] == element_to_check[field_to_check]:
                return element

        return {}

    def print_missing_items(self):
        """
        Prints the missing elements report.
        """
        changed = []

        for missing_type in self.missing_items_report:
            click.secho(f"Missing {missing_type}:\n", fg='bright_red')

            for item in self.missing_items_report[missing_type]:
                if 'changed_field' in item:
                    changed.append(item['message'])

                else:
                    click.secho(item['message'], fg='bright_red')

            if changed:
                click.secho(f"\nChanged {missing_type}:\n", fg='bright_red')
                click.secho("\n".join(changed), fg='bright_red')
                changed.clear()

            click.secho("")

    def print_items_in_docs_format(self, secho_result: bool = True) -> str:
        """
        Prints the version differences report in docs format so the user will can copy it to the README file.

        Args:
            secho_result: whether to print the result in the terminal.

        Return:
            The section result as a string to print.
        """
        result = ''

        if secho_result:
            result = f'\n## Breaking changes from the previous version of this integration - ' \
                     f'{self.new_yaml_data.get("display", "")}\n' \
                     'The following sections list the changes in this version.\n\n'

        if 'commands' in self.missing_items_report:
            result += '### Commands\n#### The following commands were removed in this version:\n'

            for command in self.missing_items_report['commands']:
                result += f'* *{command["name"]}* - this command was replaced by XXX.\n'

        if 'arguments' in self.missing_items_report:
            result += '\n### Arguments'
            # Divide the arguments between removed and changed.
            removed_arguments = self.missing_items_report['arguments'].copy()
            changed_arguments = [removed_arguments.pop(removed_arguments.index(arg)) for arg in removed_arguments
                                 if 'changed_field' in arg]

            if removed_arguments:
                result += '\n#### The following arguments were removed in this version:\n'
                argument_per_command = self.get_elements_per_command(removed_arguments)
                result += self.get_elements_per_command_in_docs_format(argument_per_command, 'argument')

            if changed_arguments:
                result += '\n#### The behavior of the following arguments was changed:\n'
                argument_per_command = self.get_elements_per_command(changed_arguments)
                result += self.get_elements_per_command_in_docs_format(argument_per_command, 'argument', True)

        if 'outputs' in self.missing_items_report:
            result += '\n### Outputs\n#### The following outputs were removed in this version:\n'
            # Get only the removed outputs, and removed the changed.
            removed_outputs = [output for output in self.missing_items_report['outputs'] if 'changed_field' not in output]
            output_per_command = self.get_elements_per_command(removed_outputs)
            result += self.get_elements_per_command_in_docs_format(output_per_command, 'output')

        if secho_result:
            result += '\n## Additional Considerations for this version\n* Insert any API changes, ' \
                      'any behavioral changes, limitations, or restrictions that would be new to this version.\n'
            click.secho(result)

        return result

    def print_items(self) -> bool:
        """
        Prints the different elements report.

        Return:
            bool. return true if found items to print and false if not.
        """
        if not self.missing_items_report:
            click.secho("The integrations are backwards compatible", fg='green')
            return False

        if self.docs_format_output:
            self.print_items_in_docs_format()
            return False

        else:
            self.print_missing_items()
        return True

    @staticmethod
    def get_elements_per_command_in_docs_format(elements_per_command, element_type, message=False) -> str:
        """
        Gets the elements split per commands in docs format.

        Args:
            elements_per_command: A dictionary contains the split elements by command name.
            element_type: The type of elements.
            message: whether to use the element message.

        Return:
            String of the section result.
        """
        result = ''

        for command in elements_per_command:
            result += f'\nIn the *{command}* command:\n'

            for element in elements_per_command[command]:
                result += f'* *{element["name"]}* - {element["message"].rsplit(" - ", 1)[1]}\n' if message \
                    else f'* *{element["name"]}* - this {element_type} was replaced by XXX.\n'

        return result

    @staticmethod
    def get_elements_per_command(list_of_elements) -> dict:
        """
        Split a given list of elements by command.

        Args:
            list_of_elements: The list of elements to split.

        Return:
            Dict contains the split elements by command name.
        """
        result: dict = {}

        for element in list_of_elements:
            if element['command_name'] in result:
                result[element['command_name']].append(element)

            else:
                result[element['command_name']] = [element]

        return result


# demisto_sdk/commands/lint/commands_builder.py

# STD python packages
import os
from pathlib import Path
from typing import List, Optional

from packaging.version import parse

from demisto_sdk.commands.lint.resources.pylint_plugins.base_checker import base_msg
from demisto_sdk.commands.lint.resources.pylint_plugins.certified_partner_level_checker import cert_partner_msg
from demisto_sdk.commands.lint.resources.pylint_plugins.community_level_checker import community_msg
from demisto_sdk.commands.lint.resources.pylint_plugins.partner_level_checker import partner_msg
from demisto_sdk.commands.lint.resources.pylint_plugins.xsoar_level_checker import xsoar_msg

# Third party packages
# Local imports

excluded_files = ["CommonServerPython.py", "demistomock.py", "CommonServerUserPython.py", "conftest.py", ".venv"]


def build_flake8_command(files: List[Path]) -> str:
    """ Build command for executing flake8 lint check
        https://flake8.pycqa.org/en/latest/user/invocation.html
    Args:
        files(List[Path]): files to execute lint
        py_num(str): The python version in use

    Returns:
        str: flake8 command
    """

    command = "flake8 "

    # This is the same config used in `tox.ini` file in content
    # We will probably want to use that in the future for this and all linters
    command += '--ignore=W605,F403,F405,W503 '
    command += '--exclude=_script_template_docker.py,./CommonServerPython.py,./demistomock.py '
    command += '--max-line-length 130 '
    command += '--per-file-ignores=nudge_external_prs.py:E231,E251,E999 '

    # Generating file patterns - path1,path2,path3,..
    files_list = [file.name for file in files]
    command += ' '.join(files_list)

    return command


def build_bandit_command(files: List[Path]) -> str:
    """ Build command for executing bandit lint check
        https://github.com/PyCQA/bandit
    Args:
        files(List(Path)):  files to execute lint

    Returns:
        str: bandit command
    """
    command = "bandit"
    # Reporting only issues with high and medium severity level
    command += " -ll"
    # Reporting only issues of a high confidence level
    command += " -iii"
    # Skip the following tests: Pickle usage, Use of insecure hash func, Audit url open,
    # Using xml.etree.ElementTree.fromstring,  Using xml.dom.minidom.parseString
    command += " -s B301,B303,B310,B314,B318"
    # Aggregate output by filename
    command += " -a file"
    # File to be excluded when performing lints check
    command += f" --exclude={','.join(excluded_files)}"
    # Only show output in the case of an error
    command += " -q"
    # Setting error format
    command += " --format custom --msg-template '{abspath}:{line}: {test_id} " \
               "[Severity: {severity} Confidence: {confidence}] {msg}'"
    # Generating path patterns - path1,path2,path3,..
    files_list = [str(item) for item in files]
    command += f" -r {','.join(files_list)}"

    return command


def build_xsoar_linter_command(files: List[Path], support_level: str = "base") -> str:
    """ Build command to execute with xsoar linter module
    Args:
        py_num(str): The python version in use
        files(List[Path]): files to execute lint
        support_level: Support level for the file

    Returns:
       str: xsoar linter command using pylint load plugins
    """
    if not support_level:
        support_level = 'base'

    # linters by support level
    support_levels = {
        'base': 'base_checker',
        'community': 'base_checker,community_level_checker',
        'partner': 'base_checker,community_level_checker,partner_level_checker',
        'certified partner': 'base_checker,community_level_checker,partner_level_checker,'
                             'certified_partner_level_checker',
        'xsoar': 'base_checker,community_level_checker,partner_level_checker,certified_partner_level_checker,'
                 'xsoar_level_checker'
    }

    # messages from all level linters
    Msg_XSOAR_linter = {'base_checker': base_msg, 'community_level_checker': community_msg,
                        'partner_level_checker': partner_msg, 'certified_partner_level_checker': cert_partner_msg,
                        'xsoar_level_checker': xsoar_msg}

    checker_path = ""
    message_enable = ""
    if support_levels.get(support_level):
        checkers = support_levels.get(support_level)
        support = checkers.split(',') if checkers else []
        for checker in support:
            checker_path += f"{checker},"
            checker_msgs_list = Msg_XSOAR_linter.get(checker, {}).keys()
            for msg in checker_msgs_list:
                message_enable += f"{msg},"

    command = "pylint"
    # Excluded files
    command += f" --ignore={','.join(excluded_files)}"
    # Disable all errors
    command += " -E --disable=all"
    # Message format
    command += " --msg-template='{abspath}:{line}:{column}: {msg_id} {obj}: {msg}'"
    # Enable only Demisto Plugins errors.
    command += f" --enable={message_enable}"
    # Load plugins
    if checker_path:
        command += f" --load-plugins {checker_path}"
    # Generating path patterns - file1 file2 file3,..
    files_list = [str(file) for file in files]
    command += " " + " ".join(files_list)
    return command


def build_mypy_command(files: List[Path], version: str, content_repo: Path = None) -> str:
    """ Build command to execute with mypy module
        https://mypy.readthedocs.io/en/stable/command_line.html
    Args:
        files(List[Path]): files to execute lint
        version(str): python varsion X.Y (3.7, 2.7 ..)
        content_repo(Path): path of the content repo

    Returns:
        str: mypy command
    """
    command = "mypy"
    # Define python versions
    command += f" --python-version {version}"
    # This flag enable type checks the body of every function, regardless of whether it has type annotations.
    command += " --check-untyped-defs"
    # This flag makes mypy ignore all missing imports.
    command += " --ignore-missing-imports"
    # This flag adjusts how mypy follows imported modules that were not explicitly passed in via the command line
    command += " --follow-imports=silent"
    # This flag will add column offsets to error messages.
    command += " --show-column-numbers"
    # This flag will precede all errors with “note” messages explaining the context of the error.
    command += " --show-error-codes"
    # Use visually nicer output in error messages
    command += " --pretty"
    # This flag enables redefinion of a variable with an arbitrary type in some contexts.
    command += " --allow-redefinition"
    # Get the full path to the file.
    command += " --show-absolute-path"

    # If the command fails due to mypy bug, print the traceback
    command += " --show-traceback"

    # Point cache to be .mypy_cache in the content repo
    command += f" --cache-dir={content_repo/'.mypy_cache' if content_repo else '/dev/null'}"
    # Generating path patterns - file1 file2 file3,..
    files_list = [str(item) for item in files]
    command += " " + " ".join(files_list)

    return command


def build_vulture_command(files: List[Path], pack_path: Path) -> str:
    """ Build command to execute with pylint module
        https://github.com/jendrikseipp/vulture
    Args:
        py_num(str): The python version in use
        files(List[Path]): files to execute lint
        pack_path(Path): Package path

    Returns:
       str: vulture command
    """
    command = "vulture"
    # Excluded files
    command += f" --min-confidence {os.environ.get('VULTURE_MIN_CONFIDENCE_LEVEL', '100')}"
    # File to be excluded when performing lints check
    command += f" --exclude={','.join(excluded_files)}"
    # Whitelist vulture
    whitelist = Path(pack_path) / '.vulture_whitelist.py'
    if whitelist.exists():
        command += f" {whitelist.name}"
    files_list = [file.name for file in files]
    command += " " + " ".join(files_list)
    return command


def build_pylint_command(files: List[Path], docker_version: Optional[str] = None) -> str:
    """ Build command to execute with pylint module
        https://docs.pylint.org/en/1.6.0/run.html#invoking-pylint
    Args:
        files(List[Path]): files to execute lint
        docker_version: The version of the python docker image.
    Returns:
       str: pylint command
    """
    command = "pylint"
    # Excluded files
    command += f" --ignore={','.join(excluded_files)}"
    # Prints only errors
    command += " -E"
    # disable xsoar linter messages
    disable = ['bad-option-value']
    # TODO: remove when pylint will update its version to support py3.9

    if docker_version:
        py_ver = parse(docker_version)
        major = py_ver.major  # type: ignore
        minor = py_ver.minor  # type: ignore

        if major == 3 and minor >= 9:
            disable.append('unsubscriptable-object')
    command += f" --disable={','.join(disable)}"
    # Disable specific errors
    command += " -d duplicate-string-formatting-argument"
    # Message format
    command += " --msg-template='{abspath}:{line}:{column}: {msg_id} {obj}: {msg}'"
    # List of members which are set dynamically and missed by pylint inference system, and so shouldn't trigger
    # E1101 when accessed.
    command += " --generated-members=requests.packages.urllib3,requests.codes.ok"
    # Generating path patterns - file1 file2 file3,..
    files_list = [file.name for file in files]
    command += " " + " ".join(files_list)
    return command


def build_pytest_command(test_xml: str = "", json: bool = False, cov: str = "") -> str:
    """ Build command to execute with pytest module
        https://docs.pytest.org/en/latest/usage.html
    Args:
        test_xml(str): path indicate if required or not
        json(bool): Define json creation after test

    Returns:
        str: pytest command
    """
    command = "pytest -ra"
    # Generating junit-xml report - used in circle ci
    if test_xml:
        command += " --junitxml=/devwork/report_pytest.xml"
    # Generating json report
    if json:
        command += " --json=/devwork/report_pytest.json"

    if cov:
        command += f' --cov-report= --cov={cov}'

    return command


def build_pwsh_analyze_command(file: Path) -> str:
    """ Build command for powershell analyze

    Args:
        file(Path): files to execute lint

    Returns:
       str: powershell analyze command
    """
    # Invoke script analyzer
    command = "Invoke-ScriptAnalyzer"
    # Return exit code when finished
    command += " -EnableExit"
    # Lint Files paths
    command += f" -Path {file.name}"

    return f"pwsh -Command {command}"


def build_pwsh_test_command() -> str:
    """ Build command for powershell test

    Returns:
       str: powershell test command
    """
    command = "Invoke-Pester"
    # Return exit code when finished
    command += ' -Configuration \'@{Run=@{Exit=$true}; Output=@{Verbosity="Detailed"}}\''

    return f"pwsh -Command {command}"


# demisto_sdk/commands/lint/helpers.py

# STD python packages
import io
import logging
import os
import re
import shlex
import shutil
import sqlite3
import tarfile
import textwrap
from contextlib import contextmanager
from functools import lru_cache
from pathlib import Path
from typing import Callable, Dict, Generator, List, Optional, Union

# Third party packages
import coverage
import docker
import docker.errors
import git
import requests
from docker.models.containers import Container
from packaging.version import parse

# Local packages
from demisto_sdk.commands.common.constants import TYPE_PWSH, TYPE_PYTHON, DemistoException
from demisto_sdk.commands.common.docker_helper import init_global_docker_client

# Python2 requirements
PYTHON2_REQ = ["flake8", "vulture"]

# Define check exit code if failed
EXIT_CODES = {
    "flake8": 0b1,
    "XSOAR_linter": 0b1000000000,
    "bandit": 0b10,
    "mypy": 0b100,
    "vulture": 0b1000,
    "pytest": 0b10000,
    "pylint": 0b100000,
    "pwsh_analyze": 0b1000000,
    "pwsh_test": 0b10000000,
    "image": 0b100000000,
}

# Execution exit codes
SUCCESS = 0b0
FAIL = 0b1
RERUN = 0b10
WARNING = 0b100

# Power shell checks
PWSH_CHECKS = ["pwsh_analyze", "pwsh_test"]
PY_CHCEKS = ["flake8", "XSOAR_linter", "bandit", "mypy", "vulture", "pytest", "pylint"]

# Line break
RL = '\n'

logger = logging.getLogger('demisto-sdk')


def build_skipped_exit_code(no_flake8: bool, no_bandit: bool, no_mypy: bool, no_pylint: bool, no_vulture: bool,
                            no_xsoar_linter: bool,
                            no_test: bool, no_pwsh_analyze: bool, no_pwsh_test: bool, docker_engine: bool) -> float:
    """
    no_flake8(bool): Whether to skip flake8.
    no_xsoar_linter(bool): Whether to skip xsoar linter.
    no_bandit(bool): Whether to skip bandit.
    no_mypy(bool): Whether to skip mypy.
    no_vulture(bool): Whether to skip vulture
    no_pylint(bool): Whether to skip pylint.
    no_test(bool): Whether to skip pytest.
    docker_engine(bool): docker engine exists.
    """
    skipped_code = 0b0
    # When the CI env var is not set - on local env - check if any linters should be skipped
    # Otherwise - When the CI env var is set - Run all linters without skipping
    if not os.environ.get('CI'):
        if no_flake8:
            skipped_code |= EXIT_CODES["flake8"]
        if no_xsoar_linter:
            skipped_code |= EXIT_CODES["XSOAR_linter"]
        if no_bandit:
            skipped_code |= EXIT_CODES["bandit"]
        if no_mypy:
            skipped_code |= EXIT_CODES["mypy"]
        if no_vulture:
            skipped_code |= EXIT_CODES["vulture"]
        if no_pylint or not docker_engine:
            skipped_code |= EXIT_CODES["pylint"]
        if no_test or not docker_engine:
            skipped_code |= EXIT_CODES["pytest"]
        if no_pwsh_analyze or not docker_engine:
            skipped_code |= EXIT_CODES["pwsh_analyze"]
        if no_pwsh_test or not docker_engine:
            skipped_code |= EXIT_CODES["pwsh_test"]

    return skipped_code


def get_test_modules(content_repo: Optional[git.Repo], is_external_repo: bool) -> Dict[Path, bytes]:
    """ Get required test modules from content repository - {remote}/master
    1. Tests/demistomock/demistomock.py
    2. Tests/scripts/dev_envs/pytest/conftest.py
    3. Scripts/CommonServerPython/CommonServerPython.py
    4. CommonServerUserPython.py

    Returns:
        dict: path and file content - see below modules dict
    """
    if is_external_repo:
        modules = [Path("demistomock.py"),
                   Path("dev_envs/pytest/conftest.py"),
                   Path("CommonServerPython.py"),
                   Path("demistomock.ps1"),
                   Path("CommonServerPowerShell.ps1")
                   ]
    else:
        modules = [Path("Tests/demistomock/demistomock.py"),
                   Path("Tests/scripts/dev_envs/pytest/conftest.py"),
                   Path("Packs/Base/Scripts/CommonServerPython/CommonServerPython.py"),
                   Path("Tests/demistomock/demistomock.ps1"),
                   Path("Packs/Base/Scripts/CommonServerPowerShell/CommonServerPowerShell.ps1")
                   ]
    modules_content = {}
    if content_repo:
        # Trying to get file from local repo before downloading from GitHub repo (Get it from disk), Last fetch
        module_not_found = False

        for module in modules:
            try:
                module_full_path = content_repo.working_dir / module  # type: ignore
                logger.debug(f'read file {module_full_path}')
                if module.match('*CommonServerPython.py'):
                    # Remove import of DemistoClassApiModule in CommonServerPython,
                    # since tests don't use this class and the import fails the tests.
                    modules_content[module] = (module_full_path).read_bytes().replace(
                        b'from DemistoClassApiModule import *', b'')
                    logger.debug(f'Changed file {module_full_path} without demisto import')
                else:
                    modules_content[module] = (module_full_path).read_bytes()
            except FileNotFoundError:
                module_not_found = True
                logger.warning(f'Module {module} was not found, possibly deleted due to being in a feature branch')

        if module_not_found and is_external_repo:
            raise DemistoException('Unable to find module in external repository')
    else:
        # If not succeed to get from local repo copy, Download the required modules from GitHub
        for module in modules:
            url = f'https://raw.githubusercontent.com/demisto/content/master/{module}'
            for trial in range(2):
                res = requests.get(url=url,
                                   verify=False)
                if res.ok:
                    logger.debug('got file {module} - ok - not 4XX or 5XX')
                    modules_content[module] = res.content
                    break
                elif trial == 2:
                    raise requests.exceptions.ConnectionError

    modules_content[Path("CommonServerUserPython.py")] = b''

    return modules_content


@contextmanager
def add_typing_module(lint_files: List[Path], python_version: str):
    """ Check for typing import for python2 packages
            1. Entrance - Add import typing in the begining of the file.
            2. Closing - change back to original.

        Args:
            lint_files(list): File to execute lint - for adding typing in python 2.7
            python_version(str): The package python version.

        Raises:
            IOError: if can't write to files due permissions or other reasons
    """
    added_modules: List[Path] = []
    back_lint_files: List[Path] = []
    try:
        # Add typing import if needed to python version 2 packages
        py_ver = parse(python_version).major  # type: ignore
        if py_ver < 3:
            for lint_file in lint_files:
                data = lint_file.read_text(encoding="utf-8")
                typing_regex = "(from typing import|import typing)"
                module_match = re.search(typing_regex, data)
                if not module_match:
                    original_file = lint_file
                    back_file = lint_file.with_suffix('.bak')
                    original_file.rename(back_file)
                    data = back_file.read_text()
                    original_file.write_text("from typing import *  # noqa: F401" + '\n' + data)
                    back_lint_files.append(back_file)
                    added_modules.append(original_file)
        yield
    except Exception:
        pass
    finally:
        for added_module in added_modules:
            if added_module.exists():
                added_module.unlink()
        for back_file in back_lint_files:
            if back_file.exists():
                original_name = back_file.with_suffix('.py')
                back_file.rename(original_name)


@contextmanager
def add_tmp_lint_files(content_repo: Path, pack_path: Path, lint_files: List[Path], modules: Dict[Path, bytes],
                       pack_type: str):
    """ LintFiles is context manager to mandatory files for lint and test
            1. Entrance - download missing files to pack.
            2. Closing - Remove downloaded files from pack.

        Args:
            pack_path(Path): Absolute path of pack
            lint_files(list): File to execute lint - for adding typing in python 2.7
            modules(dict): modules content to locate in pack path
            content_repo(Path): Repository object
            pack_type(st): Pack type.

        Raises:
            IOError: if can't write to files due permissions or other reasons
    """
    added_modules: List[Path] = []
    try:
        # Add mandatory test,lint modules
        for module, content in modules.items():
            pwsh_module = TYPE_PWSH == pack_type and module.suffix == '.ps1'
            python_module = TYPE_PYTHON == pack_type and module.suffix == '.py'
            if pwsh_module or python_module:
                cur_path = pack_path / module.name
                if not cur_path.exists():
                    cur_path.write_bytes(content)
                    added_modules.append(cur_path)
        if pack_type == TYPE_PYTHON:
            # Append empty so it will exists
            cur_path = pack_path / "CommonServerUserPython.py"
            if not cur_path.exists():
                cur_path.touch()
                added_modules.append(cur_path)

            # Add API modules to directory if needed
            module_regex = r'from ([\w\d]+ApiModule) import \*(?:  # noqa: E402)?'
            for lint_file in lint_files:
                module_name = ""
                data = lint_file.read_text()
                module_match = re.search(module_regex, data)
                if module_match:
                    module_name = module_match.group(1)
                    rel_api_path = Path('Packs/ApiModules/Scripts') / module_name / f'{module_name}.py'
                    cur_path = pack_path / f'{module_name}.py'
                    if content_repo:
                        module_path = content_repo / rel_api_path
                        shutil.copy(src=module_path,
                                    dst=cur_path)
                    else:
                        url = f'https://raw.githubusercontent.com/demisto/content/master/{rel_api_path}'
                        api_content = requests.get(url=url,
                                                   verify=False).content
                        cur_path.write_bytes(api_content)

                    added_modules.append(cur_path)
        yield
    except Exception as e:
        logger.error(f'add_tmp_lint_files unexpected exception: {str(e)}')
        raise


@lru_cache(maxsize=300)
def get_python_version_from_image(image: str, timeout: int = 60) -> str:
    """ Get python version from docker image

    Args:
        image(str): Docker image id or name
        timeout(int): Docker client request timeout

    Returns:
        str: Python version X.Y (3.7, 3.6, ..)
    """
    # skip pwoershell images
    if 'pwsh' in image or 'powershell' in image:
        return '3.8'

    match_group = re.match(r'[\d\w]+/python3?:(?P<python_version>[23]\.\d+)', image)
    if match_group:
        return match_group.groupdict()['python_version']
    py_num = None
    # Run three times
    log_prompt = f'Get python version from image {image}'
    docker_client = init_global_docker_client(timeout=timeout, log_prompt=log_prompt)
    logger.info(f'{log_prompt} - Start')
    try:
        logger.debug(f'{log_prompt} - Running `sys.version_info` in the image')
        command = "python -c \"import sys; print('{}.{}'.format(sys.version_info[0], sys.version_info[1]))\""

        py_num = docker_client.containers.run(
            image=image,
            command=shlex.split(command),
            remove=True,
            restart_policy={"Name": "on-failure", "MaximumRetryCount": 3}
        )
        # Wait for container to finish
        logger.debug(f'{log_prompt} - Container finished running. {py_num=}')

        # Get python version
        py_num = parse(py_num.decode("utf-8")).base_version

    except Exception:
        logger.exception(f'{log_prompt} - Failed detecting Python version for image {image}')
    logger.info(f'{log_prompt} - End. Python version is {py_num}')
    return py_num if py_num else '3.8'


def get_file_from_container(container_obj: Container, container_path: str, encoding: str = "") -> Union[str, bytes]:
    """ Copy file from container.

    Args:
        container_obj(Container): Container ID to copy file from
        container_path(Path): Path in container image (file)
        encoding(str): valid encoding e.g. utf-8

    Returns:
        str or bytes: file as string decoded in utf-8

    Raises:
        IOError: Raise IO error if unable to create temp file
    """
    data: Union[str, bytes] = b''
    archive, stat = container_obj.get_archive(container_path)
    file_like = io.BytesIO(b"".join(archive))
    with tarfile.open(fileobj=file_like) as tar:
        before_read = tar.extractfile(stat['name'])
    if isinstance(before_read, io.BufferedReader):
        data = before_read.read()
    if encoding and isinstance(data, bytes):
        data = data.decode(encoding)

    return data


def copy_dir_to_container(container_obj: Container, host_path: Path, container_path: Path):
    """ Copy all content directory from container.

    Args:
        container_obj(Container): Container ID to copy file from
        host_path(Path): Path in host (directory)
        container_path(Path): Path in container (directory)

    Returns:
        str: file as string decode as utf-8

    Raises:
        IOError: Rase IO error if unable to create temp file
    """
    excluded_regex = "(__init__.py|.*.back)"
    file_like_object = io.BytesIO()
    old_cwd = os.getcwd()
    with tarfile.open(fileobj=file_like_object, mode='w:gz') as archive:
        os.chdir(host_path)
        archive.add('.', recursive=True, filter=lambda tarinfo: (
            tarinfo if not re.search(excluded_regex, Path(tarinfo.name).name) else None))
        os.chdir(old_cwd)

    for trial in range(2):
        status = container_obj.put_archive(path=container_path,
                                           data=file_like_object.getvalue())
        if status:
            break
        elif trial == 1:
            raise docker.errors.APIError(message="unable to copy dir to container")


def stream_docker_container_output(streamer: Generator, logging_level: Callable = logger.info) -> None:
    """ Stream container logs

    Args:
        streamer(Generator): Generator created by docker-sdk
    """
    try:
        wrapper = textwrap.TextWrapper(initial_indent='\t',
                                       subsequent_indent='\t',
                                       width=150)
        for chunk in streamer:
            logging_level(wrapper.fill(str(chunk.decode('utf-8'))))
    except Exception:
        logger.info('Failed to stream a container log.')


@contextmanager
def pylint_plugin(dest: Path):
    """
    Function which links the given path with the content of pylint plugins folder in resources.
    The main purpose is to link each pack with the pylint plugins.
    Args:
        dest: Pack path.
    """
    plugin_dirs = Path(__file__).parent / 'resources' / 'pylint_plugins'

    try:
        for file in plugin_dirs.iterdir():
            if file.is_file() and file.name != '__pycache__' and file.name.split('.')[1] != 'pyc':
                os.symlink(file, dest / file.name)

        yield
    finally:
        for file in plugin_dirs.iterdir():
            if file.is_file() and file.name != '__pycache__' and file.name.split('.')[1] != 'pyc':
                if os.path.lexists(dest / f'{file.name}'):
                    (dest / f'{file.name}').unlink()


def split_warnings_errors(output: str):
    """
        Function which splits the given string into warning messages and error using W or E in the beginning of string
        For error messages that do not start with E , they will be returned as other.
        The output of a certain pack can both include:
            - Fail msgs
            - Fail msgs and warnings msgs
            - Passed msgs
            - Passed msgs and warnings msgs
            - warning msgs
        Args:
            output(str): string which contains messages from linters.
        return:
            list of error messags, list of warnings messages, list of all undetected messages
        """
    output_lst = output.split('\n')
    # Warnings and errors lists currently relevant for XSOAR Linter
    warnings_list = []
    error_list = []
    # Others list is relevant for mypy and flake8.
    other_msg_list = []
    for msg in output_lst:
        # 'W:' for python2 xsoar linter
        # 'W[0-9]' for python3 xsoar linter
        if (msg.startswith('W') and msg[1].isdigit()) or 'W:' in msg or 'W90' in msg:
            warnings_list.append(msg)
        elif (msg.startswith('E') and msg[1].isdigit()) or 'E:' in msg or 'E90' in msg:
            error_list.append(msg)
        else:
            other_msg_list.append(msg)

    return error_list, warnings_list, other_msg_list


def coverage_report_editor(coverage_file, code_file_absolute_path):
    """

    Args:
        coverage_file: the .coverage file this contains the coverage data in sqlite format.
        code_file_absolute_path: the real absolute path to the measured code file.

    Notes:
        the .coverage files contain all the files list with their absolute path.
        but our tests (pytest step) are running inside a docker container.
        so we have to change the path to the correct one.
    """
    logger.info(f'Editing coverage report for {coverage_file}')
    with sqlite3.connect(coverage_file) as sql_connection:
        cursor = sql_connection.cursor()
        index = cursor.execute('SELECT count(*) FROM file').fetchall()[0][0]
        if not index == 1:
            logger.debug('unexpected file list in coverage report')
        else:
            cursor.execute('UPDATE file SET path = ? WHERE id = ?', (code_file_absolute_path, 1))
            sql_connection.commit()
        cursor.close()
    if not index == 1:
        os.remove(coverage_file)


def coverage_files():
    packs_pass = Path('Packs')
    for cov_path in packs_pass.glob('*/Integrations/*/.coverage'):
        yield str(cov_path)
    for cov_path in packs_pass.glob('*/Scripts/*/.coverage'):
        yield str(cov_path)


def generate_coverage_report(html=False, xml=False, report=True, cov_dir='coverage_report'):
    """
    Args:
        html(bool): should generate an html report. default false
        xml(bool): should generate an xml report. default false
        report(bool): should print the coverage report. default true
        cov_dir(str): the directory to place the report files (.coverage, html and xml report)
    """
    cov_file = os.path.join(cov_dir, '.coverage')
    cov = coverage.Coverage(data_file=cov_file)
    cov.combine(coverage_files())
    if not os.path.exists(cov_file):
        logger.warning(f'skipping coverage report {cov_file} file not found. '
                       f'Should not expect this if code files were changed or when linting all with pytest.')
        return

    export_msg = 'exporting {0} coverage report to {1}'
    if report:
        report_data = io.StringIO()
        report_data.write('\n\n############################\n unit-tests coverage report\n############################\n')
        try:
            cov.report(file=report_data)
        except coverage.misc.CoverageException as warning:
            if isinstance(warning.args, tuple) and warning.args and warning.args[0] == 'No data to report.':
                logger.info(f'No coverage data in file {cov_file}')
                return
            raise warning
        report_data.seek(0)
        logger.info(report_data.read())

    if html:
        html_dir = os.path.join(cov_dir, 'html')
        logger.info(export_msg.format('html', os.path.join(html_dir, 'index.html')))
        try:
            cov.html_report(directory=html_dir)
        except coverage.misc.CoverageException as warning:
            logger.warning(str(warning))
            return
    if xml:
        xml_file = os.path.join(cov_dir, 'coverage.xml')
        logger.info(export_msg.format('xml', xml_file))
        try:
            cov.xml_report(outfile=xml_file)
        except coverage.misc.CoverageException as warning:
            logger.warning(str(warning))
            return


# demisto_sdk/commands/lint/lint_manager.py

# STD packages
import concurrent.futures
import logging
import os
import platform
import re
import sys
import textwrap
from typing import Any, Dict, List, Set, Tuple, Union

import docker
import docker.errors
import git
import requests.exceptions
import urllib3.exceptions
from packaging.version import Version
from wcmatch.pathlib import Path, PosixPath

from demisto_sdk.commands.common.constants import PACKS_PACK_META_FILE_NAME, TYPE_PWSH, TYPE_PYTHON, DemistoException
from demisto_sdk.commands.common.docker_helper import init_global_docker_client
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.logger import Colors
from demisto_sdk.commands.common.timers import report_time_measurements
from demisto_sdk.commands.common.tools import (find_file, find_type, get_api_module_dependencies, get_content_path,
                                               get_file_displayed_name, get_json, is_external_repository, print_error,
                                               print_v, print_warning, retrieve_file_ending)
from demisto_sdk.commands.lint.helpers import (EXIT_CODES, FAIL, PWSH_CHECKS, PY_CHCEKS, SUCCESS,
                                               build_skipped_exit_code, generate_coverage_report, get_test_modules)
from demisto_sdk.commands.lint.linter import Linter

json = JSON_Handler()

# Third party packages

# Local packages
logger = logging.getLogger('demisto-sdk')

sha1Regex = re.compile(r'\b[0-9a-fA-F]{40}\b', re.M)


class LintManager:
    """ LintManager used to activate lint command using Linters in a single or multi thread.

    Attributes:
        input(str): Directories to run lint on.
        git(bool): Perform lint and test only on chaged packs.
        all_packs(bool): Whether to run on all packages.
        verbose(int): Whether to output a detailed response.
        quiet(bool): Whether to output a quiet response.
        log_path(str): Path to all levels of logs.
        prev_ver(str): Previous branch or SHA1 commit to run checks against.
        json_file_path(str): Path to a json file to write the run resutls to.
        id_set_path(str): Path to an existing id_set.json.
        check_dependent_api_module(bool): Whether to run lint also on the packs dependent on the modified api modules
        files.
    """

    def __init__(self, input: str, git: bool, all_packs: bool, quiet: bool, verbose: int, prev_ver: str,
                 json_file_path: str = '', id_set_path: str = None, check_dependent_api_module: bool = False):

        # Verbosity level
        self._verbose = not quiet if quiet else verbose
        # Gather facts for manager
        self._facts: dict = self._gather_facts()
        self._prev_ver = prev_ver
        self._all_packs = all_packs
        # Set 'git' to true if no packs have been specified, 'lint' should operate as 'lint -g'
        lint_no_packs_command = not git and not all_packs and not input
        git = True if lint_no_packs_command else git
        # Filter packages to lint and test check
        self._pkgs: List[PosixPath] = self._get_packages(content_repo=self._facts["content_repo"],
                                                         input=input,
                                                         git=git,
                                                         all_packs=all_packs,
                                                         base_branch=self._prev_ver)

        self._id_set_path = id_set_path
        if check_dependent_api_module:
            print('Checking for packages dependent on the modified API module', end='... ')
            dependent_on_api_module = get_api_module_dependencies(self._pkgs, self._id_set_path, self._verbose)
            dependent_on_api_module = self._get_packages(content_repo=self._facts["content_repo"],
                                                         input=dependent_on_api_module)
            self._pkgs = list(set(self._pkgs + dependent_on_api_module))
            if dependent_on_api_module:
                print(f'Found {Colors.Fg.cyan}{len(dependent_on_api_module)}{Colors.reset} dependent packages. '
                      f'Executing lint and test on those as well.')
            else:
                print('No dependent packages found.')
        if json_file_path:
            if os.path.isdir(json_file_path):
                json_file_path = os.path.join(json_file_path, 'lint_outputs.json')
        self.json_file_path = json_file_path
        self.linters_error_list: list = []

    @staticmethod
    def _gather_facts() -> Dict[str, Any]:
        """ Gather shared required facts for lint command execution - Also perform mandatory resource checkup.
            1. Content repo object.
            2. Requirements file for docker images.
            3. Mandatory test modules - demisto-mock.py etc
            3. Docker daemon check.

        Returns:
            dict: facts
        """
        global logger
        facts = {
            "content_repo": None,
            "requirements_3": None,
            "requirements_2": None,
            "test_modules": None,
            "docker_engine": True
        }
        # Get content repo object
        is_external_repo = False
        try:
            git_repo = git.Repo(os.getcwd(),
                                search_parent_directories=True)
            remote_url = git_repo.remote().urls.__next__()
            is_fork_repo = 'content' in remote_url
            is_external_repo = is_external_repository()

            if not is_fork_repo and not is_external_repo:
                raise git.InvalidGitRepositoryError

            facts["content_repo"] = git_repo  # type: ignore
            logger.debug(f"Content path {git_repo.working_dir}")
        except (git.InvalidGitRepositoryError, git.NoSuchPathError) as e:
            print_warning("You are running demisto-sdk lint not in content repository!")
            logger.warning(f"can't locate content repo {e}")
        # Get global requirements file
        pipfile_dir = Path(__file__).parent / 'resources'
        try:
            pipfile_lock_path = pipfile_dir / 'pipfile_python3/Pipfile.lock'
            with open(file=pipfile_lock_path) as f:
                lock_file: dict = json.load(fp=f)["develop"]
                facts["requirements_3"] = [key + value["version"] for key, value in  # type: ignore
                                           lock_file.items()]
                logger.debug("Test requirements successfully collected for python 3:\n"
                             f" {facts[f'requirements_3']}")
            python2_requirements = pipfile_dir / 'pipfile_python2/dev-requirements.txt'
            facts["requirements_2"] = python2_requirements.read_text().strip().split('\n')  # type: ignore
        except (json.JSONDecodeError, IOError, FileNotFoundError, KeyError) as e:
            print_error("Can't parse pipfile.lock - Aborting!")
            logger.critical(f"demisto-sdk-can't parse pipfile.lock {e}")
            sys.exit(1)
        # Get mandatory modulestest modules and Internet connection for docker usage
        try:
            facts["test_modules"] = get_test_modules(content_repo=facts["content_repo"],  # type: ignore
                                                     is_external_repo=is_external_repo)
            logger.debug("Test mandatory modules successfully collected")
        except (git.GitCommandError, DemistoException) as e:
            if is_external_repo:
                print_error('You are running on an external repo - '
                            'run `.hooks/bootstrap` before running the demisto-sdk lint command\n'
                            'See here for additional information: https://xsoar.pan.dev/docs/concepts/dev-setup')
            else:
                print_error(
                    "Unable to get test-modules demisto-mock.py etc - Aborting! corrupt repository or pull from master")
            logger.error(f"demisto-sdk-unable to get mandatory test-modules demisto-mock.py etc {e}")
            sys.exit(1)
        except (requests.exceptions.ConnectionError, urllib3.exceptions.NewConnectionError) as e:
            print_error("Unable to get mandatory test-modules demisto-mock.py etc - Aborting! (Check your internet "
                        "connection)")
            logger.error(f"demisto-sdk-unable to get mandatory test-modules demisto-mock.py etc {e}")
            sys.exit(1)
        # Validating docker engine connection
        logger.debug('creating docker client from env')
        docker_client: docker.DockerClient = init_global_docker_client(log_prompt='LintManager')
        try:
            logger.debug('pinging docker daemon')
            docker_client.ping()
        except (requests.exceptions.ConnectionError, urllib3.exceptions.ProtocolError, docker.errors.APIError) as ex:
            if os.getenv("CI") and os.getenv("CIRCLE_PROJECT_REPONAME") == "content":
                # when running lint in content we fail if docker isn't available for some reason
                raise ValueError("Docker engine not available and we are in content CI env. Can not run lint!!") from ex
            facts["docker_engine"] = False
            print_warning("Can't communicate with Docker daemon - check your docker Engine is ON - Skipping lint, "
                          "test which require docker!")
            logger.info("can not communicate with Docker daemon")
        logger.debug("Docker daemon test passed")
        return facts

    def _get_packages(self, content_repo: git.Repo, input: Union[str, List[str]], git: bool = False,
                      all_packs: bool = False,
                      base_branch: str = 'master') -> List[PosixPath]:
        """ Get packages paths to run lint command.

        Args:
            content_repo(git.Repo): Content repository object.
            input(str): dir pack specified as argument.
            git(bool): Perform lint and test only on changed packs.
            all_packs(bool): Whether to run on all packages.
            base_branch (str): Name of the branch or sha1 commit to run the diff on.

        Returns:
            List[PosixPath]: Pkgs to run lint
        """
        pkgs: list
        if all_packs or git:
            pkgs = LintManager._get_all_packages(content_dir=content_repo.working_dir)  # type: ignore
        else:  # specific pack as input, -i flag has been used
            pkgs = []
            if isinstance(input, str):
                input = input.split(',')
            for item in input:
                is_pack = os.path.isdir(item) and os.path.exists(os.path.join(item, PACKS_PACK_META_FILE_NAME))
                if is_pack:
                    pkgs.extend(LintManager._get_all_packages(content_dir=item))
                else:
                    pkgs.append(Path(item))
        if git:
            pkgs = self._filter_changed_packages(content_repo=content_repo, pkgs=pkgs,
                                                 base_branch=base_branch)
            for pkg in pkgs:
                print_v(f"Found changed package {Colors.Fg.cyan}{pkg}{Colors.reset}",
                        log_verbose=self._verbose)
        if pkgs:
            pkgs_str = ", ".join(map(str, pkgs))
            print(f"Executing lint and test on integrations and scripts in {Colors.Fg.cyan}{pkgs_str}{Colors.reset}")

        return pkgs

    @staticmethod
    def _get_all_packages(content_dir: str) -> List[str]:
        """Gets all integration, script in packages and packs in content repo.

        Returns:
            list: A list of integration, script and beta_integration names.
        """
        # Get packages from main content path
        content_main_pkgs: set = set(Path(content_dir).glob(['Integrations/*/',
                                                             'Scripts/*/', ]))
        # Get packages from packs path
        packs_dir: Path = Path(content_dir) / 'Packs'
        content_packs_pkgs: set = set(packs_dir.glob(['*/Integrations/*/',
                                                      '*/Scripts/*/']))
        all_pkgs = content_packs_pkgs.union(content_main_pkgs)

        return list(all_pkgs)

    @staticmethod
    def _get_packages_from_modified_files(modified_files):
        r"""
        Out of all modified files, return only the files relevant for linting, which are the packages
        (scripts\integrations) under the pack.
        Args:
            modified_files: A list of paths of files recognized as modified.

        Returns:
            A list of paths of modified packages (scripts/integrations)
        """
        return [path for path in modified_files if 'Scripts' in path.parts or 'Intergations' in path.parts]

    @staticmethod
    def _filter_changed_packages(content_repo: git.Repo, pkgs: List[PosixPath], base_branch: str) -> List[PosixPath]:
        """ Checks which packages had changes in them and should run on Lint.
        The diff is calculated using git, and is done by the following cases:
        - case 1: If the active branch is 'master', the diff is between master and the previous commit.
        - case 2: If the active branch is not master, and no other base branch is specified to comapre to,
         the diff is between the active branch and master.
        - case 3: If the base branch is specified, the diff is between the active branch (master\not master) and the given base branch.

        Args:
            pkgs(List[PosixPath]): pkgs to check
            base_branch (str): Name of the branch or sha1 commit to run the diff on.

        Returns:
            List[PosixPath]: A list of names of packages that should run.
        """

        staged_files = {content_repo.working_dir / Path(item.b_path).parent for item in  # type: ignore
                        content_repo.active_branch.commit.tree.diff(None, paths=pkgs)}

        if base_branch == 'master' and content_repo.active_branch.name == 'master':
            # case 1: comparing master against the latest previous commit
            last_common_commit = content_repo.remote().refs.master.commit.parents[0]
            print(f"Comparing {Colors.Fg.cyan}master{Colors.reset} to its {Colors.Fg.cyan}previous commit: "
                  f"{last_common_commit} {Colors.reset}")

        else:
            # cases 2+3: compare the active branch (master\not master) against the given base branch (master\not master)
            if sha1Regex.match(base_branch):  # if the base branch is given as a commit hash
                last_common_commit = base_branch
            else:
                last_common_commit = content_repo.merge_base(content_repo.active_branch.commit,
                                                             f'{content_repo.remote()}/{base_branch}')[0]
            print(f"Comparing {Colors.Fg.cyan}{content_repo.active_branch}{Colors.reset} to"
                  f" last common commit with {Colors.Fg.cyan}{last_common_commit}{Colors.reset}")

        changed_from_base = {content_repo.working_dir / Path(item.b_path).parent for item in  # type: ignore
                             content_repo.active_branch.commit.tree.diff(last_common_commit, paths=pkgs)}
        all_changed = staged_files.union(changed_from_base)
        pkgs_to_check = all_changed.intersection(pkgs)

        return list(pkgs_to_check)  # type: ignore

    def execute_all_packages(self,
                             parallel: int,
                             no_flake8: bool,
                             no_xsoar_linter: bool,
                             no_bandit: bool,
                             no_mypy: bool,
                             no_pylint: bool,
                             no_coverage: bool,
                             no_vulture: bool,
                             no_test: bool,
                             no_pwsh_analyze: bool,
                             no_pwsh_test: bool,
                             keep_container: bool,
                             test_xml: str,
                             docker_timeout: int,
                             lint_status: dict,
                             pkgs_status: dict,
                             pkgs_type: list,
                             ) -> Tuple[int, int]:
        """ Runs the Lint command on all given packages.

        Args:
            parallel(int): Whether to run command on multiple threads
            no_flake8(bool): Whether to skip flake8
            no_xsoar_linter(bool): Whether to skip xsoar linter
            no_bandit(bool): Whether to skip bandit
            no_mypy(bool): Whether to skip mypy
            no_vulture(bool): Whether to skip vulture
            no_pylint(bool): Whether to skip pylint
            no_coverage(bool): Run pytest without coverage report
            no_test(bool): Whether to skip pytest
            no_pwsh_analyze(bool): Whether to skip powershell code analyzing
            no_pwsh_test(bool): whether to skip powershell tests
            keep_container(bool): Whether to keep the test container
            test_xml(str): Path for saving pytest xml results
            docker_timeout(int): timeout for docker requests
            pkgs_type: List of the pack types
            pkgs_status: Dictionary for pack status (keys are packs, the values are their status)
            lint_status: Dictionary for the lint status  (the keys are the linters, the values are a list of packs)

        Returns:
            Tuple[int, int]: exit code, warning code
        """
        try:
            with concurrent.futures.ThreadPoolExecutor(max_workers=parallel) as executor:
                return_exit_code: int = 0
                return_warning_code: int = 0
                results = []
                # Executing lint checks in different threads
                for pack in sorted(self._pkgs):
                    linter: Linter = Linter(pack_dir=pack,
                                            content_repo="" if not self._facts["content_repo"] else  # type: ignore
                                            Path(self._facts["content_repo"].working_dir),
                                            req_2=self._facts["requirements_2"],
                                            req_3=self._facts["requirements_3"],
                                            docker_engine=self._facts["docker_engine"],
                                            docker_timeout=docker_timeout)
                    results.append(executor.submit(linter.run_pack,
                                                   no_flake8=no_flake8,
                                                   no_bandit=no_bandit,
                                                   no_mypy=no_mypy,
                                                   no_vulture=no_vulture,
                                                   no_xsoar_linter=no_xsoar_linter,
                                                   no_pylint=no_pylint,
                                                   no_test=no_test,
                                                   no_pwsh_analyze=no_pwsh_analyze,
                                                   no_pwsh_test=no_pwsh_test,
                                                   modules=self._facts["test_modules"],
                                                   keep_container=keep_container,
                                                   test_xml=test_xml,
                                                   no_coverage=no_coverage))

                logger.info('Waiting for futures to complete')
                for i, future in enumerate(concurrent.futures.as_completed(results)):
                    logger.debug(f'checking output of future {i=}')
                    pkg_status = future.result()
                    logger.info(f'Got lint results for {pkg_status["pkg"]}')
                    pkgs_status[pkg_status["pkg"]] = pkg_status
                    if pkg_status["exit_code"]:
                        for check, code in EXIT_CODES.items():
                            if pkg_status["exit_code"] & code:
                                lint_status[f"fail_packs_{check}"].append(pkg_status["pkg"])

                        if not return_exit_code & pkg_status["exit_code"]:
                            return_exit_code += pkg_status["exit_code"]
                    if pkg_status["warning_code"]:
                        for check, code in EXIT_CODES.items():
                            if pkg_status["warning_code"] & code:
                                lint_status[f"warning_packs_{check}"].append(pkg_status["pkg"])
                        if not return_warning_code & pkg_status["warning_code"]:
                            return_warning_code += pkg_status["warning_code"]
                    if pkg_status["pack_type"] not in pkgs_type:
                        pkgs_type.append(pkg_status["pack_type"])
                logger.info('Finished all futures')
                return return_exit_code, return_warning_code
        except KeyboardInterrupt:
            msg = "Stop demisto-sdk lint - Due to 'Ctrl C' signal"
            print_warning(msg)
            logger.warning(msg)
            executor.shutdown(wait=False)  # If keyboard interrupt no need to wait to clean resources
            return 1, 0
        except Exception as e:
            msg = f"Stop demisto-sdk lint - Due to Exception {e}"
            print_warning(msg)
            logger.error(msg)

            if Version(platform.python_version()) > Version('3.9'):
                executor.shutdown(wait=True, cancel_futures=True)  # type: ignore[call-arg]
            else:
                logger.info('Using Python under 3.8, we will cancel futures manually.')
                executor.shutdown(wait=True)  # Note that `cancel_futures` not supported in python 3.8
                for res in results:
                    res.cancel()
            return 1, 0

    def run(self, parallel: int, no_flake8: bool, no_xsoar_linter: bool, no_bandit: bool, no_mypy: bool,
            no_pylint: bool, no_coverage: bool, coverage_report: str,
            no_vulture: bool, no_test: bool, no_pwsh_analyze: bool, no_pwsh_test: bool,
            keep_container: bool,
            test_xml: str, failure_report: str, docker_timeout: int,
            time_measurements_dir: str = None) -> int:
        """ Runs the Lint command on all given packages.

        Args:
            parallel(int): Whether to run command on multiple threads
            no_flake8(bool): Whether to skip flake8
            no_xsoar_linter(bool): Whether to skip xsoar linter
            no_bandit(bool): Whether to skip bandit
            no_mypy(bool): Whether to skip mypy
            no_vulture(bool): Whether to skip vulture
            no_pylint(bool): Whether to skip pylint
            no_coverage(bool): Run pytest without coverage report
            coverage_report(str): the directory fo exporting the coverage data
            no_test(bool): Whether to skip pytest
            no_pwsh_analyze(bool): Whether to skip powershell code analyzing
            no_pwsh_test(bool): whether to skip powershell tests
            keep_container(bool): Whether to keep the test container
            test_xml(str): Path for saving pytest xml results
            failure_report(str): Path for store failed packs report
            docker_timeout(int): timeout for docker requests
            time_measurements_dir(str): the directory fo exporting the time measurements info
            total_timeout (int): amount of seconds for the task

        Returns:
            int: exit code by fail exit codes by var EXIT_CODES
        """
        lint_status: Dict = {
            "fail_packs_flake8": [],
            "fail_packs_XSOAR_linter": [],
            "fail_packs_bandit": [],
            "fail_packs_mypy": [],
            "fail_packs_vulture": [],
            "fail_packs_pylint": [],
            "fail_packs_pytest": [],
            "fail_packs_pwsh_analyze": [],
            "fail_packs_pwsh_test": [],
            "fail_packs_image": [],
            "warning_packs_flake8": [],
            "warning_packs_XSOAR_linter": [],
            "warning_packs_bandit": [],
            "warning_packs_mypy": [],
            "warning_packs_vulture": [],
            "warning_packs_pylint": [],
            "warning_packs_pytest": [],
            "warning_packs_pwsh_analyze": [],
            "warning_packs_pwsh_test": [],
            "warning_packs_image": [],
        }

        # Python or powershell or both
        pkgs_type: list = []

        # Detailed packages status
        pkgs_status: dict = {}

        # Skiped lint and test codes
        skipped_code = build_skipped_exit_code(no_flake8=no_flake8, no_bandit=no_bandit, no_mypy=no_mypy,
                                               no_vulture=no_vulture, no_xsoar_linter=no_xsoar_linter,
                                               no_pylint=no_pylint, no_test=no_test, no_pwsh_analyze=no_pwsh_analyze,
                                               no_pwsh_test=no_pwsh_test, docker_engine=self._facts["docker_engine"])

        return_exit_code, return_warning_code = self.execute_all_packages(
            parallel=parallel,
            no_flake8=no_flake8,
            no_xsoar_linter=no_xsoar_linter,
            no_bandit=no_bandit,
            no_mypy=no_mypy,
            no_pylint=no_pylint,
            no_coverage=no_coverage,
            no_vulture=no_vulture,
            no_test=no_test,
            no_pwsh_test=no_pwsh_test,
            keep_container=keep_container,
            test_xml=test_xml,
            docker_timeout=docker_timeout,
            no_pwsh_analyze=no_pwsh_analyze,
            lint_status=lint_status,
            pkgs_status=pkgs_status,
            pkgs_type=pkgs_type,
        )

        if time_measurements_dir:
            report_time_measurements(group_name='lint', time_measurements_dir=time_measurements_dir)

        self._report_results(lint_status=lint_status,
                             pkgs_status=pkgs_status,
                             return_exit_code=return_exit_code,
                             return_warning_code=return_warning_code,
                             skipped_code=int(skipped_code),
                             pkgs_type=pkgs_type,
                             no_coverage=no_coverage,
                             coverage_report=coverage_report)

        self._create_failed_packs_report(lint_status=lint_status, path=failure_report)

        # check if there were any errors during lint run , if so set to FAIL as some error codes are bigger
        # then 512 and will not cause failure on the exit code.
        if return_exit_code:
            # allow all_packs to fail on mypy
            if self._all_packs and return_exit_code == EXIT_CODES['mypy']:
                return_exit_code = SUCCESS
            else:
                return_exit_code = FAIL
        return return_exit_code

    def _report_results(self, lint_status: dict, pkgs_status: dict, return_exit_code: int, return_warning_code: int,
                        skipped_code: int,
                        pkgs_type: list,
                        no_coverage: bool, coverage_report: str):
        """ Log report to console

        Args:
            lint_status(dict): Overall lint status
            pkgs_status(dict): All pkgs status dict
            return_exit_code(int): exit code will indicate which lint or test failed
            return_warning_code(int): warning code will indicate which lint or test caused warning messages
            skipped_code(int): skipped test code
            pkgs_type(list): list determine which pack type exits.
            no_coverage(bool): Do NOT create coverage report.

     """
        if not no_coverage:
            if coverage_report:
                generate_coverage_report(html=True, xml=True, cov_dir=coverage_report)
            else:
                generate_coverage_report()

        self.report_pass_lint_checks(return_exit_code=return_exit_code,
                                     skipped_code=skipped_code,
                                     pkgs_type=pkgs_type)
        self.report_warning_lint_checks(return_warning_code=return_warning_code,
                                        pkgs_status=pkgs_status,
                                        lint_status=lint_status,
                                        all_packs=self._all_packs)
        self.report_unit_tests(return_exit_code=return_exit_code,
                               pkgs_status=pkgs_status,
                               lint_status=lint_status)
        self.report_failed_lint_checks(return_exit_code=return_exit_code,
                                       pkgs_status=pkgs_status,
                                       lint_status=lint_status)

        self.report_failed_image_creation(return_exit_code=return_exit_code,
                                          pkgs_status=pkgs_status,
                                          lint_status=lint_status)

        self.report_summary(pkg=self._pkgs, pkgs_status=pkgs_status, lint_status=lint_status, all_packs=self._all_packs)
        self.create_json_output()

    @staticmethod
    def report_pass_lint_checks(return_exit_code: int, skipped_code: int, pkgs_type: list):
        """ Log PASS/FAIL on each lint/test

        Args:
            return_exit_code(int): exit code will indicate which lint or test failed
            skipped_code(int): skipped test code.
            pkgs_type(list): list determine which pack type exits.
         """
        longest_check_key = len(max(EXIT_CODES.keys(), key=len))
        for check, code in EXIT_CODES.items():
            spacing = longest_check_key - len(check)
            if 'XSOAR_linter' in check:
                check_str = check.replace('_', ' ')
            else:
                check_str = check.capitalize().replace('_', ' ')
            if (check in PY_CHCEKS and TYPE_PYTHON in pkgs_type) or (check in PWSH_CHECKS and TYPE_PWSH in pkgs_type):
                if code & skipped_code:
                    print(f"{check_str} {' ' * spacing}- {Colors.Fg.cyan}[SKIPPED]{Colors.reset}")
                elif code & return_exit_code:
                    print(f"{check_str} {' ' * spacing}- {Colors.Fg.red}[FAIL]{Colors.reset}")
                else:
                    print(f"{check_str} {' ' * spacing}- {Colors.Fg.green}[PASS]{Colors.reset}")
            elif check != 'image':
                print(f"{check_str} {' ' * spacing}- {Colors.Fg.cyan}[SKIPPED]{Colors.reset}")

    def report_failed_lint_checks(self, lint_status: dict, pkgs_status: dict, return_exit_code: int):
        """ Log failed lint log if exsits

        Args:
            lint_status(dict): Overall lint status
            pkgs_status(dict): All pkgs status dict
            return_exit_code(int): exit code will indicate which lint or test failed
        """
        for check in ["flake8", "XSOAR_linter", "bandit", "mypy", "vulture"]:
            if EXIT_CODES[check] & return_exit_code:
                sentence = f" {check.capitalize()} errors "
                print(f"\n{Colors.Fg.red}{'#' * len(sentence)}{Colors.reset}")
                print(f"{Colors.Fg.red}{sentence}{Colors.reset}")
                print(f"{Colors.Fg.red}{'#' * len(sentence)}{Colors.reset}\n")
                for fail_pack in lint_status[f"fail_packs_{check}"]:
                    print(f"{Colors.Fg.red}{pkgs_status[fail_pack]['pkg']}{Colors.reset}")
                    print(pkgs_status[fail_pack][f"{check}_errors"])
                    self.linters_error_list.append({
                        'linter': check,
                        'pack': fail_pack,
                        'type': 'error',
                        'messages': pkgs_status[fail_pack][f"{check}_errors"]
                    })

        for check in ["pylint", "pwsh_analyze", "pwsh_test"]:
            check_str = check.capitalize().replace('_', ' ')
            if EXIT_CODES[check] & return_exit_code:
                sentence = f" {check_str} errors "
                print(f"\n{Colors.Fg.red}{'#' * len(sentence)}{Colors.reset}")
                print(f"{Colors.Fg.red}{sentence}{Colors.reset}")
                print(f"{Colors.Fg.red}{'#' * len(sentence)}{Colors.reset}\n")
                for fail_pack in lint_status[f"fail_packs_{check}"]:
                    print(f"{Colors.Fg.red}{fail_pack}{Colors.reset}")
                    for image in pkgs_status[fail_pack]["images"]:
                        print(image[f"{check}_errors"])

    def report_warning_lint_checks(self, lint_status: dict, pkgs_status: dict, return_warning_code: int,
                                   all_packs: bool):
        """ Log warnings lint log if exists

        Args:
            lint_status(dict): Overall lint status
            pkgs_status(dict): All pkgs status dict
            return_warning_code(int): exit code will indicate which lint or test caused warnings
            all_packs(bool) if all packs runs then no need for warnings messages.
        """
        if not all_packs:
            for check in ["flake8", "XSOAR_linter", "bandit", "mypy", "vulture"]:
                if EXIT_CODES[check] & return_warning_code:
                    sentence = f" {check.capitalize()} warnings "
                    print(f"\n{Colors.Fg.orange}{'#' * len(sentence)}{Colors.reset}")
                    print(f"{Colors.Fg.orange}{sentence}{Colors.reset}")
                    print(f"{Colors.Fg.orange}{'#' * len(sentence)}{Colors.reset}\n")
                    for fail_pack in lint_status[f"warning_packs_{check}"]:
                        print(f"{Colors.Fg.orange}{pkgs_status[fail_pack]['pkg']}{Colors.reset}")
                        print(pkgs_status[fail_pack][f"{check}_warnings"])
                        self.linters_error_list.append({
                            'linter': check,
                            'pack': fail_pack,
                            'type': 'warning',
                            'messages': pkgs_status[fail_pack][f"{check}_warnings"]
                        })

    def report_unit_tests(self, lint_status: dict, pkgs_status: dict, return_exit_code: int):
        """ Log failed unit-tests , if verbosity specified will log also success unit-tests

        Args:
            lint_status(dict): Overall lint status
            pkgs_status(dict): All pkgs status dict
            return_exit_code(int): exit code will indicate which lint or test failed
        """
        # Indentation config
        preferred_width = 100
        pack_indent = 2
        pack_prefix = " " * pack_indent + "- Package: "
        wrapper_pack = textwrap.TextWrapper(initial_indent=pack_prefix,
                                            width=preferred_width,
                                            subsequent_indent=' ' * len(pack_prefix))
        docker_indent = 6
        docker_prefix = " " * docker_indent + "- Image: "
        wrapper_docker_image = textwrap.TextWrapper(initial_indent=docker_prefix,
                                                    width=preferred_width,
                                                    subsequent_indent=' ' * len(docker_prefix))
        test_indent = 9
        test_prefix = " " * test_indent + "- "
        wrapper_test = textwrap.TextWrapper(initial_indent=test_prefix, width=preferred_width,
                                            subsequent_indent=' ' * len(test_prefix))
        error_indent = 9
        error_first_prefix = " " * error_indent + "  Error: "
        error_sec_prefix = " " * error_indent + "         "
        wrapper_first_error = textwrap.TextWrapper(initial_indent=error_first_prefix, width=preferred_width,
                                                   subsequent_indent=' ' * len(error_first_prefix))
        wrapper_sec_error = textwrap.TextWrapper(initial_indent=error_sec_prefix, width=preferred_width,
                                                 subsequent_indent=' ' * len(error_sec_prefix))

        # Log passed unit-tests
        headline_printed = False
        passed_printed = False
        for pkg, status in pkgs_status.items():
            if status.get("images"):
                if status.get("images")[0].get("pytest_json", {}).get("report", {}).get("tests"):
                    if (not headline_printed and self._verbose) and (EXIT_CODES["pytest"] & return_exit_code):
                        # Log unit-tests
                        sentence = " Unit Tests "
                        print(f"\n{Colors.Fg.cyan}{'#' * len(sentence)}")
                        print(f"{sentence}")
                        print(f"{'#' * len(sentence)}{Colors.reset}")
                        headline_printed = True
                    if not passed_printed:
                        print_v(f"\n{Colors.Fg.green}Passed Unit-tests:{Colors.reset}", log_verbose=self._verbose)
                        passed_printed = True
                    print_v(wrapper_pack.fill(f"{Colors.Fg.green}{pkg}{Colors.reset}"), log_verbose=self._verbose)
                    for image in status["images"]:
                        if not image.get("image_errors"):
                            tests = image.get("pytest_json", {}).get("report", {}).get("tests")
                            if tests:
                                print_v(wrapper_docker_image.fill(image['image']), log_verbose=self._verbose)
                                for test_case in tests:
                                    outcome = test_case.get("call", {}).get("outcome")
                                    if outcome != "failed":
                                        name = re.sub(pattern=r"\[.*\]",
                                                      repl="",
                                                      string=test_case.get("name"))
                                        if outcome and outcome != "passed":
                                            name = f'{name} ({outcome.upper()})'
                                        print_v(wrapper_test.fill(name), log_verbose=self._verbose)

        # Log failed unit-tests
        if EXIT_CODES["pytest"] & return_exit_code:
            if not headline_printed:
                # Log unit-tests
                sentence = " Unit Tests "
                print(f"\n{Colors.Fg.cyan}{'#' * len(sentence)}")
                print(f"{sentence}")
                print(f"{'#' * len(sentence)}{Colors.reset}")
            print(f"\n{Colors.Fg.red}Failed Unit-tests:{Colors.reset}")
            for fail_pack in lint_status["fail_packs_pytest"]:
                print(wrapper_pack.fill(f"{Colors.Fg.red}{fail_pack}{Colors.reset}"))
                for image in pkgs_status[fail_pack]["images"]:
                    tests = image.get("pytest_json", {}).get("report", {}).get("tests")
                    if tests:
                        for test_case in tests:
                            if test_case.get("call", {}).get("outcome") == "failed":
                                name = re.sub(pattern=r"\[.*\]",
                                              repl="",
                                              string=test_case.get("name"))
                                print(wrapper_test.fill(name))
                                if test_case.get("call", {}).get("longrepr"):
                                    print(wrapper_docker_image.fill(image['image']))
                                    for i in range(len(test_case.get("call", {}).get("longrepr"))):
                                        if i == 0:
                                            print(wrapper_first_error.fill(
                                                test_case.get("call", {}).get("longrepr")[i]))
                                        else:
                                            print(wrapper_sec_error.fill(test_case.get("call", {}).get("longrepr")[i]))
                                    print('\n')
                    else:
                        print(wrapper_docker_image.fill(image['image']))
                        errors = image.get("pytest_errors", {})
                        if errors:
                            print(wrapper_sec_error.fill(errors))

    @staticmethod
    def report_failed_image_creation(lint_status: dict, pkgs_status: dict, return_exit_code: int):
        """ Log failed image creation if occured

        Args:
            lint_status(dict): Overall lint status
            pkgs_status(dict): All pkgs status dict
            return_exit_code(int): exit code will indicate which lint or test failed
     """
        # Indentation config
        preferred_width = 100
        indent = 2
        pack_prefix = " " * indent + "- Package: "
        wrapper_pack = textwrap.TextWrapper(initial_indent=pack_prefix,
                                            width=preferred_width,
                                            subsequent_indent=' ' * len(pack_prefix))
        image_prefix = " " * indent + "  Image: "
        wrapper_image = textwrap.TextWrapper(initial_indent=image_prefix, width=preferred_width,
                                             subsequent_indent=' ' * len(image_prefix))
        indent_error = 4
        error_prefix = " " * indent_error + "  Error: "
        wrapper_error = textwrap.TextWrapper(initial_indent=error_prefix, width=preferred_width,
                                             subsequent_indent=' ' * len(error_prefix))
        # Log failed images creation
        if EXIT_CODES["image"] & return_exit_code:
            sentence = " Image creation errors "
            print(f"\n{Colors.Fg.red}{'#' * len(sentence)}{Colors.reset}")
            print(f"{Colors.Fg.red}{sentence}{Colors.reset}")
            print(f"{Colors.Fg.red}{'#' * len(sentence)}{Colors.reset}")
            for fail_pack in lint_status["fail_packs_image"]:
                print(wrapper_pack.fill(f"{Colors.Fg.cyan}{fail_pack}{Colors.reset}"))
                for image in pkgs_status[fail_pack]["images"]:
                    print(wrapper_image.fill(image["image"]))
                    print(wrapper_error.fill(image["image_errors"]))

    @staticmethod
    def report_summary(pkg, pkgs_status: dict, lint_status: dict, all_packs: bool = False):
        """ Log failed image creation if occured

        Args:
            pkgs_status: The packs status
            lint_status(dict): Overall lint status
            all_packs(bool): True when running lint command with -a flag.
     """
        preferred_width = 100
        fail_pack_indent = 3
        fail_pack_prefix = " " * fail_pack_indent + "- "
        wrapper_fail_pack = textwrap.TextWrapper(initial_indent=fail_pack_prefix, width=preferred_width,
                                                 subsequent_indent=' ' * len(fail_pack_prefix))
        # intersection of all failed packages
        failed: Set[str] = set()

        # intersection of all warnings packages
        warnings: Set[str] = set()
        # each pack is checked for warnings and failures . A certain pack can appear in both failed packages and
        # warnings packages.
        for key in lint_status:
            # ignore mypy errors in all_packs report
            if all_packs and 'mypy' in key:
                continue

            if key.startswith('fail'):
                failed = failed.union(lint_status[key])
            if key.startswith('warning'):
                warnings = warnings.union(lint_status[key])
        if all_packs:
            num_passed = len([pack for pack, result in pkgs_status.items() if result.get('exit_code') == 0 or result.get('exit_code') == EXIT_CODES['mypy']])
        else:
            num_passed = len([pack for pack, result in pkgs_status.items() if result.get('exit_code') == 0])
        # Log unit-tests summary
        sentence = " Summary "
        print(f"\n{Colors.Fg.cyan}{'#' * len(sentence)}")
        print(f"{sentence}")
        print(f"{'#' * len(sentence)}{Colors.reset}")
        print(f"Packages: {len(pkg)}")
        print(f"Packages PASS: {Colors.Fg.green}{num_passed}{Colors.reset}")
        print(f"Packages FAIL: {Colors.Fg.red}{len(failed)}{Colors.reset}")
        print(f"Packages WARNING (can either PASS or FAIL): {Colors.Fg.orange}{len(warnings)}{Colors.reset}\n")

        if not all_packs:
            if warnings:
                print("Warning packages:")
            for warning in warnings:
                print(f"{Colors.Fg.orange}{wrapper_fail_pack.fill(warning)}{Colors.reset}")

        if failed:
            print("Failed packages:")
        for fail_pack in failed:
            print(f"{Colors.Fg.red}{wrapper_fail_pack.fill(fail_pack)}{Colors.reset}")

    @staticmethod
    def _create_failed_packs_report(lint_status: dict, path: str):
        """
        Creates and saves a file containing all lint failed packs
        :param lint_status: dict
            Dictionary containing type of failures and corresponding failing tests. Looks like this:
             lint_status = {
            "fail_packs_flake8": [],
            "fail_packs_bandit": [],
            "fail_packs_mypy": [],
            "fail_packs_vulture": [],
            "fail_packs_pylint": [],
            "fail_packs_pytest": [],
            "fail_packs_pwsh_analyze": [],
            "fail_packs_pwsh_test": [],
            "fail_packs_image": []
        }
        :param path: str
            The path to save the report.
        """
        failed_ut: set = set()
        for key in lint_status:
            if key.startswith('fail') and 'mypy' not in key:  # TODO remove this when reduce the number of failed `mypy` packages.
                failed_ut = failed_ut.union(lint_status[key])
        if path and failed_ut:
            file_path = Path(path) / "failed_lint_report.txt"
            file_path.write_text('\n'.join(failed_ut))

    def create_json_output(self):
        """Creates a JSON file output for lints"""
        if not self.json_file_path:
            return

        if os.path.exists(self.json_file_path):
            json_contents = get_json(self.json_file_path)
            if not (isinstance(json_contents, list)):
                json_contents = []
        else:
            json_contents = []
        logger.info('Collecting results to write to file')
        # format all linters to JSON format -
        # if any additional linters are added, please add a formatting function here
        for check in self.linters_error_list:
            if check.get('linter') == 'flake8':
                self.flake8_error_formatter(check, json_contents)
            elif check.get('linter') == 'mypy':
                self.mypy_error_formatter(check, json_contents)
            elif check.get('linter') == 'bandit':
                self.bandit_error_formatter(check, json_contents)
            elif check.get('linter') == 'vulture':
                self.vulture_error_formatter(check, json_contents)
            elif check.get('linter') == 'XSOAR_linter':
                self.xsoar_linter_error_formatter(check, json_contents)
        with open(self.json_file_path, 'w+') as f:
            json.dump(json_contents, f, indent=4)

        logger.info(f'Logs saved to {self.json_file_path}')

    def flake8_error_formatter(self, errors: Dict, json_contents: List) -> None:
        """Format flake8 error strings to JSON format and add them the json_contents

        Args:
            errors (Dict): A dictionary containing flake8 error strings
            json_contents (List): The JSON file outputs
        """
        error_messages = errors.get('messages', '')
        error_messages = error_messages.split('\n') if error_messages else []
        for message in error_messages:
            if message:
                file_path, line_number, column_number, _ = message.split(':', 3)
                code = message.split()[1]
                output = {
                    'linter': 'flake8',
                    'severity': errors.get('type'),
                    'errorCode': code,
                    'message': message.split(code)[1].lstrip(),
                    'row': line_number,
                    'col': column_number
                }
                self.add_to_json_outputs(output, file_path, json_contents)

    @staticmethod
    def gather_mypy_errors(error_messages: List) -> List:
        """Gather multi-line mypy errors to a single line

        Args:
            error_messages (List): A list of mypy error outputs

        Returns:
            List. A list of strings, each element is a full mypy error message
        """
        mypy_errors: list = []
        gather_error: list = []
        for line in error_messages:
            if os.path.isfile(line.split(':')[0]):
                if gather_error:
                    mypy_errors.append('\n'.join(gather_error))
                    gather_error = []
            gather_error.append(line)

        # handle final error
        # last line is irrelevant
        if gather_error:
            mypy_errors.append('\n'.join(gather_error[:-1]))

        return mypy_errors

    def mypy_error_formatter(self, errors: Dict, json_contents: List) -> None:
        """Format mypy error strings to JSON format and add them the json_contents

        Args:
            errors (Dict): A dictionary containing mypy error strings
            json_contents (List): The JSON file outputs
        """
        error_messages = errors.get('messages', '')
        error_messages = error_messages.split('\n') if error_messages else []
        mypy_errors = self.gather_mypy_errors(error_messages)

        for message in mypy_errors:
            if message:
                file_path, line_number, column_number, _ = message.split(':', 3)
                output_message = message  # default
                for prefix in ('error:', 'note:'):
                    if prefix in message:
                        output_message = message.split(prefix)[1].lstrip()
                        break
                output = {
                    'linter': 'mypy',
                    'severity': errors.get('type'),
                    'message': output_message,
                    'row': line_number,
                    'col': column_number
                }
                self.add_to_json_outputs(output, file_path, json_contents)

    def bandit_error_formatter(self, errors: Dict, json_contents: List) -> None:
        """Format bandit error strings to JSON format and add them the json_contents

        Args:
            errors (Dict): A dictionary containing bandit error strings
            json_contents (List): The JSON file outputs
        """
        error_messages = errors.get('messages', '')
        error_messages = error_messages.split('\n') if error_messages else []
        for message in error_messages:
            if message:
                file_path, line_number, _ = message.split(':', 2)
                output = {
                    'linter': 'bandit',
                    'severity': errors.get('type'),
                    'errorCode': message.split(' ')[1],
                    'message': message.split('[')[1].replace(']', ' -'),
                    'row': line_number,
                }
                self.add_to_json_outputs(output, file_path, json_contents)

    @staticmethod
    def get_full_file_path_for_vulture(file_name: str, content_path: str) -> str:
        """Get the full file path to a file with a given name name from the content path

        Args:
            file_name (str): The file name of the file to find
            content_path (str): The content file path

        Returns:
            str. The path to the file
        """
        file_ending = retrieve_file_ending(file_name)
        if not file_ending:
            file_name = f'{file_name}.py'
        elif file_ending != 'py':
            file_name = file_name.replace(file_ending, 'py')
        return find_file(content_path, file_name)

    def vulture_error_formatter(self, errors: Dict, json_contents: List) -> None:
        """Format vulture error strings to JSON format and add them the json_contents

        Args:
            errors (Dict): A dictionary containing vulture error strings
            json_contents (List): The JSON file outputs
        """
        error_messages = errors.get('messages', '')
        error_messages = error_messages.split('\n') if error_messages else []
        content_path = get_content_path()
        for message in error_messages:
            if message:
                file_name, line_number, error_contents = message.split(':', 2)
                file_path = self.get_full_file_path_for_vulture(file_name, content_path)  # type: ignore
                output = {
                    'linter': 'vulture',
                    'severity': errors.get('type'),
                    'message': error_contents.lstrip(),
                    'row': line_number,
                }
                self.add_to_json_outputs(output, file_path, json_contents)

    def xsoar_linter_error_formatter(self, errors: Dict, json_contents: List) -> None:
        """Format XSOAR linter error strings to JSON format and add them the json_contents

        Args:
            errors (Dict): A dictionary containing XSOAR linter error strings
            json_contents (List): The JSON file outputs
        """
        error_messages = errors.get('messages', '')
        error_messages = error_messages.split('\n') if error_messages else []
        for message in error_messages:
            if message:
                split_message = message.split(':')
                file_path = split_message[0] if len(split_message) >= 1 else ''
                code = message.split(' ')[1] if len(message.split(' ')) >= 2 else ''
                output = {
                    'linter': 'xsoar_linter',
                    'severity': errors.get('type'),
                    'errorCode': code,
                    'message': message.split(code)[-1].lstrip() if len(message.split(code)) >= 1 else '',
                    'row': split_message[1] if len(split_message) >= 2 else '',
                    'col': split_message[2] if len(split_message) >= 3 else ''
                }
                self.add_to_json_outputs(output, file_path, json_contents)

    @staticmethod
    def add_to_json_outputs(output: Dict, file_path: str, json_contents: List):
        """Adds an error entry to the JSON file contents

        Args:
            output (Dict): The information about an error entry
            file_path (str): The file path where the error occurred
            json_contents (List): The JSON file outputs
        """
        yml_file_path = file_path.replace('.py', '.yml').replace('.ps1', '.yml')
        file_type = find_type(yml_file_path)
        full_error_output = {
            'filePath': file_path,
            'fileType': os.path.splitext(file_path)[1].replace('.', ''),
            'entityType': file_type.value if file_type else '',
            'errorType': 'Code',
            'name': get_file_displayed_name(yml_file_path),  # type: ignore[arg-type]
            **output
        }
        json_contents.append(full_error_output)


# demisto_sdk/commands/lint/linter.py

# STD python packages
import copy
import hashlib
import logging
import os
import platform
import traceback
from typing import Any, Dict, List, Tuple

import docker
import docker.errors
import docker.models.containers
import git
import requests.exceptions
import urllib3.exceptions
from packaging.version import parse
from wcmatch.pathlib import NEGATE, Path

from demisto_sdk.commands.common.constants import INTEGRATIONS_DIR, PACKS_PACK_META_FILE_NAME, TYPE_PWSH, TYPE_PYTHON
from demisto_sdk.commands.common.docker_helper import get_docker, init_global_docker_client
from demisto_sdk.commands.common.handlers import JSON_Handler, YAML_Handler
from demisto_sdk.commands.common.timers import timer
from demisto_sdk.commands.common.tools import get_all_docker_images, run_command_os
from demisto_sdk.commands.lint.commands_builder import (build_bandit_command, build_flake8_command, build_mypy_command,
                                                        build_pwsh_analyze_command, build_pwsh_test_command,
                                                        build_pylint_command, build_pytest_command,
                                                        build_vulture_command, build_xsoar_linter_command)
from demisto_sdk.commands.lint.helpers import (EXIT_CODES, FAIL, RERUN, RL, SUCCESS, WARNING, add_tmp_lint_files,
                                               add_typing_module, coverage_report_editor, get_file_from_container,
                                               get_python_version_from_image, pylint_plugin, split_warnings_errors,
                                               stream_docker_container_output)

json = JSON_Handler()


# 3-rd party packages

# Local packages

logger = logging.getLogger('demisto-sdk')


class Linter:
    """ Linter used to activate lint command on single package

        Attributes:
            pack_dir(Path): Pack to run lint on.
            content_repo(Path): Git repo object of content repo.
            req_2(list): requirements for docker using python2.
            req_3(list): requirements for docker using python3.
            docker_engine(bool):  Whether docker engine detected by docker-sdk.
    """

    def __init__(self, pack_dir: Path, content_repo: Path, req_3: list, req_2: list, docker_engine: bool,
                 docker_timeout: int):
        self._req_3 = req_3
        self._req_2 = req_2
        self._content_repo = content_repo

        # For covering the case when a path file is sent instead of a directory
        self._pack_abs_dir = pack_dir if pack_dir.is_dir() else pack_dir.parent

        self._pack_name = None
        self.docker_timeout = docker_timeout
        # Docker client init
        if docker_engine:
            self._docker_client: docker.DockerClient = init_global_docker_client(timeout=docker_timeout, log_prompt='Linter')
            self._docker_hub_login = self._docker_login()
        # Facts gathered regarding pack lint and test
        self._facts: Dict[str, Any] = {
            "images": [],
            "python_version": 0,
            "env_vars": {},
            "test": False,
            "lint_files": [],
            "support_level": None,
            "is_long_running": False,
            "lint_unittest_files": [],
            "additional_requirements": [],
            "docker_engine": docker_engine,
            "is_script": False,
            "commands": None,
        }
        # Pack lint status object - visualize it
        self._pkg_lint_status: Dict = {
            "pkg": None,
            "pack_type": None,
            "path": str(self._content_repo),
            "errors": [],
            "images": [],
            "flake8_errors": None,
            "XSOAR_linter_errors": None,
            "bandit_errors": None,
            "mypy_errors": None,
            "vulture_errors": None,
            "flake8_warnings": None,
            "XSOAR_linter_warnings": None,
            "bandit_warnings": None,
            "mypy_warnings": None,
            "vulture_warnings": None,
            "exit_code": SUCCESS,
            "warning_code": SUCCESS,
        }
        self._pack_name = None
        yml_file = self._pack_abs_dir.glob([r'*.yaml', r'*.yml', r'!*unified*.yml'], flags=NEGATE)
        if not yml_file:
            logger.info(f"{self._pack_abs_dir} - Skipping no yaml file found {yml_file}")
            self._pkg_lint_status["errors"].append('Unable to find yml file in package')
        else:
            try:
                self._yml_file = next(yml_file)  # type: ignore
                self._pack_name = self._yml_file.stem
            except StopIteration:
                logger.info(f"{self._pack_abs_dir} - Skipping no yaml file found {yml_file}")
                self._pkg_lint_status["errors"].append('Unable to find yml file in package')

    @timer(group_name='lint')
    def run_pack(self, no_flake8: bool, no_bandit: bool, no_mypy: bool, no_pylint: bool, no_vulture: bool,
                 no_xsoar_linter: bool, no_pwsh_analyze: bool, no_pwsh_test: bool, no_test: bool, modules: dict,
                 keep_container: bool, test_xml: str, no_coverage: bool) -> dict:
        """ Run lint and tests on single package
        Performing the follow:
            1. Run the lint on OS - flake8, bandit, mypy.
            2. Run in package docker - pylint, pytest.

        Args:
            no_xsoar_linter(bool): Whether to skip xsoar-linter
            no_flake8(bool): Whether to skip flake8
            no_bandit(bool): Whether to skip bandit
            no_mypy(bool): Whether to skip mypy
            no_vulture(bool): Whether to skip vulture
            no_pylint(bool): Whether to skip pylint
            no_test(bool): Whether to skip pytest
            no_pwsh_analyze(bool): Whether to skip powershell code analyzing
            no_pwsh_test(bool): whether to skip powershell tests
            modules(dict): Mandatory modules to locate in pack path (CommonServerPython.py etc)
            keep_container(bool): Whether to keep the test container
            test_xml(str): Path for saving pytest xml results
            no_coverage(bool): Run pytest without coverage report

        Returns:
            dict: lint and test all status, pkg status)
        """
        # Gather information for lint check information
        log_prompt = f'{self._pack_name} - Run'
        logger.info(f'{log_prompt} - Start')
        try:
            skip = self._gather_facts(modules)
        # If not python pack - skip pack
            if skip:
                return self._pkg_lint_status
            # Locate mandatory files in pack path - for more info checkout the context manager LintFiles
            with add_tmp_lint_files(content_repo=self._content_repo,
                                    pack_path=self._pack_abs_dir,
                                    lint_files=self._facts["lint_files"],
                                    modules=modules,
                                    pack_type=self._pkg_lint_status["pack_type"]):
                # Run lint check on host - flake8, bandit, mypy
                if self._pkg_lint_status["pack_type"] == TYPE_PYTHON:
                    self._run_lint_in_host(no_bandit=no_bandit,
                                           no_mypy=no_mypy,
                                           no_xsoar_linter=no_xsoar_linter,)

                # Run lint and test check on pack docker image
                if self._facts["docker_engine"]:
                    self._run_lint_on_docker_image(no_pylint=no_pylint,
                                                   no_test=no_test,
                                                   no_pwsh_analyze=no_pwsh_analyze,
                                                   no_pwsh_test=no_pwsh_test,
                                                   keep_container=keep_container,
                                                   test_xml=test_xml,
                                                   no_coverage=no_coverage,
                                                   no_vulture=no_vulture,
                                                   no_flake8=no_flake8,
                                                   )
        except Exception as ex:
            err = f'{self._pack_abs_dir}: Unexpected fatal exception: {str(ex)}'
            logger.error(f"{err}. Traceback: {traceback.format_exc()}")
            self._pkg_lint_status["errors"].append(err)
            self._pkg_lint_status['exit_code'] += FAIL
        logger.info(f'{log_prompt} - Finished Successfully')
        return self._pkg_lint_status

    @timer(group_name='lint')
    def _gather_facts(self, modules: dict) -> bool:
        """ Gathering facts about the package - python version, docker images, valid docker image, yml parsing
        Args:
            modules(dict): Test mandatory modules to be ignore in lint check

        Returns:
            bool: Indicating if to continue further or not, if False exit Thread, Else continue.
        """
        # Looking for pkg yaml
        log_prompt = f"{self._pack_name} - Facts"
        self._pkg_lint_status["pkg"] = self._pack_name
        logger.info(f"{log_prompt} - Using yaml file {self._yml_file}")
        # Parsing pack yaml - in order to verify if check needed
        try:
            script_obj: Dict = {}
            yml_obj: Dict = YAML_Handler().load(self._yml_file)
            if isinstance(yml_obj, dict):
                script_obj = yml_obj.get('script', {}) if isinstance(yml_obj.get('script'), dict) else yml_obj
            self._facts['is_script'] = True if 'Scripts' in self._yml_file.parts else False
            self._facts['is_long_running'] = script_obj.get('longRunning')
            self._facts['commands'] = self._get_commands_list(script_obj)
            self._pkg_lint_status["pack_type"] = script_obj.get('type')
        except (FileNotFoundError, IOError, KeyError):
            self._pkg_lint_status["errors"].append('Unable to parse package yml')
            return True
        # return no check needed if not python pack
        if self._pkg_lint_status["pack_type"] not in (TYPE_PYTHON, TYPE_PWSH):
            logger.info(f"{log_prompt} - Skipping due to not Python, Powershell package - Pack is"
                        f" {self._pkg_lint_status['pack_type']}")
            return True

        # Docker images
        if self._facts["docker_engine"]:
            logger.info(f'{log_prompt} - Collecting all docker images to pull')
            self._facts["images"] = [[image, -1] for image in get_all_docker_images(script_obj=script_obj)]
            if os.getenv('GITLAB_CI', False):
                self._facts["images"] = [[f'docker-io.art.code.pan.run/{image[0]}', -1] for image in self._facts["images"]]
            # Gather environment variables for docker execution
            self._facts["env_vars"] = {
                "CI": os.getenv("CI", False),
                "DEMISTO_LINT_UPDATE_CERTS": os.getenv('DEMISTO_LINT_UPDATE_CERTS', "yes")
            }

        lint_files = set()
        # Facts for python pack
        if self._pkg_lint_status["pack_type"] == TYPE_PYTHON:
            self._update_support_level()
            if self._facts["docker_engine"]:
                # Getting python version from docker image - verifying if not valid docker image configured
                for image in self._facts["images"]:
                    py_num: str = get_python_version_from_image(image=image[0], timeout=self.docker_timeout)
                    image[1] = py_num
                    logger.info(f"{self._pack_name} - Facts - {image[0]} - Python {py_num}")
                    if not self._facts["python_version"]:
                        self._facts["python_version"] = py_num

                # Checking whatever *test* exists in package
                self._facts["test"] = True if next(
                    self._pack_abs_dir.glob([r'test_*.py', r'*_test.py']), None  # type: ignore
                ) else False
                if self._facts["test"]:
                    logger.info(f"{log_prompt} - Tests found")
                else:
                    logger.info(f"{log_prompt} - Tests not found")
                # Gather package requirements embedded in test-requirements.py file
                test_requirements = self._pack_abs_dir / 'test-requirements.txt'
                if test_requirements.exists():
                    try:
                        additional_req = test_requirements.read_text(encoding='utf-8').strip().split('\n')
                        self._facts["additional_requirements"].extend(additional_req)
                        logger.info(f"{log_prompt} - Additional package Pypi packages found - {additional_req}")
                    except (FileNotFoundError, IOError):
                        self._pkg_lint_status["errors"].append('Unable to parse test-requirements.txt in package')
            elif not self._facts["python_version"]:
                # get python version from yml
                pynum = '3.7' if (script_obj.get('subtype', 'python3') == 'python3') else '2.7'
                self._facts["python_version"] = pynum
                logger.info(f"{log_prompt} - Using python version from yml: {pynum}")

            # Get lint files
            lint_files = set(self._pack_abs_dir.glob(["*.py", "!__init__.py", "!*.tmp"],
                                                     flags=NEGATE))
        # Facts for Powershell pack
        elif self._pkg_lint_status["pack_type"] == TYPE_PWSH:
            # Get lint files
            lint_files = set(
                self._pack_abs_dir.glob(["*.ps1", "!*Tests.ps1", "CommonServerPowerShell.ps1", "demistomock.ps1'"],
                                        flags=NEGATE))

        # Add CommonServer to the lint checks
        if 'commonserver' in self._pack_abs_dir.name.lower():
            # Powershell
            if self._pkg_lint_status["pack_type"] == TYPE_PWSH:
                self._facts["lint_files"] = [Path(self._pack_abs_dir / 'CommonServerPowerShell.ps1')]  # type: ignore
            # Python
            elif self._pkg_lint_status["pack_type"] == TYPE_PYTHON:
                self._facts["lint_files"] = [Path(self._pack_abs_dir / 'CommonServerPython.py')]  # type: ignore
        else:
            test_modules = {self._pack_abs_dir / module.name for module in modules.keys()}
            lint_files = lint_files.difference(test_modules)
            self._facts["lint_files"] = list(lint_files)

        if self._facts["lint_files"]:
            self._remove_gitignore_files(log_prompt)
            for lint_file in self._facts["lint_files"]:
                logger.info(f"{log_prompt} - Lint file {lint_file}")
        else:
            logger.info(f"{log_prompt} - Lint files not found")

        # Remove files that are in gitignore

        self._split_lint_files()

        self._linter_to_commands()

        return False

    def _linter_to_commands(self):
        self._facts['lint_to_commands'] = {
            'pylint': build_pylint_command(self._facts["lint_files"], docker_version=self._facts.get('python_version')),
            'flake8': build_flake8_command(self._facts['lint_files'] + self._facts['lint_unittest_files']),
            'vulture': build_vulture_command(self._facts['lint_files'], self._pack_abs_dir)
        }

    def _remove_gitignore_files(self, log_prompt: str) -> None:
        """
        Skipping files that matches gitignore patterns.
        Args:
            log_prompt(str): log prompt string

        Returns:

        """
        try:
            repo = git.Repo(self._content_repo)
            files_to_ignore = repo.ignored(self._facts['lint_files'])
            for file in files_to_ignore:
                logger.info(f"{log_prompt} - Skipping gitignore file {file}")
            self._facts["lint_files"] = [path for path in self._facts['lint_files'] if path not in files_to_ignore]

        except (git.InvalidGitRepositoryError, git.NoSuchPathError):
            logger.debug("No gitignore files is available")

    def _split_lint_files(self):
        """ Remove unit test files from _facts['lint_files'] and put into their own list _facts['lint_unittest_files']
        This is because not all lints should be done on unittest files.
        """
        lint_files_list = copy.deepcopy(self._facts["lint_files"])
        for lint_file in lint_files_list:
            if lint_file.name.startswith('test_') or lint_file.name.endswith('_test.py'):
                self._facts['lint_unittest_files'].append(lint_file)
                self._facts["lint_files"].remove(lint_file)

    @timer(group_name='lint')
    def _run_lint_in_host(self, no_bandit: bool, no_mypy: bool,
                          no_xsoar_linter: bool):
        """ Run lint check on host

        Args:
            no_flake8(bool): Whether to skip flake8.
            no_bandit(bool): Whether to skip bandit.
            no_mypy(bool): Whether to skip mypy.
            no_vulture(bool): Whether to skip Vulture.
        """
        log_prompt = f'{self._pack_name} - Run Lint In Host'
        logger.info(f'{log_prompt} - Started')
        exit_code: int = SUCCESS
        for lint_check in ["XSOAR_linter", "bandit", "mypy"]:
            exit_code = SUCCESS
            output = ""
            if self._facts["lint_files"]:
                if lint_check == "XSOAR_linter" and not no_xsoar_linter:
                    exit_code, output = self._run_xsoar_linter(py_num=self._facts["python_version"],
                                                               lint_files=self._facts["lint_files"])
                elif lint_check == "bandit" and not no_bandit:
                    exit_code, output = self._run_bandit(lint_files=self._facts["lint_files"])

                elif lint_check == "mypy" and not no_mypy and parse(
                    self._facts['python_version']
                ).major >= 3:  # type: ignore
                    # mypy does not support python2 now
                    exit_code, output = self._run_mypy(py_num=self._facts["python_version"],
                                                       lint_files=self._facts["lint_files"])

            self._handle_lint_results(exit_code, lint_check, output)
        logger.info(f'{log_prompt} - Finished successfully')

    def _handle_lint_results(self, exit_code, lint_check, output):
        # check for any exit code other than 0
        if exit_code:
            error, warning, other = split_warnings_errors(output)
        if exit_code and warning:
            self._pkg_lint_status["warning_code"] |= EXIT_CODES[lint_check]
            self._pkg_lint_status[f"{lint_check}_warnings"] = "\n".join(warning)
        if exit_code & FAIL:
            self._pkg_lint_status["exit_code"] |= EXIT_CODES[lint_check]
            # if the error were extracted correctly as they start with E
            if error:
                self._pkg_lint_status[f"{lint_check}_errors"] = "\n".join(error)
                # if there were errors but they do not start with E
            else:
                self._pkg_lint_status[f"{lint_check}_errors"] = "\n".join(other)

    @timer(group_name='lint')
    def _run_xsoar_linter(self, py_num: str, lint_files: List[Path]) -> Tuple[int, str]:
        """ Runs Xsaor linter in pack dir

        Args:
            lint_files(List[Path]): file to perform lint

        Returns:
           int:  0 on successful else 1, errors
           str: Xsoar linter errors
        """
        status = SUCCESS
        FAIL_PYLINT = 0b10
        with pylint_plugin(self._pack_abs_dir):
            log_prompt = f"{self._pack_name} - XSOAR Linter"
            logger.info(f"{log_prompt} - Start")
            myenv = os.environ.copy()
            if myenv.get('PYTHONPATH'):
                myenv['PYTHONPATH'] += ':' + str(self._pack_abs_dir)
            else:
                myenv['PYTHONPATH'] = str(self._pack_abs_dir)
            if self._facts['is_long_running']:
                myenv['LONGRUNNING'] = 'True'

            py_ver = parse(py_num).major  # type: ignore
            if py_ver < 3:
                myenv['PY2'] = 'True'
            myenv['is_script'] = str(self._facts['is_script'])
            # as Xsoar checker is a pylint plugin and runs as part of pylint code, we can not pass args to it.
            # as a result we can use the env vars as a getway.
            myenv['commands'] = ','.join([str(elem) for elem in self._facts['commands']]) \
                if self._facts['commands'] else ''
            stdout, stderr, exit_code = run_command_os(
                command=build_xsoar_linter_command(
                    lint_files, self._facts.get('support_level', 'base')  # type: ignore
                ),
                cwd=self._pack_abs_dir, env=myenv)
        if exit_code & FAIL_PYLINT:
            logger.error(f"{log_prompt} - Finished, errors found")
            status = FAIL
        if exit_code & WARNING:
            logger.warning(f"{log_prompt} - Finished, warnings found")
            if not status:
                status = WARNING
        # if pylint did not run and failure exit code has been returned from run commnad
        elif exit_code & FAIL:
            status = FAIL
            logger.debug(f"{log_prompt} - Actual XSOAR linter error -")
            logger.debug(f"{log_prompt} - Full format stdout: {RL if stdout else ''}{stdout}")
            # for contrib prs which are not merged from master and do not have pylint in dev-requirements-py2.
            if os.environ.get('CI'):
                stdout = "Xsoar linter could not run, Please merge from master"
            else:
                stdout = "Xsoar linter could not run, please make sure you have" \
                         " the necessary Pylint version for both py2 and py3"
            logger.error(f"{log_prompt} - Finished, errors found")

        logger.debug(f"{log_prompt} - Finished, exit-code: {exit_code}")
        logger.debug(f"{log_prompt} - Finished, stdout: {RL if stdout else ''}{stdout}")
        logger.debug(f"{log_prompt} - Finished, stderr: {RL if stderr else ''}{stderr}")

        if not exit_code:
            logger.info(f"{log_prompt} - Successfully finished")

        return status, stdout

    @timer(group_name='lint')
    def _run_bandit(self, lint_files: List[Path]) -> Tuple[int, str]:
        """ Run bandit in pack dir

        Args:
            lint_files(List[Path]): file to perform lint

        Returns:
           int:  0 on successful else 1, errors
           str: Bandit errors
        """
        log_prompt = f"{self._pack_name} - Bandit"
        logger.info(f"{log_prompt} - Start")
        stdout, stderr, exit_code = run_command_os(command=build_bandit_command(lint_files),  # type: ignore
                                                   cwd=self._pack_abs_dir)
        logger.debug(f"{log_prompt} - Finished, exit-code: {exit_code}")
        logger.debug(f"{log_prompt} - Finished, stdout: {RL if stdout else ''}{stdout}")
        logger.debug(f"{log_prompt} - Finished, stderr: {RL if stderr else ''}{stderr}")
        if stderr or exit_code:
            logger.error(f"{log_prompt} - Finished, errors found")
            if stderr:
                return FAIL, stderr
            else:
                return FAIL, stdout

        logger.info(f"{log_prompt} - Successfully finished")

        return SUCCESS, ""

    @timer(group_name='lint')
    def _run_mypy(self, py_num: str, lint_files: List[Path]) -> Tuple[int, str]:
        """ Run mypy in pack dir

        Args:
            py_num(str): The python version in use
            lint_files(List[Path]): file to perform lint

        Returns:
           int:  0 on successful else 1, errors
           str: Bandit errors
        """
        log_prompt = f"{self._pack_name} - Mypy"
        logger.info(f"{log_prompt} - Start")
        with add_typing_module(lint_files=lint_files, python_version=py_num):  # type: ignore
            mypy_command = build_mypy_command(
                files=lint_files, version=py_num, content_repo=self._content_repo  # type: ignore
            )
            stdout, stderr, exit_code = run_command_os(command=mypy_command, cwd=self._pack_abs_dir)
        logger.debug(f"{log_prompt} - Finished, exit-code: {exit_code}")
        logger.debug(f"{log_prompt} - Finished, stdout: {RL if stdout else ''}{stdout}")
        logger.debug(f"{log_prompt} - Finished, stderr: {RL if stderr else ''}{stderr}")
        if stderr or exit_code:
            logger.error(f"{log_prompt} - Finished, errors found")
            if stderr:
                return FAIL, stderr
            else:
                return FAIL, stdout

        logger.info(f"{log_prompt} - Successfully finished")

        return SUCCESS, ""

    @timer(group_name='lint')
    def _run_lint_on_docker_image(self, no_pylint: bool, no_test: bool, no_pwsh_analyze: bool, no_pwsh_test: bool,
                                  keep_container: bool, test_xml: str, no_coverage: bool, no_flake8: bool, no_vulture: bool):
        """ Run lint check on docker image

        Args:
            no_pylint(bool): Whether to skip pylint
            no_test(bool): Whether to skip pytest
            no_pwsh_analyze(bool): Whether to skip powershell code analyzing
            no_pwsh_test(bool): whether to skip powershell tests
            keep_container(bool): Whether to keep the test container
            test_xml(str): Path for saving pytest xml results
            no_coverage(bool): Run pytest without coverage report

        """
        log_promopt = f'{self._pack_name} - Run Lint On Docker Image'
        logger.info(f'{log_promopt} - Running lint: Number of images={self._facts["images"]}')
        for image in self._facts["images"]:
            logger.info(f'{log_promopt} - Running lint on docker image {image[0]}')
            # Docker image status - visualize
            status = {
                "image": image[0],
                "image_errors": "",
                "pylint_errors": "",
                "pytest_errors": "",
                "pytest_json": {},
                "pwsh_analyze_errors": "",
                "pwsh_test_errors": ""
            }
            # Creating image if pylint specified or found tests and tests specified
            image_id = ""
            errors = ""
            for trial in range(2):
                image_id, errors = self._docker_image_create(docker_base_image=image)
                if not errors:
                    break

            if image_id and not errors:
                # Set image creation status
                for check in ["pylint", "pytest", "pwsh_analyze", "pwsh_test", "flake8", "vulture"]:
                    exit_code = SUCCESS
                    output = ""
                    for trial in range(2):
                        if self._pkg_lint_status["pack_type"] == TYPE_PYTHON:
                            if not no_flake8 and check == 'flake8' and (self._facts['lint_files'] or self._facts['lint_unittest_files']):
                                exit_code, output = self._docker_run_linter(linter=check,
                                                                            test_image=image_id,
                                                                            keep_container=keep_container)
                            if not no_vulture and check == 'vulture' and self._facts['lint_files']:
                                exit_code, output = self._docker_run_linter(linter=check,
                                                                            test_image=image_id,
                                                                            keep_container=keep_container)
                            # Perform pylint
                            if not no_pylint and check == "pylint" and self._facts["lint_files"]:
                                exit_code, output = self._docker_run_linter(linter=check,
                                                                            test_image=image_id,
                                                                            keep_container=keep_container)
                            # Perform pytest
                            elif not no_test and self._facts["test"] and check == "pytest":
                                exit_code, output, test_json = self._docker_run_pytest(test_image=image_id,
                                                                                       keep_container=keep_container,
                                                                                       test_xml=test_xml,
                                                                                       no_coverage=no_coverage)
                                status["pytest_json"] = test_json
                        elif self._pkg_lint_status["pack_type"] == TYPE_PWSH:
                            # Perform powershell analyze
                            if not no_pwsh_analyze and check == "pwsh_analyze" and self._facts["lint_files"]:
                                exit_code, output = self._docker_run_pwsh_analyze(test_image=image_id,
                                                                                  keep_container=keep_container)
                            # Perform powershell test
                            elif not no_pwsh_test and check == "pwsh_test":
                                exit_code, output = self._docker_run_pwsh_test(test_image=image_id,
                                                                               keep_container=keep_container)
                        # If lint check perfrom and failed on reason related to enviorment will run twice,
                        # But it failing in second time it will count as test failure.
                        if (exit_code == RERUN and trial == 1) or exit_code == FAIL or exit_code == SUCCESS:
                            if exit_code in [RERUN, FAIL]:
                                if check in {'flake8', 'vulture'}:
                                    self._handle_lint_results(exit_code, check, output)
                                else:
                                    self._pkg_lint_status["exit_code"] |= EXIT_CODES[check]
                                    status[f"{check}_errors"] = output
                            break
            else:
                status["image_errors"] = str(errors)
                self._pkg_lint_status["exit_code"] += EXIT_CODES["image"]
            # Add image status to images
            self._pkg_lint_status["images"].append(status)
            logger.info(f'{log_promopt} - Finished linting on docker image {image[0]}')

        logger.info(f'{log_promopt} - Finished linting. Number of images={self._facts["images"]}')

    def _docker_login(self) -> bool:
        """ Login to docker-hub using environment variables:
                1. DOCKERHUB_USER - User for docker hub.
                2. DOCKERHUB_PASSWORD - Password for docker-hub.
            Used in Circle-CI for pushing into repo devtestdemisto

        Returns:
            bool: True if logged in successfully.
        """
        docker_user = os.getenv('DOCKERHUB_USER')
        docker_pass = os.getenv('DOCKERHUB_PASSWORD')
        if docker_user and docker_pass:
            try:
                self._docker_client.login(username=docker_user,
                                          password=docker_pass,
                                          registry="https://index.docker.io/v1")
                return self._docker_client.ping()
            except docker.errors.APIError:
                return False
        return False

    @timer(group_name='lint')
    def _docker_image_create(self, docker_base_image: List[Any]) -> Tuple[str, str]:
        """ Create docker image:
            1. Installing 'build base' if required in alpine images version - https://wiki.alpinelinux.org/wiki/GCC
            2. Installing pypi packs - if only pylint required - only pylint installed otherwise all pytest and pylint
               installed, packages which being install can be found in path demisto_sdk/commands/lint/dev_envs
            3. The docker image build done by Dockerfile template located in
                demisto_sdk/commands/lint/templates/dockerfile.jinja2

        Args:
            docker_base_image(list): docker image to use as base for installing dev deps and python version.

        Returns:
            str, str. image name to use and errors string.
        """
        log_prompt = f"{self._pack_name} - Image create"
        # Get requirements file for image
        requirements = []

        if docker_base_image[1] != -1:
            py_ver = parse(docker_base_image[1]).major  # type: ignore
            if py_ver == 2:
                requirements = self._req_2
            elif py_ver == 3:
                requirements = self._req_3
        # Using DockerFile template
        pip_requirements = requirements + self._facts["additional_requirements"]
        # Trying to pull image based on dockerfile hash, will check if something changed
        errors = ""
        identifier = hashlib.md5("\n".join(sorted(pip_requirements)).encode("utf-8")).hexdigest()
        test_image_name = f'{docker_base_image[0].replace("demisto", "devtestdemisto")}-{identifier}'
        test_image = None
        try:
            logger.info(f"{log_prompt} - Trying to pull existing image {test_image_name}")
            test_image = get_docker().pull_image(test_image_name)
        except (docker.errors.APIError, docker.errors.ImageNotFound):
            logger.info(f"{log_prompt} - Unable to find image {test_image_name}")
        # Creatng new image if existing image isn't found
        if not test_image:
            logger.info(
                f"{log_prompt} - Creating image based on {docker_base_image[0]} - Could take 2-3 minutes at first "
                f"time")
            try:
                get_docker().create_image(docker_base_image[0], test_image_name, container_type=self._pkg_lint_status["pack_type"],
                                          install_packages=pip_requirements)

                if self._docker_hub_login:
                    for _ in range(2):
                        try:
                            test_image_name_to_push = test_image_name.replace('docker-io.art.code.pan.run/', '')
                            self._docker_client.images.push(test_image_name_to_push)
                            logger.info(f"{log_prompt} - Image {test_image_name_to_push} pushed to repository")
                            break
                        except (requests.exceptions.ConnectionError, urllib3.exceptions.ReadTimeoutError,
                                requests.exceptions.ReadTimeout):
                            logger.info(f"{log_prompt} - Unable to push image {test_image_name} to repository")

            except (docker.errors.BuildError, docker.errors.APIError, Exception) as e:
                logger.critical(f"{log_prompt} - Build errors occurred {e}")
                errors = str(e)
        return test_image_name, errors

    def _docker_remove_container(self, container_name: str):
        try:
            container = self._docker_client.containers.get(container_name)
            container.remove(force=True)
        except docker.errors.NotFound:
            pass
        except requests.exceptions.ChunkedEncodingError as err:
            # see: https://github.com/docker/docker-py/issues/2696#issuecomment-721322548
            if platform.system() != 'Darwin' or 'Connection broken' not in str(err):
                raise

    def _docker_run_linter(self, linter: str, test_image: str, keep_container: bool) -> Tuple[int, str]:
        log_prompt = f'{self._pack_name} - {linter} - Image {test_image}'
        logger.info(f"{log_prompt} - Start")
        container_name = f"{self._pack_name}-{linter}"
        # Check if previous run left container a live if it do, we remove it
        self._docker_remove_container(container_name)

        # Run container
        exit_code = SUCCESS
        output = ""
        try:
            container: docker.models.containers.Container = get_docker().create_container(
                name=container_name,
                image=test_image,
                command=[self._facts['lint_to_commands'][linter]],
                user=f"{os.getuid()}:4000",
                files_to_push=[(self._pack_abs_dir, '/devwork')],
                environment=self._facts["env_vars"],
            )
            container.start()
            stream_docker_container_output(container.logs(stream=True))
            # wait for container to finish
            container_status = container.wait(condition="exited")
            # Get container exit code
            container_exit_code = container_status.get("StatusCode")
            # Getting container logs
            container_log = container.logs().decode("utf-8")
            logger.info(f"{log_prompt} - exit-code: {container_exit_code}")
            if container_exit_code in [1, 2, 127]:
                # 1-fatal message issued
                # 2-Error message issued
                # 127-Command failure (for instance the linter is not exists)
                exit_code = FAIL
                output = container_log
                logger.error(f"{log_prompt} - Finished, errors found")
            elif container_exit_code in [4, 8, 16]:
                # 4-Warning message issued
                # 8-refactor message issued
                # 16-convention message issued
                logger.info(f"{log_prompt} - Successfully finished - warnings found")
                exit_code = SUCCESS
            elif container_exit_code == 32:
                # 32-usage error
                logger.critical(f"{log_prompt} - Finished - Usage error")
                exit_code = RERUN
            else:
                logger.info(f"{log_prompt} - Successfully finished")
            # Keeping container if needed or remove it
            if keep_container:
                print(f"{log_prompt} - container name {container_name}")
                container.commit(repository=container_name.lower(), tag=linter)
            else:
                try:
                    container.remove(force=True)
                except docker.errors.NotFound as e:
                    logger.critical(f"{log_prompt} - Unable to delete container - {e}")
        except Exception as e:
            logger.exception(f"{log_prompt} - Unable to run {linter}")
            exit_code = RERUN
            output = str(e)
        return exit_code, output

    @timer(group_name='lint')
    def _docker_run_pytest(self, test_image: str, keep_container: bool, test_xml: str, no_coverage: bool = False) -> Tuple[int, str, dict]:
        """ Run Pytest in created test image

        Args:
            test_image(str): Test image id/name
            keep_container(bool): True if to keep container after execution finished
            test_xml(str): Xml saving path
            no_coverage(bool): Run pytest without coverage report
        Returns:
            int: 0 on successful, errors 1, need to retry 2
            str: Unit test json report
        """
        log_prompt = f'{self._pack_name} - Pytest - Image {test_image}'
        logger.info(f"{log_prompt} - Start")
        container_name = f"{self._pack_name}-pytest"
        # Check if previous run left container a live if it does, Remove it
        self._docker_remove_container(container_name)
        # Collect tests
        exit_code = SUCCESS
        output = ''
        test_json = {}
        try:
            # Running pytest container
            cov_file_path = Path.joinpath(self._pack_abs_dir, '.coverage')
            cov = self._pack_abs_dir.stem if not no_coverage else ''
            uid = os.getuid() or 4000
            logger.debug(f'{log_prompt} - user uid for running lint/test: {uid}')  # lgtm[py/clear-text-logging-sensitive-data]
            container: docker.models.containers.Container = get_docker().create_container(
                name=container_name,
                image=test_image,
                command=[build_pytest_command(test_xml=test_xml, json=True, cov=cov)],
                user=f"{uid}:4000",
                files_to_push=[(self._pack_abs_dir, '/devwork')],
                environment=self._facts["env_vars"],
            )
            container.start()
            stream_docker_container_output(container.logs(stream=True))
            # Waiting for container to be finished
            container_status: dict = container.wait(condition="exited")
            # Getting container exit code
            container_exit_code = container_status.get("StatusCode")
            # Getting container logs
            logger.info(f"{log_prompt} - exit-code: {container_exit_code}")
            if container_exit_code in [0, 1, 2, 5]:
                # 0-All tests passed
                # 1-Tests were collected and run but some of the tests failed
                # 2-Test execution was interrupted by the user
                # 5-No tests were collected

                if test_xml:
                    test_data_xml = get_file_from_container(container_obj=container,
                                                            container_path="/devwork/report_pytest.xml")
                    xml_apth = Path(test_xml) / f'{self._pack_name}_pytest.xml'

                    with open(file=xml_apth, mode='bw') as f:
                        f.write(test_data_xml)  # type: ignore

                if cov:
                    cov_data = get_file_from_container(container_obj=container,
                                                       container_path="/devwork/.coverage")
                    cov_data = cov_data if isinstance(cov_data, bytes) else cov_data.encode()
                    with open(cov_file_path, 'wb') as coverage_file:
                        coverage_file.write(cov_data)
                    coverage_report_editor(cov_file_path, os.path.join(self._pack_abs_dir, f'{self._pack_abs_dir.stem}.py'))

                test_json = json.loads(get_file_from_container(container_obj=container,
                                                               container_path="/devwork/report_pytest.json",
                                                               encoding="utf-8"))
                for test in test_json.get('report', {}).get("tests"):
                    if test.get("call", {}).get("longrepr"):
                        test["call"]["longrepr"] = test["call"]["longrepr"].split('\n')
                if container_exit_code in [0, 5]:
                    logger.info(f"{log_prompt} - Successfully finished")
                    exit_code = SUCCESS
                elif container_exit_code in [2]:
                    output = container.logs().decode('utf-8')
                    exit_code = FAIL
                else:
                    logger.error(f"{log_prompt} - Finished, errors found")
                    exit_code = FAIL
            elif container_exit_code in [3, 4]:
                # 3-Internal error happened while executing tests
                # 4-pytest command line usage error
                logger.critical(f"{log_prompt} - Usage error")
                exit_code = RERUN
                output = container.logs().decode('utf-8')
            else:
                # Any other container exit code
                logger.error(f"{log_prompt} - Finished, docker container error found ({container_exit_code})")
                exit_code = FAIL
            # Remove container if not needed
            if keep_container:
                print(f"{log_prompt} - Container name {container_name}")
                container.commit(repository=container_name.lower(), tag="pytest")
            else:
                try:
                    container.remove(force=True)
                except docker.errors.NotFound as e:
                    logger.critical(f"{log_prompt} - Unable to remove container {e}")
        except (docker.errors.ImageNotFound, docker.errors.APIError) as e:
            logger.critical(f"{log_prompt} - Unable to run pytest container {e}")
            exit_code = RERUN
        logger.info(f'{self._pack_name} - Pytest finished image {test_image}')
        return exit_code, output, test_json

    def _docker_run_pwsh_analyze(self, test_image: str, keep_container: bool) -> Tuple[int, str]:
        """ Run Powershell code analyze in created test image

        Args:
            test_image(str): test image id/name
            keep_container(bool): True if to keep container after excution finished

        Returns:
            int: 0 on successful, errors 1, need to retry 2
            str: Container log
        """
        log_prompt = f'{self._pack_name} - Powershell analyze - Image {test_image}'
        logger.info(f"{log_prompt} - Start")
        container_name = f"{self._pack_name}-pwsh-analyze"
        # Check if previous run left container a live if it do, we remove it
        container: docker.models.containers.Container
        try:
            container = self._docker_client.containers.get(container_name)
            container.remove(force=True)
        except docker.errors.NotFound:
            pass

        # Run container
        exit_code = SUCCESS
        output = ""
        try:
            uid = os.getuid() or 4000
            logger.debug(f'{log_prompt} - user uid for running lint/test: {uid}')  # lgtm[py/clear-text-logging-sensitive-data]
            container = get_docker().create_container(name=container_name, image=test_image,
                                                      user=f"{uid}:4000", environment=self._facts["env_vars"],
                                                      files_to_push=[(self._pack_abs_dir, '/devwork')],
                                                      command=build_pwsh_analyze_command(
                                                          self._facts["lint_files"][0])
                                                      )
            container.start()
            stream_docker_container_output(container.logs(stream=True))
            # wait for container to finish
            container_status = container.wait(condition="exited")
            # Get container exit code
            container_exit_code = container_status.get("StatusCode")
            # Getting container logs
            container_log = container.logs().decode("utf-8")
            logger.info(f"{log_prompt} - exit-code: {container_exit_code}")
            if container_exit_code:
                # 1-fatal message issued
                # 2-Error message issued
                logger.error(f"{log_prompt} - Finished, errors found")
                output = container_log
                exit_code = FAIL
            else:
                logger.info(f"{log_prompt} - Successfully finished")
            # Keeping container if needed or remove it
            if keep_container:
                print(f"{log_prompt} - container name {container_name}")
                container.commit(repository=container_name.lower(), tag="pwsh_analyze")
            else:
                try:
                    container.remove(force=True)
                except docker.errors.NotFound as e:
                    logger.critical(f"{log_prompt} - Unable to delete container - {e}")
        except (docker.errors.ImageNotFound, docker.errors.APIError, requests.exceptions.ReadTimeout) as e:
            logger.critical(f"{log_prompt} - Unable to run powershell test - {e}")
            exit_code = RERUN

        return exit_code, output

    def _update_support_level(self):
        logger.debug(f'Updating support level for {self._pack_name}')
        pack_dir = self._pack_abs_dir.parent if self._pack_abs_dir.parts[-1] == INTEGRATIONS_DIR else \
            self._pack_abs_dir.parent.parent
        pack_metadata_file = pack_dir / PACKS_PACK_META_FILE_NAME
        logger.debug(f'Before reading content of {pack_metadata_file}')
        with pack_metadata_file.open() as f:
            pack_meta_content: Dict = json.load(f)
        logger.debug(f'After reading content of {pack_metadata_file}')
        self._facts['support_level'] = pack_meta_content.get('support')
        if self._facts['support_level'] == 'partner' and pack_meta_content.get('Certification'):
            self._facts['support_level'] = 'certified partner'

    def _docker_run_pwsh_test(self, test_image: str, keep_container: bool) -> Tuple[int, str]:
        """ Run Powershell tests in created test image

        Args:
            test_image(str): test image id/name
            keep_container(bool): True if to keep container after excution finished

        Returns:
            int: 0 on successful, errors 1, neet to retry 2
            str: Container log
        """
        log_prompt = f'{self._pack_name} - Powershell test - Image {test_image}'
        logger.info(f"{log_prompt} - Start")
        container_name = f"{self._pack_name}-pwsh-test"
        # Check if previous run left container a live if it do, we remove it
        self._docker_remove_container(container_name)

        # Run container
        exit_code = SUCCESS
        output = ""
        try:
            uid = os.getuid() or 4000
            logger.debug(f'{log_prompt} - user uid for running lint/test: {uid}')  # lgtm[py/clear-text-logging-sensitive-data]
            container: docker.models.containers.Container = get_docker().create_container(
                files_to_push=[(self._pack_abs_dir, '/devwork')],
                name=container_name, image=test_image, command=build_pwsh_test_command(),
                user=f"{uid}:4000", environment=self._facts["env_vars"])
            container.start()
            stream_docker_container_output(container.logs(stream=True))
            # wait for container to finish
            container_status = container.wait(condition="exited")
            # Get container exit code
            container_exit_code = container_status.get("StatusCode")
            # Getting container logs
            container_log = container.logs().decode("utf-8")
            logger.info(f"{log_prompt} - exit-code: {container_exit_code}")
            if container_exit_code:
                # 1-fatal message issued
                # 2-Error message issued
                logger.error(f"{log_prompt} - Finished, errors found")
                output = container_log
                exit_code = FAIL
            else:
                logger.info(f"{log_prompt} - Successfully finished")
            # Keeping container if needed or remove it
            if keep_container:
                print(f"{log_prompt} - container name {container_name}")
                container.commit(repository=container_name.lower(), tag='pwsh_test')
            else:
                try:
                    container.remove(force=True)
                except docker.errors.NotFound as e:
                    logger.critical(f"{log_prompt} - Unable to delete container - {e}")
        except (docker.errors.ImageNotFound, docker.errors.APIError, requests.exceptions.ReadTimeout) as e:
            logger.critical(f"{log_prompt} - Unable to run powershell test - {e}")
            exit_code = RERUN

        return exit_code, output

    def _get_commands_list(self, script_obj: dict):
        """ Get all commands from yml file of the pack
           Args:
               script_obj(dict): the script section of the yml file.
           Returns:
               list: list of all commands
        """
        commands_list = []
        try:
            commands_obj = script_obj.get('commands', {})
            for command in commands_obj:
                commands_list.append(command.get('name', ''))
        except Exception:
            logger.debug("Failed getting the commands from the yml file")
        return commands_list


# demisto_sdk/commands/openapi_codegen/openapi_codegen.py

import os
import re
import shutil
import sys
from distutils.util import strtobool
from typing import Any, List, Optional, Union

import autopep8

from demisto_sdk.commands.common.handlers import JSON_Handler, YAML_Handler
from demisto_sdk.commands.common.hook_validations.docker import DockerImageValidator
from demisto_sdk.commands.common.tools import camel_to_snake, print_error
from demisto_sdk.commands.generate_integration.base_code import (BASE_ARGUMENT, BASE_BASIC_AUTH, BASE_CLIENT,
                                                                 BASE_CODE_TEMPLATE, BASE_CREDENTIALS, BASE_DATA,
                                                                 BASE_FUNCTION, BASE_HEADER, BASE_LIST_FUNCTIONS,
                                                                 BASE_PARAMS, BASE_PROPS, BASE_REQUEST_FUNCTION,
                                                                 BASE_TOKEN)
from demisto_sdk.commands.generate_integration.XSOARIntegration import XSOARIntegration

json = JSON_Handler()


yaml = YAML_Handler()

ILLEGAL_DESCRIPTION_CHARS = ['\n', 'br', '*', '\r', '\t', 'para', 'span', '«', '»', '<', '>']
ILLEGAL_CODE_CHARS = ILLEGAL_DESCRIPTION_CHARS + [' ', '.', ',', '(', ')', '`', ':', "'", '"', '[', ']', '-']
ILLEGAL_CODE_NAMES = ['type', 'from', 'id', 'filter', 'list']
NAME_FIX = '_'
OUTPUT_TYPES = {
    'string': 'String',
    'integer': 'Number',
    'object': 'Unknown',
    'array': 'Unknown',
    'boolean': 'Boolean',
}
ARGUMENT_TYPES = {
    'string': 'str',
    'integer': 'int',
    'boolean': 'bool',
    'file': 'str',
    'int': 'int',
    'str': 'str',
}
JSON_TYPE_HEADER = 'application/json'
ALL_TYPE_HEADER = '*/*'
DEFAULT_HOST = 'https://www.example.com/api'
BEARER_AUTH_TYPE = 'apiKey'
BASIC_AUTH_TYPE = 'basic'
MAX_DESCRIPTION_WORDS = 3000


class OpenAPIIntegration:
    def __init__(self, file_path: str, base_name: str, command_prefix: str, context_path: str,
                 unique_keys: Optional[str] = None, root_objects: Optional[str] = None,
                 verbose: bool = False, fix_code: bool = False, configuration: Optional[dict] = None):
        self.json: dict = {}
        self.file_path = file_path
        self.base_name = base_name
        self.command_prefix = command_prefix
        self.context_path = context_path
        self.unique_keys = unique_keys.split(',') if unique_keys is not None else []
        self.root_objects = root_objects.split(',') if root_objects is not None else []
        self.configuration = configuration if configuration else {}
        self.security_definitions = None
        self.host = ''
        self.base_path = ''
        self.name = ''
        self.description = ''
        self.definitions: dict = {}
        self.components: dict = {}
        self.reference: dict = {}
        self.functions: list = []
        self.parameters: list = []
        self.verbose = verbose
        self.fix_code = fix_code

    def load_file(self):
        """
        Loads a swagger file and parses it into a functions list.
        """

        try:
            with open(self.file_path, 'rb') as json_file:
                self.json = json.load(json_file)
        except Exception as e:
            print_error(f'Failed to load the swagger file: {e}')
            sys.exit(1)

        if self.json.get('host', None):
            self.host = self.json.get('host', None)
        elif self.json.get('servers', None):
            self.host = self.json.get('servers', [])[0]['url']
        else:
            self.host = ''
        self.base_path = self.json.get('basePath', '')
        self.name = self.json['info']['title']
        self.description = self.clean_description(self.json.get('info', {}).get('description', ''))
        self.definitions = self.json.get('definitions', {})
        self.components = self.json.get('components', {})
        self.reference = self.definitions or self.components.get('schemas', {}) or {}
        self.security_definitions = self.json.get('securityDefinitions', {})
        self.functions = []

        for path, function in self.json['paths'].items():
            try:
                for method, data in function.items():
                    if not data:
                        continue
                    if isinstance(data, list):
                        data = data[0]
                    self.print_with_verbose(f'Adding command for the path: {path}')
                    self.add_function(path, method, data, function.get(
                        'parameters', []))
            except Exception as e:
                print_error(f'Failed adding the command for the path {path}: {e}')
                raise
        self.handle_duplicates(self.functions)
        self.functions = sorted(self.functions, key=lambda x: x['name'])
        if not self.configuration:
            self.generate_configuration()

    def generate_configuration(self):
        """
        Generates an integration configuration file according to parsed functions from a swagger file.
        """

        security_types: list = []
        if self.security_definitions:
            all_security_types = [s.get('type') for s in self.security_definitions.values()]
            security_types = [t for t in all_security_types if t in [BEARER_AUTH_TYPE, BASIC_AUTH_TYPE]]
        if not security_types:
            security_types = [BEARER_AUTH_TYPE]

        configuration = {
            'name': self.name or 'GeneratedIntegration',
            'description': self.description or 'This integration was auto generated by the Cortex XSOAR SDK.',
            'category': 'Utilities',
            'url': self.host or DEFAULT_HOST,
            'auth': security_types,
            'context_path': self.context_path,
            'commands': []
        }
        for function in self.functions:
            command = {
                'name': function['name'].replace('_', '-'),
                'path': function['path'],
                'method': function['method'],
                'description': function['description'],
                'arguments': [],
                'outputs': [],
                'context_path': function.get('context_path', ''),
                'root_object': function.get('root_object', '')
            }

            headers = []
            if function['consumes'] and JSON_TYPE_HEADER not in function['consumes'] \
                    and ALL_TYPE_HEADER not in function['consumes']:
                headers.append({'Content-Type': function['consumes'][0]})

            if function['produces'] and JSON_TYPE_HEADER not in function['produces'] \
                    and ALL_TYPE_HEADER not in function['produces']:
                headers.append({'Accept': function['produces'][0]})
            command['headers'] = headers

            for arg in function['arguments']:
                arg_name = arg.get('name', '')
                arg_ref = arg.get('ref', '')
                for char in ILLEGAL_CODE_CHARS:
                    arg_name = arg_name.replace(char, '_')
                    arg_ref = arg_ref.replace(char, '_')
                command['arguments'].append({
                    'name': arg_name,
                    'description': arg.get('description', ''),
                    'required': arg.get('required'),
                    'default': arg.get('default', ''),
                    'in': arg.get('in', ''),
                    'ref': arg_ref,
                    'type': arg.get('type', 'string'),
                    'options': arg.get('enums'),
                    'properties': arg.get('properties', {})
                })

            for output in function['outputs']:
                command['outputs'].append({
                    'name': output['name'],
                    'description': output['description'],
                    'type': output['type']
                })

                if output['name'] in self.unique_keys:
                    command['unique_key'] = output['name']

            if 'unique_key' not in command:
                command['unique_key'] = ''

            configuration['commands'].append(command)  # type: ignore

        configuration['code_type'] = 'python'
        configuration['code_subtype'] = 'python3'

        try:
            latest_tag = DockerImageValidator.get_docker_image_latest_tag_request('demisto/python3')
            configuration['docker_image'] = f'demisto/python3:{latest_tag}'
        except Exception as e:
            self.print_with_verbose(f'Failed getting latest docker image for demisto/python3: {e}')

        self.configuration = configuration

    def generate_python_code(self) -> str:
        """
        Generate python code from the base template with the commands from the integration configuration file.

        Returns:
            code: The integration python code.
        """

        # Use the code from base_code in py as the basis
        code: str = BASE_CODE_TEMPLATE

        # Build the functions from configuration file
        functions: list = []
        req_functions: list = []
        for command in self.configuration['commands']:
            function, req_function = self.get_python_command_and_request_functions(command)
            functions.append(function)
            req_functions.append(req_function)

        code = code.replace('$FUNCTIONS$', '\n'.join(functions))
        code = code.replace('$BASEURL$', self.base_path)
        client = BASE_CLIENT.replace('$REQUESTFUNCS$', '\n'.join(req_functions))
        code = code.replace('$CLIENT$', client).replace('$CLIENT_API_KEY$', '')

        if BEARER_AUTH_TYPE in self.configuration['auth']:
            code = code.replace('$BEARERAUTHPARAMS$', BASE_TOKEN)
        else:
            code = '\n'.join(
                [x for x in code.split('\n') if any(ext not in x for ext in ['$BEARERAUTHPARAMS$'])])

        if BASIC_AUTH_TYPE in self.configuration['auth']:
            code = code.replace('$BASEAUTHPARAMS$', BASE_CREDENTIALS)
            code = code.replace('$BASEAUTH$', BASE_BASIC_AUTH)
        else:
            code = '\n'.join(
                [x for x in code.split('\n') if '$BASEAUTHPARAMS$' not in x])
            code = code.replace('$BASEAUTH$', 'None')

        list_functions = []

        # Add the command mappings:
        for command in self.functions:
            prefix = f'{self.command_prefix}-' if self.command_prefix else ''

            if self.command_prefix:
                prefix = f'{self.command_prefix}-'

            function = BASE_LIST_FUNCTIONS.replace(
                '$FUNCTIONNAME$', f"{prefix}{command['name']}".replace('_', '-'))
            fn = command['name'].replace('-', '_')
            function = function.replace('$FUNCTIONCOMMAND$', f'{fn}_command')
            list_functions.append(function)

        code = code.replace('$COMMANDSLIST$', '\n\t'.join(list_functions))
        self.print_with_verbose('Finished generating the Python code.')

        if self.fix_code:
            self.print_with_verbose('Fixing the code with autopep8...')
            code = autopep8.fix_code(code)

        return code

    def get_python_command_and_request_functions(self, command: dict) -> tuple:
        """
        Generates a command function and a request function in python
        according to a given command in the integration configuration file.

        Args:
            command: The integration command from the configuration file.

        Returns:
            function:  The command function in python according to the command.
            req_function: The request function in python according to the command.
        """
        function_name = command['name'].replace('-', '_')
        headers = command['headers']
        self.print_with_verbose(f'Adding the function {function_name} to the code...')
        function = BASE_FUNCTION.replace('$FUNCTIONNAME$', function_name)
        req_function = BASE_REQUEST_FUNCTION.replace('$FUNCTIONNAME$', function_name)
        argument_names, arguments, arguments_found, body_data, params_data = self.process_command_arguments(command)
        if arguments_found:
            function = function.replace('$ARGUMENTS$', '\n    '.join(arguments))
            function = function.replace('$REQARGS$', ', '.join(argument_names))
            req_function = req_function.replace('$REQARGS$', ', $REQARGS$')
            req_function = req_function.replace('$REQARGS$', ', '.join(argument_names))
        else:
            req_function = req_function.replace('$REQARGS$', '')
            function = function.replace('$REQARGS$', '')
            function = '\n'.join(
                [x for x in function.split('\n') if '$ARGUMENTS$' not in x])
        req_function = req_function.replace('$METHOD$', command['method'])
        command['path'] = f"'{command['path']}'" if "'" not in command['path'] else command['path']
        command['path'] = f"f{command['path']}" if "{" in command['path'] else command['path']
        for param in re.findall(r'{([^}]+)}', command['path']):  # get content inside curly brackets
            if param in ILLEGAL_CODE_NAMES:
                command['path'] = command['path'].replace(param, f'{param}{NAME_FIX}')
        req_function = req_function.replace('$PATH$', command['path'])
        if params_data:
            params = self.format_params(params_data, BASE_PARAMS, '$PARAMS$')
            req_function = req_function.replace('$PARAMETERS$', params)
        else:
            req_function = '\n'.join(
                [x for x in req_function.split('\n') if '$PARAMETERS$' not in x])
        if body_data:
            body_data = self.format_params(body_data, BASE_DATA, '$DATAOBJ$')
            req_function = req_function.replace('$DATA$', body_data)
        else:
            req_function = '\n'.join(
                [x for x in req_function.split('\n') if '$DATA$' not in x])
        if params_data:
            req_function = req_function.replace(
                '$NEWPARAMS$', ', params=params')
        else:
            req_function = req_function.replace('$NEWPARAMS$', '')
        if body_data:
            req_function = req_function.replace('$NEWDATA$', ', json_data=data')
        else:
            req_function = req_function.replace('$NEWDATA$', '')
        if headers:
            new_headers = []
            for header in headers:
                for k, v in header.items():
                    new_headers.append(BASE_HEADER.replace('$HEADERKEY$', f"'{k}'")
                                       .replace('$HEADERVALUE$', f"'{v}'"))

            req_function = req_function.replace('$HEADERSOBJ$', ' \n        '.join(new_headers))
        else:
            req_function = '\n'.join([x for x in req_function.split('\n') if '$HEADERSOBJ$' not in x])

        if self.configuration['context_path']:
            context_name = self.context_path
        else:
            context_name = command['name'].title().replace('_', '')
        if command.get('context_path'):
            function = function.replace('$CONTEXTPATH$', f'.{command["context_path"]}')
        else:
            function = function.replace('$CONTEXTPATH$', '')
        if command.get('root_object'):
            function = function.replace('$OUTPUTS$', f"response.get('{command['root_object']}')")
        else:
            function = function.replace('$OUTPUTS$', 'response')
        if command['unique_key']:
            function = function.replace('$UNIQUEKEY$', command['unique_key'])
        else:
            function = function.replace('$UNIQUEKEY$', '')
        function = function.replace('$CONTEXTNAME$', context_name)
        return function, req_function

    def process_command_arguments(self, command: dict) -> tuple:
        """
        Processes the arguments for a command and set them up to paste in the python code.
        Args:
            command: The command from the integration configuration file.

        Returns:
            argument_names: A list of processed argument names.
            arguments: A list of the processed arguments to paste in the python code, e.g.:
            >>> str(args.get('accountId', ''))
            arguments_found: Whether any arguments were found for the command.
            body_data: The arguments to set in the body of the request.
            params_data: The arguments to set in the query of the request.
        """
        params_data = []
        body_data = []
        arguments = []
        argument_names = []
        arguments_found = False
        for arg in command['arguments']:
            arguments_found = True
            code_arg_name = arg['name']
            ref_arg_name = arg['name']
            arg_type = arg.get('type')
            arg_props = []
            if arg.get('ref'):
                ref_arg_name = f'{arg["ref"]}_{ref_arg_name}'.lower()
                code_arg_name = f'{arg["ref"]}_{code_arg_name}'.lower()
            if ref_arg_name in ILLEGAL_CODE_NAMES:
                ref_arg_name = f'{ref_arg_name}{NAME_FIX}'
            if code_arg_name in ILLEGAL_CODE_NAMES:
                code_arg_name = f'{code_arg_name}{NAME_FIX}'
            if arg['properties']:
                for k, v in arg['properties'].items():
                    prop_default = self.get_arg_default(v)
                    prop_arg_name = f'{code_arg_name}_{k}'.lower()
                    prop_arg_type = ARGUMENT_TYPES.get(v.get('type', 'string'), 'str')
                    if prop_arg_type == 'bool':
                        prop_arg_type = 'argToBoolean'
                    if prop_arg_name in ILLEGAL_CODE_NAMES:
                        prop_arg_name = f'{prop_arg_name}{NAME_FIX}'
                    this_prop_argument = f"{BASE_ARGUMENT.replace('$DARGNAME$', prop_arg_name)}{prop_default})"
                    this_prop_argument = this_prop_argument.replace('$SARGNAME$', prop_arg_name)
                    if 'None' not in prop_default:
                        this_prop_argument = this_prop_argument.replace('$ARGTYPE$', f'{prop_arg_type}(') + ')'
                    else:
                        this_prop_argument = this_prop_argument.replace('$ARGTYPE$', '')
                    arg_props.append({k: prop_arg_name})
                    arguments.append(this_prop_argument)
                all_props = self.format_params(arg_props, BASE_PROPS, '$PROPS$')
                this_argument = f'{code_arg_name} = {all_props}'
                if arg_type == 'array':
                    this_argument = f'argToList({this_argument})'
            else:
                if arg_type == 'array':
                    argument_default = ', []'
                    new_arg_type = 'argToList'
                else:
                    argument_default = self.get_arg_default(arg)
                    new_arg_type = ARGUMENT_TYPES.get(arg['type'], 'str')
                    if new_arg_type == 'bool':
                        new_arg_type = 'argToBoolean'

                this_argument = f"{BASE_ARGUMENT.replace('$DARGNAME$', ref_arg_name)}{argument_default})"
                if 'None' not in argument_default:
                    this_argument = this_argument.replace('$ARGTYPE$', f'{new_arg_type}(') + ')'
                else:
                    this_argument = this_argument.replace('$ARGTYPE$', '')

            this_argument = this_argument.replace('$SARGNAME$', code_arg_name)
            argument_names.append(code_arg_name)
            arguments.append(this_argument)
            if 'query' in arg['in']:
                params_data.append({
                    arg['name']: ref_arg_name
                })
            elif arg['in'] in ['formData', 'body']:
                body_data.append({
                    arg['name']: ref_arg_name
                })

        return argument_names, arguments, arguments_found, body_data, params_data

    def generate_yaml(self) -> XSOARIntegration:
        """
        Generates the yaml structure of the integration.

        Returns:
            integration: An object representation of the integration yaml structure.

        """
        # Create the commands section
        commands = self.get_yaml_commands()
        commonfields = XSOARIntegration.CommonFields(self.configuration['name'])
        name = self.configuration['name']
        display = self.configuration['name']
        category = self.configuration['category']
        description = self.configuration['description']
        configurations = self.get_yaml_params()

        int_script = XSOARIntegration.Script('', self.configuration['code_type'],
                                             self.configuration['code_subtype'], self.configuration['docker_image'],
                                             self.configuration.get('fetch_incidents', False), commands)

        integration = XSOARIntegration(commonfields, name, display, category, description, configuration=configurations,
                                       script=int_script)
        return integration

    def get_yaml_params(self) -> list:
        """
        Gets the configuration params for the integration.

        Returns:
            params: A list of integration params.
        """
        url = self.configuration['url']
        params = [XSOARIntegration.Configuration(display=f'Server URL (e.g. {url})',
                                                 name='url',
                                                 defaultvalue=url,
                                                 type_=0,
                                                 required=True)]
        if not isinstance(self.configuration['auth'], list):
            self.configuration['auth'] = [self.configuration['auth']]
        if BEARER_AUTH_TYPE in self.configuration['auth']:
            params.append(XSOARIntegration.Configuration(display='API Key',
                                                         name='api_key',
                                                         required=True,
                                                         type_=4))
        if BASIC_AUTH_TYPE in self.configuration['auth']:
            params.append(XSOARIntegration.Configuration(display='Username',
                                                         name='credentials',
                                                         required=True,
                                                         type_=9))
        if self.configuration.get('fetch_incidents', False):
            params.extend([
                XSOARIntegration.Configuration(display='Fetch incidents',
                                               name='isFetch',
                                               type_=8,
                                               required=False),
                XSOARIntegration.Configuration(display='Incident type',
                                               name='incidentType',
                                               type_=13,
                                               required=False),
                XSOARIntegration.Configuration(display='Maximum number of incidents per fetch',
                                               name='max_fetch',
                                               defaultvalue='10',
                                               type_=0,
                                               required=False),
                XSOARIntegration.Configuration(display='Fetch alerts with status (ACTIVE, CLOSED)',
                                               name='alert_status',
                                               defaultvalue='ACTIVE',
                                               type_=15,
                                               required=False,
                                               options=['ACTIVE', 'CLOSED']),
                XSOARIntegration.Configuration(display='Fetch alerts with type',
                                               name='alert_type',
                                               type_=0,
                                               required=False),
                XSOARIntegration.Configuration(display='Minimum severity of alerts to fetch',
                                               name='min_severity',
                                               defaultvalue='Low',
                                               type_=15,
                                               required=True,
                                               options=['Low', 'Medium', 'High', 'Critical'])])
        params.extend([XSOARIntegration.Configuration(display='Trust any certificate (not secure)',
                                                      name='insecure',
                                                      type_=8,
                                                      required=False),
                       XSOARIntegration.Configuration(display='Use system proxy settings',
                                                      name='proxy',
                                                      type_=8,
                                                      required=False)])
        return params

    def get_yaml_commands(self) -> list:
        """
        Gets the integration commands in yaml format (in object representation) according to the configuration.

        Returns:
            commands: A list of integration commands in yaml format.
        """
        commands = []
        for command in self.configuration['commands']:
            args = []
            for arg in command['arguments']:
                options = None
                auto = None
                arg_name = arg['name']
                if arg.get('ref'):
                    arg_name = f"{arg['ref']}_{arg_name}".lower()
                if arg_name in ILLEGAL_CODE_NAMES:
                    arg_name = f'{arg_name}{NAME_FIX}'
                required = True if arg['required'] else False
                description = arg.get('description', '')
                is_array = True if arg['type'] == 'array' else False
                if arg['properties']:
                    for k, v in arg['properties'].items():
                        prop_arg_name = f'{arg_name}_{k}'
                        if description:
                            prop_description = f'{description} - {k}'
                        else:
                            prop_description = f'{arg_name} {k}'
                        prop_required = True if required is True and v.get('required') else False
                        options = None
                        auto = None
                        if v.get('options'):
                            auto = 'PREDEFINED'
                            options = v['options']
                        args.append(XSOARIntegration.Script.Command.Argument(prop_arg_name, prop_description,
                                                                             prop_required, auto, options))
                else:
                    if arg['options']:
                        auto = 'PREDEFINED'
                        options = arg['options']

                    args.append(XSOARIntegration.Script.Command.Argument(arg_name, description, required,
                                                                         auto, options, is_array))

            outputs = []
            context_path = command['context_path']
            for output in command['outputs']:
                output_name = output['name']
                if context_path:
                    output_name = f'{context_path}.{output_name}'
                if self.context_path:
                    output_name = f'{self.context_path}.{output_name}'
                output_description = output['description']
                output_type = output['type']

                outputs.append(XSOARIntegration.Script.Command.Output(output_type, output_name, output_description))

            prefix = ''
            if self.command_prefix:
                prefix = f'{self.command_prefix}-'
            command_name = f"{prefix}{command['name']}".replace('_', '-')
            command_description = command['description']
            commands.append(XSOARIntegration.Script.Command(command_name, command_description, args, outputs))

        return commands

    def extract_properties(self, obj: Union[dict, list], context: list) -> list:
        """
        Extracts properties recursively from an object in the swagger file.
        Args:
            obj: The swagger file object.
            context: The context of the object in the swagger file.

        Returns:
            results: A list of the object's properties.
        """
        arr: list = []

        def extract(extracted_object: Union[dict, list], prop_arr: list, current_context: list) -> list:

            if isinstance(extracted_object, list):
                for item in extracted_object:
                    extract(item, prop_arr, current_context)

            elif isinstance(extracted_object, dict):
                if extracted_object.get('type') and type(extracted_object.get('type')) == str:
                    if extracted_object.get('type') in ['array', 'object']:
                        refs = self.extract_values(extracted_object, '$ref')
                        if refs:
                            ref = refs[0].split('/')[-1]
                            prop_arr = extract(self.extract_values(self.reference.get(ref, {}), 'properties'), prop_arr,
                                               current_context)
                        elif extracted_object.get('items'):
                            for k, v in extracted_object.get('items', {}).items():
                                prop_arr = extract(v, prop_arr, current_context)
                        elif extracted_object.get('properties'):
                            prop_arr = extract(
                                extracted_object.get('properties', {}), prop_arr, current_context)
                        elif extracted_object.get('allOf'):
                            for item in extracted_object.get('allOf', []):
                                prop_arr = extract(item, prop_arr, current_context)
                    else:
                        prop_arr.append({'name': '.'.join(current_context), 'type': extracted_object.get('type',
                                                                                                         'Unknown'),
                                         'description': extracted_object.get('description', '')})
                elif extracted_object.get('type') and type(extracted_object.get('type')) == dict:
                    for k, v in extracted_object.items():
                        if k not in current_context:
                            current_context.append(k)
                            prop_arr = extract(v, prop_arr, current_context)
                            current_context.pop()
                else:
                    for k, v in extracted_object.items():
                        if k not in current_context:
                            current_context.append(k)
                            prop_arr = extract(v, prop_arr, current_context)
                            current_context.pop()

            return prop_arr

        results = extract(obj, arr, context)
        return results

    def extract_outputs(self, data: dict) -> tuple:
        """
        Extracts outputs from a function(path) in a swagger file.
        Args:
            data: The data of the swagger function.

        Returns:
            outputs: The extracted outputs.
            root_object: The root object of the command output.
            context_path: The context path of the command output.
        """
        outputs: list = []
        root_object: str = ''
        context_path: str = ''
        for response_code, response in data.get('responses', {}).items():
            new_response = {}
            try:
                if int(response_code) != 200:
                    continue
            except Exception:
                self.print_with_verbose(f'Could not get the code for the response {response}')

            new_response['description'] = response.get('description', None)
            all_items = []
            schemas = self.extract_values(response, 'schema')
            if schemas:
                schema = schemas[0]
                refs = self.extract_values(schema, '$ref')
                found_root = False
                if refs:
                    for ref in refs:
                        ref = ref.split('/')[-1]
                        ref_props = self.extract_values(self.reference.get(ref, {}), 'properties')
                        # Addition of filtering dicts only was added because some swaggers contain example files
                        # Which are written in string and caused errors on ref_props[0].items()
                        ref_props = [ref_prop for ref_prop in ref_props if isinstance(ref_prop, dict)]
                        if ref_props:
                            for k, prop in ref_props[0].items():
                                if k in self.root_objects:
                                    # We found a root, we need the output of the root only
                                    root_object = k
                                    all_items = [prop]
                                    path = self.extract_values(prop, '$ref')
                                    if path:
                                        context_path = self.clean_function_name(path[0].split('/')[-1], False)
                                    found_root = True
                            if found_root:
                                break

                            all_items.extend(ref_props)
                        if not context_path:
                            context_path = self.clean_function_name(ref, False)
                else:
                    all_items.extend(self.extract_values(schema, 'properties'))

                properties = self.extract_properties(all_items, [])
                for prop in properties:
                    description = prop.get('description', '')
                    this_type = prop.get('type', 'object')
                    this_type = OUTPUT_TYPES.get(this_type, 'Unknown')
                    resp_name = prop.get('name')
                    description = self.clean_description(description)
                    outputs.append({'name': resp_name, 'type': this_type, 'description': description})

        return outputs, root_object, context_path

    def extract_args(self, arguments: list) -> list:
        """
        Extracts arguments from a function(path) in a swagger file.
        Args:
            arguments: The list of arguments from a function(path) in the swagger file.

        Returns:
            extracted_arguments: The extracted arguments.
        """
        extracted_arguments: list = []
        for arg in arguments:
            refs = []
            if 'schema' in arg:
                refs = self.extract_values(arg['schema'], '$ref')
                for ref in refs:
                    ref = ref.split('/')[-1]
                    ref_args = self.extract_values(self.reference.get(ref, {}), 'properties')
                    # Addition of filtering dicts only was added because some swaggers contain example files
                    # Which are written in string and caused errors on ref_props[0].items()
                    ref_args = [ref_arg for ref_arg in ref_args if isinstance(ref_arg, dict)]
                    for ref_arg in ref_args:
                        for k, v in ref_arg.items():
                            new_ref_arg = {'name': k, 'in': arg.get('in'),
                                           'required': True if k in self.reference.get(ref, {}).get('required', [])
                                           else False}
                            if '$ref' in ref_arg[k]:
                                new_ref_arg['properties'] = {}
                                c_ref = ref_arg[k]['$ref'].split('/')[-1]
                                complex_refs = self.extract_values(self.reference.get(c_ref, {}), 'properties')
                                for complex_ref in complex_refs:
                                    for ck, cv in complex_ref.items():
                                        new_ref_arg['properties'][ck] = {}
                                        new_ref_arg['properties'][ck]['type'] = cv.get('type', 'string')
                                        new_ref_arg['properties'][ck]['description'] = cv.get('description', '')
                                        new_ref_arg['properties'][ck]['required'] = True \
                                            if ck in self.reference.get(c_ref, {}).get('required', []) else False
                                        new_ref_arg['properties'][ck]['options'] = [str(x) for x in cv.get('enum')] \
                                            if arg.get('enum', None) else None
                            else:
                                new_ref_arg.update(v)
                            new_ref_arg['ref'] = ref
                            extracted_arguments.append(self.init_arg(new_ref_arg))
            if not refs:
                if arg.get('schema', {}).get('type'):
                    arg['type'] = arg['schema']['type']
                extracted_arguments.append(self.init_arg(arg))

        return extracted_arguments

    def add_function(self, path: str, method: str, data: dict, params: list):
        """
        Adds a function parsed from the swagger data.
        Args:
            path: The function path.
            method: The function request method.
            data: The function data.
            params: The function parameters.

        """
        new_function: dict = {'path': '/'.join(x.split(' ')[0] for x in path.split('/')).strip('/'), 'method': method}
        name = data.get('operationId', None)
        if not name:
            name = data.get('summary', '').lower()
            name = name.replace(' ', '-')
        if not name:
            name = '_'.join([re.sub(r'{[^)]*\}', '', x) for x in path.split('/')])

        name = self.clean_function_name(name)
        new_function['name'] = name
        func_desc = data.get('summary', None)
        if not func_desc:
            func_desc = data.get('description', '')
        new_function['description'] = self.clean_description(func_desc)
        new_function['arguments'] = []
        new_function['parameters'] = data.get('parameters', None)
        new_function['consumes'] = data.get('consumes', [])
        new_function['produces'] = data.get('produces', [])
        new_function['outputs'] = []
        if not new_function['parameters']:
            new_function['parameters'] = params
            iter_item = params
        else:
            iter_item = data.get('parameters', [])
        new_function['arguments'] = self.extract_args(iter_item)
        outputs, root_object, context_path = self.extract_outputs(data)
        new_function['outputs'] = outputs
        new_function['root_object'] = root_object
        new_function['context_path'] = context_path
        self.functions.append(new_function)

    def save_python_code(self, directory: str) -> str:
        """
        Writes the python code to a file.
        Args:
            directory: The directory to save the file to.

        Returns:
            python_file: The path to the python file.
        """
        self.print_with_verbose('Creating python file...')
        python_file = os.path.join(directory, f'{self.base_name}.py')
        try:
            with open(python_file, 'w') as fp:
                fp.write(self.generate_python_code())
                return python_file
        except Exception as err:
            print_error(f'Error writing {python_file} - {err}')
            raise

    def save_yaml(self, directory: str) -> str:
        """
        Writes the yaml to a file.
        Args:
            directory: The directory to save the file to.

        Returns:
            yaml_file: The path to the yaml file.
        """
        self.print_with_verbose('Creating yaml file...')
        yaml_file = os.path.join(directory, f'{self.base_name}.yml')
        try:
            with open(yaml_file, 'w') as fp:
                yaml.dump(self.generate_yaml().to_dict(), fp)
            return yaml_file
        except Exception as err:
            print_error(f'Error writing {yaml_file} - {err}')
            raise

    def save_config(self, config: dict, directory: str) -> str:
        """
        Writes the integration configuration to a file in JSON format.
        Args:
            config: The integration configuration.
            directory: The directory to save the file to.

        Returns:
            config_file: The path to the configuration file.
        """
        self.print_with_verbose('Creating configuration file...')
        config_file = os.path.join(directory, f'{self.base_name}_config.json')
        try:
            with open(config_file, 'w') as fp:
                json.dump(config, fp, indent=4)
            return config_file
        except Exception as err:
            print_error(f'Error writing {config_file} - {err}')
            raise

    def save_image_and_desc(self, directory: str) -> tuple:
        """
        Writes template image and description.
        Args:
            directory: The directory to save the file to.

        Returns:
            image_path: The path to the image file.
            desc_path: The path to the description file.
        """
        self.print_with_verbose('Creating image and description files...')
        image_path = os.path.join(directory, f'{self.base_name}_image.png')
        desc_path = os.path.join(directory, f'{self.base_name}_description.md')
        try:
            shutil.copy(os.path.join(os.path.dirname(__file__), 'resources', 'Generated_image.png'), image_path)
            shutil.copy(os.path.join(os.path.dirname(__file__), 'resources', 'Generated_description.md'), desc_path)
            return image_path, desc_path
        except Exception as err:
            print_error(f'Error copying image and description files - {err}')
            return '', ''

    def save_package(self, directory: str) -> tuple:
        """
        Creates a package for the integration including python, yaml, image and description files.
        Args:
            directory: The directory to create the package in.

        Returns:
            code_path: The path to the python code file.
            yml_path: the path to the yaml file.
            image_path: The path to the image file.
            desc_path: The path to the description file.
        """
        code_path = self.save_python_code(directory)
        yml_path = self.save_yaml(directory)
        image_path, desc_path = self.save_image_and_desc(directory)
        return code_path, yml_path, image_path, desc_path

    def print_with_verbose(self, text: str):
        """
        Prints a text verbose is set to true.
        Args:
            text: The text to print.
        """
        if self.verbose:
            print(text)

    def init_arg(self, arg: dict):
        """
        Parses an argument.
        Args:
            arg: The argument to parse.

        Returns:
            new_arg: The parsed argument.
        """
        new_arg = {}
        arg_desc = arg.get('description', '')
        arg_desc = self.clean_description(arg_desc)
        new_arg['name'] = arg.get('name', '')
        new_arg['description'] = arg_desc
        new_arg['required'] = arg.get('required')
        new_arg['default'] = arg.get('default', '')
        new_arg['in'] = arg.get('in', 'path')
        new_arg['type'] = arg.get('type', 'string')
        new_arg['properties'] = arg.get('properties', {})
        new_arg['ref'] = arg.get('ref', '')
        new_arg['enums'] = [str(x) for x in arg.get('enum', [])] if arg.get('enum', None) else None

        return new_arg

    @staticmethod
    def extract_values(obj: Union[dict, list], key: str) -> list:
        """
        Extracts values from an object by a provided key.
        Args:
            obj: The object to extract.
            key: The key to extract by.

        Returns:
            results: The extracted values.
        """
        arr: list = []

        def extract(extracted_object: Union[dict, list], values: list, key_to_extract: str) -> list:
            if isinstance(extracted_object, dict):
                for k, v in extracted_object.items():
                    if k == key_to_extract:
                        values.append(v)
                    elif isinstance(v, (dict, list)):
                        extract(v, values, key_to_extract)

            elif isinstance(extracted_object, list):
                for item in extracted_object:
                    extract(item, values, key_to_extract)
            return values
        results = extract(obj, arr, key)
        return results

    @staticmethod
    def get_arg_default(arg: dict):
        """
        Gets the format for an argument default value, e.g.:
        >>> , ''
        >>> , False
        Args:
            arg: The argument to get the default format for.

        Returns:
            argument_default: The default format for the argument.
        """
        arg_type = ARGUMENT_TYPES.get(arg.get('type', 'string'), 'str')
        arg_default = arg.get('default', '')
        default: Optional[Any]
        if arg_type == 'int':
            try:
                default = int(arg_default)
            except Exception:
                default = None
            argument_default = f', {default}'
        elif arg_type == 'bool':
            try:
                default = strtobool(arg_default)
            except Exception:
                default = False
            argument_default = f', {default}'
        else:
            argument_default = f", '{arg_default}'"

        return argument_default

    @staticmethod
    def clean_description(description: str) -> str:
        """
        Cleans a description string.
        Args:
            description: The description string to clean.

        Returns:
            description: The clean description.
        """
        if len(description) > MAX_DESCRIPTION_WORDS:
            description = description[:MAX_DESCRIPTION_WORDS] + '...'
        for i in ILLEGAL_DESCRIPTION_CHARS:
            description = description.replace(i, ' ')

        return description

    @staticmethod
    def clean_function_name(name: str, snakeify: bool = True) -> str:
        """
        Cleans a function name.
        Args:
            name: The function name to clean.
            snakeify: Whether to format the name to snake-case.

        Returns:
            name: The clean name.
        """
        for i in ILLEGAL_CODE_CHARS:
            name = name.replace(i, '')

        if snakeify:
            name = camel_to_snake(name)
            name = name.replace('-', '_').replace('__', '_').strip('_')

        return name

    @staticmethod
    def format_params(params: list, base: str, base_string: str) -> str:
        """
        Formats a list of params to a string of params assignment, e.g. param1=param_name, param2=param_name2
        Args:
            params: A list of params with mapping of name to code name
            base: The base code to use
            base_string: The base string to replace with the params

        Returns:
            params: Formatted params string
        """
        modified_params = []
        for p in params:
            for name, code_name in p.items():
                modified_params.append(f'{name}={code_name}')

        params_string = base.replace(base_string, ', '.join(modified_params))
        return params_string

    def handle_duplicates(self, functions: List):
        """
        Find duplicates command names and update the names according to path
        Args:
            functions: the list of functions
        """
        duplicate_names = [d['name'] for d in functions]
        duplicate_names = [n for n in duplicate_names if duplicate_names.count(n) > 1]
        if duplicate_names:
            for func in functions:
                name = func.get('name')
                if name in duplicate_names:
                    path = func.get('path')
                    method = func.get('method')
                    # getting the last curly brackets is exists and keeping its value
                    function_path = re.sub(r'{([\w]*)\}$', r'by/\g<1>', path)
                    # Remove the rest curly brackets from the path.
                    path_name = '_'.join([re.sub(r'{[^)]*\}', '', x) for x in function_path.split('/')])
                    name = self.clean_function_name(path_name)
                    func['name'] = f'{method}_{name}'


# demisto_sdk/commands/postman_codegen/postman_codegen.py

import logging
import re
from collections import defaultdict
from typing import Any, Dict, List, Union

import demisto_sdk.commands.common.tools as tools
from demisto_sdk.commands.common.constants import DemistoException
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.hook_validations.docker import DockerImageValidator
from demisto_sdk.commands.generate_integration.code_generator import (IntegrationGeneratorArg,
                                                                      IntegrationGeneratorCommand,
                                                                      IntegrationGeneratorConfig,
                                                                      IntegrationGeneratorOutput,
                                                                      IntegrationGeneratorParam, ParameterType)
from demisto_sdk.commands.generate_outputs.json_to_outputs.json_to_outputs import determine_type, flatten_json

json = JSON_Handler()


logger: logging.Logger = logging.getLogger('demisto-sdk')


def postman_headers_to_conf_headers(postman_headers, skip_authorization_header: bool = False):
    """
    postman_headers = [
        {
            "key": "Content-Type",
            "value": "application/json",
            "type": "text"
        },
        {
            "key": "Accept",
            "value": "application/json",
            "type": "text"
        }
    ]

    to =>
    [{'Content-Type': 'application/json'}, {'Accept': 'application/json'}]

    """
    if not postman_headers:
        return None

    headers = []
    for ph in postman_headers:
        if skip_authorization_header and ph['key'] == 'Authorization':
            continue

        headers.append({
            ph['key']: ph['value']
        })

    return headers


def create_body_format(body: Union[dict, list], args: List[IntegrationGeneratorArg] = None):
    """
    Gets the raw body of a request and creates a request holding the argument's names in it (that we will get from the users),
    instead of the values that the argument's had in postman. The argument's name are generated surrounded with {}.

    :param body: the raw body from postman
    :param args: list of the arguments, holding their name and path in the object
    """

    def format_value(input_body: Dict, output_body: Dict, path: tuple = ()):
        for key, value in input_body.items():
            if isinstance(value, dict):
                output_body[key] = {}
                output_body[key] = format_value(value, output_body[key], path + (key,))
            elif isinstance(value, list) and len(value) > 0 and isinstance(value[0], dict):
                output_body[key] = [{}]
                output_body[key][0] = format_value(value[0], output_body[key][0], path + (key,))
            else:
                output_body[key] = '{' + key + '}'
                if args:
                    for arg in args:
                        if arg.in_object == list(path) and key == arg.name.split('_')[-1]:
                            output_body[key] = '{' + arg.name + '}'
                            break
        return output_body

    if isinstance(body, dict):
        return format_value(body, {})
    if isinstance(body, list) and len(body) > 0 and isinstance(body[0], dict):
        return [format_value(body[0], {})]

    return None


def flatten_collections(items: list) -> list:
    """When creating nested collections, will flatten all the requests to a list of dicts."""
    lst = list()
    for item in items:
        if isinstance(item, list):  # A list of collections
            lst += flatten_collections(item)
        elif isinstance(item, dict) and 'item' in item:  # Get requests from collection
            lst += flatten_collections(item['item'])
        else:  # Request itself is a dict.
            lst.append(item)
    return lst


def postman_to_autogen_configuration(
        collection: dict,
        name,
        command_prefix,
        context_path_prefix,
        category=None
) -> IntegrationGeneratorConfig:
    info = collection.get('info', {})
    items = collection.get('item', [])
    postman_auth = collection.get('auth', {})
    variable = collection.get('variable', [])

    logger.debug('trying to find the default base url')
    for v in variable:
        if v['key'] in ('url', 'server'):
            host = v['value']
            logger.debug(f'base url found: {host}')
            break
    else:
        host = ''

    docker_image = get_docker_image()

    description = ''

    commands = []
    items = flatten_collections(items)  # in case of nested collections
    commands_names = build_commands_names_dict(items)
    duplicate_requests_check(commands_names)

    for item in items:
        command = convert_request_to_command(item)

        if command is None:
            # skip command in case is None
            # probably something was wrong with the request and command is not created
            continue

        commands.append(command)

    params = [
        IntegrationGeneratorParam(
            name='url',
            display='Server URL',
            type_=ParameterType.STRING,
            required=True,
            defaultvalue=host or 'https://www.example.com'
        ),
        IntegrationGeneratorParam(
            name='proxy',
            display='Use system proxy',
            type_=ParameterType.BOOLEAN,
            required=False
        ),
        IntegrationGeneratorParam(
            name='insecure',
            display='Trust any certificate',
            type_=ParameterType.BOOLEAN,
            required=False
        )
    ]

    if postman_auth:
        if postman_auth['type'] == 'apikey':
            params.append(IntegrationGeneratorParam(
                name='api_key',
                display='API Key',
                type_=ParameterType.ENCRYPTED,
                required=True
            ))
        elif postman_auth['type'] == 'bearer':
            params.append(IntegrationGeneratorParam(
                name='token',
                display='API Token',
                type_=ParameterType.ENCRYPTED,
                required=True
            ))
        elif postman_auth['type'] == 'basic':
            params.append(IntegrationGeneratorParam(
                name='credentials',
                display='Username',
                type_=ParameterType.AUTH,
                required=True
            ))
    else:
        # look for apikey in headers
        logger.debug('trying to find apikey in request headers')
        auth_header_found = False
        for item in items:
            if auth_header_found:
                break

            request = item.get('request', {})
            for header in request.get('header', []):
                if header.get('key') == 'Authorization':
                    params.append(IntegrationGeneratorParam(
                        name='api_key',
                        display='API Key',
                        type_=ParameterType.ENCRYPTED,
                        required=True
                    ))
                    logger.debug('found Authorization header')

                    if '{{' in header.get('value'):
                        # if header value contains {{ means it has a format like 'Authorization': 'SWSS {{apikey}}'
                        # header_format will be f'SWSS {api_key}'
                        header_format = "f'" + re.sub(r'\{\{.*\}\}', '{params["api_key"]}', header.get('value')) + "'"
                    else:
                        header_format = 'params[\'api_key\']'

                    postman_auth = {
                        'type': 'apikey',
                        'apikey': [
                            {
                                "key": "format",
                                "value": header_format,
                                "type": "string"
                            },
                            {
                                "key": "in",
                                "value": "header",
                                "type": "string"
                            },
                            {
                                "key": "key",
                                "value": "Authorization",
                                "type": "string"
                            }
                        ]
                    }

                    auth_header_found = True
                    break

    if name:
        display_name = name
        id_ = ''.join(e for e in name if e.isalnum())
    elif info.get('name'):
        display_name = info.get('name')
        id_ = ''.join(e for e in info.get('name') if e.isalnum())
    else:
        display_name = 'Generated Name Replace It'
        id_ = 'GeneratedNameReplaceIt'

    config = IntegrationGeneratorConfig(
        name=id_,
        display_name=display_name,
        description=description or info.get('description', 'Generated description - REPLACE THIS'),
        params=params,
        category=category or 'Utilities',
        command_prefix=command_prefix or tools.to_kebab_case(id_),
        commands=commands,
        docker_image=docker_image,
        context_path=context_path_prefix or id_,
        url=host,
        base_url_path='',
        auth=postman_auth,
    )

    return config


def get_docker_image():
    try:
        latest_tag = DockerImageValidator.get_docker_image_latest_tag_request('demisto/python3')
        docker_image = f'demisto/python3:{latest_tag}'
        logger.debug(f'docker image set to: {docker_image}')
    except Exception as e:
        # set default docker image
        docker_image = 'demisto/python3:3.9.1.14969'
        logger.warning(f'Failed getting latest docker image for demisto/python3: {e}')
    return docker_image


def convert_request_to_command(item: dict):
    logger.debug(f'converting request to command: {item.get("name")}')
    name = item.get('name')
    assert isinstance(name, str), 'Could not find name. Is this a valid postman 2.1 collection?'
    command_name = tools.to_kebab_case(name)
    context_prefix = tools.to_pascal_case(name)

    request = item.get('request')
    if request is None:
        raise DemistoException('Could not find request in the collection. Is it a valid postman collection?')

    logger.debug(f'converting postman headers of request: {name}')
    headers = postman_headers_to_conf_headers(request.get('header'), skip_authorization_header=True)

    args = []
    outputs = []
    returns_file = False

    logger.debug(f'creating url arguments of request: {name}')
    request_url_object = request.get('url')

    if not request_url_object:
        logger.error(f'failed to get item.request.url.path object of request {name}. '
                     f'Go to Postman, Save the request and try again with the updated collection.')
        return None

    url_path = '/'.join(request_url_object.get('path')).replace('{{', '{').replace('}}', '}')
    for url_path_item in request_url_object.get('path'):
        if re.match(r'\{\{.*\}\}', url_path_item):
            arg = IntegrationGeneratorArg(
                name=url_path_item.replace('{{', '').replace('}}', ''),
                description='',
                in_='url'
            )
            args.append(arg)

    for url_path_variable in request_url_object.get('variable', []):
        variable_name = url_path_variable.get('key')
        if not variable_name:
            continue
        arg = IntegrationGeneratorArg(
            name=variable_name,
            description='',
            in_='url'
        )
        args.append(arg)
        url_path = url_path.replace(f'/:{variable_name}', f'/{{{variable_name}}}')

    logger.debug(f'creating query arguments of request: {name}')
    for q in request_url_object.get('query', []):
        arg = IntegrationGeneratorArg(
            name=q.get('key'),
            description='',
            in_='query'
        )
        args.append(arg)

    logger.debug(f'creating arguments which will be passed to the request body of request: {name}')
    request_body = request.get('body')
    body_format = None
    if request_body:
        if request_body.get('mode') == 'raw':
            try:
                body_obj = json.loads(request_body.get('raw'))
                flattened_json = flatten_json(body_obj)
                shared_arg_to_split_position_dict = find_shared_args_path(flattened_json)

                for key, value in flattened_json.items():
                    path_split = key.split('.')
                    json_path = path_split[:-1]
                    min_unique_path_length = shared_arg_to_split_position_dict[path_split[-1].lower()] + 1
                    arg_name = '_'.join(path_split[-min_unique_path_length:])
                    arg = IntegrationGeneratorArg(
                        name=arg_name,
                        description='',
                        in_='body',
                        in_object=json_path
                    )
                    args.append(arg)

                body_format = create_body_format(body_obj, args)

            except Exception:
                logger.exception(f'Failed to parse {name} request body as JSON.')

    if not item.get('response') or item.get('response') == 0:
        logger.error(f'[{name}] request is missing response. Make sure to save at least one successful '
                     f'response in Postman')
    else:
        try:
            response = item.get('response')[0]  # type: ignore[index]  # It will be catched in the except
            if response.get('_postman_previewlanguage') == 'json':
                outputs = generate_command_outputs(json.loads(response.get('body')))
            elif response.get('_postman_previewlanguage') == 'raw':
                returns_file = True

        except (ValueError, IndexError, TypeError):
            logger.exception(f'Failed to parse to JSON response body of {name} request.')

    command = IntegrationGeneratorCommand(
        name=command_name,
        url_path=url_path,
        http_method=request.get('method'),
        headers=headers,
        description=request.get('description') or '',
        arguments=args,
        outputs=outputs,
        context_path=context_prefix,
        root_object='',
        unique_key='',
        returns_file=returns_file,
        body_format=body_format
    )

    return command


def generate_command_outputs(body: Union[Dict, List]) -> List[IntegrationGeneratorOutput]:
    """
    Parses postman body to list of command outputs.
    Args:
        body (Union[Dict, List]): Body returned from HTTP request.

    Returns:
        (List[IntegrationGeneratorOutput]): List of outputs returned from the HTTP request.
    """
    flattened_body = flatten_json(body)
    # If body is list, remove first item of every key as it generates additional not needed dot.
    if isinstance(body, list):
        flattened_body = {k[1:]: v for k, v in flattened_body.items()}
    return [IntegrationGeneratorOutput(
        name=key,
        description='',
        type_=determine_type(value)
    ) for key, value in flattened_body.items()]


def build_commands_names_dict(items: list) -> dict:
    names_dict = defaultdict(list)
    for item in items:
        request_name = item.get('name', None)
        if request_name:
            command_name = tools.to_kebab_case(request_name)
            names_dict[command_name].append(request_name)
    return names_dict


def duplicate_requests_check(commands_names_dict: dict) -> None:
    duplicates_list = []
    for key in commands_names_dict:
        if len(commands_names_dict[key]) > 1:
            duplicates_list.extend(commands_names_dict[key])

    assert len(duplicates_list) == 0, f'There are requests with non-unique names: {duplicates_list}.\n' \
                                      f'You should give a unique name to each request.\n' \
                                      f'Names are case-insensitive and whitespaces are ignored.'


def find_shared_args_path(flattened_json: Dict[str, Any]) -> Dict[str, int]:
    """
    Finds the minimum unique path length needed for all arguments with the same name.
    Args:
        flattened_json (Dict[str, any]): Flattened json returned after running the flatten_json function.

    Returns:
        (Dict[str, int]): Dictionary from argument name to the minimum unique path length for all arguments with this name.
    """
    arg_name_to_split_path_dict: Dict[str, List[List[str]]] = defaultdict(list)
    shared_arg_to_split_position_dict: Dict[str, int] = defaultdict(int)
    for key in flattened_json:
        split_path = key.split('.')
        arg_name = split_path[-1].lower()

        shared_arg_to_split_position_dict[arg_name] = update_min_unique_path(split_path,
                                                                             arg_name_to_split_path_dict[arg_name],
                                                                             shared_arg_to_split_position_dict[arg_name])
        arg_name_to_split_path_dict[arg_name].append(split_path)

    return shared_arg_to_split_position_dict


def update_min_unique_path(split_path: List[str], other_args_split_paths: List[List[str]], current_min_unique: int):
    """
    Finds the minimum unique path length needed for all arguments with the same name.
    """
    for other_arg_split_path in other_args_split_paths:
        for max_same_path, (other_path, arg_path) in enumerate(zip(other_arg_split_path[::-1], split_path[::-1])):
            if other_path == arg_path:
                current_min_unique = max(max_same_path + 1, current_min_unique)
            else:
                break

    return current_min_unique


# demisto_sdk/commands/run_cmd/runner.py

import ast
import re
import tempfile

import demisto_client

from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.tools import LOG_COLORS, print_color, print_error, print_v, print_warning
from demisto_sdk.commands.generate_outputs.json_to_outputs.json_to_outputs import json_to_outputs

json = JSON_Handler()


class DemistoRunTimeError(RuntimeError):
    """Demisto run time error"""
    pass


class Runner:
    """Used to run a command on Demisto and print the results.
        Attributes:
            query (str): The query to execute.
            log_verbose (bool): Whether to output a detailed response.
            debug (str): Holds the path of the debug log file (or '-' if the logs will be printed in stdout).
            debug_path (str): The path in which you will save the debug file.
            client (DefaultApi): Demisto-SDK client object.
        """
    ERROR_ENTRY_TYPE = 4
    DEBUG_FILE_ENTRY_TYPE = 16
    SECTIONS_HEADER_REGEX = re.compile(r'^(Context Outputs|Human Readable section|Raw Response section)')
    RAW_RESPONSE_HEADER = re.compile(r'^Raw Response section')
    CONTEXT_HEADER = re.compile(r'Context Outputs:')
    HUMAN_READABLE_HEADER = re.compile(r'Human Readable section')
    FULL_LOG_REGEX = re.compile(r'.*Full Integration Log')

    def __init__(self, query: str, insecure: bool = False, debug: str = None, debug_path: str = None,
                 verbose: bool = False, json_to_outputs: bool = False, prefix: str = '', raw_response: bool = False):
        self.query = query if query.startswith('!') else f'!{query}'
        self.log_verbose = verbose
        self.debug = debug
        self.debug_path = debug_path
        verify = (not insecure) if insecure else None  # set to None so demisto_client will use env var DEMISTO_VERIFY_SSL
        self.client = demisto_client.configure(verify_ssl=verify)
        self.json2outputs = json_to_outputs
        self.prefix = prefix
        self.raw_response = raw_response

        if self.debug or self.json2outputs:
            self.query += ' debug-mode="true"'

    def run(self):
        """Runs an integration command on Demisto and prints the result.
        """
        playground_id = self._get_playground_id()

        try:
            log_ids = self._run_query(playground_id)
        except DemistoRunTimeError as err:
            log_ids = None
            print_error(str(err))

        if self.debug:
            if not log_ids:
                print_warning('Entry with debug log not found')
            else:
                self._export_debug_log(log_ids)

        if self.json2outputs:
            if not self.prefix:
                print_error("A prefix for the outputs is needed for this command. Please provide one")
                return 1
            else:
                raw_output_json = self._return_context_dict_from_log(log_ids)
                if raw_output_json:
                    with tempfile.NamedTemporaryFile(mode='w+') as f:
                        if isinstance(raw_output_json, dict):
                            f.write(json.dumps(raw_output_json))
                        if isinstance(raw_output_json, list):
                            f.write(json.dumps(raw_output_json[0]))
                        f.seek(0)
                        file_path = f.name
                        command = self.query.split(' ')[0]
                        json_to_outputs(command, json=file_path, prefix=self.prefix)
                else:
                    print_error("Could not extract raw output as JSON from command")
                    return 1

    def _get_playground_id(self):
        """Retrieves Playground ID from the remote Demisto instance.
        """
        playground_filter = {'filter': {'type': [9]}}
        answer = self.client.search_investigations(filter=playground_filter)
        if answer.total == 0:
            raise RuntimeError(
                'No playgrounds were detected in the environment.'
            )
        playgrounds = answer.data
        if len(playgrounds) > 1:
            # if found more than one playground, try to filter to results against the current user
            user_data, response, _ = self.client.generic_request(path='/user',
                                                                 method='GET',
                                                                 content_type='application/json',
                                                                 response_type=object)
            if response != 200:
                raise RuntimeError('Cannot find username')
            username = user_data.get('username')
            playgrounds = [playground for playground in playgrounds if playground.creating_user_id == username]
            if len(playgrounds) != 1:
                raise RuntimeError(
                    f'There is more than one playground to the user. '
                    f'Number of playgrounds is: {len(playgrounds)}'
                )

        result = playgrounds[0].id

        print_v(f'Playground ID: {result}', self.log_verbose)

        return result

    def _run_query(self, playground_id: str):
        """Runs a query on Demisto instance and prints the output.

        Args:
            playground_id: The investigation ID of the playground.

        Returns:
            list. A list of the log IDs if debug mode is on, otherwise an empty list.
        """
        update_entry = {
            'investigationId': playground_id,
            'data': self.query
        }

        answer = self.client.investigation_add_entries_sync(update_entry=update_entry)
        if not answer:
            raise DemistoRunTimeError('Command did not run, make sure it was written correctly.')

        log_ids = []

        for entry in answer:
            # answer should have entries with `contents` - the readable output of the command
            if entry.parent_content:
                print_color('### Command:', LOG_COLORS.YELLOW)
            if entry.contents:
                print_color('## Readable Output', LOG_COLORS.YELLOW)
                if entry.type == self.ERROR_ENTRY_TYPE:
                    print_error(f'{entry.contents}\n')
                else:
                    print(f'{entry.contents}\n')

            # and entries with `file_id`s defined, that is the fileID of the debug log file
            if entry.type == self.DEBUG_FILE_ENTRY_TYPE:
                log_ids.append(entry.id)

        return log_ids

    def _export_debug_log(self, log_ids: list):
        """Retrieve & export debug mode log files

        Args:
            log_ids (list): artifact ids of the log files
        """
        if self.debug_path:
            with open(self.debug_path, 'w+b') as output_file:
                for log_id in log_ids:
                    result = self.client.download_file(log_id)
                    with open(result, 'r+') as log_info:
                        for line in log_info:
                            output_file.write(line.encode('utf-8'))
            print_color(f'Debug Log successfully exported to {self.debug_path}', LOG_COLORS.GREEN)
        else:
            print_color('## Detailed Log', LOG_COLORS.YELLOW)
            for log_id in log_ids:
                result = self.client.download_file(log_id)
                with open(result, 'r+') as log_info:
                    for line in log_info:
                        if self.SECTIONS_HEADER_REGEX.match(line):
                            print_color(line, LOG_COLORS.YELLOW)
                        elif self.FULL_LOG_REGEX.match(line):
                            print_color('Full Integration Log:', LOG_COLORS.YELLOW)
                        else:
                            print(line)

    def _return_context_dict_from_log(self, log_ids: list) -> dict:
        """
            retrieves the context section from the debug_log. If context is empty ({}) or doesn't exist, returns
            the raw output section.
        Args:
            log_ids (list): artifact ids of the log files

        Returns:
            the context of the executed query
        """
        if not self.debug_path:
            for log_id in log_ids:
                result = self.client.download_file(log_id)
                with open(result, 'r+') as log_info:
                    for line in log_info:
                        if self.RAW_RESPONSE_HEADER.match(line):
                            try:
                                return json.loads(log_info.readline())
                            except Exception:
                                pass
                        if self.CONTEXT_HEADER.match(line) and not self.raw_response:
                            context = ''
                            line = log_info.readline()
                            while not self.HUMAN_READABLE_HEADER.match(line):
                                context = context + line
                                line = log_info.readline()
                            context = re.sub(r"\(val\..+\)", "", context)  # noqa: W605
                            try:
                                temp_dict = json.loads(context)
                                if temp_dict:
                                    return temp_dict
                            except Exception:
                                pass
            return dict()
        else:
            temp_dict = dict()
            with open(self.debug_path, 'w+b') as output_file:
                for log_id in log_ids:
                    result = self.client.download_file(log_id)
                    with open(result, 'r+') as log_info:
                        for line in log_info:
                            if self.RAW_RESPONSE_HEADER.match(line) and not temp_dict:
                                output_file.write(line.encode('utf-8'))
                                line = log_info.readline()
                                try:
                                    temp_dict = json.loads(line)
                                except Exception:
                                    pass
                            if self.CONTEXT_HEADER.match(line) and not self.raw_response:
                                context = ''
                                output_file.write(line.encode('utf-8'))
                                line = log_info.readline()
                                while not self.HUMAN_READABLE_HEADER.match(line):
                                    output_file.write(line.encode('utf-8'))
                                    context = context + line
                                    line = log_info.readline()
                                context = re.sub(r'\(val\..+\)', '', context)
                                try:
                                    temp_dict = json.loads(context)
                                except Exception:
                                    pass
                            output_file.write(line.encode('utf-8'))
            print_color(f'Debug Log successfully exported to {self.debug_path}', LOG_COLORS.GREEN)
            return temp_dict

    def execute_command(self, command: str):
        playground_id = self._get_playground_id()

        # delete context
        update_entry = {
            'investigationId': playground_id,
            'data': '!DeleteContext all=yes'
        }

        self.client.investigation_add_entries_sync(update_entry=update_entry)

        # execute the command in playground
        update_entry = {
            'investigationId': playground_id,
            'data': command
        }
        res = self.client.investigation_add_entries_sync(update_entry=update_entry)

        body = {'query': '${.}'}
        context = self.client.generic_request(f'investigation/{playground_id}/context', 'POST', body)[0]

        context = ast.literal_eval(context)

        return res, context


# demisto_sdk/commands/run_playbook/playbook_runner.py

import os
import time

import demisto_client
from demisto_client.demisto_api.rest import ApiException

from demisto_sdk.commands.common.tools import LOG_COLORS, print_color, print_error


class PlaybookRunner:
    """PlaybookRunner is a class that's designed to run a playbook in a given Demisto instance.

    Attributes:
        playbook_id (str): the ID of the playbook to run
        should_wait (bool): whether to wait until the playbook run is completed or not.
        timeout (int): timeout for the command. The playbook will continue to run in Demisto
        demisto_client (DefaultApi): Demisto-SDK client object.
        base_link_to_workplan (str): the base link to see the full playbook run in Demisto.
    """

    def __init__(self, playbook_id: str, url: str, wait: bool, timeout: int, insecure: bool = False):
        self.playbook_id = playbook_id
        self.should_wait = wait
        self.timeout = timeout
        verify = (not insecure) if insecure else None  # we set to None so demisto_client will use env var DEMISTO_VERIFY_SSL
        # if url parameter is not provided, demisto_client will search the DEMISTO_BASE_URL env variable
        self.demisto_client = demisto_client.configure(
            base_url=url,
            verify_ssl=verify)

        self.base_link_to_workplan = self.get_base_link_to_workplan(url)

    def run_playbook(self):
        # type: () -> int
        """Run a playbook in Demisto.

        Returns:
            int. 0 in success, 1 in a failure.
        """
        # create an incident with the given playbook
        try:
            incident_id = self.create_incident_with_playbook(
                incident_name=f'inc_{self.playbook_id}', playbook_id=self.playbook_id)
        except ApiException as a:
            print_error(str(a))
            return 1

        work_plan_link = self.base_link_to_workplan + str(incident_id)
        if self.should_wait:
            print(f'Waiting for the playbook to finish running.. \n'
                  f'To see the playbook run in real-time please go to : {work_plan_link}',
                  LOG_COLORS.GREEN)

            elasped_time = 0
            start_time = time.time()

            while elasped_time < self.timeout:
                playbook_results = self.get_playbook_results_dict(incident_id)
                if playbook_results["state"] == "inprogress":
                    time.sleep(10)
                    elasped_time = int(time.time() - start_time)
                else:   # the playbook has finished running
                    break

            # Ended the loop because of timeout
            if elasped_time >= self.timeout:
                print_error(f'The command had timed out while the playbook is in progress.\n'
                            f'To keep tracking the playbook please go to : {work_plan_link}')
            else:
                if playbook_results["state"] == "failed":
                    print_error("The playbook finished running with status: FAILED")
                else:
                    print_color("The playbook has completed its run successfully", LOG_COLORS.GREEN)

        # The command does not wait for the playbook to finish running
        else:
            print(f'To see results please go to : {work_plan_link}')

        return 0

    def create_incident_with_playbook(self, incident_name, playbook_id):
        # type: (str, str) -> int
        """Create an incident in Demisto with the given incident_name and the given playbook_id

        Args:
            incident_name (str): The name of the incident
            playbook_id (str): The id of the playbook

        Raises:
            ApiException: if the client has failed to create an incident

        Returns:
            int. The new incident's ID.
        """

        create_incident_request = demisto_client.demisto_api.CreateIncidentRequest()
        create_incident_request.create_investigation = True
        create_incident_request.playbook_id = playbook_id
        create_incident_request.name = incident_name

        try:
            response = self.demisto_client.create_incident(create_incident_request=create_incident_request)
        except ApiException as e:
            print_error(f'Failed to create incident with playbook id : "{playbook_id}", '
                        'possible reasons are:\n'
                        '1. This playbook name does not exist \n'
                        '2. Schema problems in the playbook \n'
                        '3. Unauthorized api key')
            raise e

        print_color(f'The playbook: {self.playbook_id} was triggered successfully.', LOG_COLORS.GREEN)
        return response.id

    def get_playbook_results_dict(self, inc_id):
        playbook_results = self.demisto_client.generic_request(method='GET', path=f'/inv-playbook/{inc_id}')
        return eval(playbook_results[0])

    def get_base_link_to_workplan(self, url):
        """Create a base link to the workplan in the specified demisto instance

        Args:
            url(str): URL to a demisto instance. Could be None if not provided

        Returns:
            str: The link to the workplan
        """

        if url:
            return f'{url}/#/WorkPlan/'

        else:
            base_url = os.environ.get('DEMISTO_BASE_URL')
            return f'{base_url}/#/WorkPlan/'


# demisto_sdk/commands/run_test_playbook/test_playbook_runner.py

import os
import re
import time

import demisto_client
from demisto_client.demisto_api.rest import ApiException

from demisto_sdk.commands.common.tools import LOG_COLORS, get_yaml, print_color
from demisto_sdk.commands.upload.uploader import Uploader

SUCCESS_RETURN_CODE = 0
ERROR_RETURN_CODE = 1
ENTRY_TYPE_ERROR = 4


class TestPlaybookRunner:
    """TestPlaybookRunner is a class that's designed to run a test playbook in a given XSOAR instance.

    Attributes:
        test_playbook_path (str): the input of the test playbook to run
        all (bool): whether to wait until the playbook run is completed or not.
        should_wait (bool): whether to wait until the test playbook run is completed or not.
        timeout (int): timeout for the command. The test playbook will continue to run in your xsoar instance.
        demisto_client (DefaultApi): Demisto-SDK client object.
        base_link_to_workplan (str): the base link to see the full test playbook run in your xsoar instance.
    """

    def __init__(self, test_playbook_path: str = '', all: bool = False, wait: bool = True, timeout: int = 90,
                 insecure: bool = False):
        self.test_playbook_path = test_playbook_path
        self.all_test_playbooks = all
        self.should_wait = wait
        self.timeout = timeout

        # we set to None so demisto_client will use env var DEMISTO_VERIFY_SSL
        self.verify = (not insecure) if insecure else None
        self.demisto_client = demisto_client.configure(verify_ssl=self.verify)
        self.base_link_to_workplan = self.get_base_link_to_workplan()

    def manage_and_run_test_playbooks(self):
        """
        Manages all ru-test-playbook command flows
        return The exit code of each flow
        """
        status_code = SUCCESS_RETURN_CODE
        test_playbooks: list = []

        if not self.validate_tpb_path():
            status_code = ERROR_RETURN_CODE

        test_playbooks.extend(self.collect_all_tpb_files_paths())

        for tpb in test_playbooks:
            self.upload_tpb(tpb_file=tpb)
            test_playbook_id = self.get_test_playbook_id(tpb)
            status_code = self.run_test_playbook_by_id(test_playbook_id)

        return status_code

    def collect_all_tpb_files_paths(self):
        test_playbooks: list = []
        # Run all repo test playbooks
        if self.all_test_playbooks:
            test_playbooks.extend(self.get_all_test_playbooks())

        # Run all pack test playbooks
        elif os.path.isdir(self.test_playbook_path):
            test_playbooks.extend(self.get_test_playbooks_from_folder(self.test_playbook_path))

        # Run specific test playbook
        elif os.path.isfile(self.test_playbook_path):
            test_playbooks.append(self.test_playbook_path)

        return test_playbooks

    def validate_tpb_path(self) -> bool:
        """
        Verifies that the input path configuration given by the user is correct
        :return: The verification result
        """
        is_path_valid = True
        if not self.all_test_playbooks:
            if not self.test_playbook_path:
                print_color("Error: Missing option '-tpb' / '--test-playbook-path'.", LOG_COLORS.RED)
                is_path_valid = False

            elif not os.path.exists(self.test_playbook_path):
                print_color(f'Error: Given input path: {self.test_playbook_path} does not exist', LOG_COLORS.RED)
                is_path_valid = False

        return is_path_valid

    def get_test_playbook_id(self, file_path):
        """
        Get test playbook ID from test playbook file name
        """
        test_playbook_data = get_yaml(file_path=file_path)
        return test_playbook_data.get('id')

    def get_test_playbooks_from_folder(self, folder_path):
        """
        Get all pack test playbooks
        """
        full_path = f'{folder_path}/TestPlaybooks'
        list_test_playbooks_files = os.listdir(full_path)
        list_test_playbooks_files = [f'{full_path}/{tpb}' for tpb in list_test_playbooks_files]
        return list_test_playbooks_files

    def get_all_test_playbooks(self):
        """
        Get all the repo test playbooks
        """
        tpb_list: list = []
        packs_list = os.listdir('Packs')
        for pack in packs_list:
            if os.path.isdir(f'Packs/{pack}'):
                tpb_list.extend(self.get_test_playbooks_from_folder(f'Packs/{pack}'))
        return tpb_list

    def run_test_playbook_by_id(self, test_playbook_id):
        """Run a test playbook in your xsoar instance.

        Returns:
            int. 0 in success, 1 in a failure.
        """
        status_code: int = SUCCESS_RETURN_CODE
        # create an incident with the given playbook
        try:
            incident_id = self.create_incident_with_test_playbook(
                incident_name=f'inc_{test_playbook_id}', test_playbook_id=test_playbook_id)
        except ApiException as a:
            print_color(str(a), LOG_COLORS.RED)
            status_code = ERROR_RETURN_CODE

        work_plan_link = self.base_link_to_workplan + str(incident_id)
        if self.should_wait:
            status_code = self.run_and_check_tpb_status(test_playbook_id, work_plan_link, incident_id)

        else:
            print_color(f'To see results please go to : {work_plan_link}', LOG_COLORS.NATIVE)

        return status_code

    def run_and_check_tpb_status(self, test_playbook_id, work_plan_link, incident_id):
        status_code = SUCCESS_RETURN_CODE
        print_color(f'Waiting for the test playbook to finish running.. \n'
                    f'To see the test playbook run in real-time please go to : {work_plan_link}', LOG_COLORS.GREEN)

        elapsed_time = 0
        start_time = time.time()

        while elapsed_time < self.timeout:
            test_playbook_result = self.get_test_playbook_results_dict(incident_id)
            if test_playbook_result['state'] == "inprogress":
                time.sleep(10)
                elapsed_time = int(time.time() - start_time)
            else:  # the test playbook has finished running
                break

        # Ended the loop because of timeout
        if elapsed_time >= self.timeout:
            print_color(f'The command had timed out while the playbook is in progress.\n'
                        f'To keep tracking the test playbook please go to : {work_plan_link}', LOG_COLORS.RED)
        else:
            if test_playbook_result['state'] == "failed":
                self.print_tpb_error_details(test_playbook_result, test_playbook_id)
                print_color("The test playbook finished running with status: FAILED", LOG_COLORS.RED)
                status_code = ERROR_RETURN_CODE
            else:
                print_color("The test playbook has completed its run successfully", LOG_COLORS.GREEN)

        return status_code

    def create_incident_with_test_playbook(self, incident_name, test_playbook_id):
        # type: (str, str) -> int
        """Create an incident in your xsoar instance with the given incident_name and the given test_playbook_id

        Args:
            incident_name (str): The name of the incident
            test_playbook_id (str): The id of the playbook

        Raises:
            ApiException: if the client has failed to create an incident

        Returns:
            int. The new incident's ID.
        """

        create_incident_request = demisto_client.demisto_api.CreateIncidentRequest()
        create_incident_request.create_investigation = True
        create_incident_request.playbook_id = test_playbook_id
        create_incident_request.name = incident_name

        try:
            response = self.demisto_client.create_incident(create_incident_request=create_incident_request)
        except ApiException as e:
            print_color(f'Failed to create incident with playbook id : "{test_playbook_id}", '
                        'possible reasons are:\n'
                        '1. This playbook name does not exist \n'
                        '2. Schema problems in the playbook \n'
                        '3. Unauthorized api key', LOG_COLORS.RED)
            raise e

        print_color(f'The test playbook: {self.test_playbook_path} was triggered successfully.', LOG_COLORS.GREEN)
        return response.id

    def get_test_playbook_results_dict(self, inc_id):
        test_playbook_results = self.demisto_client.generic_request(method='GET', path=f'/inv-playbook/{inc_id}')
        return eval(test_playbook_results[0])

    def print_tpb_error_details(self, tpb_res, tpb_id):
        entries = tpb_res.get('entries')
        if entries:
            print_color(f'Test Playbook {tpb_id} has failed:', LOG_COLORS.RED)
            for entry in entries:
                if entry['type'] == ENTRY_TYPE_ERROR and entry['parentContent']:
                    print_color(f'- Task ID: {entry["taskId"]}', LOG_COLORS.RED)
                    # Checks for passwords and replaces them with "******"
                    parent_content = re.sub(r' (P|p)assword="[^";]*"', ' password=******', entry['parentContent'])
                    print_color(f'  Command: {parent_content}', LOG_COLORS.RED)
                    print_color(f'  Body:\n{entry["contents"]}', LOG_COLORS.RED)

    def get_base_link_to_workplan(self):
        """Create a base link to the workplan in the specified xsoar instance
        Returns:
            str: The link to the workplan
        """

        base_url = os.environ.get('DEMISTO_BASE_URL')
        return f'{base_url}/#/WorkPlan/'

    def upload_tpb(self, tpb_file):
        uploader = Uploader(input=tpb_file, insecure=self.verify)  # type: ignore
        uploader.upload()


# demisto_sdk/commands/secrets/secrets.py

import io
import math
import os
import string
from collections import defaultdict
from typing import DefaultDict

import PyPDF2
from bs4 import BeautifulSoup

from demisto_sdk.commands.common.configuration import Configuration
from demisto_sdk.commands.common.constants import (PACKS_DIR, PACKS_INTEGRATION_README_REGEX, PACKS_WHITELIST_FILE_NAME,
                                                   FileType, re)
from demisto_sdk.commands.common.content import Content
from demisto_sdk.commands.common.git_util import GitUtil
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.tools import (LOG_COLORS, find_type, get_pack_name, is_file_path_in_pack, print_color,
                                               print_error, print_warning, run_command)

json = JSON_Handler()


# secrets settings
# Entropy score is determined by shanon's entropy algorithm, most English words will score between 1.5 and 3.5

ENTROPY_THRESHOLD = 4.0
ACCEPTED_FILE_STATUSES = ['m', 'a']
SKIPPED_FILES = {
    'secrets_white_list', 'id_set.json', 'conf.json', 'Pipfile', 'secrets-ignore', 'ami_builds.json',
    'secrets_test.py', 'secrets.py', 'constants.py', 'core.py', 'pack_metadata.json',
    'dev-requirements-py2.txt', 'dev-requirements-py3.txt', '.vscode/extensions.json', '.devcontainer/devcontainer.json',
}
TEXT_FILE_TYPES = {'.yml', '.py', '.json', '.md', '.txt', '.sh', '.ini', '.eml', '', '.csv', '.js', '.pdf', '.html',
                   '.ps1'}
SKIP_FILE_TYPE_ENTROPY_CHECKS = {'.eml'}
SKIP_DEMISTO_TYPE_ENTROPY_CHECKS = {'playbook-'}
YML_FILE_EXTENSION = '.yml'

# disable-secrets-detection-start
# secrets
URLS_REGEX = r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+'
EMAIL_REGEX = r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+'
IPV6_REGEX = r'(?:(?:[0-9A-Fa-f]{1,4}:){6}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1' \
             r'[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|::' \
             r'(?:[0-9A-Fa-f]{1,4}:){5}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]' \
             r'{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|' \
             r'(?:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){4}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|' \
             r'(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}' \
             r'|2[0-4][0-9]|25[0-5]))|(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){3}' \
             r'(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])' \
             r'\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,2}' \
             r'[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:){2}(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|' \
             r'(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}' \
             r'|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]{1,4}:){,3}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}:' \
             r'(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4]' \
             r'[0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|(?:(?:[0-9A-Fa-f]' \
             r'{1,4}:){,4}[0-9A-Fa-f]{1,4})?::(?:[0-9A-Fa-f]{1,4}:[0-9A-Fa-f]{1,4}|(?:(?:[0-9]|[1-9][0-9]' \
             r'|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))|' \
             r'(?:(?:[0-9A-Fa-f]{1,4}:){,5}[0-9A-Fa-f]{1,4})?::[0-9A-Fa-f]{1,4}|' \
             r'(?:(?:[0-9A-Fa-f]{1,4}:){,6}[0-9A-Fa-f]{1,4})?::)'
IPV4_REGEX = r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b'
DATES_REGEX = r'((\d{4}[/.-]\d{2}[/.-]\d{2})[T\s](\d{2}:?\d{2}:?\d{2}:?(\.\d{5,10})?([+-]\d{2}:?\d{2})?Z?)?)'
# false positives
UUID_REGEX = r'([\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{8,12})'
# find any substring
WHILEIST_REGEX = r'\S*{}\S*'


# disable-secrets-detection-end


class SecretsValidator(object):

    def __init__(
            self,
            configuration=Configuration(), is_circle=False, ignore_entropy=False, white_list_path='',
            input_path='', prev_ver=None
    ):
        self.input_paths = input_path.split(',') if input_path else None
        self.configuration = configuration
        self.is_circle = is_circle
        self.white_list_path = white_list_path
        self.ignore_entropy = ignore_entropy
        self.prev_ver = prev_ver
        if self.prev_ver and not self.prev_ver.startswith('origin'):
            self.prev_ver = 'origin/' + self.prev_ver

    def get_secrets(self, branch_name, is_circle):
        secret_to_location_mapping = {}
        if self.input_paths:
            secrets_file_paths = self.input_paths
        else:
            secrets_file_paths = self.get_all_diff_text_files(branch_name, is_circle)
        # If a input path supplied, should not run on git. If not supplied make sure not in middle of merge.
        if not run_command('git rev-parse -q --verify MERGE_HEAD') or self.input_paths:
            secret_to_location_mapping = self.search_potential_secrets(secrets_file_paths, self.ignore_entropy)
            if secret_to_location_mapping:
                secrets_found_string = 'Secrets were found in the following files:'
                for file_name in secret_to_location_mapping:
                    for line in sorted(secret_to_location_mapping[file_name]):
                        secrets_found_string += ('\nIn File: ' + f'{file_name}:{line}' + '\n')
                        if len(secret_to_location_mapping[file_name][line]) == 1:
                            secrets_found_string += f'Secret found: {secret_to_location_mapping[file_name][line][0]}\n'
                        else:
                            secrets_found_string += f'Secrets found: {secret_to_location_mapping[file_name][line]}\n'

                if not is_circle:
                    secrets_found_string += '\n\nRemove or whitelist secrets in order to proceed, then re-commit\n'

                else:
                    secrets_found_string += '\n\nThe secrets were exposed in public repository,' \
                                            ' remove the files asap and report it.\n'

                secrets_found_string += 'For more information about whitelisting visit: ' \
                                        'https://xsoar.pan.dev/docs/concepts/demisto-sdk#secrets'
                print_error(secrets_found_string)
        return secret_to_location_mapping

    def reformat_secrets_output(self, secrets_list):
        """
        Get a list of secrets and reformat it's output
        :param secrets_list: List of secrets
        :return: str: List of secrets
        """
        return '\n'.join(secrets_list) if secrets_list else ''

    def get_all_diff_text_files(self, branch_name, is_circle):
        """
        Get all new/modified text files that need to be searched for secrets
        :param branch_name: current branch being worked on
        :param is_circle: boolean to check if being ran from circle
        :return: list: list of text files
        """
        if is_circle:
            prev_ver = self.prev_ver
            if not prev_ver:
                self.git_util = GitUtil(repo=Content.git())
                prev_ver = self.git_util.handle_prev_ver()[1]
            if not prev_ver.startswith('origin'):
                prev_ver = 'origin/' + prev_ver
            print(f"Running secrets validation against {prev_ver}")

            changed_files_string = run_command(f"git diff --name-status {prev_ver}...{branch_name}")
        else:
            print("Running secrets validation on all changes")
            changed_files_string = run_command("git diff --name-status --no-merges HEAD")
        return list(self.get_diff_text_files(changed_files_string))

    def get_diff_text_files(self, files_string):
        """Filter out only added/modified text files from git diff
        :param files_string: string representing the git diff files
        :return: text_files_list: string of full path to text files
        """
        # file statuses to filter from the diff, no need to test deleted files.
        all_files = files_string.split('\n')
        text_files_list = set()
        for file_name in all_files:
            file_data: list = list(filter(None, file_name.split('\t')))
            if not file_data:
                continue
            file_status = file_data[0]
            if 'r' in file_status.lower():
                file_path = file_data[2]
            else:
                file_path = file_data[1]
            # only modified/added file, text readable, exclude white_list file
            if (file_status.lower() in ACCEPTED_FILE_STATUSES or 'r' in file_status.lower()) and self.is_text_file(
                    file_path):
                if not any(skipped_file in file_path for skipped_file in SKIPPED_FILES):
                    text_files_list.add(file_path)
        return text_files_list

    @staticmethod
    def is_text_file(file_path):
        file_extension = os.path.splitext(file_path)[1]
        if file_extension in TEXT_FILE_TYPES:
            return True
        return False

    def search_potential_secrets(self, secrets_file_paths: list, ignore_entropy: bool = False):
        """Returns potential secrets(sensitive data) found in committed and added files
        :param secrets_file_paths: paths of files that are being commited to git repo
        :param ignore_entropy: If True then will ignore running entropy algorithm for finding potential secrets

        :return: dictionary(filename: (list)secrets) of strings sorted by file name for secrets found in files
        """
        secret_to_location_mapping: DefaultDict[str, defaultdict] = defaultdict(lambda: defaultdict(list))
        for file_path in secrets_file_paths:
            # Get if file path in pack and pack name
            is_pack = is_file_path_in_pack(file_path)
            pack_name = get_pack_name(file_path)
            # Get generic/ioc/files white list sets based on if pack or not
            secrets_white_list, ioc_white_list, files_white_list = self.get_white_listed_items(is_pack, pack_name)
            # Skip white listed files

            if file_path in files_white_list:
                print("Skipping secrets detection for file: {} as it is white listed".format(file_path))
                continue
            # Init vars for current loop
            file_name = os.path.basename(file_path)
            _, file_extension = os.path.splitext(file_path)
            # get file contents
            file_contents = self.get_file_contents(file_path, file_extension)
            # if detected disable-secrets comments, removes the line/s
            file_contents = self.remove_secrets_disabled_line(file_contents)
            # in packs regard all items as regex as well, reset pack's whitelist in order to avoid repetition later
            if is_pack:
                file_contents = self.remove_whitelisted_items_from_file(file_contents, secrets_white_list)

            yml_file_contents = self.get_related_yml_contents(file_path)
            # Add all context output paths keywords to whitelist temporary
            if file_extension == YML_FILE_EXTENSION or yml_file_contents:
                temp_white_list = self.create_temp_white_list(yml_file_contents if yml_file_contents else file_contents)
                secrets_white_list = secrets_white_list.union(temp_white_list)
            # Search by lines after strings with high entropy / IoCs regex as possibly suspicious
            for line_num, line in enumerate(file_contents.split('\n')):
                # REGEX scanning for IOCs and false positive groups
                regex_secrets, false_positives = self.regex_for_secrets(line)
                for regex_secret in regex_secrets:
                    if not any(ioc.lower() in regex_secret.lower() for ioc in ioc_white_list):
                        secret_to_location_mapping[file_path][line_num + 1].append(regex_secret)
                # added false positives into white list array before testing the strings in line

                secrets_white_list = secrets_white_list.union(false_positives)

                if not ignore_entropy:
                    # due to nature of eml files, skip string by string secret detection - only regex
                    if file_extension in SKIP_FILE_TYPE_ENTROPY_CHECKS or \
                            any(demisto_type in file_name for demisto_type in SKIP_DEMISTO_TYPE_ENTROPY_CHECKS):
                        continue
                    line = self.remove_false_positives(line)
                    # calculate entropy for each string in the file
                    for string_ in line.split():
                        # compare the lower case of the string against both generic whitelist & temp white list
                        if not any(
                                white_list_string.lower() in string_.lower()
                                for white_list_string in secrets_white_list):

                            entropy = self.calculate_shannon_entropy(string_)
                            if entropy >= ENTROPY_THRESHOLD:
                                secret_to_location_mapping[file_path][line_num + 1].append(string_)

        return secret_to_location_mapping

    @staticmethod
    def remove_whitelisted_items_from_file(file_content: str, secrets_white_list: set) -> str:
        """Removes whitelisted items from file content

        Arguments:
            file_content (str): The content of the file to remove the whitelisted item from
            secrets_white_list (set): List of whitelist items to remove from the file content.

        Returns:
            str: The file content with the whitelisted items removed.
        """
        for item in secrets_white_list:
            try:
                file_content = re.sub(WHILEIST_REGEX.format(re.escape(item)), '', file_content)
            except re.error as err:
                error_string = f"Could not use secrets with item: {item}"
                print_error(error_string)
                raise re.error(error_string, str(err))
        return file_content

    @staticmethod
    def create_temp_white_list(file_contents) -> set:
        temp_white_list: set = set()
        context_paths = re.findall(r'contextPath: (\S+\.+\S+)', file_contents)
        for context_path in context_paths:
            context_path = context_path.split('.')
            context_path = [white_item.lower() for white_item in context_path if len(white_item) > 4]
            temp_white_list = temp_white_list.union(context_path)

        return temp_white_list

    def get_related_yml_contents(self, file_path):
        # if script or readme file, search for yml in order to retrieve temp white list
        yml_file_contents = ''
        # Validate if it is integration documentation file or supported file extension
        if find_type(file_path) in [FileType.PYTHON_FILE, FileType.README, FileType.POWERSHELL_FILE]:
            yml_file_contents = self.retrieve_related_yml(os.path.dirname(file_path))
        return yml_file_contents

    @staticmethod
    def retrieve_related_yml(integration_path):
        matching_yml_file_contents = None
        yml_file = os.path.join(integration_path, os.path.basename(integration_path) + '.yml')
        if os.path.exists(yml_file):
            with io.open(yml_file, mode="r", encoding="utf-8") as matching_yml_file:
                matching_yml_file_contents = matching_yml_file.read()
        return matching_yml_file_contents

    @staticmethod
    def regex_for_secrets(line):
        """Scans for IOCs with potentially low entropy score
        :param line: line to test as string representation (string)
        :return  potential_secrets (list) IOCs found via regex, false_positives (list) Non secrets with high entropy
        """
        potential_secrets = []
        false_positives = []

        # Dates REGEX for false positive preventing since they have high entropy
        dates = re.findall(DATES_REGEX, line)
        if dates:
            false_positives += [date[0].lower() for date in dates]
        # UUID REGEX - for false positives
        uuids = re.findall(UUID_REGEX, line)
        if uuids:
            false_positives += uuids
        # docker images version are detected as ips. so we ignore and whitelist them
        # example: dockerimage: demisto/duoadmin:1.0.0.147
        re_res = re.search(r'dockerimage:\s*\w*demisto/\w+:(\d+.\d+.\d+.\d+)', line)
        if re_res:
            docker_version = re_res.group(1)
            false_positives.append(docker_version)
            line = line.replace(docker_version, '')
        # URL REGEX
        urls = re.findall(URLS_REGEX, line)
        if urls:
            potential_secrets += urls
        # EMAIL REGEX
        emails = re.findall(EMAIL_REGEX, line)
        if emails:
            potential_secrets += emails
        # IPV6 REGEX
        ipv6_list = re.findall(IPV6_REGEX, line)
        if ipv6_list:
            for ipv6 in ipv6_list:
                if ipv6 != '::' and len(ipv6) > 4:
                    potential_secrets.append(ipv6)
        # IPV4 REGEX
        ipv4_list = re.findall(IPV4_REGEX, line)
        if ipv4_list:
            potential_secrets += ipv4_list

        return potential_secrets, false_positives

    @staticmethod
    def calculate_shannon_entropy(data) -> float:
        """Algorithm to determine the randomness of a given data.
        Higher is more random/complex, most English words will yield in average result of 3
        :param data: could be either a list/dict or a string.
        :return: entropy: entropy score.
        """
        if not data:
            return 0
        entropy = 0.0
        # each unicode code representation of all characters which are considered printable
        for char in (ord(c) for c in string.printable):
            # probability of event X
            p_x = float(data.count(chr(char))) / len(data)
            if p_x > 0:
                # the information in every possible news, in bits
                entropy += - p_x * math.log(p_x, 2)
        return entropy

    def get_white_listed_items(self, is_pack, pack_name):
        final_white_list, ioc_white_list, files_white_list = self.get_generic_white_list(self.white_list_path)
        if is_pack:
            pack_whitelist_path = os.path.join(PACKS_DIR, pack_name, PACKS_WHITELIST_FILE_NAME)
            pack_white_list, _, pack_files_white_list = self.get_packs_white_list(pack_whitelist_path, pack_name)
            final_white_list.extend(pack_white_list)
            files_white_list.extend(pack_files_white_list)

        final_white_list = set(final_white_list)
        if '' in final_white_list:
            # remove('') is ignoring empty lines in whitelists - users can accidentally add empty lines and those will
            # cause whitelisting of every string
            final_white_list.remove('')

        return final_white_list, set(ioc_white_list), set(files_white_list)

    @staticmethod
    def get_generic_white_list(whitelist_path):
        final_white_list = []
        ioc_white_list = []
        files_while_list = []
        if os.path.isfile(whitelist_path):
            with io.open(whitelist_path, mode="r", encoding="utf-8") as secrets_white_list_file:
                secrets_white_list_file = json.load(secrets_white_list_file)
                for name, white_list in secrets_white_list_file.items():  # type: ignore
                    if name == 'iocs':
                        for sublist in white_list:
                            ioc_white_list += [white_item for white_item in white_list[sublist] if len(white_item) > 4]
                        final_white_list += ioc_white_list
                    elif name == 'files':
                        files_while_list = white_list
                    else:
                        final_white_list += [white_item for white_item in white_list if len(white_item) > 4]

        return final_white_list, ioc_white_list, files_while_list

    @staticmethod
    def get_packs_white_list(whitelist_path, pack_name=None):
        final_white_list = []
        files_white_list = []

        if os.path.isfile(whitelist_path):
            with io.open(whitelist_path, mode="r", encoding="utf-8") as secrets_white_list_file:
                temp_white_list = secrets_white_list_file.read().split('\n')
            for white_list_line in temp_white_list:
                if white_list_line.startswith('file:'):
                    white_list_line = os.path.join(PACKS_DIR, pack_name, white_list_line[5:])
                    if not os.path.isfile(os.path.join(white_list_line)):
                        print_warning(f'{white_list_line} not found.\n'
                                      'please add the file name in the following format\n'
                                      'file:[Scripts|Integrations|Playbooks]/name/file.example\n'
                                      'e.g. file:Scripts/HelloWorldScript/HelloWorldScript.py')
                    files_white_list.append(white_list_line)
                else:
                    final_white_list.append(white_list_line)
        return final_white_list, [], files_white_list

    def get_file_contents(self, file_path, file_extension):
        try:
            # if pdf or README.md file, parse text
            integration_readme = re.match(pattern=PACKS_INTEGRATION_README_REGEX,
                                          string=file_path,
                                          flags=re.IGNORECASE)
            if file_extension == '.pdf':
                file_contents = self.extract_text_from_pdf(file_path)
            elif file_extension == '.md' and integration_readme:
                file_contents = self.extract_text_from_md_html(file_path)
            else:
                # Open each file, read its contents in UTF-8 encoding to avoid unicode characters
                with io.open(file_path, mode="r", encoding="utf-8", errors='ignore') as commited_file:
                    file_contents = commited_file.read()
            file_contents = self.ignore_base64(file_contents)
            return file_contents
        except Exception as ex:
            print("Failed opening file: {}. Exception: {}".format(file_path, ex))
            raise

    @staticmethod
    def extract_text_from_pdf(file_path):
        page_num = 0
        file_contents = ''
        try:
            pdf_file_obj = open('./' + file_path, 'rb')
            pdf_reader = PyPDF2.PdfFileReader(pdf_file_obj)
            num_pages = pdf_reader.numPages
        except PyPDF2.utils.PdfReadError:
            print('ERROR: Could not parse PDF file in path: {} - ***Review Manually***'.format(file_path))
            return file_contents
        while page_num < num_pages:
            pdf_page = pdf_reader.getPage(page_num)
            page_num += 1
            file_contents += pdf_page.extractText()

        return file_contents

    @staticmethod
    def extract_text_from_md_html(file_path):
        try:
            with open(file_path, mode='r') as html_page:
                soup = BeautifulSoup(html_page, features="html.parser")
                file_contents = soup.text
                return file_contents
        except Exception as ex:
            print_error('Unable to parse the following file {} due to error {}'.format(file_path, ex))
            raise

    @staticmethod
    def remove_false_positives(line):
        false_positive = re.search(r'([^\s]*[(\[{].*[)\]}][^\s]*)', line)
        if false_positive:
            false_positive_result = false_positive.group(1)
            line = line.replace(false_positive_result, '')
        return line

    @staticmethod
    def is_secrets_disabled(line, skip_secrets):
        if bool(re.findall(r'(disable-secrets-detection-start)', line)):
            skip_secrets['skip_multi'] = True
        elif bool(re.findall(r'(disable-secrets-detection-end)', line)):
            skip_secrets['skip_multi'] = False
        elif bool(re.findall(r'(disable-secrets-detection)', line)):
            skip_secrets['skip_once'] = True
        return skip_secrets

    @staticmethod
    def ignore_base64(file_contents):
        base64_strings = re.findall(r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|'
                                    r'[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})', file_contents)
        for base64_string in base64_strings:
            if len(base64_string) > 500:
                file_contents = file_contents.replace(base64_string, '')
        return file_contents

    @staticmethod
    def get_branch_name() -> str:
        branches = run_command('git branch')
        branch_name_reg = re.search(r'\* (.*)', branches)
        if not branch_name_reg:
            return ''
        return branch_name_reg.group(1)

    def find_secrets(self):
        print_color('Starting secrets detection', LOG_COLORS.GREEN)
        is_circle = self.is_circle
        branch_name = self.get_branch_name()
        secrets_found = self.get_secrets(branch_name, is_circle)
        if secrets_found:
            return True
        else:
            print_color('Finished validating secrets, no secrets were found.', LOG_COLORS.GREEN)
            return False

    def remove_secrets_disabled_line(self, file_content: str) -> str:
        """Removes lines that have "disable-secrets-detection" from file content

        Arguments:
            file_content (str): The content of the file to remove the "disable-secrets-detection" lines from

        Returns:
            str: The new file content with the "disable-secrets-detection" lines removed.
        """
        skip_secrets = {'skip_once': False, 'skip_multi': False}
        new_file_content = ""
        for line in file_content.split('\n'):
            skip_secrets = self.is_secrets_disabled(line, skip_secrets)
            if skip_secrets['skip_once'] or skip_secrets['skip_multi']:
                skip_secrets['skip_once'] = False
            else:
                new_file_content += f'{line}\n'
        return new_file_content

    def run(self):
        if self.find_secrets():
            return 1

        else:
            return 0


# demisto_sdk/commands/split/jsonsplitter.py

import os

import click

from demisto_sdk.commands.common.constants import DASHBOARDS_DIR, GENERIC_MODULES_DIR, PACKS_DIR
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.tools import get_pack_name, is_external_repository

json = JSON_Handler()


class JsonSplitter:
    """
    JsonSplitter is a class intended to split Generic Modules from their intrinsic dashboards

    Attributes:
        input (str): The path to the Generic module json file to split.
        output (str): The path to a directory in which the new dashboards should be put (and the module
                    if we create a new module file).
        no_auto_create_dir (bool): Whether to auto create new directories in content repo.
        no_logging (bool): Whether to print logs to the user.
        new_module_file (bool): Whether to create a new module file.
    """

    def __init__(self, input: str, output: str, no_auto_create_dir: bool = False,
                 no_logging: bool = False, new_module_file: bool = False):
        self.input = input
        self.dashboard_dir = output if output else ''
        self.module_dir = output if output else ''
        self.logging = not no_logging
        self.autocreate_dir = not no_auto_create_dir
        self.new_module_file = new_module_file

        with open(self.input, 'rb') as json_file:
            self.module_json_data = json.load(json_file)

    def split_json(self):
        if self.logging:
            click.secho(f"Starting dashboard extraction from generic module {self.module_json_data.get('name')}",
                        fg='bright_cyan')
        self.create_output_dirs()
        self.create_dashboards()
        self.create_module()
        return 0

    def create_output_dir(self, path):
        try:
            os.mkdir(path)
        except FileExistsError:
            pass

    def create_output_dirs(self):
        # no output given
        if not self.dashboard_dir:
            # check if in content repository
            if not is_external_repository() and self.autocreate_dir:
                pack_name = get_pack_name(self.input)
                if self.logging:
                    click.echo(f"No output path given creating Dashboards and GenericModules "
                               f"directories in pack {pack_name}")
                pack_path = os.path.join(PACKS_DIR, pack_name)
                self.dashboard_dir = os.path.join(pack_path, DASHBOARDS_DIR)
                self.module_dir = os.path.join(pack_path, GENERIC_MODULES_DIR)

                # create the dirs, dont fail if exist
                self.create_output_dir(self.dashboard_dir)
                self.create_output_dir(self.module_dir)

            # if not in content create the files locally
            else:
                if self.logging:
                    click.echo("No output path given and not running in content repo, creating "
                               "files in the current working directory")
                self.dashboard_dir = '.'
                self.module_dir = '.'

    def create_dashboards(self):
        if self.logging:
            click.echo("Starting dashboard creation")

        for view in self.module_json_data.get('views', []):
            for tab in view.get('tabs', []):
                dashboard_data = tab.get('dashboard')

                if dashboard_data:
                    dashboard_file_name = dashboard_data.get('name').replace(' ', '') + '.json'
                    full_dashboard_path = os.path.join(self.dashboard_dir, dashboard_file_name)

                    if self.logging:
                        click.echo(f"Creating dashboard: {full_dashboard_path}")

                    with open(full_dashboard_path, 'w') as dashboard_file:
                        json.dump(dashboard_data, dashboard_file, indent=4)

                    tab['dashboard'] = {'id': dashboard_data.get('id')}

    def create_module(self):
        if not self.new_module_file:
            if self.logging:
                click.echo(f"Updating module file {self.input}")
            module_file_path = self.input

        else:
            if self.logging:
                click.echo("Creating new module file")

            given_name = False
            while not given_name:
                file_name = str(input("\nPlease enter a new module file name: "))
                if not file_name or ' ' in file_name:
                    click.echo("File name cannot be empty nor have spaces in it")

                else:
                    given_name = True

                if not file_name.endswith('.json'):
                    file_name = file_name + '.json'

            module_file_path = os.path.join(self.module_dir, file_name)

        with open(module_file_path, 'w') as module_file:
            json.dump(self.module_json_data, module_file, indent=4)


# demisto_sdk/commands/split/ymlsplitter.py

import base64
import os
import re
import shutil
import subprocess
import tempfile
from io import open
from pathlib import Path
from typing import Optional

from ruamel.yaml.scalarstring import PlainScalarString, SingleQuotedScalarString

from demisto_sdk.commands.common.configuration import Configuration
from demisto_sdk.commands.common.constants import TYPE_PWSH, TYPE_PYTHON, TYPE_TO_EXTENSION
from demisto_sdk.commands.common.handlers import YAML_Handler
from demisto_sdk.commands.common.tools import (LOG_COLORS, get_all_docker_images, get_pipenv_dir, get_python_version,
                                               pascal_case, print_color, print_error)
from demisto_sdk.commands.unify.integration_script_unifier import IntegrationScriptUnifier

yaml = YAML_Handler()

REGEX_MODULE = r"### GENERATED CODE ###((.|\s)+?)### END GENERATED CODE ###"
INTEGRATIONS_DOCS_REFERENCE = 'https://xsoar.pan.dev/docs/reference/integrations/'


def get_pip_requirements(docker_image: str):
    return subprocess.check_output(["docker", "run", "--rm", docker_image,
                                    "pip", "freeze", "--disable-pip-version-check"],
                                   universal_newlines=True, stderr=subprocess.DEVNULL).strip()


class YmlSplitter:
    """YmlSplitter is a class that's designed to split a yml file to it's components.

    Attributes:
        input (str): input yml file path
        output (str): output path
        no_demisto_mock (bool): whether to add an import for demistomock
        no_common_server (bool): whether to add an import for common server
        no_auto_create_dir (bool): whether to create a dir
        base_name (str): the base name of all extracted files
        no_readme (bool): whether to extract readme
        no_pipenv (boo): whether to create pipenv
        no_code_formatting (bool): whether to avoid basic formatting on the code, i.e. autopep8 and isort
        file_type (str): yml file type (integration/script/modeling or parsing rule)
        configuration (Configuration): Configuration object
        lines_inserted_at_code_start (int): the amount of lines inserted at the beginning of the code file
    """

    def __init__(
            self,
            input: str,
            output: str = '',
            file_type: str = '',
            no_demisto_mock: bool = False,
            no_common_server: bool = False,
            no_auto_create_dir: bool = False,
            configuration: Configuration = None,
            base_name: str = '',
            no_readme: bool = False,
            no_pipenv: bool = False,
            no_logging: bool = False,
            no_code_formatting: bool = False,
            **_,  # ignoring unexpected kwargs
    ):
        self.input = Path(input).resolve()
        self.output = (Path(output) if output else Path(self.input.parent)).resolve()
        self.demisto_mock = not no_demisto_mock
        self.common_server = not no_common_server
        self.file_type = file_type
        self.base_name = base_name
        self.readme = not no_readme
        self.pipenv = not no_pipenv
        self.logging = not no_logging
        self.run_code_formatting: bool = not no_code_formatting
        self.lines_inserted_at_code_start = 0
        self.config = configuration or Configuration()
        self.auto_create_dir = not no_auto_create_dir
        with open(self.input, 'rb') as yml_file:
            self.yml_data = yaml.load(yml_file)
        self.api_module_path: Optional[str] = None

    def get_output_path(self):
        """Get processed output path
        """
        output_path = Path(self.output)
        if self.auto_create_dir and output_path.name in {'Integrations', 'Scripts', 'ModelingRules', 'ParsingRules'}:
            code_name = self.yml_data.get("name")
            if not code_name:
                raise ValueError(f'Failed determining Integration/Script/ModelingRule/ParsingRule name '
                                 f'when trying to auto create sub dir at: {self.output}'
                                 f'\nRun with option --no-auto-create-dir to skip auto creation of target dir.')
            output_path = output_path / pascal_case(code_name)
        return output_path

    def extract_to_package_format(self, executed_from_contrib_converter: bool = False) -> int:
        """Extracts the self.input yml file into several files according to the XSOAR standard of the package format.

        Returns:
             int. status code for the operation.
        """
        try:
            output_path = self.get_output_path()
        except ValueError as ex:
            print_error(str(ex))
            return 1
        self.print_logs("Starting migration of: {} to dir: {}".format(self.input, output_path),
                        log_color=LOG_COLORS.NATIVE)
        output_path.mkdir(parents=True, exist_ok=True)
        base_name = output_path.name if not self.base_name else self.base_name
        code_file = output_path / base_name
        self.extract_code(code_file, executed_from_contrib_converter)
        script = self.yml_data.get('script', {})
        lang_type: str = script.get('type', '') if self.file_type == 'integration' else self.yml_data.get('type')
        self.extract_image("{}/{}_image.png".format(output_path, base_name))
        self.extract_long_description("{}/{}_description.md".format(output_path, base_name))
        yaml_out = "{}/{}.yml".format(output_path, base_name)
        self.print_logs("Creating yml file: {} ...".format(yaml_out), log_color=LOG_COLORS.NATIVE)
        with open(self.input, 'r') as yf:
            yaml_obj = yaml.load(yf)
        script_obj = yaml_obj

        if self.file_type in ('modelingrule', 'parsingrule'):
            self.extract_rules(f'{output_path}/{base_name}.xif')
            if 'rules' in yaml_obj:
                yaml_obj['rules'] = PlainScalarString('')
            if 'schema' in yaml_obj:
                self.extract_rule_schema_and_samples(f'{output_path}/{base_name}_schema.json')
                yaml_obj['schema'] = PlainScalarString('')
            if 'samples' in yaml_obj:
                self.extract_rule_schema_and_samples(f'{output_path}/{base_name}.json')
                del yaml_obj['samples']
            with open(yaml_out, 'w') as yf:
                yaml.dump(yaml_obj, yf)
        else:
            code_file = f"{code_file}{TYPE_TO_EXTENSION[lang_type]}"
            if self.file_type == 'integration':
                script_obj = yaml_obj['script']
                if 'image' in yaml_obj:
                    del yaml_obj['image']
                if 'detaileddescription' in yaml_obj:
                    del yaml_obj['detaileddescription']
            script_obj['script'] = SingleQuotedScalarString('')
            code_type = script_obj['type']
            if code_type == TYPE_PWSH and not yaml_obj.get('fromversion'):
                self.print_logs("Setting fromversion for PowerShell to: 5.5.0", log_color=LOG_COLORS.NATIVE)
                yaml_obj['fromversion'] = "5.5.0"
            with open(yaml_out, 'w') as yf:
                yaml.dump(yaml_obj, yf)
            # check if there is a README and if found, set found_readme to True
            found_readme = False
            if self.readme:
                yml_readme = self.input.parent / f'{self.input.stem}_README.md'
                readme = output_path / 'README.md'
                if yml_readme.exists():
                    found_readme = True
                    self.print_logs(f"Copying {readme} to {readme}", log_color=LOG_COLORS.NATIVE)
                    shutil.copy(yml_readme, readme)
                else:
                    # open an empty file
                    with open(readme, 'w'):
                        pass

            # Python code formatting and dev env setup
            if code_type == TYPE_PYTHON:
                if self.run_code_formatting:
                    self.print_logs("Running autopep8 on file: {} ...".format(code_file), log_color=LOG_COLORS.NATIVE)
                    try:
                        subprocess.call(["autopep8", "-i", "--max-line-length", "130", code_file])
                    except FileNotFoundError:
                        self.print_logs("autopep8 skipped! It doesn't seem you have autopep8 installed.\n"
                                        "Make sure to install it with: pip install autopep8.\n"
                                        "Then run: autopep8 -i {}".format(code_file), LOG_COLORS.YELLOW)

                    self.print_logs("Running isort on file: {} ...".format(code_file), LOG_COLORS.NATIVE)
                    try:
                        subprocess.call(["isort", code_file])
                    except FileNotFoundError:
                        self.print_logs("isort skipped! It doesn't seem you have isort installed.\n"
                                        "Make sure to install it with: pip install isort.\n"
                                        "Then run: isort {}".format(code_file), LOG_COLORS.YELLOW)
                if self.pipenv:
                    try:
                        self.print_logs("Detecting python version and setting up pipenv files ...", log_color=LOG_COLORS.NATIVE)
                        docker = get_all_docker_images(script_obj)[0]
                        py_ver = get_python_version(docker, self.config.log_verbose)
                        pip_env_dir = get_pipenv_dir(py_ver, self.config.envs_dirs_base)
                        self.print_logs("Copying pipenv files from: {}".format(pip_env_dir), log_color=LOG_COLORS.NATIVE)
                        shutil.copy("{}/Pipfile".format(pip_env_dir), output_path)
                        shutil.copy("{}/Pipfile.lock".format(pip_env_dir), output_path)
                        env = os.environ.copy()
                        env["PIPENV_IGNORE_VIRTUALENVS"] = "1"
                        try:
                            subprocess.call(["pipenv", "install", "--dev"], cwd=output_path, env=env)
                            self.print_logs("Installing all py requirements from docker: [{}] into pipenv".format(docker),
                                            LOG_COLORS.NATIVE)
                            requirements = get_pip_requirements(docker)
                            fp = tempfile.NamedTemporaryFile(delete=False)
                            fp.write(requirements.encode('utf-8'))
                            fp.close()

                            try:
                                subprocess.check_call(["pipenv", "install", "-r", fp.name], cwd=output_path, env=env)

                            except Exception:
                                self.print_logs("Failed installing requirements in pipenv.\n "
                                                "Please try installing manually after extract ends\n", LOG_COLORS.RED)

                            os.unlink(fp.name)
                            self.print_logs("Installing flake8 for linting", log_color=LOG_COLORS.NATIVE)
                            subprocess.call(["pipenv", "install", "--dev", "flake8"], cwd=output_path, env=env)
                        except FileNotFoundError as err:
                            self.print_logs("pipenv install skipped! It doesn't seem you have pipenv installed.\n"
                                            "Make sure to install it with: pip3 install pipenv.\n"
                                            f"Then run in the package dir: pipenv install --dev\n.Err: {err}", LOG_COLORS.YELLOW)
                        arg_path = output_path.relative_to()
                        self.print_logs("\nCompleted: setting up package: {}\n".format(arg_path), LOG_COLORS.GREEN)
                        next_steps: str = "Next steps: \n" \
                                          "* Install additional py packages for unit testing (if needed): cd {};" \
                                          " pipenv install <package>\n".format(arg_path) if code_type == TYPE_PYTHON else ''
                        next_steps += "* Create unit tests\n" \
                                      "* Check linting and unit tests by running: demisto-sdk lint -i {}\n".format(arg_path)
                        next_steps += "* When ready, remove from git the old yml and/or README and add the new package:\n" \
                                      "    git rm {}\n".format(self.input)
                        if found_readme:
                            next_steps += "    git rm {}\n".format(self.input.parent / f'{self.input.stem}_README.md')
                        next_steps += "    git add {}\n".format(arg_path)
                        self.print_logs(next_steps, log_color=LOG_COLORS.NATIVE)

                    except Exception:
                        self.print_logs("An unexpected error has occurred while trying to install "
                                        "the requirements in pipenv.\n"
                                        "Please try installing manually after extract ends.\n", LOG_COLORS.RED)

                else:
                    self.print_logs("Skipping pipenv and requirements installation - Note: no Pipfile will be created",
                                    log_color=LOG_COLORS.YELLOW)

        self.print_logs(f"Finished splitting the yml file - you can find the split results here: {output_path}",
                        log_color=LOG_COLORS.GREEN)
        return 0

    def extract_code(self, code_file_path, executed_from_contrib_converter: bool = False) -> int:
        """Extracts the code from the yml_file.
        If code_file_path doesn't contain the proper extension will add it.

        Returns:
             int. status code for the operation.
        """
        common_server = self.common_server
        if common_server:
            common_server = "CommonServerPython" not in str(self.input) and 'CommonServerPowerShell' not in str(self.input)
        if self.file_type == 'modelingrule' or self.file_type == 'parsingrule':
            # no need to extract code
            return 0

        script = self.yml_data['script']
        if self.file_type == 'integration':  # in integration the script is stored at a second level
            lang_type = script['type']
            script = script['script']
        else:
            lang_type = self.yml_data['type']
        ext = TYPE_TO_EXTENSION[lang_type]
        code_file_path = code_file_path.with_suffix(ext)
        self.print_logs("Extracting code to: {} ...".format(code_file_path), log_color=LOG_COLORS.NATIVE)
        with open(code_file_path, 'wt') as code_file:
            if lang_type == TYPE_PYTHON and self.demisto_mock:
                code_file.write("import demistomock as demisto  # noqa: F401\n")
                self.lines_inserted_at_code_start += 1
            if common_server:
                if lang_type == TYPE_PYTHON:
                    code_file.write("from CommonServerPython import *  # noqa: F401\n")
                    self.lines_inserted_at_code_start += 1
                if lang_type == TYPE_PWSH:
                    code_file.write(". $PSScriptRoot\\CommonServerPowerShell.ps1\n")
                    self.lines_inserted_at_code_start += 1
            script = self.replace_imported_code(script, executed_from_contrib_converter)
            script = self.replace_section_headers_code(script)
            code_file.write(script)
            if script and script[-1] != '\n':
                # make sure files end with a new line (pyml seems to strip the last newline)
                code_file.write("\n")
        return 0

    def extract_image(self, output_path) -> int:
        """Extracts the image from the yml_file.

        Returns:
             int. status code for the operation.
        """
        if self.file_type == 'script':
            return 0  # no image in script type
        self.print_logs("Extracting image to: {} ...".format(output_path), log_color=LOG_COLORS.NATIVE)
        im_field = self.yml_data.get('image')
        if im_field and len(im_field.split(',')) >= 2:
            image_b64 = self.yml_data['image'].split(',')[1]
            with open(output_path, 'wb') as image_file:
                image_file.write(base64.decodebytes(image_b64.encode('utf-8')))
        return 0

    def remove_integration_documentation(self, detailed_description):
        if "[View Integration Documentation]" in detailed_description:
            normalized_integration_id = IntegrationScriptUnifier.normalize_integration_id(self.yml_data['commonfields']['id'])
            integration_doc_link = INTEGRATIONS_DOCS_REFERENCE + normalized_integration_id
            documentation = f'[View Integration Documentation]({integration_doc_link})'
            if '\n\n---\n' + documentation in detailed_description:
                detailed_description = detailed_description.replace('\n\n---\n' + documentation, "")
            elif documentation in detailed_description:
                detailed_description = detailed_description.replace(documentation, "")

        return detailed_description

    def extract_long_description(self, output_path) -> int:
        """Extracts the detailed description from the yml_file.

        Returns:
             int. status code for the operation.
        """
        if self.file_type == 'script':
            return 0  # no long description in script type
        long_description = self.yml_data.get('detaileddescription')
        if long_description:
            long_description = self.remove_integration_documentation(long_description)
            self.print_logs("Extracting long description to: {} ...".format(output_path), log_color=LOG_COLORS.NATIVE)
            with open(output_path, 'w', encoding='utf-8') as desc_file:
                desc_file.write(long_description)
        return 0

    def extract_rules(self, output_path) -> int:
        """Extracts the parsing and modeling rules from the yml_file.

        Returns:
             int. status code for the operation.
        """
        rules = self.yml_data.get('rules')
        if rules:
            self.print_logs("Extracting rules to: {} ...".format(output_path), log_color=LOG_COLORS.NATIVE)
            with open(output_path, 'w', encoding='utf-8') as rules_file:
                rules_file.write(rules)
        return 0

    def extract_rule_schema_and_samples(self, output_path) -> int:
        """
        Extracts the schema of the modeling rules from the yml_file.
        Extracts the samples of the parsing rules from the yml_file.
        Returns:
             int. status code for the operation.
        """
        # Modeling rules
        schema = self.yml_data.get('schema')
        if schema:
            self.print_logs("Extracting rules schema to: {} ...".format(output_path), log_color=LOG_COLORS.NATIVE)
            with open(output_path, 'w', encoding='utf-8') as rules_file:
                rules_file.write(schema)

        # Parsing rules
        samples = self.yml_data.get('samples')
        if samples:
            self.print_logs("Extracting rules samples to: {} ...".format(output_path), log_color=LOG_COLORS.NATIVE)
            with open(output_path, 'w', encoding='utf-8') as rules_file:
                rules_file.write(samples)
        return 0

    def print_logs(self, log_msg: str, log_color: str) -> None:
        """
        Prints the logging message if logging is enabled
        :param log_msg: The logging message
        :param log_color: The printing color
        :return: None
        """
        if self.logging:
            print_color(log_msg, log_color)

    def update_api_module_contribution(self, lines: list, imported_line: str):
        """
            save the api module changes done by the contributor to the api module file before it is replaced in the
            integration code.
            :param lines: the integration lines.
            :param imported_line: the imported line in the code, represents the Api Module used.
            :return: None
        """
        imported_line_arr = imported_line.split(' ')  # example: imported_line = from CorIRApiModule import *
        updated_lines = lines[4: -3]  # ignore first 4 lines and last 3 line.
        if len(imported_line_arr) >= 3 and imported_line_arr[0] == 'from' and imported_line_arr[2] == 'import':
            module_name = imported_line_arr[1]
            self.api_module_path = os.path.join('./Packs', 'ApiModules', 'Scripts', module_name, module_name + '.py')
            with open(self.api_module_path, 'w') as f:
                f.write('from CommonServerPython import *  # noqa: F401\n')
                f.write('import demistomock as demisto  # noqa: F401\n')
                f.write('\n'.join(updated_lines))

    def replace_imported_code(self, script, executed_from_contrib_converter: bool = False):
        # this is how we check that generated code exists, and the syntax of the generated code is up to date
        if '### GENERATED CODE ###:' in script and \
                '### END GENERATED CODE ###' in script:
            matches = re.finditer(REGEX_MODULE, script)
            for match in matches:
                code = match.group(1)
                lines = code.split('\n')
                imported_line = lines[0][2:]  # the first two chars are not part of the code
                if executed_from_contrib_converter:
                    self.update_api_module_contribution(lines, imported_line)
                self.print_logs(f'Replacing code block with `{imported_line}`', LOG_COLORS.NATIVE)
                script = script.replace(match.group(), imported_line)
        return script

    def replace_section_headers_code(self, script):
        """
        remove the auto-generated section headers if they exist.
        """
        return re.sub(r"register_module_line\('.+', '(?:start|end)', __line__\(\)\)\n", '', script)


# demisto_sdk/commands/unify/generic_module_unifier.py

import os
import sys
from typing import Dict, Optional

import click

from demisto_sdk.commands.common.constants import PACKS_DIR, FileType
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.tools import find_type, get_pack_name

json = JSON_Handler()


class GenericModuleUnifier:
    """
    Unifies a GenericModule object with it's Dashboards
    """

    def __init__(self, input: str, output: str = '', force: bool = False, marketplace: Optional[str] = None):
        """
        Init a GenericModuleUnifier
        Args:
            input: a path of the GenericModule file to unify.
            output: The output dir to write the unified GenericModule json to.
            force: if True - Forcefully overwrites the preexisting unified GenericModule file if one exists.
        """

        self.input_path = input
        self.pack_name = get_pack_name(file_path=self.input_path)
        self.pack_path = os.path.join(PACKS_DIR, self.pack_name)

        self.input_file_name = os.path.basename(self.input_path).rstrip('.json')
        self.use_force = force
        self.marketplace = marketplace

        if output:
            if not os.path.isdir(output):
                click.secho('You have failed to provide a legal dir path', fg='bright_red')
                sys.exit(1)

            self.dest_dir = output

        else:
            # an output wasn't given, save the unified file in the input's file dir
            self.dest_dir = os.path.dirname(self.input_path)

        self.dest_path = os.path.join(self.dest_dir, f'{self.input_file_name}_unified.json')

    def find_dashboard_by_id(self, dashboard_id: str) -> Optional[Dict]:
        """
        Search for a dashboard with the given id in the relevant pack path.
        Args:
            dashboard_id: dashboard id to search for

        Returns: if found - the content of the dashboard, else - None.
        """
        dashboards_dir_path = f'{self.pack_path}/Dashboards/'
        for file_name in os.listdir(dashboards_dir_path):
            file_path = os.path.join(dashboards_dir_path, file_name)
            if find_type(file_path) == FileType.DASHBOARD:
                # it's a dashboard
                with open(file_path) as f:
                    dashboard = json.load(f)
                if dashboard.get('id') == dashboard_id:
                    # the searched dashboard was found
                    return dashboard
        return None

    def merge_generic_module_with_its_dashboards(self) -> Dict:
        """
        Unifies a GenericModule object with it's Dashboards

        Returns: the unified GenericModule
        """
        with open(self.input_path) as f:
            generic_module = json.load(f)

        views = generic_module.get('views', [])
        for view in views:
            tabs = view.get('tabs', [])
            for tab in tabs:
                dashboard_id = tab.get('dashboard', {}).get('id')
                if dashboard_id:
                    # search dashboard in the GenericModule's pack
                    dashboard_content = self.find_dashboard_by_id(dashboard_id=dashboard_id)
                    if dashboard_content:
                        tab['dashboard'] = dashboard_content

                    else:
                        click.secho(f'Dashboard {dashboard_id} was not found in pack: {self.pack_name} '
                                    f'and therefore was not unified', fg="bright_red")

        self.save_unified_generic_module(generic_module)

        return generic_module

    def save_unified_generic_module(self, unified_generic_module_json: Dict):
        """
        Save the unified GenericModule to a json file.
        Args:
            unified_generic_module_json: unified GenericModule

        """
        if os.path.isfile(self.dest_path) and self.use_force is False:
            raise ValueError(f'Output file already exists: {self.dest_path}.'
                             ' Make sure to remove this file from source control, set a different output dir or set the'
                             '-f argument to True in order to overwrite the preexisting file.')

        with open(self.dest_path, mode='w') as file:
            json.dump(unified_generic_module_json, file, indent=4)


# demisto_sdk/commands/unify/integration_script_unifier.py

import base64
import copy
import glob
import io
import os
import re
import shutil
from typing import Dict, List, Optional, Union

import click
from inflection import dasherize, underscore
from ruamel.yaml.scalarstring import FoldedScalarString

from demisto_sdk.commands.common.constants import (DEFAULT_CONTENT_ITEM_FROM_VERSION, DEFAULT_CONTENT_ITEM_TO_VERSION,
                                                   DEFAULT_IMAGE_PREFIX, INTEGRATIONS_DIR, SCRIPTS_DIR,
                                                   TYPE_TO_EXTENSION, FileType)
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.tools import (LOG_COLORS, arg_to_list, find_type, get_mp_tag_parser, get_pack_name,
                                               get_yaml, get_yml_paths_in_dir, print_color, print_warning,
                                               server_version_compare)
from demisto_sdk.commands.unify.yaml_unifier import YAMLUnifier

json = JSON_Handler()

PACK_METADATA_PATH = 'pack_metadata.json'
CONTRIBUTOR_DISPLAY_NAME = ' ({} Contribution)'
CONTRIBUTOR_DETAILED_DESC = '### {} Contributed Integration\n' \
                            '#### Integration Author: {}\n' \
                            'Support and maintenance for this integration are provided by the author. ' \
                            'Please use the following contact details:'

CONTRIBUTOR_COMMUNITY_DETAILED_DESC = '### Community Contributed Integration\n ' \
                                      '#### Integration Author: {}\n ' \
                                      'No support or maintenance is provided by the author. Customers are encouraged ' \
                                      'to engage with the user community for questions and guidance at the ' \
                                      '[Cortex XSOAR Live Discussions](https://live.paloaltonetworks.com/' \
                                      't5/cortex-xsoar-discussions/bd-p/Cortex_XSOAR_Discussions).'

CONTRIBUTORS_LIST = ['partner', 'developer', 'community']
COMMUNITY_CONTRIBUTOR = 'community'
INTEGRATIONS_DOCS_REFERENCE = 'https://xsoar.pan.dev/docs/reference/integrations/'


class IntegrationScriptUnifier(YAMLUnifier):

    def __init__(self, input: str, dir_name=INTEGRATIONS_DIR, output: str = '',
                 image_prefix=DEFAULT_IMAGE_PREFIX, force: bool = False, yml_modified_data=None, custom: str = '',
                 marketplace: Optional[str] = None):

        self.image_prefix = image_prefix
        self.custom = custom
        if yml_modified_data:
            self.yml_data = yml_modified_data

        super().__init__(
            input=input,
            output=output,
            force=force,
            marketplace=marketplace,
        )

        # script key for scripts is a string.
        # script key for integrations is a dictionary.
        self.is_script_package = isinstance(self.yml_data.get('script'), str)
        self.dir_name = SCRIPTS_DIR if self.is_script_package else dir_name

    def write_yaml_with_docker(self, yml_unified, yml_data, script_obj):
        """Write out the yaml file taking into account the dockerimage45 tag.
        If it is present will create 2 integration files
        One for 4.5 and below and one for 5.0.

        Arguments:
            output_path {str} -- output path
            yml_unified {dict} -- unified yml dict
            yml_data {dict} -- yml object
            script_obj {dict} -- script object

        Returns:
            dict -- dictionary mapping output path to unified data
        """
        output_map = {self.dest_path: yml_unified}
        if 'dockerimage45' in script_obj:
            # we need to split into two files 45 and 50. Current one will be from version 5.0
            if self.is_script_package:  # scripts
                del yml_unified['dockerimage45']
            else:  # integrations
                del yml_unified['script']['dockerimage45']

            yml_unified45 = copy.deepcopy(yml_unified)

            # validate that this is a script/integration which targets both 4.5 and 5.0+.
            if server_version_compare(yml_data.get('fromversion', DEFAULT_CONTENT_ITEM_FROM_VERSION), '5.0.0') >= 0:
                raise ValueError(f'Failed: {self.dest_path}. dockerimage45 set for 5.0 and later only')

            yml_unified['fromversion'] = '5.0.0'

            # validate that this is a script/integration which targets both 4.5 and 5.0+.
            if server_version_compare(yml_data.get('toversion', DEFAULT_CONTENT_ITEM_TO_VERSION), '5.0.0') < 0:
                raise ValueError(f'Failed: {self.dest_path}. dockerimage45 set for 4.5 and earlier only')

            yml_unified45['toversion'] = '4.5.9'

            if script_obj.get('dockerimage45'):  # we have a value for dockerimage45 set it as dockerimage
                if self.is_script_package:  # scripts
                    yml_unified45['dockerimage'] = script_obj.get('dockerimage45')
                else:  # integrations
                    yml_unified45['script']['dockerimage'] = script_obj.get('dockerimage45')

            else:  # no value for dockerimage45 remove the dockerimage entry
                del yml_unified45['dockerimage']

            output_path45 = re.sub(r'\.yml$', '_45.yml', self.dest_path)  # type: ignore[arg-type]
            output_map = {
                self.dest_path: yml_unified,
                output_path45: yml_unified45,
            }
        for file_path, file_data in output_map.items():
            self._output_yaml(file_path, file_data)

        return output_map

    def unify(self, file_name_suffix=None):
        print(f"Merging package: {self.package_path}")
        self._set_dest_path(file_name_suffix)

        script_obj = self.yml_data

        if not self.is_script_package:  # integration
            self.update_hidden_parameters_value()  # changes self.yml_data
            script_obj = self.yml_data['script']
        script_type = TYPE_TO_EXTENSION[script_obj['type']]

        yml_unified = copy.deepcopy(self.yml_data)

        yml_unified, script_path = self.insert_script_to_yml(script_type, yml_unified, self.yml_data)
        image_path = None
        desc_path = None
        if not self.is_script_package:
            yml_unified, image_path = self.insert_image_to_yml(self.yml_data, yml_unified)
            yml_unified, desc_path = self.insert_description_to_yml(self.yml_data, yml_unified)
            contributor_type, metadata_data = self.get_contributor_data()

            if self.is_contributor_pack(contributor_type):
                contributor_email = metadata_data.get('email', '')
                contributor_url = metadata_data.get('url', '')
                author = metadata_data.get('author')
                yml_unified = self.add_contributors_support(yml_unified, contributor_type, contributor_email,
                                                            contributor_url, author)

        if self.custom:
            yml_unified = self.add_custom_section(yml_unified)

        output_map = self.write_yaml_with_docker(yml_unified, self.yml_data, script_obj)
        unifier_outputs = list(output_map.keys()), self.yml_path, script_path, image_path, desc_path
        print_color(f'Created unified yml: {list(output_map.keys())}', LOG_COLORS.GREEN)

        return unifier_outputs[0]

    def update_hidden_parameters_value(self) -> None:
        """
        The `hidden` attribute of each param may be a bool (affecting both marketplaces),
        or a list of marketplaces where this param is hidden.

        This method replaces a list with a boolean, according to the self.marketplace value.
        Boolean values are left untouched.
        """
        if not self.marketplace:
            return

        for i, param in enumerate(self.yml_data.get('configuration', ())):
            if isinstance(hidden := (param.get('hidden')), list):
                # converts list to bool
                self.yml_data['configuration'][i]['hidden'] = self.marketplace in hidden

    def add_custom_section(self, unified_yml: Dict) -> Dict:
        """
            Args:
                unified_yml - The unified_yml
            Returns:
                 the unified yml with the id/name/display appended with the custom label
                 if the fields exsits.
        """
        to_append = f' - {self.custom}'
        if unified_yml.get('name'):
            unified_yml['name'] += to_append
        if unified_yml.get('commonfields', {}).get('id'):
            unified_yml['commonfields']['id'] += to_append
        if not self.is_script_package:
            if unified_yml.get('display'):
                unified_yml['display'] += to_append

        return unified_yml

    def insert_image_to_yml(self, yml_data, yml_unified):
        image_data, found_img_path = self.get_data(self.package_path, "*png")
        if image_data:
            image_data = self.image_prefix + base64.b64encode(image_data).decode('utf-8')

            if yml_data.get('image') and self.use_force is False:
                raise ValueError('Please move the image from the yml to an image file (.png)'
                                 f' in the package: {self.package_path}')

            yml_unified['image'] = image_data
        else:
            click.secho(f'Failed getting image data for {self.package_path}', fg="yellow")

        return yml_unified, found_img_path

    def insert_description_to_yml(self, yml_data, yml_unified):
        desc_data, found_desc_path = self.get_data(self.package_path, '*_description.md')

        if yml_data.get('detaileddescription') and self.use_force is False:
            raise ValueError('Please move the detailed description from the yml to a description file (.md)'
                             f' in the package: {self.package_path}')

        detailed_description = ''
        if desc_data:
            detailed_description = get_mp_tag_parser().parse_text(FoldedScalarString(desc_data.decode('utf-8')))

        integration_doc_link = ''
        if '[View Integration Documentation]' not in detailed_description:
            integration_doc_link = self.get_integration_doc_link(yml_data)
        if integration_doc_link:
            if detailed_description:
                detailed_description += '\n\n---\n' + integration_doc_link
            else:
                detailed_description += integration_doc_link
        if detailed_description:
            yml_unified['detaileddescription'] = detailed_description
        return yml_unified, found_desc_path

    def get_data(self, path, extension):
        data_path = glob.glob(os.path.join(path, extension))
        data = None
        found_data_path = None
        if not self.is_script_package and data_path:
            found_data_path = data_path[0]
            with open(found_data_path, 'rb') as data_file:
                data = data_file.read()

        return data, found_data_path

    def get_code_file(self, script_type):
        """Return the first code file in the specified directory path
        :param script_type: script type: .py, .js, .ps1
        :type script_type: str
        :return: path to found code file
        :rtype: str
        """

        ignore_regex = (r'CommonServerPython\.py|CommonServerUserPython\.py|demistomock\.py|_test\.py'
                        r'|conftest\.py|__init__\.py|ApiModule\.py|vulture_whitelist\.py'
                        r'|CommonServerPowerShell\.ps1|CommonServerUserPowerShell\.ps1|demistomock\.ps1|\.Tests\.ps1')
        if self.package_path.endswith('/'):
            self.package_path = self.package_path[:-1]  # remove the last / as we use os.path.join
        if self.package_path.endswith(os.path.join('Scripts', 'CommonServerPython')):
            return os.path.join(self.package_path, 'CommonServerPython.py')
        if self.package_path.endswith(os.path.join('Scripts', 'CommonServerUserPython')):
            return os.path.join(self.package_path, 'CommonServerUserPython.py')
        if self.package_path.endswith(os.path.join('Scripts', 'CommonServerPowerShell')):
            return os.path.join(self.package_path, 'CommonServerPowerShell.ps1')
        if self.package_path.endswith(os.path.join('Scripts', 'CommonServerUserPowerShell')):
            return os.path.join(self.package_path, 'CommonServerUserPowerShell.ps1')
        if self.package_path.endswith('ApiModule'):
            return os.path.join(self.package_path, os.path.basename(os.path.normpath(self.package_path)) + '.py')

        script_path_list = list(filter(lambda x: not re.search(ignore_regex, x, flags=re.IGNORECASE),
                                       sorted(glob.glob(os.path.join(self.package_path, '*' + script_type)))))
        if script_path_list:
            return script_path_list[0]
        else:
            raise Exception("the provided code file is not supported")

    def insert_script_to_yml(self, script_type, yml_unified, yml_data):
        script_path = self.get_code_file(script_type)
        with io.open(script_path, mode='r', encoding='utf-8') as script_file:
            script_code = script_file.read()

        # Check if the script imports an API module. If it does,
        # the API module code will be pasted in place of the import.
        imports_to_names = self.check_api_module_imports(script_code)
        script_code = self.insert_module_code(script_code, imports_to_names)

        if script_type == '.py':
            clean_code = self.clean_python_code(script_code)
            if 'CommonServer' not in yml_data['name']:
                # CommonServerPython has those line hard-coded so there is no need to add them here.
                clean_code = f"register_module_line('{yml_data['name']}', 'start', __line__())\n" \
                    f'{clean_code}\n' \
                    f"register_module_line('{yml_data['name']}', 'end', __line__())\n"
        elif script_type == '.ps1':
            clean_code = self.clean_pwsh_code(script_code)
        else:
            # for JS scripts
            clean_code = script_code

        if self.is_script_package:
            if yml_data.get('script', '') not in ('', '-'):
                print_warning(f'Script section is not empty in package {self.package_path}.'
                              f'It should be blank or a dash(-).')

            yml_unified['script'] = FoldedScalarString(clean_code)

        else:
            if yml_data['script'].get('script', '') not in ('', '-'):
                print_warning(f'Script section is not empty in package {self.package_path}.'
                              f'It should be blank or a dash(-).')

            yml_unified['script']['script'] = FoldedScalarString(clean_code)

        return yml_unified, script_path

    def get_script_or_integration_package_data(self):
        # should be static method
        _, yml_path = get_yml_paths_in_dir(self.package_path, error_msg='')

        if not yml_path:
            raise Exception(f'No yml files found in package path: {self.package_path}. '
                            'Is this really a package dir?')

        if find_type(yml_path) in (FileType.SCRIPT, FileType.TEST_SCRIPT):
            code_type = get_yaml(yml_path).get('type')
        else:
            code_type = get_yaml(yml_path).get('script', {}).get('type')
        unifier = IntegrationScriptUnifier(self.package_path)
        code_path = unifier.get_code_file(TYPE_TO_EXTENSION[code_type])
        with io.open(code_path, 'r', encoding='utf-8') as code_file:
            code = code_file.read()

        return yml_path, code

    @staticmethod
    def check_api_module_imports(script_code: str) -> Dict[str, str]:
        """
        Checks integration code for API module imports
        :param script_code: The integration code
        :return: The import string and the imported module name
        """

        # General regex to find API module imports, for example: "from MicrosoftApiModule import *  # noqa: E402"
        module_regex = r'from ([\w\d]+ApiModule) import \*(?:  # noqa: E402)?'

        module_matches = re.finditer(module_regex, script_code)

        return {module_match.group(): module_match.group(1) for module_match in module_matches}

    @staticmethod
    def insert_module_code(script_code: str, import_to_name: Dict[str, str]) -> str:
        """
        Inserts API module in place of an import to the module according to the module name
        :param script_code: The integration code
        :param import_to_name: A dictionary where the keys are The module import string to replace
        and the values are The module name
        :return: The integration script with the module code appended in place of the import
        """
        for module_import, module_name in import_to_name.items():

            module_path = os.path.join('./Packs', 'ApiModules', 'Scripts', module_name, module_name + '.py')
            module_code = IntegrationScriptUnifier._get_api_module_code(module_name, module_path)

            # the wrapper numbers represents the number of generated lines added
            # before (negative) or after (positive) the registration line
            module_code = f'\n### GENERATED CODE ###: {module_import}\n' \
                          f'# This code was inserted in place of an API module.\n' \
                          f"register_module_line('{module_name}', 'start', __line__(), wrapper=-3)\n" \
                          f'{module_code}\n' \
                          f"register_module_line('{module_name}', 'end', __line__(), wrapper=1)\n" \
                          f'### END GENERATED CODE ###'

            script_code = script_code.replace(module_import, module_code)
        return script_code

    @staticmethod
    def _get_api_module_code(module_name, module_path):
        """
        Attempts to get the API module code from the ApiModules pack.
        :param module_name: The API module name
        :param module_path: The API module code file path
        :return: The API module code
        """
        try:
            with io.open(module_path, mode='r', encoding='utf-8') as script_file:
                module_code = script_file.read()
        except Exception as exc:
            raise ValueError('Could not retrieve the module [{}] code: {}'.format(module_name, str(exc)))

        return module_code

    @staticmethod
    def clean_python_code(script_code, remove_print_future=True):
        # we use '[ \t]' and not \s as we don't want to match newline
        script_code = re.sub(r'import demistomock as demisto[ \t]*(#.*)?', "", script_code)
        script_code = re.sub(r'from CommonServerPython import \*[ \t]*(#.*)?', "", script_code)
        script_code = re.sub(r'from CommonServerUserPython import \*[ \t]*(#.*)?', "", script_code)
        # print function is imported in python loop
        if remove_print_future:  # docs generation requires to leave this
            script_code = re.sub(r'from __future__ import print_function[ \t]*(#.*)?', "", script_code)
        return script_code

    @staticmethod
    def clean_pwsh_code(script_code):
        script_code = script_code.replace(". $PSScriptRoot\\demistomock.ps1", "")
        script_code = script_code.replace(". $PSScriptRoot\\CommonServerPowerShell.ps1", "")
        return script_code

    def get_pack_path(self):
        return self.package_path.split('Integrations')[0]

    def is_contributor_pack(self, contributor_type):
        """Checks if the pack is a contribution.
        Args:
            contributor_type (str): The contributor type.
        Returns:
            (bool). True if it is a contributed pack, False otherwise.
        """
        if contributor_type in CONTRIBUTORS_LIST:
            return True
        return False

    def get_contributor_data(self):
        """Gets contributor data.

        Returns:
            (str, dict). Contributor type and file data.
        """
        pack_path = self.get_pack_path()
        pack_metadata_data, pack_metadata_path = self.get_data(pack_path, PACK_METADATA_PATH)

        if pack_metadata_data:
            try:
                json_pack_metadata = json.loads(pack_metadata_data)
            except json.JSONDecodeError as e:
                pack_name: str = get_pack_name(pack_path)
                raise Exception(f'Failed to load pack metadata of pack {pack_name}: {str(e)}') from e
            support_field = json_pack_metadata.get('support')
            return support_field, json_pack_metadata
        return None, None

    def add_contributors_support(self, unified_yml: Dict, contributor_type: str,
                                 contributor_email: Union[str, List[str]],
                                 contributor_url: str, author: str = '') -> Dict:
        """Add contributor support to the unified file - text in the display name and detailed description.

        Args:
            unified_yml (dict): The unified yaml file.
            contributor_type (str): The contributor type - partner / developer / community
            contributor_email (str): The contributor email.
            contributor_url (str): The contributor url.
            author (str): The packs author.

        Returns:
            The unified yaml file (dict).
        """
        if ' Contribution)' not in unified_yml['display']:
            unified_yml['display'] += CONTRIBUTOR_DISPLAY_NAME.format(contributor_type.capitalize())
        existing_detailed_description = unified_yml.get('detaileddescription', '')
        if contributor_type == COMMUNITY_CONTRIBUTOR:
            contributor_description = CONTRIBUTOR_COMMUNITY_DETAILED_DESC.format(author)
        else:
            contributor_description = CONTRIBUTOR_DETAILED_DESC.format(contributor_type.capitalize(), author)
            if contributor_email:
                email_list: List[str] = arg_to_list(contributor_email, ",")
                for email in email_list:
                    contributor_description += f'\n- **Email**: [{email}](mailto:{email})'
            if contributor_url:
                contributor_description += f'\n- **URL**: [{contributor_url}]({contributor_url})'

        contrib_details = re.findall(r'### .* Contributed Integration', existing_detailed_description)

        if not contrib_details:
            unified_yml['detaileddescription'] = contributor_description + '\n***\n' + existing_detailed_description

        return unified_yml

    def get_integration_doc_link(self, unified_yml: Dict) -> str:
        """Generates the integration link to the integration documentation

        Args:
            unified_yml (Dict): The integration YAML dictionary object

        Returns:
            str: The integration doc markdown link to add to the detailed description (if reachable)
        """
        normalized_integration_id = self.normalize_integration_id(unified_yml['commonfields']['id'])
        integration_doc_link = INTEGRATIONS_DOCS_REFERENCE + normalized_integration_id

        readme_path = os.path.join(self.package_path, 'README.md')
        if os.path.isfile(readme_path) and os.stat(readme_path).st_size != 0:
            # verify README file exists and is not empty
            return f'[View Integration Documentation]({integration_doc_link})'
        else:
            click.secho(
                f'Did not find README in {self.package_path}, not adding integration doc link',
                fg="bright_cyan"
            )
            return ''

    @staticmethod
    def normalize_integration_id(integration_id: str) -> str:
        """Normalizes integration ID to an identifier to be used as the integration documentation ID

        Examples
            >>> normalize_integration_id('Cortex XDR - IOC')
            cortex-xdr---ioc
            >>> normalize_integration_id('Whois')
            whois
            >>> normalize_integration_id('SomeIntegration')
            some-integration

        Args:
            integration_id (str): The integration ID to normalize

        Returns:
            str: The normalized identifier
        """
        dasherized_integration_id = dasherize(underscore(integration_id)).replace(' ', '-')
        # remove all non-word characters (dash is ok)
        return re.sub(r'[^\w-]', '', dasherized_integration_id)

    def move_readme_next_to_unified(self, yml_path):
        """
            Args:
                yml_path: The path to the yml file.

            Return:
                README new location next to the unified yml (in the same folder).

        """
        dst_path_readme = yml_path.replace('.yml', '_README.md')
        src_path_readme = os.path.join(self.package_path, 'README.md')
        shutil.move(src_path_readme, dst_path_readme)
        return dst_path_readme


# demisto_sdk/commands/unify/rule_unifier.py

import io
import json
import os
from collections import defaultdict
from pathlib import Path
from typing import Optional

import click
from ruamel.yaml.scalarstring import FoldedScalarString

from demisto_sdk.commands.common.constants import SAMPLES_DIR
from demisto_sdk.commands.unify.yaml_unifier import YAMLUnifier


class RuleUnifier(YAMLUnifier):
    def __init__(
        self,
        input: str,
        output: Optional[str] = None,
        force: bool = False,
        marketplace: Optional[str] = None,
    ):
        self.input_rule = input

        super().__init__(
            input=input,
            output=output,
            force=force,
            marketplace=marketplace
        )

        self.dir_name = os.path.basename(os.path.dirname(self.package_path))

    def unify(self):
        click.echo(f'Unifiying {self.package_path}...')
        self._set_dest_path()
        self._insert_rules()
        self._insert_schema()
        self._insert_samples()
        self._output_yaml(file_path=self.dest_path, file_data=self.yml_data)
        click.secho(f'Successfully created unifyed YAML in {self.dest_path}', fg="green")

        return [str(self.dest_path)]

    def _insert_rules(self):
        rules_path = Path(self.yml_path).with_suffix('.xif')
        with io.open(rules_path, mode='r', encoding='utf-8') as rules_file:
            rules = rules_file.read()
            self.yml_data['rules'] = FoldedScalarString(rules)

    def _insert_samples(self):
        samples_dir = os.path.join(os.path.dirname(self.package_path), SAMPLES_DIR)
        if os.path.isdir(samples_dir):
            samples = defaultdict(list)
            for sample_file in os.listdir(samples_dir):
                with io.open(os.path.join(samples_dir, sample_file), mode='r', encoding='utf-8') as samples_file_object:
                    sample = json.loads(samples_file_object.read())
                    if self.yml_data.get('id') in sample.get('rules', []):
                        samples[f'{sample.get("vendor")}_{sample.get("product")}'].extend(sample.get('samples'))
            if samples:
                self.yml_data['samples'] = FoldedScalarString(json.dumps(samples, indent=4))
                click.echo(f'Added {len(samples)} samples.')
            else:
                click.echo('Did not find matching samples.')

    def _insert_schema(self):
        schema_path = self.yml_path.replace('.yml', '_schema.json')
        if os.path.exists(schema_path):
            with io.open(schema_path, mode='r', encoding='utf-8') as schema_file:
                schema = json.loads(schema_file.read())
                self.yml_data['schema'] = FoldedScalarString(json.dumps(schema, indent=4))
        else:
            click.echo('No schema file was found.')


# demisto_sdk/commands/unify/xdrc_template_unifier.py

import base64
import json
import os
from pathlib import Path

import click

from demisto_sdk.commands.common.constants import DIR_TO_PREFIX
from demisto_sdk.commands.common.errors import Errors
from demisto_sdk.commands.common.tools import get_yml_paths_in_dir


class XDRCTemplateUnifier:
    def __init__(
        self,
        input: str,
        output: Path = None,
        dir_name: str = 'XDRCTemplates',
    ):
        self.input_xdrc_template = input

        if dir_name:
            self.dir_name = dir_name

        input = os.path.abspath(input)
        self.package_path = input.rstrip(os.sep)

        _, self.yml_path = get_yml_paths_in_dir(self.package_path, Errors.no_yml_file(self.package_path))

        self.dest_path = os.path.abspath(output) if output else None

    def unify(self):
        click.echo(f'Unifying {self.package_path}...')
        self._set_dest_path()
        output_data = self._insert_xdrc_template()
        self._insert_yaml_template(output_data)
        self._output_json(file_data=output_data)
        click.secho(f'Successfully created unified JSON in {self.dest_path}', fg="green")

        return [str(self.dest_path)]

    def _set_dest_path(self, file_name_suffix: str = None,):
        """Sets the target (destination) output path for the unified JSON"""
        package_dir_name = os.path.basename(self.package_path)
        output_filename = '{}-{}-{}.json'.format('external', DIR_TO_PREFIX[self.dir_name], package_dir_name)

        if file_name_suffix:
            # append suffix to output file name
            output_filename = file_name_suffix.join(os.path.splitext(output_filename))

        if self.dest_path:
            self.dest_path = os.path.join(self.dest_path, output_filename)
        else:
            self.dest_path = os.path.join(self.package_path, output_filename)

    def _insert_xdrc_template(self):
        xdrc_template_path = Path(self.yml_path).with_suffix('.json')

        with open(xdrc_template_path) as xdrc_template_file:
            xdrc_template = json.load(xdrc_template_file)

        return xdrc_template

    def _insert_yaml_template(self, output_data):
        yaml_template_path = Path(self.yml_path)
        encoding = 'utf-8'
        with open(yaml_template_path, "r") as yaml_template_file:
            output_data['yaml_template'] = base64.b64encode(yaml_template_file.read().encode(encoding)).decode(encoding)

    def _output_json(self, file_data):
        with open(self.dest_path, mode='w+') as dest_file:  # type: ignore
            json.dump(file_data, dest_file)


# demisto_sdk/commands/unify/yaml_unifier.py

import io
import os
import sys
from abc import ABC, abstractmethod
from typing import Optional

from demisto_sdk.commands.common.constants import DIR_TO_PREFIX, INTEGRATIONS_DIR, SCRIPTS_DIR
from demisto_sdk.commands.common.errors import Errors
from demisto_sdk.commands.common.handlers import YAML_Handler
from demisto_sdk.commands.common.tools import get_yml_paths_in_dir, print_error

UNSUPPORTED_INPUT_ERR_MSG = '''Unsupported input. Please provide either:
1. Path to directory of an integration or a script.
2. Path to directory of a Parsing/Modeling rule.'''


class YAMLUnifier(ABC):
    """Interface to YAML objects that need to be unified

    Attributes:
        package_path (str): The directory path to the files to unify.
        dest_path (str, optional): The output dir to write the unified YAML to.
        use_force(bool): Forcefully overwrites the preexisting yml if one exists.
        yaml(YAML_Handler): Wrapper object to handle YAML files.
        yml_path(str): The YAML file path.
        yml_data(dict): The YAML doucment Python object.
    """

    def __init__(
        self,
        input: str,
        output: Optional[str] = None,
        force: bool = False,
        marketplace: Optional[str] = None,
    ):
        directory_name = ''
        # Changing relative path to current abspath fixed problem with default output file name.
        input = os.path.abspath(input)
        if not os.path.isdir(input):
            print_error(UNSUPPORTED_INPUT_ERR_MSG)
            sys.exit(1)
        for optional_dir_name in DIR_TO_PREFIX:
            if optional_dir_name in input:
                directory_name = optional_dir_name

        if not directory_name:
            print_error(UNSUPPORTED_INPUT_ERR_MSG)

        self.package_path = input
        self.package_path = self.package_path.rstrip(os.sep)

        self.use_force = force
        self.dest_path = output
        self.dir_name = ''
        self.marketplace = marketplace
        yml_paths, self.yml_path = get_yml_paths_in_dir(self.package_path, Errors.no_yml_file(self.package_path))
        for path in yml_paths:
            # The plugin creates a unified YML file for the package.
            # In case this script runs locally and there is a unified YML file in the package we need to ignore it.
            # Also,
            # we don't take the unified file by default because
            # there might be packages that were not created by the plugin.
            if 'unified' not in path and os.path.basename(os.path.dirname(path)) not in [SCRIPTS_DIR, INTEGRATIONS_DIR]:
                self.yml_path = path
                break

        self.yaml = YAML_Handler(width=50000)  # make sure long lines will not break (relevant for code section)

        if self.yml_path:
            with io.open(self.yml_path, 'r', encoding='utf8') as yml_file:
                self.yml_data = self.yaml.load(yml_file)
        else:
            self.yml_data = {}
            print_error(f'No yml found in path: {self.package_path}')

    @abstractmethod
    def unify(self):
        """Merges the various components to create an output yml file."""
        ...

    def _set_dest_path(
        self,
        file_name_suffix: Optional[str] = None,
    ):
        """Sets the target (destination) output path for the unified YAML, based on:
            - Integration/Script directory name.
            - Content item type (Integration/Script/Rule).
            - Content item prefix (integration/script/parsingrule/modelingrule).
            - Provided file name suffix.

        Args:
            file_name_suffix(str): An optional suffix to concat to the filename.
        """
        package_dir_name = os.path.basename(self.package_path)
        output_filename = '{}-{}.yml'.format(DIR_TO_PREFIX[self.dir_name], package_dir_name)

        if file_name_suffix:
            # append suffix to output file name
            output_filename = file_name_suffix.join(os.path.splitext(output_filename))

        if self.dest_path:
            self.dest_path = os.path.join(self.dest_path, output_filename)
        else:
            self.dest_path = os.path.join(self.package_path, output_filename)

    def _output_yaml(
        self,
        file_path: Optional[str],
        file_data: dict,
    ):
        """Writes the YAML unified to the given path.
        Checks whether the unified YAML already exists, and either fail or overwrite it forced to.

        Args:
            file_path(str): The file path to output the YAML to.
            file_data(dict): The unified YAML contents.
        """
        if os.path.isfile(file_path) and not self.use_force:  # type: ignore[arg-type]
            raise ValueError(
                f'Output file already exists: {self.dest_path}.'
                ' Make sure to remove this file from source control'
                ' or rename this package (for example if it is a v2).'
            )

        with io.open(file_path, mode='w', encoding='utf-8') as file_:  # type: ignore[arg-type]
            self.yaml.dump(file_data, file_)


# demisto_sdk/commands/update_release_notes/update_rn.py

"""
This script is used to create a release notes template
"""
import copy
import errno
import os
import re
from distutils.version import LooseVersion
from pathlib import Path
from typing import Any, Optional, Tuple, Union

from demisto_sdk.commands.common.constants import (ALL_FILES_VALIDATION_IGNORE_WHITELIST, DEPRECATED_REGEXES,
                                                   IGNORED_PACK_NAMES, RN_HEADER_BY_FILE_TYPE, XSIAM_DASHBOARDS_DIR,
                                                   XSIAM_REPORTS_DIR, FileType)
from demisto_sdk.commands.common.content import Content
from demisto_sdk.commands.common.content.objects.pack_objects import Integration, Playbook, Script
from demisto_sdk.commands.common.content.objects.pack_objects.abstract_pack_objects.yaml_content_object import \
    YAMLContentObject
from demisto_sdk.commands.common.content_constant_paths import DEFAULT_ID_SET_PATH
from demisto_sdk.commands.common.git_util import GitUtil
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.tools import (LOG_COLORS, find_type, get_api_module_ids,
                                               get_api_module_integrations_set, get_definition_name, get_display_name,
                                               get_from_version, get_json, get_latest_release_notes_text, get_pack_name,
                                               get_remote_file, get_yaml, pack_name_to_path, print_color, print_error,
                                               print_warning, run_command)

json = JSON_Handler()

CLASS_BY_FILE_TYPE = {FileType.INTEGRATION: Integration, FileType.SCRIPT: Script, FileType.PLAYBOOK: Playbook}


def get_deprecated_comment_from_desc(description: str) -> str:
    """
    find deprecated comment from description
    Args:
        description: The yml description

    Returns:
        If a deprecated description is found return it for rn.
    """
    deprecate_line_with_replacement = re.findall(DEPRECATED_REGEXES[0], description)
    deprecate_line_no_replacement = re.findall(DEPRECATED_REGEXES[1], description)

    deprecate_line = deprecate_line_with_replacement + deprecate_line_no_replacement
    return deprecate_line[0] if deprecate_line else ''


def deprecated_commands(commands: list) -> set:
    """return a set of the deprecated commands only"""
    return {command.get('name') for command in commands if command.get('deprecated')}


def get_yml_objects(path: str, file_type) -> Tuple[Any, YAMLContentObject]:
    """
    Generate yml files from master and from the current branch
    Args:
        path: The requested file path
        file_type: the file type

    Returns:
        Two YML objects, the first of the yml at master (old yml) and the second from the current branch (new yml)
    """
    old_yml_obj = get_remote_file(path)
    new_yml_obj = CLASS_BY_FILE_TYPE[file_type](path)

    return old_yml_obj, new_yml_obj


def get_deprecated_rn(path: str, file_type):
    """Generate rn for deprecated items"""
    old_yml, new_yml = get_yml_objects(path, file_type)

    if not old_yml.get('deprecated') and new_yml.is_deprecated:
        description = new_yml.get('comment') if file_type == file_type.SCRIPT else new_yml.get('description')
        rn_from_description = get_deprecated_comment_from_desc(description)
        return f'- Deprecated. {rn_from_description or "Use %%% instead"}.\n'

    if file_type != FileType.INTEGRATION:
        return ''

    # look for deprecated commands
    rn = ''
    old_commands = deprecated_commands(old_yml.get('script', {}).get('commands'))
    new_commands = deprecated_commands(new_yml.script.get('commands'))

    for command_name in new_commands:
        # if command is deprecated in new yml, and not in old yml
        if command_name not in old_commands:
            rn += f'- Command ***{command_name}*** is deprecated. Use %%% instead.\n'
    return rn


class UpdateRN:
    def __init__(self, pack_path: str, update_type: Union[str, None], modified_files_in_pack: set, added_files: set,
                 specific_version: str = None, pre_release: bool = False, pack: str = None,
                 pack_metadata_only: bool = False, text: str = '', existing_rn_version_path: str = '',
                 is_force: bool = False, is_bc: bool = False):
        self.pack = pack if pack else get_pack_name(pack_path)
        self.update_type = update_type
        self.pack_path = pack_path
        # renamed files will appear in the modified list as a tuple: (old path, new path)
        modified_files_in_pack = {file_[1] if isinstance(file_, tuple) else file_ for file_ in modified_files_in_pack}
        self.modified_files_in_pack = set()
        for file_path in modified_files_in_pack:
            self.modified_files_in_pack.add(self.change_image_or_desc_file_path(file_path))

        self.added_files = added_files
        self.pre_release = pre_release
        self.specific_version = specific_version
        self.existing_rn_changed = False
        self.text = text
        self.existing_rn_version_path = existing_rn_version_path
        self.should_delete_existing_rn = False
        self.pack_metadata_only = pack_metadata_only
        self.is_force = is_force
        git_util = GitUtil(repo=Content.git())
        self.main_branch = git_util.handle_prev_ver()[1]
        self.metadata_path = os.path.join(self.pack_path, 'pack_metadata.json')
        self.master_version = self.get_master_version()
        self.rn_path = ''
        self.is_bc = is_bc
        self.bc_path = ''

    @staticmethod
    def change_image_or_desc_file_path(file_path: str) -> str:
        """ Changes image and description file paths to the corresponding yml file path.
            if a non-image or description file path is given, it remains unchanged.

            :param file_path: The file path to check

            :rtype: ``str``
            :return
                The new file path if was changed
        """

        def validate_new_path(expected_path: str):
            if not Path(expected_path).exists():
                print_warning(f"file {file_path} implies the existence of {str(expected_path)}, which is missing. "
                              f"Did you mistype {file_path}?")

        if file_path.endswith('_image.png'):
            if Path(file_path).parent.name in (XSIAM_DASHBOARDS_DIR, XSIAM_REPORTS_DIR):
                new_path = file_path.replace('_image.png', '.json')
            else:
                new_path = file_path.replace('_image.png', '.yml')
            validate_new_path(new_path)
            return new_path

        elif file_path.endswith('_description.md'):
            new_path = file_path.replace('_description.md', '.yml')
            validate_new_path(new_path)
            return new_path

        return file_path

    def handle_existing_rn_version_path(self, rn_path: str) -> str:
        """ Checks whether the existing RN version path exists and return it's content.

            :param rn_path: The rn path to check

            :rtype: ``str``
            :return
                The content of the rn
        """
        if self.existing_rn_version_path:
            existing_rn_abs_path = Path(self.existing_rn_version_path).absolute()
            rn_path_abs_path = Path(rn_path).absolute()
            self.should_delete_existing_rn = str(existing_rn_abs_path) != str(rn_path_abs_path)
            try:
                return existing_rn_abs_path.read_text()
            except Exception as e:
                print_error(f'Failed to load the previous release notes file content: {e}')
        return ''

    def execute_update(self) -> bool:
        """ Obtains the information needed in order to update the pack and executes the update.

            :rtype: ``bool``
            :return
                Whether the RN was updated successfully or not
        """
        if self.pack in IGNORED_PACK_NAMES:
            print_warning(f"Release notes are not required for the {self.pack} pack since this pack"
                          f" is not versioned.")
            return False

        new_version, new_metadata = self.get_new_version_and_metadata()
        rn_path = self.get_release_notes_path(new_version)
        self.check_rn_dir(rn_path)
        self.rn_path = rn_path
        self.find_added_pack_files()
        changed_files = {}
        for packfile in self.modified_files_in_pack:
            file_name, file_type = self.get_changed_file_name_and_type(packfile)
            if 'yml' in packfile and file_type in [FileType.INTEGRATION, FileType.BETA_INTEGRATION,
                                                   FileType.SCRIPT] and packfile not in self.added_files:
                docker_image_name: Optional[str] = check_docker_image_changed(main_branch=self.main_branch,
                                                                              packfile=packfile)
            else:
                docker_image_name = None
            changed_files[(file_name, file_type)] = {
                'description': get_file_description(packfile, file_type),
                'is_new_file': packfile in self.added_files,
                'fromversion': get_from_version_at_update_rn(packfile),
                'dockerimage': docker_image_name,
                'path': packfile
            }
        return self.create_pack_rn(rn_path, changed_files, new_metadata, new_version)

    def create_pack_rn(self, rn_path: str, changed_files: dict, new_metadata: dict, new_version: str) -> bool:
        """ Checks whether the pack requires a new rn and if so, creates it.

            :param
                rn_path (str): The rn path
                changed_files (dict): The changed files details
                new_metadata (dict): The new pack metadata
                new_version (str): The new version str representation, e.g 1.0.2, 1.11.2 etc.


            :rtype: ``bool``
            :return
                Whether the RN was updated successfully or not
        """
        rn_string = self.handle_existing_rn_version_path(rn_path)
        if not rn_string:
            rn_string = self.build_rn_template(changed_files)
        if len(rn_string) > 0 or self.is_force:
            if self.is_bump_required():
                self.write_metadata_to_file(new_metadata)
            self.create_markdown(rn_path, rn_string, changed_files)
            self.build_rn_config_file(new_version)
            try:
                run_command(f'git add {rn_path}', exit_on_error=False)
            except RuntimeError:
                print_warning(f'Could not add the release note files to git: {rn_path}')
            if self.is_bc and self.bc_path:
                try:
                    run_command(f'git add {self.bc_path}', exit_on_error=False)
                except RuntimeError:
                    print_warning(f'Could not add the release note config file to git: {rn_path}')
            if self.existing_rn_changed:
                print_color(f"Finished updating release notes for {self.pack}.", LOG_COLORS.GREEN)
                if not self.text:
                    print_color(f"\nNext Steps:\n - Please review the "
                                f"created release notes found at {rn_path} and document any changes you "
                                f"made by replacing '%%UPDATE_RN%%'.\n - Commit "
                                f"the new release notes to your branch.\nFor information regarding proper"
                                f" format of the release notes, please refer to "
                                f"https://xsoar.pan.dev/docs/integrations/changelog", LOG_COLORS.GREEN)
                return True
            else:
                print_color(f"No changes to {self.pack} pack files were detected from the previous time "
                            "this command was run. The release notes have not been "
                            "changed.", LOG_COLORS.GREEN)
        else:
            print_color("No changes which would belong in release notes were detected.", LOG_COLORS.YELLOW)
        return False

    def build_rn_config_file(self, new_version: str) -> None:
        """
        Builds RN config file if needed. Currently, we use RN config file only for cases where version has breaking
        changes.
        Args:
            new_version (str): The new version number representation, e.g 1.2.1, 1.22.1, etc.

        Returns:
            (None): Creates/updates config file with BC entries, if -bc flag was given.
        """
        # Currently, we only use config file if version is BC. If version is not BC no need to create config file.
        if not self.is_bc:
            return
        bc_file_path: str = f'''{self.pack_path}/ReleaseNotes/{new_version.replace('.', '_')}.json'''
        self.bc_path = bc_file_path
        bc_file_data: dict = dict()
        if os.path.exists(bc_file_path):
            with open(bc_file_path, 'r') as f:
                bc_file_data = json.loads(f.read())
        bc_file_data['breakingChanges'] = True
        bc_file_data['breakingChangesNotes'] = bc_file_data.get('breakingChangesNotes')
        with open(bc_file_path, 'w') as f:
            f.write(json.dumps(bc_file_data))
        print_color(f'Finished creating config file for RN version {new_version}.\n'
                    'If you wish only specific text to be shown as breaking changes, please fill the '
                    '`breakingChangesNotes` field with the appropriate breaking changes text.', LOG_COLORS.GREEN)

    def get_new_version_and_metadata(self) -> Tuple[str, dict]:
        """
            Gets the new version and the new metadata after version bump or by getting it from the pack metadata if
            bump is not required.

            :rtype: ``(str, dict)``
            :return: The new version and new metadata dictionary
        """
        if self.is_bump_required():
            if self.update_type is None:
                self.update_type = "revision"
            new_version, new_metadata = self.bump_version_number(self.specific_version, self.pre_release)
            if self.is_force:
                print_color(f"Bumping {self.pack} to version: {new_version}",
                            LOG_COLORS.NATIVE)
            else:
                print_color(f"Changes were detected. Bumping {self.pack} to version: {new_version}",
                            LOG_COLORS.NATIVE)
        else:
            new_metadata = self.get_pack_metadata()
            new_version = new_metadata.get('currentVersion', '99.99.99')
        return new_version, new_metadata

    def _does_pack_metadata_exist(self) -> bool:
        """ Check if pack_metadata.json exists

            :rtype: ``bool``
            :return
                Whether the pack metadata exists
        """
        if not os.path.isfile(self.metadata_path):
            print_error(f'"{self.metadata_path}" file does not exist, create one in the root of the pack')
            return False

        return True

    def get_master_version(self) -> str:
        """
            Gets the current version from origin/master or origin/main if available, otherwise return '0.0.0'.

            :rtype: ``str``
            :return
                The master version

        """
        master_current_version = '0.0.0'
        master_metadata = None
        try:
            master_metadata = get_remote_file(self.metadata_path, tag=self.main_branch)
        except Exception as e:
            print_error(f"master branch is unreachable.\n The reason is:{e} \n "
                        f"The updated version will be taken from local metadata file instead of master")
        if master_metadata:
            master_current_version = master_metadata.get('currentVersion', '0.0.0')
        return master_current_version

    def is_bump_required(self) -> bool:
        """
            Checks if the currentVersion in the pack metadata has been changed or not. Additionally, it will verify
            that there is no conflict with the currentVersion in then Master branch.

            :rtype: ``bool``
            :return
                Whether a version bump is required
        """
        try:
            if self.only_docs_changed() and not self.is_force:
                return False
            new_metadata = self.get_pack_metadata()
            new_version = new_metadata.get('currentVersion', '99.99.99')
            if LooseVersion(self.master_version) >= LooseVersion(new_version):
                return True
            return False
        except RuntimeError as e:
            raise RuntimeError(f"Unable to locate a pack with the name {self.pack} in the git diff.\n"
                               f"Please verify the pack exists and the pack name is correct.") from e

    def only_docs_changed(self) -> bool:
        """
            Checks if the only files that were changed are documentation files.

            :rtype: ``bool``
            :return
                Whether only the docs were changed
        """
        changed_files = self.added_files.union(self.modified_files_in_pack)
        changed_files_copy = copy.deepcopy(changed_files)  # copying as pop will leave the file out of the set
        if (len(changed_files) == 1 and 'README' in changed_files_copy.pop()) or \
                (all('README' in file or ('.png' in file and '_image.png' not in file) for file in changed_files)):
            return True
        return False

    def find_added_pack_files(self):
        """
            Checks if the added files in the given pack require RN and if so, adds them to the modified files in the
            pack.
        """
        for a_file in self.added_files:
            if self.pack in a_file:
                if any(item in a_file for item in ALL_FILES_VALIDATION_IGNORE_WHITELIST):
                    continue
                else:
                    self.modified_files_in_pack.add(self.change_image_or_desc_file_path(a_file))

    def get_release_notes_path(self, input_version: str) -> str:
        """ Gets the release notes path.

            :param input_version: The new rn version

            :rtype: ``bool``
            :return
            Whether the RN was updated successfully or not
        """
        _new_version = input_version.replace('.', '_')
        new_version = _new_version.replace('_prerelease', '')
        return os.path.join(self.pack_path, 'ReleaseNotes', f'{new_version}.md')

    @staticmethod
    def find_corresponding_yml(file_path) -> str:
        """ Gets the pack's corresponding yml file from the python/yml file.

            :param file_path: The pack python/yml file

            :rtype: ``str``
            :return
            The path to the pack's yml file
        """
        if file_path.endswith('.py'):
            yml_filepath = file_path.replace('.py', '.yml')
        else:
            yml_filepath = file_path
        return yml_filepath

    def get_changed_file_name_and_type(self, file_path) -> Tuple[str, Optional[FileType]]:
        """ Gets the changed file name and type.

            :param file_path: The file path

            :rtype: ``str, FileType``
            :return
            The changed file name and type
        """
        _file_type = None
        file_name = 'N/A'

        if self.pack + '/' in file_path and ('README' not in file_path):
            _file_path = self.find_corresponding_yml(file_path)
            file_name = get_display_name(_file_path)
            _file_type = find_type(_file_path)

        return file_name, _file_type

    def get_pack_metadata(self) -> dict:
        """ Gets the pack metadata.

            :rtype: ``dict``
            :return
            The pack metadata dictionary
        """
        try:
            data_dictionary = get_json(self.metadata_path, cache_clear=True)
        except FileNotFoundError as e:
            raise FileNotFoundError(f'Pack {self.pack} was not found. Please verify the pack name is correct.') from e
        return data_dictionary

    def bump_version_number(self, specific_version: str = None, pre_release: bool = False) -> Tuple[str, dict]:
        """ Increases the version number by user input or update type.

            :param
                specific_version: The specific version to change the version to
                pre_release: Indicates that the change should be designated a pre-release version

            :rtype: ``str, dict``
            :return
            The new version number (for example: 1.0.3) and the new pack metadata after version bump
        """
        if self.update_type is None and specific_version is None:
            raise ValueError("Received no update type when one was expected.")
        new_version = ''  # This will never happen since we pre-validate the argument
        data_dictionary = self.get_pack_metadata()
        current_version = self.master_version if self.master_version != '0.0.0' else self.get_pack_metadata().get(
            'currentVersion', '99.99.99')
        if specific_version:
            print_color(f"Bumping {self.pack} to the version {specific_version}. If you need to update"
                        f" the release notes a second time, please remove the -v flag.", LOG_COLORS.NATIVE)
            data_dictionary['currentVersion'] = specific_version
            return specific_version, data_dictionary
        elif self.update_type == 'major':
            version = current_version.split('.')
            version[0] = str(int(version[0]) + 1)
            if int(version[0]) > 99:
                raise ValueError(f"Version number is greater than 99 for the {self.pack} pack. "
                                 f"Please verify the currentVersion is correct.")
            version[1] = '0'
            version[2] = '0'
            new_version = '.'.join(version)
        elif self.update_type == 'minor':
            version = current_version.split('.')
            version[1] = str(int(version[1]) + 1)
            if int(version[1]) > 99:
                raise ValueError(f"Version number is greater than 99 for the {self.pack} pack. "
                                 f"Please verify the currentVersion is correct. If it is, "
                                 f"then consider bumping to a new Major version.")
            version[2] = '0'
            new_version = '.'.join(version)
        # We validate the input via click

        elif self.update_type in ['revision', 'documentation']:
            version = current_version.split('.')
            version[2] = str(int(version[2]) + 1)
            if int(version[2]) > 99:
                raise ValueError(f"Version number is greater than 99 for the {self.pack} pack. "
                                 f"Please verify the currentVersion is correct. If it is, "
                                 f"then consider bumping to a new Minor version.")
            new_version = '.'.join(version)
        elif self.update_type == 'maintenance':
            raise ValueError("The *maintenance* option is no longer supported."
                             " Please use the \"revision\" option and make sure to provide informative release notes.")
        if pre_release:
            new_version = new_version + '_prerelease'
        data_dictionary['currentVersion'] = new_version
        return new_version, data_dictionary

    def write_metadata_to_file(self, metadata_dict: dict):
        """ Writes the new metadata to the pack metadata file.

            :param
                metadata_dict: The new metadata to write

        """
        if self._does_pack_metadata_exist():
            with open(self.metadata_path, 'w') as file_path:
                json.dump(metadata_dict, file_path, indent=4)
                print_color(f"Updated pack metadata version at path : {self.metadata_path}",
                            LOG_COLORS.GREEN)

    @staticmethod
    def check_rn_dir(rn_path: str):
        """ Checks whether the release notes folder exists and if not creates it.

            :param rn_path: The RN path to check/create

        """
        if not os.path.exists(os.path.dirname(rn_path)):
            try:
                os.makedirs(os.path.dirname(rn_path))
            except OSError as exc:  # Guard against race condition
                if exc.errno != errno.EEXIST:
                    raise

    def build_rn_template(self, changed_items: dict) -> str:
        """ Builds the new release notes template.

            :param
                changed_items: The changed items data dictionary

            :rtype: ``str``
            :return
            The new release notes template
        """
        rn_string = ''

        if self.pack_metadata_only:
            rn_string += f'\n#### Integrations\n##### {self.pack}\n- Documentation and metadata improvements.\n'
            return rn_string
        rn_template_as_dict: dict = {}
        if self.is_force:
            rn_string = self.build_rn_desc(content_name=self.pack, text=self.text)
        # changed_items.items() looks like that: [((name, type), {...}), (name, type), {...}] and we want to sort
        # them by type (x[0][1])

        for (content_name, _type), data in sorted(changed_items.items(),
                                                  key=lambda x: RN_HEADER_BY_FILE_TYPE[x[0][1]] if x[0] and x[0][1]
                                                  else ''):  # Sort RN by header
            desc = data.get('description', '')
            is_new_file = data.get('is_new_file', False)
            from_version = data.get('fromversion', '')
            docker_image = data.get('dockerimage')
            path = data.get('path')
            # Skipping the invalid files
            if not _type or content_name == 'N/A':
                continue
            rn_desc = self.build_rn_desc(_type=_type, content_name=content_name, desc=desc, is_new_file=is_new_file,
                                         text=self.text, docker_image=docker_image, from_version=from_version,
                                         path=path)

            header = f'\n#### {RN_HEADER_BY_FILE_TYPE[_type]}\n'
            rn_template_as_dict[header] = rn_template_as_dict.get(header, '') + rn_desc

        for key, val in rn_template_as_dict.items():
            rn_string = f"{rn_string}{key}{val}"

        return rn_string

    def build_rn_desc(self, _type: FileType = None, content_name: str = '', desc: str = '', is_new_file: bool = False,
                      text: str = '', docker_image: Optional[str] = '', from_version: str = '', path: str = '') -> str:
        """ Builds the release notes description.

            :param
                _type: The file type
                content_name: The pack name
                desc: The pack description
                is_new_file: True if the file is new
                text: Text to add to the release notes files
                from_version: From version

            :rtype: ``str``
            :return
            The release notes description
        """
        if _type in (FileType.CONNECTION, FileType.INCIDENT_TYPE, FileType.REPUTATION, FileType.LAYOUT,
                     FileType.INCIDENT_FIELD, FileType.INDICATOR_FIELD):

            rn_desc = f'- **{content_name}**\n'

        elif _type in (FileType.GENERIC_TYPE, FileType.GENERIC_FIELD):
            definition_name = get_definition_name(path, self.pack_path)
            rn_desc = f'- **({definition_name}) - {content_name}**\n'
        else:
            if is_new_file:
                rn_desc = f'##### New: {content_name}\n- {desc}'
                if from_version:
                    rn_desc += f' (Available from Cortex XSOAR {from_version}).'
                rn_desc += '\n'
            else:
                rn_desc = f'##### {content_name}\n'
                if self.update_type == 'documentation':
                    rn_desc += '- Documentation and metadata improvements.\n'
                else:
                    deprecate_rn = ''
                    if _type in (FileType.INTEGRATION, FileType.SCRIPT, FileType.PLAYBOOK):
                        deprecate_rn = get_deprecated_rn(path, _type)
                    if deprecate_rn:
                        if text:
                            rn_desc += f'- {text}\n'
                        rn_desc += deprecate_rn
                    else:
                        rn_desc += f'- {text or "%%UPDATE_RN%%"}\n'

        if _type == FileType.TRIGGER:
            rn_desc = f'- {desc}'  # Issue - https://github.com/demisto/etc/issues/48153#issuecomment-1111988526

        if docker_image:
            rn_desc += f'- Updated the Docker image to: *{docker_image}*.\n'
        return rn_desc

    def update_existing_rn(self, current_rn, changed_files) -> str:
        """ Update the existing release notes.

            :param
                current_rn: The existing rn
                changed_files: The new data to add

            :rtype: ``str``
            :return
            The updated release notes
        """
        update_docker_image_regex = r'- Updated the Docker image to: \*.*\*\.'
        # Deleting old entry for docker images, will re-write later, this allows easier generating of updated rn.
        current_rn_without_docker_images = re.sub(update_docker_image_regex, '', current_rn)
        new_rn = current_rn_without_docker_images
        # changed_files.items() looks like that: [((name, type), {...}), (name, type), {...}] and we want to sort
        # them by name (x[0][0])
        for (content_name, _type), data in sorted(changed_files.items(),
                                                  key=lambda x: x[0][0] if x[0][0] else '', reverse=True):
            is_new_file = data.get('is_new_file')
            desc = data.get('description', '')
            docker_image = data.get('dockerimage')
            path = data.get('path')

            if _type is None:
                continue

            _header_by_type = RN_HEADER_BY_FILE_TYPE.get(_type)
            if _type in (FileType.CONNECTION, FileType.INCIDENT_TYPE, FileType.REPUTATION, FileType.LAYOUT,
                         FileType.INCIDENT_FIELD, FileType.JOB, FileType.WIZARD):
                rn_desc = f'\n- **{content_name}**'

            elif _type in (FileType.GENERIC_TYPE, FileType.GENERIC_FIELD):
                definition_name = get_definition_name(path, self.pack_path)
                rn_desc = f'\n- **({definition_name}) - {content_name}**'

            elif _type == FileType.TRIGGER:
                rn_desc = f'\n- {desc}'  # Issue https://github.com/demisto/etc/issues/48153#issuecomment-1111988526

            else:
                rn_desc = f'\n##### New: {content_name}\n- {desc}\n' if is_new_file \
                    else f'\n##### {content_name}\n- %%UPDATE_RN%%\n'
            if docker_image:
                rn_desc += f'- Updated the Docker image to: *{docker_image}*.'

            if _header_by_type and _header_by_type in current_rn_without_docker_images:
                if content_name in current_rn_without_docker_images:
                    if docker_image:
                        new_rn = self.handle_existing_rn_with_docker_image(new_rn, _header_by_type, docker_image,
                                                                           content_name)
                else:
                    self.existing_rn_changed = True
                    rn_parts = new_rn.split(_header_by_type)
                    new_rn_part = rn_desc
                    if len(rn_parts) > 1:
                        new_rn = f"{rn_parts[0]}{_header_by_type}{new_rn_part}{rn_parts[1]}"
                    else:
                        new_rn = ''.join(rn_parts) + new_rn_part
            else:
                self.existing_rn_changed = True
                if _header_by_type and _header_by_type in new_rn:
                    rn_parts = new_rn.split(_header_by_type)
                    new_rn_part = rn_desc
                    if len(rn_parts) > 1:
                        new_rn = f"{rn_parts[0]}{_header_by_type}{new_rn_part}{rn_parts[1]}"
                else:
                    new_rn_part = f'\n#### {_header_by_type}{rn_desc}'
                    new_rn += new_rn_part
        if new_rn != current_rn:
            self.existing_rn_changed = True
        return new_rn

    @staticmethod
    def handle_existing_rn_with_docker_image(new_rn: str, header_by_type: str, docker_image: str,
                                             content_name: str) -> str:
        """
        Receives the new RN to be written, performs operations to add the docker image to the given RN.
        Args:
            new_rn (str): new RN.
            header_by_type (str): Header of the RN to add docker image to, e.g 'Integrations', 'Scripts'
            docker_image (str): Docker image to add
            content_name (str): The content name to add the docker image entry to, e.g integration name, script name.

        Returns:
            (str): Updated RN
        """
        # Writing or re-writing docker image to release notes.
        rn_parts = new_rn.split(header_by_type)
        new_rn_part = f'- Updated the Docker image to: *{docker_image}*.'
        if len(rn_parts) > 1:
            # Splitting again by content name to append the docker image release note to corresponding
            # content entry only
            content_parts = rn_parts[1].split(f'{content_name}\n')
            new_rn = f'{rn_parts[0]}{header_by_type}{content_parts[0]}{content_name}\n{new_rn_part}\n' \
                     f'{content_parts[1]}'
        else:
            print_warning(f'Could not parse release notes {new_rn} by header type: {header_by_type}')
        return new_rn

    def create_markdown(self, release_notes_path: str, rn_string: str, changed_files: dict):
        """ Creates the new markdown and writes it to the release notes file.

            :param
                release_notes_path: The release notes file path
                rn_string: The rn data (if exists)
                changed_files: The changed files details
                docker_image_name: The docker image name

        """
        if os.path.exists(release_notes_path) and self.update_type is not None:
            print_warning(f"Release notes were found at {release_notes_path}. Skipping")
        elif self.update_type is None and self.specific_version is None:
            current_rn = get_latest_release_notes_text(release_notes_path)
            updated_rn = self.update_existing_rn(current_rn, changed_files)
            with open(release_notes_path, 'w') as fp:
                fp.write(updated_rn)
        else:
            self.existing_rn_changed = True
            with open(release_notes_path, 'w') as fp:
                fp.write(rn_string)

    def rn_with_docker_image(self, rn_string: str, docker_image: Optional[str]) -> str:
        """
            Receives existing release notes, if docker image was updated, adds docker_image to release notes.
            Taking care of cases s.t:
            1) no docker image update have occurred ('docker_image' is None).
            2) Release notes did not contain updated docker image note.
            3) Release notes contained updated docker image notes, with the newest updated docker image.
            4) Release notes contained updated docker image notes, but docker image was updated again since last time
               release notes have been updated.

            param:
                rn_string (str): The current text contained in the release note
                docker_image (Optional[str]): The docker image str, if given

            :rtype: ``str``
            :return
                The release notes, with the most updated docker image release note, if given
        """
        if not docker_image:
            return rn_string
        docker_image_str = f'- Updated the Docker image to: *{docker_image}*.'
        if docker_image_str in rn_string:
            return rn_string
        self.existing_rn_changed = True
        if '- Updated the Docker image to' not in rn_string:
            return rn_string + f'{docker_image_str}\n'
        update_docker_image_regex = r'- Updated the Docker image to: \*.*\*\.'
        updated_rn = re.sub(update_docker_image_regex, docker_image_str, rn_string)
        self.existing_rn_changed = True
        return updated_rn


def get_file_description(path, file_type) -> str:
    """ Gets the file description.

        :param
            path: The file path
            file_type: The file type

        :rtype: ``str``
        :return
        The file description if exists otherwise returns %%UPDATE_RN%%
    """
    if not os.path.isfile(path):
        print_warning(f'Cannot get file description: "{path}" file does not exist')
        return ''

    elif file_type in (FileType.PLAYBOOK, FileType.INTEGRATION, FileType.CORRELATION_RULE):
        yml_file = get_yaml(path)
        return yml_file.get('description', '')

    elif file_type == FileType.SCRIPT:
        yml_file = get_yaml(path)
        return yml_file.get('comment', '')

    elif file_type in (FileType.CLASSIFIER, FileType.REPORT, FileType.WIDGET, FileType.DASHBOARD, FileType.JOB,
                       FileType.TRIGGER, FileType.WIZARD):
        json_file = get_json(path)
        return json_file.get('description', '')

    return '%%UPDATE_RN%%'


def update_api_modules_dependents_rn(pre_release: bool, update_type: Union[str, None],
                                     added: Union[list, set], modified: Union[list, set],
                                     id_set_path: Optional[Path] = None, text: str = '') -> set:
    """ Updates release notes for any pack that depends on API module that has changed.

        :param
            pre_release: The file type
            update_type: The update type
            added: The added files
            modified: The modified files
            id_set_path: The id set path
            text: Text to add to the release notes files

        :rtype: ``set``
        :return
        A set of updated packs
    """
    total_updated_packs: set = set()
    if not id_set_path:
        if not os.path.isfile(DEFAULT_ID_SET_PATH):
            print_error("Failed to update integrations dependent on the APIModule pack - no id_set.json is "
                        "available. Please run `demisto-sdk create-id-set` to generate it, and rerun this command.")
            return total_updated_packs
        id_set_path = DEFAULT_ID_SET_PATH
    with open(id_set_path, 'r') as conf_file:
        id_set = json.load(conf_file)
    api_module_set = get_api_module_ids(added)
    api_module_set = api_module_set.union(get_api_module_ids(modified))
    print_warning(f"Changes were found in the following APIModules: {api_module_set}, updating all dependent "
                  f"integrations.")
    integrations = get_api_module_integrations_set(api_module_set, id_set.get('integrations', []))
    for integration in integrations:
        integration_path = integration.get('file_path')
        integration_pack = integration.get('pack')
        integration_pack_path = pack_name_to_path(integration_pack)
        update_pack_rn = UpdateRN(pack_path=integration_pack_path, update_type=update_type,
                                  modified_files_in_pack={integration_path}, pre_release=pre_release,
                                  added_files=set(), pack=integration_pack, text=text)
        updated = update_pack_rn.execute_update()
        if updated:
            total_updated_packs.add(integration_pack)
    return total_updated_packs


def check_docker_image_changed(main_branch: str, packfile: str) -> Optional[str]:
    """ Checks whether the docker image was changed in master.

        :param
            main_branch: The git main branch
            packfile: The added or modified yml path

        :rtype: ``Optional[str]``
        :return
        The latest docker image
    """
    try:
        diff = run_command(f'git diff {main_branch} -- {packfile}', exit_on_error=False)
    except RuntimeError as e:
        if any(['is outside repository' in exp for exp in e.args]):
            return None
        else:
            print_warning(f'skipping docker image check, Encountered the following error:\n{e.args[0]}')
            return None
    else:
        diff_lines = diff.splitlines()
        for diff_line in diff_lines:
            if 'dockerimage:' in diff_line:  # search whether exists a line that notes that the Docker image was
                # changed.
                split_line = diff_line.split()
                if split_line[0].startswith('+'):
                    return split_line[-1]
        return None


def get_from_version_at_update_rn(path: str) -> Optional[str]:
    """
        param:
            path (str): path to yml file, if exists

        :rtype: ``Optional[str]``
        :return:
            Fromversion if there is a fromversion key in the yml file

    """
    if not os.path.isfile(path):
        print_warning(f'Cannot get file fromversion: "{path}" file does not exist')
        return None
    return get_from_version(path)


# demisto_sdk/commands/update_release_notes/update_rn_manager.py

import os
from pathlib import Path
from typing import Optional, Tuple

import git

from demisto_sdk.commands.common.constants import API_MODULES_PACK, SKIP_RELEASE_NOTES_FOR_TYPES
from demisto_sdk.commands.common.tools import (LOG_COLORS, filter_files_by_type, filter_files_on_pack, get_pack_name,
                                               get_pack_names_from_files, pack_name_to_path, print_color, print_warning,
                                               suppress_stdout)
from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN, update_api_modules_dependents_rn
from demisto_sdk.commands.validate.validate_manager import ValidateManager


class UpdateReleaseNotesManager:
    def __init__(self, user_input: Optional[str] = None, update_type: Optional[str] = None,
                 pre_release: bool = False, is_all: Optional[bool] = False, text: Optional[str] = None,
                 specific_version: Optional[str] = None, id_set_path: Optional[Path] = None,
                 prev_ver: Optional[str] = None, is_force: bool = False, is_bc: bool = False):
        self.given_pack = user_input
        self.changed_packs_from_git: set = set()
        self.update_type = update_type
        self.pre_release: bool = False if pre_release is None else pre_release
        # update release notes to every required pack if not specified.
        self.is_all = True if not self.given_pack else is_all
        self.text: str = '' if text is None else text
        self.is_force = is_force
        self.specific_version = specific_version
        self.id_set_path = id_set_path
        self.prev_ver = prev_ver
        self.packs_existing_rn: dict = {}
        self.total_updated_packs: set = set()
        # When a user choose a specific pack to update rn, the -g flag should not be passed
        if self.given_pack and self.is_all:
            raise ValueError('Please remove the -g flag when specifying only one pack.')
        self.rn_path: list = list()
        self.is_bc = is_bc

    def manage_rn_update(self):
        """
            Manages the entire update release notes process.
        """
        print_color('Starting to update release notes.', LOG_COLORS.NATIVE)
        # The given_pack can be both path or pack name thus, we extract the pack name from the path if needed.
        if self.given_pack and '/' in self.given_pack:
            self.given_pack = get_pack_name(self.given_pack)  # extract pack from path

        # Find which files were changed from git
        modified_files, added_files, old_format_files = self.get_git_changed_files()
        self.changed_packs_from_git = get_pack_names_from_files(modified_files).union(
            get_pack_names_from_files(added_files)).union(get_pack_names_from_files(old_format_files))
        # Check whether the packs have some existing RNs already (created manually or by the command)
        self.check_existing_rn(added_files)

        self.handle_api_module_change(modified_files, added_files)
        self.create_release_notes(modified_files, added_files, old_format_files)
        if len(self.total_updated_packs) > 1:
            print_color('\nSuccessfully updated the following packs:\n' + '\n'.join(self.total_updated_packs),
                        LOG_COLORS.GREEN)

    def filter_to_relevant_files(self, file_set: set, validate_manager: ValidateManager) -> Tuple[set, set, bool]:
        """
        Given a file set, filter it to only files which require RN and if given, from a specific pack
        """
        filtered_set = set()
        for file in file_set:
            if isinstance(file, tuple):
                file_path = str(file[1])

            else:
                file_path = str(file)

            if self.given_pack:
                file_pack_name = get_pack_name(file_path)
                if not file_pack_name or file_pack_name not in self.given_pack:
                    continue

            filtered_set.add(file)

        return validate_manager.filter_to_relevant_files(filtered_set)

    def filter_files_from_git(self, modified_files: set, added_files: set, renamed_files: set,
                              validate_manager: ValidateManager):
        """
        Filter the raw file sets to only the relevant files for RN
        """
        filtered_modified, old_format_files, _ = self.filter_to_relevant_files(modified_files, validate_manager)
        filtered_renamed, _, _ = self.filter_to_relevant_files(renamed_files, validate_manager)
        filtered_modified = filtered_modified.union(filtered_renamed)
        filtered_added, new_files_in_old_format, _ = self.filter_to_relevant_files(added_files, validate_manager)
        old_format_files = old_format_files.union(new_files_in_old_format)
        return filtered_modified, filtered_added, old_format_files

    def setup_validate_manager(self):
        return ValidateManager(skip_pack_rn_validation=True, prev_ver=self.prev_ver,
                               silence_init_prints=True, skip_conf_json=True, check_is_unskipped=False,
                               file_path=self.given_pack)

    def get_git_changed_files(self) -> Tuple[set, set, set]:
        """ Get the changed files from git (added, modified, old format, metadata).

            :return:
                - The filtered modified files (including the renamed files)
                - The filtered added files
                - The modified old-format files (legacy unified python files)
        """
        try:
            validate_manager = self.setup_validate_manager()
            if not validate_manager.git_util:  # in case git utils can't be initialized.
                raise git.InvalidGitRepositoryError('unable to connect to git.')
            validate_manager.setup_git_params()
            if self.given_pack:
                with suppress_stdout():
                    # The Validator prints errors which are related to all changed files that
                    # were changed against prev version. When the user is giving a specific pack to update,
                    # we want to suppress the error messages which are related to other packs.
                    modified_files, added_files, renamed_files = \
                        validate_manager.get_unfiltered_changed_files_from_git()
                    return self.filter_files_from_git(modified_files, added_files, renamed_files, validate_manager)

            modified_files, added_files, renamed_files = \
                validate_manager.get_unfiltered_changed_files_from_git()
            return self.filter_files_from_git(modified_files, added_files, renamed_files, validate_manager)

        except (git.InvalidGitRepositoryError, git.NoSuchPathError, FileNotFoundError) as e:
            raise FileNotFoundError(
                "You are not running `demisto-sdk update-release-notes` command in the content repository.\n"
                "Please run `cd content` from your terminal and run the command again") from e

    def check_existing_rn(self, added_files: set):
        """
            Checks whether the packs already have an existing release notes files and adds
            them to the packs_existing_rn dictionary.

            :param added_files: A set of new added files
        """
        for file_path in added_files:
            if 'ReleaseNotes' in file_path:
                self.packs_existing_rn[get_pack_name(file_path)] = file_path

    def handle_api_module_change(self, modified_files: set, added_files: set):
        """ Checks whether the modified file is in the API modules pack and if so, updates every pack which depends
            on that API module.

            :param
                added_files: A set of new added files
                modified_files: A set of modified files
        """
        # We want to handle ApiModules changes when:
        # (1) The user gave a path to the api module which was changed.
        # (2) The user did not give a specific path at all (is_all = True) but some ApiModules were changed.
        api_module_was_given = self.given_pack and API_MODULES_PACK in self.given_pack
        api_module_changed_in_git = self.changed_packs_from_git and API_MODULES_PACK in self.changed_packs_from_git

        if api_module_was_given or (api_module_changed_in_git and self.is_all):
            updated_packs = update_api_modules_dependents_rn(self.pre_release, self.update_type, added_files,
                                                             modified_files, self.id_set_path, self.text)
            self.total_updated_packs = self.total_updated_packs.union(updated_packs)

    def create_release_notes(self, modified_files: set, added_files: set, old_format_files: set):
        """ Iterates over the packs which needs an update and creates a release notes for them.

            :param
                modified_files: A set of modified files
                added_files: A set of added files
                old_format_files: A set of old formatted files
        """
        # Certain file types do not require release notes update
        filtered_modified_files = filter_files_by_type(modified_files, skip_file_types=SKIP_RELEASE_NOTES_FOR_TYPES)
        filtered_added_files = filter_files_by_type(added_files, skip_file_types=SKIP_RELEASE_NOTES_FOR_TYPES)
        if self.given_pack:  # A specific pack was chosen to update
            self.create_pack_release_notes(self.given_pack, filtered_modified_files, filtered_added_files,
                                           old_format_files)

        elif self.changed_packs_from_git:  # update all changed packs
            for pack in self.changed_packs_from_git:
                if API_MODULES_PACK in pack:  # We already handled Api Modules so we can skip it.
                    continue
                self.create_pack_release_notes(pack, filtered_modified_files, filtered_added_files, old_format_files)
        else:
            print_warning('No changes that require release notes were detected. If such changes were made, '
                          'please commit the changes and rerun the command.')

    def create_pack_release_notes(self, pack: str, filtered_modified_files: set, filtered_added_files: set,
                                  old_format_files: set):
        """ Creates the release notes for a given pack if was changed.

            :param
                pack: The pack to create release notes for
                filtered_modified_files: A set of filtered modified files
                filtered_added_files: A set of filtered added files
                old_format_files: A set of old formatted files
        """
        existing_rn_version = self.get_existing_rn(pack)
        if existing_rn_version is None:  # New release notes file already found for the pack
            raise RuntimeError(f"New release notes file already found for {pack}. "
                               f"Please update manually or run `demisto-sdk update-release-notes "
                               f"-i {pack}` without specifying the update_type.")
        pack_modified = filter_files_on_pack(pack, filtered_modified_files)
        pack_added = filter_files_on_pack(pack, filtered_added_files)
        pack_old = filter_files_on_pack(pack, old_format_files)

        # Checks if update is required
        if pack_modified or pack_added or pack_old or self.is_force:
            pack_path = pack_name_to_path(pack)
            update_pack_rn = UpdateRN(pack_path=pack_path, update_type=self.update_type,
                                      modified_files_in_pack=pack_modified.union(pack_old),
                                      pre_release=self.pre_release,
                                      added_files=pack_added, specific_version=self.specific_version,
                                      text=self.text, is_force=self.is_force,
                                      existing_rn_version_path=existing_rn_version, is_bc=self.is_bc)
            updated = update_pack_rn.execute_update()
            self.rn_path.append(update_pack_rn.rn_path)

            # If new release notes were created add it to the total number of packs that were updated.
            if updated:
                self.total_updated_packs.add(pack)
                # If there is an outdated previous release notes, remove it (for example: User updated his version to
                # 1.0.4 and meanwhile the master version changed to 1.0.4, so we want to remove the user's 1_0_4 file
                # and add a 1_0_5 file.)
                if update_pack_rn.should_delete_existing_rn:
                    os.unlink(self.packs_existing_rn[pack])
        else:
            print_warning(f'Either no changes were found in {pack} pack '
                          f'or the changes found should not be documented in the release notes file.\n'
                          f'If relevant changes were made, please commit the changes and rerun the command.')

    def get_existing_rn(self, pack) -> Optional[str]:
        """ Gets the existing rn of the pack is exists.

            :param
                pack: The pack to check
            :return
                The existing rn version if exists, otherwise an empty string
                None on error when pack has rn already and update type was given
        """
        if pack not in self.packs_existing_rn:
            return ''

        if self.update_type is None:
            return self.packs_existing_rn[pack]
        else:
            return None


# demisto_sdk/commands/update_xsoar_config_file/update_xsoar_config_file.py

import logging
import os
from typing import Dict, List

import demisto_client

from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.tools import LOG_COLORS, print_color

json = JSON_Handler()


XSOAR_CONFIG_FILE_JSON = "xsoar_config.json"
MARKETPLACE_PACKS_SECTION = "marketplace_packs"
CUSTOM_PACKS_SECTION = "custom_packs"


class XSOARConfigFileUpdater:
    """
    XSOARConfigFileUpdater is a class that's designed to update and edit the XSOAR Config File.

    Attributes:
        pack_id (str): The Pack ID to add to XSOAR Configuration File
        pack_data (str): The Pack Data to add to XSOAR Configuration File -
        Pack URL for Custom Pack and Pack Version for OOTB Pack
        add_marketplace_pack: (bool) Add the Pack to the MarketPlace Packs section in the Configuration File
        add_custom_pack (bool): Add the Pack to the Custom Packs section in the Configuration File
        add_all_marketplace_packs (bool): Add all the installed MarketPlace Packs to the XSOAR Configuration File
        file_path (str): XSOAR Configuration File path, the default value is in the repo level
    """

    def __init__(self, pack_id: str, pack_data: str, add_marketplace_pack: bool = False, add_custom_pack: bool = False,
                 add_all_marketplace_packs: bool = False, insecure: bool = False,
                 file_path: str = XSOAR_CONFIG_FILE_JSON):
        logging.disable(logging.CRITICAL)
        self.pack_id = pack_id
        self.pack_data = pack_data
        self.add_marketplace_pack = add_marketplace_pack
        self.add_custom_pack = add_custom_pack
        self.add_all_marketplace_packs = add_all_marketplace_packs
        self.insecure = insecure
        self.file_path = file_path
        self.client = None

    def update(self) -> int:
        """
        Update the Configuration File according the usr input.
        :return: The exit code
        """
        exit_code: int = self.update_config_file_manager()
        return exit_code

    def update_config_file_manager(self) -> int:
        """
        Manages all XSOAR Configuration File command flows
        :return The exit code of each flow
        """
        if not self.verify_flags():
            return 1
        elif self.add_all_marketplace_packs:
            self.add_all_installed_packs_to_config_file()
        elif self.add_marketplace_pack:
            self.update_marketplace_pack()
        elif self.add_custom_pack:
            self.update_custom_pack()
        return 0

    def verify_flags(self) -> bool:
        """
        Verifies that the flags configuration given by the user is correct
        :return: The verification result
        """
        is_valid_pack_structure = True

        if self.add_marketplace_pack or self.add_custom_pack:
            if not self.pack_id:
                is_valid_pack_structure = False
                print_color("Error: Missing option '-pi' / '--pack-id'.", LOG_COLORS.RED)
            if not self.pack_data:
                is_valid_pack_structure = False
                print_color("Error: Missing option '-pd' / '--pack-data'.", LOG_COLORS.RED)
        return is_valid_pack_structure

    def add_all_installed_packs_to_config_file(self):
        """
        Update the MarketPlace Packs Section in the Configuration File with all the installed packs on the machine.
        """
        marketplace_packs = self.get_installed_packs()
        self.update_xsoar_config_data(section_name=MARKETPLACE_PACKS_SECTION, data_to_update=marketplace_packs)

    def get_installed_packs(self) -> List[Dict[str, str]]:
        """
        Gets the current installed packs on the machine.
        """
        client = demisto_client.configure(verify_ssl=self.insecure)
        res = client.generic_request('/contentpacks/metadata/installed', "GET")
        installed_packs_data = eval(res[0])

        installed_packs = [
            {
                "id": pack["id"],
                "version": pack['currentVersion']
            }
            for pack in installed_packs_data
        ]
        return installed_packs

    def update_marketplace_pack(self):
        """
        Add / Update the MarketPlace Packs Section in the Configuration File with the new pack data.
        """
        new_pack = {
            "id": self.pack_id,
            "version": self.pack_data
        }
        self.update_xsoar_config_data(section_name=MARKETPLACE_PACKS_SECTION, data_to_update=new_pack)

    def update_custom_pack(self):
        """
        Add / Update the Custom Packs Section in the Configuration File with the new pack data.
        """
        new_pack = {
            "id": self.pack_id,
            "url": self.pack_data
        }
        self.update_xsoar_config_data(section_name=CUSTOM_PACKS_SECTION, data_to_update=new_pack)

    def update_xsoar_config_data(self, section_name, data_to_update):
        """
        Add / Update the Configuration File according to the section and data that provided.
        """
        config_file_info = self.get_xsoar_config_data()
        if config_file_info.get(section_name):
            if isinstance(data_to_update, list):
                config_file_info[section_name].extend(data_to_update)
            if isinstance(data_to_update, dict):
                config_file_info[section_name].append(data_to_update)
        else:
            config_file_info[section_name] = [data_to_update] if isinstance(data_to_update, dict) else data_to_update
        self.set_xsoar_config_data(config_file_info=config_file_info)

    def get_xsoar_config_data(self):
        config_file_info = {}
        if os.path.exists(self.file_path):
            with open(self.file_path, 'r') as config_file:
                try:
                    config_file_info = json.load(config_file)
                except json.JSONDecodeError:  # In case that the file exits but empty
                    pass
        return config_file_info

    def set_xsoar_config_data(self, config_file_info):
        with open(self.file_path, 'w') as config_file:
            json.dump(config_file_info, config_file, indent=4)


# demisto_sdk/commands/upload/upload.py

import os
import shutil
import tempfile

from demisto_sdk.commands.common.constants import ENV_DEMISTO_SDK_MARKETPLACE, MarketplaceVersions
from demisto_sdk.utils.utils import check_configuration_file


def upload_content_entity(**kwargs):
    from demisto_sdk.commands.upload.uploader import ConfigFileParser, Uploader
    from demisto_sdk.commands.zip_packs.packs_zipper import EX_FAIL, PacksZipper
    keep_zip = kwargs.pop('keep_zip')
    is_zip = kwargs.pop('zip', False)
    config_file_path = kwargs.pop('input_config_file')
    is_xsiam = kwargs.pop('xsiam', False)
    if is_zip or config_file_path:
        if is_zip:
            pack_path = kwargs['input']

        else:
            config_file_to_parse = ConfigFileParser(config_file_path=config_file_path)
            pack_path = config_file_to_parse.parse_file()
            kwargs['detached_files'] = True
        if is_xsiam:
            marketplace = MarketplaceVersions.MarketplaceV2.value
        else:
            marketplace = MarketplaceVersions.XSOAR.value
        os.environ[ENV_DEMISTO_SDK_MARKETPLACE] = marketplace.lower()

        output_zip_path = keep_zip or tempfile.mkdtemp()
        packs_unifier = PacksZipper(pack_paths=pack_path, output=output_zip_path,
                                    content_version='0.0.0', zip_all=True, quiet_mode=True, marketplace=marketplace)
        packs_zip_path, pack_names = packs_unifier.zip_packs()
        if packs_zip_path is None and not kwargs.get('detached_files'):
            return EX_FAIL

        kwargs['input'] = packs_zip_path
        kwargs['pack_names'] = pack_names

    check_configuration_file('upload', kwargs)
    upload_result = Uploader(**kwargs).upload()
    if (is_zip or config_file_path) and not keep_zip:
        shutil.rmtree(output_zip_path, ignore_errors=True)
    return upload_result


# demisto_sdk/commands/upload/uploader.py

import ast
import glob
import logging
import os
from typing import List, Tuple, Union

import click
import demisto_client
from demisto_client.demisto_api.rest import ApiException
from packaging.version import Version
from tabulate import tabulate

from demisto_sdk.commands.common.constants import (CLASSIFIERS_DIR, CONTENT_ENTITIES_DIRS, DASHBOARDS_DIR,
                                                   INCIDENT_FIELDS_DIR, INCIDENT_TYPES_DIR, INDICATOR_FIELDS_DIR,
                                                   INDICATOR_TYPES_DIR, INTEGRATIONS_DIR, JOBS_DIR, LAYOUTS_DIR,
                                                   LISTS_DIR, PLAYBOOKS_DIR, REPORTS_DIR, SCRIPTS_DIR,
                                                   TEST_PLAYBOOKS_DIR, WIDGETS_DIR, FileType)
from demisto_sdk.commands.common.content.errors import ContentFactoryError
from demisto_sdk.commands.common.content.objects.abstract_objects import JSONObject, YAMLObject
from demisto_sdk.commands.common.content.objects.pack_objects.pack import Pack
from demisto_sdk.commands.common.content.objects_factory import path_to_pack_object
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.tools import (find_type, get_child_directories, get_demisto_version, get_file,
                                               get_parent_directory_name, print_v)

json = JSON_Handler()


# These are the class names of the objects in demisto_sdk.commands.common.content.objects
UPLOAD_SUPPORTED_ENTITIES = [
    FileType.INTEGRATION,
    FileType.BETA_INTEGRATION,
    FileType.SCRIPT,
    FileType.TEST_SCRIPT,

    FileType.PLAYBOOK,
    FileType.TEST_PLAYBOOK,

    FileType.OLD_CLASSIFIER,
    FileType.CLASSIFIER,
    FileType.MAPPER,

    FileType.INCIDENT_TYPE,
    FileType.INCIDENT_FIELD,
    FileType.REPUTATION,
    FileType.INDICATOR_FIELD,

    FileType.WIDGET,
    FileType.REPORT,
    FileType.DASHBOARD,
    FileType.LAYOUT,
    FileType.LAYOUTS_CONTAINER,
    FileType.LISTS,
    FileType.JOB
]


UNIFIED_ENTITIES_DIR = [INTEGRATIONS_DIR, SCRIPTS_DIR]

CONTENT_ENTITY_UPLOAD_ORDER = [
    INTEGRATIONS_DIR,
    SCRIPTS_DIR,
    PLAYBOOKS_DIR,
    TEST_PLAYBOOKS_DIR,
    INCIDENT_TYPES_DIR,
    INCIDENT_FIELDS_DIR,
    INDICATOR_FIELDS_DIR,
    INDICATOR_TYPES_DIR,
    CLASSIFIERS_DIR,
    WIDGETS_DIR,
    LAYOUTS_DIR,
    LISTS_DIR,
    JOBS_DIR,
    DASHBOARDS_DIR,
    REPORTS_DIR
]
SUCCESS_RETURN_CODE = 0
ERROR_RETURN_CODE = 1
ABORTED_RETURN_CODE = 2


class Uploader:
    """Upload a pack specified in self.infile to a remote Cortex XSOAR instance.
        Attributes:
            path (str): The path of a pack / directory / file to upload.
            verbose (bool): Whether to output a detailed response.
            client (DefaultApi): Demisto-SDK client object.
        """

    def __init__(self, input: str, insecure: bool = False, verbose: bool = False, pack_names: list = None,
                 skip_validation: bool = False, detached_files: bool = False, reattach: bool = False):
        self.path = input
        self.log_verbose = verbose
        verify = (not insecure) if insecure else None  # set to None so demisto_client will use env var DEMISTO_VERIFY_SSL
        self.client = demisto_client.configure(verify_ssl=verify)
        self.successfully_uploaded_files: List[Tuple[str, str]] = []
        self.failed_uploaded_files: List[Tuple[str, str, str]] = []
        self.unuploaded_due_to_version: List[Tuple[str, str, Version, Version, Version]] = []
        self.demisto_version = get_demisto_version(self.client)
        self.pack_names = pack_names
        self.skip_upload_packs_validation = skip_validation
        self.is_files_to_detached = detached_files
        self.reattach_files = reattach

    def upload(self):
        """Upload the pack / directory / file to the remote Cortex XSOAR instance.
        """
        if self.demisto_version == "0":
            click.secho("Could not connect to XSOAR server. Try checking your connection configurations.",
                        fg="bright_red")
            return ERROR_RETURN_CODE

        status_code = SUCCESS_RETURN_CODE

        if self.is_files_to_detached:
            item_detacher = ItemDetacher(client=self.client)
            list_detach_items_ids: list = item_detacher.detach_item_manager(upload_file=True)

            if self.reattach_files:
                item_reattacher = ItemReattacher(client=self.client)
                item_reattacher.reattach_item_manager(detached_files_ids=list_detach_items_ids)

            if not self.path:
                return SUCCESS_RETURN_CODE
        host = self.client.api_client.configuration.host
        click.secho(f'Using {host=}')
        click.secho(f"Uploading {self.path} ...")
        if self.path is None or not os.path.exists(self.path):
            click.secho(f'Error: Given input path: {self.path} does not exist', fg='bright_red')
            return ERROR_RETURN_CODE

        # uploading a pack zip
        elif self.path.endswith('.zip'):
            status_code = self.zipped_pack_uploader(path=self.path, skip_validation=self.skip_upload_packs_validation)

        # Uploading a file
        elif os.path.isfile(self.path):
            status_code = self.file_uploader(self.path) or status_code

        # Uploading an entity directory
        elif os.path.isdir(self.path):
            parent_dir_name = get_parent_directory_name(self.path)
            if parent_dir_name in UNIFIED_ENTITIES_DIR:
                status_code = self.unified_entity_uploader(self.path) or status_code
            elif os.path.basename(self.path.rstrip('/')) in CONTENT_ENTITIES_DIRS:
                status_code = self.entity_dir_uploader(self.path) or status_code
            else:
                status_code = self.pack_uploader(self.path) or status_code

        if status_code == ABORTED_RETURN_CODE:
            return status_code

        if not self.successfully_uploaded_files \
                and not self.failed_uploaded_files \
                and not self.unuploaded_due_to_version:
            # if not uploaded any file
            click.secho(
                f'\nError: Given input path: {self.path} is not uploadable. '
                f'Input path should point to one of the following:\n'
                f'  1. Pack\n'
                f'  2. A content entity directory that is inside a pack. For example: an Integrations directory or '
                f'a Layouts directory\n'
                f'  3. Valid file that can be imported to Cortex XSOAR manually. '
                f'For example a playbook: helloWorld.yml',
                fg='bright_red'
            )
            return ERROR_RETURN_CODE

        print_summary(self.successfully_uploaded_files, self.unuploaded_due_to_version, self.failed_uploaded_files)
        return status_code

    def file_uploader(self, path: str) -> int:
        """
        Upload a file.
        Args:
            path: The path of the file to upload. The rest of the parameters are taken from self.

        Returns:

        """
        try:
            upload_object: Union[YAMLObject, JSONObject] = path_to_pack_object(path)
        except ContentFactoryError:
            file_name = os.path.split(path)[-1]
            message = f"Cannot upload {path} as the file type is not supported for upload."
            if self.log_verbose:
                click.secho(message, fg='bright_red')
            self.failed_uploaded_files.append((file_name, "Unknown", message))
            return ERROR_RETURN_CODE

        file_name = upload_object.path.name  # type: ignore

        entity_type = find_type(str(upload_object.path))
        if entity_type in UPLOAD_SUPPORTED_ENTITIES:
            if upload_object.from_version <= self.demisto_version <= upload_object.to_version:  # type: ignore
                try:
                    result = upload_object.upload(self.client)  # type: ignore
                    if self.log_verbose:
                        if hasattr(result, 'to_str'):
                            print_v(f'Result:\n{result.to_str()}', self.log_verbose)
                        else:
                            print_v(f'Result:\n{result}', self.log_verbose)
                        click.secho(f'Uploaded {entity_type} - \'{os.path.basename(path)}\': successfully', fg='green')
                    self.successfully_uploaded_files.append((file_name, entity_type.value))
                    return SUCCESS_RETURN_CODE
                except Exception as err:
                    message = parse_error_response(err, entity_type, file_name, self.log_verbose)
                    self.failed_uploaded_files.append((file_name, entity_type.value, message))
                    return ERROR_RETURN_CODE
            else:
                if self.log_verbose:
                    click.secho(f"Input path {path} is not uploading due to version mismatch.\n"
                                f"XSOAR version is: {self.demisto_version} while the file's version is "
                                f"{upload_object.from_version} - {upload_object.to_version}", fg='bright_red')
                self.unuploaded_due_to_version.append((file_name, entity_type.value, self.demisto_version,  # type: ignore
                                                       upload_object.from_version, upload_object.to_version))
                return ERROR_RETURN_CODE
        else:
            if self.log_verbose:
                click.secho(
                    f'\nError: Given input path: {path} is not uploadable. '
                    f'Input path should point to one of the following:\n'
                    f'  1. Pack\n'
                    f'  2. A content entity directory that is inside a pack. For example: an Integrations directory or '
                    f'a Layouts directory\n'
                    f'  3. Valid file that can be imported to Cortex XSOAR manually. '
                    f'For example a playbook: helloWorld.yml',
                    fg='bright_red'
                )
            self.failed_uploaded_files.append((file_name, entity_type.value, 'Unsuported file path/type'))
            return ERROR_RETURN_CODE

    def unified_entity_uploader(self, path) -> int:
        """
        Uploads unified entity folder

        Args:
            path: the folder path of a unified entity in the format `Pack/{Pack_Name}/Integration/{Integration_Name}`

        Returns:
            status code
        """
        if get_parent_directory_name(path) not in UNIFIED_ENTITIES_DIR:
            return ERROR_RETURN_CODE
        yml_files = []
        for file in glob.glob(f"{path}/*.yml"):
            if not file.endswith('_unified.yml'):
                yml_files.append(file)
        if len(yml_files) > 1:
            self.failed_uploaded_files.append((path, "Entity Folder",
                                               "The folder contains more than one `.yml` file "
                                               "(not including `_unified.yml`)"))
            return ERROR_RETURN_CODE
        if not yml_files:
            self.failed_uploaded_files.append((path, "Entity Folder", "The folder does not contain a `.yml` file"))
            return ERROR_RETURN_CODE
        return self.file_uploader(yml_files[0])

    def entity_dir_uploader(self, path: str) -> int:
        """
        Uploads an entity path directory
        Args:
            path: an entity path in the following format `Packs/{Pack_Name}/{Entity_Type}`

        Returns:
            The status code of the operation.

        """
        status_code = SUCCESS_RETURN_CODE
        dir_name = os.path.basename(path.rstrip('/'))
        if dir_name in UNIFIED_ENTITIES_DIR:
            for entity_folder in glob.glob(f"{path}/*/"):
                status_code = self.unified_entity_uploader(entity_folder) or status_code
        if dir_name in CONTENT_ENTITIES_DIRS:
            # upload json or yml files. Other files such as `.md`, `.png` should be ignored
            for file in glob.glob(f"{path}/*.yml"):
                status_code = self.file_uploader(file) or status_code
            for file in glob.glob(f"{path}/*.json"):
                status_code = self.file_uploader(file) or status_code
        return status_code

    def pack_uploader(self, path: str) -> int:
        status_code = SUCCESS_RETURN_CODE
        sorted_directories = sort_directories_based_on_dependencies(get_child_directories(path))
        for entity_folder in sorted_directories:
            if os.path.basename(entity_folder.rstrip('/')) in CONTENT_ENTITIES_DIRS:
                status_code = self.entity_dir_uploader(entity_folder) or status_code
        return status_code

    def zipped_pack_uploader(self, path: str, skip_validation: bool) -> int:

        zipped_pack = Pack(path)

        try:
            logger = logging.getLogger('demisto-sdk')

            if not self.pack_names:
                self.pack_names = [zipped_pack.path.stem]

            if self.notify_user_should_override_packs():
                zipped_pack.upload(logger, self.client, skip_validation)
                self.successfully_uploaded_files.extend([(pack_name, FileType.PACK.value) for pack_name in self.pack_names])
                return SUCCESS_RETURN_CODE

            return ABORTED_RETURN_CODE

        except (Exception, KeyboardInterrupt) as err:
            file_name = zipped_pack.path.name  # type: ignore
            message = parse_error_response(err, FileType.PACK.value, file_name, self.log_verbose)
            self.failed_uploaded_files.append((file_name, FileType.PACK.value, message))
            return ERROR_RETURN_CODE

    def notify_user_should_override_packs(self):
        """Notify the user about possible overridden packs"""

        response = self.client.generic_request('/contentpacks/metadata/installed', "GET")
        installed_packs = eval(response[0])
        if installed_packs:
            installed_packs = {pack['name'] for pack in installed_packs}
            common_packs = installed_packs & set(self.pack_names)  # type: ignore
            if common_packs:
                pack_names = '\n'.join(common_packs)
                click.secho(f'This command will overwrite the following packs:\n{pack_names}.\n'
                            'Any changes made on XSOAR will be lost.\n'
                            'Are you sure you want to continue? Y/[N]', fg='bright_red')
                answer = str(input())
                return answer in ['y', 'Y', 'yes']

        return True


def parse_error_response(error: ApiException, file_type: str, file_name: str, print_error: bool = False):
    """
    Parses error message from exception raised in call to client to upload a file

    error (ApiException): The exception which was raised in call in to client
    file_type (str): The file type which was attempted to be uploaded
    file_name (str): The file name which was attempted to be uploaded
    """
    message = error
    if hasattr(error, 'reason'):
        if '[SSL: CERTIFICATE_VERIFY_FAILED]' in str(error.reason):
            message = '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate.\n' \
                      'Try running the command with --insecure flag.'

        elif 'Failed to establish a new connection:' in str(error.reason):
            message = 'Failed to establish a new connection: Connection refused.\n' \
                      'Try checking your BASE url configuration.'

        elif error.reason in ('Bad Request', 'Forbidden'):
            error_body = json.loads(error.body)
            message = error_body.get('error')

            if error_body.get('status') == 403:
                message += '\nTry checking your API key configuration.'
    if print_error:
        click.secho(str(f'\nUpload {file_type}: {file_name} failed:'), fg='bright_red')
        click.secho(str(message), fg='bright_red')
    if isinstance(error, KeyboardInterrupt):
        message = 'Aborted due to keyboard interrupt.'
    return message


def print_summary(successfully_uploaded_files, unuploaded_due_to_version, failed_uploaded_files):
    """Prints uploaded files summary
    Successful uploads grid based on `successfully_uploaded_files` attribute in green color
    Failed uploads grid based on `failed_uploaded_files` attribute in red color
    """
    click.secho('\n\nUPLOAD SUMMARY:')
    if successfully_uploaded_files:
        click.secho('\nSUCCESSFUL UPLOADS:', fg='green')
        click.secho(tabulate(successfully_uploaded_files, headers=['NAME', 'TYPE'],
                             tablefmt="fancy_grid") + '\n', fg='green')
    if unuploaded_due_to_version:
        click.secho('\nNOT UPLOADED DUE TO VERSION MISMATCH:', fg='yellow')
        click.secho(tabulate(unuploaded_due_to_version, headers=['NAME', 'TYPE', 'XSOAR Version',
                                                                 'FILE_FROM_VERSION', 'FILE_TO_VERSION'],
                             tablefmt="fancy_grid") + '\n', fg='yellow')
    if failed_uploaded_files:
        click.secho('\nFAILED UPLOADS:', fg='bright_red')
        click.secho(tabulate(failed_uploaded_files, headers=['NAME', 'TYPE', 'ERROR'],
                             tablefmt="fancy_grid") + '\n', fg='bright_red')


def sort_directories_based_on_dependencies(dir_list: list) -> list:
    """
    Sorts given list of directories based on logic order of content entities that depend on each other.
    If a given directory does not appear in the CONTENT_ENTITY_UPLOAD_ORDER list it will be ignored

    Args:
        dir_list (List): List of directories to sort

    Returns:
        List. The sorted list of directories.
    """
    srt = {item: index for index, item in enumerate(CONTENT_ENTITY_UPLOAD_ORDER)}
    dir_list_copy = dir_list.copy()
    for dir_path in dir_list_copy:
        if os.path.basename(dir_path) not in CONTENT_ENTITY_UPLOAD_ORDER:
            dir_list.remove(dir_path)
    dir_list.sort(
        key=lambda item: srt.get(os.path.basename(item))  # type: ignore[arg-type, return-value]
    )
    return dir_list


class ConfigFileParser:
    """Parse configuration file to get a list of custom packs to upload to a remote Cortex XSOAR instance.
        Attributes:
            config_file_path (str): The path of the configuration file.
        """

    def __init__(self, config_file_path: str):
        self.config_file_path = config_file_path

    def parse_file(self):
        config_file_data = self.get_file_data()
        custom_packs_paths = self.get_custom_packs_paths(config_file_data)
        return custom_packs_paths

    def get_file_data(self):
        with open(self.config_file_path) as config_file:
            config_file_data = json.load(config_file)
        return config_file_data

    def get_custom_packs_paths(self, config_file_data):
        custom_packs = config_file_data.get('custom_packs', [])
        custom_packs_paths = ",".join(pack.get('url') for pack in custom_packs)
        return custom_packs_paths


class ItemDetacher:
    def __init__(self, client, file_path: str = 'SystemPacks'):
        self.file_path = file_path
        self.client = client

    DETACH_ITEM_TYPE_TO_ENDPOINT: dict = {
        'IncidentTypes': '/incidenttype/detach/:id/',
        'Layouts': '/layout/:id/detach/',
        'Playbooks': '/playbook/detach/:id/',
        'Scripts': '/automation/detach/:id/',
    }

    VALID_FILES_FOR_DETACH = ['Playbooks', 'Scripts', 'IncidentTypes', 'Layouts']

    def detach_item(self, file_id, file_path):
        endpoint: str = ''
        for file_type, file_endpoint in self.DETACH_ITEM_TYPE_TO_ENDPOINT.items():
            if file_type in file_path:
                endpoint = file_endpoint
                break
        endpoint = endpoint.replace(':id', file_id)

        try:
            self.client.generic_request(endpoint, "POST")
            click.secho(f'\nFile: {file_id} was detached', fg='green')
        except Exception as e:
            raise Exception(f'Exception raised when fetching custom content:\n{e}')

    def extract_items_from_dir(self):
        detach_files_list: list = []

        all_files = glob.glob(f'{self.file_path}/**/*', recursive=True)
        for file_path in all_files:
            if os.path.isfile(file_path) and self.is_valid_file_for_detach(file_path):
                file_type = self.find_item_type_to_detach(file_path)
                file_data = get_file(file_path, file_type)
                file_id = file_data.get('id')
                if file_id:
                    detach_files_list.append({'file_id': file_id, 'file_type': file_type, 'file_path': file_path})
        return detach_files_list

    def is_valid_file_for_detach(self, file_path: str) -> bool:
        for file in self.VALID_FILES_FOR_DETACH:
            if file in file_path and (file_path.endswith('yml') or file_path.endswith('json')):
                return True
        return False

    def find_item_type_to_detach(self, file_path) -> str:
        return 'yml' if 'Playbooks' in file_path or 'Scripts' in file_path else 'json'

    def find_item_id_to_detach(self):
        file_type = self.find_item_type_to_detach(self.file_path)
        file_data = get_file(self.file_path, file_type)
        file_id = file_data.get('id')
        return file_id

    def detach_item_manager(self, upload_file: bool = False):
        detach_files_list: list = []
        if os.path.isdir(self.file_path):
            detach_files_list = self.extract_items_from_dir()
            for file in detach_files_list:
                self.detach_item(file.get('file_id'), file_path=file.get('file_path'))
                if upload_file:
                    uploader = Uploader(input=file.get('file_path'))
                    uploader.upload()

        elif os.path.isfile(self.file_path):
            file_id = self.find_item_id_to_detach()
            detach_files_list.append({'file_id': file_id, 'file_path': self.file_path})
            self.detach_item(file_id=file_id, file_path=self.file_path)
            if upload_file:
                uploader = Uploader(input=self.file_path)
                uploader.upload()

        detached_items_ids = [file.get('file_id') for file in detach_files_list]
        return detached_items_ids


class ItemReattacher:

    def __init__(self, client, file_path: str = ''):
        self.file_path = file_path
        self.client = client

    REATTACH_ITEM_TYPE_TO_ENDPOINT: dict = {
        'IncidentType': '/incidenttype/attach/:id',
        'Layouts': '/layout/:id/attach',
        'Playbooks': '/playbook/attach/:id',
        'Automations': '/automation/attach/:id',
    }

    def download_all_detach_supported_items(self) -> dict:
        all_detach_supported_items: dict = {}
        yml_req_body = {"query": "system:T"}

        for endpoint in ['/playbook/search', '/automation/search']:
            res = self.client.generic_request(endpoint, 'POST', body=yml_req_body)
            res_result = ast.literal_eval(res[0])
            if 'playbook' in endpoint:
                all_detach_supported_items['Playbooks'] = res_result.get('playbooks')
            else:
                all_detach_supported_items['Automations'] = res_result.get('scripts')

        for item_type in ['IncidentType', 'Layouts']:
            endpoint = item_type.lower()
            res = self.client.generic_request(endpoint, 'GET')
            all_detach_supported_items[item_type] = ast.literal_eval(res[0])

        return all_detach_supported_items

    def reattach_item(self, item_id, item_type):
        endpoint: str = self.REATTACH_ITEM_TYPE_TO_ENDPOINT[item_type]
        endpoint = endpoint.replace(':id', item_id)
        try:
            self.client.generic_request(endpoint, 'POST')
            click.secho(f'\n{item_type}: {item_id} was reattached', fg='green')
        except Exception as e:
            raise Exception(f'Exception raised when fetching custom content:\n{e}')

    def reattach_item_manager(self, detached_files_ids=None):
        if not self.file_path and detached_files_ids:
            all_files: dict = self.download_all_detach_supported_items()
            for item_type, item_list in all_files.items():
                for item in item_list:
                    if not item.get('detached', '') or item.get('detached', '') == 'false':
                        continue
                    item_id = item.get('id')
                    if item_id and item_id not in detached_files_ids:
                        self.reattach_item(item_id, item_type)


# demisto_sdk/commands/validate/validate_manager.py

import os
from concurrent.futures._base import Future, as_completed
from configparser import ConfigParser, MissingSectionHeaderError
from pathlib import Path
from typing import Callable, List, Optional, Set, Tuple

import click
import pebble
from colorama import Fore
from git import InvalidGitRepositoryError
from packaging import version

from demisto_sdk.commands.common import tools
from demisto_sdk.commands.common.configuration import Configuration
from demisto_sdk.commands.common.constants import (API_MODULES_PACK, AUTHOR_IMAGE_FILE_NAME, CONTENT_ENTITIES_DIRS,
                                                   DEFAULT_CONTENT_ITEM_TO_VERSION, GENERIC_FIELDS_DIR,
                                                   GENERIC_TYPES_DIR, IGNORED_PACK_NAMES, OLDEST_SUPPORTED_VERSION,
                                                   PACKS_DIR, PACKS_PACK_META_FILE_NAME, SKIP_RELEASE_NOTES_FOR_TYPES,
                                                   VALIDATION_USING_GIT_IGNORABLE_DATA, FileType,
                                                   FileType_ALLOWED_TO_DELETE, PathLevel)
from demisto_sdk.commands.common.content import Content
from demisto_sdk.commands.common.content_constant_paths import DEFAULT_ID_SET_PATH
from demisto_sdk.commands.common.errors import (FOUND_FILES_AND_ERRORS, FOUND_FILES_AND_IGNORED_ERRORS,
                                                PRESET_ERROR_TO_CHECK, PRESET_ERROR_TO_IGNORE, Errors,
                                                get_all_error_codes)
from demisto_sdk.commands.common.git_util import GitUtil
from demisto_sdk.commands.common.hook_validations.author_image import AuthorImageValidator
from demisto_sdk.commands.common.hook_validations.base_validator import BaseValidator, error_codes
from demisto_sdk.commands.common.hook_validations.classifier import ClassifierValidator
from demisto_sdk.commands.common.hook_validations.conf_json import ConfJsonValidator
from demisto_sdk.commands.common.hook_validations.correlation_rule import CorrelationRuleValidator
from demisto_sdk.commands.common.hook_validations.dashboard import DashboardValidator
from demisto_sdk.commands.common.hook_validations.deprecation import DeprecationValidator
from demisto_sdk.commands.common.hook_validations.description import DescriptionValidator
from demisto_sdk.commands.common.hook_validations.generic_definition import GenericDefinitionValidator
from demisto_sdk.commands.common.hook_validations.generic_field import GenericFieldValidator
from demisto_sdk.commands.common.hook_validations.generic_module import GenericModuleValidator
from demisto_sdk.commands.common.hook_validations.generic_type import GenericTypeValidator
from demisto_sdk.commands.common.hook_validations.id import IDSetValidations
from demisto_sdk.commands.common.hook_validations.image import ImageValidator
from demisto_sdk.commands.common.hook_validations.incident_field import IncidentFieldValidator
from demisto_sdk.commands.common.hook_validations.incident_type import IncidentTypeValidator
from demisto_sdk.commands.common.hook_validations.indicator_field import IndicatorFieldValidator
from demisto_sdk.commands.common.hook_validations.integration import IntegrationValidator
from demisto_sdk.commands.common.hook_validations.job import JobValidator
from demisto_sdk.commands.common.hook_validations.layout import LayoutsContainerValidator, LayoutValidator
from demisto_sdk.commands.common.hook_validations.lists import ListsValidator
from demisto_sdk.commands.common.hook_validations.mapper import MapperValidator
from demisto_sdk.commands.common.hook_validations.modeling_rule import ModelingRuleValidator
from demisto_sdk.commands.common.hook_validations.pack_unique_files import PackUniqueFilesValidator
from demisto_sdk.commands.common.hook_validations.parsing_rule import ParsingRuleValidator
from demisto_sdk.commands.common.hook_validations.playbook import PlaybookValidator
from demisto_sdk.commands.common.hook_validations.pre_process_rule import PreProcessRuleValidator
from demisto_sdk.commands.common.hook_validations.python_file import PythonFileValidator
from demisto_sdk.commands.common.hook_validations.readme import ReadMeValidator
from demisto_sdk.commands.common.hook_validations.release_notes import ReleaseNotesValidator
from demisto_sdk.commands.common.hook_validations.release_notes_config import ReleaseNotesConfigValidator
from demisto_sdk.commands.common.hook_validations.report import ReportValidator
from demisto_sdk.commands.common.hook_validations.reputation import ReputationValidator
from demisto_sdk.commands.common.hook_validations.script import ScriptValidator
from demisto_sdk.commands.common.hook_validations.structure import StructureValidator
from demisto_sdk.commands.common.hook_validations.test_playbook import TestPlaybookValidator
from demisto_sdk.commands.common.hook_validations.triggers import TriggersValidator
from demisto_sdk.commands.common.hook_validations.widget import WidgetValidator
from demisto_sdk.commands.common.hook_validations.wizard import WizardValidator
from demisto_sdk.commands.common.hook_validations.xsiam_dashboard import XSIAMDashboardValidator
from demisto_sdk.commands.common.hook_validations.xsiam_report import XSIAMReportValidator
from demisto_sdk.commands.common.hook_validations.xsoar_config_json import XSOARConfigJsonValidator
from demisto_sdk.commands.common.tools import (_get_file_id, find_type, get_api_module_ids,
                                               get_api_module_integrations_set, get_content_path, get_file,
                                               get_pack_ignore_file_path, get_pack_name, get_pack_names_from_files,
                                               get_relative_path_from_packs_dir, get_remote_file, get_yaml,
                                               open_id_set_file, print_warning, run_command_os)
from demisto_sdk.commands.create_id_set.create_id_set import IDSetCreator

SKIPPED_FILES = ['CommonServerPython.py', 'CommonServerUserPython.py', 'demistomock.py', 'DemistoClassApiModule.py']


class ValidateManager:
    def __init__(
            self, is_backward_check=True, prev_ver=None, use_git=False, only_committed_files=False,
            print_ignored_files=False, skip_conf_json=True, validate_id_set=False, file_path=None,
            validate_all=False, is_external_repo=False, skip_pack_rn_validation=False, print_ignored_errors=False,
            silence_init_prints=False, no_docker_checks=False, skip_dependencies=False, id_set_path=None, staged=False,
            create_id_set=False, json_file_path=None, skip_schema_check=False, debug_git=False, include_untracked=False,
            pykwalify_logs=False, check_is_unskipped=True, quiet_bc=False, multiprocessing=True,
            specific_validations=None,
    ):
        # General configuration
        self.skip_docker_checks = False
        self.no_configuration_prints = silence_init_prints
        self.skip_conf_json = skip_conf_json
        self.is_backward_check = is_backward_check
        self.is_circle = only_committed_files
        self.validate_all = validate_all
        self.use_git = use_git
        self.skip_pack_rn_validation = skip_pack_rn_validation
        self.print_ignored_files = print_ignored_files
        self.print_ignored_errors = print_ignored_errors
        self.skip_dependencies = skip_dependencies or not use_git
        self.skip_id_set_creation = not create_id_set or skip_dependencies
        self.compare_type = '...'
        self.staged = staged
        self.skip_schema_check = skip_schema_check
        self.debug_git = debug_git
        self.include_untracked = include_untracked
        self.pykwalify_logs = pykwalify_logs
        self.quiet_bc = quiet_bc
        self.check_is_unskipped = check_is_unskipped
        self.conf_json_data = {}
        self.run_with_multiprocessing = multiprocessing
        self.is_possible_validate_readme = self.is_node_exist() or ReadMeValidator.is_docker_available()

        if json_file_path:
            self.json_file_path = os.path.join(json_file_path, 'validate_outputs.json') if \
                os.path.isdir(json_file_path) else json_file_path
        else:
            self.json_file_path = ''

        self.specific_validations = specific_validations
        if specific_validations:
            self.specific_validations = specific_validations.split(',')

        # Class constants
        self.handle_error = BaseValidator(print_as_warnings=print_ignored_errors,
                                          json_file_path=json_file_path).handle_error
        self.should_run_validation = BaseValidator(specific_validations=specific_validations).should_run_validation
        self.file_path = file_path
        self.id_set_path = id_set_path or DEFAULT_ID_SET_PATH
        # create the id_set only once per run.
        self.id_set_file = self.get_id_set_file(self.skip_id_set_creation, self.id_set_path)

        self.id_set_validations = IDSetValidations(is_circle=self.is_circle,
                                                   configuration=Configuration(),
                                                   ignored_errors=None,
                                                   print_as_warnings=self.print_ignored_errors,
                                                   id_set_file=self.id_set_file,
                                                   json_file_path=json_file_path,
                                                   specific_validations=self.specific_validations) if validate_id_set else None

        self.deprecation_validator = DeprecationValidator(id_set_file=self.id_set_file)

        try:
            self.git_util = GitUtil(repo=Content.git())
            self.branch_name = self.git_util.get_current_git_branch_or_hash()
        except (InvalidGitRepositoryError, TypeError):
            # if we are using git - fail the validation by raising the exception.
            if self.use_git:
                raise
            # if we are not using git - simply move on.
            else:
                click.echo('Unable to connect to git')
                self.git_util = None  # type: ignore[assignment]
                self.branch_name = ''

        if prev_ver and not prev_ver.startswith('origin'):
            self.prev_ver = self.setup_prev_ver('origin/' + prev_ver)
        else:
            self.prev_ver = self.setup_prev_ver(prev_ver)

        self.check_only_schema = False
        self.always_valid = False
        self.ignored_files = set()
        self.new_packs = set()
        self.skipped_file_types = (FileType.CHANGELOG,
                                   FileType.DOC_IMAGE,
                                   FileType.MODELING_RULE_SCHEMA,
                                   FileType.XSIAM_DASHBOARD_IMAGE,
                                   FileType.XSIAM_REPORT_IMAGE,
                                   FileType.XSIAM_DASHBOARD_IMAGE,
                                   FileType.XDRC_TEMPLATE_YML,
                                   FileType.XDRC_TEMPLATE,)

        self.is_external_repo = is_external_repo
        if is_external_repo:
            if not self.no_configuration_prints:
                click.echo('Running in a private repository')
            self.skip_conf_json = True

        self.print_percent = False
        self.completion_percentage = 0

        if validate_all:
            # No need to check docker images on build branch hence we do not check on -a mode
            # also do not skip id set creation unless the flag is up
            self.skip_docker_checks = True
            self.skip_pack_rn_validation = True
            self.print_percent = not self.run_with_multiprocessing  # the Multiprocessing will mismatch the percent
            self.check_is_unskipped = False

        if no_docker_checks:
            self.skip_docker_checks = True

        if self.skip_conf_json:
            self.check_is_unskipped = False

        if not self.skip_conf_json:
            self.conf_json_validator = ConfJsonValidator(specific_validations=self.specific_validations)
            self.conf_json_data = self.conf_json_validator.conf_data

    def is_node_exist(self) -> bool:
        """ Check if node interpreter exists.
        Returns:
            bool: True if node exist, else False
        """
        # Check node exist
        content_path = get_content_path()
        stdout, stderr, exit_code = run_command_os('node -v', cwd=content_path)  # type: ignore
        if exit_code:
            return False
        return True

    def print_final_report(self, valid):
        self.print_ignored_files_report(self.print_ignored_files)
        self.print_ignored_errors_report(self.print_ignored_errors)

        if valid:
            click.secho('\nThe files are valid', fg='green')
            return 0

        else:
            all_failing_files = '\n'.join(set(FOUND_FILES_AND_ERRORS))
            click.secho(f"\n=========== Found errors in the following files ===========\n\n{all_failing_files}\n",
                        fg="bright_red")

            if self.always_valid:
                click.secho('Found the errors above, but not failing build', fg='yellow')
                return 0

            click.secho('The files were found as invalid, the exact error message can be located above',
                        fg='red')
            return 1

    def run_validation(self):
        """Initiates validation in accordance with mode (i,g,a)
        """
        if self.validate_all:
            is_valid = self.run_validation_on_all_packs()
        elif self.use_git:
            is_valid = self.run_validation_using_git()
        elif self.file_path:
            is_valid = self.run_validation_on_specific_files()
        else:
            # default validate to -g --post-commit
            self.use_git = True
            self.is_circle = True
            is_valid = self.run_validation_using_git()
        return self.print_final_report(is_valid)

    @staticmethod
    def detect_file_level(file_path: str) -> PathLevel:
        """
        Detect the whether the path points to a file, a content entity dir, a content generic entity dir
        (i.e GenericFields or GenericTypes), a pack dir or package dir

        Args:
             file_path(str): the path to check.

        Returns:
            PathLevel. File, ContentDir, ContentGenericDir, Pack or Package - depending on the file path level.
        """
        if os.path.isfile(file_path):
            return PathLevel.FILE

        file_path = file_path.rstrip('/')
        dir_name = os.path.basename(file_path)
        if dir_name in CONTENT_ENTITIES_DIRS:
            return PathLevel.CONTENT_ENTITY_DIR

        if str(os.path.dirname(file_path)).endswith(GENERIC_TYPES_DIR) or \
                str(os.path.dirname(file_path)).endswith(GENERIC_FIELDS_DIR):
            return PathLevel.CONTENT_GENERIC_ENTITY_DIR

        if os.path.basename(os.path.dirname(file_path)) == PACKS_DIR:
            return PathLevel.PACK

        else:
            return PathLevel.PACKAGE

    def run_validation_on_specific_files(self):
        """Run validations only on specific files
        """
        files_validation_result = set()

        for path in self.file_path.split(','):
            error_ignore_list = self.get_error_ignore_list(get_pack_name(path))
            file_level = self.detect_file_level(path)

            if file_level == PathLevel.FILE:
                click.secho(f'\n================= Validating file {path} =================', fg="bright_cyan")
                files_validation_result.add(self.run_validations_on_file(path, error_ignore_list))

            elif file_level == PathLevel.CONTENT_ENTITY_DIR:
                click.secho(f'\n================= Validating content directory {path} =================',
                            fg="bright_cyan")
                files_validation_result.add(self.run_validation_on_content_entities(path, error_ignore_list))

            elif file_level == PathLevel.CONTENT_GENERIC_ENTITY_DIR:
                click.secho(f'\n================= Validating content directory {path} =================',
                            fg="bright_cyan")
                files_validation_result.add(self.run_validation_on_generic_entities(path, error_ignore_list))

            elif file_level == PathLevel.PACK:
                click.secho(f'\n================= Validating pack {path} =================',
                            fg="bright_cyan")
                files_validation_result.add(self.run_validations_on_pack(path)[0])

            else:
                click.secho(f'\n================= Validating package {path} =================',
                            fg="bright_cyan")
                files_validation_result.add(self.run_validation_on_package(path, error_ignore_list))

        return all(files_validation_result)

    def wait_futures_complete(self, futures_list: List[Future], done_fn: Callable):
        """Wait for all futures to complete, Raise exception if occurred.
        Args:
            futures_list: futures to wait for.
            done_fn: Function to run on result.
        Raises:
            Exception: Raise caught exception for further cleanups.
        """
        for future in as_completed(futures_list):
            try:
                result = future.result()
                done_fn(result[0], result[1])
            except Exception as e:
                click.secho(f'An error occurred while tried to collect result, Error: {e}', fg="bright_red")
                raise

    def run_validation_on_all_packs(self):
        """Runs validations on all files in all packs in repo (-a option)

        Returns:
            bool. true if all files are valid, false otherwise.
        """
        click.secho('\n================= Validating all files =================', fg="bright_cyan")
        all_packs_valid = set()

        if not self.skip_conf_json:
            all_packs_valid.add(self.conf_json_validator.is_valid_conf_json())

        count = 1
        # Filter non-pack files that might exist locally (e.g, .DS_STORE on MacOS)
        all_packs = list(filter(os.path.isdir, [os.path.join(PACKS_DIR, p) for p in os.listdir(PACKS_DIR)]))
        num_of_packs = len(all_packs)
        all_packs.sort(key=str.lower)

        ReadMeValidator.add_node_env_vars()
        if self.is_possible_validate_readme:
            with ReadMeValidator.start_mdx_server(handle_error=self.handle_error):
                return self.validate_packs(all_packs, all_packs_valid, count, num_of_packs)
        else:
            return self.validate_packs(all_packs, all_packs_valid, count, num_of_packs)

    def validate_packs(self, all_packs: list, all_packs_valid: set,
                       count: int, num_of_packs: int) -> bool:

        if self.run_with_multiprocessing:
            with pebble.ProcessPool(max_workers=4) as executor:
                futures = []
                for pack_path in all_packs:
                    futures.append(executor.schedule(self.run_validations_on_pack, args=(pack_path,)))
                self.wait_futures_complete(futures_list=futures,
                                           done_fn=lambda x, y: (all_packs_valid.add(x),  # type: ignore
                                                                 FOUND_FILES_AND_ERRORS.extend(y)))  # type: ignore
        else:
            for pack_path in all_packs:
                self.completion_percentage = format((count / num_of_packs) * 100, ".2f")  # type: ignore
                all_packs_valid.add(self.run_validations_on_pack(pack_path)[0])
                count += 1
        return all(all_packs_valid)

    def run_validations_on_pack(self, pack_path):
        """Runs validation on all files in given pack. (i,g,a)

        Args:
            pack_path: the path to the pack.

        Returns:
            bool. true if all files in pack are valid, false otherwise.
        """
        pack_entities_validation_results = set()
        pack_error_ignore_list = self.get_error_ignore_list(os.path.basename(pack_path))

        pack_entities_validation_results.add(self.validate_pack_unique_files(pack_path, pack_error_ignore_list))

        for content_dir in os.listdir(pack_path):
            content_entity_path = os.path.join(pack_path, content_dir)
            if content_dir in CONTENT_ENTITIES_DIRS:
                pack_entities_validation_results.add(self.run_validation_on_content_entities(content_entity_path,
                                                                                             pack_error_ignore_list))
            else:
                self.ignored_files.add(content_entity_path)

        return all(pack_entities_validation_results), FOUND_FILES_AND_ERRORS

    def run_validation_on_content_entities(self, content_entity_dir_path, pack_error_ignore_list):
        """Gets non-pack folder and runs validation within it (Scripts, Integrations...)

        Returns:
            bool. true if all files in directory are valid, false otherwise.
        """
        content_entities_validation_results = set()
        if content_entity_dir_path.endswith(GENERIC_FIELDS_DIR) or content_entity_dir_path.endswith(GENERIC_TYPES_DIR):
            for dir_name in os.listdir(content_entity_dir_path):
                dir_path = os.path.join(content_entity_dir_path, dir_name)
                if not os.path.isfile(dir_path):
                    # should be only directories (not files) in generic types/fields directory
                    content_entities_validation_results.add(
                        self.run_validation_on_generic_entities(dir_path, pack_error_ignore_list))
                else:
                    self.ignored_files.add(dir_path)
        else:
            for file_name in os.listdir(content_entity_dir_path):
                file_path = os.path.join(content_entity_dir_path, file_name)
                if os.path.isfile(file_path):
                    if file_path.endswith('.json') or file_path.endswith('.yml') or file_path.endswith('.md'):
                        content_entities_validation_results.add(self.run_validations_on_file(file_path,
                                                                                             pack_error_ignore_list))
                    else:
                        self.ignored_files.add(file_path)

                else:
                    content_entities_validation_results.add(self.run_validation_on_package(file_path,
                                                                                           pack_error_ignore_list))

        return all(content_entities_validation_results)

    def run_validation_on_package(self, package_path, pack_error_ignore_list):
        package_entities_validation_results = set()

        for file_name in os.listdir(package_path):
            file_path = os.path.join(package_path, file_name)
            should_validate_py_file = file_path.endswith('.py') and file_name not in SKIPPED_FILES
            if file_path.endswith('.yml') or file_path.endswith('.md') or should_validate_py_file:
                package_entities_validation_results.add(self.run_validations_on_file(file_path, pack_error_ignore_list))

            else:
                self.ignored_files.add(file_path)

        return all(package_entities_validation_results)

    def run_validation_on_generic_entities(self, dir_path, pack_error_ignore_list):
        """
        Gets a generic content entity directory (i.e a sub-directory of GenericTypes or GenericFields)
        and runs validation within it.

        Returns:
            bool. true if all files in directory are valid, false otherwise.
        """
        package_entities_validation_results = set()

        for file_name in os.listdir(dir_path):
            file_path = os.path.join(dir_path, file_name)
            if file_path.endswith('.json'):  # generic types/fields are jsons
                package_entities_validation_results.add(self.run_validations_on_file(file_path, pack_error_ignore_list))
            else:
                self.ignored_files.add(file_path)

        return all(package_entities_validation_results)

    @error_codes('BA114')
    def is_valid_pack_name(self, file_path, old_file_path):
        """
        Valid pack name is currently considered to be a new pack name or an existing pack.
        If pack name is changed, will return `False`.
        """
        if not old_file_path:
            return True
        original_pack_name = get_pack_name(old_file_path)
        new_pack_name = get_pack_name(file_path)
        if original_pack_name != new_pack_name:
            error_message, error_code = Errors.changed_pack_name(original_pack_name)
            if self.handle_error(error_message=error_message, error_code=error_code, file_path=file_path,
                                 drop_line=True):
                return False
        return True

    @error_codes('BA102,IM110')
    def is_valid_file_type(self, file_type: FileType, file_path: str):
        """
        If a file_type is unsupported, will return `False`.
        """
        if not file_type:
            error_message, error_code = Errors.file_type_not_supported(file_type, file_path)
            if str(file_path).endswith('.png'):
                error_message, error_code = Errors.invalid_image_name_or_location()
            if self.handle_error(error_message=error_message, error_code=error_code, file_path=file_path,
                                 drop_line=True):
                return False

        return True

    # flake8: noqa: C901
    def run_validations_on_file(self, file_path, pack_error_ignore_list, is_modified=False,
                                old_file_path=None, modified_files=None, added_files=None):
        """Choose a validator to run for a single file. (i)

        Args:
            modified_files: A set of modified files - used for RN validation
            added_files: A set of added files - used for RN validation
            old_file_path: The old file path for renamed files
            pack_error_ignore_list: A dictionary of all pack ignored errors
            file_path: the file on which to run.
            is_modified: whether the file is modified or added.

        Returns:
            bool. true if file is valid, false otherwise.
        """
        if not self.is_valid_pack_name(file_path, old_file_path):
            return False
        file_type = find_type(file_path)

        is_added_file = file_path in added_files if added_files else False

        if file_type in self.skipped_file_types or file_path.endswith('_unified.yml'):
            self.ignored_files.add(file_path)
            return True
        elif not self.is_valid_file_type(file_type, file_path):
            return False

        if file_type == FileType.XSOAR_CONFIG:
            xsoar_config_validator = XSOARConfigJsonValidator(file_path, specific_validations=self.specific_validations)
            return xsoar_config_validator.is_valid_xsoar_config_file()

        if not self.check_only_schema:
            validation_print = f"\nValidating {file_path} as {file_type.value}"
            if self.print_percent:
                if FOUND_FILES_AND_ERRORS:
                    validation_print += f' {Fore.RED}[{self.completion_percentage}%]{Fore.RESET}'
                else:
                    validation_print += f' {Fore.GREEN}[{self.completion_percentage}%]{Fore.RESET}'

            click.echo(validation_print)

        structure_validator = StructureValidator(file_path, predefined_scheme=file_type,
                                                 ignored_errors=pack_error_ignore_list,
                                                 print_as_warnings=self.print_ignored_errors, tag=self.prev_ver,
                                                 old_file_path=old_file_path, branch_name=self.branch_name,
                                                 is_new_file=not is_modified,
                                                 json_file_path=self.json_file_path,
                                                 skip_schema_check=self.skip_schema_check,
                                                 pykwalify_logs=self.pykwalify_logs,
                                                 quiet_bc=self.quiet_bc,
                                                 specific_validations=self.specific_validations)

        # schema validation
        if file_type not in {FileType.TEST_PLAYBOOK, FileType.TEST_SCRIPT, FileType.DESCRIPTION}:
            if not structure_validator.is_valid_file():
                return False

        # Passed schema validation
        # if only schema validation is required - stop check here
        if self.check_only_schema:
            return True

        # id_set validation
        if self.id_set_validations and not self.id_set_validations.is_file_valid_in_set(file_path, file_type,
                                                                                        pack_error_ignore_list):
            return False

        # conf.json validation
        valid_in_conf = True
        if self.check_is_unskipped and file_type in {FileType.INTEGRATION, FileType.SCRIPT, FileType.BETA_INTEGRATION}:
            if not self.conf_json_validator.is_valid_file_in_conf_json(structure_validator.current_file, file_type,
                                                                       file_path):
                valid_in_conf = False

        # Note: these file are not ignored but there are no additional validators for connections
        if file_type == FileType.CONNECTION:
            return True

        # test playbooks and test scripts are using the same validation.
        elif file_type in {FileType.TEST_PLAYBOOK, FileType.TEST_SCRIPT}:
            return self.validate_test_playbook(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.RELEASE_NOTES:
            if not self.skip_pack_rn_validation:
                return self.validate_release_notes(file_path, added_files, modified_files, pack_error_ignore_list,
                                                   is_modified)
            else:
                click.secho('Skipping release notes validation', fg='yellow')

        elif file_type == FileType.RELEASE_NOTES_CONFIG:
            return self.validate_release_notes_config(file_path, pack_error_ignore_list)

        elif file_type == FileType.DESCRIPTION:
            return self.validate_description(file_path, pack_error_ignore_list)

        elif file_type == FileType.README:
            if not self.is_possible_validate_readme:
                error_message, error_code = Errors.error_uninstall_node()
                if self.handle_error(error_message=error_message, error_code=error_code,
                                     file_path=file_path):
                    return False
            if not self.validate_all:
                ReadMeValidator.add_node_env_vars()
                if not ReadMeValidator.are_modules_installed_for_verify(
                    get_content_path()  # type: ignore
                ) and not ReadMeValidator.is_docker_available():  # shows warning message
                    return True
                with ReadMeValidator.start_mdx_server(handle_error=self.handle_error):
                    return self.validate_readme(file_path, pack_error_ignore_list)
            return self.validate_readme(file_path, pack_error_ignore_list)

        elif file_type == FileType.REPORT:
            return self.validate_report(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.PLAYBOOK:
            return self.validate_playbook(structure_validator, pack_error_ignore_list, file_type, is_modified)

        elif file_type == FileType.INTEGRATION:
            return all([self.validate_integration(structure_validator, pack_error_ignore_list, is_modified,
                                                  file_type), valid_in_conf])

        elif file_type == FileType.SCRIPT:
            return all([self.validate_script(structure_validator, pack_error_ignore_list, is_modified,
                                             file_type), valid_in_conf])
        elif file_type == FileType.PYTHON_FILE:
            return self.validate_python_file(file_path, pack_error_ignore_list)

        elif file_type == FileType.BETA_INTEGRATION:
            return self.validate_beta_integration(structure_validator, pack_error_ignore_list)

        # Validate only images of packs
        elif file_type == FileType.IMAGE:
            return self.validate_image(file_path, pack_error_ignore_list)

        elif file_type == FileType.AUTHOR_IMAGE:
            return self.validate_author_image(file_path, pack_error_ignore_list)

        elif file_type == FileType.INCIDENT_FIELD:
            return self.validate_incident_field(structure_validator, pack_error_ignore_list, is_modified, is_added_file)

        elif file_type == FileType.INDICATOR_FIELD:
            return self.validate_indicator_field(structure_validator, pack_error_ignore_list, is_modified,
                                                 is_added_file)

        elif file_type == FileType.REPUTATION:
            return self.validate_reputation(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.LAYOUT:
            return self.validate_layout(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.LAYOUTS_CONTAINER:
            return self.validate_layoutscontainer(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.PRE_PROCESS_RULES:
            return self.validate_pre_process_rule(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.LISTS:
            return self.validate_lists(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.DASHBOARD:
            return self.validate_dashboard(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.INCIDENT_TYPE:
            return self.validate_incident_type(structure_validator, pack_error_ignore_list, is_modified)

        elif file_type == FileType.MAPPER:
            return self.validate_mapper(structure_validator, pack_error_ignore_list, is_modified)

        elif file_type in (FileType.OLD_CLASSIFIER, FileType.CLASSIFIER):
            return self.validate_classifier(structure_validator, pack_error_ignore_list, file_type)

        elif file_type == FileType.WIDGET:
            return self.validate_widget(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.TRIGGER:
            return self.validate_triggers(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.PARSING_RULE:
            return self.validate_parsing_rule(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.MODELING_RULE:
            return self.validate_modeling_rule(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.CORRELATION_RULE:
            return self.validate_correlation_rule(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.XSIAM_DASHBOARD:
            return self.validate_xsiam_dashboard(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.XSIAM_REPORT:
            return self.validate_xsiam_report(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.WIZARD:
            return self.validate_wizard(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.GENERIC_FIELD:
            return self.validate_generic_field(structure_validator, pack_error_ignore_list, is_added_file)

        elif file_type == FileType.GENERIC_TYPE:
            return self.validate_generic_type(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.GENERIC_MODULE:
            return self.validate_generic_module(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.GENERIC_DEFINITION:
            return self.validate_generic_definition(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.JOB:
            return self.validate_job(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.CONTRIBUTORS:
            # This is temporarily - need to add a proper contributors validations
            return True

        else:
            return self.file_type_not_supported(file_type, file_path)
        return True

    @error_codes('BA102')
    def file_type_not_supported(self, file_type: FileType, file_path: str):
        error_message, error_code = Errors.file_type_not_supported(file_type, file_path)
        if self.handle_error(error_message=error_message, error_code=error_code, file_path=file_path):
            return False
        return True

    @staticmethod
    def get_file_by_status(modified_files: Set, old_format_files: Set,
                           file_path: str) -> Tuple[Set, Set, Set]:
        """Given a specific file path identify in which git status set
        it exists and return a set containing that file and 2 additional empty sets.

        Args:
            modified_files(Set): A set of modified and renamed files.
            old_format_files(Set): A set of old format files.
            file_path(str): The file path to check.

        Returns:
            Tuple[Set, Set, Set]. 3 sets representing modified, added or old format files respectively
            where the file path is in the appropriate set
        """
        filtered_modified_files: Set = set()
        filtered_added_files: Set = set()
        filtered_old_format: Set = set()

        # go through modified files and try to identify if the file is there
        for file in modified_files:
            if isinstance(file, str) and file == file_path:
                filtered_modified_files.add(file_path)
                return filtered_modified_files, filtered_added_files, filtered_old_format

            # handle renamed files which are in tuples
            elif file_path in file:
                filtered_modified_files.add(file)
                return filtered_modified_files, filtered_added_files, filtered_old_format

        # if the file is not modified check if it is in old format files
        if file_path in old_format_files:
            filtered_old_format.add(file_path)

        else:
            # if not found in either modified or old format consider the file newly added
            filtered_added_files.add(file_path)

        return filtered_modified_files, filtered_added_files, filtered_old_format

    @staticmethod
    def specify_files_from_directory(file_set: Set, directory_path: str) -> Set:
        """Filter a set of file paths to only include ones which are from a specified directory.

        Args:
            file_set(Set): A set of file paths - could be stings or tuples for rename files.
            directory_path(str): the directory path in which to check for the files.

        Returns:
            Set. A set of all the paths of files that appear in the given directory.
        """
        filtered_set: Set = set()
        for file in file_set:
            if isinstance(file, str) and directory_path in file:
                filtered_set.add(file)

            # handle renamed files
            elif isinstance(file, tuple) and directory_path in file[1]:
                filtered_set.add(file)

        return filtered_set

    def specify_files_by_status(self, modified_files: Set, added_files: Set, old_format_files: Set) -> \
            Tuple[Set, Set, Set]:
        """Filter the files identified from git to only specified files.

        Args:
            modified_files(Set): A set of modified and renamed files.
            added_files(Set): A set of added files.
            old_format_files(Set): A set of old format files.

        Returns:
            Tuple[Set, Set, Set]. 3 sets for modified, added an old format files where the only files that
            appear are the ones specified by the 'file_path' ValidateManager parameter
        """
        filtered_modified_files: Set = set()
        filtered_added_files: Set = set()
        filtered_old_format: Set = set()

        for path in self.file_path.split(','):
            path = get_relative_path_from_packs_dir(path)
            file_level = self.detect_file_level(path)
            if file_level == PathLevel.FILE:
                temp_modified, temp_added, temp_old_format = self.get_file_by_status(modified_files,
                                                                                     old_format_files, path)
                filtered_modified_files = filtered_modified_files.union(temp_modified)
                filtered_added_files = filtered_added_files.union(temp_added)
                filtered_old_format = filtered_old_format.union(temp_old_format)

            else:
                filtered_modified_files = filtered_modified_files.union(
                    self.specify_files_from_directory(modified_files, path))
                filtered_added_files = filtered_added_files.union(
                    self.specify_files_from_directory(added_files, path))
                filtered_old_format = filtered_old_format.union(
                    self.specify_files_from_directory(old_format_files, path))

        return filtered_modified_files, filtered_added_files, filtered_old_format

    def run_validation_using_git(self):
        """Runs validation on only changed packs/files (g)
        """
        valid_git_setup = self.setup_git_params()
        if not self.no_configuration_prints:
            self.print_git_config()

        modified_files, added_files, changed_meta_files, old_format_files, valid_types = \
            self.get_changed_files_from_git()

        # filter to only specified paths if given
        if self.file_path:
            modified_files, added_files, old_format_files = self.specify_files_by_status(modified_files, added_files,
                                                                                         old_format_files)
        deleted_files = self.git_util.deleted_files(prev_ver=self.prev_ver, committed_only=self.is_circle,
                                                    staged_only=self.staged, include_untracked=self.include_untracked)

        validation_results = {valid_git_setup, valid_types}

        validation_results.add(self.validate_modified_files(modified_files))
        validation_results.add(self.validate_added_files(added_files, modified_files))
        validation_results.add(self.validate_changed_packs_unique_files(modified_files, added_files, old_format_files,
                                                                        changed_meta_files))
        validation_results.add(self.validate_deleted_files(deleted_files, added_files))

        if old_format_files:
            click.secho(f'\n================= Running validation on old format files =================',
                        fg="bright_cyan")
            validation_results.add(self.validate_no_old_format(old_format_files))

        if not self.skip_pack_rn_validation:
            validation_results.add(self.validate_no_duplicated_release_notes(added_files))
            validation_results.add(self.validate_no_missing_release_notes(modified_files, old_format_files,
                                                                          added_files))

        return all(validation_results)

    """ ######################################## Unique Validations ####################################### """

    def validate_description(self, file_path, pack_error_ignore_list):
        description_validator = DescriptionValidator(file_path, ignored_errors=pack_error_ignore_list,
                                                     print_as_warnings=self.print_ignored_errors,
                                                     json_file_path=self.json_file_path,
                                                     specific_validations=self.specific_validations)
        return description_validator.is_valid_file()

    def validate_readme(self, file_path, pack_error_ignore_list):
        readme_validator = ReadMeValidator(file_path, ignored_errors=pack_error_ignore_list,
                                           print_as_warnings=self.print_ignored_errors,
                                           json_file_path=self.json_file_path,
                                           specific_validations=self.specific_validations)
        return readme_validator.is_valid_file()

    def validate_python_file(self, file_path, pack_error_ignore_list):
        python_file_validator = PythonFileValidator(file_path, ignored_errors=pack_error_ignore_list,
                                                    print_as_warnings=self.print_ignored_errors,
                                                    json_file_path=self.json_file_path,
                                                    specific_validations=self.specific_validations)
        return python_file_validator.is_valid_file()

    def validate_test_playbook(self, structure_validator, pack_error_ignore_list):
        test_playbook_validator = TestPlaybookValidator(structure_validator=structure_validator,
                                                        ignored_errors=pack_error_ignore_list,
                                                        print_as_warnings=self.print_ignored_errors,
                                                        json_file_path=self.json_file_path)
        return test_playbook_validator.is_valid_test_playbook(validate_rn=False)

    @error_codes('RN108')
    def validate_no_release_notes_for_new_pack(self, pack_name, file_path):
        if pack_name in self.new_packs:
            error_message, error_code = Errors.added_release_notes_for_new_pack(pack_name)
            if self.handle_error(error_message=error_message, error_code=error_code, file_path=file_path):
                return False
        return True

    def validate_release_notes(self, file_path, added_files, modified_files, pack_error_ignore_list, is_modified):
        pack_name = get_pack_name(file_path)

        # added new RN to a new pack
        if not self.validate_no_release_notes_for_new_pack(pack_name, file_path):
            return False

        if pack_name != 'NonSupported':
            if not added_files:
                added_files = {file_path}

            release_notes_validator = ReleaseNotesValidator(file_path, pack_name=pack_name,
                                                            modified_files=modified_files,
                                                            added_files=added_files,
                                                            ignored_errors=pack_error_ignore_list,
                                                            print_as_warnings=self.print_ignored_errors,
                                                            json_file_path=self.json_file_path,
                                                            specific_validations=self.specific_validations)
            return release_notes_validator.is_file_valid()

        return True

    def validate_release_notes_config(self, file_path: str, pack_error_ignore_list: list) -> bool:
        """
        Builds validator for RN config file and returns its validation results.
        Args:
            file_path (str): Path to RN config file.
            pack_error_ignore_list (list): Pack error ignore list.

        Returns:
            (bool): Whether RN config file is valid.
        """
        pack_name = get_pack_name(file_path)
        if pack_name == 'NonSupported':
            return True
        release_notes_config_validator = ReleaseNotesConfigValidator(file_path, ignored_errors=pack_error_ignore_list,
                                                                     print_as_warnings=self.print_ignored_errors,
                                                                     json_file_path=self.json_file_path,
                                                                     specific_validations=self.specific_validations)
        return release_notes_config_validator.is_file_valid()

    def validate_playbook(self, structure_validator, pack_error_ignore_list, file_type, is_modified):
        playbook_validator = PlaybookValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                               print_as_warnings=self.print_ignored_errors,
                                               json_file_path=self.json_file_path,
                                               validate_all=self.validate_all,
                                               deprecation_validator=self.deprecation_validator)

        deprecated_result = self.check_and_validate_deprecated(file_type=file_type,
                                                               file_path=structure_validator.file_path,
                                                               current_file=playbook_validator.current_file,
                                                               is_modified=True,
                                                               is_backward_check=False,
                                                               validator=playbook_validator)
        if deprecated_result is not None:
            return deprecated_result

        return playbook_validator.is_valid_playbook(validate_rn=False,
                                                    id_set_file=self.id_set_file,
                                                    is_modified=is_modified)

    def validate_integration(self, structure_validator, pack_error_ignore_list, is_modified, file_type):
        integration_validator = IntegrationValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                     print_as_warnings=self.print_ignored_errors,
                                                     skip_docker_check=self.skip_docker_checks,
                                                     json_file_path=self.json_file_path,
                                                     validate_all=self.validate_all,
                                                     deprecation_validator=self.deprecation_validator
                                                     )

        deprecated_result = self.check_and_validate_deprecated(file_type=file_type,
                                                               file_path=structure_validator.file_path,
                                                               current_file=integration_validator.current_file,
                                                               is_modified=is_modified,
                                                               is_backward_check=self.is_backward_check,
                                                               validator=integration_validator)
        if deprecated_result is not None:
            return deprecated_result
        if is_modified and self.is_backward_check:
            return all([integration_validator.is_valid_file(validate_rn=False, skip_test_conf=self.skip_conf_json,
                                                            check_is_unskipped=self.check_is_unskipped,
                                                            conf_json_data=self.conf_json_data,
                                                            is_modified=is_modified),
                        integration_validator.is_backward_compatible()])
        else:
            return integration_validator.is_valid_file(validate_rn=False, skip_test_conf=self.skip_conf_json,
                                                       check_is_unskipped=self.check_is_unskipped,
                                                       conf_json_data=self.conf_json_data)

    def validate_script(self, structure_validator, pack_error_ignore_list, is_modified, file_type):
        script_validator = ScriptValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                           print_as_warnings=self.print_ignored_errors,
                                           skip_docker_check=self.skip_docker_checks,
                                           json_file_path=self.json_file_path,
                                           validate_all=self.validate_all,
                                           deprecation_validator=self.deprecation_validator)

        deprecated_result = self.check_and_validate_deprecated(file_type=file_type,
                                                               file_path=structure_validator.file_path,
                                                               current_file=script_validator.current_file,
                                                               is_modified=is_modified,
                                                               is_backward_check=self.is_backward_check,
                                                               validator=script_validator)
        if deprecated_result is not None:
            return deprecated_result

        if is_modified and self.is_backward_check:
            return all([script_validator.is_valid_file(validate_rn=False),
                        script_validator.is_backward_compatible()])
        else:
            return script_validator.is_valid_file(validate_rn=False)

    def validate_beta_integration(self, structure_validator, pack_error_ignore_list):
        integration_validator = IntegrationValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                     print_as_warnings=self.print_ignored_errors,
                                                     skip_docker_check=self.skip_docker_checks,
                                                     json_file_path=self.json_file_path,
                                                     validate_all=self.validate_all,
                                                     )
        return integration_validator.is_valid_beta_integration()

    def validate_image(self, file_path, pack_error_ignore_list):
        pack_name = get_pack_name(file_path)
        if pack_name == 'NonSupported':
            return True
        image_validator = ImageValidator(file_path, ignored_errors=pack_error_ignore_list,
                                         print_as_warnings=self.print_ignored_errors,
                                         json_file_path=self.json_file_path,
                                         specific_validations=self.specific_validations)
        return image_validator.is_valid()

    def validate_author_image(self, file_path, pack_error_ignore_list):
        author_image_validator: AuthorImageValidator = AuthorImageValidator(file_path,
                                                                            ignored_errors=pack_error_ignore_list,
                                                                            print_as_warnings=self.print_ignored_errors,
                                                                            specific_validations=self.specific_validations)
        return author_image_validator.is_valid()

    def validate_report(self, structure_validator, pack_error_ignore_list):
        report_validator = ReportValidator(structure_validator=structure_validator,
                                           ignored_errors=pack_error_ignore_list,
                                           print_as_warnings=self.print_ignored_errors,
                                           json_file_path=self.json_file_path)
        return report_validator.is_valid_file(validate_rn=False)

    def validate_incident_field(self, structure_validator, pack_error_ignore_list, is_modified, is_added_file):
        incident_field_validator = IncidentFieldValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                          print_as_warnings=self.print_ignored_errors,
                                                          json_file_path=self.json_file_path,
                                                          id_set_file=self.id_set_file)
        if is_modified and self.is_backward_check:
            return all([incident_field_validator.is_valid_file(validate_rn=False, is_new_file=not is_modified,
                                                               use_git=self.use_git,
                                                               is_added_file=is_added_file),
                        incident_field_validator.is_backward_compatible()])
        else:
            return incident_field_validator.is_valid_file(validate_rn=False, is_new_file=not is_modified,
                                                          use_git=self.use_git,
                                                          is_added_file=is_added_file)

    def validate_indicator_field(self, structure_validator, pack_error_ignore_list, is_modified, is_added_file):
        indicator_field_validator = IndicatorFieldValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                            print_as_warnings=self.print_ignored_errors,
                                                            json_file_path=self.json_file_path)
        if is_modified and self.is_backward_check:
            return all([indicator_field_validator.is_valid_file(validate_rn=False, is_new_file=not is_modified,
                                                                use_git=self.use_git,
                                                                is_added_file=is_added_file),
                        indicator_field_validator.is_backward_compatible()])
        else:
            return indicator_field_validator.is_valid_file(validate_rn=False, is_new_file=not is_modified,
                                                           use_git=self.use_git,
                                                           is_added_file=is_added_file)

    def validate_reputation(self, structure_validator, pack_error_ignore_list):
        reputation_validator = ReputationValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                   print_as_warnings=self.print_ignored_errors,
                                                   json_file_path=self.json_file_path)
        return reputation_validator.is_valid_file(validate_rn=False)

    def validate_layout(self, structure_validator, pack_error_ignore_list):
        layout_validator = LayoutValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                           print_as_warnings=self.print_ignored_errors,
                                           json_file_path=self.json_file_path)
        return layout_validator.is_valid_layout(validate_rn=False, id_set_file=self.id_set_file,
                                                is_circle=self.is_circle)

    def validate_layoutscontainer(self, structure_validator, pack_error_ignore_list):
        layout_validator = LayoutsContainerValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                     print_as_warnings=self.print_ignored_errors,
                                                     json_file_path=self.json_file_path)
        return layout_validator.is_valid_layout(validate_rn=False, id_set_file=self.id_set_file,
                                                is_circle=self.is_circle)

    def validate_pre_process_rule(self, structure_validator, pack_error_ignore_list):
        pre_process_rules_validator = PreProcessRuleValidator(structure_validator,
                                                              ignored_errors=pack_error_ignore_list,
                                                              print_as_warnings=self.print_ignored_errors,
                                                              json_file_path=self.json_file_path)
        return pre_process_rules_validator.is_valid_pre_process_rule(validate_rn=False, id_set_file=self.id_set_file,
                                                                     is_ci=self.is_circle)

    def validate_lists(self, structure_validator, pack_error_ignore_list):
        lists_validator = ListsValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                         print_as_warnings=self.print_ignored_errors,
                                         json_file_path=self.json_file_path)
        return lists_validator.is_valid_list()

    def validate_dashboard(self, structure_validator, pack_error_ignore_list):
        dashboard_validator = DashboardValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                 print_as_warnings=self.print_ignored_errors,
                                                 json_file_path=self.json_file_path)
        return dashboard_validator.is_valid_dashboard(validate_rn=False)

    def validate_incident_type(self, structure_validator, pack_error_ignore_list, is_modified):
        incident_type_validator = IncidentTypeValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                        print_as_warnings=self.print_ignored_errors,
                                                        json_file_path=self.json_file_path)
        if is_modified and self.is_backward_check:
            return all([incident_type_validator.is_valid_incident_type(validate_rn=False),
                        incident_type_validator.is_backward_compatible()])
        else:
            return incident_type_validator.is_valid_incident_type(validate_rn=False)

    def validate_mapper(self, structure_validator, pack_error_ignore_list, is_modified):
        mapper_validator = MapperValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                           print_as_warnings=self.print_ignored_errors,
                                           json_file_path=self.json_file_path)
        if is_modified and self.is_backward_check:
            return all([mapper_validator.is_valid_mapper(validate_rn=False, id_set_file=self.id_set_file,
                                                         is_circle=self.is_circle),
                        mapper_validator.is_backward_compatible()])

        return mapper_validator.is_valid_mapper(validate_rn=False, id_set_file=self.id_set_file,
                                                is_circle=self.is_circle)

    def validate_classifier(self, structure_validator, pack_error_ignore_list, file_type):
        if file_type == FileType.CLASSIFIER:
            new_classifier_version = True

        else:
            new_classifier_version = False

        classifier_validator = ClassifierValidator(structure_validator, new_classifier_version=new_classifier_version,
                                                   ignored_errors=pack_error_ignore_list,
                                                   print_as_warnings=self.print_ignored_errors,
                                                   json_file_path=self.json_file_path)
        return classifier_validator.is_valid_classifier(validate_rn=False,
                                                        id_set_file=self.id_set_file,
                                                        is_circle=self.is_circle)

    def validate_widget(self, structure_validator, pack_error_ignore_list):
        widget_validator = WidgetValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                           print_as_warnings=self.print_ignored_errors,
                                           json_file_path=self.json_file_path)
        return widget_validator.is_valid_file(validate_rn=False)

    def validate_triggers(self, structure_validator, pack_error_ignore_list):
        triggers_validator = TriggersValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                               print_as_warnings=self.print_ignored_errors,
                                               json_file_path=self.json_file_path)
        return triggers_validator.is_valid_file(validate_rn=False)

    def validate_xsiam_report(self, structure_validator, pack_error_ignore_list):
        xsiam_report_validator = XSIAMReportValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                      print_as_warnings=self.print_ignored_errors,
                                                      json_file_path=self.json_file_path)
        return xsiam_report_validator.is_valid_file(validate_rn=False)

    def validate_xsiam_dashboard(self, structure_validator, pack_error_ignore_list):
        xsiam_dashboard_validator = XSIAMDashboardValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                            print_as_warnings=self.print_ignored_errors,
                                                            json_file_path=self.json_file_path)
        return xsiam_dashboard_validator.is_valid_file(validate_rn=False)

    def validate_parsing_rule(self, structure_validator, pack_error_ignore_list):
        parsing_rule_validator = ParsingRuleValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                      print_as_warnings=self.print_ignored_errors,
                                                      json_file_path=self.json_file_path)
        return parsing_rule_validator.is_valid_file(validate_rn=False)

    def validate_correlation_rule(self, structure_validator, pack_error_ignore_list):
        correlation_rule_validator = CorrelationRuleValidator(structure_validator,
                                                              ignored_errors=pack_error_ignore_list,
                                                              print_as_warnings=self.print_ignored_errors,
                                                              json_file_path=self.json_file_path)
        return correlation_rule_validator.is_valid_file(validate_rn=False)

    def validate_modeling_rule(self, structure_validator, pack_error_ignore_list):
        modeling_rule_validator = ModelingRuleValidator(structure_validator,
                                                        ignored_errors=pack_error_ignore_list,
                                                        print_as_warnings=self.print_ignored_errors,
                                                        json_file_path=self.json_file_path)
        return modeling_rule_validator.is_valid_file(validate_rn=False)

    def validate_wizard(self, structure_validator, pack_error_ignore_list):
        wizard_validator = WizardValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                           print_as_warnings=self.print_ignored_errors,
                                           json_file_path=self.json_file_path)
        return wizard_validator.is_valid_file(validate_rn=False, id_set_file=self.id_set_file)

    def validate_generic_field(self, structure_validator, pack_error_ignore_list, is_added_file):
        generic_field_validator = GenericFieldValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                        print_as_warnings=self.print_ignored_errors,
                                                        json_file_path=self.json_file_path)

        return generic_field_validator.is_valid_file(validate_rn=False, is_added_file=is_added_file)

    def validate_generic_type(self, structure_validator, pack_error_ignore_list):
        generic_type_validator = GenericTypeValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                      print_as_warnings=self.print_ignored_errors,
                                                      json_file_path=self.json_file_path)

        return generic_type_validator.is_valid_file(validate_rn=False)

    def validate_generic_module(self, structure_validator, pack_error_ignore_list):
        generic_module_validator = GenericModuleValidator(structure_validator, ignored_errors=pack_error_ignore_list,
                                                          print_as_warnings=self.print_ignored_errors,
                                                          json_file_path=self.json_file_path)

        return generic_module_validator.is_valid_file(validate_rn=False)

    def validate_generic_definition(self, structure_validator, pack_error_ignore_list):
        generic_definition_validator = GenericDefinitionValidator(structure_validator,
                                                                  ignored_errors=pack_error_ignore_list,
                                                                  print_as_warnings=self.print_ignored_errors,
                                                                  json_file_path=self.json_file_path)

        return generic_definition_validator.is_valid_file(validate_rn=False)

    def validate_pack_unique_files(self, pack_path: str, pack_error_ignore_list: dict,
                                   should_version_raise=False) -> bool:
        """
        Runs validations on the following pack files:
        * .secret-ignore: Validates that the file exist and that the file's secrets can be parsed as a list delimited by '\n'
        * .pack-ignore: Validates that the file exists and that all regexes in it can be compiled
        * README.md file: Validates that the file exists and image links are valid
        * 2.pack_metadata.json: Validates that the file exists and that it has a valid structure
        * Author_image.png: Validates that the file isn't empty, dimension of 120*50 and that image size is up to 4kb
        Runs validation on the pack dependencies
        Args:
            should_version_raise: Whether we should check if the version of the metadata was raised
            pack_error_ignore_list: A dictionary of all pack ignored errors
            pack_path: A path to a pack
        """
        files_valid = True
        author_valid = True

        click.echo(f'\nValidating {pack_path} unique pack files')
        pack_unique_files_validator = PackUniqueFilesValidator(pack=os.path.basename(pack_path),
                                                               pack_path=pack_path,
                                                               ignored_errors=pack_error_ignore_list,
                                                               print_as_warnings=self.print_ignored_errors,
                                                               should_version_raise=should_version_raise,
                                                               validate_dependencies=not self.skip_dependencies,
                                                               id_set_path=self.id_set_path,
                                                               private_repo=self.is_external_repo,
                                                               skip_id_set_creation=self.skip_id_set_creation,
                                                               prev_ver=self.prev_ver,
                                                               json_file_path=self.json_file_path,
                                                               specific_validations=self.specific_validations)
        pack_errors = pack_unique_files_validator.are_valid_files(self.id_set_validations)
        if pack_errors:
            click.secho(pack_errors, fg="bright_red")
            files_valid = False

        # check author image
        author_image_path = os.path.join(pack_path, AUTHOR_IMAGE_FILE_NAME)
        if os.path.exists(author_image_path):
            click.echo("Validating pack author image")
            author_valid = self.validate_author_image(author_image_path, pack_error_ignore_list)

        return files_valid and author_valid

    def validate_job(self, structure_validator, pack_error_ignore_list):
        job_validator = JobValidator(structure_validator,
                                     pack_error_ignore_list,
                                     print_as_warnings=self.print_ignored_errors,
                                     json_file_path=self.json_file_path)
        is_valid = job_validator.is_valid_file()
        if not is_valid:
            click.secho(job_validator.get_errors(), fg="bright_red")

        return is_valid

    def validate_modified_files(self, modified_files):
        click.secho(f'\n================= Running validation on modified files =================',
                    fg="bright_cyan")
        valid_files = set()
        for file_path in modified_files:
            # handle renamed files
            if isinstance(file_path, tuple):
                old_file_path = file_path[0]
                file_path = file_path[1]

            else:
                old_file_path = None

            pack_name = get_pack_name(file_path)
            valid_files.add(self.run_validations_on_file(file_path, self.get_error_ignore_list(pack_name),
                                                         is_modified=True, old_file_path=old_file_path))
        return all(valid_files)

    def validate_added_files(self, added_files, modified_files):
        click.secho(f'\n================= Running validation on newly added files =================',
                    fg="bright_cyan")

        valid_files = set()
        for file_path in added_files:
            pack_name = get_pack_name(file_path)
            valid_files.add(self.run_validations_on_file(file_path, self.get_error_ignore_list(pack_name),
                                                         is_modified=False, modified_files=modified_files,
                                                         added_files=added_files))
        return all(valid_files)

    @staticmethod
    def should_raise_pack_version(pack: str) -> bool:
        """
        Args:
            pack: The pack name.

        Returns: False if pack is in IGNORED_PACK_NAMES else True.

        """
        return pack not in IGNORED_PACK_NAMES

    @staticmethod
    def is_file_allowed_to_be_deleted(file_path):
        """
        Args:
            file_path: The file path.

        Returns: True if the file allowed to be deleted, else False.

        """
        file_path = str(file_path)
        file_dict = get_remote_file(file_path, tag='master')
        file_type = find_type(file_path, file_dict)
        return file_type in FileType_ALLOWED_TO_DELETE or not file_type

    @staticmethod
    def was_file_renamed_but_labeled_as_deleted(deleted_file_path, added_files):
        """ Check if a file was renamed and not deleted (git false label the file as deleted)
        Args:
            file_path: The file path.

        Returns: True if the file was renamed and not deleted, else False.

        """
        if added_files:
            deleted_file_path = str(deleted_file_path)
            deleted_file_dict = get_remote_file(deleted_file_path, tag='master')  # for detecting deleted files
            if deleted_file_type := find_type(deleted_file_path, deleted_file_dict):
                deleted_file_id = _get_file_id(deleted_file_type.value, deleted_file_dict)
                if deleted_file_id:
                    for file in added_files:
                        file = str(file)
                        file_type = find_type(file)
                        file_dict = get_file(file, file_type.value)
                        if deleted_file_id == _get_file_id(file_type.value, file_dict):
                            return True
        return False

    @error_codes('BA115')
    def validate_deleted_files(self, deleted_files: set, added_files: set) -> bool:
        click.secho(f'\n================= Checking for prohibited deleted files =================',
                    fg="bright_cyan")

        is_valid = True
        for file_path in deleted_files:
            file_path = Path(file_path)
            if 'Packs' not in file_path.absolute().parts:
                # not allowed to delete non-content files
                file_path = str(file_path)
                error_message, error_code = Errors.file_cannot_be_deleted(file_path)
                if self.handle_error(error_message, error_code, file_path):
                    is_valid = False

            else:
                if not self.was_file_renamed_but_labeled_as_deleted(file_path, added_files):
                    if not self.is_file_allowed_to_be_deleted(file_path):
                        file_path = str(file_path)
                        error_message, error_code = Errors.file_cannot_be_deleted(file_path)
                        if self.handle_error(error_message, error_code, file_path):
                            is_valid = False

        return is_valid

    def validate_changed_packs_unique_files(self, modified_files, added_files, old_format_files, changed_meta_files):
        click.secho(f'\n================= Running validation on changed pack unique files =================',
                    fg="bright_cyan")
        valid_pack_files = set()

        added_packs = get_pack_names_from_files(added_files)
        modified_packs = get_pack_names_from_files(modified_files).union(get_pack_names_from_files(old_format_files))
        changed_meta_packs = get_pack_names_from_files(changed_meta_files)

        packs_that_should_have_version_raised = self.get_packs_that_should_have_version_raised(modified_files,
                                                                                               added_files,
                                                                                               old_format_files)

        changed_packs = modified_packs.union(added_packs).union(changed_meta_packs)

        for pack in changed_packs:
            raise_version = False
            pack_path = tools.pack_name_to_path(pack)
            if pack in packs_that_should_have_version_raised:
                raise_version = self.should_raise_pack_version(pack)
            valid_pack_files.add(self.validate_pack_unique_files(pack_path,
                                                                 self.get_error_ignore_list(pack),
                                                                 should_version_raise=raise_version))

        return all(valid_pack_files)

    @error_codes('ST106')
    def validate_no_old_format(self, old_format_files):
        """ Validate there are no files in the old format (unified yml file for the code and configuration
        for python integration).

        Args:
            old_format_files(set): file names which are in the old format.
        """
        handle_error = True
        for file_path in old_format_files:
            click.echo(f"Validating old-format file {file_path}")
            yaml_data = get_yaml(file_path)
            # we only fail on old format if no toversion (meaning it is latest) or if the ynl is not deprecated.
            if 'toversion' not in yaml_data and not yaml_data.get('deprecated'):
                error_message, error_code = Errors.invalid_package_structure()
                if self.handle_error(error_message, error_code, file_path=file_path):
                    handle_error = False
        return handle_error

    @error_codes('RN105')
    def validate_no_duplicated_release_notes(self, added_files):
        """Validated that among the added files - there are no duplicated RN for the same pack.

        Args:
            added_files(set): The added files

        Returns:
            bool. True if no duplications found, false otherwise
        """
        click.secho(f'\n================= Verifying no duplicated release notes =================',
                    fg="bright_cyan")
        added_rn = set()
        for file in added_files:
            if find_type(file) == FileType.RELEASE_NOTES:
                pack_name = get_pack_name(file)
                if pack_name not in added_rn:
                    added_rn.add(pack_name)
                else:
                    error_message, error_code = Errors.multiple_release_notes_files()
                    if self.handle_error(error_message, error_code, file_path=pack_name):
                        return False

        click.secho("\nNo duplicated release notes found.\n", fg="bright_green")
        return True

    @error_codes('RN106')
    def validate_no_missing_release_notes(self, modified_files, old_format_files, added_files):
        """Validate that there are no missing RN for changed files

        Args:
            modified_files (set): a set of modified files.
            old_format_files (set): a set of old format files that were changed.
            added_files (set): a set of files that were added.

        Returns:
            bool. True if no missing RN found, False otherwise
        """
        click.secho("\n================= Checking for missing release notes =================\n", fg="bright_cyan")
        packs_that_should_have_new_rn_api_module_related: set = set()
        # existing packs that have files changed (which are not RN, README nor test files) - should have new RN
        changed_files = modified_files.union(old_format_files).union(added_files)
        packs_that_should_have_new_rn = get_pack_names_from_files(
            changed_files,
            skip_file_types=SKIP_RELEASE_NOTES_FOR_TYPES
        )
        if API_MODULES_PACK in packs_that_should_have_new_rn:
            api_module_set = get_api_module_ids(changed_files)
            integrations = get_api_module_integrations_set(api_module_set,
                                                           self.id_set_file.get('integrations', []))
            packs_that_should_have_new_rn_api_module_related = set(map(lambda integration: integration.get('pack'),
                                                                       integrations))
            packs_that_should_have_new_rn = packs_that_should_have_new_rn.union(
                packs_that_should_have_new_rn_api_module_related)

            # APIModules pack is without a version and should not have RN
            packs_that_should_have_new_rn.remove(API_MODULES_PACK)

        # new packs should not have RN
        packs_that_should_have_new_rn = packs_that_should_have_new_rn - self.new_packs

        packs_that_have_new_rn = self.get_packs_with_added_release_notes(added_files)

        packs_that_have_missing_rn = packs_that_should_have_new_rn.difference(packs_that_have_new_rn)

        if packs_that_have_missing_rn:
            is_valid = set()
            for pack in packs_that_have_missing_rn:
                # # ignore RN in NonSupported pack
                if 'NonSupported' in pack:
                    continue
                ignored_errors_list = self.get_error_ignore_list(pack)
                if pack in packs_that_should_have_new_rn_api_module_related:
                    error_message, error_code = Errors.missing_release_notes_for_pack(API_MODULES_PACK)
                else:
                    error_message, error_code = Errors.missing_release_notes_for_pack(pack)
                if not BaseValidator(ignored_errors=ignored_errors_list,
                                     print_as_warnings=self.print_ignored_errors,
                                     json_file_path=self.json_file_path,
                                     specific_validations=self.specific_validations).handle_error(
                    error_message, error_code,
                    file_path=os.path.join(os.getcwd(), PACKS_DIR, pack, PACKS_PACK_META_FILE_NAME)
                ):
                    is_valid.add(True)

                else:
                    is_valid.add(False)

            return all(is_valid)

        else:
            click.secho("No missing release notes found.\n", fg="bright_green")
            return True

    """ ######################################## Git Tools and filtering ####################################### """

    def setup_prev_ver(self, prev_ver: Optional[str]):
        """Setting up the prev_ver parameter"""
        # if prev_ver parameter is set, use it
        if prev_ver:
            return prev_ver

        # If git is connected - Use it to get prev_ver
        if self.git_util:
            # If demisto exists in remotes if so set prev_ver as 'demisto/master'
            if self.git_util.check_if_remote_exists('demisto'):
                return 'demisto/master'

            # Otherwise, use git to get the primary branch
            _, branch = self.git_util.handle_prev_ver()
            return 'origin/' + branch

        # Default to 'origin/master'
        return 'origin/master'

    def setup_git_params(self):
        """Setting up the git relevant params"""
        self.branch_name = self.git_util.get_current_git_branch_or_hash() if (self.git_util and not self.branch_name) \
            else self.branch_name

        # check remote validity
        if '/' in self.prev_ver and not self.git_util.check_if_remote_exists(self.prev_ver):
            non_existing_remote = self.prev_ver.split("/")[0]
            click.secho(f'Could not find remote {non_existing_remote} reverting to '
                        f'{str(self.git_util.repo.remote())}', fg='bright_red')
            self.prev_ver = self.prev_ver.replace(non_existing_remote, str(self.git_util.repo.remote()))

        # if running on release branch check against last release.
        if self.branch_name.startswith('21.') or self.branch_name.startswith('22.'):
            self.skip_pack_rn_validation = True
            self.prev_ver = os.environ.get('GIT_SHA1')
            self.is_circle = True

            # when running against git while on release branch - show errors but don't fail the validation
            self.always_valid = True

        # On main or master don't check RN
        elif self.branch_name in ['master', 'main']:
            self.skip_pack_rn_validation = True
            error_message, error_code = Errors.running_on_master_with_git()
            if self.handle_error(error_message, error_code, file_path='General',
                                 warning=(not self.is_external_repo or self.is_circle), drop_line=True):
                return False
        return True

    def print_git_config(self):
        click.secho(f'\n================= Running validation on branch {self.branch_name} =================',
                    fg="bright_cyan")
        if not self.no_configuration_prints:
            click.echo(f"Validating against {self.prev_ver}")

            if self.branch_name in [self.prev_ver, self.prev_ver.replace('origin/', '')]:  # pragma: no cover
                click.echo("Running only on last commit")

            elif self.is_circle:
                click.echo("Running only on committed files")

            elif self.staged:
                click.echo("Running only on staged files")

            else:
                click.echo("Running on committed and staged files")

            if self.skip_pack_rn_validation:
                click.echo("Skipping release notes validation")

            if self.skip_docker_checks:
                click.echo("Skipping Docker checks")

            if not self.is_backward_check:
                click.echo("Skipping backwards compatibility checks")

            if self.skip_dependencies:
                click.echo("Skipping pack dependencies check")

    def get_unfiltered_changed_files_from_git(self) -> Tuple[Set, Set, Set]:
        """
        Get the added and modified before file filtration to only relevant files

        Returns:
            3 sets:
            - The unfiltered modified files
            - The unfiltered added files
            - The unfiltered renamed files
        """
        # get files from git by status identification against prev-ver
        modified_files = self.git_util.modified_files(prev_ver=self.prev_ver,
                                                      committed_only=self.is_circle, staged_only=self.staged,
                                                      debug=self.debug_git, include_untracked=self.include_untracked)
        added_files = self.git_util.added_files(prev_ver=self.prev_ver, committed_only=self.is_circle,
                                                staged_only=self.staged, debug=self.debug_git,
                                                include_untracked=self.include_untracked)
        renamed_files = self.git_util.renamed_files(prev_ver=self.prev_ver, committed_only=self.is_circle,
                                                    staged_only=self.staged, debug=self.debug_git,
                                                    include_untracked=self.include_untracked)

        return modified_files, added_files, renamed_files

    def get_changed_files_from_git(self) -> Tuple[Set, Set, Set, Set, bool]:
        """Get the added and modified after file filtration to only relevant files for validate

        Returns:
            - The filtered modified files (including the renamed files)
            - The filtered added files
            - The changed metadata files
            - The modified old-format files (legacy unified python files)
            - Boolean flag that indicates whether all file types are supported
        """

        modified_files, added_files, renamed_files = self.get_unfiltered_changed_files_from_git()

        # filter files only to relevant files
        filtered_modified, old_format_files, _ = self.filter_to_relevant_files(modified_files)
        filtered_renamed, _, valid_types_renamed = self.filter_to_relevant_files(renamed_files)
        filtered_modified = filtered_modified.union(filtered_renamed)

        filtered_added, old_format_added, valid_types_added = self.filter_to_relevant_files(added_files)
        old_format_files |= old_format_added

        valid_types = all((valid_types_added, valid_types_renamed))

        # extract metadata files from the recognised changes
        changed_meta = self.pack_metadata_extraction(modified_files, added_files, renamed_files)
        filtered_changed_meta, old_format_changed, _ = self.filter_to_relevant_files(changed_meta,
                                                                                     check_metadata_files=True)
        old_format_files |= old_format_changed
        return filtered_modified, filtered_added, filtered_changed_meta, old_format_files, valid_types

    def pack_metadata_extraction(self, modified_files, added_files, renamed_files):
        """Extract pack metadata files from the modified and added files

        Return all modified metadata file paths
        and get all newly added packs from the added metadata files."""
        changed_metadata_files = set()
        for path in modified_files.union(renamed_files):
            file_path = str(path[1]) if isinstance(path, tuple) else str(path)

            if file_path.endswith(PACKS_PACK_META_FILE_NAME):
                changed_metadata_files.add(file_path)

        for path in added_files:
            if str(path).endswith(PACKS_PACK_META_FILE_NAME):
                self.new_packs.add(get_pack_name(str(path)))

        return changed_metadata_files

    def filter_to_relevant_files(self, file_set, check_metadata_files=False):
        """Goes over file set and returns only a filtered set of only files relevant for validation"""
        filtered_set: set = set()
        old_format_files: set = set()
        valid_types: set = set()
        for path in file_set:
            old_path = None
            if isinstance(path, tuple):
                file_path = str(path[1])
                old_path = str(path[0])

            else:
                file_path = str(path)

            try:
                formatted_path, old_path, valid_file_extension = self.check_file_relevance_and_format_path(
                    file_path,
                    old_path,
                    old_format_files,
                    check_metadata_files=check_metadata_files)
                valid_types.add(valid_file_extension)
                if formatted_path:
                    if old_path:
                        filtered_set.add((old_path, formatted_path))
                    else:
                        filtered_set.add(formatted_path)

            # handle a case where a file was deleted locally though recognised as added against master.
            except FileNotFoundError:
                if file_path not in self.ignored_files:
                    if self.print_ignored_files:
                        click.secho(f"ignoring file {file_path}", fg='yellow')
                    self.ignored_files.add(file_path)

        return filtered_set, old_format_files, all(valid_types)

    def check_file_relevance_and_format_path(self, file_path, old_path, old_format_files, check_metadata_files=False):
        """
        Determines if a file is relevant for validation and create any modification to the file_path if needed
        :returns a tuple(string, string, bool) where
            - the first element is the path of the file that should be returned, if the file isn't relevant then returns an empty string
            - the second element is the old path in case the file was renamed, if the file wasn't renamed then return an empty string
            - true if the file type is supported, false otherwise
        """
        irrelevant_file_output = '', '', True
        if file_path.split(os.path.sep)[0] in ('.gitlab', '.circleci', '.github', '.devcontainer', '.vscode'):
            return irrelevant_file_output

        file_type = find_type(file_path)

        if self.ignore_files_irrelevant_for_validation(file_path, check_metadata_files=check_metadata_files):
            return irrelevant_file_output

        if not file_type:
            if str(file_path).endswith('.png'):
                error_message, error_code = Errors.invalid_image_name_or_location()
            else:
                error_message, error_code = Errors.file_type_not_supported(None, file_path)

            self.handle_error(error_message, error_code, file_path=file_path)
            return '', '', False

        # redirect non-test code files to the associated yml file
        if file_type in [FileType.PYTHON_FILE, FileType.POWERSHELL_FILE, FileType.JAVASCRIPT_FILE, FileType.XIF_FILE]:
            if not (str(file_path).endswith('_test.py') or str(file_path).endswith('.Tests.ps1') or
                    str(file_path).endswith('_test.js')):
                file_path = file_path.replace('.py', '.yml').replace('.ps1', '.yml').replace('.js', '.yml').replace(
                    '.xif', '.yml')

                if old_path:
                    old_path = old_path.replace('.py', '.yml').replace('.ps1', '.yml').replace('.js', '.yml')
            else:
                return irrelevant_file_output

        if file_type == FileType.XDRC_TEMPLATE_YML:
            file_path = file_path.replace('.yml', '.json')

            if old_path:
                old_path = old_path.replace('.yml', '.json')

        # redirect schema file when updating release notes
        if file_type == FileType.MODELING_RULE_SCHEMA:
            file_path = file_path.replace('_schema', '').replace('.json', '.yml')

            if old_path:
                old_path = old_path.replace('_schema', '').replace('.json', '.yml')

        # check for old file format
        if self.is_old_file_format(file_path, file_type):
            old_format_files.add(file_path)
            return irrelevant_file_output

        # if renamed file - return a tuple
        if old_path:
            return file_path, old_path, True

        # else return the file path
        else:
            return file_path, '', True

    def ignore_files_irrelevant_for_validation(self, file_path: str, check_metadata_files: bool = False) -> bool:
        """
        Will ignore files that are not in the packs directory, are .txt files or are in the
        VALIDATION_USING_GIT_IGNORABLE_DATA tuple.

        Args:
            file_path: path of file to check if should be ignored.
            check_metadata_files: If True will not ignore metadata files.
        Returns: True if file is ignored, false otherwise
        """

        if PACKS_DIR not in file_path:
            self.ignore_file(file_path)
            return True

        if check_metadata_files and find_type(file_path) == FileType.METADATA:
            return False

        if file_path.endswith(".txt"):
            self.ignore_file(file_path)
            return True

        if any(name in str(file_path) for name in VALIDATION_USING_GIT_IGNORABLE_DATA):
            self.ignore_file(file_path)
            return True
        return False

    def ignore_file(self, file_path: str) -> None:
        if self.print_ignored_files:
            click.secho(f"ignoring file {file_path}", fg='yellow')
        self.ignored_files.add(file_path)

    """ ######################################## Validate Tools ############################################### """

    @staticmethod
    def create_ignored_errors_list(errors_to_check):
        """ Creating a list of errors without the errors in the errors_to_check list """
        ignored_error_list = []
        all_errors = get_all_error_codes()
        for error_code in all_errors:
            error_type = error_code[:2]
            if error_code not in errors_to_check and error_type not in errors_to_check:
                ignored_error_list.append(error_code)

        return ignored_error_list

    def add_ignored_errors_to_list(self, config, section, key, ignored_errors_list):
        if key == 'ignore':
            ignored_errors_list.extend(str(config[section][key]).split(','))

        if key in PRESET_ERROR_TO_IGNORE:
            ignored_errors_list.extend(PRESET_ERROR_TO_IGNORE.get(key))

        if key in PRESET_ERROR_TO_CHECK:
            ignored_errors_list.extend(
                self.create_ignored_errors_list(PRESET_ERROR_TO_CHECK.get(key)))

    def get_error_ignore_list(self, pack_name):
        ignored_errors_list: dict = {}
        if pack_name:
            pack_ignore_path = get_pack_ignore_file_path(pack_name)

            if os.path.isfile(pack_ignore_path):
                try:
                    config = ConfigParser(allow_no_value=True)
                    config.read(pack_ignore_path)

                    # create file specific ignored errors list
                    for section in config.sections():
                        if section.startswith("file:"):
                            file_name = section[5:]
                            ignored_errors_list[file_name] = []
                            for key in config[section]:
                                self.add_ignored_errors_to_list(config, section, key, ignored_errors_list[file_name])

                except MissingSectionHeaderError:
                    pass
            else:
                click.secho(f'Could not find pack-ignore file at path {pack_ignore_path}', fg="bright_red")

        return ignored_errors_list

    @staticmethod
    def is_old_file_format(file_path, file_type):
        file_yml = get_yaml(file_path)
        # check for unified integration
        if file_type == FileType.INTEGRATION and file_yml.get('script', {}).get('script', '-') not in ['-', '']:
            if file_yml.get('script', {}).get('type', 'javascript') != 'python':
                return False
            return True

        # check for unified script
        if file_type == FileType.SCRIPT and file_yml.get('script', '-') not in ['-', '']:
            if file_yml.get('type', 'javascript') != 'python':
                return False
            return True
        return False

    @staticmethod
    def get_packs_with_added_release_notes(added_files):
        added_rn = set()
        for file in added_files:
            if find_type(path=file) == FileType.RELEASE_NOTES:
                added_rn.add(get_pack_name(file))

        return added_rn

    def print_ignored_errors_report(self, print_ignored_errors):
        if print_ignored_errors:
            all_ignored_errors = '\n'.join(FOUND_FILES_AND_IGNORED_ERRORS)
            click.secho(f"\n=========== Found ignored errors "
                        f"in the following files ===========\n\n{all_ignored_errors}",
                        fg="yellow")

    def print_ignored_files_report(self, print_ignored_files):
        if print_ignored_files:
            all_ignored_files = '\n'.join(list(self.ignored_files))
            click.secho(f"\n=========== Ignored the following files ===========\n\n{all_ignored_files}",
                        fg="yellow")

    def get_packs_that_should_have_version_raised(self, modified_files, added_files, old_format_files):
        # modified packs (where the change is not test-playbook, test-script, readme, metadata file, release notes or
        # doc/author images)
        all_modified_files = modified_files.union(old_format_files)
        modified_packs_that_should_have_version_raised = get_pack_names_from_files(all_modified_files, skip_file_types={
            FileType.RELEASE_NOTES, FileType.README, FileType.TEST_PLAYBOOK, FileType.TEST_SCRIPT,
            FileType.DOC_IMAGE, FileType.AUTHOR_IMAGE, FileType.CONTRIBUTORS})

        # also existing packs with added files which are not test-playbook, test-script readme or release notes
        # should have their version raised
        modified_packs_that_should_have_version_raised = modified_packs_that_should_have_version_raised.union(
            get_pack_names_from_files(added_files, skip_file_types={
                FileType.RELEASE_NOTES, FileType.README, FileType.TEST_PLAYBOOK,
                FileType.TEST_SCRIPT, FileType.DOC_IMAGE, FileType.AUTHOR_IMAGE,
                FileType.CONTRIBUTORS}) - self.new_packs)

        return modified_packs_that_should_have_version_raised

    @staticmethod
    def get_packs(changed_files):
        packs = set()
        for changed_file in changed_files:
            if isinstance(changed_file, tuple):
                changed_file = changed_file[1]
            pack = get_pack_name(changed_file)
            if pack:
                packs.add(pack)

        return packs

    def get_id_set_file(self, skip_id_set_creation, id_set_path):
        """

        Args:
            skip_id_set_creation (bool): whether should skip id set validation or not
            this will also determine whether a new id_set can be created by validate.
            id_set_path (str): id_set.json path file

        Returns:
            str: is_set file path
        """
        id_set = {}
        if not os.path.isfile(id_set_path):
            if not skip_id_set_creation:
                id_set, _, _ = IDSetCreator(print_logs=False).create_id_set()

        else:
            id_set = open_id_set_file(id_set_path)

        if not id_set and not self.no_configuration_prints:
            error_message, error_code = Errors.no_id_set_file()
            self.handle_error(error_message, error_code, file_path=os.path.join(os.getcwd(), id_set_path), warning=True)

        return id_set

    def check_and_validate_deprecated(self, file_type, file_path, current_file, is_modified, is_backward_check,
                                      validator):
        """If file is deprecated, validate it. Return None otherwise.

        Files with 'deprecated: true' or 'toversion < OLDEST_SUPPORTED_VERSION' fields are considered deprecated.

        Args:
            file_type: (FileType) Type of file to validate.
            file_path: (str) file path to validate.
            current_file: (dict) file in json format to validate.
            is_modified: (boolean) for whether the file was modified.
            is_backward_check: (boolean) for whether to preform backwards compatibility validation.
            validator: (ContentEntityValidator) validator object to run backwards compatibility validation from.

        Returns:
            True if current_file is deprecated and valid.
            False if current_file is deprecated and invalid.
            None if current_file is not deprecated.
        """
        is_deprecated = current_file.get("deprecated")

        toversion_is_old = "toversion" in current_file and \
                           version.parse(current_file.get("toversion", DEFAULT_CONTENT_ITEM_TO_VERSION)) < \
                           version.parse(OLDEST_SUPPORTED_VERSION)

        if is_deprecated or toversion_is_old:
            click.echo(f"Validating deprecated file: {file_path}")

            is_valid_as_deprecated = True
            if hasattr(validator, "is_valid_as_deprecated"):
                is_valid_as_deprecated = validator.is_valid_as_deprecated()

            if is_modified and is_backward_check:
                return all([is_valid_as_deprecated, validator.is_backward_compatible()])

            self.ignored_files.add(file_path)
            if self.print_ignored_files:
                click.echo(f"Skipping validation for: {file_path}")

            return is_valid_as_deprecated
        return None


# demisto_sdk/commands/zip_packs/packs_zipper.py

import logging
import os
import sys
from contextlib import contextmanager
from pathlib import Path
from shutil import make_archive

import click

from demisto_sdk.commands.common.constants import PACKS_DIR, MarketplaceVersions
from demisto_sdk.commands.common.content.objects.pack_objects.pack import Pack
from demisto_sdk.commands.common.tools import arg_to_list
from demisto_sdk.commands.create_artifacts.content_artifacts_creator import (IGNORED_PACKS, ArtifactsManager,
                                                                             ContentObject, create_dirs, delete_dirs,
                                                                             dump_pack, zip_packs)

EX_SUCCESS = 0
EX_FAIL = 1


class PacksZipper:

    def __init__(self, pack_paths: str, output: str, content_version: str, zip_all: bool,
                 marketplace: str = MarketplaceVersions.XSOAR.value, quiet_mode: bool = False):
        self.artifacts_manager = PacksManager(
            pack_paths=pack_paths,
            artifacts_path=output,
            content_version=content_version,
            all_in_one_zip=zip_all,
            quiet_mode=quiet_mode,
            marketplace=marketplace
        )

    def zip_packs(self):
        """Compress the given packs to one zip file or zip for each pack (depended on the zip_all flag).

        Returns:
            tuple: (the created zip path or the directory of the pack zips, list of the pack names)
                    or (None, None) if the pack names are empty

        """
        if self.artifacts_manager.pack_names:
            self.artifacts_manager.dump_packs()
            return self.artifacts_manager.output_path, self.artifacts_manager.pack_names

        else:
            return None, None


class PacksManager(ArtifactsManager):
    """Manages the work with packs in zip-packs command.

    Attributes:
        zip_all (bool): Flag indicating whether to zip all the packs in one zip or not.
        quiet_mode (bool): Flag indicating is in quiet mode or not.
        output_path (str): The target of the created zip

    """

    def __init__(self, pack_paths: str, all_in_one_zip: bool, quiet_mode: bool,
                 marketplace: str = MarketplaceVersions.XSOAR.value, **kwargs):
        super().__init__(packs=True, zip=True, cpus=1, suffix='', marketplace=marketplace, **kwargs)
        self.init_packs(pack_paths)
        self.zip_all = all_in_one_zip
        self.quiet_mode = quiet_mode
        self.output_path = f'{self.content_uploadable_zips_path}.zip' if self.zip_all \
            else str(self.content_uploadable_zips_path)

    def init_packs(self, pack_paths):
        """Init dict that map pack name to Pack object and init also the pack_names property.

        Args:
            pack_paths (str): CSV str with the pack paths.

        """
        self.packs = {}
        for path_str in arg_to_list(pack_paths):
            path = Path(path_str)
            if len(path.parts) == 2 and path.parts[0] == PACKS_DIR:  # relative path from Packs/...
                path = self.content.path / path

            if not os.path.exists(path):
                click.secho(f'Error: Given input path: {path} does not exist, ignored', fg='bright_red')
                continue

            self.packs.update({path.name: Pack(path)})

        self.pack_names = [*self.packs]

    def get_relative_pack_path(self, content_object: ContentObject):
        """Get the path of the given object relatively from the Packs directory, for example HelloWorld/Scripts.

        Args:
            content_object (ContentObject): The object to get the relative path for.

        Returns:
            Path: The relative path.
        """
        for part in content_object.path.parts:
            if part in self.packs:
                return content_object.path.relative_to(self.packs[part].path.parent)

    def get_dir_to_delete(self):
        """Get list of directories to delete after the unify process finished.

        Returns:
            list: The value from ArtifactsManager.get_dir_to_delete, if zip_all is True - the uploadable_packsalso dir also
        """

        result = super().get_dir_to_delete()
        if self.zip_all:
            result.append(self.content_uploadable_zips_path)
        return result

    def dump_packs(self):
        """Dump all the packs stored in the packs field
        and will print summery according to the quit_mode field

        """
        reports = []
        # we quiet the outputs and in case we want the output - a summery will be printed
        with QuietModeController(quiet_logger=True, quiet_output=True), PacksDirsHandler(self):
            for pack_name in self.pack_names:
                if pack_name not in IGNORED_PACKS:
                    reports.append(dump_pack(self, self.packs[pack_name]))

        if not self.quiet_mode:
            logger = logging.getLogger('demisto-sdk')
            for report in reports:
                logger.info(report.to_str(src_relative_to=None))

            created_zip_path = self.output_path if self.zip_all \
                else '\n'.join([f'{self.output_path}/{pack_name}.zip' for pack_name in self.pack_names])
            logger.info(f'\nCreated zips:\n{created_zip_path}')


class QuietModeController:
    """Control a quiet mode for loggers and stdout.

    Attributes:
        quiet_modes (dict): Dict with bool flags for the various outputs (logger, stdout).
        old_stdout (TextIO): The previous target of the system stdout.
        logger (logging.Logger): The active logger.
        prev_logger_level (int): The previous log level before quiet mode was activated.
    """

    def __init__(self, quiet_logger: bool, quiet_output):
        self.quiet_modes = dict(quiet_logger=quiet_logger, quiet_output=quiet_output)
        self.old_stdout = sys.stdout
        self.logger = logging.getLogger('demisto-sdk')
        self.prev_logger_level = self.logger.getEffectiveLevel()

    def __enter__(self):
        if self.quiet_modes['quiet_output']:
            sys.stdout = open(os.devnull, 'w')
        if self.quiet_modes['quiet_logger']:
            self.logger.setLevel(logging.ERROR)

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.logger.debug('{} {} {}'.format(exc_type, exc_val, exc_tb))
        sys.stdout = self.old_stdout
        self.logger.setLevel(self.prev_logger_level)


def zip_uploadable_packs(artifact_manager: ArtifactsManager):
    """Zip the zipped packs directory"""
    pack_zips_dir = artifact_manager.content_uploadable_zips_path
    make_archive(pack_zips_dir, 'zip', pack_zips_dir)


@contextmanager
def PacksDirsHandler(artifact_manager: PacksManager):
    """ Artifacts Directories handler.
    Logic by time line:
        1. Delete artifacts directories if exists.
        2. Create directories.
        3. If any error occurred -> Delete artifacts directories -> Exit.
        4. If finish successfully:
            a. If zip all:
                 Zip artifacts zip.
               Else
                 Zip packs for uploading.
            b. Delete artifacts directories.

    Args:
        artifact_manager: Packs manager object.
    """
    try:
        delete_dirs(artifact_manager)
        create_dirs(artifact_manager)
        yield
    except (Exception, KeyboardInterrupt) as e:
        delete_dirs(artifact_manager)
        artifact_manager.exit_code = EX_FAIL
        logger = logging.getLogger('demisto-sdk')
        logger.error(e)
        exit(EX_FAIL)
    else:
        if artifact_manager.zip_all:
            zip_packs(artifact_manager)
            zip_uploadable_packs(artifact_manager)
        else:
            zip_packs(artifact_manager)

        delete_dirs(artifact_manager)


# demisto_sdk/__main__.py

# Site packages
import copy
import logging
import os
import sys
from pathlib import Path
from typing import IO, Any, Dict

import click
import git
from pkg_resources import DistributionNotFound, get_distribution

from demisto_sdk.commands.common.configuration import Configuration
from demisto_sdk.commands.common.constants import (ENV_DEMISTO_SDK_MARKETPLACE, MODELING_RULES_DIR, PARSING_RULES_DIR,
                                                   FileType)
from demisto_sdk.commands.common.content_constant_paths import ALL_PACKS_DEPENDENCIES_DEFAULT_PATH
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.tools import (find_type, get_last_remote_release_version, get_release_note_entries,
                                               is_external_repository, print_error, print_success, print_warning)
from demisto_sdk.commands.content_graph.interface.neo4j.neo4j_graph import Neo4jContentGraphInterface
from demisto_sdk.commands.split.ymlsplitter import YmlSplitter
from demisto_sdk.commands.upload.upload import upload_content_entity
from demisto_sdk.utils.utils import check_configuration_file

json = JSON_Handler()

# Third party packages

# Common tools


class PathsParamType(click.Path):
    """
    Defines a click options type for use with the @click.option decorator

    The type accepts a string of comma-separated values where each individual value adheres
    to the definition for the click.Path type. The class accepts the same parameters as the
    click.Path type, applying those arguments for each comma-separated value in the list.
    See https://click.palletsprojects.com/en/8.0.x/parameters/#implementing-custom-types for
    more details.
    """

    def convert(self, value, param, ctx):
        if ',' not in value:
            return super(PathsParamType, self).convert(value, param, ctx)

        split_paths = value.split(',')
        # check the validity of each of the paths
        _ = [super(PathsParamType, self).convert(path, param, ctx) for path in split_paths]
        return value


class VersionParamType(click.ParamType):
    """
    Defines a click options type for use with the @click.option decorator

    The type accepts a string represents a version number.
    """

    name = "version"

    def convert(self, value, param, ctx):
        version_sections = value.split('.')
        if len(version_sections) == 3 and \
                all(version_section.isdigit() for version_section in version_sections):
            return value
        else:
            self.fail(f"Version {value} is not according to the expected format. "
                      f"The format of version should be in x.y.z format, e.g: <2.1.3>", param, ctx)


json = JSON_Handler()


class DemistoSDK:
    """
    The core class for the SDK.
    """

    def __init__(self):
        self.configuration = None


pass_config = click.make_pass_decorator(DemistoSDK, ensure=True)


@click.group(invoke_without_command=True, no_args_is_help=True, context_settings=dict(max_content_width=100), )
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-v', '--version', help='Get the demisto-sdk version.',
    is_flag=True, default=False, show_default=True
)
@click.option(
    '-rn', '--release-notes', help='Get the release notes of the current demisto-sdk version.',
    is_flag=True, default=False, show_default=True
)
@pass_config
def main(config, version, release_notes):
    config.configuration = Configuration()
    import dotenv

    from demisto_sdk.commands.common.tools import get_content_path
    dotenv.load_dotenv(Path(get_content_path()) / '.env', override=True)    # type: ignore # load .env file from the cwd
    if not os.getenv('DEMISTO_SDK_SKIP_VERSION_CHECK') or version:  # If the key exists/called to version
        try:
            __version__ = get_distribution('demisto-sdk').version
        except DistributionNotFound:
            __version__ = 'dev'
            print_warning('Cound not find the version of the demisto-sdk. This usually happens when running in a development environment.')
        else:
            last_release = get_last_remote_release_version()
            print_warning(f'You are using demisto-sdk {__version__}.')
            if last_release and __version__ != last_release:
                print_warning(f'however version {last_release} is available.\n'
                              f'To update, run pip3 install --upgrade demisto-sdk')
            if release_notes:
                rn_entries = get_release_note_entries(__version__)

                if not rn_entries:
                    print_warning('\nCould not get the release notes for this version.')
                else:
                    click.echo('\nThe following are the release note entries for the current version:\n')
                    for rn in rn_entries:
                        click.echo(rn)
                    click.echo('')


# ====================== split ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input', help='The yml/json file to extract from', required=True
)
@click.option(
    '-o', '--output',
    help="The output dir to write the extracted code/description/image/json to."
)
@click.option(
    '--no-demisto-mock',
    help="Don't add an import for demisto mock. (only for yml files)",
    is_flag=True,
    show_default=True
)
@click.option(
    '--no-common-server',
    help="Don't add an import for CommonServerPython. (only for yml files)",
    is_flag=True,
    show_default=True
)
@click.option(
    '--no-auto-create-dir',
    help="Don't auto create the directory if the target directory ends with *Integrations/*Scripts/*Dashboards"
         "/*GenericModules.",
    is_flag=True,
    show_default=True
)
@click.option(
    '--no-pipenv',
    help="Don't auto create pipenv for requirements installation. (only for yml files)",
    is_flag=True,
    show_default=True
)
@click.option(
    '--new-module-file',
    help="Create a new module file instead of editing the existing file. (only for json files)",
    is_flag=True,
    show_default=True
)
@pass_config
def split(config, **kwargs):
    """Split the code, image and description files from a Demisto integration or script yaml file
    to multiple files(To a package format - https://demisto.pan.dev/docs/package-dir).
    """
    from demisto_sdk.commands.split.jsonsplitter import JsonSplitter

    check_configuration_file('split', kwargs)
    file_type: FileType = find_type(kwargs.get('input', ''), ignore_sub_categories=True)
    if file_type not in [FileType.INTEGRATION, FileType.SCRIPT, FileType.GENERIC_MODULE, FileType.MODELING_RULE,
                         FileType.PARSING_RULE]:
        print_error('File is not an Integration, Script, Generic Module, Modeling Rule or Parsing Rule.')
        return 1

    if file_type in [FileType.INTEGRATION, FileType.SCRIPT, FileType.MODELING_RULE, FileType.PARSING_RULE]:
        yml_splitter = YmlSplitter(configuration=config.configuration, file_type=file_type.value, **kwargs)
        return yml_splitter.extract_to_package_format()

    else:
        json_splitter = JsonSplitter(input=kwargs.get('input'), output=kwargs.get('output'),  # type: ignore[arg-type]
                                     no_auto_create_dir=kwargs.get('no_auto_create_dir'),  # type: ignore[arg-type]
                                     no_logging=kwargs.get('no_logging'),  # type: ignore[arg-type]
                                     new_module_file=kwargs.get('new_module_file'))  # type: ignore[arg-type]
        return json_splitter.split_json()


# ====================== extract-code ====================== #
@main.command(hidden=True)
@click.help_option(
    '-h', '--help'
)
@click.option(
    '--input', '-i',
    help='The yml file to extract from',
    required=True
)
@click.option(
    '--output', '-o',
    required=True,
    help="The output file to write the code to"
)
@click.option(
    '--no-demisto-mock',
    help="Don't add an import for demisto mock, false by default",
    is_flag=True,
    show_default=True
)
@click.option(
    '--no-common-server',
    help="Don't add an import for CommonServerPython."
         "If not specified will import unless this is CommonServerPython",
    is_flag=True,
    show_default=True
)
@pass_config
def extract_code(config, **kwargs):
    """Extract code from a Demisto integration or script yaml file."""
    from demisto_sdk.commands.split.ymlsplitter import YmlSplitter
    check_configuration_file('extract-code', kwargs)
    file_type: FileType = find_type(kwargs.get('input', ''), ignore_sub_categories=True)
    if file_type not in [FileType.INTEGRATION, FileType.SCRIPT]:
        print_error('File is not an Integration or Script.')
        return 1
    extractor = YmlSplitter(configuration=config.configuration, file_type=file_type.value, **kwargs)
    return extractor.extract_code(kwargs['outfile'])


# ====================== unify ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-i", "--input", help="The directory path to the files or path to the file to unify", required=True, type=click.Path(dir_okay=True)
)
@click.option(
    "-o", "--output", help="The output dir to write the unified yml to", required=False
)
@click.option(
    "-c", "--custom", help="Add test label to unified yml id/name/display", required=False,
)
@click.option(
    "-f", "--force", help="Forcefully overwrites the preexisting yml if one exists",
    is_flag=True,
    show_default=False
)
@click.option('-mp', '--marketplace',
              help='The marketplace the content items are created for, that determines usage of marketplace '
                   'unique text. Default is the XSOAR marketplace.',
              default='xsoar', type=click.Choice(['xsoar', 'marketplacev2', 'v2']))
def unify(**kwargs):
    """
    This command has three main functions:

    1. Integration/Script Unifier - Unifies integration/script code, image, description and yml files to a single XSOAR yml file.
     * Note that this should be used on a single integration/script and not a pack, not multiple scripts/integrations.
     * To use this function - set as input a path to the *directory* of the integration/script to unify.

    2. GenericModule Unifier - Unifies a GenericModule with its Dashboards to a single JSON object.
     * To use this function - set as input a path to a GenericModule *file*.

    3. Parsing/Modeling Rule Unifier - Unifies Parsing/Modeling rule YML, XIF and samples JSON files to a single YML file.
     * Note that this should be used on a single parsing/modeling rule and not a pack, not multiple rules.
     * To use this function - set as input a path to the *directory* of the parsing/modeling rule to unify.
    """
    check_configuration_file('unify', kwargs)
    # Input is of type Path.
    kwargs['input'] = str(kwargs['input'])
    file_type = find_type(kwargs['input'])
    custom = kwargs.pop('custom')
    if marketplace := kwargs.get('marketplace'):
        os.environ[ENV_DEMISTO_SDK_MARKETPLACE] = marketplace.lower()
    if file_type == FileType.GENERIC_MODULE:
        from demisto_sdk.commands.unify.generic_module_unifier import GenericModuleUnifier

        # pass arguments to GenericModule unifier and call the command
        generic_module_unifier = GenericModuleUnifier(**kwargs)
        generic_module_unifier.merge_generic_module_with_its_dashboards()
    elif any(rule_dir in os.path.abspath(kwargs['input']) for rule_dir in [PARSING_RULES_DIR, MODELING_RULES_DIR]):
        from demisto_sdk.commands.unify.rule_unifier import RuleUnifier
        rule_unifier = RuleUnifier(**kwargs)
        rule_unifier.unify()
    else:
        from demisto_sdk.commands.unify.integration_script_unifier import IntegrationScriptUnifier

        # pass arguments to YML unifier and call the command
        yml_unifier = IntegrationScriptUnifier(**kwargs, custom=custom)
        yml_unifier.unify()

    return 0


# ====================== zip-packs ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option('-i', '--input',
              help="The packs to be zipped as csv list of pack paths.",
              type=PathsParamType(exists=True, resolve_path=True),
              required=True)
@click.option('-o', '--output', help='The destination directory to create the packs.',
              type=click.Path(file_okay=False, resolve_path=True), required=True)
@click.option('-v', '--content-version', help='The content version in CommonServerPython.', default='0.0.0')
@click.option('-u', '--upload', is_flag=True, help='Upload the unified packs to the marketplace.', default=False)
@click.option('--zip-all', is_flag=True, help='Zip all the packs in one zip file.', default=False)
def zip_packs(**kwargs) -> int:
    """Generating zipped packs that are ready to be uploaded to Cortex XSOAR machine."""
    from demisto_sdk.commands.common.logger import logging_setup
    from demisto_sdk.commands.upload.uploader import Uploader
    from demisto_sdk.commands.zip_packs.packs_zipper import EX_FAIL, EX_SUCCESS, PacksZipper
    logging_setup(3)
    check_configuration_file('zip-packs', kwargs)

    # if upload is true - all zip packs will be compressed to one zip file
    should_upload = kwargs.pop('upload', False)
    zip_all = kwargs.pop('zip_all', False) or should_upload

    if marketplace := kwargs.get('marketplace'):
        os.environ[ENV_DEMISTO_SDK_MARKETPLACE] = marketplace.lower()

    packs_zipper = PacksZipper(zip_all=zip_all, pack_paths=kwargs.pop('input'), quiet_mode=zip_all, **kwargs)
    zip_path, unified_pack_names = packs_zipper.zip_packs()

    if should_upload and zip_path:
        return Uploader(input=zip_path, pack_names=unified_pack_names).upload()

    return EX_SUCCESS if zip_path is not None else EX_FAIL


# ====================== validate ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    '--no-conf-json', is_flag=True,
    default=False, show_default=True, help='Skip conf.json validation.')
@click.option(
    '-s', '--id-set', is_flag=True,
    default=False, show_default=True, help='Perform validations using the id_set file.')
@click.option(
    "-idp", "--id-set-path", help="The path of the id-set.json used for validations.",
    type=click.Path(resolve_path=True))
@click.option(
    '--prev-ver', help='Previous branch or SHA1 commit to run checks against.')
@click.option(
    '--no-backward-comp', is_flag=True, show_default=True,
    help='Whether to check backward compatibility or not.')
@click.option(
    '-g', '--use-git', is_flag=True, show_default=True,
    default=False,
    help='Validate changes using git - this will check current branch\'s changes against origin/master or origin/main. '
         'If the --post-commit flag is supplied: validation will run only on the current branch\'s changed files '
         'that have been committed. '
         'If the --post-commit flag is not supplied: validation will run on all changed files in the current branch, '
         'both committed and not committed. ')
@click.option(
    '-pc', '--post-commit',
    is_flag=True,
    help='Whether the validation should run only on the current branch\'s committed changed files. '
         'This applies only when the -g flag is supplied.'
)
@click.option(
    '-st', '--staged',
    is_flag=True,
    help='Whether the validation should ignore unstaged files.'
         'This applies only when the -g flag is supplied.'
)
@click.option(
    '-iu', '--include-untracked',
    is_flag=True,
    help='Whether to include untracked files in the validation. '
         'This applies only when the -g flag is supplied.'
)
@click.option(
    '-a', '--validate-all', is_flag=True, show_default=True, default=False,
    help='Whether to run all validation on all files or not.'
)
@click.option(
    '-i', '--input', type=click.Path(exists=True, resolve_path=True),
    help='The path of the content pack/file to validate specifically.'
)
@click.option(
    '--skip-pack-release-notes', is_flag=True,
    help='Skip validation of pack release notes.')
@click.option(
    '--print-ignored-errors', is_flag=True,
    help='Print ignored errors as warnings.')
@click.option(
    '--print-ignored-files', is_flag=True,
    help='Print which files were ignored by the command.')
@click.option(
    '--no-docker-checks', is_flag=True,
    help='Whether to run docker image validation.')
@click.option(
    '--silence-init-prints', is_flag=True,
    help='Whether to skip the initialization prints.')
@click.option(
    '--skip-pack-dependencies', is_flag=True,
    help='Skip validation of pack dependencies.')
@click.option(
    '--create-id-set', is_flag=True,
    help='Whether to create the id_set.json file.')
@click.option(
    '-j', '--json-file', help='The JSON file path to which to output the command results.')
@click.option(
    '--skip-schema-check', is_flag=True,
    help='Whether to skip the file schema check.')
@click.option(
    '--debug-git', is_flag=True,
    help='Whether to print debug logs for git statuses.')
@click.option(
    '--print-pykwalify', is_flag=True,
    help='Whether to print the pykwalify log errors.')
@click.option(
    "--quiet-bc-validation",
    help="Set backwards compatibility validation's errors as warnings.",
    is_flag=True)
@click.option(
    "--allow-skipped",
    help="Don't fail on skipped integrations or when all test playbooks are skipped.",
    is_flag=True)
@click.option(
    "--no-multiprocessing",
    help="run validate all without multiprocessing, for debugging purposes.",
    is_flag=True, default=False)
@click.option(
    '-sv', '--run-specific-validations',
    help="Run specific validations by stating the error codes.",
    is_flag=False)
@pass_config
def validate(config, **kwargs):
    """Validate your content files. If no additional flags are given, will validated only committed files."""
    from demisto_sdk.commands.validate.validate_manager import ValidateManager
    run_with_mp = not kwargs.pop('no_multiprocessing')
    check_configuration_file('validate', kwargs)
    sys.path.append(config.configuration.env_dir)

    file_path = kwargs['input']

    if kwargs['post_commit'] and kwargs['staged']:
        print_error('Could not supply the staged flag with the post-commit flag')
        sys.exit(1)
    try:
        is_external_repo = is_external_repository()
        # default validate to -g --post-commit
        if not kwargs.get('validate_all') and not kwargs['use_git'] and not file_path:
            kwargs['use_git'] = True
            kwargs['post_commit'] = True
        validator = ValidateManager(
            is_backward_check=not kwargs['no_backward_comp'],
            only_committed_files=kwargs['post_commit'], prev_ver=kwargs['prev_ver'],
            skip_conf_json=kwargs['no_conf_json'], use_git=kwargs['use_git'],
            file_path=file_path,
            validate_all=kwargs.get('validate_all'),
            validate_id_set=kwargs['id_set'],
            skip_pack_rn_validation=kwargs['skip_pack_release_notes'],
            print_ignored_errors=kwargs['print_ignored_errors'],
            is_external_repo=is_external_repo,
            print_ignored_files=kwargs['print_ignored_files'],
            no_docker_checks=kwargs['no_docker_checks'],
            silence_init_prints=kwargs['silence_init_prints'],
            skip_dependencies=kwargs['skip_pack_dependencies'],
            id_set_path=kwargs.get('id_set_path'),
            staged=kwargs['staged'],
            create_id_set=kwargs.get('create_id_set'),
            json_file_path=kwargs.get('json_file'),
            skip_schema_check=kwargs.get('skip_schema_check'),
            debug_git=kwargs.get('debug_git'),
            include_untracked=kwargs.get('include_untracked'),
            quiet_bc=kwargs.get('quiet_bc_validation'),
            multiprocessing=run_with_mp,
            check_is_unskipped=not kwargs.get('allow_skipped', False),
            specific_validations=kwargs.get('run_specific_validations'),
        )
        return validator.run_validation()
    except (git.InvalidGitRepositoryError, git.NoSuchPathError, FileNotFoundError) as e:
        print_error(e)
        print_error("\nYou may not be running `demisto-sdk validate` command in the content directory.\n"
                    "Please run the command from content directory")
        sys.exit(1)


# ====================== create-content-artifacts ====================== #
@main.command(hidden=True)
@click.help_option(
    '-h', '--help'
)
@click.option('-a', '--artifacts_path', help='Destination directory to create the artifacts.',
              type=click.Path(file_okay=False, resolve_path=True), required=True)
@click.option('--zip/--no-zip', help='Zip content artifacts folders', default=True)
@click.option('--packs', help='Create only content_packs artifacts. '
                              'Used for server version 5.5.0 and earlier.', is_flag=True)
@click.option('-v', '--content_version', help='The content version in CommonServerPython.', default='0.0.0')
@click.option('-s', '--suffix', help='Suffix to add all yaml/json/yml files in the created artifacts.')
@click.option('--cpus',
              help='Number of cpus/vcpus available - only required when os not reflect number of cpus (CircleCI'
                   'always show 32, but medium has 3.', hidden=True, default=os.cpu_count())
@click.option('-idp', '--id-set-path', help='The full path of id_set.json', hidden=True,
              type=click.Path(exists=True, resolve_path=True))
@click.option('-p', '--pack-names',
              help=("Packs to create artifacts for. Optional values are: `all` or "
                    "csv list of packs. "
                    "Default is set to `all`"),
              default="all", hidden=True)
@click.option('-sk', '--signature-key', help='Base64 encoded signature key used for signing packs.', hidden=True)
@click.option('-sd', '--sign-directory', help='Path to the signDirectory executable file.',
              type=click.Path(exists=True, resolve_path=True), hidden=True)
@click.option('-rt', '--remove-test-playbooks', is_flag=True,
              help='Should remove test playbooks from content packs or not.', default=True, hidden=True)
@click.option('-mp', '--marketplace', help='The marketplace the artifacts are created for, that '
                                           'determines which artifacts are created for each pack. '
                                           'Default is the XSOAR marketplace, that has all of the packs '
                                           'artifacts.', default='xsoar', type=click.Choice(['xsoar', 'marketplacev2', 'v2', 'xpanse']))
@click.option('-fbi', '--filter-by-id-set', is_flag=True,
              help='Whether to use the id set as content items guide, meaning only include in the packs the '
                   'content items that appear in the id set.', default=False, hidden=True)
@click.option('-af', '--alternate-fields', is_flag=True,
              help='Use the alternative fields if such are present in the yml or json of the content item.', default=False, hidden=True)
def create_content_artifacts(**kwargs) -> int:
    """Generating the following artifacts:
       1. content_new - Contains all content objects of type json,yaml (from_version < 6.0.0)
       2. content_packs - Contains all packs from Packs - Ignoring internal files (to_version >= 6.0.0).
       3. content_test - Contains all test scripts/playbooks (from_version < 6.0.0)
       4. content_all - Contains all from content_new and content_test.
       5. uploadable_packs - Contains zipped packs that are ready to be uploaded to Cortex XSOAR machine.
    """
    from demisto_sdk.commands.common.logger import logging_setup
    from demisto_sdk.commands.create_artifacts.content_artifacts_creator import ArtifactsManager
    logging_setup(3)
    check_configuration_file('create-content-artifacts', kwargs)
    if marketplace := kwargs.get('marketplace'):
        os.environ[ENV_DEMISTO_SDK_MARKETPLACE] = marketplace.lower()
    artifacts_conf = ArtifactsManager(**kwargs)
    return artifacts_conf.create_content_artifacts()


# ====================== secrets ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input', help='Specify file of to check secret on.'
)
@click.option(
    '--post-commit',
    is_flag=True,
    show_default=True,
    help='Whether the secretes is done after you committed your files, '
         'this will help the command to determine which files it should check in its '
         'run. Before you commit the files it should not be used. Mostly for build '
         'validations.'
)
@click.option(
    '-ie', '--ignore-entropy',
    is_flag=True,
    help='Ignore entropy algorithm that finds secret strings (passwords/api keys).'
)
@click.option(
    '-wl', '--whitelist',
    default='./Tests/secrets_white_list.json',
    show_default=True,
    help='Full path to whitelist file, file name should be "secrets_white_list.json"'
)
@click.option(
    '--prev-ver',
    help='The branch against which to run secrets validation.'
)
@pass_config
def secrets(config, **kwargs):
    """Run Secrets validator to catch sensitive data before exposing your code to public repository.
     Attach path to whitelist to allow manual whitelists.
     """
    from demisto_sdk.commands.secrets.secrets import SecretsValidator
    check_configuration_file('secrets', kwargs)
    sys.path.append(config.configuration.env_dir)
    secrets_validator = SecretsValidator(
        configuration=config.configuration,
        is_circle=kwargs['post_commit'],
        ignore_entropy=kwargs['ignore_entropy'],
        white_list_path=kwargs['whitelist'],
        input_path=kwargs.get('input')
    )
    return secrets_validator.run()


# ====================== lint ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-i", "--input", help="Specify directory(s) of integration/script",
    type=PathsParamType(exists=True, resolve_path=True)
)
@click.option("-g", "--git", is_flag=True, help="Will run only on changed packages")
@click.option("-a", "--all-packs", is_flag=True, help="Run lint on all directories in content repo")
@click.option('-v', "--verbose", count=True, help="Verbosity level -v / -vv / .. / -vvv",
              type=click.IntRange(0, 3, clamp=True), default=2, show_default=True)
@click.option('-q', "--quiet", is_flag=True, help="Quiet output, only output results in the end")
@click.option("-p", "--parallel", default=1, help="Run tests in parallel", type=click.IntRange(0, 15, clamp=True),
              show_default=True)
@click.option("--no-flake8", is_flag=True, help="Do NOT run flake8 linter")
@click.option("--no-bandit", is_flag=True, help="Do NOT run bandit linter")
@click.option("--no-xsoar-linter", is_flag=True, help="Do NOT run XSOAR linter")
@click.option("--no-mypy", is_flag=True, help="Do NOT run mypy static type checking")
@click.option("--no-vulture", is_flag=True, help="Do NOT run vulture linter")
@click.option("--no-pylint", is_flag=True, help="Do NOT run pylint linter")
@click.option("--no-test", is_flag=True, help="Do NOT test (skip pytest)")
@click.option("--no-pwsh-analyze", is_flag=True, help="Do NOT run powershell analyze")
@click.option("--no-pwsh-test", is_flag=True, help="Do NOT run powershell test")
@click.option("-kc", "--keep-container", is_flag=True, help="Keep the test container")
@click.option("--prev-ver", help="Previous branch or SHA1 commit to run checks against", default='master')
@click.option("--test-xml", help="Path to store pytest xml results", type=click.Path(exists=True, resolve_path=True))
@click.option("--failure-report", help="Path to store failed packs report",
              type=click.Path(exists=True, resolve_path=True))
@click.option("-lp", "--log-path", help="Path to store all levels of logs",
              type=click.Path(resolve_path=True))
@click.option("-j", "--json-file", help="The JSON file path to which to output the command results.",
              type=click.Path(resolve_path=True))
@click.option("--no-coverage", is_flag=True, help="Do NOT run coverage report.")
@click.option(
    "--coverage-report", help="Specify directory for the coverage report files",
    type=PathsParamType()
)
@click.option("-dt", "--docker-timeout", default=60,
              help="The timeout (in seconds) for requests done by the docker client.", type=int)
@click.option("-idp", "--id-set-path", help="Path to id_set.json, relevant for when using the "
                                            "--check-dependent-api-module flag.",
              type=click.Path(resolve_path=True),
              default='Tests/id_set.json')
@click.option("-cdam", "--check-dependent-api-module", is_flag=True, help="Run unit tests and lint on all packages that "
              "are dependent on the found "
              "modified api modules.", default=False)
@click.option("--time-measurements-dir", help="Specify directory for the time measurements report file",
              type=PathsParamType())
def lint(**kwargs):
    """Lint command will perform:
        1. Package in host checks - flake8, bandit, mypy, vulture.
        2. Package in docker image checks -  pylint, pytest, powershell - test, powershell - analyze.
        Meant to be used with integrations/scripts that use the folder (package) structure.
        Will lookup up what docker image to use and will setup the dev dependencies and file in the target folder.
        If no additional flags specifying the packs are given,will lint only changed files.
    """
    from demisto_sdk.commands.common.logger import logging_setup
    from demisto_sdk.commands.lint.lint_manager import LintManager
    logging_setup(verbose=kwargs.get('verbose'),  # type: ignore[arg-type]
                  quiet=kwargs.get('quiet'),  # type: ignore[arg-type]
                  log_path=kwargs.get('log_path'))  # type: ignore[arg-type]

    check_configuration_file('lint', kwargs)
    lint_manager = LintManager(
        input=kwargs.get('input'),  # type: ignore[arg-type]
        git=kwargs.get('git'),  # type: ignore[arg-type]
        all_packs=kwargs.get('all_packs'),  # type: ignore[arg-type]
        verbose=kwargs.get('verbose'),  # type: ignore[arg-type]
        quiet=kwargs.get('quiet'),  # type: ignore[arg-type]
        prev_ver=kwargs.get('prev_ver'),  # type: ignore[arg-type]
        json_file_path=kwargs.get('json_file'),  # type: ignore[arg-type]
        id_set_path=kwargs.get('id_set_path'),  # type: ignore[arg-type]
        check_dependent_api_module=kwargs.get('check_dependent_api_module'),  # type: ignore[arg-type]
    )
    return lint_manager.run(
        parallel=kwargs.get('parallel'),  # type: ignore[arg-type]
        no_flake8=kwargs.get('no_flake8'),  # type: ignore[arg-type]
        no_bandit=kwargs.get('no_bandit'),  # type: ignore[arg-type]
        no_mypy=kwargs.get('no_mypy'),  # type: ignore[arg-type]
        no_vulture=kwargs.get('no_vulture'),  # type: ignore[arg-type]
        no_xsoar_linter=kwargs.get('no_xsoar_linter'),  # type: ignore[arg-type]
        no_pylint=kwargs.get('no_pylint'),  # type: ignore[arg-type]
        no_test=kwargs.get('no_test'),  # type: ignore[arg-type]
        no_pwsh_analyze=kwargs.get('no_pwsh_analyze'),  # type: ignore[arg-type]
        no_pwsh_test=kwargs.get('no_pwsh_test'),  # type: ignore[arg-type]
        keep_container=kwargs.get('keep_container'),  # type: ignore[arg-type]
        test_xml=kwargs.get('test_xml'),  # type: ignore[arg-type]
        failure_report=kwargs.get('failure_report'),  # type: ignore[arg-type]
        no_coverage=kwargs.get('no_coverage'),  # type: ignore[arg-type]
        coverage_report=kwargs.get('coverage_report'),  # type: ignore[arg-type]
        docker_timeout=kwargs.get('docker_timeout'),  # type: ignore[arg-type]
        time_measurements_dir=kwargs.get('time_measurements_dir'),  # type: ignore[arg-type]
    )


# ====================== coverage-analyze ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-i", "--input", help="The .coverage file to analyze.",
    default=os.path.join('coverage_report', '.coverage'),
    type=PathsParamType(exists=True, resolve_path=True)
)
@click.option(
    "--default-min-coverage", help="Default minimum coverage (for new files).",
    default=70.0, type=click.FloatRange(0.0, 100.0)
)
@click.option(
    "--allowed-coverage-degradation-percentage", help="Allowed coverage degradation percentage (for modified files).",
    default=1.0, type=click.FloatRange(0.0, 100.0)
)
@click.option(
    "--no-cache", help="Force download of the previous coverage report file.",
    is_flag=True, type=bool)
@click.option(
    "--report-dir", help="Directory of the coverage report files.",
    default='coverage_report', type=PathsParamType(resolve_path=True))
@click.option(
    "--report-type", help="The type of coverage report (posible values: 'text', 'html', 'xml', 'json' or 'all').",
    type=str)
@click.option("--no-min-coverage-enforcement", help="Do not enforce minimum coverage.", is_flag=True)
@click.option(
    "--previous-coverage-report-url", help="URL of the previous coverage report.",
    default='https://storage.googleapis.com/marketplace-dist-dev/code-coverage-reports/coverage-min.json', type=str
)
def coverage_analyze(**kwargs):
    from demisto_sdk.commands.coverage_analyze.coverage_report import CoverageReport
    try:
        no_degradation_check = kwargs['allowed_coverage_degradation_percentage'] == 100.0
        no_min_coverage_enforcement = kwargs['no_min_coverage_enforcement']

        cov_report = CoverageReport(
            default_min_coverage=kwargs['default_min_coverage'],
            allowed_coverage_degradation_percentage=kwargs['allowed_coverage_degradation_percentage'],
            coverage_file=kwargs['input'],
            no_cache=kwargs.get('no_cache', False),
            report_dir=kwargs['report_dir'],
            report_type=kwargs['report_type'],
            no_degradation_check=no_degradation_check,
            previous_coverage_report_url=kwargs['previous_coverage_report_url']
        )
        cov_report.coverage_report()
        # if no_degradation_check=True we will suppress the minimum coverage check
        if no_degradation_check or cov_report.coverage_diff_report() or no_min_coverage_enforcement:
            return 0
    except Exception as error:
        logging.getLogger('demisto-sdk').error(error)

    return 1


# ====================== format ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-i", "--input", help="The path of the script yml file\n"
                          "If no input is specified, the format will be executed on all new/changed files.",
    type=click.Path(exists=True, resolve_path=True))
@click.option(
    "-o", "--output", help="The path where the formatted file will be saved to",
    type=click.Path(resolve_path=True))
@click.option(
    "-fv", "--from-version", help="Specify fromversion of the pack")
@click.option(
    "-nv", "--no-validate", help="Set when validate on file is not wanted", is_flag=True)
@click.option(
    "-ud", "--update-docker", help="Set if you want to update the docker image of the integration/script", is_flag=True)
@click.option(
    "-v", "--verbose", help="Verbose output", is_flag=True)
@click.option(
    "-y", "--assume-yes",
    help="Automatic yes to prompts; assume 'yes' as answer to all prompts and run non-interactively",
    is_flag=True)
@click.option(
    "-d", "--deprecate", help="Set if you want to deprecate the integration/script/playbook", is_flag=True)
@click.option(
    "-g", "--use-git",
    help="Use git to automatically recognize which files changed and run format on them.",
    is_flag=True)
@click.option(
    '--prev-ver', help='Previous branch or SHA1 commit to run checks against.')
@click.option(
    '-iu', '--include-untracked',
    is_flag=True,
    help='Whether to include untracked files in the formatting.'
)
@click.option(
    '-at', '--add-tests',
    is_flag=True,
    help='Whether to answer manually to add tests configuration prompt when running interactively.'
)
@click.option(
    "-s", "--id-set-path", help="The path of the id_set json file.",
    type=click.Path(exists=True, resolve_path=True))
def format(
        input: Path,
        output: Path,
        from_version: str,
        no_validate: bool,
        update_docker: bool,
        verbose: bool,
        assume_yes: bool,
        deprecate: bool,
        use_git: bool,
        prev_ver: str,
        include_untracked: bool,
        add_tests: bool,
        id_set_path: str
):
    """Run formatter on a given script/playbook/integration/incidentfield/indicatorfield/
    incidenttype/indicatortype/layout/dashboard/classifier/mapper/widget/report file/genericfield/generictype/
    genericmodule/genericdefinition.
    """
    from demisto_sdk.commands.format.format_module import format_manager
    return format_manager(
        str(input) if input else None,
        str(output) if output else None,
        from_version=from_version,
        no_validate=no_validate,
        update_docker=update_docker,
        assume_yes=assume_yes,
        verbose=verbose,
        deprecate=deprecate,
        use_git=use_git,
        prev_ver=prev_ver,
        include_untracked=include_untracked,
        add_tests=add_tests,
        id_set_path=id_set_path,
    )


# ====================== upload ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-i", "--input",
    type=PathsParamType(exists=True, resolve_path=True),
    help="The path of file or a directory to upload. The following are supported:\n"
         "- Pack\n"
         "- A content entity directory that is inside a pack. For example: an Integrations "
         "directory or a Layouts directory.\n"
         "- Valid file that can be imported to Cortex XSOAR manually. For example a playbook: "
         "helloWorld.yml", required=False
)
@click.option(
    "--input-config-file",
    type=PathsParamType(exists=True, resolve_path=True),
    help="The path to the config file to download all the custom packs from", required=False
)
@click.option(
    "-z", "--zip",
    help="Compress the pack to zip before upload, this flag is relevant only for packs.", is_flag=True
)
@click.option(
    "-x", "--xsiam",
    help="Upload the pack to XSIAM server. Must be used together with -z", is_flag=True
)
@click.option(
    "--keep-zip", help="Directory where to store the zip after creation, this argument is relevant only for packs "
                       "and in case the --zip flag is used.", required=False, type=click.Path(exists=True))
@click.option(
    "--insecure",
    help="Skip certificate validation", is_flag=True
)
@click.option(
    "--skip_validation", is_flag=True,
    help="Only for upload zipped packs, "
         "if true will skip upload packs validation, use just when migrate existing custom content to packs."
)
@click.option(
    "-v", "--verbose",
    help="Verbose output", is_flag=True
)
@click.option(
    "--reattach",
    help="Reattach the detached files in the XSOAR instance"
         "for the CI/CD Flow. If you set the --input-config-file flag, "
         "any detached item in your XSOAR instance that isn't currently in the repo's SystemPacks folder "
         "will be re-attached.)", is_flag=True
)
def upload(**kwargs):
    """Upload integration or pack to Demisto instance.
    DEMISTO_BASE_URL environment variable should contain the Demisto server base URL.
    DEMISTO_API_KEY environment variable should contain a valid Demisto API Key.
    * Note: Uploading classifiers to Cortex XSOAR is available from version 6.0.0 and up. *
    """
    return upload_content_entity(**kwargs)


# ====================== download ====================== #


@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-o", "--output", help="The path of a package directory to download custom content to", required=False,
    multiple=False)
@click.option(
    "-i", "--input", help="Custom content file name to be downloaded. Can be provided multiple times",
    required=False, multiple=True)
@click.option(
    "-r", "--regex", help="Regex Pattern, download all the custom content files that match this regex pattern.",
    required=False)
@click.option(
    "--insecure", help="Skip certificate validation", is_flag=True)
@click.option(
    "-v", "--verbose", help="Verbose output", is_flag=True)
@click.option(
    "-f", "--force", help="Whether to override existing files or not", is_flag=True)
@click.option(
    "-lf", "--list-files", help="Prints a list of all custom content files available to be downloaded", is_flag=True)
@click.option(
    "-a", "--all-custom-content", help="Download all available custom content files", is_flag=True)
@click.option(
    "-fmt", "--run-format", help="Whether to run demisto-sdk format on downloaded files or not", is_flag=True)
@click.option(
    "--system", help="Download system items", is_flag=True, default=False)
@click.option(
    "-it", "--item-type", help="The items type to download, use just when downloading system items, should be one "
                               "form the following list: [IncidentType, IndicatorType, Field, Layout, Playbook, "
                               "Automation, Classifier, Mapper]",
    type=click.Choice(['IncidentType', 'IndicatorType', 'Field', 'Layout', 'Playbook', 'Automation', 'Classifier',
                       'Mapper'], case_sensitive=False))
@click.option(
    "--no-code-formatting", help="Use this flag to avoid running Autopep8 and isort on Python files.",
    is_flag=True, default=False
)
def download(**kwargs):
    """Download custom content from Demisto instance.
    DEMISTO_BASE_URL environment variable should contain the Demisto server base URL.
    DEMISTO_API_KEY environment variable should contain a valid Demisto API Key.
    """
    from demisto_sdk.commands.download.downloader import Downloader
    check_configuration_file('download', kwargs)
    downloader: Downloader = Downloader(**kwargs)
    return downloader.download()


# ====================== update-xsoar-config-file ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-pi", "--pack-id", help="The Pack ID to add to XSOAR Configuration File", required=False,
    multiple=False)
@click.option(
    "-pd", "--pack-data", help="The Pack Data to add to XSOAR Configuration File - "
                               "Pack URL for Custom Pack and Pack Version for OOTB Pack", required=False,
    multiple=False)
@click.option(
    "-mp", "--add-marketplace-pack", help="Add a Pack to the MarketPlace Packs section in the Configuration File",
    required=False, is_flag=True)
@click.option(
    "-cp", "--add-custom-pack", help="Add the Pack to the Custom Packs section in the Configuration File",
    is_flag=True)
@click.option(
    "-all", "--add-all-marketplace-packs",
    help="Add all the installed MarketPlace Packs to the marketplace_packs in XSOAR Configuration File", is_flag=True)
@click.option(
    "--insecure", help="Skip certificate validation", is_flag=True)
@click.option(
    "--file-path", help="XSOAR Configuration File path, the default value is in the repo level", is_flag=False)
def xsoar_config_file_update(**kwargs):
    """Handle your XSOAR Configuration File.
    Add automatically all the installed MarketPlace Packs to the marketplace_packs section in XSOAR Configuration File.
    Add a Pack to both marketplace_packs and custom_packs sections in the Configuration File.
    """
    from demisto_sdk.commands.update_xsoar_config_file.update_xsoar_config_file import XSOARConfigFileUpdater
    file_updater: XSOARConfigFileUpdater = XSOARConfigFileUpdater(**kwargs)
    return file_updater.update()


# ====================== run ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-q", "--query", help="The query to run", required=True)
@click.option(
    "--insecure", help="Skip certificate validation", is_flag=True)
@click.option(
    "-v", "--verbose", help="Verbose output", is_flag=True)
@click.option(
    "-D", "--debug", help="Whether to enable the debug-mode feature or not, if you want to save the output file "
                          "please use the --debug-path option", is_flag=True)
@click.option(
    "--debug-path", help="The path to save the debug file at, if not specified the debug file will be printed to the "
                         "terminal")
@click.option(
    "--json-to-outputs", help="Whether to run json_to_outputs command on the context output of the query. If the "
                              "context output does not exists or the `-r` flag is used, will use the raw"
                              " response of the query", is_flag=True)
@click.option(
    "-p", "--prefix", help="Used with `json-to-outputs` flag. Output prefix e.g. Jira.Ticket, VirusTotal.IP, "
                           "the base path for the outputs that the script generates")
@click.option(
    "-r", "--raw-response", help="Used with `json-to-outputs` flag. Use the raw response of the query for"
                                 " `json-to-outputs`", is_flag=True)
def run(**kwargs):
    """Run integration command on remote Demisto instance in the playground.
    DEMISTO_BASE_URL environment variable should contain the Demisto base URL.
    DEMISTO_API_KEY environment variable should contain a valid Demisto API Key.
    """
    from demisto_sdk.commands.run_cmd.runner import Runner
    check_configuration_file('run', kwargs)
    runner = Runner(**kwargs)
    return runner.run()


# ====================== run-playbook ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    '--url', '-u',
    help='URL to a Demisto instance. You can also specify the URL as an environment variable named: DEMISTO_BASE_URL'
)
@click.option(
    '--playbook_id', '-p',
    help="The playbook ID to run.",
    required=True
)
@click.option(
    '--wait', '-w', is_flag=True,
    help="Wait until the playbook run is finished and get a response."
)
@click.option(
    '--timeout', '-t',
    default=90,
    show_default=True,
    help="Timeout for the command. The playbook will continue to run in Demisto"
)
@click.option(
    "--insecure", help="Skip certificate validation.", is_flag=True)
def run_playbook(**kwargs):
    """Run a playbook in Demisto.
    DEMISTO_API_KEY environment variable should contain a valid Demisto API Key.
    Example: DEMISTO_API_KEY=<API KEY> demisto-sdk run-playbook -p 'p_name' -u
    'https://demisto.local'.
    """
    from demisto_sdk.commands.run_playbook.playbook_runner import PlaybookRunner
    check_configuration_file('run-playbook', kwargs)
    playbook_runner = PlaybookRunner(**kwargs)
    return playbook_runner.run_playbook()


# ====================== run-test-playbook ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-tpb', '--test-playbook-path',
    help="Path to test playbook to run, "
         "can be a path to specific test playbook or path to pack name for example: Packs/GitHub.",
    required=False
)
@click.option(
    '--all', is_flag=True,
    help="Run all the test playbooks from this repository."
)
@click.option(
    '--wait', '-w', is_flag=True, default=True,
    help="Wait until the test-playbook run is finished and get a response."
)
@click.option(
    '--timeout', '-t',
    default=90,
    show_default=True,
    help="Timeout for the command. The test-playbook will continue to run in your instance"
)
@click.option(
    "--insecure", help="Skip certificate validation.", is_flag=True)
def run_test_playbook(**kwargs):
    """Run a test playbooks in your instance."""
    from demisto_sdk.commands.run_test_playbook.test_playbook_runner import TestPlaybookRunner
    check_configuration_file('run-test-playbook', kwargs)
    test_playbook_runner = TestPlaybookRunner(**kwargs)
    return test_playbook_runner.manage_and_run_test_playbooks()


# ====================== generate-outputs ====================== #
@main.command(short_help='''Generates outputs (from json or examples).''')
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-c", "--command", help="Specific command name (e.g. xdr-get-incidents)",
    required=False)
@click.option(
    "-j", "--json",
    help="Valid JSON file path. If not specified, the script will wait for user input in the terminal. "
         "The response can be obtained by running the command with `raw-response=true` argument.",
    required=False)
@click.option(
    "-p", "--prefix",
    help="Output prefix like Jira.Ticket, VirusTotal.IP, the base path for the outputs that the "
         "script generates", required=False)
@click.option(
    "-o", "--output",
    help="Output file path, if not specified then will print to stdout",
    required=False)
@click.option(
    "-v", "--verbose", is_flag=True,
    help="Verbose output - mainly for debugging purposes")
@click.option(
    "--ai", is_flag=True,
    help="**Experimental** - Help generate context descriptions via AI transformers (must have a valid AI21 key at ai21.com)")
@click.option(
    "--interactive",
    help="If passed, then for each output field will ask user interactively to enter the "
         "description. By default is interactive mode is disabled. No need to use with --ai (it is already interactive)",
    is_flag=True)
@click.option(
    "-d", "--descriptions",
    help="A JSON or a path to a JSON file, mapping field names to their descriptions. "
         "If not specified, the script prompt the user to input the JSON content.",
    is_flag=True)
@click.option(
    "-i", "--input",
    help="Valid YAML integration file path.",
    required=False)
@click.option(
    "-e", "--examples",
    help="Integrations: path for file containing command examples."
         " Each command should be in a separate line."
         " Scripts: the script example surrounded by quotes."
         " For example: -e '!ConvertFile entry_id=<entry_id>'")
@click.option(
    "--insecure",
    help="Skip certificate validation to run the commands in order to generate the docs.",
    is_flag=True)
def generate_outputs(**kwargs):
    """Demisto integrations/scripts have a YAML file that defines them.
    Creating the YAML file is a tedious and error-prone task of manually copying outputs from the API result to the
    file/UI/PyCharm. This script auto generates the YAML for a command from the JSON result of the relevant API call
    In addition you can supply examples files and generate the context description directly in the YML from those examples.
    """
    from demisto_sdk.commands.generate_outputs.generate_outputs import run_generate_outputs
    check_configuration_file('generate-outputs', kwargs)
    return run_generate_outputs(**kwargs)


# ====================== generate-test-playbook ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input',
    required=True,
    help='Specify integration/script yml path')
@click.option(
    '-o', '--output',
    required=False,
    help='Specify output directory or path to an output yml file. '
         'If a path to a yml file is specified - it will be the output path.\n'
         'If a folder path is specified - a yml output will be saved in the folder.\n'
         'If not specified, and the input is located at `.../Packs/<pack_name>/Integrations`, '
         'the output will be saved under `.../Packs/<pack_name>/TestPlaybooks`.\n'
         'Otherwise (no folder in the input hierarchy is named `Packs`), '
         'the output will be saved in the current directory.')
@click.option(
    '-n', '--name',
    required=True,
    help='Specify test playbook name. The output file name will be `playbook-<name>_Test.yml')
@click.option(
    '--no-outputs', is_flag=True,
    help='Skip generating verification conditions for each output contextPath. Use when you want to decide which '
         'outputs to verify and which not')
@click.option(
    "-v", "--verbose", help="Verbose output for debug purposes - shows full exception stack trace", is_flag=True)
@click.option(
    "-ab", "--all-brands", "use_all_brands",
    help="Generate a test-playbook which calls commands using integrations of all available brands. "
         "When not used, the generated playbook calls commands using instances of the provided integration brand.",
    is_flag=True
)
@click.option(
    "-c", "--commands", help="A comma-separated command names to generate playbook tasks for, "
                             "will ignore the rest of the commands."
                             "e.g xdr-get-incidents,xdr-update-incident",
    required=False
)
@click.option(
    "-e", "--examples", help="For integrations: path for file containing command examples."
                             " Each command should be in a separate line."
                             " For scripts: the script example surrounded by quotes."
                             " For example: -e '!ConvertFile entry_id=<entry_id>'"
)
@click.option(
    "-u", "--upload", help="Whether to upload the test playbook after the generation.", is_flag=True)
def generate_test_playbook(**kwargs):
    """Generate test playbook from integration or script"""
    from demisto_sdk.commands.generate_test_playbook.test_playbook_generator import PlaybookTestsGenerator
    check_configuration_file('generate-test-playbook', kwargs)
    file_type: FileType = find_type(kwargs.get('input', ''), ignore_sub_categories=True)
    if file_type not in [FileType.INTEGRATION, FileType.SCRIPT]:
        print_error('Generating test playbook is possible only for an Integration or a Script.')
        return 1

    try:
        generator = PlaybookTestsGenerator(file_type=file_type.value, **kwargs)
        if generator.run():
            sys.exit(0)
        sys.exit(1)
    except PlaybookTestsGenerator.InvalidOutputPathError as e:
        print_error(str(e))
        return 1


# ====================== init ====================== #


@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-n", "--name", help="The name of the directory and file you want to create")
@click.option(
    "--id", help="The id used in the yml file of the integration or script"
)
@click.option(
    "-o", "--output", help="The output dir to write the object into. The default one is the current working "
                           "directory.")
@click.option(
    '--integration', is_flag=True, help="Create an Integration based on BaseIntegration template")
@click.option(
    '--script', is_flag=True, help="Create a Script based on BaseScript example")
@click.option(
    "--pack", is_flag=True, help="Create pack and its sub directories")
@click.option(
    "-t", "--template", help="Create an Integration/Script based on a specific template.\n"
                             "Integration template options: HelloWorld, HelloIAMWorld, FeedHelloWorld.\n"
                             "Script template options: HelloWorldScript")
@click.option(
    "-a", "--author-image", help="Path of the file 'Author_image.png'. \n "
                                 "Image will be presented in marketplace under PUBLISHER section. File should be up to 4kb and dimensions of 120x50"
)
@click.option(
    '--demisto_mock', is_flag=True,
    help="Copy the demistomock. Relevant for initialization of Scripts and Integrations within a Pack.")
@click.option(
    '--common-server', is_flag=True,
    help="Copy the CommonServerPython. Relevant for initialization of Scripts and Integrations within a Pack.")
def init(**kwargs):
    """Initialize a new Pack, Integration or Script.
    If the script/integration flags are not present, we will create a pack with the given name.
    Otherwise when using the flags we will generate a script/integration based on your selection.
    """
    from demisto_sdk.commands.init.initiator import Initiator
    check_configuration_file('init', kwargs)
    initiator = Initiator(**kwargs)
    initiator.init()
    return 0


# ====================== generate-docs ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-i", "--input", help="Path of the yml file.", required=True)
@click.option(
    "-o", "--output", help="The output dir to write the documentation file into,"
                           " documentation file name is README.md. If not specified, will be in the yml dir.",
    required=False)
@click.option(
    "-uc", "--use_cases", help="For integration - Top use-cases. Number the steps by '*' (i.e. '* foo. * bar.')",
    required=False)
@click.option(
    "-c", "--command", help="A comma-separated command names to generate doc for, will ignore the rest of the commands."
                            "e.g xdr-get-incidents,xdr-update-incident",
    required=False
)
@click.option(
    "-e", "--examples", help="Integrations: path for file containing command examples."
                             " Each command should be in a separate line."
                             " Scripts: the script example surrounded by quotes."
                             " For example: -e '!ConvertFile entry_id=<entry_id>'")
@click.option(
    "-p", "--permissions", type=click.Choice(["none", "general", "per-command"]), help="Permissions needed.",
    required=True, default='none')
@click.option(
    "-cp", "--command-permissions", help="Path for file containing commands permissions"
                                         " Each command permissions should be in a separate line."
                                         " (i.e. '<command-name> Administrator READ-WRITE')", required=False)
@click.option(
    "-l", "--limitations", help="Known limitations. Number the steps by '*' (i.e. '* foo. * bar.')", required=False)
@click.option(
    "--insecure", help="Skip certificate validation to run the commands in order to generate the docs.",
    is_flag=True)
@click.option(
    "-v", "--verbose", is_flag=True, help="Verbose output - mainly for debugging purposes.")
@click.option(
    "--old-version", help="Path of the old integration version yml file.")
@click.option(
    "--skip-breaking-changes", is_flag=True, help="Skip generating of breaking changes section.")
@click.option(
    "--custom-image-path", help="A custom path to a playbook image. If not stated, a default link will be added to the file.")
def generate_docs(**kwargs):
    """Generate documentation for integration, playbook or script from yaml file."""

    check_configuration_file('generate-docs', kwargs)
    input_path_str: str = kwargs.get('input', '')
    if not (input_path := Path(input_path_str)).exists():
        print_error(f'input {input_path_str} does not exist')
        return 1

    if (output_path := kwargs.get('output')) and not Path(output_path).is_dir():
        print_error(f'Output directory {output_path} is not a directory.')
        return 1

    if input_path.is_file():
        if input_path.suffix.lower() != '.yml':
            print_error(f'input {input_path} is not a valid yml file.')
            return 1
        _generate_docs_for_file(kwargs)

    # Add support for input which is a Playbooks directory and not a single yml file
    elif input_path.is_dir() and input_path.name == 'Playbooks':
        for yml in input_path.glob('*.yml'):
            file_kwargs = copy.deepcopy(kwargs)
            file_kwargs['input'] = str(yml)
            _generate_docs_for_file(file_kwargs)

    else:
        print_error(F'Input {input_path} is neither a valid yml file, nor a folder named Playbooks.')
        return 1

    return 0


def _generate_docs_for_file(kwargs: Dict[str, Any]):
    """Helper function for supporting Playbooks directory as an input and not only a single yml file."""

    from demisto_sdk.commands.generate_docs.generate_integration_doc import generate_integration_doc
    from demisto_sdk.commands.generate_docs.generate_playbook_doc import generate_playbook_doc
    from demisto_sdk.commands.generate_docs.generate_script_doc import generate_script_doc

    # Extract all the necessary arguments
    input_path: str = kwargs.get('input', '')
    output_path = kwargs.get('output')
    command = kwargs.get('command')
    examples: str = kwargs.get('examples', '')
    permissions = kwargs.get('permissions')
    limitations = kwargs.get('limitations')
    insecure: bool = kwargs.get('insecure', False)
    verbose: bool = kwargs.get('verbose', False)
    old_version: str = kwargs.get('old_version', '')
    skip_breaking_changes: bool = kwargs.get('skip_breaking_changes', False)
    custom_image_path: str = kwargs.get('custom_image_path', '')

    if command:
        if output_path and (not os.path.isfile(os.path.join(output_path, "README.md"))) \
                or (not output_path) \
                and (not os.path.isfile(os.path.join(os.path.dirname(os.path.realpath(input_path)), "README.md"))):
            print_error("The `command` argument must be presented with existing `README.md` docs.")
            return 1

    file_type = find_type(kwargs.get('input', ''), ignore_sub_categories=True)
    if file_type not in [FileType.INTEGRATION, FileType.SCRIPT, FileType.PLAYBOOK]:
        print_error('File is not an Integration, Script or a Playbook.')
        return 1

    if old_version and not os.path.isfile(old_version):
        print_error(F'Input old version file {old_version} was not found.')
        return 1

    if old_version and not old_version.lower().endswith('.yml'):
        print_error(F'Input old version {old_version} is not a valid yml file.')
        return 1

    if file_type == FileType.INTEGRATION:
        print(f'Generating {file_type.value.lower()} documentation')
        use_cases = kwargs.get('use_cases')
        command_permissions = kwargs.get('command_permissions')
        return generate_integration_doc(input_path=input_path, output=output_path, use_cases=use_cases,
                                        examples=examples, permissions=permissions,
                                        command_permissions=command_permissions, limitations=limitations,
                                        insecure=insecure, verbose=verbose, command=command,
                                        old_version=old_version,
                                        skip_breaking_changes=skip_breaking_changes)
    elif file_type == FileType.SCRIPT:
        print(f'Generating {file_type.value.lower()} documentation')
        return generate_script_doc(input_path=input_path, output=output_path, examples=examples,
                                   permissions=permissions,
                                   limitations=limitations, insecure=insecure, verbose=verbose)
    elif file_type == FileType.PLAYBOOK:
        print(f'Generating {file_type.value.lower()} documentation')
        return generate_playbook_doc(input_path=input_path, output=output_path, permissions=permissions,
                                     limitations=limitations, verbose=verbose, custom_image_path=custom_image_path)
    else:
        print_error(f'File type {file_type.value} is not supported.')
        return 1


# ====================== create-id-set ====================== #
@main.command(hidden=True)
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input',
    help='Input file path, the default is the content repo.',
    default=''
)
@click.option(
    "-o", "--output",
    help="Output file path, the default is the Tests directory.",
    default=''
)
@click.option(
    '-fd',
    '--fail-duplicates',
    help="Fails the process if any duplicates are found.",
    is_flag=True
)
@click.option('-mp', '--marketplace', help='The marketplace the id set are created for, that determines which packs are'
                                           ' inserted to the id set, and which items are present in the id set for '
                                           'each pack. Default is all packs exists in the content repository.',
              default='')
def create_id_set(**kwargs):
    """Create the content dependency tree by ids."""
    from demisto_sdk.commands.create_id_set.create_id_set import IDSetCreator
    from demisto_sdk.commands.find_dependencies.find_dependencies import remove_dependencies_from_id_set

    check_configuration_file('create-id-set', kwargs)
    id_set_creator = IDSetCreator(**kwargs)
    id_set, excluded_items_by_pack, excluded_items_by_type = id_set_creator.create_id_set()

    if excluded_items_by_pack:
        remove_dependencies_from_id_set(id_set, excluded_items_by_pack, excluded_items_by_type,
                                        kwargs.get('marketplace', ''))
        id_set_creator.save_id_set()


# ====================== merge-id-sets ====================== #
@main.command(hidden=True)
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i1', '--id-set1',
    help='First id_set.json file path',
    required=True
)
@click.option(
    '-i2', '--id-set2',
    help='Second id_set.json file path',
    required=True
)
@click.option(
    '-o', '--output',
    help='File path of the united id_set',
    required=True
)
@click.option(
    '-fd',
    '--fail-duplicates',
    help="Fails the process if any duplicates are found.",
    is_flag=True
)
def merge_id_sets(**kwargs):
    """Merge two id_sets"""
    from demisto_sdk.commands.common.update_id_set import merge_id_sets_from_files
    check_configuration_file('merge-id-sets', kwargs)
    first = kwargs['id_set1']
    second = kwargs['id_set2']
    output = kwargs['output']
    fail_duplicates = kwargs['fail_duplicates']

    _, duplicates = merge_id_sets_from_files(
        first_id_set_path=first,
        second_id_set_path=second,
        output_id_set_path=output
    )
    if duplicates:
        print_error(f'Failed to merge ID sets: {first} with {second}, '
                    f'there are entities with ID: {duplicates} that exist in both ID sets')
        if fail_duplicates:
            sys.exit(1)


# ====================== update-release-notes =================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-i", "--input", help="The relative path of the content pack. For example Packs/Pack_Name"
)
@click.option(
    '-u', '--update-type', help="The type of update being done. [major, minor, revision, documentation]",
    type=click.Choice(['major', 'minor', 'revision', 'documentation'])
)
@click.option(
    '-v', '--version', help="Bump to a specific version.", type=VersionParamType()
)
@click.option(
    '-g', '--use-git',
    help="Use git to identify the relevant changed files, will be used by default if '-i' is not set",
    is_flag=True
)
@click.option(
    '-f', '--force', help="Force update release notes for a pack (even if not required).", is_flag=True
)
@click.option(
    '--text', help="Text to add to all of the release notes files.",
)
@click.option(
    '--prev-ver', help='Previous branch or SHA1 commit to run checks against.'
)
@click.option(
    "--pre_release", help="Indicates that this change should be designated a pre-release version.",
    is_flag=True)
@click.option(
    "-idp", "--id-set-path", help="The path of the id-set.json used for APIModule updates.",
    type=click.Path(resolve_path=True))
@click.option(
    '-bc', '--breaking-changes', help='If new version contains breaking changes.',
    is_flag=True)
def update_release_notes(**kwargs):
    """Auto-increment pack version and generate release notes template."""
    from demisto_sdk.commands.update_release_notes.update_rn_manager import UpdateReleaseNotesManager
    check_configuration_file('update-release-notes', kwargs)
    if kwargs.get('force') and not kwargs.get('input'):
        print_error('Please add a specific pack in order to force a release notes update.')
        sys.exit(0)

    if not kwargs.get('use_git') and not kwargs.get('input'):
        click.confirm('No specific pack was given, do you want to update all changed packs?', abort=True)

    try:
        rn_mng = UpdateReleaseNotesManager(user_input=kwargs.get('input'), update_type=kwargs.get('update_type'),
                                           pre_release=kwargs.get('pre_release', False), is_all=kwargs.get('use_git'),
                                           text=kwargs.get('text'), specific_version=kwargs.get('version'),
                                           id_set_path=kwargs.get('id_set_path'), prev_ver=kwargs.get('prev_ver'),
                                           is_force=kwargs.get('force', False),
                                           is_bc=kwargs.get('breaking_changes', False))
        rn_mng.manage_rn_update()
        sys.exit(0)
    except Exception as e:
        print_error(f'An error occurred while updating the release notes: {str(e)}')
        sys.exit(1)


# ====================== find-dependencies ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-i", "--input", help="Pack path to find dependencies. For example: Pack/HelloWorld. When using the"
                          " --get-dependent-on flag, this argument can be used multiple times.", required=False,
    type=click.Path(exists=True, dir_okay=True), multiple=True)
@click.option(
    "-idp", "--id-set-path", help="Path to id set json file.", required=False, default='')
@click.option(
    "--no-update", help="Use to find the pack dependencies without updating the pack metadata.", required=False,
    is_flag=True)
@click.option('-v', "--verbose", help="Whether to print the log to the console.", required=False,
              is_flag=True)
@click.option("--use-pack-metadata", help="Whether to update the dependencies from the pack metadata.", required=False,
              is_flag=True)
@click.option("--all-packs-dependencies", help="Return a json file with ALL content packs dependencies. "
                                               "The json file will be saved under the path given in the "
                                               "'--output-path' argument", required=False, is_flag=True)
@click.option("-o", "--output-path", help="The destination path for the packs dependencies json file. This argument is "
                                          "only relevant for when using the '--all-packs-dependecies' flag.",
              required=False)
@click.option("--get-dependent-on", help="Get only the packs dependent ON the given pack. Note: this flag can not be"
                                         " used for the packs ApiModules and Base", required=False, is_flag=True)
@click.option("-d", "--dependency", help="Find which items in a specific content pack appears as a mandatory "
                                         "dependency of the searched pack ", required=False)
def find_dependencies(**kwargs):
    """Find pack dependencies and update pack metadata."""
    from demisto_sdk.commands.find_dependencies.find_dependencies import PackDependencies
    check_configuration_file('find-dependencies', kwargs)
    update_pack_metadata = not kwargs.get('no_update')
    input_paths = kwargs.get('input')  # since it can be multiple, received as a tuple
    verbose = kwargs.get('verbose', False)
    id_set_path = kwargs.get('id_set_path', '')
    use_pack_metadata = kwargs.get('use_pack_metadata', False)
    all_packs_dependencies = kwargs.get('all_packs_dependencies', False)
    get_dependent_on = kwargs.get('get_dependent_on', False)
    output_path = kwargs.get('output_path', ALL_PACKS_DEPENDENCIES_DEFAULT_PATH)
    dependency = kwargs.get('dependency', '')
    try:

        PackDependencies.find_dependencies_manager(
            id_set_path=str(id_set_path),
            verbose=verbose,
            update_pack_metadata=update_pack_metadata,
            use_pack_metadata=use_pack_metadata,
            input_paths=input_paths,
            all_packs_dependencies=all_packs_dependencies,
            get_dependent_on=get_dependent_on,
            output_path=output_path,
            dependency=dependency,
        )

    except ValueError as exp:
        print_error(str(exp))


# ====================== postman-codegen ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input',
    help='The Postman collection 2.1 JSON file',
    required=True, type=click.File())
@click.option(
    '-o', '--output',
    help='The output directory to save the config file or the integration',
    type=click.Path(dir_okay=True, exists=True),
    default=Path('.'),
    show_default=True
)
@click.option(
    '-n', '--name',
    help='The output integration name')
@click.option(
    '-op', '--output-prefix',
    help='The global integration output prefix. By default it is the product name.'
)
@click.option(
    '-cp', '--command-prefix',
    help='The prefix for each command in the integration. By default is the product name in lower case'
)
@click.option(
    '--config-out',
    help='Used for advanced integration customisation. Generates a config json file instead of integration.',
    is_flag=True
)
@click.option(
    '--verbose', help='Print debug level logs', is_flag=True)
@click.option(
    '-p', '--package',
    help='Generated integration will be split to package format instead of a yml file.',
    is_flag=True
)
@pass_config
def postman_codegen(
        config,
        input: IO,
        output: Path,
        name: str,
        output_prefix: str,
        command_prefix: str,
        config_out: bool,
        verbose: bool,
        package: bool
):
    """Generates a Cortex XSOAR integration given a Postman collection 2.1 JSON file."""
    from demisto_sdk.commands.common.logger import logging_setup
    from demisto_sdk.commands.postman_codegen.postman_codegen import postman_to_autogen_configuration
    from demisto_sdk.commands.split.ymlsplitter import YmlSplitter

    if verbose:
        logger = logging_setup(verbose=3)
    else:
        logger = logging.getLogger('demisto-sdk')

    postman_config = postman_to_autogen_configuration(
        collection=json.load(input),
        name=name,
        command_prefix=command_prefix,
        context_path_prefix=output_prefix
    )

    if config_out:
        path = Path(output) / f'config-{postman_config.name}.json'
        path.write_text(json.dumps(postman_config.to_dict(), indent=4))
        logger.info(f'Config file generated at:\n{str(path.absolute())}')
    else:
        # generate integration yml
        yml_path = postman_config.generate_integration_package(output, is_unified=True)
        if package:
            yml_splitter = YmlSplitter(
                configuration=config.configuration,
                file_type=FileType.INTEGRATION,
                input=str(yml_path),
                output=str(output),
            )
            yml_splitter.extract_to_package_format()
            print_success(f'Package generated at {str(Path(output).absolute())} successfully')
        else:
            print_success(f'Integration generated at {str(yml_path.absolute())} successfully')


# ====================== generate-integration ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input',
    help='config json file produced by commands like postman-codegen and openapi-codegen',
    required=True,
    type=click.File()
)
@click.option(
    '-o', '--output',
    help='The output directory to save the integration package',
    type=click.Path(dir_okay=True, exists=True),
    default=Path('.')
)
@click.option(
    '--verbose',
    help='Print debug level logs',
    is_flag=True
)
def generate_integration(input: IO, output: Path, verbose: bool):
    """Generates a Cortex XSOAR integration from a config json file,
    which is generated by commands like postman-codegen
    """
    from demisto_sdk.commands.common.logger import logging_setup
    from demisto_sdk.commands.generate_integration.code_generator import IntegrationGeneratorConfig
    if verbose:
        logging_setup(verbose=3)

    config_dict = json.load(input)
    config = IntegrationGeneratorConfig(**config_dict)

    config.generate_integration_package(output, True)


# ====================== openapi-codegen ====================== #
@main.command(short_help='''Generates a Cortex XSOAR integration given an OpenAPI specification file.''')
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input_file', help='The swagger file to load in JSON format', required=True)
@click.option(
    '-cf', '--config_file', help='The integration configuration file. It is created in the first run of the command',
    required=False)
@click.option(
    '-n', '--base_name', help='The base filename to use for the generated files', required=False)
@click.option(
    '-o', '--output_dir', help='Directory to store the output in (default is current working directory)',
    required=False)
@click.option(
    '-pr', '--command_prefix', help='Add a prefix to each command in the code', required=False)
@click.option(
    '-c', '--context_path', help='Context output path', required=False)
@click.option(
    '-u', '--unique_keys', help='Comma separated unique keys to use in context paths (case sensitive)', required=False)
@click.option(
    '-r', '--root_objects', help='Comma separated JSON root objects to use in command outputs (case sensitive)',
    required=False)
@click.option(
    '-v', '--verbose', is_flag=True, help='Be verbose with the log output')
@click.option(
    '-f', '--fix_code', is_flag=True, help='Fix the python code using autopep8')
@click.option(
    '-a', '--use_default', is_flag=True, help='Use the automatically generated integration configuration'
                                              ' (Skip the second run).')
def openapi_codegen(**kwargs):
    """Generates a Cortex XSOAR integration given an OpenAPI specification file.
    In the first run of the command, an integration configuration file is created, which can be modified.
    Then, the command is run a second time with the integration configuration to generate the actual integration files.
    """
    from demisto_sdk.commands.openapi_codegen.openapi_codegen import OpenAPIIntegration
    check_configuration_file('openapi-codegen', kwargs)
    if not kwargs.get('output_dir'):
        output_dir = os.getcwd()
    else:
        output_dir = kwargs['output_dir']

    # Check the directory exists and if not, try to create it
    if not os.path.exists(output_dir):
        try:
            os.mkdir(output_dir)
        except Exception as err:
            print_error(f'Error creating directory {output_dir} - {err}')
            sys.exit(1)
    if not os.path.isdir(output_dir):
        print_error(f'The directory provided "{output_dir}" is not a directory')
        sys.exit(1)

    input_file = kwargs['input_file']
    base_name = kwargs.get('base_name')
    if base_name is None:
        base_name = 'GeneratedIntegration'

    command_prefix = kwargs.get('command_prefix')
    if command_prefix is None:
        command_prefix = '-'.join(base_name.split(' ')).lower()

    context_path = kwargs.get('context_path')
    if context_path is None:
        context_path = base_name.replace(' ', '')

    unique_keys = kwargs.get('unique_keys', '')
    if unique_keys is None:
        unique_keys = ''

    root_objects = kwargs.get('root_objects', '')
    if root_objects is None:
        root_objects = ''

    verbose = kwargs.get('verbose', False)
    fix_code = kwargs.get('fix_code', False)

    configuration = None
    if kwargs.get('config_file'):
        try:
            with open(kwargs['config_file'], 'r') as config_file:
                configuration = json.load(config_file)
        except Exception as e:
            print_error(f'Failed to load configuration file: {e}')

    click.echo('Processing swagger file...')
    integration = OpenAPIIntegration(input_file, base_name, command_prefix, context_path,
                                     unique_keys=unique_keys, root_objects=root_objects,
                                     verbose=verbose, fix_code=fix_code, configuration=configuration)

    integration.load_file()
    if not kwargs.get('config_file'):
        integration.save_config(integration.configuration, output_dir)
        print_success(f'Created configuration file in {output_dir}')
        if not kwargs.get('use_default', False):
            config_path = os.path.join(output_dir, f'{base_name}_config.json')
            command_to_run = f'demisto-sdk openapi-codegen -i "{input_file}" -cf "{config_path}" -n "{base_name}" ' \
                             f'-o "{output_dir}" -pr "{command_prefix}" -c "{context_path}"'
            if unique_keys:
                command_to_run = command_to_run + f' -u "{unique_keys}"'
            if root_objects:
                command_to_run = command_to_run + f' -r "{root_objects}"'
            if verbose:
                command_to_run = command_to_run + ' -v'
            if fix_code:
                command_to_run = command_to_run + ' -f'

            click.echo(f'Run the command again with the created configuration file(after a review): {command_to_run}')
            sys.exit(0)

    if integration.save_package(output_dir):
        print_success(f'Successfully finished generating integration code and saved it in {output_dir}')
    else:
        print_error(f'There was an error creating the package in {output_dir}')
        sys.exit(1)


# ====================== test-content command ====================== #
@main.command(
    short_help='''Created incidents for selected test-playbooks and gives a report about the results''',
    hidden=True
)
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-k', '--api-key', help='The Demisto API key for the server', required=True)
@click.option(
    '-s', '--server', help='The server URL to connect to')
@click.option(
    '-c', '--conf', help='Path to content conf.json file', required=True)
@click.option(
    '-e', '--secret', help='Path to content-test-conf conf.json file')
@click.option(
    '-n', '--nightly', type=bool, help='Run nightly tests')
@click.option(
    '-t', '--slack', help='The token for slack', required=True)
@click.option(
    '-a', '--circleci', help='The token for circleci', required=True)
@click.option(
    '-b', '--build-number', help='The build number', required=True)
@click.option(
    '-g', '--branch-name', help='The current content branch name', required=True)
@click.option(
    '-i', '--is-ami', type=bool, help='is AMI build or not', default=False)
@click.option(
    '-m',
    '--mem-check',
    type=bool,
    help='Should trigger memory checks or not. The slack channel to check the data is: '
         'dmst_content_nightly_memory_data',
    default=False)
@click.option(
    '-d',
    '--server-version',
    help='Which server version to run the tests on(Valid only when using AMI)',
    default="NonAMI")
@click.option(
    '-u',
    '--use-retries',
    is_flag=True,
    help='Should use retries mechanism or not (if test-playbook fails, it will execute it again few times and '
         'determine success according to most of the runs',
    default=False)
@click.option(
    '--server-type',
    help='Which server runs the tests? XSIAM or XSOAR',
    default='XSOAR')
@click.option(
    '-x',
    '--xsiam-machine',
    help='XSIAM machine to use, if it is XSIAM build.')
@click.option('--xsiam-servers-path', help='Path to secret xsiam server metadata file.')
@click.option('--xsiam-servers-api-keys-path', help='Path to file with XSIAM Servers api keys.')
def test_content(**kwargs):
    """Configure instances for the integration needed to run tests_to_run tests.
    Run test module on each integration.
    create an investigation for each test.
    run test playbook on the created investigation using mock if possible.
    Collect the result and give a report.
    """
    from demisto_sdk.commands.test_content.execute_test_content import execute_test_content
    check_configuration_file('test-content', kwargs)
    execute_test_content(**kwargs)


# ====================== doc-review ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input', type=str, help='The path to the file to check', multiple=True)
@click.option(
    '--no-camel-case', is_flag=True, help='Whether to check CamelCase words', default=False)
@click.option(
    '--known-words', type=str, help="The path to a file containing additional known words", multiple=True
)
@click.option(
    '--always-true', is_flag=True, help="Whether to fail the command if misspelled words are found"
)
@click.option(
    '--expand-dictionary', is_flag=True, help="Whether to expand the base dictionary to include more words - "
                                              "will download 'brown' corpus from nltk package"
)
@click.option(
    '--templates', is_flag=True, help="Whether to print release notes templates"
)
@click.option(
    '-g', '--use-git', is_flag=True, help="Use git to identify the relevant changed files, "
                                          "will be used by default if '-i' and '--templates' are not set"
)
@click.option(
    '--prev-ver', type=str, help="The branch against which changes will be detected "
                                 "if '-g' flag is set. Default is 'demisto/master'"
)
@click.option(
    '-rn', '--release-notes', is_flag=True, help="Will run only on release notes files"
)
@click.option(
    '-pkw', '--use-packs-known-words', is_flag=True, help="Will find and load the known_words file from the pack. "
                                                          "To use this option make sure you are running from the "
                                                          "content directory.", default=False
)
def doc_review(**kwargs):
    """Check the spelling in .md and .yml files as well as review release notes"""
    from demisto_sdk.commands.doc_reviewer.doc_reviewer import DocReviewer
    doc_reviewer = DocReviewer(
        file_paths=kwargs.get('input', []),
        known_words_file_paths=kwargs.get('known_words', []),
        no_camel_case=kwargs.get('no_camel_case'),
        no_failure=kwargs.get('always_true'),
        expand_dictionary=kwargs.get('expand_dictionary'),
        templates=kwargs.get('templates'),
        use_git=kwargs.get('use_git'),
        prev_ver=kwargs.get('prev_ver'),
        release_notes_only=kwargs.get('release_notes'),
        load_known_words_from_pack=kwargs.get('use_packs_known_words'),
    )
    result = doc_reviewer.run_doc_review()
    if result:
        sys.exit(0)

    sys.exit(1)


# ====================== integration-diff ====================== #
@main.command(name="integration-diff",
              help='''Given two versions of an integration, Check that everything in the old integration is covered in
              the new integration''')
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-n', '--new', type=str, help='The path to the new version of the integration', required=True)
@click.option(
    '-o', '--old', type=str, help='The path to the old version of the integration', required=True)
@click.option(
    '--docs-format', is_flag=True,
    help='Whether output should be in the format for the version differences section in README.')
def integration_diff(**kwargs):
    """
    Checks for differences between two versions of an integration, and verified that the new version covered the old version.
    """
    from demisto_sdk.commands.integration_diff.integration_diff_detector import IntegrationDiffDetector
    integration_diff_detector = IntegrationDiffDetector(
        new=kwargs.get('new', ''),
        old=kwargs.get('old', ''),
        docs_format=kwargs.get('docs_format', False)
    )
    result = integration_diff_detector.check_different()

    if result:
        sys.exit(0)

    sys.exit(1)


# ====================== generate_yml_from_python ====================== #
@main.command(name="generate-yml-from-python",
              help='''Generate YML file from Python code that includes special syntax.\n
                      The output file name will be the same as the Python code with the `.yml` extension instead of `.py`.\n
                      The generation currently supports integrations only.\n
                      For more information on usage and installation visit the command's README.md file.''')
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input', type=click.Path(exists=True), help='The path to the python code to generate from', required=True)
@click.option(
    '-v', '--verbose', is_flag=True, type=bool, help='Indicate for extended prints.', required=False)
@click.option(
    '-f', '--force', is_flag=True, type=bool, help='Override existing yml file.', required=False)
def generate_yml_from_python(**kwargs):
    """
    Checks for differences between two versions of an integration, and verified that the new version covered the old version.
    """
    from demisto_sdk.commands.generate_yml_from_python.generate_yml import YMLGenerator

    yml_generator = YMLGenerator(filename=kwargs.get('input', ''), verbose=kwargs.get('verbose', False),
                                 force=kwargs.get('force', False))
    yml_generator.generate()
    yml_generator.save_to_yml_file()


# ====================== convert ====================== #
@main.command()
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input', type=click.Path(exists=True), required=True,
    help='The path of the content pack/directory/file to convert.'
)
@click.option(
    '-v', '--version', required=True, help="Version the input to be compatible with."
)
@pass_config
def convert(config, **kwargs):
    """
    Convert the content of the pack/directory in the given input to be compatible with the version given by
    version command.
    """
    from demisto_sdk.commands.convert.convert_manager import ConvertManager
    check_configuration_file('convert', kwargs)
    sys.path.append(config.configuration.env_dir)

    input_path = kwargs['input']
    server_version = kwargs['version']
    convert_manager = ConvertManager(input_path, server_version)
    result = convert_manager.convert()

    if result:
        sys.exit(1)

    sys.exit(0)


# ====================== generate-unit-tests ====================== #


@main.command(short_help='''Generates unit tests for integration code.''')
@click.help_option(
    '-h', '--help'
)
@click.option(
    "-c", "--commands", help="Specific commands name to generate unit test for (e.g. xdr-get-incidents)",
    required=False)
@click.option(
    '-o', '--output_dir', help='Directory to store the output in (default is the input integration directory)',
    required=False)
@click.option('-v', "--verbose", count=True, help="Verbosity level -v / -vv / .. / -vvv",
              type=click.IntRange(0, 3, clamp=True), default=1, show_default=True)
@click.option(
    "-i", "--input_path",
    help="Valid integration file path.",
    required=True)
@click.option('-q', "--quiet", is_flag=True, help="Quiet output, only output results in the end")
@click.option("-lp", "--log-path", help="Path to store all levels of logs",
              type=click.Path(resolve_path=True))
@click.option(
    '-d', '--use_demisto',
    help="Run commands at Demisto automatically.", is_flag=True
)
@click.option(
    "--insecure",
    help="Skip certificate validation", is_flag=True
)
@click.option(
    "-e", "--examples",
    help="Integrations: path for file containing command examples."
         " Each command should be in a separate line.")
@click.option(
    "-a", "--append",
    help="Append generated test file to the existing <integration_name>_test.py. Else, overwriting existing UT",
    is_flag=True)
def generate_unit_tests(input_path: str = '',
                        commands: list = [],
                        output_dir: str = '',
                        examples: str = '',
                        insecure: bool = False,
                        use_demisto: bool = False,
                        append: bool = False,
                        verbose: int = 1,
                        quiet: bool = False,
                        log_path: str = ''):
    """
    This command is used to generate unit tests automatically from an  integration python code.
    Also supports generating unit tests for specific commands.
    """

    klara_logger = logging.getLogger('PYSCA')
    klara_logger.propagate = False
    from demisto_sdk.commands.common.logger import logging_setup
    from demisto_sdk.commands.generate_unit_tests.generate_unit_tests import run_generate_unit_tests
    logging_setup(verbose=verbose,  # type: ignore[arg-type]
                  quiet=quiet,  # type: ignore[arg-type]
                  log_path=log_path)  # type: ignore[arg-type]
    return run_generate_unit_tests(
        input_path,
        commands,
        output_dir,
        examples,
        insecure,
        use_demisto,
        append
    )


@main.command(
    name='error-code',
    help='Quickly find relevant information regarding an error code.',
)
@click.help_option(
    '-h', '--help'
)
@click.option(
    '-i', '--input', required=True,
    help='The error code to search for.',
)
@pass_config
def error_code(config, **kwargs):
    from demisto_sdk.commands.error_code_info.error_code_info import generate_error_code_information
    check_configuration_file('error-code-info', kwargs)
    sys.path.append(config.configuration.env_dir)

    result = generate_error_code_information(kwargs.get('input'))

    sys.exit(result)


# ====================== create-content-graph ====================== #
@main.command(
    hidden=True,
)
@click.help_option(
    '-h', '--help'
)
@click.option('-ud', '--use-docker', is_flag=True, help="Use docker service to run the content graph")
@click.option('-us', '--use-existing', is_flag=True, help="Use existing service", default=False)
@click.option('-d', '--dependencies', is_flag=True, help="Whether dependencies should be included in the graph", default=False)
@click.option('-o', '--output-file', type=click.Path(), help="dump file output", default=None)
@click.option('-v', "--verbose", count=True, help="Verbosity level -v / -vv / .. / -vvv",
              type=click.IntRange(0, 3, clamp=True), default=2, show_default=True)
@click.option('-q', "--quiet", is_flag=True, help="Quiet output, only output results in the end")
@click.option("-lp", "--log-path", help="Path to store all levels of logs",
              type=click.Path(resolve_path=True))
def create_content_graph(
    use_docker: bool = False,
    use_existing: bool = False,
    dependencies: bool = False,
    output_file: Path = None,
    **kwargs,
):
    from demisto_sdk.commands.common.logger import logging_setup
    from demisto_sdk.commands.content_graph.content_graph_commands import \
        create_content_graph as create_content_graph_command
    logging_setup(verbose=kwargs.get('verbose'),  # type: ignore[arg-type]
                  quiet=kwargs.get('quiet'),  # type: ignore[arg-type]
                  log_path=kwargs.get('log_path'))  # type: ignore[arg-type]
    with Neo4jContentGraphInterface(
        start_service=not use_existing,
        output_file=Path(output_file) if output_file else None,
        use_docker=use_docker,
    ) as content_graph_interface:
        create_content_graph_command(content_graph_interface, dependencies)


@main.result_callback()
def exit_from_program(result=0, **kwargs):
    sys.exit(result)


if __name__ == '__main__':
    main()


# demisto_sdk/commands/test_content/tests/DemistoClientMock.py

import re
from typing import List
from unittest.mock import MagicMock

from demisto_sdk.commands.common.constants import PB_Status


class DemistoClientMock:
    INCIDNET_NUMNER = 1
    INCIDENT_SEARCH_PATTERN = re.compile(r'/inv-playbook/[\d]+')
    INVESTIGATION_SEARCH_PATTERN = re.compile(r'/investigation/[\d]+')
    INTEGRATION_SEARCH_PATTERN = re.compile(r'/settings/integration/(.*)')
    INCIDENT_QUERY_PATTERN = re.compile(r'id: ([\d]+)')
    GENERIC_RESPONSE = ("{}", 200, "")
    INSTANCES = []
    CONFIGURATIONS = []
    api_client = MagicMock()
    demisto_api = MagicMock()
    api_client.configuration.host = 'https://1.1.1.1'

    def __init__(self, integrations: List[str] = None):
        """
        Configures the integrations known by the server which will be returned by the search integration request.
        Args:
            integrations: A list of integration names to be returned.
        """
        if integrations:
            for integration in integrations:
                self.add_integration_configuration(integration)

    @staticmethod
    def configure(*_, **__):
        return DemistoClientMock

    @staticmethod
    def create_incident(**_):
        DemistoClientMock.INCIDNET_NUMNER += 1
        return MagicMock(id=DemistoClientMock.INCIDNET_NUMNER,
                         investigation_id=DemistoClientMock.INCIDNET_NUMNER)

    @staticmethod
    def search_incidents(filter):
        incident_id = DemistoClientMock.INCIDENT_QUERY_PATTERN.findall(filter.filter.query)[0]
        return MagicMock(total=1,
                         data=[MagicMock(id=incident_id, investigation_id=incident_id)])
        pass

    @classmethod
    def generic_request_func(cls, self, method, path, **kwargs):
        response_mapper = {
            '/incident/batchDelete': {'POST': cls._delete_incident},
            '/settings/integration/search': {'POST': cls._search_integration_instances},
            '/settings/integration': {'PUT': cls._create_integration_instance},
            '/settings/integration/test': {'POST': cls._test_integration_instance},
            '/containers/reset': {'POST': cls._reset_containers},
            '/system/config': {'GET': cls._get_system_config,
                               'POST': cls._update_system_config}
        }
        if method == 'GET' and cls.INCIDENT_SEARCH_PATTERN.match(path):
            return cls._get_investigation_playbook_state(**kwargs)
        if method == 'POST' and cls.INVESTIGATION_SEARCH_PATTERN.match(path):
            return cls._get_investigation_playbook_state(**kwargs)
        if method == 'DELETE' and cls.INTEGRATION_SEARCH_PATTERN.match(path):
            integration_id = cls.INTEGRATION_SEARCH_PATTERN.findall(path)[0]
            return cls._delete_integration_instance(integration_id)
        mock_function = response_mapper.get(path, {}).get(method)
        if not mock_function:
            return cls.GENERIC_RESPONSE
        return mock_function(**kwargs)

    @staticmethod
    def _delete_incident(*_, **__):
        return DemistoClientMock.GENERIC_RESPONSE

    @staticmethod
    def _search_integration_instances(*_, **__):
        return str({'configurations': DemistoClientMock.CONFIGURATIONS,
                    'instances': DemistoClientMock.INSTANCES}), 200, ""

    @staticmethod
    def _create_integration_instance(*_, **kwargs):
        module_instance = kwargs.get('body')
        module_instance['id'] = module_instance['brand']
        DemistoClientMock.INSTANCES.append(module_instance)
        return str(module_instance), 200, ""

    @staticmethod
    def _delete_integration_instance(integration_id):
        for instance in DemistoClientMock.INSTANCES:
            if instance.get('id') == integration_id:
                DemistoClientMock.INSTANCES.remove(instance)
        return DemistoClientMock.GENERIC_RESPONSE

    @staticmethod
    def _test_integration_instance(*_, **__):
        return str({"success": True, "message": ""}), 200, ""

    @staticmethod
    def _get_investigation_playbook_state(*_, **__):
        return str({'state': PB_Status.COMPLETED}), 200, ""

    @staticmethod
    def _reset_containers(*_, **__):
        return DemistoClientMock.GENERIC_RESPONSE

    @staticmethod
    def _get_system_config(*_, **__):
        return str({'sysConf': {}}), 200, ""

    @staticmethod
    def _update_system_config(*_, **__):
        return DemistoClientMock.GENERIC_RESPONSE

    @staticmethod
    def add_integration_configuration(integration_name):
        DemistoClientMock.CONFIGURATIONS.append({
            'id': integration_name,
            'name': integration_name,
            'category': 'some_category',
            'configuration': [
                {'display': 'proxy',
                 'name': 'proxy',
                 'defaultValue': True},
                {'display': 'insecure',
                 'name': 'insecure',
                 'defaultValue': True}
            ]
        })


# demisto_sdk/commands/test_content/tests/ParallelLoggingManager_test.py

from threading import Thread, currentThread

import pytest

from demisto_sdk.commands.test_content.ParallelLoggingManager import ParallelLoggingManager


class TestParallelLoggingManager:

    def test_queue_listener_sanity(self, tmp_path):
        """
        Given:
            - a ParallelLoggingManager
        When:
            - writing logs to the log file
        Then:
            - assert logs are not written to the file until execute_logs method is called
            - assert all logs appear in file after execute_logs method is called
        """
        pytest.skip("For unknown reason this test cannot pass in the current infra")
        log_file_path = f'{tmp_path}/log_file.log'
        logging_manager = ParallelLoggingManager(log_file_path)
        logging_manager.debug('debug1')
        logging_manager.info('info1')
        logging_manager.warning('warning1')
        logging_manager.error('error1')
        logging_manager.critical('critical1')
        logging_manager.success('success1')
        # Generating exception
        try:
            _ = 1 / 0
        except Exception:
            logging_manager.exception('exception1')
        log_file_lines = self.get_log_content(log_file_path)
        assert not log_file_lines
        logging_manager.execute_logs()
        log_file_lines = self.get_log_content(log_file_path)
        for expected_in_log, log_line in zip(self.get_expected_content('1').values(), log_file_lines):
            thread_name, level, content = expected_in_log
            self.assert_log_line(log_line, thread_name, level, content)

    def test_real_time_logger_sanity(self, tmp_path):
        """
        Given:
            - a ParallelLoggingManager
        When:
            - writing logs to the log file using the flag 'real_time=True'
        Then:
            - assert logs are written to the file immediately
        """
        pytest.skip("For unknown reason this test cannot pass in the current infra")
        log_file_path = f'{tmp_path}/log_file.log'
        logging_manager = ParallelLoggingManager(log_file_path)
        expected_logs = self.get_expected_content('1')
        logging_manager.debug('debug1', real_time=True)
        self.assert_latest_log_line(log_file_path, *expected_logs['debug'])
        logging_manager.info('info1', real_time=True)
        self.assert_latest_log_line(log_file_path, *expected_logs['info'])
        logging_manager.warning('warning1', real_time=True)
        self.assert_latest_log_line(log_file_path, *expected_logs['warning'])
        logging_manager.error('error1', real_time=True)
        self.assert_latest_log_line(log_file_path, *expected_logs['error'])
        logging_manager.critical('critical1', real_time=True)
        self.assert_latest_log_line(log_file_path, *expected_logs['critical'])
        logging_manager.success('success1', real_time=True)
        self.assert_latest_log_line(log_file_path, *expected_logs['success'])

    def test_listeners_with_multiple_threads(self, tmp_path):
        """
        Given:
            - a ParallelLoggingManager
        When:
            - writing logs to the log file with 5 different threads
        Then:
            - assert logs does not appear in the file before each thread has called the 'execute_logs' method
            - assert logs are written to the file grouped together for each thread
        """
        pytest.skip("For unknown reason this test cannot pass in the current infra")
        log_file_path = f'{tmp_path}/log_file.log'
        logging_manager = ParallelLoggingManager(log_file_path)
        successful_threads_results = set()

        def thread_log_function():
            thread_name = currentThread().getName()
            logging_manager.info('test1')
            logging_manager.info('test2')
            logging_manager.info('test3')
            log_content_lines = TestParallelLoggingManager.get_log_content(logging_manager.log_file_name)
            thread_logs = [line for line in log_content_lines if thread_name in line]
            assert not thread_logs
            logging_manager.execute_logs()
            log_content_lines = TestParallelLoggingManager.get_log_content(logging_manager.log_file_name)
            first_thread_log_line = next(line for line in log_content_lines if thread_name in line)
            first_thread_log_line_index = log_content_lines.index(first_thread_log_line)
            self.assert_log_line(log_content_lines[first_thread_log_line_index], thread_name, 'INFO', 'test1')
            self.assert_log_line(log_content_lines[first_thread_log_line_index + 1], thread_name, 'INFO', 'test2')
            self.assert_log_line(log_content_lines[first_thread_log_line_index + 2], thread_name, 'INFO', 'test3')
            successful_threads_results.add(thread_name)

        threads = []
        for i in range(5):
            threads.append(Thread(target=thread_log_function))
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()
            assert thread.getName() in successful_threads_results

    @staticmethod
    def get_expected_content(content_postfix: str) -> dict:
        """
        Generate a dict that contains for each log level a tuple with the thread name, the log level and the content
        of the log message
        Args:
            content_postfix: the suffix that is added to each log file
        Returns:
            A dict that contains for each log level a tuple with the thread name, the log level and the content
        """
        thread_name = currentThread().getName()
        return {'debug': (thread_name, 'DEBUG', f'debug{content_postfix}'),
                'info': (thread_name, 'INFO', f'info{content_postfix}'),
                'warning': (thread_name, 'WARNING', f'warning{content_postfix}'),
                'error': (thread_name, 'ERROR', f'error{content_postfix}'),
                'critical': (thread_name, 'CRITICAL', f'critical{content_postfix}'),
                'success': (thread_name, 'SUCCESS', f'success{content_postfix}'),
                'exception': (thread_name, 'ERROR', f'exception{content_postfix}'),
                }

    @staticmethod
    def get_log_content(log_file_path: str) -> list:
        """
        Reads the log file and returns it's lines
        Args:
            log_file_path: The path of the file

        Returns:
            A list with the log file lines
        """
        with open(log_file_path, 'r') as log_file:
            log_file_lines = log_file.readlines()
        return log_file_lines

    def assert_latest_log_line(self, log_file_path: str, thread_name: str, level: str, content: str) -> None:
        """
        Assert that the last line in the log file has the thread name, log level name, and content as expected
        Args:
            log_file_path: The path to the log file
            thread_name: The expected thread name
            level: The expected level
            content: The expected log message
        """
        latest_log_line = self.get_log_content(log_file_path)[-1]
        self.assert_log_line(latest_log_line, thread_name, level, content)

    @staticmethod
    def assert_log_line(log_line: str, thread_name: str, level: str, content: str) -> None:
        """
        Assert that the line of the log file has the thread name, log level name, and content as expected
        Args:
            log_line: Given log line
            thread_name: The expected thread name
            level: The expected level
            content: The expected log message
        """
        assert thread_name in log_line
        assert level in log_line
        assert content in log_line


# demisto_sdk/commands/test_content/tests/build_context_test.py

import pytest

from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.test_content.ParallelLoggingManager import ParallelLoggingManager
from demisto_sdk.commands.test_content.TestContentClasses import BuildContext

json = JSON_Handler()


def generate_test_configuration(playbook_id: str,
                                integrations: list = None,
                                instance_names: list = None,
                                nightly: bool = None,
                                fromversion: str = '',
                                toversion: str = '',
                                timeout: int = None,
                                memory_threshold: int = None,
                                pid_threshold: int = None,
                                is_mockable: bool = None,
                                runnable_on_docker_only: bool = None
                                ) -> dict:
    playbook_config = {
        'playbookID': playbook_id,
    }
    if integrations:
        playbook_config['integrations'] = integrations
    if instance_names:
        playbook_config['instance_names'] = instance_names
    if nightly is not None:
        playbook_config['nightly'] = nightly
    if fromversion:
        playbook_config['fromversion'] = fromversion
    if toversion:
        playbook_config['toversion'] = toversion
    if timeout:
        playbook_config['timeout'] = timeout
    if memory_threshold:
        playbook_config['memory_threshold'] = memory_threshold
    if pid_threshold:
        playbook_config['pid_threshold'] = pid_threshold
    if runnable_on_docker_only is not None:
        playbook_config['runnable_on_docker_only'] = runnable_on_docker_only
    if is_mockable is not None:
        playbook_config['is_mockable'] = is_mockable
    return playbook_config


def generate_content_conf_json(
        tests: list = None,
        skipped_tests: dict = None,
        skipped_integrations: dict = None,
        unmockable_integrations: dict = None,
        parallel_integrations: list = None,
        docker_thresholds_images: dict = None
) -> dict:
    """
    Generates a replica of the content conf.json file according to parameters
    Args:
        tests: A dict with a test playbook configuration
        skipped_tests: A dict containing playbook IDs as keys and the reason why it was skipped as value
        skipped_integrations: A list containing integration IDs
        unmockable_integrations: A dict containing integration IDs as keys and the reason why it's unmockable as value
        parallel_integrations: A list containing integration IDs
        docker_thresholds_images: A dict containing image name as key and a dict containing docker limitations as value
    Returns:
        A replica of the content conf.json file according to parameters
    """
    return {
        'testTimeout': 160,
        'testInterval': 20,
        'tests': tests or [],
        'skipped_tests': skipped_tests or {},
        'skipped_integrations': skipped_integrations or {},
        'unmockable_integrations': unmockable_integrations or {},
        'parallel_integrations': parallel_integrations or [],
        'docker_thresholds': {
            'images': docker_thresholds_images or {}
        }
    }


def generate_secret_conf_json(
        integrations: list = None
):
    return {
        'username': 'username',
        'userPassword': 'password',
        'integrations': integrations or []
    }


def generate_integration_configuration(
        name: str,
        params: dict = None,
        instance_name: str = None,
        byoi: bool = None,
        validate_test: bool = None,
) -> dict:
    """
    Generates an integration configuration according to params
    Args:
        name: The name of the integration
        params: A dict containing the params of the integration
        instance_name: the name of the instance that should be used
        byoi: is byoi integration
        validate_test: should the build validate the test-module before configuring
    """
    integration_config = {
        'name': name,
        'params': params or {}
    }
    if instance_name:
        integration_config['instance_name'] = instance_name
    if byoi is not None:
        integration_config['byoi'] = byoi
    if validate_test is not None:
        integration_config['validate_test'] = validate_test
    return integration_config


def generate_env_results_content(
        number_of_instances: int = 1,
        role: str = 'Server Master',
):
    role_to_ami_name_mapping = {
        'Server Master': 'Demisto-Marketplace-Content-AMI-Master-612266-2021-01-03',
        'Server 6.0': 'Demisto-Marketplace-Content-AMI-GA_6_0-86106-2021-01-03',
        'Server 5.5': 'Demisto-Circle-CI-Content-AMI-PreGA-5.5-87835-2021-01-03',
        'Server 5.0': 'Demisto-Circle-CI-Content-AMI-GA-5.0-62071-2021-01-03'
    }
    env_results = [{'AmiName': role_to_ami_name_mapping[role],
                    'Role': role,
                    'InstanceDNS': '1.1.1.1',
                    'TunnelPort': 4445} for _ in range(number_of_instances)]
    return env_results


def generate_xsiam_servers_data():
    return {
        "qa2-test-111111": {
            "ui_url": "https://xsiam1.paloaltonetworks.com/",
            "instance_name": "qa2-test-111111",
            "x-xdr-auth-id": 1,
            "base_url": "https://api1.paloaltonetworks.com/",
            "xsiam_version": "3.2.0",
            "demisto_version": "99.99.98"
        },
        "qa2-test-222222": {
            "ui_url": "https://xsoar-content-2.xdr-qa2-uat.us.paloaltonetworks.com/",
            "instance_name": "qa2-test-222222",
            "x-xdr-auth-id": 1,
            "base_url": "https://api-xsoar-content-2.xdr-qa2-uat.us.paloaltonetworks.com",
            "xsiam_version": "3.2.0",
            "demisto_version": "99.99.98"
        }
    }


def get_mocked_build_context(
        mocker,
        tmp_file,
        content_conf_json: dict = None,
        secret_conf_json: dict = None,
        env_results_content: dict = None,
        filtered_tests_content: list = None,
        nightly: bool = False,
        server_version: str = 'Server Master'
) -> BuildContext:
    """
    Generates a BuildContext instance with mocked data.
    Args:
        mocker: The mocker instance used in the unittest
        tmp_file: the path in which the log should be written
        content_conf_json: The contents of conf.json to load in the BuildContext instance
        secret_conf_json: The contents of content-test-conf conf.json to load in the BuildContext instance
        env_results_content: The contents of env_results.json to load in the BuildContext instance
        filtered_tests_content: The contents of filtered_tests to load in the BuildContext instance
        nightly: Indicates whether this build is a nightly build
        server_version: The server version to run the instance on
    """
    logging_manager = ParallelLoggingManager(tmp_file / 'log_file.log')
    conf_path = tmp_file / 'conf_path'
    conf_path.write_text(json.dumps(content_conf_json or generate_content_conf_json()))

    secret_conf_path = tmp_file / 'secret_conf_path'
    secret_conf_path.write_text(json.dumps(secret_conf_json or generate_secret_conf_json()))

    env_results_path = tmp_file / 'env_results_path'
    env_results_path.write_text(json.dumps(env_results_content or generate_env_results_content()))
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.ENV_RESULTS_PATH', str(env_results_path))

    filtered_tests_path = tmp_file / 'filtered_tests_path'
    filtered_tests_path.write_text('\n'.join(filtered_tests_content or []))
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.FILTER_CONF', str(filtered_tests_path))

    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.BuildContext._retrieve_slack_user_id',
                 return_value='some_user_id')
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.BuildContext._get_all_integration_config',
                 return_value=[])
    kwargs = {
        'api_key': 'api_key',
        'server': None,
        'conf': conf_path,
        'secret': secret_conf_path,
        'slack': 'slack_token',
        'nightly': nightly,
        'is_ami': True,
        'circleci': 'circle_token',
        'build_number': '11111',
        'branch_name': 'branch',
        'server_version': server_version,
        'mem_check': False,
        'server_type': 'XSOAR'
    }
    return BuildContext(kwargs, logging_manager)


def create_xsiam_build(mocker, tmp_file):
    logging_manager = ParallelLoggingManager(tmp_file / 'log_file.log')
    conf_path = tmp_file / 'conf_path'
    conf_path.write_text(json.dumps(generate_content_conf_json()))

    secret_conf_path = tmp_file / 'secret_conf_path'
    secret_conf_path.write_text(json.dumps(generate_secret_conf_json()))

    xsiam_servers_path = tmp_file / 'xsiam_servers_path.json'
    xsiam_servers_path.write_text(json.dumps(generate_xsiam_servers_data()))

    xsiam_api_keys_path = tmp_file / 'xsiam_api_keys_path.json'
    xsiam_api_keys_path.write_text(json.dumps({"qa2-test-111111": "api_key", "qa2-test-222222": "api_key"}))

    env_results_path = tmp_file / 'env_results_path'
    env_results_path.write_text(json.dumps(generate_env_results_content()))
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.ENV_RESULTS_PATH', str(env_results_path))

    filtered_tests_path = tmp_file / 'filtered_tests_path'
    filtered_tests_path.write_text('[]')
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.FILTER_CONF', str(filtered_tests_path))

    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.BuildContext._retrieve_slack_user_id',
                 return_value='some_user_id')
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.BuildContext._get_all_integration_config',
                 return_value=[])
    kwargs = {
        'api_key': 'api_key',
        'server': None,
        'conf': conf_path,
        'secret': secret_conf_path,
        'slack': 'slack_token',
        'nightly': False,
        'is_ami': True,
        'circleci': 'circle_token',
        'build_number': '11111',
        'branch_name': 'branch',
        'server_version': 'XSIAM Master',
        'mem_check': False,
        'server_type': 'XSIAM',
        'xsiam_servers_path': xsiam_servers_path,
        'xsiam_machine': 'qa2-test-111111',
        'xsiam_servers_api_keys_path': xsiam_api_keys_path
    }
    return BuildContext(kwargs, logging_manager)


def test_build_creation(mocker, tmp_path):
    """
    Given:
        - A build context for xsiam run
    When:
        - Running test-content command and creating xsiam build context
    Then:
        - All xsiam build  parameters created as expected
    """
    build_contex = create_xsiam_build(mocker, tmp_path)
    assert build_contex.is_xsiam
    assert build_contex.auth_id == 1


def test_non_filtered_tests_are_skipped(mocker, tmp_path):
    """
    Given:
        - A build context with one test in filtered tests list
    When:
        - Initializing the BuildContext instance
    Then:
        - Ensure that all tests that are not in filtered tests are skipped
        - Ensure that the test that was in  filtered tests is not skipped
    """
    filtered_tests = ['test_that_should_run']
    tests = [generate_test_configuration(playbook_id='test_that_should_run'),
             generate_test_configuration(playbook_id='test_that_should_be_skipped')]
    content_conf_json = generate_content_conf_json(tests=tests)
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             filtered_tests_content=filtered_tests)
    assert 'test_that_should_be_skipped' in build_context.tests_data_keeper.skipped_tests
    assert 'test_that_should_run' not in build_context.tests_data_keeper.skipped_tests


def test_no_tests_are_executed_when_filtered_tests_is_empty(mocker, tmp_path):
    """
    Given:
        - A build context with empty filtered tests list
    When:
        - Initializing the BuildContext instance
    Then:
        - Ensure that all tests that are skipped
    """
    tests = [generate_test_configuration(playbook_id='test_that_should_be_skipped')]
    content_conf_json = generate_content_conf_json(tests=tests)
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             filtered_tests_content=[])
    assert 'test_that_should_be_skipped' in build_context.tests_data_keeper.skipped_tests


def test_playbook_with_skipped_integrations_is_skipped(mocker, tmp_path):
    """
    Given:
        - A build context with one test in filtered tests list that has a skipped integration

    When:
        - Initializing the BuildContext instance
    Then:
        - Ensure that the playbook with the skipped integrations is skipped
    """
    filtered_tests = ['test_with_skipped_integrations']
    tests = [generate_test_configuration(playbook_id='test_with_skipped_integrations',
                                         integrations=['skipped_integration'])]
    content_conf_json = generate_content_conf_json(tests=tests,
                                                   skipped_integrations={'skipped_integration': ''})
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             filtered_tests_content=filtered_tests)
    assert 'test_with_skipped_integrations' in build_context.tests_data_keeper.skipped_tests


def test_nightly_playbook_skipping(mocker, tmp_path):
    """
    Given:
        - A build context with one nightly playbook
    When:
        - Initializing the BuildContext instance
    Then:
        - Ensure that the nightly playbook is skipped on non nightly build
        - Ensure that the nightly playbook is not skipped on nightly build
    """
    filtered_tests = ['nightly_playbook']
    tests = [generate_test_configuration(playbook_id='nightly_playbook', nightly=True)]
    content_conf_json = generate_content_conf_json(tests=tests)
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             filtered_tests_content=filtered_tests)
    assert 'nightly_playbook' in build_context.tests_data_keeper.skipped_tests
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             filtered_tests_content=filtered_tests,
                                             nightly=True)
    assert 'nightly_playbook' not in build_context.tests_data_keeper.skipped_tests


def test_playbook_with_integration(mocker, tmp_path):
    """
    Given:
        - A build context with playbook that has an integration
    When:
        - Initializing the BuildContext instance
    Then:
        - Ensure that the playbook with the integration is not skipped on nightly build
    """
    filtered_tests = ['playbook_with_integration']
    tests = [generate_test_configuration(playbook_id='playbook_with_integration',
                                         integrations=['integration'])]
    content_conf_json = generate_content_conf_json(tests=tests)
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             filtered_tests_content=filtered_tests,
                                             nightly=True)
    assert 'playbook_with_integration' not in build_context.tests_data_keeper.skipped_tests


def test_playbook_with_version_mismatch_is_skipped(mocker, tmp_path):
    """
    Given:
        - A build context for a version that does not match the playbook version
    When:
        - Initializing the BuildContext instance
    Then:
        - Ensure that the playbook with version mismatch is skipped
    """
    filtered_tests = ['playbook_with_version_mismatch']
    tests = [generate_test_configuration(playbook_id='playbook_with_version_mismatch',
                                         toversion='6.0.0')]
    content_conf_json = generate_content_conf_json(tests=tests)
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             filtered_tests_content=filtered_tests)
    assert 'playbook_with_version_mismatch' in build_context.tests_data_keeper.skipped_tests


def test_unmockable_playbook_configuration(mocker, tmp_path):
    """
    Given:
        - A build context that has a playbook configured with 'is_mockable=False'
    When:
        - Initializing the BuildContext instance
    Then:
        - Ensure that the unmockable test configuration is in the unmockable_test_ids
    """
    filtered_tests = ['unmockable_playbook']
    tests = [generate_test_configuration(playbook_id='unmockable_playbook',
                                         is_mockable=False)]
    content_conf_json = generate_content_conf_json(tests=tests)
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             filtered_tests_content=filtered_tests)
    assert 'unmockable_playbook' in build_context.unmockable_test_ids


def test_mockable_playbook_configuration(mocker, tmp_path):
    """
    Given:
        - A build context that has a playbook configured with 'is_mockable' not set
    When:
        - Initializing the BuildContext instance
    Then:
        - Ensure that the mockable test configuration is not in the unmockable_test_ids
    """
    filtered_tests = ['mockable_playbook']
    tests = [generate_test_configuration(playbook_id='mockable_playbook',
                                         integrations=['some_mockable_integration'])]
    content_conf_json = generate_content_conf_json(tests=tests)
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             filtered_tests_content=filtered_tests)
    assert 'mockable_playbook' not in build_context.unmockable_test_ids


def test_get_instances_ips(mocker, tmp_path):
    """
    Given:
        - A build context
    When:
        - Initializing the BuildContext instance
    Then:
        - Ensure that the instance ips are parsed as a dict that maps the IPs to the tunnel port.
    """
    build_context = get_mocked_build_context(mocker,
                                             tmp_path)
    assert build_context.instances_ips == {'1.1.1.1': 4445}


def test_get_public_ip_from_server_url(mocker, tmp_path):
    """
    Given:
        - A build context
    When:
        - Calling the method 'test_get_public_ip_from_server_url'
    Then:
        - Ensure it returns the internal IP with a valid port.
        - Ensure it returns the given URL as is in case there is no port in the URL prefix.
        - Ensure it raises an exception in case a non valid port is given.
    """
    build_context = get_mocked_build_context(mocker,
                                             tmp_path)
    assert build_context.get_public_ip_from_server_url('https://localhost:4445') == 'https://1.1.1.1'
    assert build_context.get_public_ip_from_server_url('https://2.2.2.2') == 'https://2.2.2.2'
    with pytest.raises(Exception) as excinfo:
        build_context.get_public_ip_from_server_url('https://localhost:4446')
    assert 'Could not find private ip for the server mapped to port 4446' in str(excinfo.value)


# demisto_sdk/commands/test_content/tests/execute_test_content_test.py

import os
from unittest import mock

import pytest
import requests_mock

from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.test_content.execute_test_content import (COVERAGE_REPORT_COMMENT, SKIPPED_CONTENT_COMMENT,
                                                                    _add_pr_comment)

json = JSON_Handler()


MOCK_ENV_VARIABLES = {
    'XSOAR_BOT_TEST_CONTENT': '123456',
    'CI_COMMIT_BRANCH': 'mock_branch',
    'CI_COMMIT_SHA': '1234567890abcdef',
    'UT_JOB_ID': '123456'
}


@pytest.mark.parametrize('comment, skipped_flow, coverage_flow', [
    (
        'The following integrations/tests were collected by the CI build but are currently skipped. '
        'The collected tests are related to this pull request and might be critical:\n- demo integration',
        True,
        False,
    ),
    (
        'Link to the coverage report of the integration:\n '
        'https://xsoar.docs.pan.run/-/content/-/jobs/123456/artifacts/artifacts/coverage_report/html/index.html',
        False,
        True
    )
])
def test_add_pr_comment(mocker, comment, skipped_flow, coverage_flow):
    """
    When:
        - A job in content pipeline is running.

    Given:
        - A pr in content.
        - A comment to add.

    Then:
        - Verify that comment was added as expected (and remove if needed)
    """

    def mock_handle_github_response(response, logging_module):
        return response.json() if response.text else ''

    mocker.patch('demisto_sdk.commands.test_content.execute_test_content._handle_github_response',
                 side_effect=mock_handle_github_response)
    url = 'https://api.github.com/search/issues'
    query = '?q=1234567890abcdef+repo:demisto/content+org:demisto+is:pr+is:open+head:mock_branch+is:open'

    with mock.patch.dict(os.environ, MOCK_ENV_VARIABLES, clear=True):
        with requests_mock.Mocker() as m:
            m.get(url + query, json={'total_count': 1,
                                     'items': [{'comments_url': 'https://api.github.com/search/issues/1'}]})
            m.get('https://api.github.com/search/issues/1', json=[
                {'body': SKIPPED_CONTENT_COMMENT, 'url': 'https://github.com/comment_12345'},
                {'body': COVERAGE_REPORT_COMMENT, 'url': 'https://github.com/comment_67890'}
            ])
            skipped_deleted = m.delete('https://github.com/comment_12345')
            coverage_deleted = m.delete('https://github.com/comment_67890')
            result = m.post('https://api.github.com/search/issues/1')

            _add_pr_comment(comment, "")

    assert json.loads(result.last_request.text).get('body') == comment
    assert skipped_deleted.called == skipped_flow
    assert coverage_deleted.called == coverage_flow


# demisto_sdk/commands/test_content/tests/integration_test.py

from copy import deepcopy

import pytest

from demisto_sdk.commands.test_content.ParallelLoggingManager import ParallelLoggingManager
from demisto_sdk.commands.test_content.TestContentClasses import BuildContext, Integration

CONFIGURATION = {
    'configuration': [
        {

            'name': 'incidentType',
            'value': ''
        },

    ],
    'name': 'example_integration',
    'category': ''
}

INCIDENT_CASES = [
    (
        {"incident_type": "Example Type"},  # case only incident type
        {'incident_type': 'Example Type', 'classifier': '', 'mapper': ''}
    ),
    (
        {
            "incident_type": "Example Type",
            "classifier_id": "Example Classifier",
            "incoming_mapper_id": "Example Mapper"},  # case both incident type and classifier/mapper
        {'incident_type': 'Example Type', 'classifier': 'Example Classifier', 'mapper': 'Example Mapper'}
    ),
    (
        {
            "classifier_id": "Example Classifier",
            "incoming_mapper_id": "Example Mapper"},  # case both incident type and classifier/mapper
        {'incident_type': '', 'classifier': 'Example Classifier', 'mapper': 'Example Mapper'}
    ),
    ({},  # case no incident configuration provided
     {'incident_type': '', 'classifier': '', 'mapper': ''}),
    (None,  # case no incident configuration provided
     {'incident_type': '', 'classifier': '', 'mapper': ''})
]


@pytest.mark.parametrize('incident_configuration, expected', INCIDENT_CASES)
def test_create_module(mocker, incident_configuration, expected):
    """
    Given:
        incident configuration with only incident type
        incident configuration with both incident type and classifier/ mapper
        incident configuration with only classifier/ mapper
        incident configuration without incident configuration
    When:
        running configuring instance for test playbook run

    Then:
        validate the module iד configured with the incident configuration.
        (the default incidentType is changed to new one, and mapper/ classifier are added)
    """

    class Dummyconf:
        unmockable_integrations = []

    test_build_params = {'api_key': '', 'server': '', 'conf': '', 'secret': '', 'nightly': '', 'circleci': '',
                         'slack': '', 'server_type': 'XSOAR',
                         'build_number': '', 'branch_name': '', 'is_ami': '', 'mem_check': '', 'server_version': ''}
    mocker.patch.object(BuildContext, '_load_conf_files', return_value=(Dummyconf(), ''))
    mocker.patch.object(BuildContext, '_load_env_results_json')
    mocker.patch.object(BuildContext, '_get_server_numeric_version')
    mocker.patch.object(BuildContext, '_get_instances_ips')
    mocker.patch.object(BuildContext, '_extract_filtered_tests')
    mocker.patch.object(BuildContext, '_get_unmockable_tests_from_conf')
    mocker.patch.object(BuildContext, '_get_tests_to_run', return_value=('', ''))
    mocker.patch.object(BuildContext, '_retrieve_slack_user_id')
    mocker.patch.object(BuildContext, '_get_all_integration_config')

    test_integration = Integration(BuildContext(test_build_params, ParallelLoggingManager('temp_log')),
                                   'example_integration', [])

    res_module = test_integration.create_module(instance_name='test', configuration=deepcopy(CONFIGURATION),
                                                incident_configuration=incident_configuration)
    assert res_module.get('configuration').get('configuration')[0].get('value') == expected.get('incident_type')
    assert res_module.get('incomingMapperId') == expected.get('mapper')
    assert res_module.get('mappingId') == expected.get('classifier')


# demisto_sdk/commands/test_content/tests/mock_unit_test.py

from __future__ import print_function

import time
from threading import Thread

from demisto_sdk.commands.test_content.mock_server import (MITMProxy, clean_filename, get_folder_path,
                                                           get_log_file_path, get_mock_file_path)


def test_clean_filename():
    assert clean_filename(u'th)))i(s) is a (test8)8   8') == 'th___i_s__is_a__test8_8___8'
    assert clean_filename(u'n&%ew $r#eplac@es', replace='&%$#@') == 'n__ew _r_eplac_es'


def test_get_paths():
    test_playbook_id = u'test_playbook'
    assert get_mock_file_path(test_playbook_id) == 'test_playbook/test_playbook.mock'
    assert get_log_file_path(test_playbook_id) == 'test_playbook/test_playbook_playback.log'
    assert get_log_file_path(test_playbook_id, record=True) == 'test_playbook/test_playbook_record.log'
    assert get_folder_path(test_playbook_id) == 'test_playbook/'


def test_push_mock_files_thread_safety(mocker):
    """
    Given:
        - A proxy instance
    When:
        - Mocking the subprocess's 'check_output' command with a method that increments a counter with 1
        and running the 'push_mock_files' with 5 threads at the same time.
    Then:
        - Ensure thread safety is achieved and that the counter equals 5 at the end of the test
        - Ensure that each thread has entered the 'increment_counter' method alone
    """
    class Counter:
        counter = 0
        modifying_dates = []

    def increment_counter(*_, **__):
        time.sleep(2)
        Counter.counter += 1
        Counter.modifying_dates.append(time.time())
        return
    # Mocking
    mocker.patch.object(MITMProxy, '__init__', lambda *args, **kwargs: None)
    proxy_instance = MITMProxy('public_ip', 'logging_module', 'build_number', 'branch_name')
    # Replacing the ami instance with MagicMock instance so that the check_output command will be mocked as well
    proxy_instance.ami = mocker.MagicMock()
    proxy_instance.logging_module = mocker.MagicMock()
    proxy_instance.counter = 0
    proxy_instance.ami.check_output = increment_counter

    # Creating 10 threads that will push together
    threads = []
    for i in range(5):
        threads.append(Thread(target=proxy_instance.push_mock_files))
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join()
    assert Counter.counter == 5
    for i in range(4):
        assert Counter.modifying_dates[i] < Counter.modifying_dates[i + 1]
        assert Counter.modifying_dates[i + 1] - Counter.modifying_dates[i] > 1


# demisto_sdk/commands/test_content/tests/server_context_test.py

from demisto_sdk.commands.test_content.mock_server import MITMProxy
from demisto_sdk.commands.test_content.TestContentClasses import BuildContext, ServerContext
from demisto_sdk.commands.test_content.tests.build_context_test import (generate_content_conf_json,
                                                                        generate_integration_configuration,
                                                                        generate_secret_conf_json,
                                                                        generate_test_configuration,
                                                                        get_mocked_build_context)
from demisto_sdk.commands.test_content.tests.DemistoClientMock import DemistoClientMock


def test_execute_tests(mocker, tmp_path):
    """
    Given:
        - A ServerContext instance that should execute three tests:
            1) A test with no integrations
            2) A test with mockable integration
            3) A test with unmockable integration

    When:
        - Running execute_tests.

    Then:
        - Ensure the server context has executed all the tests
        - Ensure the mockable and unmockable tests queues were emptied
        - Ensure no test has failed during that test
    """
    # Setting up the build context
    filtered_tests = ['playbook_without_integrations',
                      'playbook_with_mockable_integration',
                      'playbook_with_unmockable_integration',
                      'skipped_playbook']
    # Setting up the content conf.json
    tests = [generate_test_configuration(playbook_id='playbook_without_integrations'),
             generate_test_configuration(playbook_id='playbook_with_mockable_integration',
                                         integrations=['mockable_integration']),
             generate_test_configuration(playbook_id='playbook_with_unmockable_integration',
                                         integrations=['unmockable_integration']),
             generate_test_configuration(playbook_id='skipped_playbook')]
    content_conf_json = generate_content_conf_json(tests=tests,
                                                   unmockable_integrations={'unmockable_integration': 'reason'},
                                                   skipped_tests={'skipped_playbook': 'reason'})
    # Setting up the content-test-conf conf.json
    integration_names = ['mockable_integration', 'unmockable_integration']
    integrations_configurations = [generate_integration_configuration(integration_name) for integration_name in
                                   integration_names]
    secret_test_conf = generate_secret_conf_json(integrations_configurations)

    # Setting up the build_context instance
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             secret_conf_json=secret_test_conf,
                                             filtered_tests_content=filtered_tests)
    # Setting up the client
    mocked_demisto_client = DemistoClientMock(integrations=integration_names)
    server_context = generate_mocked_server_context(build_context, mocked_demisto_client, mocker)
    server_context.execute_tests()

    # Validating all tests were executed
    for test in set(filtered_tests) - {'skipped_playbook'}:
        assert test in server_context.executed_tests

    # Validating all queues were emptied
    assert build_context.mockable_tests_to_run.all_tasks_done
    assert build_context.unmockable_tests_to_run.all_tasks_done

    # Validating no failed playbooks
    assert not build_context.tests_data_keeper.failed_playbooks

    # Validating skipped test
    assert 'skipped_playbook' not in server_context.executed_tests
    assert 'skipped_playbook' in build_context.tests_data_keeper.skipped_tests


def generate_mocked_server_context(build_context: BuildContext,
                                   mocked_demisto_client: DemistoClientMock,
                                   mocker) -> ServerContext:
    """
    Creates a ServerContext with the requested build context and mocked client.
    Args:
        build_context: The build context in which the server context should be created
        mocked_demisto_client: The mocked client with which the Server context will generate requests
        mocker: Mocker object

    Returns:
        A ServerContext instance
    """
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.demisto_client', mocked_demisto_client)
    # Mocking unnecessary calls
    mocker.patch('demisto_sdk.commands.test_content.mock_server.run_with_mock').return_value.__enter__.return_value = {}
    mocker.patch('demisto_sdk.commands.test_content.IntegrationsLock.safe_lock_integrations', return_value=True)
    mocker.patch('demisto_sdk.commands.test_content.IntegrationsLock.safe_unlock_integrations')
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_docker_threshold_test')
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.is_redhat_instance', return_value=False)
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._notify_failed_test')
    mocker.patch.object(MITMProxy, '__init__', lambda *args, **kwargs: None)
    mocker.patch('time.sleep')
    # Executing the test
    server_context = ServerContext(build_context, '1.1.1.1')
    server_context.proxy = mocker.MagicMock()
    return server_context


# demisto_sdk/commands/test_content/tests/test_context_test.py

import ast
import logging
from functools import partial

import pytest

from demisto_sdk.commands.common.constants import PB_Status
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.test_content.Docker import Docker
from demisto_sdk.commands.test_content.TestContentClasses import (Integration, TestConfiguration, TestContext,
                                                                  TestPlaybook)
from demisto_sdk.commands.test_content.tests.build_context_test import (generate_content_conf_json,
                                                                        generate_integration_configuration,
                                                                        generate_secret_conf_json,
                                                                        generate_test_configuration,
                                                                        get_mocked_build_context)
from demisto_sdk.commands.test_content.tests.DemistoClientMock import DemistoClientMock
from demisto_sdk.commands.test_content.tests.server_context_test import generate_mocked_server_context

json = JSON_Handler()


def test_is_runnable_on_this_instance(mocker):
    """
    Given:
        - A test configuration configured to run only on instances that uses docker as container engine
    When:
        - The method _is_runnable_on_current_server_instance is invoked from the TestContext class
    Then:
        - Ensure that it returns False when the test is running on REHL instance that uses podman
        - Ensure that it returns True when the test is running on a regular Linux instance that uses docker
    """
    test_playbook_configuration = TestConfiguration(
        generate_test_configuration(playbook_id='playbook_runnable_only_on_docker',
                                    runnable_on_docker_only=True), default_test_timeout=30)
    test_context_builder = partial(TestContext,
                                   build_context=mocker.MagicMock(),
                                   playbook=TestPlaybook(mocker.MagicMock(),
                                                         test_playbook_configuration),
                                   client=mocker.MagicMock())

    test_context = test_context_builder(server_context=mocker.MagicMock(is_instance_using_docker=False))
    assert not test_context._is_runnable_on_current_server_instance()
    test_context = test_context_builder(server_context=mocker.MagicMock(is_instance_using_docker=True))
    assert test_context._is_runnable_on_current_server_instance()


# Retries mechanism UT

class RunIncidentTestMock:

    def __init__(self, response_list):
        self.call_count = 0
        self.count_response_list = response_list

    def run_incident_test(self):
        res = self.count_response_list[self.call_count]
        self.call_count += 1
        return res

# Unmockable


def init_server_context(mocker, tmp_path, mockable=False):
    playbook_type = 'mocked_playbook' if mockable else 'unmocked_playbook'
    playbook_id_type = 'mocked_playbook' if mockable else 'unmocked_playbook'
    integrations_type = 'mocked_integration' if mockable else 'unmocked_integration'
    mock_func = '_execute_unmockable_tests' if mockable else '_execute_mockable_tests'
    unmockable_integration = {integrations_type: 'reason'} if not mockable else {}

    filtered_tests = [playbook_type]
    tests = [generate_test_configuration(playbook_id=playbook_id_type,
                                         integrations=[integrations_type])]
    integrations_configurations = [generate_integration_configuration(integrations_type)]
    secret_test_conf = generate_secret_conf_json(integrations_configurations)
    content_conf_json = generate_content_conf_json(tests=tests, unmockable_integrations=unmockable_integration)
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             secret_conf_json=secret_test_conf,
                                             content_conf_json=content_conf_json,
                                             filtered_tests_content=filtered_tests)
    mocked_demisto_client = DemistoClientMock(integrations=[integrations_type])
    server_context = generate_mocked_server_context(build_context, mocked_demisto_client, mocker)
    mocker.patch.object(server_context, mock_func, return_value=None)

    return build_context, server_context


def test_unmockable_playbook_passes_on_first_run(mocker, tmp_path):
    """
    Given:
        - An unmockable test
    When:
        - The unmockable test passes on the first run
    Then:
        - Ensure that it exists in the succeeded_playbooks set
        - Ensure that it does not exist in the failed_playbook set
    """
    execution_results = [PB_Status.COMPLETED]
    build_context, server_context = init_server_context(mocker, tmp_path)
    incident_test_mock = RunIncidentTestMock(execution_results)
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()
    assert incident_test_mock.call_count == 1
    assert not build_context.tests_data_keeper.failed_playbooks  # empty set
    assert 'unmocked_playbook' in build_context.tests_data_keeper.succeeded_playbooks


def test_unmockable_playbook_passes_most_of_the_time(mocker, tmp_path):
    """
    Given:
        - An unmockable test
    When:
        - The unmockable test passes on the second and third run
    Then:
        - Ensure that it exists in the succeeded_playbooks set
        - Ensure that it does not exist in the failed_playbook set
    """
    execution_results = [PB_Status.FAILED, PB_Status.COMPLETED, PB_Status.COMPLETED]
    build_context, server_context = init_server_context(mocker, tmp_path)
    incident_test_mock = RunIncidentTestMock(execution_results)
    logs = build_context.logging_module = mocker.MagicMock()
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()

    assert incident_test_mock.call_count == 3
    assert not build_context.tests_data_keeper.failed_playbooks
    assert 'unmocked_playbook' in build_context.tests_data_keeper.succeeded_playbooks
    assert any('Test-Playbook was executed 3 times, and passed 2 times. Adding to succeeded playbooks.' in log_item[0][0] for
               log_item in logs.info.call_args_list)


def test_unmockable_playbook_fails_every_time(mocker, tmp_path):
    """
    Given:
        - An unmockable test
    When:
        - The unmockable test fails in all of the runs
    Then:
        - Ensure that it does not exist in the succeeded_playbooks set
        - Ensure that it exists in the failed_playbook set
    """
    execution_results = [PB_Status.FAILED, PB_Status.FAILED, PB_Status.FAILED]
    build_context, server_context = init_server_context(mocker, tmp_path)
    incident_test_mock = RunIncidentTestMock(execution_results)
    logs = build_context.logging_module = mocker.MagicMock()
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()

    assert incident_test_mock.call_count == 3
    assert 'unmocked_playbook (Mock Disabled)' in build_context.tests_data_keeper.failed_playbooks
    assert not build_context.tests_data_keeper.succeeded_playbooks
    assert any('Test-Playbook was executed 3 times, and passed only 0 times. Adding to failed playbooks.' in log_item[0][0] for
               log_item in logs.info.call_args_list)


def test_unmockable_playbook_fails_most_of_the_times(mocker, tmp_path):
    """
    Given:
        - An unmockable test
    When:
        - The unmockable test fails in most of the runs
    Then:
        - Ensure that it does not exist in the succeeded_playbooks set
        - Ensure that it exists in the failed_playbook set
    """
    execution_results = [PB_Status.FAILED, PB_Status.COMPLETED, PB_Status.FAILED]
    build_context, server_context = init_server_context(mocker, tmp_path)
    incident_test_mock = RunIncidentTestMock(execution_results)
    logs = build_context.logging_module = mocker.MagicMock()
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()

    assert incident_test_mock.call_count == 3
    assert 'unmocked_playbook (Mock Disabled)' in build_context.tests_data_keeper.failed_playbooks
    assert not build_context.tests_data_keeper.succeeded_playbooks
    assert any('Test-Playbook was executed 3 times, and passed only 1 times. Adding to failed playbooks.' in [log_item][0][0] for
               log_item in logs.info.call_args_list)


# Mockable

def test_mockable_playbook_first_playback_passes(mocker, tmp_path):
    """
    Given:
        - A mockable test
    When:
        - The mockable test passes on the first playback run
    Then:
        - Ensure that it exists in the succeeded_playbooks set
        - Ensure that it does not exist in the failed_playbook set
    """
    execution_results = [PB_Status.COMPLETED]
    build_context, server_context = init_server_context(mocker, tmp_path, mockable=True)
    incident_test_mock = RunIncidentTestMock(execution_results)
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()

    assert incident_test_mock.call_count == 1
    assert 'mocked_playbook' in build_context.tests_data_keeper.succeeded_playbooks
    assert not build_context.tests_data_keeper.failed_playbooks


def test_mockable_playbook_second_playback_passes(mocker, tmp_path):
    """
    Given:
        - A mockable test
    When:
        - The mockable test that fails on the first playback but then passes the recording and second playback
    Then:
        - Ensure that it exists in the succeeded_playbooks set
        - Ensure that it does not exist in the failed_playbook set
    """
    execution_results = [PB_Status.FAILED, PB_Status.COMPLETED, PB_Status.COMPLETED]
    build_context, server_context = init_server_context(mocker, tmp_path, mockable=True)
    incident_test_mock = RunIncidentTestMock(execution_results)
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()

    assert incident_test_mock.call_count == 3
    assert 'mocked_playbook' in build_context.tests_data_keeper.succeeded_playbooks
    assert not build_context.tests_data_keeper.failed_playbooks


def test_mockable_playbook_recording_passes_most_of_the_time_Playback_pass(mocker, tmp_path):
    """
    Given:
        - A mockable test
    When:
        - The mockable test that fails on the first playback, then passes most of the recordings and second playback.
    Then:
        - Ensure that it exists in the succeeded_playbooks set
        - Ensure that it does not exist in the failed_playbook set
    """
    execution_results = [PB_Status.FAILED, PB_Status.FAILED, PB_Status.COMPLETED, PB_Status.COMPLETED, PB_Status.COMPLETED]
    build_context, server_context = init_server_context(mocker, tmp_path, mockable=True)
    incident_test_mock = RunIncidentTestMock(execution_results)
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()

    data_keeper = build_context.tests_data_keeper
    assert incident_test_mock.call_count == 5
    assert 'mocked_playbook' in data_keeper.succeeded_playbooks
    assert not data_keeper.failed_playbooks


def test_mockable_playbook_recording_passes_most_of_the_time_playback_fails(mocker, tmp_path):
    """
    Given:
        - A mockable test
    When:
        - The mockable test that fails on the first playback, then passes most of the recordings but the second playback fails.
    Then:
        - Ensure that it does not exist in the succeeded_playbooks set
        - Ensure that it exists in the failed_playbook set
    """
    execution_results = [PB_Status.FAILED, PB_Status.FAILED, PB_Status.COMPLETED, PB_Status.COMPLETED, PB_Status.FAILED]
    build_context, server_context = init_server_context(mocker, tmp_path, mockable=True)
    incident_test_mock = RunIncidentTestMock(execution_results)
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()

    data_keeper = build_context.tests_data_keeper
    assert incident_test_mock.call_count == 5
    assert not data_keeper.succeeded_playbooks
    assert 'mocked_playbook (Second Playback)' in data_keeper.failed_playbooks
    assert data_keeper.playbook_report['mocked_playbook'][0]['number_of_executions'] == 3
    assert data_keeper.playbook_report['mocked_playbook'][0]['number_of_successful_runs'] == 2
    assert data_keeper.playbook_report['mocked_playbook'][0]['failed_stage'] == 'Second playback'


def test_mockable_playbook_recording_fails_most_of_the_time(mocker, tmp_path):
    """
    Given:
        - A mockable test
    When:
        - The mockable test that fails on the first playback, then fails most of the recordings.
    Then:
        - Ensure that it does not exist in the succeeded_playbooks set
        - Ensure that it exists in the failed_playbook set
        - no second playback is needed
    """
    execution_results = [PB_Status.FAILED, PB_Status.FAILED, PB_Status.COMPLETED, PB_Status.FAILED]
    incident_test_mock = RunIncidentTestMock(execution_results)
    build_context, server_context = init_server_context(mocker, tmp_path, mockable=True)
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()

    data_keeper = build_context.tests_data_keeper
    assert incident_test_mock.call_count == 4
    assert not data_keeper.succeeded_playbooks
    assert 'mocked_playbook' in data_keeper.failed_playbooks
    assert data_keeper.playbook_report['mocked_playbook'][0]['number_of_executions'] == 3
    assert data_keeper.playbook_report['mocked_playbook'][0]['number_of_successful_runs'] == 1
    assert data_keeper.playbook_report['mocked_playbook'][0]['failed_stage'] == 'Execution'


def test_mockable_playbook_recording_fails_every_time(mocker, tmp_path):
    """
    Given:
        - A mockable test
    When:
        - The mockable test that fails on the first playback, then fails on every record run.
    Then:
        - Ensure that it does not exist in the succeeded_playbooks set
        - Ensure that it exists in the failed_playbook set
        - no second playback is needed
    """
    execution_results = [PB_Status.FAILED, PB_Status.FAILED, PB_Status.FAILED, PB_Status.FAILED]
    incident_test_mock = RunIncidentTestMock(execution_results)
    build_context, server_context = init_server_context(mocker, tmp_path, mockable=True)
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()

    data_keeper = build_context.tests_data_keeper
    assert incident_test_mock.call_count == 4
    assert not data_keeper.succeeded_playbooks
    assert 'mocked_playbook' in data_keeper.failed_playbooks
    assert data_keeper.playbook_report['mocked_playbook'][0]['number_of_executions'] == 3
    assert data_keeper.playbook_report['mocked_playbook'][0]['number_of_successful_runs'] == 0
    assert data_keeper.playbook_report['mocked_playbook'][0]['failed_stage'] == 'Execution'


def test_mockable_playbook_second_playback_fails(mocker, tmp_path):
    """
    Given:
        - A mockable test
    When:
        - The mockable test fails on the second playback
    Then:
        - Ensure that it exists in the failed_playbooks set
        - Ensure that it does not exist in the succeeded_playbooks list
    """

    execution_results = [PB_Status.FAILED, PB_Status.COMPLETED, PB_Status.FAILED]
    incident_test_mock = RunIncidentTestMock(execution_results)
    build_context, server_context = init_server_context(mocker, tmp_path, mockable=True)
    mocker.patch('demisto_sdk.commands.test_content.TestContentClasses.TestContext._run_incident_test',
                 incident_test_mock.run_incident_test)
    server_context.execute_tests()

    data_keeper = build_context.tests_data_keeper
    assert incident_test_mock.call_count == 3
    assert not data_keeper.succeeded_playbooks
    assert 'mocked_playbook (Second Playback)' in data_keeper.failed_playbooks
    assert data_keeper.playbook_report['mocked_playbook'][0]['number_of_executions'] == 1
    assert data_keeper.playbook_report['mocked_playbook'][0]['number_of_successful_runs'] == 1
    assert data_keeper.playbook_report['mocked_playbook'][0]['failed_stage'] == 'Second playback'


def test_docker_thresholds_for_non_pwsh_integrations(mocker):
    """
    Given:
        - A test context with a playbook that uses a python integration
    When:
        - Running 'get_threshold_values' method
    Then:
        - Ensure that the memory threshold is the default python memory threshold value
        - Ensure that the pis threshold is the default python pid threshold value
    """
    test_playbook_configuration = TestConfiguration(
        generate_test_configuration(playbook_id='playbook_runnable_only_on_docker',
                                    integrations=['integration']), default_test_timeout=30)
    playbook_instance = TestPlaybook(mocker.MagicMock(), test_playbook_configuration)
    playbook_instance.integrations[0].integration_type = Docker.PYTHON_INTEGRATION_TYPE
    test_context = TestContext(build_context=mocker.MagicMock(),
                               playbook=playbook_instance,
                               client=mocker.MagicMock(),
                               server_context=mocker.MagicMock())
    memory_threshold, pid_threshold = test_context.get_threshold_values()
    assert memory_threshold == Docker.DEFAULT_CONTAINER_MEMORY_USAGE
    assert pid_threshold == Docker.DEFAULT_CONTAINER_PIDS_USAGE


def test_docker_thresholds_for_pwsh_integrations(mocker):
    """
    Given:
        - A test context with a playbook that uses a powershell integration
    When:
        - Running 'get_threshold_values' method
    Then:
        - Ensure that the memory threshold is the default powershell memory threshold value
        - Ensure that the pis threshold is the default powershell pid threshold value
    """
    test_playbook_configuration = TestConfiguration(
        generate_test_configuration(playbook_id='playbook_runnable_only_on_docker',
                                    integrations=['integration']), default_test_timeout=30)
    playbook_instance = TestPlaybook(mocker.MagicMock(), test_playbook_configuration)
    playbook_instance.integrations[0].integration_type = Docker.POWERSHELL_INTEGRATION_TYPE
    test_context = TestContext(build_context=mocker.MagicMock(),
                               playbook=playbook_instance,
                               client=mocker.MagicMock(),
                               server_context=mocker.MagicMock())
    memory_threshold, pid_threshold = test_context.get_threshold_values()
    assert memory_threshold == Docker.DEFAULT_PWSH_CONTAINER_MEMORY_USAGE
    assert pid_threshold == Docker.DEFAULT_PWSH_CONTAINER_PIDS_USAGE


class TestPrintContextToLog:

    @staticmethod
    def create_playbook_instance(mocker):
        test_playbook_configuration = TestConfiguration(generate_test_configuration(
            playbook_id='playbook_with_context',
            integrations=['integration']
        ),
            default_test_timeout=30)
        pb_instance = TestPlaybook(mocker.MagicMock(), test_playbook_configuration)
        pb_instance.build_context.logging_module = mocker.MagicMock()
        return pb_instance

    def test_print_context_to_log__success(self, mocker):
        """
        Given:
            - A test context with a context_print_dt value
            - The context result is a string "{'foo': 'goo'}"
        When:
            - Running 'print_context_to_log' method
        Then:
            - Ensure that a proper json result is being printed to the context
        """
        dt_result = "{'foo': 'goo'}"
        expected_result = json.dumps(ast.literal_eval(dt_result), indent=4)
        playbook_instance = self.create_playbook_instance(mocker)
        client = mocker.MagicMock()
        client.api_client.call_api.return_value = (dt_result, 200)
        playbook_instance.print_context_to_log(client, incident_id='1')
        assert playbook_instance.build_context.logging_module.info.call_args[0][0] == expected_result

    def test_print_context_to_log__empty(self, mocker):
        """
        Given:
            - A test context with a context_print_dt value
            - The context result is empty
        When:
            - Running 'print_context_to_log' method
        Then:
            - Ensure that an empty result is being printed to the context
        """
        expected_dt = '{}'
        playbook_instance = self.create_playbook_instance(mocker)
        client = mocker.MagicMock()
        client.api_client.call_api.return_value = (expected_dt, 200)
        playbook_instance.print_context_to_log(client, incident_id='1')
        assert playbook_instance.build_context.logging_module.info.call_args[0][0] == expected_dt

    def test_print_context_to_log__none(self, mocker):
        """
        Given:
            - A test context with a context_print_dt value
            - The context result is None
        When:
            - Running 'print_context_to_log' method
        Then:
            - Ensure that an exception is raised and handled via logging error
        """
        expected_dt = None
        expected_error = 'unable to parse result for result with value: None'
        playbook_instance = self.create_playbook_instance(mocker)
        client = mocker.MagicMock()
        client.api_client.call_api.return_value = (expected_dt, 200)
        playbook_instance.print_context_to_log(client, incident_id='1')
        assert playbook_instance.build_context.logging_module.error.call_args[0][0] == expected_error

    def test_print_context_to_log__error(self, mocker):
        """
        Given:
            - A test context with a context_print_dt value
            - The context return with an HTTP error code
        When:
            - Running 'print_context_to_log' method
        Then:
            - Ensure that an exception is raised and handled via logging error messages
        """
        expected_dt = 'No Permission'
        expected_first_error = 'incident context fetch failed with Status code 403'
        expected_second_error = f"('{expected_dt}', 403)"
        playbook_instance = self.create_playbook_instance(mocker)
        client = mocker.MagicMock()
        client.api_client.call_api.return_value = (expected_dt, 403)
        playbook_instance.print_context_to_log(client, incident_id='1')
        assert playbook_instance.build_context.logging_module.error.call_args_list[0][0][0] == expected_first_error
        assert playbook_instance.build_context.logging_module.error.call_args_list[1][0][0] == expected_second_error


def test_replacing_placeholders(mocker, tmp_path):
    """
    Given:
        - Integration with placeholders, different servers
    When:
        - Calling _set_integration_params during creating integrations configurations
    Then:
        - Ensure that replacing placeholders happens not in place,
        and next integration with same build_context, will able to replace '%%SERVER_HOST%%' placeholder.
    """
    # Setting up the build context
    filtered_tests = ['playbook_integration',
                      'playbook_second_integration']
    # Setting up the content conf.json
    tests = [generate_test_configuration(playbook_id='playbook_integration',
                                         integrations=['integration_with_placeholders']),
             generate_test_configuration(playbook_id='playbook_second_integration',
                                         integrations=['integration_with_placeholders'])
             ]
    content_conf_json = generate_content_conf_json(tests=tests,
                                                   unmockable_integrations={'FirstIntegration': 'reason'},
                                                   skipped_tests={})
    # Setting up the content-test-conf conf.json
    integration_names = ['integration_with_placeholders']
    integrations_configurations = [generate_integration_configuration(name=integration_name,
                                                                      params={'url': '%%SERVER_HOST%%/server'})
                                   for integration_name in integration_names]
    secret_test_conf = generate_secret_conf_json(integrations_configurations)

    # Setting up the build_context instance
    build_context = get_mocked_build_context(mocker,
                                             tmp_path,
                                             content_conf_json=content_conf_json,
                                             secret_conf_json=secret_test_conf,
                                             filtered_tests_content=filtered_tests)

    integration = Integration(build_context, 'integration_with_placeholders', ['instance'])
    integration._set_integration_params(server_url='1.1.1.1', playbook_id='playbook_integration', is_mockable=False)
    integration = Integration(build_context, 'integration_with_placeholders', ['instance'])
    integration._set_integration_params(server_url='1.2.3.4', playbook_id='playbook_integration', is_mockable=False)
    assert '%%SERVER_HOST%%' in build_context.secret_conf.integrations[0].params.get('url')


CASES = [
    (  # case one input is found
        {'id': 'pb_test',
         'inputs': [{'key': 'Endpoint_hostname', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The hostname of the endpoint to isolate.', 'playbookInputQuery': None},
                    {'key': 'ManualHunting.DetectedHosts', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'Hosts that were detected as infected during the manual hunting.',
                     'playbookInputQuery': None},
                    {'key': 'Endpoint_ip', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The IP of the endpoint to isolate.', 'playbookInputQuery': None},
                    {'key': 'Endpoint_id', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The ID of the endpoint to isolate.', 'playbookInputQuery': None}],
         },
        {
            "playbookID": "pb_test",
            "input_parameters": {
                "Endpoint_hostname": {
                    "simple": "test"
                },
            }
        },
        [{'key': 'Endpoint_hostname', 'value': {'simple': 'test', 'complex': None}, 'required': False,
          'description': 'The hostname of the endpoint to isolate.', 'playbookInputQuery': None},
         {'key': 'ManualHunting.DetectedHosts', 'value': {'simple': '', 'complex': None}, 'required': False,
          'description': 'Hosts that were detected as infected during the manual hunting.',
          'playbookInputQuery': None},
         {'key': 'Endpoint_ip', 'value': {'simple': '', 'complex': None},
          'required': False, 'description': 'The IP of the endpoint to isolate.',
          'playbookInputQuery': None},
         {'key': 'Endpoint_id',
          'value': {'simple': '', 'complex': None},
          'required': False,
          'description': 'The ID of the endpoint to isolate.',
          'playbookInputQuery': None}]

    ),
]


@pytest.mark.parametrize('current, new_configuration, expected', CASES)
def test_replacing_pb_inputs(mocker, current, new_configuration, expected):
    """

    Given: Configuration with inputs to change
            Found configuration but running on older versions of server.
    When: Using the external configuration in conf.json in order to change playbook inputs on testing flow in build
    Then: Make sure the server request are correct

    """
    from demisto_client.demisto_api import DefaultApi

    from demisto_sdk.commands.test_content.TestContentClasses import (demisto_client,
                                                                      replace_external_playbook_configuration)

    class clientMock(DefaultApi):
        def generic_request(self, path, method, body=None, **kwargs):
            if path == '/about' and method == 'GET':
                return ("{'demistoVersion': '6.5.0'}", None, None)

    def generic_request_func(self, path, method, body=None, **kwargs):
        if path == '/playbook/inputs/pb_test' and method == 'POST':
            assert body == expected
        elif path == '/playbook/pb_test' and method == 'GET':
            return current, None, None
        else:
            assert False  # Unexpected path

    mocker.patch.object(demisto_client, 'generic_request_func', side_effect=generic_request_func)

    replace_external_playbook_configuration(clientMock(), new_configuration)


BAD_CASES = [
    (  # case no configuration found
        {'id': 'pb_test',
         'inputs': [{'key': 'Endpoint_hostname', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The hostname of the endpoint to isolate.', 'playbookInputQuery': None},
                    {'key': 'ManualHunting.DetectedHosts', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'Hosts that were detected as infected during the manual hunting.',
                     'playbookInputQuery': None},
                    {'key': 'Endpoint_ip', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The IP of the endpoint to isolate.', 'playbookInputQuery': None},
                    {'key': 'Endpoint_id', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The ID of the endpoint to isolate.', 'playbookInputQuery': None}],
         },
        {},
        '6.5.0',
        'External Playbook Configuration not provided, skipping re-configuration.'
    ),
    (  # case configuration found in older version
        {'id': 'pb_test',
         'inputs': [{'key': 'Endpoint_hostname', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The hostname of the endpoint to isolate.', 'playbookInputQuery': None},
                    {'key': 'ManualHunting.DetectedHosts', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'Hosts that were detected as infected during the manual hunting.',
                     'playbookInputQuery': None},
                    {'key': 'Endpoint_ip', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The IP of the endpoint to isolate.', 'playbookInputQuery': None},
                    {'key': 'Endpoint_id', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The ID of the endpoint to isolate.', 'playbookInputQuery': None}],
         },
        {
            "playbookID": "pb_test",
            "input_parameters": {
                "Endpoint_hostname": {
                    "simple": "test"
                },
            }
        },
        '6.0.0',
        'External Playbook not supported in versions previous to 6.2.0, skipping re-configuration.'
    ),
]


@pytest.mark.parametrize('current, new_configuration, version, expected_error', BAD_CASES)
def test_replacing_pb_inputs_fails_with_build_pass(mocker, current, new_configuration, version, expected_error):
    """

    Given: Missing configuration
            Found configuration but running on older versions of server.
    When: Using the external configuration in conf.json in order to change playbook inputs on testing flow in build
    Then: Make sure the build pass without errors

    """
    from demisto_client.demisto_api import DefaultApi

    from demisto_sdk.commands.test_content.TestContentClasses import (demisto_client,
                                                                      replace_external_playbook_configuration)

    class clientMock(DefaultApi):
        def generic_request(self, path, method, body=None, **kwargs):
            if path == '/about' and method == 'GET':
                return str({'demistoVersion': version}), None, None

    class LoggerMock(logging.Logger):
        def info(self, text, **kwargs):
            if text not in ['External Playbook in use, starting re-configuration.', 'Saved current configuration.']:
                assert text == expected_error

    def generic_request_func(self, path, method, body=None, **kwargs):
        if path == '/playbook/inputs/pb_test' and method == 'POST':
            return
        elif path == '/playbook/pb_test' and method == 'GET':
            return current, None, None
        else:
            assert False  # Unexpected path

    mocker.patch.object(demisto_client, 'generic_request_func', side_effect=generic_request_func)

    replace_external_playbook_configuration(clientMock(), new_configuration, LoggerMock('test logger'))


BAD_CASES_BUILD_FAIL = [
    (  # case configuration not found.
        {"id": "createPlaybookErr", "status": 400, "title": "Could not create playbook",
         "detail": "Could not create playbook", "error": "Item not found (8)", "encrypted": None, "multires": None},
        {
            "playbookID": "pb_test",
            "input_parameters": {
                "Endpoint_hostname": {
                    "simple": "test"
                },
            }
        },
        '6.2.0',
        'External Playbook was not found or has no inputs.'
    ),
    (  # case configuration was found but wrong input key given.
        {'id': 'pb_test',
         'inputs': [{'key': 'Endpoint_hostname', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The hostname of the endpoint to isolate.', 'playbookInputQuery': None},
                    {'key': 'ManualHunting.DetectedHosts', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'Hosts that were detected as infected during the manual hunting.',
                     'playbookInputQuery': None},
                    {'key': 'Endpoint_ip', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The IP of the endpoint to isolate.', 'playbookInputQuery': None},
                    {'key': 'Endpoint_id', 'value': {'simple': '', 'complex': None}, 'required': False,
                     'description': 'The ID of the endpoint to isolate.', 'playbookInputQuery': None}],
         },
        {
            "playbookID": "pb_test",
            "input_parameters": {
                "Endpoint_hostnames": {
                    "simple": "test"
                },
            }
        },
        '6.2.0',
        'Some input keys was not found in playbook: Endpoint_hostnames.'
    ),

]


@pytest.mark.parametrize('current, new_configuration, version, expected_error', BAD_CASES_BUILD_FAIL)
def test_replacing_pb_inputs_fails_with_build_fail(mocker, current, new_configuration, version, expected_error):
    """

    Given: Bad configuration - external playbooks is wrong
            Bad configuration - wrong input names
    When: Using the external configuration in conf.json in order to change playbook inputs on testing flow in build
    Then: Make sure the error contains the relevant issue.

    """
    from demisto_client.demisto_api import DefaultApi

    from demisto_sdk.commands.test_content.TestContentClasses import (demisto_client,
                                                                      replace_external_playbook_configuration)

    class clientMock(DefaultApi):
        def generic_request(self, path, method, body=None, **kwargs):
            if path == '/about' and method == 'GET':
                return str({'demistoVersion': version}), None, None

    def generic_request_func(self, path, method, body=None, **kwargs):
        if path == '/playbook/inputs/pb_test' and method == 'POST':
            return
        elif path == '/playbook/pb_test' and method == 'GET':
            return current, None, None
        else:
            assert False  # Unexpected path

    mocker.patch.object(demisto_client, 'generic_request_func', side_effect=generic_request_func)

    with pytest.raises(Exception) as e:
        replace_external_playbook_configuration(clientMock(), new_configuration)
    assert expected_error in str(e)


# demisto_sdk/commands/test_content/tests/test_tools.py

from subprocess import CalledProcessError

from demisto_sdk.commands.test_content.tools import is_redhat_instance


def raise_exception():
    raise CalledProcessError(1, 'ls -l /home/ec2-user/rhel_ami'.split())


class CheckOutputMock:
    stdout = 'output'


def test_is_redhat_instance_positive(mocker):
    mocker.patch('subprocess.run', return_value=CheckOutputMock)
    assert is_redhat_instance('instance_ip')


def test_is_redhat_instance_negative(mocker):
    mocker.patch('subprocess.check_output', side_effect=raise_exception)
    assert not is_redhat_instance('instance_ip')


# demisto_sdk/commands/test_content/tests/testplaybook_test.py

from unittest.mock import ANY

import demisto_client
import pytest

from demisto_sdk.commands.test_content.TestContentClasses import (Integration, IntegrationConfiguration,
                                                                  TestConfiguration, TestPlaybook)
from demisto_sdk.commands.test_content.tests.build_context_test import create_xsiam_build, get_mocked_build_context
from demisto_sdk.commands.test_content.tests.DemistoClientMock import DemistoClientMock
from demisto_sdk.commands.test_content.tests.server_context_test import generate_mocked_server_context


def test_set_prev_server_keys(mocker, tmp_path):
    """
    Given:
        - Test playbook and integration to run
        - Integration config with server keys (python.pass.extra.keys)

    When:
        - Cleaning server config when disabling the integration instance

    Then:
        - Ensure _clear_server_keys() return True to verify it updated the server config
        - Verify update_server_conf_func called with prev server conf
    """
    prev_server_conf = {'prev': 'conf'}
    mocked_demisto_client = DemistoClientMock()
    build_context = get_mocked_build_context(mocker, tmp_path)
    server_context = generate_mocked_server_context(build_context, mocked_demisto_client, mocker)
    update_server_conf_func = mocker.patch(
        'demisto_sdk.commands.test_content.TestContentClasses.update_server_configuration',
        return_value=(None, None, prev_server_conf)
    )
    test_playbook_configuration = TestConfiguration({}, 0)

    test_playbook = TestPlaybook(build_context, test_playbook_configuration)
    integration = Integration(build_context, 'integration_with_server_keys', ['instance'])
    integration.configuration = IntegrationConfiguration({
        'params': {
            'server_keys': {
                'python.pass.extra.keys': '--hostname=HOSTNAME##-v=test1234'
            }
        }
    })

    integration._set_server_keys(mocked_demisto_client, server_context)
    test_playbook.integrations = [integration]

    assert test_playbook._set_prev_server_keys(mocked_demisto_client, server_context)

    update_server_conf_func.assert_called_with(
        client=mocked_demisto_client,
        server_configuration=prev_server_conf,
        error_msg='Failed to set server keys',
        logging_manager=ANY
    )


@pytest.mark.parametrize('expected_res,client_res', [(True, (None, 200, None)),
                                                     (False, (None, 400, None))])
def test_close_incident(mocker, tmp_path, expected_res, client_res):
    """
    Given:
        - Test playbook to run on XSIAM build

    When:
        - Cleaning server config when closing the incidents

    Then:
        - Ensure incident closed if status code is 200
        - incident not closed but there is no exception, when code is 400
    """
    mocked_demisto_client = DemistoClientMock()
    build_context = create_xsiam_build(mocker, tmp_path)
    test_playbook_configuration = TestConfiguration({}, 0)
    mocker.patch.object(demisto_client, 'generic_request_func', return_value=client_res)

    test_playbook = TestPlaybook(build_context, test_playbook_configuration)
    res = test_playbook.close_incident(client=mocked_demisto_client, incident_id='1')
    assert res == expected_res


# demisto_sdk/commands/test_content/tests/timestamp_replacer_test.py

import os
import sys
from unittest.mock import MagicMock, mock_open

import mitmproxy
import pytest
from mitmproxy.http import Headers, HTTPFlow, Request

from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.test_content.timestamp_replacer import TimestampReplacer

json = JSON_Handler()


@pytest.fixture()
def flow():
    request = Request(
        host=b'localhost',
        path=b'/test/',
        http_version=b'1.1',
        port=1234,
        method=b'',
        scheme=b'',
        headers=Headers([(b"Host", b"example.com")]),
        content=None,
        timestamp_start=111.1,
        timestamp_end=111.2,
        authority=b'',
        trailers=''
    )
    flow = HTTPFlow(client_conn=MagicMock(),
                    server_conn=MagicMock())
    flow.request = request
    return flow


@pytest.fixture
def tz(request):
    original_tz = os.getenv('TZ')
    os.environ['TZ'] = 'UTC'

    def teardown():
        if original_tz:
            os.environ['TZ'] = original_tz
    request.addfinalizer(teardown)
    return request


TIMESTAMP_FORMATS = [
    '2021-01-11T13:18:12+00:00',
    '2021-01-14 17:44:00.571043',
    '1610639147',
    'Thu, 14 Jan 2021 15:45:47',
    'Thursday, 14-Jan-21 15:45:47 UTC',
    '2021-01-14T15:45:47+00:00'
]


@pytest.mark.usefixtures('tz')
class TestTimeStampReplacer:
    DEFAULT_OPTIONS_MAPPING = {
        'detect_timestamps': False,
        'keys_filepath': 'problematic_keys.json',
        'script_mode': 'playback',
        'debug': False
    }

    @classmethod
    def setup_class(cls):
        mitmproxy.ctx.options = MagicMock(**cls.DEFAULT_OPTIONS_MAPPING)

    def test_loader_defaults(self, mocker):
        """
        Given:
            - A timestamp replacer instance
        When:
            - Loading it's options
        Then:
            - Ensure that a call with the expected defaults is called
        """
        master_mock = mocker.MagicMock()
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.load(master_mock)
        python_version = sys.version_info
        if python_version.major == 3 and python_version.minor == 7:
            pytest.skip('The current mock syntax is supported only in python 3.8+')
        assert master_mock.add_option.call_count == 4
        for call in master_mock.add_option.mock_calls:
            assert self.DEFAULT_OPTIONS_MAPPING[call.kwargs['name']] == call.kwargs['default']

    def test_running(self, mocker):
        """
        Given:
            - A timestamp replacer instance
        When:
            - executing the 'running' method
        Then:
            - Ensure all problematic keys are loaded
        """
        problematic_keys = {'server_replay_ignore_params': 'query_key1 query_key2',
                            'server_replay_ignore_payload_params': 'payload_key1 payload_key2',
                            'keys_to_replace': 'key_to_replace1 key_to_replace2'}
        mocker.patch('os.path.exists', return_value=True)
        mocker.patch('builtins.open', mock_open(read_data=json.dumps(problematic_keys)))
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.running()
        assert time_stamp_replacer.query_keys == {'query_key2', 'query_key1'}
        assert time_stamp_replacer.form_keys == {'payload_key2', 'payload_key1'}
        assert time_stamp_replacer.json_keys == {'key_to_replace2', 'key_to_replace1'}

    @pytest.mark.parametrize('time', TIMESTAMP_FORMATS)
    def test_finding_problematic_keys_in_url_query(self, mocker, flow, time):
        """
        Given:
            - A timestamp replacer instance
        When:
            - searching a request for problematic keys that has timestamp values in the query params
        Then:
            - Ensure the problematic key is loaded into query_keys
            - Ensure the valid key is not loaded into query_keys
        """
        mocker.patch('builtins.open', mock_open())
        mitmproxy.ctx.options.detect_timestamps = True
        mitmproxy.ctx.options.script_mode = 'record'
        flow.request._set_query([('key1', 'value1'), ('timestamp_key', time)])
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.request(flow)
        assert 'timestamp_key' in time_stamp_replacer.query_keys
        assert 'key1' not in time_stamp_replacer.query_keys

    @pytest.mark.parametrize('time', TIMESTAMP_FORMATS)
    def test_finding_problematic_keys_in_form_keys(self, mocker, flow, time):
        """
        Given:
            - A timestamp replacer instance
        When:
            - searching a request for problematic keys that has timestamp values
        Then:
            - Ensure the problematic key is loaded into form_keys
            - Ensure the valid key is not loaded into form_keys
        """
        mocker.patch('builtins.open', mock_open())
        mitmproxy.ctx.options.detect_timestamps = True
        mitmproxy.ctx.options.script_mode = 'record'
        flow.request._set_urlencoded_form([('key1', 'value1'), ('timestamp_key', time)])
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.request(flow)
        assert 'timestamp_key' in time_stamp_replacer.form_keys
        assert 'key1' not in time_stamp_replacer.form_keys

    @pytest.mark.parametrize('time', TIMESTAMP_FORMATS)
    def test_finding_problematic_keys_in_json_keys(self, mocker, flow, time):
        """
        Given:
            - A timestamp replacer instance
        When:
            - searching a request for problematic keys that has timestamp values
        Then:
            - Ensure the problematic key is loaded into json_keys
            - Ensure the problematic key is loaded into json_keys even when it's nested
            - Ensure the valid key is not loaded into json_keys
        """
        mocker.patch('builtins.open', mock_open())
        mitmproxy.ctx.options.detect_timestamps = True
        mitmproxy.ctx.options.script_mode = 'record'
        flow.request.method = 'POST'
        flow.request.set_content(json.dumps({'key1': 'value1',
                                             'timestamp_key': time,
                                             'dict1': {'list': ['test', time]}}).encode())
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.request(flow)
        assert 'timestamp_key' in time_stamp_replacer.json_keys
        assert 'dict1.list.1' in time_stamp_replacer.json_keys
        assert 'key1' not in time_stamp_replacer.json_keys

    def test_json_body_parsing(self, mocker, flow):
        """
        Given:
            - A timestamp replacer instance
        When:
            - Trying to parse json body
        Then:
            - Ensure the body is parsed when is't json body
            - Ensure the body is parsed when it's json body with single quotes
        """
        mocker.patch('builtins.open', mock_open())
        mitmproxy.ctx.options.detect_timestamps = True
        mitmproxy.ctx.options.script_mode = 'record'
        flow.request.method = 'POST'
        flow.request.set_content(str({'timestamp_key': '2021-01-11T13:18:12+00:00'}).encode())
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.request(flow)
        assert 'timestamp_key' in time_stamp_replacer.json_keys

    @pytest.mark.parametrize('time', TIMESTAMP_FORMATS)
    def test_cleaning_problematic_keys_from_url_query(self, mocker, flow, time):
        """
        Given:
            - A timestamp replacer instance
        When:
            - digesting a request in playback mode
        Then:
            - Ensure the problematic key passed as query param is replaced with constant value
        """
        mocker.patch('builtins.open', mock_open())
        mitmproxy.ctx.options.detect_timestamps = True
        mitmproxy.ctx.options.script_mode = 'playback'
        flow.request._set_query([('key1', 'value1'), ('timestamp_key', time)])
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.query_keys = ['timestamp_key']
        time_stamp_replacer.request(flow)
        for key, val in flow.request._get_query():
            if key == 'timestamp_key':
                assert val == time_stamp_replacer.constant

    @pytest.mark.parametrize('time', TIMESTAMP_FORMATS)
    def test_cleaning_problematic_keys_from_form_keys(self, mocker, flow, time):
        """
        Given:
            - A timestamp replacer instance
        When:
            - digesting a request in playback mode
        Then:
            - Ensure the problematic key passed as form key is replaced with constant value
        """
        mocker.patch('builtins.open', mock_open())
        mitmproxy.ctx.options.detect_timestamps = True
        mitmproxy.ctx.options.script_mode = 'playback'
        flow.request._set_urlencoded_form([('key1', 'value1'), ('timestamp_key', time)])
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.form_keys = ['timestamp_key']
        time_stamp_replacer.request(flow)
        for key, val in flow.request._get_urlencoded_form():
            if key == 'timestamp_key':
                assert val == time_stamp_replacer.constant

    @pytest.mark.parametrize('time', TIMESTAMP_FORMATS)
    def test_cleaning_problematic_keys_from_json_keys(self, mocker, flow, time):
        """
        Given:
            - A timestamp replacer instance
        When:
            - digesting a request in playback mode
        Then:
            - Ensure the problematic keys passed in json body are replaced with constant value
        """
        mocker.patch('builtins.open', mock_open())
        mitmproxy.ctx.options.detect_timestamps = True
        mitmproxy.ctx.options.script_mode = 'playback'
        flow.request.method = 'POST'
        flow.request.set_content(json.dumps({'key1': 'value1',
                                             'timestamp_key': time,
                                             'dict1': {'list': ['test', time]}}).encode())
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.json_keys = ['timestamp_key', 'dict1.list.1']
        time_stamp_replacer.request(flow)
        content = json.loads(flow.request.get_content())
        for key, val in content.items():
            if key == 'timestamp_key':
                assert val == time_stamp_replacer.constant
            elif key == 'dict1':
                assert time_stamp_replacer.constant in val['list']

    def test_url_query_is_sorted(self, mocker, flow):
        """
        Given:
            - A timestamp replacer instance
        When:
            - searching a request for problematic keys that has timestamp values in the query params
        Then:
            - Ensure that the request query params are sorted
        """
        mocker.patch('builtins.open', mock_open())
        mitmproxy.ctx.options.detect_timestamps = True
        mitmproxy.ctx.options.script_mode = 'clean'
        flow.request._set_query([('key2', 'value2'), ('key1', 'value1')])
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.request(flow)
        updated_request = flow.request._get_query()
        assert updated_request[0] == ('key1', 'value1')
        assert updated_request[1] == ('key2', 'value2')

    def test_live_false_when_running_in_playback_state(self, flow):
        """
            Given:
                - A flow
            When:
                - script is in playback mode
            Then:
                - Ensure that the request will not go out to the real world
        """
        mitmproxy.ctx.options.script_mode = 'playback'
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.request(flow)
        assert flow.live is False

    def test_fixed_boundary(self, flow):
        """
           Given:
               - A multipart/form-data flow with a random boundary
           Then:
               - Ensure that the boundary will be replaced to 'fixed_boundary'
       """
        original_boundary = 'original_boundary'
        flow.request.headers['Content-Type'] = f'multipart/form-data; boundary={original_boundary}'
        flow.request.content = f'--{original_boundary}\nContent-Disposision: form-data; ' \
                               f'name="test"\n\nsomething\n--{original_boundary}--'.encode()
        time_stamp_replacer = TimestampReplacer()
        time_stamp_replacer.request(flow)
        assert flow.request.content == b'--fixed_boundary\nContent-Disposision: form-data; ' \
                                       b'name="test"\n\nsomething\n--fixed_boundary--'


# demisto_sdk/commands/extract_outputs.py

from json import dumps, load
from typing import Optional

from ruamel.yaml import YAML


def get_outputs(yml: dict, ignore_prefix: Optional[str]):
    result = {}
    for command in yml.get('script', {}).get('commands', []):
        for output in command.get('outputs', []):
            path = output['contextPath']
            if ignore_prefix is None or not path.startswith(ignore_prefix + "."):
                result[path] = output['description']
    return result


path = '/Users/dschwartz/dev/demisto/content/Packs/AutoFocus/Integrations/AutofocusV2/AutofocusV2.yml'
d2 = get_outputs(YAML().load(open(path)), 'AutoFocus')
d1 = load(open('/Users/dschwartz/dev/demisto/demisto-sdk/demisto_sdk/commands/common/default_output_descriptions.json'))
d1.update(d2)
print(dumps(d1))


# demisto_sdk/commands/test_content/Docker.py

import logging
import re
from subprocess import PIPE, Popen

from demisto_sdk.commands.common.handlers import JSON_Handler

json = JSON_Handler()


class Docker:
    """ Client for running docker commands on remote machine using ssh connection.

    """
    PYTHON_INTEGRATION_TYPE = 'python'
    POWERSHELL_INTEGRATION_TYPE = 'powershell'
    JAVASCRIPT_INTEGRATION_TYPE = 'javascript'
    DEFAULT_PYTHON2_IMAGE = 'demisto/python'
    DEFAULT_PYTHON3_IMAGE = 'demisto/python3'
    COMMAND_FORMAT = '{{json .}}'
    MEMORY_USAGE = 'MemUsage'
    PIDS_USAGE = 'PIDs'
    CONTAINER_NAME = 'Name'
    CONTAINER_ID = 'ID'
    DEFAULT_CONTAINER_MEMORY_USAGE = 75
    DEFAULT_CONTAINER_PIDS_USAGE = 3
    DEFAULT_PWSH_CONTAINER_MEMORY_USAGE = 140
    DEFAULT_PWSH_CONTAINER_PIDS_USAGE = 24
    REMOTE_MACHINE_USER = 'ec2-user'
    SSH_OPTIONS = 'ssh -o StrictHostKeyChecking=no'

    @classmethod
    def _build_ssh_command(cls, server_ip, remote_command, force_tty=False):
        """Add and returns ssh prefix and escapes remote command

            Args:
                server_ip (str): remote machine ip to connect using ssh.
                remote_command (str): command to execute in remote machine.
                force_tty (bool): adds -t flag in order to force tty allocation.

            Returns:
                str: full ssh command

        """
        remote_server = '{}@{}'.format(cls.REMOTE_MACHINE_USER, server_ip)
        ssh_prefix = '{} {}'.format(cls.SSH_OPTIONS, remote_server)
        if force_tty:
            ssh_prefix += ' -t'
        # escaping the remote command with single quotes
        cmd = "{} '{}'".format(ssh_prefix, remote_command)

        return cmd

    @classmethod
    def _build_stats_cmd(cls, server_ip, docker_images):
        """ Builds docker stats and grep command string.

        Example of returned value:
        ssh -o StrictHostKeyChecking=no ec2-user@server_ip
        'sudo docker stats --no-stream --no-trunc --format "{{json .}}" | grep -Ei "demistopython33.7.2.214--"'
        Grep is based on docker images names regex.

            Args:
                server_ip (str): Remote machine ip to connect using ssh.
                docker_images (set): Set of docker images.

            Returns:
                str: String command to run later as subprocess.

        """
        # docker stats command with json output
        docker_command = 'sudo docker stats --no-stream --no-trunc --format "{}"'.format(cls.COMMAND_FORMAT)
        # replacing : and / in docker images names in order to grep the stats by container name
        docker_images_regex = ['{}--'.format(re.sub('[:/]', '', docker_image)) for docker_image in docker_images]
        pipe = ' | '
        grep_command = 'grep -Ei "{}"'.format('|'.join(docker_images_regex))
        remote_command = docker_command + pipe + grep_command
        cmd = cls._build_ssh_command(server_ip, remote_command)

        return cmd

    @classmethod
    def _build_kill_cmd(cls, server_ip, container_name):
        """ Constructs docker kll command string to run on remote machine.

            Args:
                server_ip (str): Remote machine ip to connect using ssh.
                container_name (str): Docker container name to kill.

            Returns:
                str: String of docker kill command on remote machine.
        """
        remote_command = 'sudo docker kill {}'.format(container_name)
        cmd = cls._build_ssh_command(server_ip, remote_command)

        return cmd

    @classmethod
    def _build_pid_info_cmd(cls, server_ip, container_id):
        """Constructs docker exec ps command string to run on remote machine.

            Args:
                server_ip (str): Remote machine ip to connect using ssh.
                container_id (str): Docker container id.

            Returns:
                str: String of docker exec ps command on remote machine.

        """
        remote_command = 'sudo docker exec -it {} ps -fe'.format(container_id)
        cmd = cls._build_ssh_command(server_ip, remote_command, force_tty=True)

        return cmd

    @classmethod
    def _parse_stats_result(cls, stats_lines, logging_module=logging):
        """Parses the docker statics str and converts to Mib.

            Args:
                stats_lines (str): String that contains docker stats.
                logging_module: The logging module that should be used.
            Returns:
                list: List of dictionaries with parsed docker container statistics.

        """
        stats_result = []
        try:
            containers_stats = [json.loads(c) for c in stats_lines.splitlines()]

            for container_stat in containers_stats:
                memory_usage_stats = container_stat.get(cls.MEMORY_USAGE, '').split('/')[0].lower()

                if 'kib' in memory_usage_stats:
                    mib_usage = float(memory_usage_stats.replace('kib', '').strip()) / 1024
                elif 'gib' in memory_usage_stats:
                    mib_usage = float(memory_usage_stats.replace('kib', '').strip()) * 1024
                else:
                    mib_usage = float(memory_usage_stats.replace('mib', '').strip())

                stats_result.append({
                    'memory_usage': mib_usage,
                    'pids': int(container_stat.get(cls.PIDS_USAGE)),
                    'container_name': container_stat.get(cls.CONTAINER_NAME),
                    'container_id': container_stat.get(cls.CONTAINER_ID)
                })
        except Exception:
            logging_module.exception("Failed in parsing docker stats result, returned empty list.")
        finally:
            return stats_result

    @classmethod
    def run_shell_command(cls, cmd):
        """Executes shell command and returns outputs of the process.

            Args:
                cmd (str): command to execute.

            Returns:
                str: stdout of the executed command.
                str: stderr of the executed command.

        """
        process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, universal_newlines=True)
        stdout, stderr = process.communicate()

        return stdout, stderr

    @classmethod
    def get_image_for_container_id(cls, server_ip, container_id, logging_module=logging):
        cmd = cls._build_ssh_command(server_ip, "sudo docker inspect -f {{.Config.Image}} " + container_id,
                                     force_tty=False)
        stdout, stderr = cls.run_shell_command(cmd)
        if stderr:
            logging_module.warning(f"Received stderr from docker inspect command. Additional information: {stderr}")
        res = stdout or ""
        return res.strip()

    @classmethod
    def get_integration_image(cls, integration_config):
        """ Returns docker image of integration that was configured using rest api call via demisto_client

            Args:
                integration_config (dict): Integration config that included script section.
            Returns:
                list: List that includes integration docker image name. If no docker image was found,
                      default python2 and python3 images are returned.

        """
        integration_script = integration_config.get('configuration', {}).get('integrationScript', {}) or {}
        integration_type = integration_script.get('type')
        docker_image = integration_script.get('dockerImage')

        if integration_type == cls.JAVASCRIPT_INTEGRATION_TYPE:
            return None
        elif integration_type in {cls.PYTHON_INTEGRATION_TYPE, cls.POWERSHELL_INTEGRATION_TYPE} and docker_image:
            return [docker_image]
        else:
            return [cls.DEFAULT_PYTHON2_IMAGE, cls.DEFAULT_PYTHON3_IMAGE]

    @classmethod
    def docker_stats(cls, server_ip, docker_images, logging_module=logging):
        """ Executes docker stats command and greps all containers with prefix of docker images names.

            Args:
                server_ip (str): Remote machine ip to connect using ssh.
                docker_images (set): Set of docker images to check their resource usage.
                logging_module: The logging module that should be used.

            Returns:
                list: List of dictionaries with parsed container memory statistics.
        """
        cmd = cls._build_stats_cmd(server_ip, docker_images)
        stdout, stderr = cls.run_shell_command(cmd)

        if stderr:
            logging_module.warning(f"Failed running docker stats command. Additional information: {stderr}")
            return []

        return cls._parse_stats_result(stdout, logging_module)

    @classmethod
    def kill_container(cls, server_ip, container_name, logging_module):
        """ Executes docker kill command on remote machine using ssh.

            Args:
                server_ip (str): The remote server ip address.
                container_name (str): The container name to kill
                logging_module: The logging module to use

        """
        cmd = cls._build_kill_cmd(server_ip, container_name)
        _, stderr = cls.run_shell_command(cmd)

        if stderr:
            logging_module.debug(f"Failed killing container: {container_name}\nAdditional information: {stderr}")

    @classmethod
    def get_docker_pid_info(cls, server_ip, container_id, logging_module):
        """Executes docker exec ps command on remote machine using ssh.

            Args:
                server_ip (str): The remote server ip address.
                container_id (str): Docker container id.
                logging_module: The logging module to use

            Returns:
                str: output of executed command.
        """
        cmd = cls._build_pid_info_cmd(server_ip, container_id)
        stdout, stderr = cls.run_shell_command(cmd)

        if stderr:
            ignored_warning_message = "Connection to {} closed".format(server_ip)
            if ignored_warning_message not in stderr:
                logging_module.debug(f"Failed getting pid info for container id: {container_id}.\n"
                                     f"Additional information: {stderr}")

        return stdout

    @classmethod
    def check_resource_usage(cls,
                             server_url,
                             docker_images,
                             def_memory_threshold,
                             def_pid_threshold,
                             docker_thresholds,
                             logging_module):
        """
        Executes docker stats command on remote machine and returns error message in case of exceeding threshold.

        Args:
            server_url (str): Target machine full url.
            docker_images (set): Set of docker images to check their resource usage.
            def_memory_threshold (int): Memory threshold of specific docker container, in Mib.
            def_pids_threshold (int): PIDs threshold of specific docker container, in Mib.
            docker_thresholds: thresholds per docker image
            logging_module: The logging module that should be used.

        Returns:
            str: The error message. Empty in case that resource check passed.

        """
        server_ip = server_url.lstrip("https://")
        containers_stats = cls.docker_stats(server_ip, docker_images, logging_module)
        error_message = ""

        for container_stat in containers_stats:
            failed_memory_test = False
            container_name = container_stat['container_name']
            container_id = container_stat['container_id']
            memory_usage = container_stat['memory_usage']
            pids_usage = container_stat['pids']
            image_full = cls.get_image_for_container_id(server_ip,
                                                        container_id,
                                                        logging_module)  # get full name (ex: demisto/slack:1.0.0.4978)
            image_name = image_full.split(':')[0]  # just the name such as demisto/slack

            memory_threshold = (docker_thresholds.get(image_full, {}).get('memory_threshold') or docker_thresholds.get(
                image_name, {}).get('memory_threshold') or def_memory_threshold)
            pid_threshold = (docker_thresholds.get(image_full, {}).get('pid_threshold') or
                             docker_thresholds.get(image_name, {}).get('pid_threshold') or def_pid_threshold)
            logging_module.debug(
                f"Checking container: {container_name} "
                f"(image: {image_full}) for memory: {memory_threshold} pid: {pid_threshold} thresholds "
                f"with actual values: memory {memory_usage} pids: {pids_usage} ..."
            )
            if memory_usage > memory_threshold:
                error_message += ('Failed docker resource test. Docker container {} exceeded the memory threshold, '
                                  'configured: {} MiB and actual memory usage is {} MiB.\n'
                                  'Fix container memory usage or add `memory_threshold` key to failed test '
                                  'in conf.json with value that is greater than {}\n'
                                  .format(container_name, memory_threshold, memory_usage, memory_usage))
                failed_memory_test = True
            if pids_usage > pid_threshold:
                error_message += ('Failed docker resource test. Docker container {} exceeded the pids threshold, '
                                  'configured: {} and actual pid number is {}.\n'
                                  'Fix container pid usage or add `pid_threshold` key to failed test '
                                  'in conf.json with value that is greater than {}\n'
                                  .format(container_name, pid_threshold, pids_usage, pids_usage))
                additional_pid_info = cls.get_docker_pid_info(server_ip, container_id, logging_module)
                if additional_pid_info:
                    error_message += 'Additional pid information:\n{}'.format(additional_pid_info)
                failed_memory_test = True

            if failed_memory_test:
                # killing current container in case of memory resource test failure
                cls.kill_container(server_ip, container_name, logging_module)

        return error_message


# demisto_sdk/commands/test_content/IntegrationsLock.py

import datetime
import os
from contextlib import contextmanager
from typing import Generator

import pytz
import requests
from google.api_core.exceptions import PreconditionFailed
from google.cloud import storage

LOCKS_PATH = 'content-locks'
BUCKET_NAME = os.environ.get('GCS_ARTIFACTS_BUCKET')
BUILD_NUM = os.environ.get('CI_BUILD_ID')
WORKFLOW_ID = os.environ.get('CI_PIPELINE_ID')
CIRCLE_STATUS_TOKEN = os.environ.get('CIRCLECI_STATUS_TOKEN')
GITLAB_STATUS_TOKEN = os.environ.get('GITLAB_STATUS_TOKEN', '')


@contextmanager
def acquire_test_lock(test_playbook) -> Generator:
    """
    This is a context manager that handles all the locking and unlocking of integrations.
    Execution is as following:
    * Attempts to lock the test's integrations and yields the result of this attempt
    * If lock attempt has failed - yields False, if it succeeds - yields True
    * Once the test is done- will unlock all integrations
    Args:
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context
    Yields:
        A boolean indicating the lock attempt result
    """
    locked = safe_lock_integrations(test_playbook)
    try:
        yield locked
    except Exception:
        test_playbook.build_context.logging_module.exception('Failed with test lock')
    finally:
        if not locked:
            return
        safe_unlock_integrations(test_playbook)


def safe_unlock_integrations(test_playbook):
    """
    This integration safely unlocks the test's integrations.
    If an unexpected error occurs - this method will log it's details and other tests execution will continue
    Args:
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context
    """
    try:
        # executing the test could take a while, re-instancing the storage client
        storage_client = storage.Client()
        unlock_integrations(test_playbook.integrations, test_playbook, storage_client)
    except Exception:
        test_playbook.build_context.logging_module.exception('attempt to unlock integration failed for unknown reason.')


def safe_lock_integrations(test_playbook) -> bool:
    """
    This integration safely locks the test's integrations and return it's result
    If an unexpected error occurs - this method will log it's details and return False
    Args:
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context

    Returns:
        A boolean indicating the lock attempt result
    """

    integration_names = [integration.name for integration in test_playbook.integrations_to_lock]
    if integration_names:
        print_msg = f'Attempting to lock integrations {integration_names}, ' \
                    f'with timeout {test_playbook.configuration.timeout}'
    else:
        print_msg = 'No integrations to lock'
    test_playbook.build_context.logging_module.debug(print_msg)
    try:
        storage_client = storage.Client()
        locked = lock_integrations(test_playbook, storage_client)
    except Exception:
        test_playbook.build_context.logging_module.exception('attempt to lock integration failed for unknown reason.')
        locked = False
    return locked


def workflow_still_running(workflow_id: str, test_playbook) -> bool:
    """
    This method takes a workflow id and checks if the workflow is still running
    If given workflow ID is the same as the current workflow, will simply return True
    else it will query circleci api for the workflow and return the status
    Args:
        workflow_id: The ID of the workflow
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context

    Returns:
        True if the workflow is running, else False
    """
    # If this is the current workflow_id
    if workflow_id == WORKFLOW_ID:
        return True
    else:
        try:
            test_playbook.build_context.logging_module.debug(
                f'Getting status for gitlab pipeline with id: {workflow_id}')
            api_v4_url = os.environ.get('CI_API_V4_URL')
            ci_project_id = os.environ.get('CI_PROJECT_ID')
            workflow_details_response = requests.get(
                f'{api_v4_url}/projects/{ci_project_id}/pipelines/{workflow_id}',
                headers={'PRIVATE-TOKEN': GITLAB_STATUS_TOKEN})
            workflow_details_response.raise_for_status()
        except Exception:
            test_playbook.build_context.logging_module.exception(
                f'Failed to get gitlab-ci response about pipeline with id {workflow_id}.')
            return True
        return workflow_details_response.json().get('status') not in ('canceled', 'success', 'failed')


def lock_integrations(test_playbook,
                      storage_client: storage.Client) -> bool:
    """
    Locks all the test's integrations
    Args:
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context
        storage_client: The GCP storage client

    Returns:
        True if all the test's integrations were successfully locked, else False
    """
    integrations = [integration.name for integration in test_playbook.integrations_to_lock]
    if not integrations:
        return True
    existing_integrations_lock_files = get_locked_integrations(integrations, storage_client)
    for integration, lock_file in existing_integrations_lock_files.items():
        # Each file has content in the form of <circleci-build-number>:<timeout in seconds>
        # If it has not expired - it means the integration is currently locked by another test.
        workflow_id, build_number, lock_timeout = lock_file.download_as_string().decode().split(':')
        if not lock_expired(lock_file, lock_timeout) and workflow_still_running(workflow_id, test_playbook):
            # there is a locked integration for which the lock is not expired - test cannot be executed at the moment
            test_playbook.build_context.logging_module.warning(
                f'Could not lock integration {integration}, another lock file was exist with '
                f'build number: {build_number}, timeout: {lock_timeout}, last update at {lock_file.updated}.\n'
                f'Delaying test execution')
            return False
    integrations_generation_number = {}
    # Gathering generation number with which the new file will be created,
    # See https://cloud.google.com/storage/docs/generations-preconditions for details.
    for integration in integrations:
        if integration in existing_integrations_lock_files:
            integrations_generation_number[integration] = existing_integrations_lock_files[integration].generation
        else:
            integrations_generation_number[integration] = 0
    return create_lock_files(integrations_generation_number,
                             storage_client,
                             test_playbook)


def create_lock_files(integrations_generation_number: dict,
                      storage_client: storage.Client,
                      test_playbook) -> bool:
    """
    This method tries to create a lock files for all integrations specified in 'integrations_generation_number'.
    Each file should contain <circle-ci-build-number>:<test-timeout>
    where the <circle-ci-build-number> part is for debugging and troubleshooting
    and the <test-timeout> part is to be able to unlock revoked test files.
    If for any of the integrations, the lock file creation will fail- the already created files will be cleaned.
    Args:
        integrations_generation_number: A dict in the form of {<integration-name>:<integration-generation>}
        storage_client: The GCP storage client
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context

    Returns:

    """
    locked_integrations = []
    bucket = storage_client.bucket(BUCKET_NAME)
    for integration, generation_number in integrations_generation_number.items():
        blob = bucket.blob(f'{LOCKS_PATH}/{integration}')
        try:
            blob.upload_from_string(f'{WORKFLOW_ID}:{BUILD_NUM}:{test_playbook.configuration.timeout + 30}',
                                    if_generation_match=generation_number)
            test_playbook.build_context.logging_module.debug(f'integration {integration} locked')
            locked_integrations.append(integration)
        except PreconditionFailed:
            # if this exception occurs it means that another build has locked this integration
            # before this build managed to do it.
            # we need to unlock all the integrations we have already locked and try again later
            test_playbook.build_context.logging_module.warning(
                f'Could not lock integration {integration}, Create file with precondition failed.'
                f'delaying test execution.')
            unlock_integrations(locked_integrations, test_playbook, storage_client)
            return False
    return True


def unlock_integrations(integrations_to_unlock: list,
                        test_playbook,
                        storage_client: storage.Client) -> None:
    """
    Delete all integration lock files for integrations specified in 'locked_integrations'
    Args:
        integrations_to_unlock (List[Integration]): The test playbook instance we want to test under the lock's context
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context
        storage_client: The GCP storage client
    """
    locked_integrations = [integration.name for integration in integrations_to_unlock]
    locked_integration_blobs = get_locked_integrations(locked_integrations, storage_client)
    for integration, lock_file in locked_integration_blobs.items():
        try:
            # Verifying build number is the same as current build number to avoid deleting other tests lock files
            _, build_number, _ = lock_file.download_as_string().decode().split(':')
            if build_number == BUILD_NUM:
                lock_file.delete(if_generation_match=lock_file.generation)
                test_playbook.build_context.logging_module.debug(
                    f'Integration {integration} unlocked')
        except PreconditionFailed:
            test_playbook.build_context.logging_module.error(
                f'Could not unlock integration {integration} precondition failure')


def get_locked_integrations(integrations: list, storage_client: storage.Client) -> dict:
    """
    Getting all locked integrations files
    Args:
        integrations: Integrations that we want to get lock files for
        storage_client: The GCP storage client

    Returns:
        A dict of the form {<integration-name>:<integration-blob-object>} for all integrations that has a blob object.
    """
    # Listing all files in lock folder
    # Wrapping in 'list' operator because list_blobs return a generator which can only be iterated once
    lock_files_ls = list(storage_client.list_blobs(BUCKET_NAME, prefix=f'{LOCKS_PATH}'))
    current_integrations_lock_files = {}
    # Getting all existing files details for integrations that we want to lock
    for integration in integrations:
        current_integrations_lock_files.update({integration: [lock_file_blob for lock_file_blob in lock_files_ls if
                                                              lock_file_blob.name == f'{LOCKS_PATH}/{integration}']})
    # Filtering 'current_integrations_lock_files' from integrations with no files
    current_integrations_lock_files = {integration: blob_files[0] for integration, blob_files in
                                       current_integrations_lock_files.items() if blob_files}
    return current_integrations_lock_files


def lock_expired(lock_file: storage.Blob, lock_timeout: str) -> bool:
    """
    Checks if the time that passed since the creation of the 'lock_file' is more then 'lock_timeout'.
    If not- it means that the integration represented by the lock file is currently locked and is tested in another build
    Args:
        lock_file: The lock file blob object
        lock_timeout: The expiration timeout of the lock in seconds

    Returns:
        True if the lock has expired it's timeout, else False
    """
    return datetime.datetime.now(tz=pytz.utc) - lock_file.updated >= datetime.timedelta(seconds=int(lock_timeout))


# demisto_sdk/commands/test_content/ParallelLoggingManager.py

import logging
import os
import sys
from logging.handlers import QueueHandler, QueueListener
from queue import Queue
from threading import Lock, currentThread
from typing import Any, Dict, Set

import coloredlogs

ARTIFACTS_PATH = os.environ.get('ARTIFACTS_FOLDER', '.')
LOGGING_FORMAT = '[%(asctime)s] - [%(threadName)s] - [%(levelname)s] - %(message)s'
LEVEL_STYLES = {
    'critical': {'bold': True, 'color': 'red'},
    'debug': {'color': 'cyan'},
    'error': {'color': 'red'},
    'info': {},
    'warning': {'color': 'yellow'},
    'success': {'color': 'green'}
}


def _add_logging_level(level_name: str, level_num: int, method_name: str = None) -> None:
    """
    Comprehensively adds a new logging level to the `logging` module and the
    currently configured logging class.

    `level_name` becomes an attribute of the `logging` module with the value
    `level_num`. `method_name` becomes a convenience method for both `logging`
    itself and the class returned by `logging.getLoggerClass()` (usually just
    `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is
    used.


    To avoid accidental clobberings of existing attributes, this method will
    raise an `AttributeError` if the level name is already an attribute of the
    `logging` module or if the method name is already present

    Example
    -------
    >>> _add_logging_level('TRACE', logging.DEBUG - 5)
    >>> logging.getLogger(__name__).setLevel("TRACE")
    >>> logging.getLogger(__name__).trace('that worked')
    >>> logging.trace('so did this')
    >>> logging.TRACE
    5

    Args:
        level_name: The name of the level that will become an attribute of the `logging` module
        level_num: The logging value of the new level
        method_name: The method name with which the new level will be called

    """
    if not method_name:
        method_name = level_name.lower()

    if hasattr(logging, level_name):
        raise AttributeError(f'{level_name} already defined in logging module')
    if hasattr(logging, method_name):
        raise AttributeError(f'{method_name} already defined in logging module')
    if hasattr(logging.getLoggerClass(), method_name):
        raise AttributeError(f'{method_name} already defined in logger class')

    # This method was inspired by the answers to Stack Overflow post
    # http://stackoverflow.com/q/2183233/2988730, especially
    # http://stackoverflow.com/a/13638084/2988730
    def logForLevel(self, message, *args, **kwargs):
        if self.isEnabledFor(level_num):
            self._log(level_num, message, args, **kwargs)

    def logToRoot(message, *args, **kwargs):
        logging.log(level_num, message, *args, **kwargs)

    logging.addLevelName(level_num, level_name)
    setattr(logging, level_name, level_num)
    setattr(logging.getLoggerClass(), method_name, logForLevel)
    setattr(logging, method_name, logToRoot)


class ParallelLoggingManager:
    """
    This is a parallel logging manager meant to be used when using logging with multithreading when some of the logs
    should be grouped together in the log file for better readability.

    This class should be created only once in every process exceution.
    Example
    -------
    >>> logging_manager = ParallelLoggingManager('debug_file.log')
    >>> logging_manager.debug('debug message')
    >>> logging_manager.info('info message')
    >>> logging_manager.warning('warning message')
    >>> logging_manager.error('error message')
    >>> logging_manager.critical('critical message')
    >>> logging_manager.success('success message')
    # The 'exception method should be called only within exception handling context
    >>> logging_manager.exception('exception message')
    # The logs will be sent to the handlers only when the 'execute_logs' method will be called by
     the thread who wrote the log messages in the first place
    >>> logging_manager.execute_logs()
    # In case we want to write a log now we can use the 'real_time=True' flag.
    >>> logging_manager.debug('debug message', real_time=True)
    >>> logging_manager.info('info message', real_time=True)
    >>> logging_manager.warning('warning message', real_time=True)
    >>> logging_manager.error('error message', real_time=True)
    >>> logging_manager.critical('critical message', real_time=True)
    >>> logging_manager.success('success message', real_time=True)
    """

    def __init__(self, log_file_name: str, real_time_logs_only: bool = False):
        """
        Initializes the logging manager:
        - Uses a different colored format for each log level (see LEVEL_STYLES)
        - Defines a FileHandler with the given log_file_name.
        - Defines a SteamHandler for stdout.
        - Defines a logger for real time logs with those handlers
        Args:
            real_time_logs_only: When True - all logs will be handled in real time
            log_file_name: The path to where the log file will be saved
        """
        if not hasattr(logging, 'success'):
            _add_logging_level('SUCCESS', 25)
        self.real_time_logs_only = real_time_logs_only
        self.log_file_name = log_file_name
        formatter = coloredlogs.ColoredFormatter(fmt=LOGGING_FORMAT,
                                                 level_styles=LEVEL_STYLES)
        self.console_handler = logging.StreamHandler(sys.stdout)
        self.console_handler.setFormatter(formatter)
        log_file_path = os.path.join(ARTIFACTS_PATH, 'logs', log_file_name) if os.path.exists(
            os.path.join(ARTIFACTS_PATH, 'logs')) else os.path.join(ARTIFACTS_PATH, log_file_name)
        self.file_handler = logging.FileHandler(log_file_path)
        self.file_handler.setFormatter(formatter)
        self.console_handler.setLevel(logging.INFO)
        self.file_handler.setLevel(logging.DEBUG)
        self.real_time_logger = logging.getLogger(f'real_time-{log_file_path}')
        self.real_time_logger.addHandler(self.file_handler)
        self.real_time_logger.addHandler(self.console_handler)
        self.real_time_logger.setLevel(logging.DEBUG)
        self.real_time_logger.propagate = False
        self.loggers: Dict[str, Any] = {}
        self.listeners: Dict[str, QueueListener] = {}
        self.logs_lock = Lock()
        self.thread_names: Set[str] = set()

    def _add_logger(self, thread_name: str):
        """
        Defines a new logger, queueHandler and QueueListener for a new thread
        This method is only called if the thread name is not in the 'thread_names' set.
        Args:
            thread_name: The name of the thread that should be added
        """
        log_queue: Queue = Queue(-1)
        queue_handler = QueueHandler(log_queue)
        queue_handler.setLevel(logging.DEBUG)
        logger = logging.getLogger(thread_name)
        logger.propagate = False
        logger.setLevel(logging.DEBUG)
        logger.addHandler(queue_handler)
        listener = QueueListener(log_queue, self.console_handler, self.file_handler, respect_handler_level=True)
        self.loggers[thread_name] = logger
        self.listeners[thread_name] = listener
        self.thread_names.add(thread_name)

    def debug(self, message: str, real_time: bool = False) -> None:
        """
        Executes a debug log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = self.real_time_logger.debug if real_time or self.real_time_logs_only else self.loggers[
            thread_name].debug
        log_method(message)

    def info(self, message: str, real_time: bool = False) -> None:
        """
        Executes an info log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = self.real_time_logger.info if real_time or self.real_time_logs_only else self.loggers[
            thread_name].info
        log_method(message)

    def warning(self, message: str, real_time: bool = False) -> None:
        """
        Executes a warning log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = self.real_time_logger.warning if real_time or self.real_time_logs_only else self.loggers[
            thread_name].warning
        log_method(message)

    def error(self, message: str, real_time: bool = False) -> None:
        """
        Executes a error log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = self.real_time_logger.error if real_time or self.real_time_logs_only else self.loggers[
            thread_name].error
        log_method(message)

    def critical(self, message: str, real_time: bool = False) -> None:
        """
        Executes a critical log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = self.real_time_logger.critical if real_time or self.real_time_logs_only else self.loggers[
            thread_name].critical
        log_method(message)

    def exception(self, message: str, real_time: bool = False) -> None:
        """
        Executes a exception log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = self.real_time_logger.exception if real_time or self.real_time_logs_only else self.loggers[
            thread_name].exception
        log_method(message)

    def success(self, message: str, real_time: bool = False) -> None:
        """
        Executes a success log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = self.real_time_logger.success if real_time or self.real_time_logs_only else self.loggers[  # type: ignore
            thread_name].success
        log_method(message)

    def execute_logs(self) -> None:
        """
        Writes the logs from the queue to the handlers.
        """
        thread_name = currentThread().getName()
        self.logs_lock.acquire()
        self.listeners[thread_name].start()
        self.listeners[thread_name].stop()
        self.logs_lock.release()


# demisto_sdk/commands/test_content/TestContentClasses.py

import ast
import logging
import os
import re
import subprocess
import sys
import time
import urllib.parse
import uuid
from copy import deepcopy
from distutils.version import LooseVersion
from math import ceil
from pprint import pformat
from queue import Empty, Queue
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union

import demisto_client
import prettytable
import requests
import urllib3
from demisto_client.demisto_api import DefaultApi, Incident
from demisto_client.demisto_api.rest import ApiException
from slack import WebClient as SlackClient

from demisto_sdk.commands.common.constants import (DEFAULT_CONTENT_ITEM_FROM_VERSION, DEFAULT_CONTENT_ITEM_TO_VERSION,
                                                   FILTER_CONF, PB_Status)
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.tools import get_demisto_version
from demisto_sdk.commands.test_content.constants import CONTENT_BUILD_SSH_USER, LOAD_BALANCER_DNS
from demisto_sdk.commands.test_content.Docker import Docker
from demisto_sdk.commands.test_content.IntegrationsLock import acquire_test_lock
from demisto_sdk.commands.test_content.mock_server import RESULT, MITMProxy, run_with_mock
from demisto_sdk.commands.test_content.ParallelLoggingManager import ParallelLoggingManager
from demisto_sdk.commands.test_content.tools import get_ui_url, is_redhat_instance, update_server_configuration

json = JSON_Handler()

ENV_RESULTS_PATH = './artifacts/env_results.json'
FAILED_MATCH_INSTANCE_MSG = "{} Failed to run.\n There are {} instances of {}, please select one of them by using " \
                            "the instance_name argument in conf.json. The options are:\n{}"
ENTRY_TYPE_ERROR = 4
DEFAULT_INTERVAL = 4
MAX_RETRIES = 3
RETRIES_THRESHOLD = ceil(MAX_RETRIES / 2)

SLACK_MEM_CHANNEL_ID = 'CM55V7J8K'
XSIAM_SERVER_TYPE = 'XSIAM'
# For now we run all xsiam tests
IS_XSIAM = False

__all__ = ['BuildContext',
           'Conf',
           'Integration',
           'IntegrationConfiguration',
           'SecretConf',
           'TestConfiguration',
           'TestContext',
           'TestPlaybook',
           'TestResults',
           'ServerContext']


class IntegrationConfiguration:
    def __init__(self, integration_configuration):
        self.raw_dict = integration_configuration
        self.name: str = integration_configuration.get('name', '')
        self.instance_name: str = integration_configuration.get('instance_name', self.name)
        self.is_byoi: bool = integration_configuration.get('byoi', True)
        self.should_validate_test_module: bool = integration_configuration.get('validate_test', True)
        self.has_integration: bool = integration_configuration.get('has_integration', True)
        self.params: dict = integration_configuration.get('params', {})

    def __str__(self):
        return json.dumps(self.raw_dict)

    def __repr__(self):
        return str(self)


class SecretConf:
    def __init__(self, secret_conf: dict):
        """
        Args:
            secret_conf: The contents of the content-test-conf conf.json file.
        """
        self.server_username = secret_conf['username']
        self.server_password = secret_conf['userPassword']
        self.integrations = [
            IntegrationConfiguration(integration_configuration=configuration) for configuration in
            secret_conf['integrations']
        ]


class TestConfiguration:
    def __init__(self, test_configuration: dict, default_test_timeout: int):
        """
        Args:
            test_configuration: A record of a test from 'tests' list in conf.json file in content repo..
            default_test_timeout: The default timeout to use in case no timeout is specified in the configuration
        """
        self.raw_dict = test_configuration
        self.playbook_id = test_configuration.get('playbookID', '')
        self.nightly_test = test_configuration.get('nightly', False)
        self.from_version = test_configuration.get('fromversion', DEFAULT_CONTENT_ITEM_FROM_VERSION)
        self.to_version = test_configuration.get('toversion', DEFAULT_CONTENT_ITEM_TO_VERSION)
        self.timeout = test_configuration.get('timeout', default_test_timeout)
        self.memory_threshold = test_configuration.get('memory_threshold', Docker.DEFAULT_CONTAINER_MEMORY_USAGE)
        self.pid_threshold = test_configuration.get('pid_threshold', Docker.DEFAULT_CONTAINER_PIDS_USAGE)
        self.runnable_on_docker_only: bool = test_configuration.get('runnable_on_docker_only', False)
        self.is_mockable = test_configuration.get('is_mockable')
        self.context_print_dt = test_configuration.get('context_print_dt')
        self.test_integrations: List[str] = self._parse_integrations_conf(test_configuration)
        self.test_instance_names: List[str] = self._parse_instance_names_conf(test_configuration)
        self.instance_configuration: dict = test_configuration.get('instance_configuration', {})
        self.external_playbook_config: dict = test_configuration.get('external_playbook_config', {})
        self.is_first_playback_failed: bool = False
        self.number_of_executions: int = 0
        self.number_of_successful_runs: int = 0

    @staticmethod
    def _parse_integrations_conf(test_configuration):
        integrations_conf = test_configuration.get('integrations', [])
        if not isinstance(integrations_conf, list):
            integrations_conf = [integrations_conf]
        return integrations_conf

    @staticmethod
    def _parse_instance_names_conf(test_configuration):
        instance_names_conf = test_configuration.get('instance_names', [])
        if not isinstance(instance_names_conf, list):
            instance_names_conf = [instance_names_conf]
        return instance_names_conf

    def __str__(self):
        return str(self.raw_dict)


class Conf:
    def __init__(self, conf: dict):
        """
        Args:
            conf: The contents of the content conf.json file.
        """
        self.default_timeout: int = conf.get('testTimeout', 30)
        self.tests: list = [
            TestConfiguration(test_configuration, self.default_timeout) for test_configuration in conf.get('tests', [])
        ]
        self.skipped_tests: Dict[str, str] = conf.get('skipped_tests')  # type: ignore
        self.skipped_integrations: Dict[str, str] = conf.get('skipped_integrations')  # type: ignore
        self.unmockable_integrations: Dict[str, str] = conf.get('unmockable_integrations')  # type: ignore
        self.parallel_integrations: List[str] = conf['parallel_integrations']
        self.docker_thresholds = conf.get('docker_thresholds', {}).get('images', {})


class TestPlaybook:

    def __init__(self,
                 build_context,
                 test_configuration: TestConfiguration):
        """
        This class has all the info related to a test playbook during test execution
        Args:
            build_context (BuildContext): The build context to use in the build
            test_configuration: The configuration from content conf.json file
        """
        self.build_context = build_context
        self.configuration: TestConfiguration = test_configuration
        self.is_mockable: bool = self.configuration.playbook_id not in build_context.unmockable_test_ids
        self.integrations: List[Integration] = [
            Integration(self.build_context, integration_name, self.configuration.test_instance_names)
            for integration_name in self.configuration.test_integrations]
        self.integrations_to_lock = [
            integration for integration in self.integrations if
            integration.name not in self.build_context.conf.parallel_integrations]

    def __str__(self):
        return f'"{self.configuration.playbook_id}"'

    def __repr__(self):
        return str(self)

    def should_test_run(self):
        skipped_tests_collected = self.build_context.tests_data_keeper.skipped_tests

        def in_filtered_tests():
            """
            Checks if there are a list of filtered tests that the playbook is in them.
            """
            if not self.build_context.filtered_tests or self.configuration.playbook_id not in self.build_context.filtered_tests:
                self.build_context.logging_module.debug(f'Skipping {self} because it\'s not in filtered tests')
                skipped_tests_collected[self.configuration.playbook_id] = 'not in filtered tests'
                return False
            return True

        def nightly_test_in_non_nightly_build():
            """
            Checks if we are on a build which is not nightly, and the test should run only on nightly builds.
            """
            if self.configuration.nightly_test and not self.build_context.is_nightly:
                log_message = f'Skipping {self} because it\'s a nightly test in a non nightly build'
                if self.configuration.playbook_id in self.build_context.filtered_tests:
                    self.build_context.logging_module.warning(log_message)
                else:
                    self.build_context.logging_module.debug(log_message)
                skipped_tests_collected[self.configuration.playbook_id] = 'nightly test in a non nightly build'
                return True
            return False

        def skipped_test():
            if self.configuration.playbook_id in self.build_context.conf.skipped_tests:
                if self.configuration.playbook_id in self.build_context.filtered_tests:
                    # Add warning log, as the playbook is supposed to run according to the filters, but it's skipped
                    self.build_context.logging_module.warning(f'Skipping test {self} because it\'s in skipped test list')
                else:
                    self.build_context.logging_module.debug(f'Skipping test {self} because it\'s in skipped test list')
                reason = self.build_context.conf.skipped_tests[self.configuration.playbook_id]
                skipped_tests_collected[self.configuration.playbook_id] = reason
                return True
            return False

        def version_mismatch():
            if not (LooseVersion(self.configuration.from_version) <= LooseVersion(
                    self.build_context.server_numeric_version) <= LooseVersion(self.configuration.to_version)):
                self.build_context.logging_module.warning(
                    f'Test {self} ignored due to version mismatch '
                    f'(test versions: {self.configuration.from_version}-{self.configuration.to_version})\n')
                skipped_tests_collected[self.configuration.playbook_id] = \
                    f'(test versions: {self.configuration.from_version}-{self.configuration.to_version})'
                return True
            return False

        def test_has_skipped_integration():
            for integration in self.configuration.test_integrations:
                # So now we know that the test is in the filtered tests
                if integration in self.build_context.conf.skipped_integrations:
                    self.build_context.logging_module.debug(f'Skipping {self} because it has a skipped integration {integration}')
                    # The playbook should be run but has a skipped integration
                    if self.build_context.filtered_tests and self.configuration.playbook_id in self.build_context.filtered_tests:
                        # Adding the playbook ID to playbook_skipped_integration so that we can send a PR comment about it
                        skip_reason = self.build_context.conf.skipped_integrations[integration]
                        self.build_context.tests_data_keeper.playbook_skipped_integration.add(
                            f'{self.configuration.playbook_id} - reason: {skip_reason}')
                        self.build_context.logging_module.warning(
                            f'The integration {integration} is skipped and critical for the test {self}.')
                        skipped_tests_collected[self.configuration.playbook_id] = f'The integration {integration} is skipped'
                    return True

            return False

        return in_filtered_tests() and \
            not nightly_test_in_non_nightly_build() and \
            not skipped_test() and \
            not version_mismatch() and \
            not test_has_skipped_integration()

    def run_test_module_on_integrations(self, client: DefaultApi) -> bool:
        """
        runs 'test-module' on all integrations that the playbook uses and return a boolean indicating the result
        Args:
            client: The demisto_client to use

        Returns:
            True if all integrations was test-module execution was successful else False
        """
        for integration in self.integrations:
            success = integration.test_integration_instance(client)
            if not success:
                return False
        return True

    def configure_integrations(self, client: DefaultApi, server_context: 'ServerContext',
                               instance_configuration: dict) -> bool:
        """
        Configures all integrations that the playbook uses and return a boolean indicating the result
        Args:
            client: The demisto_client to use
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in

        Returns:
            True if all integrations was configured else False
        """
        configured_integrations: List[Integration] = []
        for integration in self.integrations:
            instance_created = integration.create_integration_instance(client,
                                                                       self.configuration.playbook_id,
                                                                       self.is_mockable,
                                                                       server_context,
                                                                       instance_configuration,
                                                                       )
            if not instance_created:
                self.build_context.logging_module.error(
                    f'Cannot run playbook {self}, integration {integration} failed to configure')
                # Deleting all the already configured integrations
                for configured_integration in configured_integrations:
                    configured_integration.delete_integration_instance(client)
                return False
            configured_integrations.append(integration)
        return True

    def disable_integrations(self, client: DefaultApi, server_context: 'ServerContext') -> None:
        """
        Disables all integrations that the playbook uses
        Clears server configurations set for the integration if there are such
        Reset containers if server configurations were cleared

        Args:
            client: The demisto_client to use
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in
        """
        for integration in self.integrations:
            integration.disable_integration_instance(client)
        updated_keys = False
        if not IS_XSIAM:
            updated_keys = self._set_prev_server_keys(client, server_context)
        if updated_keys:
            server_context._reset_containers()

    def _set_prev_server_keys(self, client: DefaultApi, server_context: 'ServerContext') -> bool:
        """Sets previous stored system (server) config, if existed

        Args:
            client (DefaultApi): The demisto_client to use
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in

        Returns:
             bool: Whether server configurations were updated to indicate if reset containers is required
        """
        updated = False
        for integration in self.integrations:
            if integration.configuration and 'server_keys' not in integration.configuration.params:
                continue
            if server_context.prev_system_conf:
                update_server_configuration(
                    client=client,
                    server_configuration=server_context.prev_system_conf,
                    error_msg='Failed to set server keys',
                    logging_manager=integration.build_context.logging_module,
                )
                server_context.prev_system_conf = {}
                updated = True
            else:
                integration.build_context.logging_module.error(
                    f'Failed clearing system conf. Found server_keys for integration {integration.name} but could not'
                    'find previous system conf stored'
                )
        return updated

    def delete_integration_instances(self, client: DefaultApi):
        """
        Deletes all integrations that the playbook uses
        Args:
            client: The demisto_client to use
        """
        for integration in self.integrations:
            integration.delete_integration_instance(client)

    def create_incident(self, client: DefaultApi) -> Optional[Incident]:
        """
        Creates an incident with the current playbook ID
        Args:
            client: The demisto_client to use

        Returns:
            - The created incident or None
        """
        # Preparing the incident request

        incident_name = f'inc-{self.configuration.playbook_id}--build_number:' \
                        f'{self.build_context.build_number}--{uuid.uuid4()}'
        create_incident_request = demisto_client.demisto_api.CreateIncidentRequest()
        create_incident_request.create_investigation = True
        create_incident_request.playbook_id = self.configuration.playbook_id
        create_incident_request.name = incident_name

        try:
            response = client.create_incident(create_incident_request=create_incident_request)
        except ApiException:
            self.build_context.logging_module.exception(
                f'Failed to create incident with name {incident_name} for playbook {self}')
        try:
            inc_id = response.id
        except Exception:
            inc_id = 'incCreateErr'
        # inc_id = response_json.get('id', 'incCreateErr')
        if inc_id == 'incCreateErr':
            error_message = f'Failed to create incident for playbookID: {self}.' \
                            'Possible reasons are:\nMismatch between playbookID in conf.json and ' \
                            'the id of the real playbook you were trying to use,' \
                            'or schema problems in the TestPlaybook.'
            self.build_context.logging_module.error(error_message)
            return None

        # get incident
        search_filter = demisto_client.demisto_api.SearchIncidentsData()
        inc_filter = demisto_client.demisto_api.IncidentFilter()
        inc_filter.query = f'id: {inc_id}'
        if IS_XSIAM:
            # in xsiam `create_incident` response don`t return created incident id.
            inc_filter.query = f'name:"{incident_name}"'
        # inc_filter.query
        search_filter.filter = inc_filter

        incident_search_responses = []

        found_incidents = 0
        # poll the incidents queue for a max time of 300 seconds
        timeout = time.time() + 300
        while found_incidents < 1:
            try:
                incidents = client.search_incidents(filter=search_filter)
                found_incidents = incidents.total
                incident_search_responses.append(incidents)
            except ApiException:
                if IS_XSIAM:
                    self.build_context.logging_module.exception(f'Searching incident with name {incident_name} failed')
                else:
                    self.build_context.logging_module.exception(f'Searching incident with id {inc_id} failed')
            if time.time() > timeout:
                if IS_XSIAM:
                    self.build_context.logging_module.error(f'Got timeout for searching incident with name '
                                                            f'{incident_name}')
                else:
                    self.build_context.logging_module.error(f'Got timeout for searching incident id {inc_id}')

                self.build_context.logging_module.error(f'Incident search responses: {incident_search_responses}')
                return None

            time.sleep(10)

        return incidents.data[0]

    def delete_incident(self, client: DefaultApi, incident_id: str) -> bool:
        """
        Deletes a Demisto incident
        Args:
            client: demisto_client instance to use
            incident_id: Incident to delete

        Returns:
            True if incident was deleted else False
        """
        try:
            body = {
                'ids': [incident_id],
                'filter': {},
                'all': False
            }
            res = demisto_client.generic_request_func(self=client, method='POST',
                                                      path='/incident/batchDelete', body=body)
        except ApiException:
            self.build_context.logging_module.exception(
                'Failed to delete incident, error trying to communicate with demisto server')
            return False

        if int(res[1]) != 200:
            self.build_context.logging_module.error(f'delete incident failed with Status code {res[1]}')
            self.build_context.logging_module.error(pformat(res))
            return False

        return True

    def close_incident(self, client: DefaultApi, incident_id: str) -> bool:
        """
        Closes a Demisto incident
        Args:
            client: demisto_client instance to use
            incident_id: Incident to close

        Returns:
            True if incident was closed else False
        """
        try:
            body = {
                'id': incident_id,
                'CustomFields': {}
            }
            res = demisto_client.generic_request_func(self=client, method='POST',
                                                      path='/incident/close', body=body)
            self.build_context.logging_module.info(f'Closed incident: {incident_id}.')
        except ApiException:
            self.build_context.logging_module.warning(
                'Failed to close incident, error trying to communicate with demisto server.')
            return False

        if int(res[1]) != 200:
            self.build_context.logging_module.warning(f'Close incident failed with Status code {res[1]}')
            self.build_context.logging_module.warning(pformat(res))
            return False

        return True

    def print_context_to_log(self, client: DefaultApi, incident_id: str) -> None:
        try:
            body = {
                "query": f"${{{self.configuration.context_print_dt}}}"
            }
            res = demisto_client.generic_request_func(self=client, method='POST',
                                                      path=f'/investigation/{incident_id}/context', body=body)
            if int(res[1]) != 200:
                self.build_context.logging_module.error(f'incident context fetch failed with Status code {res[1]}')
                self.build_context.logging_module.error(pformat(res))
                return
            try:
                self.build_context.logging_module.info(json.dumps(ast.literal_eval(res[0]), indent=4))
            except ValueError:
                self.build_context.logging_module.error(f"unable to parse result for result with value: {res[0]}")
        except ApiException:
            self.build_context.logging_module.exception(
                'Failed to get context, error trying to communicate with demisto server')


class BuildContext:
    def __init__(self, kwargs: dict, logging_module: ParallelLoggingManager):
        global IS_XSIAM
        self.is_xsiam = True if kwargs['server_type'] == XSIAM_SERVER_TYPE else False
        IS_XSIAM = self.is_xsiam
        self.logging_module: ParallelLoggingManager = logging_module
        self.server = kwargs['server']
        self.xsiam_machine = kwargs.get('xsiam_machine')
        self.xsiam_servers_path = kwargs.get('xsiam_servers_path')
        self.conf, self.secret_conf = self._load_conf_files(kwargs['conf'], kwargs['secret'])
        if self.is_xsiam:
            with open(kwargs.get('xsiam_servers_api_keys_path'), 'r') as json_file:  # type: ignore[arg-type]
                xsiam_servers_api_keys = json.loads(json_file.read())
            self.xsiam_conf = self._load_xsiam_file(self.xsiam_servers_path)
            self.env_json = [self.xsiam_conf.get(self.xsiam_machine, {})]
            self.api_key = xsiam_servers_api_keys.get(self.xsiam_machine)
            self.auth_id = self.env_json[0].get('x-xdr-auth-id')
            self.xsiam_ui_path = self.env_json[0].get('ui_url')
        else:
            self.api_key = kwargs['api_key']
            self.env_json = self._load_env_results_json()
            self.xsiam_conf = None
            self.auth_id = None
            self.xsiam_ui_path = None
        self.is_nightly = kwargs['nightly']
        self.slack_client = SlackClient(kwargs['slack'])
        self.circleci_token = kwargs['circleci']
        self.build_number = kwargs['build_number']
        self.build_name = kwargs['branch_name']
        self.isAMI = kwargs['is_ami'] if not self.is_xsiam else False
        self.memCheck = kwargs['mem_check']
        self.server_version = kwargs['server_version']  # AMI Role
        self.is_local_run = (self.server is not None)
        self.server_numeric_version = self._get_server_numeric_version()
        self.instances_ips = self._get_instances_ips()
        self.filtered_tests = self._extract_filtered_tests()
        self.tests_data_keeper = TestResults(self.conf.unmockable_integrations)
        self.conf_unmockable_tests = self._get_unmockable_tests_from_conf()
        self.unmockable_test_ids: Set[str] = set()
        self.mockable_tests_to_run, self.unmockable_tests_to_run = self._get_tests_to_run()
        self.test_retries_queue: Queue = Queue()
        self.slack_user_id = self._retrieve_slack_user_id()
        self.all_integrations_configurations = self._get_all_integration_config(self.instances_ips)

    def _get_all_integration_config(self, instances_ips: dict) -> Optional[list]:
        """
        Gets all integration configuration as it exists on the demisto server
        Since in all packs are installed the data returned from this request is very heavy and we want to avoid
        running it in multiple threads.
        Args:
            instances_ips: The mapping of the urls to the ports used to tunnel it's traffic

        Returns:
            A dict containing the configuration for the integration if found, else empty list
        """
        if not self.is_nightly:
            return []
        url, port = list(instances_ips.items())[0]
        if IS_XSIAM:
            server_url = url
        else:
            server_url = f'https://localhost:{port}' if port else f'https://{url}'
        return self.get_all_installed_integrations_configurations(server_url)

    def get_all_installed_integrations_configurations(self, server_url: str) -> list:
        """
        Gets all integration configuration as it exists on the demisto server
        Args:
            server_url: The url of the server to create integration in

        Returns:
            A dict containing the configuration for the integration if found else empty list
        """
        if IS_XSIAM:
            # in xsiam we dont use demisto username
            os.environ.pop('DEMISTO_USERNAME', None)
        tmp_client = demisto_client.configure(base_url=server_url,
                                              auth_id=self.auth_id,
                                              api_key=self.api_key,
                                              verify_ssl=False)
        self.logging_module.debug('Getting all integrations instances')
        try:
            res_raw = demisto_client.generic_request_func(self=tmp_client, path='/settings/integration/search',
                                                          method='POST', body={})
        except ApiException:
            self.logging_module.exception('failed to get all integrations configuration')
            return []
        res = ast.literal_eval(res_raw[0])
        TIMEOUT = 180
        SLEEP_INTERVAL = 5
        total_sleep = 0
        while 'configurations' not in res:
            if total_sleep == TIMEOUT:
                self.logging_module.error(
                    f"Timeout - failed to get all integration configuration. Error: {res}")
                return []

            time.sleep(SLEEP_INTERVAL)
            total_sleep += SLEEP_INTERVAL
        all_configurations = res['configurations']
        return all_configurations

    def _generate_tests_queue(self, tests_to_run: List[TestConfiguration]) -> Queue:
        """
        Generates a queue containing test playbooks to run
        Args:
            tests_to_run: A list containing playbook names
        """
        queue: Queue = Queue()
        for test in tests_to_run:
            playbook = TestPlaybook(self, test)
            if playbook.should_test_run():
                queue.put(playbook)
        return queue

    def _get_tests_to_run(self) -> Tuple[Queue, Queue]:
        """
        Gets tests to run in the current build and updates the unmockable tests ids set
        Returns:
            - A queue with mockable TestPlaybook instances to run in the current build
            - A queue with unmockable TestPlaybook instances to run in the current build
        """
        unmockable_tests = []
        all_tests = self._get_all_tests()
        # for xsiam, set all tests to unmockable.
        if self.server or not self.isAMI or self.is_xsiam:
            unmockable_tests = all_tests
            self.unmockable_test_ids = {test.playbook_id for test in all_tests}
        elif self.isAMI:
            unmockable_tests = self.conf_unmockable_tests
            self.unmockable_test_ids = {test.playbook_id for test in self.conf_unmockable_tests}
        self.logging_module.debug(f'Unmockable tests selected: {pformat(self.unmockable_test_ids)}')
        mockable_tests = [test for test in all_tests if test.playbook_id not in self.unmockable_test_ids]
        self.logging_module.debug(f'Mockable tests selected: {pformat([test.playbook_id for test in mockable_tests])}')
        mockable_tests_queue = self._generate_tests_queue(mockable_tests)
        unmockable_tests_queue = self._generate_tests_queue(unmockable_tests)
        return mockable_tests_queue, unmockable_tests_queue

    @staticmethod
    def _extract_filtered_tests() -> list:
        """
        Reads the content from ./artifacts/filter_file.txt and parses it into a list of test playbook IDs that should be run
        in the current build
        Returns:
            A list of playbook IDs that should be run in the current build
        """
        with open(FILTER_CONF, 'r') as filter_file:
            filtered_tests = [line.strip('\n') for line in filter_file.readlines()]

        return filtered_tests

    def _get_unmockable_tests_from_conf(self) -> list:
        """
        Extracts the unmockable test playbook by looking at all playbooks that has unmockable integrations
        Returns:
            A with unmockable playbook names
        """
        unmockable_integrations = self.conf.unmockable_integrations
        tests = self.conf.tests
        unmockable_tests = []
        for test_record in tests:
            test_name = test_record.playbook_id
            unmockable_integrations_used = [integration_name for integration_name in test_record.test_integrations if
                                            integration_name in unmockable_integrations]
            if test_name and (not test_record.test_integrations or unmockable_integrations_used) or not self.isAMI:
                unmockable_tests.append(test_record)
                # In case a test has both - an unmockable integration and is configured with is_mockable=False -
                # it will be added twice if we don't continue.
                continue
            if test_record.is_mockable is False:
                unmockable_tests.append(test_record)
        return unmockable_tests

    @staticmethod
    def _get_used_integrations(test_configuration: dict) -> list:
        """
        Gets the integration used in a test playbook configurations
        Args:
            test_configuration: A test configuration from content conf.json file.

        Returns:
            A list with integration names
        """
        tested_integrations = test_configuration.get("integrations", [])
        if isinstance(tested_integrations, list):
            return tested_integrations
        else:
            return [tested_integrations]

    def _get_all_tests(self) -> List[TestConfiguration]:
        """
        Gets a list of all playbooks configured in content conf.json
        Returns:
            A list with playbook names
        """
        tests_records = self.conf.tests
        return [test for test in tests_records if test.playbook_id]

    def _get_instances_ips(self) -> Dict[str, Any]:
        """
        Parses the env_results.json and extracts the instance ip from each server configured in it.
        Returns:
            A dict contains a mapping from server internal ip to the port used to tunnel it.
        """
        if self.server:
            return {self.server: None}
        if self.is_xsiam:
            return {env.get('base_url'): None for env in self.env_json}
        instances_ips = {env.get('InstanceDNS'): env.get('TunnelPort') for env in self.env_json if
                         env.get('Role') == self.server_version}
        return instances_ips

    def get_public_ip_from_server_url(self, server_url: str) -> str:
        """
        Gets a tunnel server url in the form of https://localhost:<port>, from that url checks in self.instance_ips
        if there is a url that is mapped into that port and return that url if found.
        Args:
            server_url: The server url to parse the port from.

        Returns:
            A URL with the private IP of the server.
        """
        port_pattern = re.compile(r'https://localhost:([0-9]+)')
        port_match = port_pattern.findall(server_url)
        if port_match:
            port = int(port_match[0])
        else:
            # If the server URL has no port in the end - it means it's a local build or XSIAM, and we can return the
            # server URL as is.
            return server_url
        for server_private_ip, tunnel_port in self.instances_ips.items():
            if tunnel_port == port:
                return f'https://{server_private_ip}'
        raise Exception(f'Could not find private ip for the server mapped to port {port}')

    @staticmethod
    def _parse_tests_list_arg(tests_list: str):
        """
        Parses the test list arguments if present.

        :param tests_list: CSV string of tests to run.
        :return: List of tests if there are any, otherwise empty list.
        """
        tests_to_run = tests_list.split(",") if tests_list else []
        return tests_to_run

    @staticmethod
    def _load_env_results_json():
        if not os.path.isfile(ENV_RESULTS_PATH):
            return {}

        with open(ENV_RESULTS_PATH, 'r') as json_file:
            return json.load(json_file)

    def _get_server_numeric_version(self) -> str:
        """
        Gets the current server version

        Returns:
            Server numeric version
        """
        default_version = '99.99.98'
        if self.is_local_run:
            self.logging_module.info(f'Local run, assuming server version is {default_version}', real_time=True)
            return default_version

        if not self.env_json:
            self.logging_module.warning(
                f'Did not find {ENV_RESULTS_PATH} file, assuming server version is {default_version}.',
                real_time=True)
            return default_version

        server_version_mapping = {
            'Server 5.0': '5.0.0',
            'Server 5.5': '5.5.0',
            'Server 6.0': '6.0.0',
            'Server 6.1': '6.1.0',
            'Server 6.2': '6.2.0',
            'Server 6.5': '6.5.0',
            'Server 6.6': '6.6.0',
            'Server 6.8': '6.8.0',
            'Server 6.9': '6.9.0',
            'Server Master': default_version,
            'XSIAM 1.2': '6.9.0',
            'XSIAM Master': default_version
        }
        server_numeric_version = server_version_mapping.get(self.server_version, default_version)
        self.logging_module.info(f'Server version: {server_numeric_version}', real_time=True)
        return server_numeric_version

    @staticmethod
    def _load_xsiam_file(xsiam_servers_path):
        conf = None
        if xsiam_servers_path:
            with open(xsiam_servers_path) as data_file:
                conf = json.load(data_file)

        return conf

    @staticmethod
    def _load_conf_files(conf_path, secret_conf_path):
        with open(conf_path) as data_file:
            conf = Conf(json.load(data_file))

        secret_conf = None
        if secret_conf_path:
            with open(secret_conf_path) as data_file:
                secret_conf = SecretConf(json.load(data_file))

        return conf, secret_conf

    def _get_user_name_from_circle(self):
        url = f'https://circleci.com/api/v1.1/project/github/demisto/content/{self.build_number}'
        res = self._http_request(url, params_dict={'circle-token': self.circleci_token})

        user_details = res.get('user', {})
        return user_details.get('name', '')

    @staticmethod
    def _http_request(url, params_dict=None):
        res = requests.request("GET",
                               url,
                               verify=True,
                               params=params_dict,
                               )
        res.raise_for_status()

        return res.json()

    def _retrieve_slack_user_id(self):
        """
        Gets the user id of the circle user who triggered the current build
        """
        user_id = ''
        try:
            user_name = os.getenv('GITLAB_USER_LOGIN') or self._get_user_name_from_circle()
            res = self.slack_client.api_call('users.list')

            user_list = res.get('members', [])  # type: ignore
            for user in user_list:
                profile = user.get('profile', {})
                name = profile.get('real_name_normalized', '')
                if name == user_name:
                    user_id = user.get('id', '')
        except Exception as exc:
            logging.debug(f'failed to retrieve the slack user ID.\nError: {exc}')

        return user_id


class TestResults:

    def __init__(self, unmockable_integrations):
        self.succeeded_playbooks = []
        self.failed_playbooks = set()
        self.playbook_report = dict()
        self.skipped_tests = dict()
        self.skipped_integrations = dict()
        self.rerecorded_tests = []
        self.empty_files = []
        self.unmockable_integrations = unmockable_integrations
        self.playbook_skipped_integration = set()

    def add_proxy_related_test_data(self, proxy):
        # Using multiple appends and not extend since append is guaranteed to be thread safe
        for playbook_id in proxy.rerecorded_tests:
            self.rerecorded_tests.append(playbook_id)
        for playbook_id in proxy.empty_files:
            self.empty_files.append(playbook_id)

    def create_result_files(self):
        with open("./Tests/succeeded_tests.txt", "w") as succeeded_tests_file:
            succeeded_tests_file.write('\n'.join(self.succeeded_playbooks))
        with open("./Tests/failed_tests.txt", "w") as failed_tests_file:
            failed_tests_file.write('\n'.join(self.failed_playbooks))
        with open('./Tests/skipped_tests.txt', "w") as skipped_tests_file:
            skipped_tests_file.write('\n'.join(self.skipped_tests))
        with open('./Tests/skipped_integrations.txt', "w") as skipped_integrations_file:
            skipped_integrations_file.write('\n'.join(self.skipped_integrations))
        with open('./Tests/test_playbooks_report.json', "w") as test_playbooks_report_file:
            json.dump(self.playbook_report, test_playbooks_report_file, indent=4)

    def print_test_summary(self,
                           is_ami: bool = True,
                           logging_module: Union[Any, ParallelLoggingManager] = logging) -> None:
        """
        Takes the information stored in the tests_data_keeper and prints it in a human readable way.
        Args:
            is_ami: indicating if the server running the tests is an AMI or not.
            logging_module: Logging module to use for test_summary
        """
        succeed_playbooks = self.succeeded_playbooks
        failed_playbooks = self.failed_playbooks
        skipped_tests = self.skipped_tests
        unmocklable_integrations = self.unmockable_integrations
        skipped_integration = self.skipped_integrations
        rerecorded_tests = self.rerecorded_tests
        empty_files = self.empty_files

        succeed_count = len(succeed_playbooks)
        failed_count = len(failed_playbooks)
        rerecorded_count = len(rerecorded_tests) if is_ami else 0
        empty_mocks_count = len(empty_files) if is_ami else 0
        logging_module.real_time_logs_only = True
        logging_module.info('TEST RESULTS:')
        logging_module.info(f'Number of playbooks tested - {succeed_count + failed_count}')
        if failed_count:
            logging_module.error(f'Number of failed tests - {failed_count}:')
            logging_module.error('Failed Tests: {}'.format(
                ''.join([f'\n\t\t\t\t\t\t\t - {playbook_id}' for playbook_id in failed_playbooks])))
        if succeed_count:
            logging_module.success(f'Number of succeeded tests - {succeed_count}')
            logging_module.success('Successful Tests: {}'.format(
                ''.join([f'\n\t\t\t\t\t\t\t - {playbook_id}' for playbook_id in succeed_playbooks])))
        if rerecorded_count > 0:
            logging_module.debug(
                f'Number of tests with failed playback and successful re-recording - {rerecorded_count}')
            logging_module.debug('Tests with failed playback and successful re-recording: {}'.format(
                ''.join([f'\n\t\t\t\t\t\t\t - {playbook_id}' for playbook_id in rerecorded_tests])))

        if empty_mocks_count > 0:
            logging_module.debug(f'Successful tests with empty mock files count- {empty_mocks_count}:\n')
            proxy_explanation = \
                '\t\t\t\t\t\t\t (either there were no http requests or no traffic is passed through the proxy.\n' \
                '\t\t\t\t\t\t\t Investigate the playbook and the integrations.\n' \
                '\t\t\t\t\t\t\t If the integration has no http traffic, add to unmockable_integrations in conf.json)'
            logging_module.debug(proxy_explanation)
            logging_module.debug('Successful tests with empty mock files: {}'.format(
                ''.join([f'\n\t\t\t\t\t\t\t - {playbook_id}' for playbook_id in empty_files])))

        if skipped_integration:
            self.print_table("Skipped Integrations", skipped_integration, logging_module.debug)

        if skipped_tests:
            self.print_table("Skipped Tests", skipped_tests, logging_module.debug)

        if unmocklable_integrations:
            self.print_table("Unmockable Integrations", unmocklable_integrations, logging_module.debug)

    @staticmethod
    def print_table(table_name: str, table_data: dict, logging_method: Callable) -> None:
        table = prettytable.PrettyTable()
        table.field_names = ['Index', 'Name', 'Reason']
        for index, record in enumerate(table_data, start=1):
            row = [index, record, table_data[record]]
            table.add_row(row)
        logging_method(f'{table_name}:\n{table}', real_time=True)


class Integration:
    def __init__(self, build_context: BuildContext, integration_name: str, potential_integration_instance_names: list):
        """
        An integration class that should represent the integrations during the build
        Args:
            build_context: The context of the build
            integration_name: The name of the integration
            potential_integration_instance_names: A list of instance names, one of those names should be the actual reason
            but we won't know for sure until we will try to filter it with conf.json from content-test-conf repo
        """
        self.build_context = build_context
        self.name = integration_name
        self.instance_names = potential_integration_instance_names
        self.instance_name = ''
        self.configuration: Optional[IntegrationConfiguration] = IntegrationConfiguration(
            {'name': self.name,
             'params': {}})
        self.docker_image: list = []
        self.integration_configuration_from_server: dict = {}
        self.integration_type: str = ""
        self.module_instance: dict = {}

    @staticmethod
    def _change_placeholders_to_values(server_url: str,
                                       config_item: IntegrationConfiguration) -> IntegrationConfiguration:
        """Some integration should be configured on the current server as host and has the string '%%SERVER_HOST%%'
        in the content-test-conf conf.json configuration.
        This method replaces these placeholders in the configuration to their real values

        Args:
            server_url: The server url that should be inserted instead of the placeholder in the configuration params
            config_item: Integration configuration object.

        Returns:
            IntegrationConfiguration class with the modified params
        """
        placeholders_map = {'%%SERVER_HOST%%': server_url}
        item_as_string = json.dumps(config_item.params)
        for key, value in placeholders_map.items():
            item_as_string = item_as_string.replace(key, value)
        config_item.params = json.loads(item_as_string)
        return config_item

    def _set_integration_params(self, server_url: str, playbook_id: str, is_mockable: bool) -> bool:
        """
        Finds the matching configuration for the integration in content-test-data conf.json file
        in accordance with the configured instance name if exist and configures the proxy parameter if needed
        Args:
            server_url: The url of the demisto server to configure the integration in
            playbook_id: The ID of the playbook for which the integration should be configured
            is_mockable: Should the proxy parameter be set to True or not

        Returns:
            True if found a matching configuration else False if found more that one configuration candidate returns False
        """
        self.build_context.logging_module.debug(f'Searching integration configuration for {self}')

        # Finding possible configuration matches
        integration_params: List[IntegrationConfiguration] = [
            deepcopy(conf) for conf in
            self.build_context.secret_conf.integrations if
            conf.name == self.name
        ]
        # Modifying placeholders if exists
        integration_params: List[IntegrationConfiguration] = [
            self._change_placeholders_to_values(server_url, conf) for conf in integration_params]

        if integration_params:
            # If we have more then one configuration for this integration - we will try to filter by instance name
            if len(integration_params) != 1:
                found_matching_instance = False
                for item in integration_params:
                    if item.instance_name in self.instance_names:
                        self.configuration = item
                        found_matching_instance = True
                        break

                if not found_matching_instance:
                    optional_instance_names = [optional_integration.instance_name
                                               for optional_integration in integration_params]
                    error_msg = FAILED_MATCH_INSTANCE_MSG.format(playbook_id,
                                                                 len(integration_params),
                                                                 self.name,
                                                                 '\n'.join(optional_instance_names))
                    self.build_context.logging_module.error(error_msg)
                    return False
            else:
                self.configuration = integration_params[0]

        elif self.name == 'Demisto REST API':
            if IS_XSIAM:
                self.build_context.logging_module.warning('Trying to configure "Demisto REST API" for XSIAM server, '
                                                          'this integration will not work on XSIAM, '
                                                          'consider using CoreRestAPI.')
            self.configuration.params = {  # type: ignore
                'url': 'https://localhost',
                'apikey': self.build_context.api_key,
                'insecure': True,
            }
        if is_mockable:
            self.build_context.logging_module.debug(f'configuring {self} with proxy params')
            for param in ('proxy', 'useProxy', 'useproxy', 'insecure', 'unsecure'):
                self.configuration.params[param] = True  # type: ignore
        return True

    def _get_integration_config(self, server_url: str) -> Optional[dict]:
        """
        Gets integration configuration as it exists on the demisto server
        Args:
            server_url: The url of the server to create integration in

        Returns:
            A dict containing the configuration for the integration if found, else None
        """
        if self.build_context.all_integrations_configurations:
            match_configurations = [x for x in self.build_context.all_integrations_configurations if
                                    x['name'] == self.name]
        else:
            all_configurations = self.build_context.get_all_installed_integrations_configurations(server_url)
            match_configurations = [x for x in all_configurations if x['name'] == self.name]

        if not match_configurations:
            self.build_context.logging_module.error('integration was not found')
            return None

        return deepcopy(match_configurations[0])

    def _delete_integration_instance_if_determined_by_name(self, client: DefaultApi, instance_name: str) -> None:
        """Deletes integration instance by it's name.

        Args:
            client (demisto_client): The configured client to use.
            instance_name (str): The name of the instance to delete.

        Notes:
            This function is needed when the name of the instance is pre-defined in the tests configuration, and the test
            itself depends on the instance to be called as the `instance name`.
            In case we need to configure another instance with the same name, the client will throw an error, so we
            will call this function first, to delete the instance with this name.

        """
        try:
            int_resp = demisto_client.generic_request_func(self=client, method='POST',
                                                           path='/settings/integration/search',
                                                           body={'size': 1000})
            int_instances = ast.literal_eval(int_resp[0])
        except ApiException:
            self.build_context.logging_module.exception(
                f'Failed to delete integration {self} instance, error trying to communicate with demisto server')
            return
        if int(int_resp[1]) != 200:
            self.build_context.logging_module.error(
                f'Get integration {self} instance failed with status code: {int_resp[1]}')
            return
        if 'instances' not in int_instances:
            self.build_context.logging_module.info(f'No integrations instances found to delete for {self}')
            return

        for instance in int_instances['instances']:
            if instance.get('name') == instance_name:
                self.build_context.logging_module.info(
                    f'Deleting integration instance {instance_name} since it is defined by name')
                self.delete_integration_instance(client, instance.get('id'))

    def _set_server_keys(self, client: DefaultApi, server_context: 'ServerContext') -> None:
        """In case the the params of the test in the content-test-conf repo has 'server_keys' key:
            Resets containers
            Adds server configuration keys using the demisto_client.

        Args:
            client (demisto_client): The configured client to use.
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in
        """
        if 'server_keys' not in self.configuration.params:  # type: ignore
            return

        server_context._reset_containers()

        self.build_context.logging_module.debug(f'Setting server keys for integration: {self}')

        data = {
            'data': {},
            'version': -1
        }

        for key, value in self.configuration.params.get('server_keys').items():  # type: ignore
            data['data'][key] = value  # type: ignore

        _, _, prev_system_conf = update_server_configuration(
            client=client,
            server_configuration=self.configuration.params.get('server_keys'),  # type: ignore
            error_msg='Failed to set server keys',
            logging_manager=self.build_context.logging_module
        )
        server_context.prev_system_conf = prev_system_conf

    def create_module(self, instance_name: str, configuration: dict, incident_configuration: dict = None):
        module_configuration = configuration['configuration']

        # If incident_type is given in Test Playbook configuration on test-conf, we change the default configuration.
        if incident_configuration and incident_configuration.get('incident_type'):
            incident_type_configuration = list(
                filter(lambda config: config.get('name') == 'incidentType', module_configuration))

            incident_type_configuration[0]['value'] = incident_configuration.get('incident_type')

        module_instance = {
            'brand': configuration['name'],
            'category': configuration['category'],
            'configuration': configuration,
            'data': [],
            'enabled': "true",
            'engine': '',
            'id': '',
            'isIntegrationScript': self.configuration.is_byoi,  # type: ignore
            'name': instance_name,
            'passwordProtected': False,
            'version': 0,
            'incomingMapperId': configuration.get('defaultMapperIn', ''),
            'mappingId': configuration.get('defaultClassifier', ''),
            'outgoingMapperId': configuration.get('defaultMapperOut', '')
        }

        # If default mapper or classifier are given in test-conf we ignore defaultMapperIn or defaultClassifier from yml.
        if incident_configuration and incident_configuration.get('classifier_id'):
            module_instance['mappingId'] = incident_configuration.get('classifier_id')
        if incident_configuration and incident_configuration.get('incoming_mapper_id'):
            module_instance['incomingMapperId'] = incident_configuration.get('incoming_mapper_id')

        return module_instance

    def create_integration_instance(self,
                                    client: DefaultApi,
                                    playbook_id: str,
                                    is_mockable: bool,
                                    server_context: 'ServerContext',
                                    instance_configuration: dict
                                    ) -> bool:
        """
        Create an instance of the integration in the server specified in the demisto client instance.
        Args:
            client: The demisto_client instance to use
            playbook_id: The playbook id for which the instance should be created
            is_mockable: Indicates whether the integration should be configured with proxy=True or not
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in

        Returns:
            The integration configuration as it exists on the server after it was configured
        """
        server_url = self.build_context.get_public_ip_from_server_url(client.api_client.configuration.host)
        self._set_integration_params(server_url, playbook_id, is_mockable)
        configuration = self._get_integration_config(client.api_client.configuration.host)
        if not configuration:
            self.build_context.logging_module.error(f'Could not find configuration for integration {self}')
            return False

        module_configuration = configuration['configuration']
        if not module_configuration:
            module_configuration = []

        if 'integrationInstanceName' in self.configuration.params:  # type: ignore
            instance_name = self.configuration.params['integrationInstanceName']  # type: ignore
            self._delete_integration_instance_if_determined_by_name(client, instance_name)
        else:
            instance_name = f'{self.configuration.instance_name.replace(" ", "_")}_test_{uuid.uuid4()}'  # type: ignore

        self.build_context.logging_module.info(
            f'Configuring instance for {self} (instance name: {instance_name}, '  # type: ignore
            f'validate "test-module": {self.configuration.should_validate_test_module})'
        )

        # define module instance:
        params = self.configuration.params  # type: ignore

        module_instance = self.create_module(instance_name, configuration, instance_configuration)

        # set server keys
        if not IS_XSIAM:
            self._set_server_keys(client, server_context)

        # set module params
        for param_conf in module_configuration:
            if param_conf['display'] in params or param_conf['name'] in params:
                # param defined in conf
                key = param_conf['display'] if param_conf['display'] in params else param_conf['name']
                if key == 'credentials':
                    credentials = params[key]
                    param_value = {
                        'credential': '',
                        'identifier': credentials['identifier'],
                        'password': credentials['password'],
                        'passwordChanged': False
                    }
                else:
                    param_value = params[key]

                param_conf['value'] = param_value
                param_conf['hasvalue'] = True
            elif param_conf['defaultValue']:
                # param is required - take default value
                param_conf['value'] = param_conf['defaultValue']
            module_instance['data'].append(param_conf)
        try:
            res = demisto_client.generic_request_func(self=client, method='PUT',
                                                      path='/settings/integration',
                                                      body=module_instance)
        except ApiException:
            self.build_context.logging_module.exception(f'Error trying to create instance for integration: {self}')
            return False

        if res[1] != 200:
            self.build_context.logging_module.error(f'create instance failed with status code  {res[1]}')
            self.build_context.logging_module.error(pformat(res[0]))
            return False

        integration_config = ast.literal_eval(res[0])
        self.integration_configuration_from_server = integration_config
        self.module_instance = module_instance
        integration_script = integration_config.get('configuration', {}).get('integrationScript', {}) or {}
        self.integration_type = integration_script.get('type', '')
        return True

    def delete_integration_instance(self, client, instance_id: Optional[str] = None) -> bool:
        """
        Deletes an integration with the given ID
        Args:
            client: Demisto client instance
            instance_id: The instance ID to Delete

        Returns:
            True if integration was deleted else False
        """
        self.build_context.logging_module.debug(f'Deleting {self} instance')
        instance_id = instance_id or self.integration_configuration_from_server.get('id')
        if not instance_id:
            self.build_context.logging_module.debug(f'no instance ID for integration {self} was supplied')
            return True
        try:
            res = demisto_client.generic_request_func(self=client, method='DELETE',
                                                      path='/settings/integration/' + urllib.parse.quote(
                                                          instance_id))
        except ApiException:
            self.build_context.logging_module.exception(
                'Failed to delete integration instance, error trying to communicate with demisto.')
            return False
        if int(res[1]) != 200:
            self.build_context.logging_module.error(f'delete integration instance failed\nStatus code {res[1]}')
            self.build_context.logging_module.error(pformat(res))
            return False
        if self.module_instance:
            self.module_instance = {}
        return True

    def test_integration_instance(self, client: DefaultApi) -> bool:
        """Runs test module on the integration instance
        Args:
            client: The demisto_client instance to use

        Returns:
            The integration configuration as it exists on the server after it was configured
        """
        if not self.configuration.should_validate_test_module:  # type: ignore
            self.build_context.logging_module.debug(
                f'Skipping test-module on {self} because the "validate_test" flag is set to False')
            return True

        connection_retries = 3
        response_code = 0
        integration_of_instance = self.integration_configuration_from_server.get('brand', '')
        instance_name = self.integration_configuration_from_server.get('name', '')
        self.build_context.logging_module.info(
            f'Running "test-module" for instance "{instance_name}" of integration "{integration_of_instance}".')
        for i in range(connection_retries):
            try:
                response_data, response_code, _ = demisto_client.generic_request_func(self=client, method='POST',
                                                                                      path='/settings/integration/test',
                                                                                      body=self.module_instance,
                                                                                      _request_timeout=120)
                break
            except ApiException:
                self.build_context.logging_module.exception(
                    f'Failed to test integration {self} instance, error trying to communicate with demisto server: '
                    f'{get_ui_url(client.api_client.configuration.host)}')
                return False
            except urllib3.exceptions.ReadTimeoutError:
                self.build_context.logging_module.warning(f"Could not connect. Trying to connect for the {i + 1} time")

        if int(response_code) != 200:
            self.build_context.logging_module.error(
                f'Integration-instance test-module failed. Bad status code: {response_code}.\n'
                f'Sever URL: {get_ui_url(client.api_client.configuration.host)}')
            return False

        result_object = ast.literal_eval(response_data)
        success, failure_message = bool(result_object.get('success')), result_object.get('message')
        if not success:
            server_url = get_ui_url(client.api_client.configuration.host)
            test_failed_msg = f'Test integration failed - server: {server_url}.\n' \
                              f'Failure message: {failure_message}' if failure_message else ' No failure message.'
            self.build_context.logging_module.error(test_failed_msg)
        return success

    def disable_integration_instance(self, client) -> None:
        """Disables the integration
        Args:
            client: The demisto_client instance to use

        Returns:
            The integration configuration as it exists on the server after it was configured
        """
        # tested with POSTMAN, this is the minimum required fields for the request.
        module_instance = {
            key: self.integration_configuration_from_server[key] for key in
            ['id', 'brand', 'name', 'data', 'isIntegrationScript', ]
        }
        module_instance['enable'] = "false"
        module_instance['version'] = -1
        self.build_context.logging_module.debug(f'Disabling integration instance "{module_instance.get("name")}"')
        try:
            res = demisto_client.generic_request_func(self=client, method='PUT',
                                                      path='/settings/integration',
                                                      body=module_instance)
        except ApiException:
            self.build_context.logging_module.exception('Failed to disable integration instance')
            return

        if res[1] != 200:
            self.build_context.logging_module.error(f'disable instance failed, Error: {pformat(res)}')

    def get_docker_images(self) -> List[str]:
        """
        Gets the docker image name from the configured integration instance's body if such body exists
        Returns:

        """
        if self.integration_configuration_from_server:
            return Docker.get_integration_image(self.integration_configuration_from_server)
        else:
            raise Exception('Cannot get docker image - integration instance was not created yet')

    def __str__(self):
        return f'"{self.name}"'

    def __repr__(self):
        return str(self)


class TestContext:
    def __init__(self,
                 build_context: BuildContext,
                 playbook: TestPlaybook,
                 client: DefaultApi,
                 server_context: 'ServerContext'):
        """
        Initializes the TestContext class
        Args:
            build_context: The context of the current build
            playbook: The TestPlaybook instance to run in the current test execution
            client: A demisto client instance to use for communication with the server
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in
        """
        self.build_context = build_context
        self.server_context = server_context
        self.playbook = playbook
        self.incident_id: Optional[str] = None
        self.test_docker_images: Set[str] = set()
        self.client: DefaultApi = client
        if IS_XSIAM:
            self.tunnel_command = ''
        else:
            self.tunnel_command = \
                f'ssh -i ~/.ssh/oregon-ci.pem -4 -o StrictHostKeyChecking=no -f -N ' \
                f'"{CONTENT_BUILD_SSH_USER}@{LOAD_BALANCER_DNS}" ' \
                f'-L "{self.server_context.tunnel_port}:{self.server_context.server_ip}:443"'

    def _get_investigation_playbook_state(self) -> str:
        """
        Queried the server for the current status of the test's investigation
        Returns:
            A string representing the status of the playbook
        """
        try:
            investigation_playbook_raw = demisto_client.generic_request_func(
                self=self.client,
                method='GET',
                path=f'/inv-playbook/{self.incident_id}')
            investigation_playbook = ast.literal_eval(investigation_playbook_raw[0])
        except ApiException:
            self.build_context.logging_module.exception(
                'Failed to get investigation playbook state, error trying to communicate with demisto server'
            )
            return PB_Status.FAILED

        try:
            state = investigation_playbook['state']
            return state
        except Exception:  # noqa: E722
            # setting state to `in progress` in XSIAM build,
            # Because `investigation_playbook` returned empty if xsiam investigation is still in progress.
            if IS_XSIAM:
                return PB_Status.IN_PROGRESS
            return PB_Status.NOT_SUPPORTED_VERSION

    def _collect_docker_images(self) -> None:
        """
        Collects docker images of the playbook's integration.
        This method can be called only after the integrations were configured in the server.
        """
        for integration in self.playbook.integrations:
            docker_images = integration.get_docker_images()
            if docker_images:
                self.test_docker_images.update(docker_images)

    def _print_investigation_error(self):
        try:
            res = demisto_client.generic_request_func(
                self=self.client,
                method='POST',
                path='/investigation/' + urllib.parse.quote(self.incident_id),  # type: ignore
                body={"pageSize": 1000})
            if res and int(res[1]) == 200:
                resp_json = ast.literal_eval(res[0])
                entries = resp_json['entries']

                self.build_context.logging_module.error(f'Playbook {self.playbook} has failed:')
                for entry in entries:
                    if entry['type'] == ENTRY_TYPE_ERROR and entry['parentContent']:
                        self.build_context.logging_module.error(f'- Task ID: {entry["taskId"]}')
                        # Checks for passwords and replaces them with "******"
                        parent_content = re.sub(
                            r' (P|p)assword="[^";]*"', ' password=******', entry['parentContent'])
                        self.build_context.logging_module.error(f'  Command: {parent_content}')
                        self.build_context.logging_module.error(f'  Body:\n{entry["contents"]}')
            else:
                self.build_context.logging_module.error(
                    f'Failed getting entries for investigation: {self.incident_id}. Res: {res}')
        except ApiException:
            self.build_context.logging_module.exception(
                'Failed to print investigation error, error trying to communicate with demisto server')

    def _poll_for_playbook_state(self) -> str:
        """
        Polls for the playbook execution in the incident and return it's state.
        Returns:
            A string representing the status of the playbook
        """
        timeout = time.time() + self.playbook.configuration.timeout
        number_of_attempts = 1
        # wait for playbook to finish run
        while True:
            # give playbook time to run
            time.sleep(5)
            try:
                # fetch status
                playbook_state = self._get_investigation_playbook_state()
            except demisto_client.demisto_api.rest.ApiException:
                playbook_state = 'Pending'
                self.build_context.logging_module.exception('Error when trying to get investigation playbook state')

            if playbook_state in (PB_Status.COMPLETED, PB_Status.NOT_SUPPORTED_VERSION):
                break
            if playbook_state == PB_Status.FAILED:
                self.build_context.logging_module.error(f'{self.playbook} failed with error/s')
                self._print_investigation_error()
                break
            if time.time() > timeout:
                self.build_context.logging_module.error(f'{self.playbook} failed on timeout')
                break

            if number_of_attempts % DEFAULT_INTERVAL == 0:
                self.build_context.logging_module.info(
                    f'{self.playbook} loop no. {number_of_attempts / DEFAULT_INTERVAL}, {playbook_state=}')
            number_of_attempts = number_of_attempts + 1
        return playbook_state

    def _run_incident_test(self) -> str:
        """
        Creates an incident in demisto server and return its status
        Returns:
            Empty string or
        """
        try:
            if not self.build_context.is_xsiam:
                self.build_context.logging_module.info(f'ssh tunnel command:\n{self.tunnel_command}')
            instance_configuration = self.playbook.configuration.instance_configuration

            if not self.playbook.configure_integrations(self.client, self.server_context, instance_configuration):
                return PB_Status.CONFIGURATION_FAILED

            test_module_result = self.playbook.run_test_module_on_integrations(self.client)
            if not test_module_result:
                self.playbook.disable_integrations(self.client, self.server_context)
                return PB_Status.FAILED

            external_playbook_configuration = self.playbook.configuration.external_playbook_config

            # Change Configuration for external configuration if needed
            restore_needed, default_vals, restore_path = replace_external_playbook_configuration(
                self.client, external_playbook_configuration)

            incident = self.playbook.create_incident(self.client)
            if not incident:
                return ''

            self.incident_id = incident.id if IS_XSIAM else incident.investigation_id
            investigation_id = self.incident_id
            if investigation_id is None:
                self.build_context.logging_module.error(f'Failed to get investigation id of incident: {incident}')
                return ''

            self.build_context.logging_module.info(f'Found incident with incident ID: {investigation_id}.')

            server_url = get_ui_url(self.client.api_client.configuration.host)
            if IS_XSIAM:
                self.build_context.logging_module.info(
                    f'Investigation URL: {self.build_context.xsiam_ui_path}incident-view/alerts_and_insights?caseId='
                    f'{investigation_id}&action:openAlertDetails={investigation_id}-work_plan')
            else:
                self.build_context.logging_module.info(f'Investigation URL: {server_url}/#/WorkPlan/{investigation_id}')
            playbook_state = self._poll_for_playbook_state()
            self.build_context.logging_module.info(f'Got incident: {investigation_id} status: {playbook_state}.')
            if self.playbook.configuration.context_print_dt:
                self.playbook.print_context_to_log(self.client, investigation_id)

            # restore Configuration for external playbook
            if restore_needed:
                restore_external_playbook_configuration(self.client, restore_path=restore_path,
                                                        restore_values=default_vals)

            self.playbook.disable_integrations(self.client, self.server_context)
            self._clean_incident_if_successful(playbook_state)
            return playbook_state
        except Exception:
            self.build_context.logging_module.exception(f'Failed to run incident test for {self.playbook}')
            return PB_Status.FAILED

    def _clean_incident_if_successful(self, playbook_state: str) -> None:
        """
        Deletes the integration instances and the incident if the test was successful or failed on docker rate limit
        Args:
            playbook_state: The state of the playbook with which we can check if the test was successful
        """
        test_passed = playbook_state in (PB_Status.COMPLETED, PB_Status.NOT_SUPPORTED_VERSION)
        # batchDelete is not supported in XSIAM, only close.
        # in XSAIAM we are closing both successful and failed incidents
        if IS_XSIAM and self.incident_id:
            self.playbook.close_incident(self.client, self.incident_id)
            self.playbook.delete_integration_instances(self.client)
        elif self.incident_id and test_passed:
            self.playbook.delete_incident(self.client, self.incident_id)
            self.playbook.delete_integration_instances(self.client)

    def _run_docker_threshold_test(self):
        self._collect_docker_images()
        if self.test_docker_images:
            memory_threshold, pid_threshold = self.get_threshold_values()
            error_message = Docker.check_resource_usage(
                server_url=self.server_context.server_ip,
                docker_images=self.test_docker_images,
                def_memory_threshold=memory_threshold,
                def_pid_threshold=pid_threshold,
                docker_thresholds=self.build_context.conf.docker_thresholds,
                logging_module=self.build_context.logging_module)
            if error_message:
                self.build_context.logging_module.error(error_message)
                return False
        return True

    def get_threshold_values(self) -> Tuple[int, int]:
        """
        Gets the memory and pid threshold values to enforce on the current test.
        If one of the playbook's integrations is a powershell integration - the threshold have to be equals or
        higher than the default powershell threshold value.
        Returns:
            - The memory threshold value
            - The pid threshold value
        """
        memory_threshold = self.playbook.configuration.memory_threshold
        pid_threshold = self.playbook.configuration.pid_threshold
        has_pwsh_integration = any(integration for integration in self.playbook.integrations if
                                   integration.integration_type == Docker.POWERSHELL_INTEGRATION_TYPE)
        if has_pwsh_integration:
            memory_threshold = max(Docker.DEFAULT_PWSH_CONTAINER_MEMORY_USAGE, memory_threshold)
            pid_threshold = max(Docker.DEFAULT_PWSH_CONTAINER_PIDS_USAGE, pid_threshold)
        return memory_threshold, pid_threshold

    def _send_slack_message(self, channel, text, user_name, as_user):
        self.build_context.slack_client.api_call(
            "chat.postMessage",
            json={
                'channel': channel,
                'username': user_name,
                'as_user': as_user,
                'text': text,
                'mrkdwn': 'true'
            }
        )

    def _notify_failed_test(self):
        text = f'{self.build_context.build_name} - {self.playbook} Failed\n' \
               f'for more details run: `{self.tunnel_command}` and browse into the following link\n' \
               f'{get_ui_url(self.client.api_client.configuration.host)}'
        text += f'/#/WorkPlan/{self.incident_id}' if self.incident_id else ''
        if self.build_context.slack_user_id:
            self.build_context.slack_client.api_call(
                "chat.postMessage",
                json={
                    'channel': self.build_context.slack_user_id,
                    'username': 'Content CircleCI',
                    'as_user': 'False',
                    'text': text
                }
            )

    def _add_to_succeeded_playbooks(self) -> None:
        """
        Adds the playbook to the succeeded playbooks list
        """
        self.build_context.tests_data_keeper.succeeded_playbooks.append(self.playbook.configuration.playbook_id)

    def _add_details_to_failed_tests_report(self, playbook_name: str, failed_stage: str) -> None:
        """
        Adds the relevant details to the failed tests report.

        Args:
            playbook_name: The test's name.
            failed_stage: The stage where the test failed.
        """
        self.build_context.tests_data_keeper.playbook_report.setdefault(playbook_name, []).append({
            'number_of_executions': self.playbook.configuration.number_of_executions,
            'number_of_successful_runs': self.playbook.configuration.number_of_successful_runs,
            'failed_stage': failed_stage,
        })

    @staticmethod
    def _get_failed_stage(status: Optional[str], is_second_playback_run: bool = False) -> str:
        """
        Gets the test failed stage.

        Args:
            status: what is the test status.
            is_second_playback_run: Is The playbook run on a second playback after a freshly created record.
        """
        if is_second_playback_run:
            return 'Second playback'
        if status == PB_Status.FAILED_DOCKER_TEST:
            return 'Docker test'
        if status == PB_Status.CONFIGURATION_FAILED:
            return 'Configuration'
        return 'Execution'

    def _add_to_failed_playbooks(self, is_second_playback_run: bool = False, status: Optional[str] = None) -> None:
        """
        Adds the playbook to the failed playbooks list

        Args:
            is_second_playback_run: Is The playbook run on a second playback after a freshly created record
        """
        failed_stage = self._get_failed_stage(status, is_second_playback_run)
        playbook_name_to_add = self.playbook.configuration.playbook_id
        if not self.playbook.is_mockable:
            playbook_name_to_add += " (Mock Disabled)"
        if is_second_playback_run:
            self.build_context.logging_module.error(
                'Playback on newly created record has failed, see the following confluence page for help:\n'
                'https://confluence.paloaltonetworks.com/display/DemistoContent/Debug+Proxy-Related+Test+Failures')
            playbook_name_to_add += ' (Second Playback)'

        self._add_details_to_failed_tests_report(self.playbook.configuration.playbook_id, failed_stage)
        self.build_context.logging_module.error(f'Test failed: {self}')
        self.build_context.tests_data_keeper.failed_playbooks.add(playbook_name_to_add)

    @staticmethod
    def _get_circle_memory_data() -> Tuple[str, str]:
        """
        Checks how many bytes are currently in use in the circle build instance
        Returns:
            The number of bytes in use
        """
        process = subprocess.Popen(['cat', '/sys/fs/cgroup/memory/memory.usage_in_bytes'], stdout=subprocess.PIPE,
                                   stderr=subprocess.STDOUT)
        stdout, stderr = process.communicate()
        return stdout.decode(), stderr.decode()

    @staticmethod
    def _get_circle_processes_data() -> Tuple[str, str]:
        """
        Returns some data about the processes currently running in the circle build instance
        """
        process = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        stdout, stderr = process.communicate()
        return stdout.decode(), stderr.decode()

    def _send_circle_memory_and_pid_stats_on_slack(self):
        """
        Sends a slack messages with the number of bytes currently in use and the number of processes currently in use
        """
        if self.build_context.is_nightly and self.build_context.memCheck and not self.build_context.is_local_run:
            stdout, stderr = self._get_circle_memory_data()
            text = 'Memory Usage: {}'.format(stdout) if not stderr else stderr
            self._send_slack_message(SLACK_MEM_CHANNEL_ID, text, 'Content CircleCI', 'False')
            stdout, stderr = self._get_circle_processes_data()
            text = stdout if not stderr else stderr
            self._send_slack_message(SLACK_MEM_CHANNEL_ID, text, 'Content CircleCI', 'False')

    def _incident_and_docker_test(self) -> str:
        """
        Runs incident and docker tests (the docker test is optional)
        Returns:
            The result of the test.
        """
        playbook_state = self._run_incident_test()
        # We don't want to run docker tests on redhat instance because it does not use docker and it does not support
        # the threshold configurations.
        if playbook_state == PB_Status.COMPLETED and self.server_context.is_instance_using_docker:
            #  currently not supported on XSIAM (etc/#47908)
            if not IS_XSIAM:
                docker_test_results = self._run_docker_threshold_test()
                if not docker_test_results:
                    playbook_state = PB_Status.FAILED_DOCKER_TEST
                self._clean_incident_if_successful(playbook_state)
        return playbook_state

    def _execute_unmockable_test(self) -> bool:
        """
        Executes an unmockable test
        In case the test failed to execute because one of the test's integrations were locked will return False
        as indication that the test was not done.
        Returns:
            True if test has finished it's execution else False
        """
        self.build_context.logging_module.info(f'------ Test {self} start ------ (Mock: Disabled)')
        with acquire_test_lock(self.playbook) as lock:
            if lock:
                status = self._incident_and_docker_test()
                self.playbook.configuration.number_of_executions += 1
                self._update_playbook_status(status)
            else:
                return False
        return True

    def _update_complete_status(self, is_first_execution: bool, is_record_run: bool, is_first_playback_run: bool,
                                is_second_playback_run: bool, use_retries_mechanism: bool,
                                number_of_executions: int):
        """
        Updates (if necessary) the playbook status in case the original status is complete.
        Args:
            is_first_execution: whether it is the first execution or not.
            is_record_run: whether it is a record execution or not.
            is_first_playback_run: whether it is the first playback execution or not.
            is_second_playback_run: whether it is the second playback execution or not.
            use_retries_mechanism: whether to use the retries mechanism or not.
            number_of_executions: the number of times the test was executed.
        Returns:
            PB_Status.COMPLETED if the Test-Playbook passed successfully and was added to succeeded playbooks.
            PB_Status.FAILED if the Test-Playbook failed and was added to failed playbooks.
            PB_Status.SECOND_PLAYBACK_REQUIRED if second playback is needed.
            PB_Status.IN_PROGRESS if more executions are needed in order to determine whether the playbook is successful or not.
        """

        self.build_context.logging_module.success(f'PASS: {self} succeed')

        # count successful run only when recording mockable or executing unmockable test.
        if not is_first_playback_run and not is_second_playback_run:
            self.playbook.configuration.number_of_successful_runs += 1

        # mockable test, first record passed, still need to check second playback.
        if is_first_execution and is_record_run:
            return PB_Status.SECOND_PLAYBACK_REQUIRED

        # first/last playback passed (mockable tests) or first execution passed (unmockable tests). adding to succeeded
        # playbooks, no more executions are needed.
        if is_first_execution or is_first_playback_run or is_second_playback_run:
            return PB_Status.COMPLETED

        if use_retries_mechanism:
            return self._update_status_based_on_retries_mechanism(number_of_executions, is_record_run)

        return PB_Status.COMPLETED

    def _update_failed_status(self, is_record_run: bool, is_first_playback_run: bool, is_second_playback_run: bool,
                              use_retries_mechanism: bool, number_of_executions: int):
        """
        Handles the playbook failed status
        - Logs according to the results
        Args:
            is_record_run: whether it is a record execution.
            is_first_playback_run: whether it is the first playback execution.
            is_second_playback_run: whether it is the second playback execution.
            use_retries_mechanism: whether to use the retries mechanism.
            number_of_executions: how many times the test was executed.
        Returns:
            PB_Status.COMPLETED if the Test-Playbook passed successfully and was added to succeeded playbooks.
            PB_Status.FAILED if the Test-Playbook failed and was added to failed playbooks.
            PB_Status.SECOND_PLAYBACK_REQUIRED if second playback is needed.
            PB_Status.IN_PROGRESS if more executions are needed in order to determine whether the playbook is successful or not.
        """
        # in case the first playback run fails, the code should continue to record the playback.
        if is_first_playback_run:
            return PB_Status.IN_PROGRESS

        # in case the second playback run fails, test-playbook is considered a failure.
        if is_second_playback_run:
            return PB_Status.FAILED

        # in case of using the retries mechanism, the function should determine whether the test-playbook is considered
        # a failure or give it another try.
        if use_retries_mechanism:
            return self._update_status_based_on_retries_mechanism(number_of_executions, is_record_run)

        # the test-playbook is considered a failed playbook.
        return PB_Status.FAILED

    def _update_status_based_on_retries_mechanism(self, number_of_executions, is_record_run):
        """
        Updates the status of a test-playbook when using the retries mechanism.
        Args:
            number_of_executions: how many times the test was executed.
            is_record_run: whether it is a record execution.
        Returns:
            PB_Status.COMPLETED if the Test-Playbook passed successfully and was added to succeeded playbooks.
            PB_Status.FAILED if the Test-Playbook failed and was added to failed playbooks.
            PB_Status.SECOND_PLAYBACK_REQUIRED if second playback is needed.
            PB_Status.IN_PROGRESS if more executions are needed in order to determine whether the playbook is successful or not.
        """
        if number_of_executions < MAX_RETRIES:
            self.build_context.logging_module.info(
                f'Using the retries mechanism for test {self}.\n'
                f'Test-Playbook was executed {number_of_executions} times, more executions are needed.')
            self.build_context.test_retries_queue.put(self.playbook)
            return PB_Status.IN_PROGRESS

        else:  # number_of_executions == MAX_RETRIES:
            # check if in most executions, the test passed.
            if self.playbook.configuration.number_of_successful_runs >= RETRIES_THRESHOLD:
                # It's not enough that the record run will pass to declare the test as successful,
                # we need the second playback to pass as well.
                if is_record_run:
                    self.build_context.logging_module.info(
                        f'Test-Playbook recording was executed {MAX_RETRIES} times, and passed {self.playbook.configuration.number_of_successful_runs} times.'
                        f' Running second playback.')
                    return PB_Status.SECOND_PLAYBACK_REQUIRED
                self.build_context.logging_module.info(
                    f'Test-Playbook was executed {MAX_RETRIES} times, and passed {self.playbook.configuration.number_of_successful_runs} times.'
                    f' Adding to succeeded playbooks.')
                return PB_Status.COMPLETED
            else:
                self.build_context.logging_module.info(
                    f'Test-Playbook was executed {MAX_RETRIES} times, and passed only {self.playbook.configuration.number_of_successful_runs} times.'
                    f' Adding to failed playbooks.')
                return PB_Status.FAILED

    def _update_playbook_status(self, status: str,
                                is_first_playback_run: bool = False,
                                is_second_playback_run: bool = False,
                                is_record_run: bool = False) -> str:
        """
        Updates the playbook status if necessary and adds the test to the right set (succeeded/failed) if test is done.
        Args:
            status: The string representation of the playbook execution
            is_first_playback_run: Is the playbook runs in playback mode
            is_second_playback_run: Is The playbook run on a second playback after a freshly created record
        Returns:
            PB_Status.COMPLETED if the Test-Playbook passed successfully and was added to succeeded playbooks.
            PB_Status.FAILED if the Test-Playbook failed and was added to failed playbooks.
            PB_Status.SECOND_PLAYBACK_REQUIRED if second playback is needed.
            PB_Status.IN_PROGRESS if more executions are needed in order to determine whether the playbook is successful or not.
        """
        use_retries_mechanism = self.server_context.use_retries_mechanism
        number_of_executions = self.playbook.configuration.number_of_executions
        is_first_execution = number_of_executions == 1

        if status == PB_Status.COMPLETED:
            updated_status = self._update_complete_status(is_first_execution, is_record_run, is_first_playback_run,
                                                          is_second_playback_run, use_retries_mechanism,
                                                          number_of_executions)

        elif status in (PB_Status.FAILED_DOCKER_TEST, PB_Status.CONFIGURATION_FAILED):
            self._add_to_failed_playbooks(status=status)
            return status

        else:  # test-playbook failed
            updated_status = self._update_failed_status(is_record_run, is_first_playback_run, is_second_playback_run,
                                                        use_retries_mechanism, number_of_executions)

        if updated_status == PB_Status.COMPLETED:
            self._add_to_succeeded_playbooks()
        elif updated_status == PB_Status.FAILED:
            self._notify_failed_test()
            self._add_to_failed_playbooks(is_second_playback_run=is_second_playback_run)
        return updated_status

    def _execute_mockable_test(self, proxy: MITMProxy):
        """
        Executes a mockable test
        In case the test failed to execute because one of the test's integrations were locked will return False
        as indication that the test was not done.
        Returns:
            True if test has finished it's execution else False
        """
        # we want to test first playback only once (we want to skip it when using retries mechanism)
        if not self.playbook.configuration.is_first_playback_failed:
            if proxy.has_mock_file(self.playbook.configuration.playbook_id):
                # Running first playback run on mock file
                self.build_context.logging_module.info(f'------ Test {self} start ------ (Mock: Playback)')
                with run_with_mock(proxy, self.playbook.configuration.playbook_id) as result_holder:
                    status = self._incident_and_docker_test()
                    status = self._update_playbook_status(status, is_first_playback_run=True)
                    result_holder[RESULT] = status == PB_Status.COMPLETED
                if status in (PB_Status.COMPLETED, PB_Status.FAILED_DOCKER_TEST):
                    return True
                self.build_context.logging_module.warning(
                    "Test failed with mock, recording new mock file. (Mock: Recording)")
                self.playbook.configuration.is_first_playback_failed = True

        # Running on record mode since playback has failed or mock file was not found
        self.build_context.logging_module.info(f'------ Test {self} start ------ (Mock: Recording)')
        with acquire_test_lock(self.playbook) as lock:
            if lock:
                with run_with_mock(proxy, self.playbook.configuration.playbook_id, record=True) as result_holder:
                    status = self._incident_and_docker_test()
                    self.playbook.configuration.number_of_executions += 1
                    status = self._update_playbook_status(status, is_record_run=True)
                    result_holder[RESULT] = status == PB_Status.SECOND_PLAYBACK_REQUIRED
            else:
                # If the integrations were not locked - the test has not finished it's execution
                return False

        # Running playback after successful record to verify the record is valid for future runs
        if status == PB_Status.SECOND_PLAYBACK_REQUIRED:
            self.build_context.logging_module.info(
                f'------ Test {self} start ------ (Mock: Second playback)')
            with run_with_mock(proxy, self.playbook.configuration.playbook_id) as result_holder:
                status = self._run_incident_test()
                self._update_playbook_status(status, is_second_playback_run=True)
                result_holder[RESULT] = status == PB_Status.COMPLETED
        return True

    def _is_runnable_on_current_server_instance(self) -> bool:
        """
        Nightly builds can have RHEL instances that uses podman instead of docker as well as the regular LINUX instance.
        In such case - if the test in runnable on docker instances **only** and the current instance uses podman -
        we will not execute the test under this instance and instead will will return it to the queue in order to run
        it under some other instance
        Returns:
            True if this instance can be run on the current instance else False
        """
        if self.playbook.configuration.runnable_on_docker_only and not self.server_context.is_instance_using_docker:
            self.build_context.logging_module.debug(
                f'Skipping test {self.playbook} since it\'s not runnable on podman instances')
            return False
        return True

    def execute_test(self, proxy: Optional[MITMProxy] = None) -> bool:
        """
        Executes the test and return a boolean that indicates whether the test was executed or not.
        In case the test was not executed - it will be returned to the queue and will be collected later in the future
        by some other ServerContext instance.
        Args:
            proxy: The MITMProxy instance to use in the current test

        Returns:
            True if the test was executed by the instance else False
        """
        try:
            if not self._is_runnable_on_current_server_instance():
                return False
            self._send_circle_memory_and_pid_stats_on_slack()
            if self.playbook.is_mockable:
                test_executed = self._execute_mockable_test(proxy)  # type: ignore
            else:
                test_executed = self._execute_unmockable_test()
            return test_executed
        except Exception:
            self.build_context.logging_module.exception(
                f'Unexpected error while running test on playbook {self.playbook}')
            self._add_to_failed_playbooks()
            return True
        finally:
            self.build_context.logging_module.info(f'------ Test {self} end ------ \n')
            self.build_context.logging_module.execute_logs()

    def __str__(self):
        test_message = f'playbook: {self.playbook}'
        if self.playbook.integrations:
            test_message += f' with integration(s): {self.playbook.integrations}'
        else:
            test_message += ' with no integrations'
        if not self.server_context.is_instance_using_docker:
            test_message += ', RedHat instance'
        return test_message

    def __repr__(self):
        return str(self)


class ServerContext:

    def __init__(self, build_context: BuildContext, server_private_ip: str, tunnel_port: int = None,
                 use_retries_mechanism: bool = True):
        self.build_context = build_context
        self.server_ip = server_private_ip
        self.tunnel_port = tunnel_port
        if IS_XSIAM:
            self.server_url = server_private_ip
            # we use client without demisto username
            os.environ.pop('DEMISTO_USERNAME', None)
        else:
            self.server_url = f'https://localhost:{tunnel_port}' if tunnel_port else f'https://{self.server_ip}'
        self.client: Optional[DefaultApi] = None
        self._configure_new_client()
        # currently not supported on XSIAM (etc/#47851)
        if IS_XSIAM:
            self.proxy = None
        else:
            self.proxy = MITMProxy(server_private_ip,
                                   self.build_context.logging_module,
                                   build_number=self.build_context.build_number,
                                   branch_name=self.build_context.build_name)
        self.is_instance_using_docker = not is_redhat_instance(self.server_ip)
        self.executed_tests: Set[str] = set()
        self.executed_in_current_round: Set[str] = set()
        self.prev_system_conf: dict = {}
        self.use_retries_mechanism: bool = use_retries_mechanism
        if IS_XSIAM:
            self.check_if_can_create_manual_alerts()

    def _execute_unmockable_tests(self):
        """
        Iterates the mockable tests queue and executes them as long as there are tests to execute
        """
        self._execute_tests(self.build_context.unmockable_tests_to_run)

    def _execute_tests(self, queue: Queue) -> None:
        """
        Iterates the tests queue and executes them as long as there are tests to execute.
        Before the tests execution starts we will reset the containers to make sure the proxy configuration is correct
        - We need it before the mockable tests because the server starts the python2 default container when it starts
            and it has no proxy configurations.
        - We need it before the unmockable tests because at that point all containers will have the proxy configured
            and we want to clean those configurations when testing unmockable playbooks
        Args:
            queue: The queue to fetch tests to execute from
        """
        self._reset_containers()
        while queue.unfinished_tasks:
            try:
                test_playbook: TestPlaybook = queue.get(block=False)
                self._reset_tests_round_if_necessary(test_playbook, queue)
            except Empty:
                continue
            self._configure_new_client()
            test_executed = TestContext(self.build_context,
                                        test_playbook,
                                        self.client,
                                        self).execute_test(self.proxy)
            if test_executed:
                self.executed_tests.add(test_playbook.configuration.playbook_id)
            else:
                queue.put(test_playbook)
            queue.task_done()

    def _execute_mockable_tests(self):
        """
        Iterates the mockable tests queue and executes them as long as there are tests to execute
        """
        # we running XSIAM without proxy. This code wont be executed on xsiam servers
        if not IS_XSIAM:
            self.proxy.configure_proxy_in_demisto(  # type: ignore[union-attr]
                proxy=self.proxy.ami.internal_ip + ':' + self.proxy.PROXY_PORT,  # type: ignore[union-attr]
                username=self.build_context.secret_conf.server_username,
                password=self.build_context.secret_conf.server_password,
                server=self.server_url)
        self._execute_tests(self.build_context.mockable_tests_to_run)
        if not IS_XSIAM:
            self.proxy.configure_proxy_in_demisto(proxy='',  # type: ignore[union-attr]
                                                  username=self.build_context.secret_conf.server_username,
                                                  password=self.build_context.secret_conf.server_password,
                                                  server=self.server_url)

    def _execute_failed_tests(self):
        self._execute_tests(self.build_context.test_retries_queue)

    def _reset_tests_round_if_necessary(self, test_playbook: TestPlaybook, queue_: Queue) -> None:
        """
        Checks if the string representation of the current test configuration is already in
        the executed_in_current_round set.
        If it is- it means we have already executed this test and the we have reached a round and there are tests that
        were not able to be locked by this execution..
        In that case we want to start a new round monitoring by emptying the 'executed_in_current_round' set and sleep
        in order to let the tests be unlocked
        Since this operation can be performed by multiple threads - this operaion is protected by the queue's lock
        Args:
            test_playbook: Test playbook to check if has been executed in the current round
            queue_: The queue from which the current tests are executed
        """
        queue_.mutex.acquire()
        if str(test_playbook.configuration) in self.executed_in_current_round:
            self.build_context.logging_module.info(
                'all tests in the queue were executed, sleeping for 30 seconds to let locked tests get unlocked.')
            self.executed_in_current_round = {str(test_playbook.configuration)}
            queue_.mutex.release()
            time.sleep(30)
            return
        else:
            self.executed_in_current_round.add(str(test_playbook.configuration))
        queue_.mutex.release()

    def _configure_new_client(self):
        if self.client:
            self.client.api_client.pool.close()
            self.client.api_client.pool.terminate()
            del self.client
        self.client = demisto_client.configure(base_url=self.server_url,
                                               api_key=self.build_context.api_key,
                                               auth_id=self.build_context.auth_id,
                                               verify_ssl=False)

    def _reset_containers(self):
        self.build_context.logging_module.info('Resetting containers\n', real_time=True)

        body, status_code, _ = demisto_client.generic_request_func(self=self.client, method='POST',
                                                                   path='/containers/reset')
        if status_code != 200:
            self.build_context.logging_module.critical(
                f'Request to reset containers failed with status code "{status_code}"\n{body}', real_time=True)
            sys.exit(1)
        time.sleep(10)

    def execute_tests(self):

        try:
            self.build_context.logging_module.info(f'Starts tests with server url - {get_ui_url(self.server_url)}',
                                                   real_time=True)
            self._execute_mockable_tests()
            self.build_context.logging_module.info('Running mock-disabled tests', real_time=True)
            self._execute_unmockable_tests()
            if self.use_retries_mechanism:
                self.build_context.logging_module.info('Running failed tests', real_time=True)
                self._execute_failed_tests()
            self.build_context.logging_module.info(f'Finished tests with server url - '
                                                   f'{get_ui_url(self.server_url)}',
                                                   real_time=True)
            # no need in xsiam, no proxy
            if not IS_XSIAM:
                self.build_context.tests_data_keeper.add_proxy_related_test_data(self.proxy)

            if self.build_context.isAMI and not IS_XSIAM:
                if self.proxy.should_update_mock_repo:  # type: ignore[union-attr]
                    self.proxy.push_mock_files()  # type: ignore[union-attr]
            self.build_context.logging_module.debug(f'Tests executed on server {self.server_ip}:\n'
                                                    f'{pformat(self.executed_tests)}')
        except Exception:
            self.build_context.logging_module.exception('~~ Thread failed ~~')
            raise
        finally:
            self.build_context.logging_module.execute_logs()

    def check_if_can_create_manual_alerts(self):
        """
        In XSIAM we can't create a new incident/alert using API call.
        We need a correlation rule in order to create an alert.
        We want to create an alert manually, when we send an API call to XSIAM server to create a new alert.
        Server check which integration sent a new alert, if the request was sent manually and not from integration it
        sets "sourceBrand" header to be "Manual". XSIAM Server looks for a correlation rule for such sourceBrand,
        and if there is no such correlation rule, no alert will be created.
        If there is a correlation rule for "Manual" integration the alert will be created.

        If this step fails please create an integration with id and name "Manual", set isFetch: true for such
        integration and make sure that the corresponding correlation rule is created.
        """
        body = {
            'query': 'id:"Manual"',
        }
        try:
            res_raw = demisto_client.generic_request_func(
                self=self.client,
                method='POST',
                path='/settings/integration/search',
                body=body,
            )
            res = ast.literal_eval(res_raw[0])
            if int(res_raw[1]) != 200:
                self.build_context.logging_module.error(
                    f'Failed to get integrations configuration with status code: {res_raw[1]}')
                return

            all_configurations = res['configurations']
            for instance in all_configurations:
                if instance.get('id') == "Manual":
                    self.build_context.logging_module.info('Server is able to create manual alerts '
                                                           '("Manual" integration exists).')
                    return
        except ApiException:
            self.build_context.logging_module.exception('Failed to get integrations configuration.')

        self.build_context.logging_module.warning('No "Manual" integration found in XSIAM instance. '
                                                  'Adding it in order to create Manual Correlation Rule.')
        self.create_manual_integration()

    def create_manual_integration(self):
        manual_integration = {
            "name": "Manual",
            "version": 1,
            "display": "Manual",
            "category": "Utilities",
            "description": "This integration creates Manual Correlation Rule.",
            "configuration": [],
            "integrationScript": {
                "script": "",
                "commands": [],
                "type": "python",
                "isFetch": True,
                "subtype": "python3"
            }
        }

        try:
            res_raw_integration = demisto_client.generic_request_func(
                self=self.client,
                method='PUT',
                path='/settings/integration-conf',
                body=manual_integration,
            )
            if int(res_raw_integration[1]) != 200:
                self.build_context.logging_module.error(
                    f'Failed to get integrations configuration with status code: {res_raw_integration[1]}')

        except ApiException:
            self.build_context.logging_module.exception('No "Manual" integration found in XSIAM instance. '
                                                        'Please add it in order to create Manual Correlation Rule.')


def replace_external_playbook_configuration(client: DefaultApi, external_playbook_configuration: dict,
                                            logger_module: logging.Logger = logging.getLogger('demisto-sdk')):
    """ takes external configuration of shape {"playbookID": "Isolate Endpoint - Generic V2",
                                               "input_parameters":{"Endpoint_hostname": {"simple", "test"}}}
        and changes the specified playbook configuration to the mentioned one.
        If playbook's inputs had changed, revert will be needed.
        Returns (Whether the Playbook changed, The values to restore, the path to use when restoring)
        Only to be used with server version 6.2 and above. """

    # Checking configuration
    if not external_playbook_configuration:
        logger_module.info("External Playbook Configuration not provided, skipping re-configuration.")
        return False, {}, ''

    # Checking server version
    server_version = get_demisto_version(client)

    if LooseVersion(server_version.base_version) < LooseVersion('6.2.0'):  # type: ignore
        logger_module.info("External Playbook not supported in versions previous to 6.2.0, skipping re-configuration.")
        return False, {}, ''

    logger_module.info("External Playbook in use, starting re-configuration.")

    # Getting current configuration
    external_playbook_id = external_playbook_configuration['playbookID']
    external_playbook_path = f'/playbook/{external_playbook_id}'
    res, _, _ = demisto_client.generic_request_func(client, method='GET',
                                                    path=external_playbook_path, response_type='object')

    inputs = res.get('inputs', [])
    if not inputs:
        raise Exception("External Playbook was not found or has no inputs.")

    # Save current for default Configuration.
    inputs_default = deepcopy(inputs)
    logger_module.info("Saved current configuration.")

    changed_keys = []
    failed_keys = []

    # Change Configuration for external pb.
    for input_ in external_playbook_configuration["input_parameters"]:
        matching_record = list(filter(lambda x: x.get('key') == input_, inputs))
        if matching_record:
            existing_val = matching_record[0]
            simple = external_playbook_configuration["input_parameters"][input_].get("simple")
            complex = external_playbook_configuration["input_parameters"][input_].get("complex")

            # If no value (simple or complex) was found, It is a typo
            if complex is None and simple is None:
                raise Exception(f'Could not found neither a `simple` nor `complex` value for field: {input_}. '
                                'A valid configuration should be of the followng format: '
                                '{<param name>: {"simple", <required value>}}')

            existing_val['value']["simple"] = simple
            existing_val['value']["complex"] = complex
            changed_keys.append(input_)

        else:
            failed_keys.append(input_)

    if failed_keys:
        raise Exception(f'Some input keys was not found in playbook: {",".join(failed_keys)}.')

    logger_module.info(f"Changing keys: {changed_keys}.")
    saving_inputs_path = f'/playbook/inputs/{external_playbook_id}'

    try:
        if changed_keys:
            demisto_client.generic_request_func(client, method='POST', path=saving_inputs_path, body=inputs)

    except Exception as e:
        raise Exception(f"Could not change inputs in playbook configuration. Error: {e}")

    logger_module.info(f"Re-configured {external_playbook_id} successfully with {len(changed_keys)} new values.")

    return True, inputs_default, saving_inputs_path


def restore_external_playbook_configuration(client: DefaultApi, restore_path: str, restore_values: dict,
                                            logger_module: logging.Logger = logging.getLogger('demisto-sdk')):
    logger_module.info("Restoring External Playbook parameters.")

    res, _, _ = demisto_client.generic_request_func(client, method='POST',
                                                    path=restore_path,
                                                    body=restore_values)

    logger_module.info("Restored External Playbook successfully.")


# demisto_sdk/commands/test_content/constants.py

LOAD_BALANCER_DNS = 'content-build-lb.demisto.works'
SSH_USER = 'ec2-user'
CONTENT_BUILD_SSH_USER = 'content-build'


# demisto_sdk/commands/test_content/execute_test_content.py

import os
import sys
from threading import Thread

import requests

from demisto_sdk.commands.test_content.ParallelLoggingManager import ParallelLoggingManager
from demisto_sdk.commands.test_content.TestContentClasses import BuildContext, ServerContext

SKIPPED_CONTENT_COMMENT = 'The following integrations/tests were collected by the CI build but are currently skipped. ' \
                          'The collected tests are related to this pull request and might be critical.'
COVERAGE_REPORT_COMMENT = 'Link to the unit tests coverage report'


def _handle_github_response(response, logging_module) -> dict:
    res_dict = response.json()
    if not response.ok:
        logging_module.error(f'Add pull request comment failed: {res_dict.get("message")}', real_time=True)
    return res_dict


def _add_pr_comment(comment, logging_module):
    token = os.environ['XSOAR_BOT_TEST_CONTENT']
    branch_name = os.environ['CI_COMMIT_BRANCH']
    sha1 = os.environ['CI_COMMIT_SHA']

    query = '?q={}+repo:demisto/content+org:demisto+is:pr+is:open+head:{}+is:open'.format(sha1, branch_name)
    url = 'https://api.github.com/search/issues'
    headers = {'Authorization': 'Bearer ' + token}

    is_skipped_tests_flow = 'The following integrations/tests were collected' in comment

    try:
        response = requests.get(url + query, headers=headers, verify=False)
        res = _handle_github_response(response, logging_module)

        if res and res.get('total_count', 0) == 1:
            issue_url = res['items'][0].get('comments_url') if res.get('items', []) else None
            if issue_url:
                # Check if a comment about skipped tests already exists. If there is delete it first and then post a
                # new comment:
                response = requests.get(issue_url, headers=headers, verify=False)
                issue_comments = _handle_github_response(response, logging_module)
                for existing_comment in issue_comments:
                    if (is_skipped_tests_flow and SKIPPED_CONTENT_COMMENT in existing_comment.get('body', '')) or \
                            (not is_skipped_tests_flow and COVERAGE_REPORT_COMMENT in existing_comment.get('body', '')):
                        comment_url = existing_comment.get('url')
                        requests.delete(comment_url, headers=headers, verify=False)
                response = requests.post(issue_url, json={'body': comment}, headers=headers, verify=False)
                _handle_github_response(response, logging_module)
        else:
            logging_module.warning('Add pull request comment failed: There is more then one open pull '
                                   f'request for branch {branch_name}.', real_time=True)
    except Exception:
        logging_module.exception('Add pull request comment failed')


def execute_test_content(**kwargs):
    logging_manager = ParallelLoggingManager('Run_Tests.log', real_time_logs_only=not kwargs['nightly'])
    build_context = BuildContext(kwargs, logging_manager)
    use_retries_mechanism = kwargs.get('use_retries', False)
    threads_list = []
    for server_ip, port in build_context.instances_ips.items():
        tests_execution_instance = ServerContext(build_context, server_private_ip=server_ip, tunnel_port=port,
                                                 use_retries_mechanism=use_retries_mechanism)
        threads_list.append(Thread(target=tests_execution_instance.execute_tests))

    logging_manager.info('Finished creating configurations, starting to run tests.')
    for thread in threads_list:
        thread.start()

    for t in threads_list:
        t.join()

    if not build_context.unmockable_tests_to_run.empty() or not build_context.mockable_tests_to_run.empty():
        raise Exception('Not all tests have been executed')
    if build_context.tests_data_keeper.playbook_skipped_integration \
            and build_context.build_name != 'master' \
            and not build_context.is_nightly:
        skipped_integrations = '\n- '.join(build_context.tests_data_keeper.playbook_skipped_integration)
        comment = f'{SKIPPED_CONTENT_COMMENT}:\n- {skipped_integrations}'
        _add_pr_comment(comment, logging_manager)
    build_context.tests_data_keeper.print_test_summary(build_context.isAMI, logging_manager)
    build_context.tests_data_keeper.create_result_files()
    if build_context.tests_data_keeper.failed_playbooks:
        logging_manager.critical("Some tests have failed. Not destroying instances.", real_time=True)
        sys.exit(1)


# demisto_sdk/commands/test_content/mock_server.py

from __future__ import print_function

import ast
import os
import string
import time
import unicodedata
from contextlib import contextmanager
from pprint import pformat
from subprocess import STDOUT, CalledProcessError, call, check_call, check_output
from threading import Lock
from typing import Dict, Iterator

import demisto_client.demisto_api
import urllib3

from demisto_sdk.commands.test_content.constants import SSH_USER

VALID_FILENAME_CHARS = f'-_.() {string.ascii_letters}{string.digits}'
PROXY_PROCESS_INIT_TIMEOUT = 20
PROXY_PROCESS_INIT_INTERVAL = 1
RESULT = 'result'
# Disable insecure warnings
urllib3.disable_warnings()


def clean_filename(playbook_or_integration_id, whitelist=VALID_FILENAME_CHARS, replace=' ()'):
    filename = playbook_or_integration_id

    # replace spaces
    for r in replace:
        filename = filename.replace(r, '_')

    # keep only valid ascii chars
    cleaned_filename = unicodedata.normalize('NFKD', filename).encode('ASCII', 'ignore').decode()

    # keep only whitelisted chars
    cleaned_filename = ''.join(c for c in cleaned_filename if c in whitelist)
    return cleaned_filename


def silence_output(cmd_method, *args, **kwargs):
    """Redirect linux command output(s) to /dev/null
    To redirect to /dev/null: pass 'null' string in 'stdout' or 'stderr' keyword args

    Args:
        cmd_method (PyFunctionObject): the "subprocess" (or wrapper) method to run
        *args: additional parameters for cmd_method
        **kwargs: additional parameters for cmd_method

    Returns:
        string. output of cmd_method
    """
    with open(os.devnull, 'w') as fnull:
        for k in ('stdout', 'stderr'):
            if kwargs.get(k) == 'null':
                kwargs[k] = fnull

        return cmd_method(*args, **kwargs)


def get_mock_file_path(playbook_or_integration_id):
    clean = clean_filename(playbook_or_integration_id)
    return os.path.join(clean + '/', clean + '.mock')


def get_log_file_path(playbook_or_integration_id, record=False):
    clean = clean_filename(playbook_or_integration_id)
    suffix = '_record' if record else '_playback'
    return os.path.join(clean + '/', clean + suffix + '.log')


def get_folder_path(playbook_or_integration_id):
    return clean_filename(playbook_or_integration_id) + '/'


class AMIConnection:
    """Wrapper for AMI communication.

    Attributes:
        internal_ip (string): The  internal IP of the instance.
    """

    REMOTE_HOME = f'/home/{SSH_USER}/'
    LOCAL_SCRIPTS_DIR = '/home/circleci/project/Tests/scripts/'

    def __init__(self, internal_ip):
        self.internal_ip = internal_ip

    def add_ssh_prefix(self, command, ssh_options=""):
        """Add necessary text before a command in order to run it on the AMI instance via SSH.

        Args:
            command (list): Command to run on the AMI machine (according to "subprocess" interface).
            ssh_options (string): optional parameters for ssh connection to AMI.

        Returns:
            string: ssh command that will run the desired command on the AMI.
        """
        if ssh_options and not isinstance(ssh_options, str):
            raise TypeError("options must be string")
        if not isinstance(command, list):
            raise TypeError("command must be list")
        prefix = "ssh {} {}@{}".format(ssh_options, SSH_USER, self.internal_ip).split()
        return prefix + command

    def call(self, command, **kwargs):
        return call(self.add_ssh_prefix(command), **kwargs)

    def check_call(self, command, **kwargs):
        return check_call(self.add_ssh_prefix(command), **kwargs)

    def check_output(self, command, **kwargs):
        return check_output(self.add_ssh_prefix(command), **kwargs)

    def copy_file(self, src, dst=REMOTE_HOME, **kwargs):
        silence_output(check_call, ['scp', src, "{}@{}:{}".format(SSH_USER, self.internal_ip, dst)],
                       stdout='null', **kwargs)
        return os.path.join(dst, os.path.basename(src))

    def run_script(self, script, *args):
        """Copy a script to the AMI and run it.

        Args:
            script (string): Name of the script file in the LOCAL_SCRIPTS_DIR.
            *args: arguments to be passed to the script.
        """
        remote_script_path = self.copy_file(os.path.join(self.LOCAL_SCRIPTS_DIR, script))

        silence_output(self.check_call, ['chmod', '+x', remote_script_path], stdout='null')
        silence_output(self.check_call, [remote_script_path] + list(args), stdout='null')


class MITMProxy:
    """Manager for MITM Proxy and the mock file structure.

    Attributes:
        logging_module: Logging module to use
        public_ip (string): The IP of the AMI instance.
        repo_folder (string): path to the local clone of the content-test-data git repo.
        tmp_folder (string): path to a temporary folder for log/mock files before pushing to git.
        current_folder (string): the current folder to use for mock/log files.
        ami (AMIConnection): Wrapper for AMI communication.
        empty_files (list): List of playbooks that have empty mock files (indicating no usage of mock mechanism).
        rerecorded_tests (list): List of playbook ids that failed on mock playback but succeeded on new recording.
        build_number (str): The number of the circleci build.
        branch_name (str): The name of the content branch in which the current job works on.
    """

    PROXY_PORT = '9997'
    MOCKS_TMP_PATH = '/tmp/Mocks/'
    MOCKS_GIT_PATH = f'{AMIConnection.REMOTE_HOME}content-test-data/'
    TIME_TO_WAIT_FOR_PROXY_SECONDS = 30
    content_data_lock = Lock()

    def __init__(self,
                 internal_ip,
                 logging_module,
                 build_number,
                 branch_name,
                 repo_folder=MOCKS_GIT_PATH,
                 tmp_folder=MOCKS_TMP_PATH,
                 ):
        is_branch_master = branch_name == 'master'
        self.internal_ip = internal_ip
        self.current_folder = self.repo_folder = repo_folder
        self.tmp_folder = tmp_folder
        self.logging_module = logging_module
        self.build_number = build_number
        self.ami = AMIConnection(self.internal_ip)
        self.should_update_mock_repo = is_branch_master
        self.should_validate_playback = not is_branch_master
        self.empty_files = []
        self.failed_tests_count = 0
        self.successful_tests_count = 0
        self.successful_rerecord_count = 0
        self.failed_rerecord_count = 0
        self.failed_rerecord_tests = []
        self.rerecorded_tests = []
        silence_output(self.ami.call, ['mkdir', '-p', tmp_folder], stderr='null')
        script_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'timestamp_replacer.py')
        self.ami.copy_file(script_filepath)

    def commit_mock_file(self, folder_name):
        self.logging_module.debug('Committing mock files')
        try:
            output = self.ami.check_output(
                'cd content-test-data && '
                'git add * -v && '
                f'git commit -m "Updated mock files for \'{folder_name}\' from build number - {self.build_number}" -v'.split(),
                stderr=STDOUT)
            self.logging_module.debug(f'Committing mock files output:\n{output.decode()}')
        except CalledProcessError as exc:
            self.logging_module.debug(f'Committing mock files output:\n{exc.output.decode()}')

    def reset_mock_files(self):
        self.logging_module.debug('Resetting mock files')
        try:
            output = self.ami.check_output(
                'cd content-test-data && git reset --hard'.split(),
                stderr=STDOUT)
            self.logging_module.debug(f'Resetting mock files output:\n{output.decode()}')
        except CalledProcessError as exc:
            self.logging_module.debug(f'Resetting mock files output:\n{exc.output.decode()}')

    def push_mock_files(self):
        self.logging_module.debug('Pushing new/updated mock files to mock git repo.', real_time=True)
        self.content_data_lock.acquire()
        try:
            output = self.ami.check_output(
                'cd content-test-data && git reset --hard && git pull -r -Xtheirs && git push -f'.split(),
                stderr=STDOUT)
            self.logging_module.debug(f'Pushing mock files output:\n{output.decode()}', real_time=True)
        except CalledProcessError as exc:
            self.logging_module.debug(f'Pushing mock files output:\n{exc.output.decode()}', real_time=True)
        except Exception as exc:
            self.logging_module.debug(f'Failed pushing mock files with error: {exc}', real_time=True)
        finally:
            self.content_data_lock.release()

    def configure_proxy_in_demisto(self, username, password, server, api_key=None, auth_id=None, proxy=''):
        client = demisto_client.configure(base_url=server, username=username,
                                          password=password, api_key=api_key, auth_id=auth_id, verify_ssl=False)
        self.logging_module.debug('Adding proxy server configurations')
        system_conf_response = demisto_client.generic_request_func(
            self=client,
            path='/system/config',
            method='GET'
        )
        system_conf = ast.literal_eval(system_conf_response[0]).get('sysConf', {})
        self.logging_module.debug(f'Server configurations before proxy server configurations:\n{pformat(system_conf)}')
        http_proxy = https_proxy = proxy
        if proxy:
            http_proxy = 'http://' + proxy
            https_proxy = 'http://' + proxy
        system_conf.update({
            'http_proxy': http_proxy,
            'https_proxy': https_proxy
        })
        data = {
            'data': system_conf,
            'version': -1
        }
        response = demisto_client.generic_request_func(self=client, path='/system/config',
                                                       method='POST', body=data)
        self.logging_module.debug(f'Server configurations response:\n{pformat(ast.literal_eval(response[0]))}')

        return response

    def get_mock_file_size(self, filepath):
        return self.ami.check_output(['stat', '-c', '%s', filepath]).strip()

    @staticmethod
    def get_script_mode(is_record: bool) -> str:
        """
        Returns the string describing script mode for the SCRIPT_MODE env variable needed for the mitmdump initialization
        Args:
            is_record: A boolean indicating this is record mode or not

        Returns:
            'record' if is_record is True else 'playback'
        """
        return 'record' if is_record else 'playback'

    def has_mock_file(self, playbook_or_integration_id):
        command = ["[", "-f", os.path.join(self.current_folder, get_mock_file_path(playbook_or_integration_id)), "]"]
        return self.ami.call(command) == 0

    def has_mock_folder(self, playbook_or_integration_id):
        command = ["[", "-d", os.path.join(self.current_folder, get_folder_path(playbook_or_integration_id)), "]"]
        return self.ami.call(command) == 0

    def set_repo_folder(self):
        """Set the repo folder as the current folder (the one used to store mock and log files)."""
        self.current_folder = self.repo_folder

    def set_tmp_folder(self):
        """Set the temp folder as the current folder (the one used to store mock and log files)."""
        self.current_folder = self.tmp_folder

    def move_mock_file_to_repo(self, playbook_or_integration_id):
        """Move the mock and log files of a (successful) test playbook run from the temp folder to the repo folder

        Args:
            playbook_or_integration_id (string): ID of the test playbook or integration of which the files should be moved.
        """
        src_filepath = os.path.join(self.tmp_folder, get_mock_file_path(playbook_or_integration_id))
        src_files = os.path.join(self.tmp_folder, get_folder_path(playbook_or_integration_id) + '*')
        dst_folder = os.path.join(self.repo_folder, get_folder_path(playbook_or_integration_id))

        if not self.has_mock_file(playbook_or_integration_id):
            self.logging_module.debug('Mock file not created!')
        elif self.get_mock_file_size(src_filepath) == '0':
            self.logging_module.debug('Mock file is empty, ignoring.')
            self.empty_files.append(playbook_or_integration_id)
        else:
            # Move to repo folder
            self.logging_module.debug(f'Moving "{src_files}" files to "{dst_folder}" directory')
            self.ami.call(['mkdir', '--parents', dst_folder])
            self.ami.call(['mv', src_files, dst_folder])

    def normalize_mock_file(self, playbook_or_integration_id: str, path: str = None):
        """Normalize the mock file of a test playbook

        Normalizes a mock file by doing the following:
        1. Replaces the timestamp/variable data in a mock file as identified by
        the keys provided in the mock file's associated 'problematic_keys.json'
        file, with constant values.
        2. Standardizes the query parameter data order for all requests.

        Args:
            playbook_or_integration_id: ID of the test playbook or
                integration for which the associated mock file will be
                normalized.
            path: Path to the directory in which to search
                for the required data (temp directory or repo). If not
                provided, method will use the current directory.
        """
        self.logging_module.debug(f'normalize_mock_file was called for test "{playbook_or_integration_id}"')
        path = path or self.current_folder
        problem_keys_filepath = os.path.join(path, get_folder_path(playbook_or_integration_id), 'problematic_keys.json')
        self.logging_module.debug(f'problem_keys_filepath="{problem_keys_filepath}"')
        problem_key_file_exists = ["[", "-f", problem_keys_filepath, "]"]
        if not self.ami.call(problem_key_file_exists) == 0:
            self.logging_module.debug('Error: The problematic_keys.json file was not written to the file path'
                                      f' "{problem_keys_filepath}" when recording '
                                      f'the "{playbook_or_integration_id}" test playbook')
            return

        mock_file_path = os.path.join(path, get_mock_file_path(playbook_or_integration_id))
        cleaned_mock_filepath = mock_file_path.strip('.mock') + '_cleaned.mock'
        log_file = os.path.join(path, get_log_file_path(playbook_or_integration_id, record=True))
        command = '/home/ec2-user/.local/bin/mitmdump -ns ~/timestamp_replacer.py ' \
                  f'--set script_mode=clean --set keys_filepath={problem_keys_filepath}' \
                  f' -r {mock_file_path} -w {cleaned_mock_filepath} | sudo tee -a {log_file}'
        self.logging_module.debug(f'command to normalize mockfile:\n\t{command}')
        self.logging_module.debug('Let\'s try and normalize the mockfile')
        try:
            check_output(self.ami.add_ssh_prefix(command.split(), ssh_options='-t'), stderr=STDOUT)
        except CalledProcessError as e:
            self.logging_module.debug(
                'There may have been a problem while normalizing the mock file.')
            err_msg = f'command `{command}` exited with return code [{e.returncode}]'
            err_msg = f'{err_msg} and the output of "{e.output}"' if e.output else err_msg
            if e.stderr:
                err_msg += f'STDERR: {e.stderr}'
            self.logging_module.debug(err_msg)
        else:
            self.logging_module.debug('Success!')

        # verify cleaned mock is different than original
        diff_cmd = f'diff -sq {cleaned_mock_filepath} {mock_file_path}'
        try:
            diff_cmd_output = self.ami.check_output(diff_cmd.split()).decode().strip()
            self.logging_module.debug(f'diff_cmd_output={diff_cmd_output}')
            if diff_cmd_output.endswith('are identical'):
                self.logging_module.debug('normalized mock file and original mock file are identical')
            else:
                self.logging_module.debug('the normalized mock file differs from the original')

        except CalledProcessError:
            self.logging_module.debug('the normalized mock file differs from the original')

        self.logging_module.debug('Replacing original mock file with the normalized one.')
        mv_cmd = f'mv {cleaned_mock_filepath} {mock_file_path}'
        self.ami.call(mv_cmd.split())

    def start(self, playbook_or_integration_id, path=None, record=False) -> None:
        """Start the proxy process and direct traffic through it.

        Args:
            playbook_or_integration_id (string): ID of the test playbook to run.
            path (string): path override for the mock/log files.
            record (bool): Select proxy mode (record/playback)
        """
        if self.is_proxy_listening():
            self.logging_module.debug('proxy service is already running, stopping it')
            self.ami.call(['sudo', 'systemctl', 'stop', 'mitmdump'])
        self.logging_module.debug(f'Attempting to start proxy in {self.get_script_mode(record)} mode')
        self.prepare_proxy_start(path, playbook_or_integration_id, record)
        # Start proxy server
        self._start_proxy_and_wait_until_its_up(is_record=record)

    def _start_proxy_and_wait_until_its_up(self, is_record: bool) -> None:
        """
        Starts mitmdump service and wait for it to listen to port 9997 with timeout of 5 seconds
        Args:
            is_record (bool):  Indicates whether this is a record run or not
        """
        self._start_mitmdump_service()
        was_proxy_up = self.wait_until_proxy_is_listening()
        if was_proxy_up:
            self.logging_module.debug(f'Proxy service started in {self.get_script_mode(is_record)} mode')
        else:
            self.logging_module.error(f'Proxy failed to start after {self.TIME_TO_WAIT_FOR_PROXY_SECONDS} seconds')

    def _start_mitmdump_service(self) -> None:
        """
        Starts mitmdump service on the remote service
        """
        self.ami.call(['sudo', 'systemctl', 'start', 'mitmdump'])

    def prepare_proxy_start(self,
                            path: str,
                            playbook_or_integration_id: str,
                            record: bool) -> bool:
        """
        Writes proxy server run configuration options to the remote host, the details of which include:
        - Creating new tmp directory on remote machine if in record mode and moves the problematic keys file in to it
        - Creating the mitmdump_rc file that includes the script mode, keys file path, mock file path and log file path
          for the mitmdump service and puts it in '/home/ec2-user/mitmdump_rc' in the remote machine.
        - starts the systemd mitmdump service

        Args:
            path: the path to the temp folder in which the record files should be created
            playbook_or_integration_id: The ID of the playbook or integration that is tested
            record: Indicates whether this is a record run or not
        """
        path = path or self.current_folder
        folder_path = get_folder_path(playbook_or_integration_id)

        repo_problem_keys_path = os.path.join(self.repo_folder, folder_path, 'problematic_keys.json')
        current_problem_keys_path = os.path.join(path, folder_path, 'problematic_keys.json')
        log_file_path = os.path.join(path, get_log_file_path(playbook_or_integration_id, record))
        mock_file_path = os.path.join(path, get_mock_file_path(playbook_or_integration_id))

        file_content = f'export KEYS_FILE_PATH="{current_problem_keys_path if record else repo_problem_keys_path}"\n'
        file_content += f'export SCRIPT_MODE={self.get_script_mode(record)}\n'
        file_content += f'export MOCK_FILE_PATH="{mock_file_path}"\n'
        file_content += f'export LOG_FILE_PATH="{log_file_path}"\n'

        # Create mock files directory
        silence_output(self.ami.call, ['mkdir', os.path.join(path, folder_path)], stderr='null')
        # when recording, copy the `problematic_keys.json` for the test to current temporary directory if it exists
        # that way previously recorded or manually added keys will only be added upon and not wiped with an overwrite
        if record:
            try:
                silence_output(self.ami.call,
                               ['mv', repo_problem_keys_path, current_problem_keys_path],
                               stdout='null',
                               stderr='null')
            except CalledProcessError as e:
                self.logging_module.debug(f'Failed to move problematic_keys.json with exit code {e.returncode}')

        return self._write_mitmdump_rc_file_to_host(file_content)

    def _write_mitmdump_rc_file_to_host(self,
                                        file_content: str) -> bool:
        """
        Does all needed preparation for starting the proxy service which include:
        - Creating the mitmdump_rc file that includes the script mode, keys file path, mock file path and log file path
          for the mitmdump service and puts it in '/home/ec2-user/mitmdump_rc' in the remote machine.

        Args:
            file_content: The content of the mitmdump_rc file that includes the script mode, keys file path,
            mock file path and log file path

        Returns:
            True if file was successfully copied to the server, else False
        """
        try:
            self.ami.call(['echo', f"'{file_content}'", '>', os.path.join(AMIConnection.REMOTE_HOME, 'mitmdump_rc')])
            return True
        except CalledProcessError:
            self.logging_module.exception(
                f'Could not copy arg file for mitmdump service to server {self.ami.internal_ip},')
        return False

    def wait_until_proxy_is_listening(self):
        """
        Checks if the mitmdump service is listening, and raises an exception if 30 seconds pass without positive answer
        """
        for i in range(self.TIME_TO_WAIT_FOR_PROXY_SECONDS):
            proxy_is_listening = self.is_proxy_listening()
            if proxy_is_listening:
                return True
            time.sleep(1)
        return False

    def is_proxy_listening(self) -> bool:
        """
        Runs 'sudo lsof -iTCP:9997 -sTCP:LISTEN' on the remote machine and returns answer according to the results
        Returns:
            True if the ssh command return exit code 0 and False otherwise
        """
        try:
            self.ami.check_output(['sudo', 'lsof', '-iTCP:9997', '-sTCP:LISTEN'])
            return True
        except CalledProcessError:
            return False

    def stop(self):
        self.logging_module.debug('Stopping mitmdump service')
        if not self.is_proxy_listening():
            self.logging_module.debug('proxy service was already down.')
        else:
            self.ami.call(['sudo', 'systemctl', 'stop', 'mitmdump'])
        self.ami.call(["rm", "-rf", "/tmp/_MEI*"])  # Clean up temp files


@contextmanager
def run_with_mock(proxy_instance: MITMProxy,
                  playbook_or_integration_id: str,
                  record: bool = False) -> Iterator[dict]:
    """
    Runs proxy in a context
    If it's a record mode:
        - Setting the current folder of the proxy to the tmp folder before mitmdump starts
        - Starts the mitmdump service in record mode
        - Handles the newly created mock files
        - Setting the current folder of the proxy to the repo folder after mitmdump stops
    If it's a playback mode:
        - Starts the mitmdump service in playback mode
        - In case the playback failed - will show the status of the mitmdump service which will include the last 10
        lines of the service's log.
    Args:
        proxy_instance: The instance of the proxy to use
        playbook_or_integration_id: The ID of the playbook or integration that is tested
        record: A boolean indicating this is record mode or not
    Yields: A result holder dict in which the calling method can add the result of the proxy run under the key 'result'
    """
    if record:
        proxy_instance.set_tmp_folder()
        # If the record files should be committed - clean the content-test-data repo first
        if proxy_instance.should_update_mock_repo:
            proxy_instance.logging_module.debug('Cleaning content-test-data repo')
            proxy_instance.reset_mock_files()
    proxy_instance.start(playbook_or_integration_id, record=record)
    result_holder: Dict[str, bool] = {}
    try:
        yield result_holder
    except Exception:
        proxy_instance.logging_module.exception('Unexpected failure in proxy context manager')
    finally:
        proxy_instance.stop()
        if record:
            if result_holder.get(RESULT):
                proxy_instance.normalize_mock_file(playbook_or_integration_id)
                proxy_instance.move_mock_file_to_repo(playbook_or_integration_id)
                proxy_instance.successful_rerecord_count += 1
                proxy_instance.rerecorded_tests.append(playbook_or_integration_id)
                if proxy_instance.should_update_mock_repo:
                    proxy_instance.logging_module.debug("committing new/updated mock files to mock git repo.")
                    proxy_instance.commit_mock_file(playbook_or_integration_id)
            else:
                proxy_instance.failed_rerecord_count += 1
                proxy_instance.failed_rerecord_tests.append(playbook_or_integration_id)
            proxy_instance.set_repo_folder()

        else:
            if result_holder.get(RESULT):
                proxy_instance.successful_tests_count += 1
            else:
                proxy_instance.failed_tests_count += 1


# demisto_sdk/commands/test_content/timestamp_replacer.py

"""BEWARE:

This file is used outside of the demisto-sdk.
DO NOT import anything from custom packages, only use builtins.
"""

import functools
import json
import logging
import urllib.parse
from ast import literal_eval
from collections import OrderedDict
from copy import deepcopy
from os import path
from time import ctime
from typing import List, Union

from dateparser import parse
from mitmproxy import ctx
from mitmproxy.addonmanager import Loader
from mitmproxy.addons.serverplayback import ServerPlayback
from mitmproxy.http import HTTPFlow, Request
from mitmproxy.script import concurrent

logging.basicConfig(
    level=logging.DEBUG,
    format='[%(asctime)s] - [%(funcName)s] - %(message)s'
)


def record_concurrently(replaying: bool = False):
    """
    A decorator to return a decorator that just executes the function it decorates normally if 'replaying' is true,
    (AKA mitmdump is executing in server-replay mode or reading in a mock file and cleaning it and saving the cleaned
    mock to a new file), otherwise pass the 'concurrent' decorator so that when mitmdump is executing in recording
    mode, that requests will be processed concurrently and not be blocking which can cause proxy errors during
    recording if multiple requests are made in a short timespan.

    Arguments:
        replaying (bool): True if timestamp replacer script is running in server playback mode or cleaning a mock file

    Returns:
        (function): decorator
    """
    logging.info(f'replaying={replaying}')
    if replaying:
        def nonconcurrent_decorator(func):
            @functools.wraps(func)
            def passthrough_wrapper(*args, **kwargs):
                value = func(*args, **kwargs)
                return value

            return passthrough_wrapper

        return nonconcurrent_decorator
    else:
        return concurrent


class TimestampReplacer:
    def __init__(self):
        self.count = 0
        self.constant = 'constant_value'
        self.json_keys = set()
        self.form_keys = set()
        self.query_keys = set()
        self.bad_keys_filepath = ''
        self.detect_timestamps = False

    def load(self, loader: Loader):
        loader.add_option(
            name='detect_timestamps',
            typespec=bool,
            default=False,
            help="""
            Set to True only if recording a mock file. Used to determine which keys need to be replaced in incoming
            request bodies during a mock playback.
            """
        )
        loader.add_option(
            name='keys_filepath',
            typespec=str,
            default='problematic_keys.json',  # disable-secrets-detection
            help="""
            The path to the file that contains the problematic keys for the test playbook recording that resides
            in the same directory.
            """
        )
        loader.add_option(
            name='script_mode',
            typespec=str,
            default='playback',
            help="""
            The mode that timestamp_replacer.py is being executed in. The options are 'record', 'clean', and
            'playback'. If no option is provided, defaults to 'playback'.
            """
        )
        loader.add_option(
            name='debug',
            typespec=bool,
            default=False,
            help="""
            Set to True to print out additional information for each request that comes in.
            """
        )

    def running(self):
        if ctx.options.debug:
            logging.info(f'ctx.options={ctx.options}')

        self.bad_keys_filepath = ctx.options.keys_filepath
        if ctx.options.detect_timestamps:
            logging.info('Detecting Timestamp Fields')
            self.detect_timestamps = True
        self.load_problematic_keys()

    def _debug_request(self, flow: HTTPFlow) -> None:
        """Print details of the request"""
        req = flow.request
        logging.info(f'{req.method} {req.pretty_url}')
        _, _, _, _, query, _ = urllib.parse.urlparse(req.url)
        queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
        logging.info(f'queriesArray={queriesArray}')
        if req.multipart_form:
            logging.info(f'multipart_form data = {req.multipart_form.items()}')
        if req.urlencoded_form:
            logging.info(f'urlencoded_form data = {req.urlencoded_form.items()}')
        logging.info(f'hashed_data={ServerPlayback._hash(self, flow)}')  # type: ignore

    # @record_concurrently(
    #     replaying=bool(
    #         ctx.options.server_replay or (ctx.options.rfile and ctx.options.save_stream_file)
    #     )
    # )
    def request(self, flow: HTTPFlow) -> None:
        self.count += 1
        if ctx.options.debug:
            self._debug_request(flow)
        req = flow.request
        if req.multipart_form and 'boundary' in req.headers.get('Content-Type', ''):
            self.replace_boundary(req)
        if ctx.options.script_mode == 'record':
            if ctx.options.detect_timestamps:
                self.run_all_key_detections(req)
                logging.info('updating problem_keys file at "{}"'.format(self.bad_keys_filepath))
                self.update_problem_keys_file()
        elif ctx.options.script_mode in {'clean', 'playback'}:
            logging.info(f'flow.live is: {flow.live}')
            flow.live = False
            logging.info(f'mode={ctx.options.script_mode} cleaning problematic key values from the request')
            self.clean_bad_keys(req)

    @staticmethod
    def replace_boundary(req: Request):
        fixed_boundary = 'fixed_boundary'
        content_type_header, old_boundary = req.headers['Content-Type'].split('boundary=')
        req.headers['Content-Type'] = 'boundary='.join([content_type_header, fixed_boundary])
        req.content = req.content.replace(old_boundary.encode(), fixed_boundary.encode())  # type: ignore

    def clean_bad_keys(self, req: Request) -> None:
        """Modify the request so that values of problematic keys are constant data

        Args:
            req (Request): The request to modify
        """
        self.clean_url_query(req)
        self.clean_urlencoded_form(req)
        self.clean_multipart_form(req)
        self.clean_json_body(req)

    def clean_url_query(self, req: Request) -> None:
        """Replace any problematic values of query parameters with constant data

        Args:
            req (Request): The request to modify
        """
        query_data = sorted(req._get_query())
        logging.info('fetched query_data: {}'.format(query_data))
        updated_query_data = []
        if query_data and self.query_keys:
            for key, val in query_data:
                if key in self.query_keys:
                    updated_query_data.append((key, self.constant))
                else:
                    updated_query_data.append((key, val))
        req._set_query(updated_query_data or query_data)
        logging.info(f'updated query_data: {req._get_query()}')

    def clean_urlencoded_form(self, req: Request) -> None:
        """Replace any problematic values of urlencoded form keys with constant data

        Args:
            req (Request): The request to modify
        """
        if req.urlencoded_form and self.form_keys:
            updated_urlencoded_form_data = []
            for key, val in req.urlencoded_form.items(multi=True):
                if key in self.form_keys:
                    updated_urlencoded_form_data.append((key, self.constant))
                else:
                    updated_urlencoded_form_data.append((key, val))
            req._set_urlencoded_form(updated_urlencoded_form_data)

    def clean_multipart_form(self, req: Request) -> None:
        """Replace any problematic values of multipart form keys with constant data

        Args:
            req (Request): The request to modify
        """
        if req.multipart_form and self.form_keys:
            updated_multipart_form_data = []
            for key, val in req.multipart_form.items(multi=True):
                if key in self.form_keys:
                    updated_multipart_form_data.append((key, self.constant))
                else:
                    updated_multipart_form_data.append((key, val))
            req._set_multipart_form(updated_multipart_form_data)

    def clean_json_body(self, req: Request) -> None:
        """Replace any problematic values of keys in the request's json body (if it has one)

        Args:
            req (Request): The request to modify
        """
        if req.method == 'POST':
            raw_content = req.raw_content
            if raw_content is not None:
                try:
                    content = raw_content.decode()
                except UnicodeDecodeError:
                    logging.error('Failed to decode request content')
                    content = ''
            else:
                content = ''
            logging.info(f'cleaning json body: content={content}')
            json_data = content.startswith('{')
            if json_data:
                try:
                    content = OrderedDict(literal_eval(content))  # type: ignore
                    self.modify_json_body(req, content)  # type: ignore
                    return
                except Exception:
                    logging.exception(f'failed to run literal_eval on content {content}')
                try:
                    logging.info('parsing the request body with "literal_eval" failed - trying with "json.loads"')
                    content = json.loads(content)
                    self.modify_json_body(req, content)  # type: ignore
                except Exception:
                    logging.exception(f'failed to run json.loads on content {content}')

    def modify_json_body(self, req: Request, json_body: dict) -> None:
        """Modify the json body of a request by replacing any timestamp data with constant data

        Args:
            req (Request): The request whose json body will be modified.
            json_body (dict): The request body to modify.
        """
        original_content = deepcopy(json_body)
        modified = False
        keys_to_replace = self.json_keys
        logging.info('{}'.format(keys_to_replace))
        for key_path in keys_to_replace:
            body = json_body
            keys = key_path.split('.')
            logging.info('keypath parts: {}'.format(keys))
            lastkey = keys[-1]
            logging.info('lastkey: {}'.format(lastkey))
            skip_key = False
            for k in keys[:-1]:
                if k in body:
                    body = body[k]
                elif isinstance(body, list) and k.isdigit():
                    if int(k) > len(body) - 1:
                        skip_key = True
                        break
                    body = body[int(k)]
                else:
                    skip_key = True
                    break
            if not skip_key:
                if lastkey in body:
                    logging.info('modifying request to "{}"'.format(req.pretty_url))
                    body[lastkey] = self.constant
                    modified = True
                elif isinstance(body, list) and lastkey.isdigit() and int(lastkey) <= len(body) - 1:
                    logging.info('modifying request to "{}"'.format(req.pretty_url))
                    body[int(lastkey)] = self.constant
                    modified = True
        if modified:
            logging.info('original request body:\n{}'.format(json.dumps(original_content, indent=4)))
            logging.info('modified request body:\n{}'.format(json.dumps(json_body, indent=4)))
            req.set_content(json.dumps(json_body).encode())

    def run_all_key_detections(self, req: Request) -> None:
        """Used to detect problematic keys in
        1. request query parameters
        2. urlencoded forms parameters and multipart forms parameters
        3. json request body

        Args:
            req (Request): The request to inspect for problematic keys
        """
        self.handle_url_query(req)
        self.handle_urlencoded_form(req)
        self.handle_multipart_form(req)
        self.handle_json_body(req)

    def handle_url_query(self, req: Request) -> None:
        query_data = req._get_query()
        logging.info('query_data: {}'.format(query_data))
        for key, val in query_data:
            # don't bother trying to interpret an argument less than 4 characters as some type of timestamp
            if len(val) > 4:
                if self.safely_parse(val):
                    self.query_keys.add(key)

    def handle_multipart_form(self, req: Request) -> None:
        """Used when detecting what keys in a multipart form to replace with constants.

        Args:
            req (Request): The request to inspect
        """
        if req.multipart_form:
            for key, val in req.multipart_form.items(multi=True):
                # don't bother trying to interpret an argument less than 4 characters as some type of timestamp
                if len(val) > 4:
                    if self.safely_parse(val):
                        self.form_keys.add(key)

    def handle_urlencoded_form(self, req: Request) -> None:
        """Used when detecting what keys in an url encoded parameters to replace with constants.

        Args:
            req (Request): The request to inspect
        """
        if req.urlencoded_form:
            for key, val in req.urlencoded_form.items(multi=True):
                # don't bother trying to interpret an argument less than 4 characters as some type of timestamp
                if len(val) > 4:
                    if self.safely_parse(val):
                        self.form_keys.add(key)

    def handle_json_body(self, req: Request) -> None:
        """Used when detecting what keys in a request's json body to replace with constants.

        Args:
            req (Request): The request to inspect
        """
        if req.method == 'POST':
            raw_content = req.raw_content
            if raw_content is not None:
                try:
                    content = raw_content.decode()
                except UnicodeDecodeError:
                    logging.error('Failed to decode request content')
                    content = ''
            else:
                content = ''
            logging.info(f'handling json body: content={content}')
            json_data = content.startswith('{')
            if json_data:
                try:
                    content = OrderedDict(literal_eval(content))  # type: ignore
                    json_keys = self.determine_problematic_keys(content)  # type: ignore
                    self.json_keys.update(json_keys)
                    return
                except Exception:
                    logging.exception(f'failed to run literal_eval content: {content}')
                try:
                    logging.info('parsing the request body with "literal_eval" failed - trying with "json.loads"')
                    content = json.loads(content)
                    json_keys = self.determine_problematic_keys(content)  # type: ignore
                    self.json_keys.update(json_keys)
                except Exception:
                    logging.exception(f'failed to run json.loads on content {content}')

    def determine_problematic_keys(self, content: dict) -> List[str]:
        """Given a json request body, return the keys (in dot notation) whose values are potentially timestamp data.

        Args:
            content (dict): The json request body to iterate through and find problematic timestamp data.

        Returns:
            List[str]: A list of keys (in dot notation, e.g. 'query.filter.time' is an example of what could be one
                problematic key) whose values are potentially timestamp data.
        """
        def travel_dict(obj: Union[dict, list], key_path='') -> List[str]:
            bad_key_paths = []
            if isinstance(obj, dict):
                for key, val in obj.items():
                    sub_key_path = '{}.{}'.format(key_path, key) if key_path else key
                    if isinstance(val, (list, dict)):
                        bad_key_paths.extend(travel_dict(val, sub_key_path))
                    else:
                        is_string = isinstance(val, str) and len(val) > 4
                        possible_timestamp = isinstance(val, (int, float)) and len(str(val)) >= 8
                        if is_string or possible_timestamp:
                            for_eval = val
                            if possible_timestamp:
                                if isinstance(for_eval, float):
                                    digits = str(val).split('.')
                                    for_eval = digits[0]
                                if len(str(for_eval)) < 13:
                                    parsed_date = self.safely_parse(ctime(val))
                                else:
                                    parsed_date = self.safely_parse(ctime(val / 1000.0))
                            else:
                                parsed_date = self.safely_parse(val)
                            # if parsed_date is not None then successfully interpreted value as some sort of
                            # time related thingieding
                            if parsed_date:
                                bad_key_paths.append(sub_key_path)
            elif isinstance(obj, list):
                for i, val in enumerate(obj):
                    sub_key_path = '{}.{}'.format(key_path, i) if key_path else i
                    if isinstance(val, (list, dict)):
                        bad_key_paths.extend(travel_dict(val, sub_key_path))
                    else:
                        is_string = isinstance(val, str) and len(val) > 4
                        possible_timestamp = isinstance(val, (int, float)) and len(str(val)) >= 8
                        if is_string or possible_timestamp:
                            for_eval = val
                            if possible_timestamp:
                                if isinstance(for_eval, float):
                                    digits = str(val).split('.')
                                    for_eval = digits[0]
                                if len(str(for_eval)) < 13:
                                    parsed_date = self.safely_parse(ctime(val))
                                else:
                                    parsed_date = self.safely_parse(ctime(val / 1000.0))
                            else:
                                parsed_date = self.safely_parse(val)
                            # if parsed_date is not None then successfully interpreted value as some sort of
                            # time related thingieding
                            if parsed_date:
                                bad_key_paths.append(sub_key_path)
            return bad_key_paths

        bad_keys = travel_dict(content)
        return bad_keys

    def update_problem_keys_file(self):
        """Update the problem keys dictionary at the keys_filepath with new problematic keys"""
        existing_problem_keys = self.read_in_problematic_keys()
        for key, val in existing_problem_keys.items():
            if key == 'keys_to_replace':
                existing_problem_keys[key] = ' '.join(set(val.split()).union(self.json_keys))
            elif key == 'server_replay_ignore_payload_params':
                existing_problem_keys[key] = ' '.join(set(val.split()).union(self.form_keys))
            elif key == 'server_replay_ignore_params':
                existing_problem_keys[key] = ' '.join(set(val.split()).union(self.query_keys))
        self.write_out_problematic_keys(existing_problem_keys)

    def read_in_problematic_keys(self):
        """Load problematic keys dictionary from the keys_filepath argument filepath in content-test-data repo
        if it exists. Otherwise, return the dictionary with empty values.
        """
        logging.info('executing "read_in_problematic_keys" method')
        repo_bad_keys_filepath = self.bad_keys_filepath.replace('/tmp/Mocks', 'content-test-data')
        logging.info('reading in problematic keys data from "{}"'.format(repo_bad_keys_filepath))
        if not path.exists(self.bad_keys_filepath) and path.exists(repo_bad_keys_filepath):
            with open(repo_bad_keys_filepath, 'r') as fp:
                problem_keys = json.load(fp)
        elif path.exists(self.bad_keys_filepath):
            with open(self.bad_keys_filepath, 'r') as fp:
                problem_keys = json.load(fp)
        else:
            problem_keys = {
                'keys_to_replace': '',
                'server_replay_ignore_params': '',
                'server_replay_ignore_payload_params': ''
            }
        return problem_keys

    def write_out_problematic_keys(self, problem_keys: dict):
        """Write updated problematic keys dictionary back to the file at the keys_filepath argument

        Args:
            problem_keys (dict): Updated dictionary of problematic keys
        """
        with open(self.bad_keys_filepath, 'w') as bad_keys_file:
            bad_keys_file.write(json.dumps(problem_keys, indent=4))

    def load_problematic_keys(self):
        """Load problematic keys from the keys_filepath argument filepath if it exists. Only necessary when running
        mitmdump in playback mode. Resets command line options with the key value pairs from the loaded dictionary.
        """
        logging.info('executing "load_problematic_keys" method')
        if path.exists(self.bad_keys_filepath):
            logging.info('"{}" path exists - loading bad keys'.format(self.bad_keys_filepath))

            problem_keys = json.load(open(self.bad_keys_filepath, 'r'))

            query_keys = problem_keys.get('server_replay_ignore_params')
            self.query_keys.update(query_keys.split() if isinstance(query_keys, str) else query_keys)
            form_keys = problem_keys.get('server_replay_ignore_payload_params')
            self.form_keys.update(form_keys.split() if isinstance(form_keys, str) else form_keys)
            json_keys = problem_keys.get('keys_to_replace')
            self.json_keys.update(json_keys.split() if isinstance(json_keys, str) else json_keys)

            logging.info('bad keys loaded\n---------------')
            logging.info(f'self.query_keys={self.query_keys}')
            logging.info(f'self.form_keys={self.form_keys}')
            logging.info(f'self.json_keys={self.json_keys}')
        else:
            logging.info('"{}" path doesn\'t exist - no bad keys to set'.format(self.bad_keys_filepath))
            logging.info('not setting bad keys from file')

    @staticmethod
    def safely_parse(val):
        """
        Safely tries to parse a value as a datetime object.
        If it fails - logs the error and the malformed output.
        Args:
            val: The val to parse

        Returns:
            True if the val was parsed to datetime object and False otherwise
        """
        try:
            if parse(val):
                return True
        except Exception as exc:
            logging.exception(f'Failed to parse as date object: {val}', exc)
        return False


# mitmproxy picks up the contents of the addons global list and loads what it finds into the addons mechanism.
addons = [TimestampReplacer()]


# demisto_sdk/commands/test_content/tools.py

import ast
import logging
from copy import deepcopy
from pprint import pformat
from subprocess import STDOUT, CalledProcessError, check_output

import demisto_client

from demisto_sdk.commands.test_content.constants import SSH_USER


def update_server_configuration(client, server_configuration,
                                error_msg, logging_manager=logging, config_keys_to_delete=None):
    """updates server configuration

    Args:
        client (demisto_client): The configured client to use.
        server_configuration (dict): The server configuration to be added
        error_msg (str): The error message
        logging_manager (logging.Logger): Logging manager object
        config_keys_to_delete (set): The server configuration keys to be deleted

    Returns:
        response_data: The response data
        status_code: The response status code
        prev_system_conf: Previous stored system conf
    """
    system_conf_response = demisto_client.generic_request_func(
        self=client,
        path='/system/config',
        method='GET'
    )
    system_conf = ast.literal_eval(system_conf_response[0]).get('sysConf', {})
    logging_manager.debug(f'Current server configurations are {pformat(system_conf)}')

    prev_system_conf = deepcopy(system_conf)

    if config_keys_to_delete:
        for key in config_keys_to_delete:
            system_conf.pop(key, None)

    if server_configuration:
        system_conf.update(server_configuration)

    data = {
        'data': system_conf,
        'version': -1
    }
    response_data, status_code, _ = demisto_client.generic_request_func(self=client, path='/system/config',
                                                                        method='POST', body=data)

    logging_manager.debug(f'Updating server configurations with {pformat(system_conf)}')

    try:
        result_object = ast.literal_eval(response_data)
        logging_manager.debug(f'Updated server configurations with response: {pformat(result_object)}')
    except ValueError as err:
        logging_manager.exception(f'failed to parse response from demisto. response is {response_data}.\nError:\n{err}')
        return

    if status_code >= 300 or status_code < 200:
        message = result_object.get('message', '')
        logging_manager.error(f'{error_msg} {status_code}\n{message}')
    return response_data, status_code, prev_system_conf


def is_redhat_instance(instance_ip: str) -> bool:
    """
    As part of the AMI creation - in case the AMI is RHEL a file named '/home/ec2-user/rhel_ami' is created as
    an indication.
    If not
    Args:
        instance_ip: The instance IP to check.

    Returns:
        True if the file '/home/ec2-user/rhel_ami' exists on the instance, else False
    """
    try:
        check_output(f'ssh {SSH_USER}@{instance_ip} ls -l /home/ec2-user/rhel_ami'.split(),
                     stderr=STDOUT)
        return True
    except CalledProcessError:
        return False


def get_ui_url(client_host):
    """

    Args:
        client_host: the client host

    Returns: the UI URL of the server. For XSIAM we remove the 'api-' prefix to get the UI URL of the server,
    whereas for XSOAR it will remain the same (the UI URL is the same).

    """
    return client_host.replace('https://api-', 'https://')


# demisto_sdk/commands/test_content/test_modeling_rule/tests/init_test_data_test.py
from pathlib import Path

from typer.testing import CliRunner

from demisto_sdk.commands.test_content.xsiam_tools.test_data import TestData

ONE_MODEL_RULE_TEXT = """
[MODEL: dataset=fake_fakerson_raw]
alter
    xdm.session_context_id = externalId,
    xdm.observer.action = act,
    xdm.event.outcome = outcome,
    xdm.event.outcome_reason = reason,
    xdm.network.http.method = requestMethod,
    xdm.network.http.url = request,
    xdm.source.host.hostname = devicehostname,
    xdm.source.host.ipv4_addresses = arraycreate(coalesce(src, "")),
    xdm.target.host.ipv4_addresses = arraycreate(coalesce(dst, "")),
    xdm.network.application_protocol_category = cat,
    xdm.network.protocol_layers = arraycreate(coalesce(app, "")),
    xdm.source.user.username = suser,
    xdm.source.zone = spriv,
    xdm.network.http.domain = dhost,
    xdm.network.http.response_code = outcome,
    xdm.target.sent_bytes = to_integer(out),
    xdm.network.http.url_category = cs2,
    xdm.network.http.content_type = contenttype,
    xdm.alert.category = cs4,
    xdm.alert.name = cs5,
    xdm.alert.severity = to_string(cn1),
    xdm.observer.name = _reporting_device_name,
    xdm.source.user_agent = requestClientApplication,
    xdm.target.interface = destinationServiceName,
    xdm.source.ipv4 = sourceTranslatedAddress,
    xdm.event.type = FakeFakersonURLClass,
    xdm.observer.product = _product,
    xdm.observer.vendor = _vendor,
    xdm.target.process.executable.file_type = fileType;
"""
DEFAULT_MODELING_RULE_NAME = 'TestModelingRule'
DEFAULT_MODELING_RULE_NAME_2 = 'TestModelingRule2'


def test_init_test_data_create(pack):
    """
    Given:
        - A path to a directory of a modeling rule.
        - The number of events to initialize the test data file for.
    When:
        - The test data file does not exist.
    Then:
        - Ensure the test data file is created.
        - Ensure the test data file contains the correct number of events.
    """
    from demisto_sdk.commands.test_content.test_modeling_rule.init_test_data import app as init_test_data_app
    runner = CliRunner()
    pack.create_modeling_rule(DEFAULT_MODELING_RULE_NAME, rules=ONE_MODEL_RULE_TEXT)
    mrule_dir = Path(pack._modeling_rules_path / DEFAULT_MODELING_RULE_NAME)
    test_data_file = mrule_dir / f'{DEFAULT_MODELING_RULE_NAME}_testdata.json'
    assert test_data_file.exists() is False
    count = 1
    result = runner.invoke(init_test_data_app, [mrule_dir.as_posix(), f'--count={count}'])
    assert result.exit_code == 0
    assert test_data_file.exists() is True
    test_data = TestData.parse_file(test_data_file.as_posix())
    assert len(test_data.data) == count


def test_init_test_data_update_with_unchanged_modeling_rule(pack):
    """
    Given:
        - A path to a directory of a modeling rule.
        - The number of events to initialize the test data file for is greater than what already
            exists in the test data file.
    When:
        - The test data file exists.
    Then:
        - Ensure the test data file contains the correct number of events.
    """
    from demisto_sdk.commands.test_content.test_modeling_rule.init_test_data import app as init_test_data_app
    runner = CliRunner()
    pack.create_modeling_rule(DEFAULT_MODELING_RULE_NAME, rules=ONE_MODEL_RULE_TEXT)
    mrule_dir = Path(pack._modeling_rules_path / DEFAULT_MODELING_RULE_NAME)
    test_data_file = mrule_dir / f'{DEFAULT_MODELING_RULE_NAME}_testdata.json'
    count = 1
    result = runner.invoke(init_test_data_app, [mrule_dir.as_posix(), f'--count={count}'])
    assert result.exit_code == 0
    assert test_data_file.exists() is True
    test_data = TestData.parse_file(test_data_file.as_posix())
    assert len(test_data.data) == count
    count = 2
    result = runner.invoke(init_test_data_app, [mrule_dir.as_posix(), f'--count={count}'])
    assert result.exit_code == 0
    assert test_data_file.exists() is True
    test_data = TestData.parse_file(test_data_file.as_posix())
    assert len(test_data.data) == count


def test_init_test_data_update_with_reduced_modeling_rule(pack):
    """
    Given:
        - A path to a directory of a modeling rule.
        - The number of events to initialize the test data file for is greater than what already
            exists in the test data file.
        - A field have been removed from the modeling rule.
    When:
        - The test data file exists.
        - The modeling rule has changed.
    Then:
        - Ensure fields that no longer exist in the modeling rule are removed from the keys of
            the expected values dictionary for the previously existing events.
        - Ensure the test data file contains the correct number of events.
    """
    from demisto_sdk.commands.test_content.test_modeling_rule.init_test_data import app as init_test_data_app
    runner = CliRunner()
    pack.create_modeling_rule(DEFAULT_MODELING_RULE_NAME, rules=ONE_MODEL_RULE_TEXT)
    mrule_dir = Path(pack._modeling_rules_path / DEFAULT_MODELING_RULE_NAME)
    test_data_file = mrule_dir / f'{DEFAULT_MODELING_RULE_NAME}_testdata.json'
    count = 1
    result = runner.invoke(init_test_data_app, [mrule_dir.as_posix(), f'--count={count}'])
    assert result.exit_code == 0
    assert test_data_file.exists() is True
    test_data = TestData.parse_file(test_data_file.as_posix())
    assert len(test_data.data) == count

    # verify field exists in the expected values dictionary for test data event 0
    field_to_remove = 'xdm.source.user.username'
    assert test_data.data[0].expected_values and field_to_remove in test_data.data[0].expected_values
    updated_rule_text = ONE_MODEL_RULE_TEXT.replace(f'\n    {field_to_remove} = suser,', '')

    # update the modeling rule with the updated rule text
    with open(mrule_dir / f'{DEFAULT_MODELING_RULE_NAME}.xif', 'w') as f:
        f.write(updated_rule_text)

    count = 2
    result = runner.invoke(init_test_data_app, [mrule_dir.as_posix(), f'--count={count}'])
    assert result.exit_code == 0
    assert test_data_file.exists() is True
    test_data = TestData.parse_file(test_data_file.as_posix())
    assert len(test_data.data) == count
    for test_data_event in test_data.data:
        assert test_data_event.expected_values and field_to_remove not in test_data_event.expected_values


def test_init_test_data_update_with_extended_modeling_rule(pack):
    """
    Given:
        - A path to a directory of a modeling rule.
        - The number of events to initialize the test data file for is greater than what already
            exists in the test data file.
        - A field have been added to the modeling rule.
    When:
        - The test data file exists.
        - The modeling rule has changed.
    Then:
        - Ensure the test data file contains the correct number of events.
        - Ensure the new field is added to the expected values dictionary for the previously existing
            events.
    """
    from demisto_sdk.commands.test_content.test_modeling_rule.init_test_data import app as init_test_data_app
    runner = CliRunner()

    field_to_add = 'xdm.source.user.username'
    reduced_rule_text = ONE_MODEL_RULE_TEXT.replace(f'\n    {field_to_add} = suser,', '')

    pack.create_modeling_rule(DEFAULT_MODELING_RULE_NAME, rules=reduced_rule_text)
    mrule_dir = Path(pack._modeling_rules_path / DEFAULT_MODELING_RULE_NAME)
    test_data_file = mrule_dir / f'{DEFAULT_MODELING_RULE_NAME}_testdata.json'
    count = 1
    result = runner.invoke(init_test_data_app, [mrule_dir.as_posix(), f'--count={count}'])
    assert result.exit_code == 0
    assert test_data_file.exists() is True
    test_data = TestData.parse_file(test_data_file.as_posix())
    assert len(test_data.data) == count

    # verify field does not exist in the expected values dictionary for test data event 0
    assert test_data.data[0].expected_values and field_to_add not in test_data.data[0].expected_values

    # update the modeling rule with the updated rule text
    with open(mrule_dir / f'{DEFAULT_MODELING_RULE_NAME}.xif', 'w') as f:
        f.write(ONE_MODEL_RULE_TEXT)

    count = 2
    result = runner.invoke(init_test_data_app, [mrule_dir.as_posix(), f'--count={count}'])
    assert result.exit_code == 0
    assert test_data_file.exists() is True
    test_data = TestData.parse_file(test_data_file.as_posix())
    assert len(test_data.data) == count
    for test_data_event in test_data.data:
        assert test_data_event.expected_values and field_to_add in test_data_event.expected_values


class TestInitTestDataMultiInput:
    """Test different multi input scenarios for init test data command."""

    def test_init_test_data_multi_input_all_valid(self, pack):
        """
        Given:
            - Paths to directories of two modeling rules.
            - The number of events to initialize the test data files for.
        When:
            - The test data files do not exist.
            - Both modeling rule directories are valid path inputs.
        Then:
            - Ensure the test data files are created.
            - Ensure the test data files contain the correct number of events.
        """
        from demisto_sdk.commands.test_content.test_modeling_rule.init_test_data import app as init_test_data_app
        runner = CliRunner()
        pack.create_modeling_rule(DEFAULT_MODELING_RULE_NAME, rules=ONE_MODEL_RULE_TEXT)
        pack.create_modeling_rule(DEFAULT_MODELING_RULE_NAME_2, rules=ONE_MODEL_RULE_TEXT)
        mrule_dir = Path(pack._modeling_rules_path / DEFAULT_MODELING_RULE_NAME)
        mrule_dir_2 = Path(pack._modeling_rules_path / DEFAULT_MODELING_RULE_NAME_2)
        test_data_file = mrule_dir / f'{DEFAULT_MODELING_RULE_NAME}_testdata.json'
        test_data_file_2 = mrule_dir_2 / f'{DEFAULT_MODELING_RULE_NAME_2}_testdata.json'
        count = 1
        result = runner.invoke(init_test_data_app, [mrule_dir.as_posix(), mrule_dir_2.as_posix(), f'--count={count}'])
        assert result.exit_code == 0
        assert test_data_file.exists() is True
        assert test_data_file_2.exists() is True
        test_data = TestData.parse_file(test_data_file.as_posix())
        test_data_2 = TestData.parse_file(test_data_file_2.as_posix())
        assert len(test_data.data) == count
        assert len(test_data_2.data) == count

    def test_init_test_data_multi_input_some_invalid(self, pack):
        """
        Given:
            - Paths to directories of two modeling rules.
            - The number of events to initialize the test data files for.
        When:
            - The test data files do not exist.
            - The first modeling rule directory is an invalid path input.
            - The second modeling rule directory is a valid path input.
        Then:
            - Ensure the test data file for the second modeling rule directory is created.
            - Ensure the test data file contain the correct number of events.
            - Ensure the command returns a non-zero exit code.
        """
        from demisto_sdk.commands.test_content.test_modeling_rule.init_test_data import app as init_test_data_app
        runner = CliRunner()
        pack.create_modeling_rule(DEFAULT_MODELING_RULE_NAME, rules=ONE_MODEL_RULE_TEXT)
        pack.create_modeling_rule(DEFAULT_MODELING_RULE_NAME_2, rules=ONE_MODEL_RULE_TEXT)
        mrule_dir = Path(pack._modeling_rules_path / DEFAULT_MODELING_RULE_NAME)
        mrule_dir_2 = Path(pack._modeling_rules_path / DEFAULT_MODELING_RULE_NAME_2)
        test_data_file = mrule_dir / f'{DEFAULT_MODELING_RULE_NAME}_testdata.json'
        test_data_file_2 = mrule_dir_2 / f'{DEFAULT_MODELING_RULE_NAME_2}_testdata.json'
        count = 1
        result = runner.invoke(init_test_data_app, [pack.path, mrule_dir.as_posix(), f'--count={count}'])
        assert result.exit_code != 0
        assert test_data_file.exists() is True
        assert test_data_file_2.exists() is False
        test_data = TestData.parse_file(test_data_file.as_posix())
        assert len(test_data.data) == count
