_module: The logging module to use

        """
        cmd = cls._build_kill_cmd(server_ip, container_name)
        _, stderr = cls.run_shell_command(cmd)

        if stderr:
            logging_module.debug(
                f"Failed killing container: {container_name}\nAdditional information: {stderr}"
            )

    @classmethod
    def get_docker_pid_info(cls, server_ip, container_id, logging_module):
        """Executes docker exec ps command on remote machine using ssh.

        Args:
            server_ip (str): The remote server ip address.
            container_id (str): Docker container id.
            logging_module: The logging module to use

        Returns:
            str: output of executed command.
        """
        cmd = cls._build_pid_info_cmd(server_ip, container_id)
        stdout, stderr = cls.run_shell_command(cmd)

        if stderr:
            ignored_warning_message = f"Connection to {server_ip} closed"
            if ignored_warning_message not in stderr:
                logging_module.debug(
                    f"Failed getting pid info for container id: {container_id}.\n"
                    f"Additional information: {stderr}"
                )

        return stdout

    @classmethod
    def check_resource_usage(
        cls,
        server_url,
        docker_images,
        def_memory_threshold,
        def_pid_threshold,
        docker_thresholds,
        logging_module,
    ):
        """
        Executes docker stats command on remote machine and returns error message in case of exceeding threshold.

        Args:
            server_url (str): Target machine full url.
            docker_images (set): Set of docker images to check their resource usage.
            def_memory_threshold (int): Memory threshold of specific docker container, in Mib.
            def_pids_threshold (int): PIDs threshold of specific docker container, in Mib.
            docker_thresholds: thresholds per docker image
            logging_module: The logging module that should be used.

        Returns:
            str: The error message. Empty in case that resource check passed.

        """
        server_ip = server_url.lstrip("https://")
        containers_stats = cls.docker_stats(server_ip, docker_images, logging_module)
        error_message = ""

        for container_stat in containers_stats:
            failed_memory_test = False
            container_name = container_stat["container_name"]
            container_id = container_stat["container_id"]
            memory_usage = container_stat["memory_usage"]
            pids_usage = container_stat["pids"]
            image_full = cls.get_image_for_container_id(
                server_ip, container_id, logging_module
            )  # get full name (ex: demisto/slack:1.0.0.4978)
            image_name = image_full.split(":")[0]  # just the name such as demisto/slack

            memory_threshold = (
                docker_thresholds.get(image_full, {}).get("memory_threshold")
                or docker_thresholds.get(image_name, {}).get("memory_threshold")
                or def_memory_threshold
            )
            pid_threshold = (
                docker_thresholds.get(image_full, {}).get("pid_threshold")
                or docker_thresholds.get(image_name, {}).get("pid_threshold")
                or def_pid_threshold
            )
            logging_module.debug(
                f"Checking container: {container_name} "
                f"(image: {image_full}) for memory: {memory_threshold} pid: {pid_threshold} thresholds "
                f"with actual values: memory {memory_usage} pids: {pids_usage} ..."
            )
            if memory_usage > memory_threshold:
                error_message += (
                    "Failed docker resource test. Docker container {} exceeded the memory threshold, "
                    "configured: {} MiB and actual memory usage is {} MiB.\n"
                    "Fix container memory usage or add `memory_threshold` key to failed test "
                    "in conf.json with value that is greater than {}\n".format(
                        container_name, memory_threshold, memory_usage, memory_usage
                    )
                )
                failed_memory_test = True
            if pids_usage > pid_threshold:
                error_message += (
                    "Failed docker resource test. Docker container {} exceeded the pids threshold, "
                    "configured: {} and actual pid number is {}.\n"
                    "Fix container pid usage or add `pid_threshold` key to failed test "
                    "in conf.json with value that is greater than {}\n".format(
                        container_name, pid_threshold, pids_usage, pids_usage
                    )
                )
                additional_pid_info = cls.get_docker_pid_info(
                    server_ip, container_id, logging_module
                )
                if additional_pid_info:
                    error_message += (
                        f"Additional pid information:\n{additional_pid_info}"
                    )
                failed_memory_test = True

            if failed_memory_test:
                # killing current container in case of memory resource test failure
                cls.kill_container(server_ip, container_name, logging_module)

        return error_message


# demisto_sdk/commands/test_content/IntegrationsLock.py

import datetime
import os
from contextlib import contextmanager
from typing import Generator

import pytz
import requests
from google.api_core.exceptions import NotFound, PreconditionFailed
from google.cloud import storage
from google.resumable_media.common import InvalidResponse

LOCKS_PATH = "content-locks"
BUCKET_NAME = os.environ.get("GCS_ARTIFACTS_BUCKET")
BUILD_NUM = os.environ.get("CI_BUILD_ID")
WORKFLOW_ID = os.environ.get("CI_PIPELINE_ID")
CIRCLE_STATUS_TOKEN = os.environ.get("CIRCLECI_STATUS_TOKEN")
GITLAB_STATUS_TOKEN = os.environ.get("GITLAB_STATUS_TOKEN", "")


@contextmanager
def acquire_test_lock(test_playbook) -> Generator:
    """
    This is a context manager that handles all the locking and unlocking of integrations.
    Execution is as following:
    * Attempts to lock the test's integrations and yields the result of this attempt
    * If lock attempt has failed - yields False, if it succeeds - yields True
    * Once the test is done- will unlock all integrations
    Args:
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context
    Yields:
        A boolean indicating the lock attempt result
    """
    locked = safe_lock_integrations(test_playbook)
    try:
        yield locked
    except Exception:
        test_playbook.build_context.logging_module.exception("Failed with test lock")
    finally:
        if not locked:
            return
        safe_unlock_integrations(test_playbook)


def safe_unlock_integrations(test_playbook):
    """
    This integration safely unlocks the test's integrations.
    If an unexpected error occurs - this method will log it's details and other tests execution will continue
    Args:
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context
    """
    try:
        # executing the test could take a while, re-instancing the storage client
        storage_client = storage.Client()
        unlock_integrations(test_playbook.integrations, test_playbook, storage_client)
    except Exception:
        test_playbook.build_context.logging_module.exception(
            "attempt to unlock integration failed for unknown reason."
        )


def safe_lock_integrations(test_playbook) -> bool:
    """
    This integration safely locks the test's integrations and return it's result
    If an unexpected error occurs - this method will log it's details and return False
    Args:
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context

    Returns:
        A boolean indicating the lock attempt result
    """

    integration_names = [
        integration.name for integration in test_playbook.integrations_to_lock
    ]
    if integration_names:
        print_msg = (
            f"Attempting to lock integrations {integration_names}, "
            f"with timeout {test_playbook.configuration.timeout}"
        )
    else:
        print_msg = "No integrations to lock"
    test_playbook.build_context.logging_module.debug(print_msg)
    try:
        storage_client = storage.Client()
        locked = lock_integrations(test_playbook, storage_client)
    except Exception:
        test_playbook.build_context.logging_module.exception(
            "attempt to lock integration failed for unknown reason."
        )
        locked = False
    return locked


def workflow_still_running(workflow_id: str, test_playbook) -> bool:
    """
    This method takes a workflow id and checks if the workflow is still running
    If given workflow ID is the same as the current workflow, will simply return True
    else it will query circleci api for the workflow and return the status
    Args:
        workflow_id: The ID of the workflow
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context

    Returns:
        True if the workflow is running, else False
    """
    # If this is the current workflow_id
    if workflow_id == WORKFLOW_ID:
        return True
    else:
        try:
            test_playbook.build_context.logging_module.debug(
                f"Getting status for gitlab pipeline with id: {workflow_id}"
            )
            api_v4_url = os.environ.get("CI_API_V4_URL")
            ci_project_id = os.environ.get("CI_PROJECT_ID")
            workflow_details_response = requests.get(
                f"{api_v4_url}/projects/{ci_project_id}/pipelines/{workflow_id}",
                headers={"PRIVATE-TOKEN": GITLAB_STATUS_TOKEN},
            )
            workflow_details_response.raise_for_status()
        except Exception:
            test_playbook.build_context.logging_module.exception(
                f"Failed to get gitlab-ci response about pipeline with id {workflow_id}."
            )
            return True
        return workflow_details_response.json().get("status") not in (
            "canceled",
            "success",
            "failed",
        )


def lock_integrations(test_playbook, storage_client: storage.Client) -> bool:
    """
    Locks all the test's integrations
    Args:
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context
        storage_client: The GCP storage client

    Returns:
        True if all the test's integrations were successfully locked, else False
    """
    integrations = [
        integration.name for integration in test_playbook.integrations_to_lock
    ]
    if not integrations:
        return True
    existing_integrations_lock_files = get_locked_integrations(
        integrations, storage_client
    )
    for integration, lock_file in existing_integrations_lock_files.items():
        # Each file has content in the form of <circleci-build-number>:<timeout in seconds>
        # If it has not expired - it means the integration is currently locked by another test.
        try:
            workflow_id, build_number, lock_timeout = (
                lock_file.download_as_string().decode().split(":")
            )
        except NotFound:
            test_playbook.build_context.logging_module.warning(
                "Integration lock file in not exists, continue running"
            )
            continue

        if not lock_expired(lock_file, lock_timeout) and workflow_still_running(
            workflow_id, test_playbook
        ):
            # there is a locked integration for which the lock is not expired - test cannot be executed at the moment
            test_playbook.build_context.logging_module.warning(
                f"Could not lock integration {integration}, another lock file was exist with "
                f"build number: {build_number}, timeout: {lock_timeout}, last update at {lock_file.updated}.\n"
                f"Delaying test execution"
            )
            return False
    integrations_generation_number = {}
    # Gathering generation number with which the new file will be created,
    # See https://cloud.google.com/storage/docs/generations-preconditions for details.
    for integration in integrations:
        if integration in existing_integrations_lock_files:
            integrations_generation_number[
                integration
            ] = existing_integrations_lock_files[integration].generation
        else:
            integrations_generation_number[integration] = 0
    return create_lock_files(
        integrations_generation_number, storage_client, test_playbook
    )


def create_lock_files(
    integrations_generation_number: dict, storage_client: storage.Client, test_playbook
) -> bool:
    """
    This method tries to create a lock files for all integrations specified in 'integrations_generation_number'.
    Each file should contain <circle-ci-build-number>:<test-timeout>
    where the <circle-ci-build-number> part is for debugging and troubleshooting
    and the <test-timeout> part is to be able to unlock revoked test files.
    If for any of the integrations, the lock file creation will fail- the already created files will be cleaned.
    Args:
        integrations_generation_number: A dict in the form of {<integration-name>:<integration-generation>}
        storage_client: The GCP storage client
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context

    Returns:

    """
    locked_integrations = []
    bucket = storage_client.bucket(BUCKET_NAME)
    for integration, generation_number in integrations_generation_number.items():
        blob = bucket.blob(f"{LOCKS_PATH}/{integration}")
        try:
            blob.upload_from_string(
                f"{WORKFLOW_ID}:{BUILD_NUM}:{test_playbook.configuration.timeout + 30}",
                if_generation_match=generation_number,
            )
            test_playbook.build_context.logging_module.debug(
                f"integration {integration} locked"
            )
            locked_integrations.append(integration)
        except (PreconditionFailed, InvalidResponse) as exception:
            # if this exception occurs it means that another build has locked this integration
            # before this build managed to do it.
            # we need to unlock all the integrations we have already locked and try again later

            if isinstance(exception, PreconditionFailed) or (
                exception.__cause__
                and isinstance(exception.__cause__, PreconditionFailed)
            ):
                test_playbook.build_context.logging_module.warning(
                    f"Could not lock integration {integration}, Create file with precondition failed."
                    f"delaying test execution."
                )
                unlock_integrations(locked_integrations, test_playbook, storage_client)
                return False

            raise

    return True


def unlock_integrations(
    integrations_to_unlock: list, test_playbook, storage_client: storage.Client
) -> None:
    """
    Delete all integration lock files for integrations specified in 'locked_integrations'
    Args:
        integrations_to_unlock (List[Integration]): The test playbook instance we want to test under the lock's context
        test_playbook (TestPlaybook): The test playbook instance we want to test under the lock's context
        storage_client: The GCP storage client
    """
    locked_integrations = [
        getattr(integration, "name", integration)
        for integration in integrations_to_unlock
    ]
    locked_integration_blobs = get_locked_integrations(
        locked_integrations, storage_client
    )
    for integration, lock_file in locked_integration_blobs.items():
        try:
            # Verifying build number is the same as current build number to avoid deleting other tests lock files
            _, build_number, _ = lock_file.download_as_string().decode().split(":")
            if build_number == BUILD_NUM:
                lock_file.delete(if_generation_match=lock_file.generation)
                test_playbook.build_context.logging_module.debug(
                    f"Integration {integration} unlocked"
                )
        except PreconditionFailed:
            test_playbook.build_context.logging_module.error(
                f"Could not unlock integration {integration} precondition failure"
            )


def get_locked_integrations(integrations: list, storage_client: storage.Client) -> dict:
    """
    Getting all locked integrations files
    Args:
        integrations: Integrations that we want to get lock files for
        storage_client: The GCP storage client

    Returns:
        A dict of the form {<integration-name>:<integration-blob-object>} for all integrations that has a blob object.
    """
    # Listing all files in lock folder
    # Wrapping in 'list' operator because list_blobs return a generator which can only be iterated once
    lock_files_ls = list(storage_client.list_blobs(BUCKET_NAME, prefix=f"{LOCKS_PATH}"))
    current_integrations_lock_files = {}
    # Getting all existing files details for integrations that we want to lock
    for integration in integrations:
        current_integrations_lock_files.update(
            {
                integration: [
                    lock_file_blob
                    for lock_file_blob in lock_files_ls
                    if lock_file_blob.name == f"{LOCKS_PATH}/{integration}"
                ]
            }
        )
    # Filtering 'current_integrations_lock_files' from integrations with no files
    current_integrations_lock_files = {
        integration: blob_files[0]
        for integration, blob_files in current_integrations_lock_files.items()
        if blob_files
    }
    return current_integrations_lock_files


def lock_expired(lock_file: storage.Blob, lock_timeout: str) -> bool:
    """
    Checks if the time that passed since the creation of the 'lock_file' is more then 'lock_timeout'.
    If not- it means that the integration represented by the lock file is currently locked and is tested in another build
    Args:
        lock_file: The lock file blob object
        lock_timeout: The expiration timeout of the lock in seconds

    Returns:
        True if the lock has expired it's timeout, else False
    """
    return datetime.datetime.now(tz=pytz.utc) - lock_file.updated >= datetime.timedelta(
        seconds=int(lock_timeout)
    )


# demisto_sdk/commands/test_content/ParallelLoggingManager.py

import logging
import os
import sys
from logging.handlers import QueueHandler, QueueListener
from queue import Queue
from threading import Lock, currentThread
from typing import Any, Dict, Set

import coloredlogs

ARTIFACTS_PATH = os.environ.get("ARTIFACTS_FOLDER", ".")
LOGGING_FORMAT = "[%(asctime)s] - [%(threadName)s] - [%(levelname)s] - %(message)s"
LEVEL_STYLES = {
    "critical": {"bold": True, "color": "red"},
    "debug": {"color": "cyan"},
    "error": {"color": "red"},
    "info": {},
    "warning": {"color": "yellow"},
    "success": {"color": "green"},
}


def _add_logging_level(
    level_name: str, level_num: int, method_name: str = None
) -> None:
    """
    Comprehensively adds a new logging level to the `logging` module and the
    currently configured logging class.

    `level_name` becomes an attribute of the `logging` module with the value
    `level_num`. `method_name` becomes a convenience method for both `logging`
    itself and the class returned by `logging.getLoggerClass()` (usually just
    `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is
    used.


    To avoid accidental clobberings of existing attributes, this method will
    raise an `AttributeError` if the level name is already an attribute of the
    `logging` module or if the method name is already present

    Example
    -------
    >>> _add_logging_level('TRACE', logging.DEBUG - 5)
    >>> logging.getLogger(__name__).setLevel("TRACE")
    >>> logging.getLogger(__name__).trace('that worked')
    >>> logging.trace('so did this')
    >>> logging.TRACE
    5

    Args:
        level_name: The name of the level that will become an attribute of the `logging` module
        level_num: The logging value of the new level
        method_name: The method name with which the new level will be called

    """
    if not method_name:
        method_name = level_name.lower()

    if hasattr(logging, level_name):
        raise AttributeError(f"{level_name} already defined in logging module")
    if hasattr(logging, method_name):
        raise AttributeError(f"{method_name} already defined in logging module")
    if hasattr(logging.getLoggerClass(), method_name):
        raise AttributeError(f"{method_name} already defined in logger class")

    # This method was inspired by the answers to Stack Overflow post
    # http://stackoverflow.com/q/2183233/2988730, especially
    # http://stackoverflow.com/a/13638084/2988730
    def logForLevel(self, message, *args, **kwargs):
        if self.isEnabledFor(level_num):
            self._log(level_num, message, args, **kwargs)

    def logToRoot(message, *args, **kwargs):
        logging.log(level_num, message, *args, **kwargs)

    logging.addLevelName(level_num, level_name)
    setattr(logging, level_name, level_num)
    setattr(logging.getLoggerClass(), method_name, logForLevel)
    setattr(logging, method_name, logToRoot)


class ParallelLoggingManager:
    """
    This is a parallel logging manager meant to be used when using logging with multithreading when some of the logs
    should be grouped together in the log file for better readability.

    This class should be created only once in every process exceution.
    Example
    -------
    >>> logging_manager = ParallelLoggingManager('debug_file.log')
    >>> logging_manager.debug('debug message')
    >>> logging_manager.info('info message')
    >>> logging_manager.warning('warning message')
    >>> logging_manager.error('error message')
    >>> logging_manager.critical('critical message')
    >>> logging_manager.success('success message')
    # The 'exception method should be called only within exception handling context
    >>> logging_manager.exception('exception message')
    # The logs will be sent to the handlers only when the 'execute_logs' method will be called by
     the thread who wrote the log messages in the first place
    >>> logging_manager.execute_logs()
    # In case we want to write a log now we can use the 'real_time=True' flag.
    >>> logging_manager.debug('debug message', real_time=True)
    >>> logging_manager.info('info message', real_time=True)
    >>> logging_manager.warning('warning message', real_time=True)
    >>> logging_manager.error('error message', real_time=True)
    >>> logging_manager.critical('critical message', real_time=True)
    >>> logging_manager.success('success message', real_time=True)
    """

    def __init__(self, log_file_name: str, real_time_logs_only: bool = False):
        """
        Initializes the logging manager:
        - Uses a different colored format for each log level (see LEVEL_STYLES)
        - Defines a FileHandler with the given log_file_name.
        - Defines a SteamHandler for stdout.
        - Defines a logger for real time logs with those handlers
        Args:
            real_time_logs_only: When True - all logs will be handled in real time
            log_file_name: The path to where the log file will be saved
        """
        if not hasattr(logging, "success"):
            _add_logging_level("SUCCESS", 25)
        self.real_time_logs_only = real_time_logs_only
        self.log_file_name = log_file_name
        formatter = coloredlogs.ColoredFormatter(
            fmt=LOGGING_FORMAT, level_styles=LEVEL_STYLES
        )
        self.console_handler = logging.StreamHandler(sys.stdout)
        self.console_handler.setFormatter(formatter)
        log_file_path = (
            os.path.join(ARTIFACTS_PATH, "logs", log_file_name)
            if os.path.exists(os.path.join(ARTIFACTS_PATH, "logs"))
            else os.path.join(ARTIFACTS_PATH, log_file_name)
        )
        self.file_handler = logging.FileHandler(log_file_path)
        self.file_handler.setFormatter(formatter)
        self.console_handler.setLevel(logging.INFO)
        self.file_handler.setLevel(logging.DEBUG)
        self.real_time_logger = logging.getLogger(f"real_time-{log_file_path}")
        self.real_time_logger.addHandler(self.file_handler)
        self.real_time_logger.addHandler(self.console_handler)
        self.real_time_logger.setLevel(logging.DEBUG)
        self.real_time_logger.propagate = False
        self.loggers: Dict[str, Any] = {}
        self.listeners: Dict[str, QueueListener] = {}
        self.logs_lock = Lock()
        self.thread_names: Set[str] = set()

    def _add_logger(self, thread_name: str):
        """
        Defines a new logger, queueHandler and QueueListener for a new thread
        This method is only called if the thread name is not in the 'thread_names' set.
        Args:
            thread_name: The name of the thread that should be added
        """
        log_queue: Queue = Queue(-1)
        queue_handler = QueueHandler(log_queue)
        queue_handler.setLevel(logging.DEBUG)
        logger = logging.getLogger(thread_name)
        logger.propagate = False
        logger.setLevel(logging.DEBUG)
        logger.addHandler(queue_handler)
        listener = QueueListener(
            log_queue,
            self.console_handler,
            self.file_handler,
            respect_handler_level=True,
        )
        self.loggers[thread_name] = logger
        self.listeners[thread_name] = listener
        self.thread_names.add(thread_name)

    def debug(self, message: str, real_time: bool = False) -> None:
        """
        Executes a debug log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = (
            self.real_time_logger.debug
            if real_time or self.real_time_logs_only
            else self.loggers[thread_name].debug
        )
        log_method(message)

    def info(self, message: str, real_time: bool = False) -> None:
        """
        Executes an info log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = (
            self.real_time_logger.info
            if real_time or self.real_time_logs_only
            else self.loggers[thread_name].info
        )
        log_method(message)

    def warning(self, message: str, real_time: bool = False) -> None:
        """
        Executes a warning log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = (
            self.real_time_logger.warning
            if real_time or self.real_time_logs_only
            else self.loggers[thread_name].warning
        )
        log_method(message)

    def error(self, message: str, real_time: bool = False) -> None:
        """
        Executes a error log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = (
            self.real_time_logger.error
            if real_time or self.real_time_logs_only
            else self.loggers[thread_name].error
        )
        log_method(message)

    def critical(self, message: str, real_time: bool = False) -> None:
        """
        Executes a critical log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = (
            self.real_time_logger.critical
            if real_time or self.real_time_logs_only
            else self.loggers[thread_name].critical
        )
        log_method(message)

    def exception(self, message: str, real_time: bool = False) -> None:
        """
        Executes a exception log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = (
            self.real_time_logger.exception
            if real_time or self.real_time_logs_only
            else self.loggers[thread_name].exception
        )
        log_method(message)

    def success(self, message: str, real_time: bool = False) -> None:
        """
        Executes a success log.
        If real_time is given - will use the real_time_logger, if not - will use the thread's logger.
        If the thread logger is used - the log will not be written until 'execute_logs' is called.
        Args:
            message: The log message
            real_time: Whether to log the message now or after the next 'execute_logs' method execution.
        """
        thread_name = currentThread().getName()
        if thread_name not in self.thread_names:
            self._add_logger(thread_name)
        log_method = (
            self.real_time_logger.success  # type: ignore[attr-defined]
            if real_time or self.real_time_logs_only
            else self.loggers[thread_name].success
        )
        log_method(message)

    def execute_logs(self) -> None:
        """
        Writes the logs from the queue to the handlers.
        """
        thread_name = currentThread().getName()
        self.logs_lock.acquire()
        self.listeners[thread_name].start()
        self.listeners[thread_name].stop()
        self.logs_lock.release()


# demisto_sdk/commands/test_content/TestContentClasses.py

import ast
import logging
import os
import re
import subprocess
import sys
import time
import urllib.parse
import uuid
from copy import deepcopy
from math import ceil
from pprint import pformat
from queue import Empty, Queue
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union

import demisto_client
import prettytable
import requests
import urllib3
from demisto_client.demisto_api import DefaultApi, Incident
from demisto_client.demisto_api.rest import ApiException
from packaging.version import Version
from slack_sdk import WebClient as SlackClient

from demisto_sdk.commands.common.constants import (
    DEFAULT_CONTENT_ITEM_FROM_VERSION,
    DEFAULT_CONTENT_ITEM_TO_VERSION,
    FILTER_CONF,
    PB_Status,
)
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.tools import get_demisto_version
from demisto_sdk.commands.test_content.Docker import Docker
from demisto_sdk.commands.test_content.IntegrationsLock import acquire_test_lock
from demisto_sdk.commands.test_content.mock_server import (
    RESULT,
    MITMProxy,
    run_with_mock,
)
from demisto_sdk.commands.test_content.ParallelLoggingManager import (
    ParallelLoggingManager,
)
from demisto_sdk.commands.test_content.tools import (
    get_ui_url,
    is_redhat_instance,
    update_server_configuration,
)

ENV_RESULTS_PATH = "./artifacts/env_results.json"
FAILED_MATCH_INSTANCE_MSG = (
    "{} Failed to run.\n There are {} instances of {}, please select one of them by using "
    "the instance_name argument in conf.json. The options are:\n{}"
)
ENTRY_TYPE_ERROR = 4
DEFAULT_INTERVAL = 4
MAX_RETRIES = 3
RETRIES_THRESHOLD = ceil(MAX_RETRIES / 2)

SLACK_MEM_CHANNEL_ID = "CM55V7J8K"
XSIAM_SERVER_TYPE = "XSIAM"
# For now we run all xsiam tests
IS_XSIAM = False

__all__ = [
    "BuildContext",
    "Conf",
    "Integration",
    "IntegrationConfiguration",
    "SecretConf",
    "TestConfiguration",
    "TestContext",
    "TestPlaybook",
    "TestResults",
    "ServerContext",
]


class IntegrationConfiguration:
    def __init__(self, integration_configuration):
        self.raw_dict = integration_configuration
        self.name: str = integration_configuration.get("name", "")
        self.instance_name: str = integration_configuration.get(
            "instance_name", self.name
        )
        self.is_byoi: bool = integration_configuration.get("byoi", True)
        self.should_validate_test_module: bool = integration_configuration.get(
            "validate_test", True
        )
        self.has_integration: bool = integration_configuration.get(
            "has_integration", True
        )
        self.params: dict = integration_configuration.get("params", {})

    def __str__(self):
        return json.dumps(self.raw_dict)

    def __repr__(self):
        return str(self)


class SecretConf:
    def __init__(self, secret_conf: dict):
        """
        Args:
            secret_conf: The contents of the content-test-conf conf.json file.
        """
        self.server_username = secret_conf["username"]
        self.server_password = secret_conf["userPassword"]
        self.integrations = [
            IntegrationConfiguration(integration_configuration=configuration)
            for configuration in secret_conf["integrations"]
        ]


class TestConfiguration:
    def __init__(self, test_configuration: dict, default_test_timeout: int):
        """
        Args:
            test_configuration: A record of a test from 'tests' list in conf.json file in content repo..
            default_test_timeout: The default timeout to use in case no timeout is specified in the configuration
        """
        self.raw_dict = test_configuration
        self.playbook_id = test_configuration.get("playbookID", "")
        self.nightly_test = test_configuration.get("nightly", False)
        self.from_version = test_configuration.get(
            "fromversion", DEFAULT_CONTENT_ITEM_FROM_VERSION
        )
        self.to_version = test_configuration.get(
            "toversion", DEFAULT_CONTENT_ITEM_TO_VERSION
        )
        self.timeout = test_configuration.get("timeout", default_test_timeout)
        self.memory_threshold = test_configuration.get(
            "memory_threshold", Docker.DEFAULT_CONTAINER_MEMORY_USAGE
        )
        self.pid_threshold = test_configuration.get(
            "pid_threshold", Docker.DEFAULT_CONTAINER_PIDS_USAGE
        )
        self.runnable_on_docker_only: bool = test_configuration.get(
            "runnable_on_docker_only", False
        )
        self.is_mockable = test_configuration.get("is_mockable")
        self.context_print_dt = test_configuration.get("context_print_dt")
        self.test_integrations: List[str] = self._parse_integrations_conf(
            test_configuration
        )
        self.test_instance_names: List[str] = self._parse_instance_names_conf(
            test_configuration
        )
        self.instance_configuration: dict = test_configuration.get(
            "instance_configuration", {}
        )
        self.external_playbook_config: dict = test_configuration.get(
            "external_playbook_config", {}
        )
        self.is_first_playback_failed: bool = False
        self.number_of_executions: int = 0
        self.number_of_successful_runs: int = 0

    @staticmethod
    def _parse_integrations_conf(test_configuration):
        integrations_conf = test_configuration.get("integrations", [])
        if not isinstance(integrations_conf, list):
            integrations_conf = [integrations_conf]
        return integrations_conf

    @staticmethod
    def _parse_instance_names_conf(test_configuration):
        instance_names_conf = test_configuration.get("instance_names", [])
        if not isinstance(instance_names_conf, list):
            instance_names_conf = [instance_names_conf]
        return instance_names_conf

    def __str__(self):
        return str(self.raw_dict)


class Conf:
    def __init__(self, conf: dict):
        """
        Args:
            conf: The contents of the content conf.json file.
        """
        self.default_timeout: int = conf.get("testTimeout", 30)
        self.tests: list = [
            TestConfiguration(test_configuration, self.default_timeout)
            for test_configuration in conf.get("tests", [])
        ]
        self.skipped_tests: Dict[str, str] = conf.get("skipped_tests")  # type: ignore
        self.skipped_integrations: Dict[str, str] = conf.get("skipped_integrations")  # type: ignore
        self.unmockable_integrations: Dict[str, str] = conf.get("unmockable_integrations")  # type: ignore
        self.parallel_integrations: List[str] = conf["parallel_integrations"]
        self.docker_thresholds = conf.get("docker_thresholds", {}).get("images", {})


class TestPlaybook:
    def __init__(self, build_context, test_configuration: TestConfiguration):
        """
        This class has all the info related to a test playbook during test execution
        Args:
            build_context (BuildContext): The build context to use in the build
            test_configuration: The configuration from content conf.json file
        """
        self.build_context = build_context
        self.configuration: TestConfiguration = test_configuration
        self.is_mockable: bool = (
            self.configuration.playbook_id not in build_context.unmockable_test_ids
        )
        self.integrations: List[Integration] = [
            Integration(
                self.build_context,
                integration_name,
                self.configuration.test_instance_names,
            )
            for integration_name in self.configuration.test_integrations
        ]
        self.integrations_to_lock = [
            integration
            for integration in self.integrations
            if integration.name not in self.build_context.conf.parallel_integrations
        ]

    def __str__(self):
        return f'"{self.configuration.playbook_id}"'

    def __repr__(self):
        return str(self)

    def should_test_run(self):
        skipped_tests_collected = self.build_context.tests_data_keeper.skipped_tests

        def in_filtered_tests():
            """
            Checks if there are a list of filtered tests that the playbook is in them.
            """
            if (
                not self.build_context.filtered_tests
                or self.configuration.playbook_id
                not in self.build_context.filtered_tests
            ):
                self.build_context.logging_module.debug(
                    f"Skipping {self} because it's not in filtered tests"
                )
                skipped_tests_collected[
                    self.configuration.playbook_id
                ] = "not in filtered tests"
                return False
            return True

        def nightly_test_in_non_nightly_build():
            """
            Checks if we are on a build which is not nightly, and the test should run only on nightly builds.
            """
            if self.configuration.nightly_test and not self.build_context.is_nightly:
                log_message = f"Skipping {self} because it's a nightly test in a non nightly build"
                if self.configuration.playbook_id in self.build_context.filtered_tests:
                    self.build_context.logging_module.warning(log_message)
                else:
                    self.build_context.logging_module.debug(log_message)
                skipped_tests_collected[
                    self.configuration.playbook_id
                ] = "nightly test in a non nightly build"
                return True
            return False

        def skipped_test():
            if self.configuration.playbook_id in self.build_context.conf.skipped_tests:
                if self.configuration.playbook_id in self.build_context.filtered_tests:
                    # Add warning log, as the playbook is supposed to run according to the filters, but it's skipped
                    self.build_context.logging_module.warning(
                        f"Skipping test {self} because it's in skipped test list"
                    )
                else:
                    self.build_context.logging_module.debug(
                        f"Skipping test {self} because it's in skipped test list"
                    )
                reason = self.build_context.conf.skipped_tests[
                    self.configuration.playbook_id
                ]
                skipped_tests_collected[self.configuration.playbook_id] = reason
                return True
            return False

        def version_mismatch():
            if not (
                Version(self.configuration.from_version)
                <= Version(self.build_context.server_numeric_version)
                <= Version(self.configuration.to_version)
            ):
                self.build_context.logging_module.warning(
                    f"Test {self} ignored due to version mismatch "
                    f"(test versions: {self.configuration.from_version}-{self.configuration.to_version})\n"
                )
                skipped_tests_collected[
                    self.configuration.playbook_id
                ] = f"(test versions: {self.configuration.from_version}-{self.configuration.to_version})"
                return True
            return False

        def test_has_skipped_integration():
            for integration in self.configuration.test_integrations:
                # So now we know that the test is in the filtered tests
                if integration in self.build_context.conf.skipped_integrations:
                    self.build_context.logging_module.debug(
                        f"Skipping {self} because it has a skipped integration {integration}"
                    )
                    # The playbook should be run but has a skipped integration
                    if (
                        self.build_context.filtered_tests
                        and self.configuration.playbook_id
                        in self.build_context.filtered_tests
                    ):
                        # Adding the playbook ID to playbook_skipped_integration so that we can send a PR comment about it
                        skip_reason = self.build_context.conf.skipped_integrations[
                            integration
                        ]
                        self.build_context.tests_data_keeper.playbook_skipped_integration.add(
                            f"{self.configuration.playbook_id} - reason: {skip_reason}"
                        )
                        self.build_context.logging_module.warning(
                            f"The integration {integration} is skipped and critical for the test {self}."
                        )
                        skipped_tests_collected[
                            self.configuration.playbook_id
                        ] = f"The integration {integration} is skipped"
                    return True

            return False

        return (
            in_filtered_tests()
            and not nightly_test_in_non_nightly_build()
            and not skipped_test()
            and not version_mismatch()
            and not test_has_skipped_integration()
        )

    def run_test_module_on_integrations(self, client: DefaultApi) -> bool:
        """
        runs 'test-module' on all integrations that the playbook uses and return a boolean indicating the result
        Args:
            client: The demisto_client to use

        Returns:
            True if all integrations was test-module execution was successful else False
        """
        for integration in self.integrations:
            success = integration.test_integration_instance(client)
            if not success:
                return False
        return True

    def configure_integrations(
        self,
        client: DefaultApi,
        server_context: "ServerContext",
        instance_configuration: dict,
    ) -> bool:
        """
        Configures all integrations that the playbook uses and return a boolean indicating the result
        Args:
            client: The demisto_client to use
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in

        Returns:
            True if all integrations was configured else False
        """
        configured_integrations: List[Integration] = []
        for integration in self.integrations:
            instance_created = integration.create_integration_instance(
                client,
                self.configuration.playbook_id,
                self.is_mockable,
                server_context,
                instance_configuration,
            )
            if not instance_created:
                self.build_context.logging_module.error(
                    f"Cannot run playbook {self}, integration {integration} failed to configure"
                )
                # Deleting all the already configured integrations
                for configured_integration in configured_integrations:
                    configured_integration.delete_integration_instance(client)
                return False
            configured_integrations.append(integration)
        return True

    def disable_integrations(
        self, client: DefaultApi, server_context: "ServerContext"
    ) -> None:
        """
        Disables all integrations that the playbook uses
        Clears server configurations set for the integration if there are such
        Reset containers if server configurations were cleared

        Args:
            client: The demisto_client to use
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in
        """
        for integration in self.integrations:
            integration.disable_integration_instance(client)
        updated_keys = False
        if not IS_XSIAM:
            updated_keys = self._set_prev_server_keys(client, server_context)
        if updated_keys:
            server_context._reset_containers()

    def _set_prev_server_keys(
        self, client: DefaultApi, server_context: "ServerContext"
    ) -> bool:
        """Sets previous stored system (server) config, if existed

        Args:
            client (DefaultApi): The demisto_client to use
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in

        Returns:
             bool: Whether server configurations were updated to indicate if reset containers is required
        """
        updated = False
        for integration in self.integrations:
            if (
                integration.configuration
                and "server_keys" not in integration.configuration.params
            ):
                continue
            if server_context.prev_system_conf:
                update_server_configuration(
                    client=client,
                    server_configuration=server_context.prev_system_conf,
                    error_msg="Failed to set server keys",
                    logging_manager=self.build_context.logging_module,
                )
                server_context.prev_system_conf = {}
                updated = True
            else:
                integration.build_context.logging_module.error(
                    f"Failed clearing system conf. Found server_keys for integration {integration.name} but could not"
                    "find previous system conf stored"
                )
        return updated

    def delete_integration_instances(self, client: DefaultApi):
        """
        Deletes all integrations that the playbook uses
        Args:
            client: The demisto_client to use
        """
        for integration in self.integrations:
            integration.delete_integration_instance(client)

    def create_incident(self, client: DefaultApi) -> Optional[Incident]:
        """
        Creates an incident with the current playbook ID
        Args:
            client: The demisto_client to use

        Returns:
            - The created incident or None
        """
        # Preparing the incident request

        incident_name = (
            f"inc-{self.configuration.playbook_id}--build_number:"
            f"{self.build_context.build_number}--{uuid.uuid4()}"
        )
        create_incident_request = demisto_client.demisto_api.CreateIncidentRequest()
        create_incident_request.create_investigation = True
        create_incident_request.playbook_id = self.configuration.playbook_id
        create_incident_request.name = incident_name

        try:
            response = client.create_incident(
                create_incident_request=create_incident_request
            )
        except ApiException:
            self.build_context.logging_module.exception(
                f"Failed to create incident with name {incident_name} for playbook {self}"
            )
        try:
            inc_id = response.id
        except Exception:
            inc_id = "incCreateErr"
        # inc_id = response_json.get('id', 'incCreateErr')
        if inc_id == "incCreateErr":
            error_message = (
                f"Failed to create incident for playbookID: {self}."
                "Possible reasons are:\nMismatch between playbookID in conf.json and "
                "the id of the real playbook you were trying to use,"
                "or schema problems in the TestPlaybook."
            )
            self.build_context.logging_module.error(error_message)
            return None

        # get incident
        search_filter = demisto_client.demisto_api.SearchIncidentsData()
        inc_filter = demisto_client.demisto_api.IncidentFilter()
        if inc_id:
            inc_filter.query = f"id: {inc_id}"
        if IS_XSIAM:
            # in xsiam `create_incident` response don`t return created incident id.
            inc_filter.name = [incident_name]
        # inc_filter.query
        search_filter.filter = inc_filter

        incident_search_responses = []

        found_incidents = 0
        # poll the incidents queue for a max time of 300 seconds
        timeout = time.time() + 300
        while found_incidents < 1:
            try:
                incidents = client.search_incidents(filter=search_filter)
                found_incidents = len(incidents.data)
                incident_search_responses.append(incidents)
            except ApiException:
                if IS_XSIAM:
                    self.build_context.logging_module.exception(
                        f"Searching incident with name {incident_name} failed"
                    )
                else:
                    self.build_context.logging_module.exception(
                        f"Searching incident with id {inc_id} failed"
                    )
            if time.time() > timeout:
                if IS_XSIAM:
                    self.build_context.logging_module.error(
                        f"Got timeout for searching incident with name "
                        f"{incident_name}"
                    )
                else:
                    self.build_context.logging_module.error(
                        f"Got timeout for searching incident id {inc_id}"
                    )

                self.build_context.logging_module.error(
                    f"Incident search responses: {incident_search_responses}"
                )
                return None

            time.sleep(10)

        return incidents.data[0]

    def delete_incident(self, client: DefaultApi, incident_id: str) -> bool:
        """
        Deletes a Demisto incident
        Args:
            client: demisto_client instance to use
            incident_id: Incident to delete

        Returns:
            True if incident was deleted else False
        """
        try:
            body = {"ids": [incident_id], "filter": {}, "all": False}
            res = demisto_client.generic_request_func(
                self=client, method="POST", path="/incident/batchDelete", body=body
            )
        except ApiException:
            self.build_context.logging_module.exception(
                "Failed to delete incident, error trying to communicate with demisto server"
            )
            return False

        if int(res[1]) != 200:
            self.build_context.logging_module.error(
                f"delete incident failed with Status code {res[1]}"
            )
            self.build_context.logging_module.error(pformat(res))
            return False

        return True

    def close_incident(self, client: DefaultApi, incident_id: str) -> bool:
        """
        Closes a Demisto incident
        Args:
            client: demisto_client instance to use
            incident_id: Incident to close

        Returns:
            True if incident was closed else False
        """
        try:
            body = {"id": incident_id, "CustomFields": {}}
            res = demisto_client.generic_request_func(
                self=client, method="POST", path="/incident/close", body=body
            )
            self.build_context.logging_module.info(f"Closed incident: {incident_id}.")
        except ApiException:
            self.build_context.logging_module.warning(
                "Failed to close incident, error trying to communicate with demisto server."
            )
            return False

        if int(res[1]) != 200:
            self.build_context.logging_module.warning(
                f"Close incident failed with Status code {res[1]}"
            )
            self.build_context.logging_module.warning(pformat(res))
            return False

        return True

    def print_context_to_log(self, client: DefaultApi, incident_id: str) -> None:
        try:
            body = {"query": f"${{{self.configuration.context_print_dt}}}"}
            res = demisto_client.generic_request_func(
                self=client,
                method="POST",
                path=f"/investigation/{incident_id}/context",
                body=body,
            )
            if int(res[1]) != 200:
                self.build_context.logging_module.error(
                    f"incident context fetch failed with Status code {res[1]}"
                )
                self.build_context.logging_module.error(pformat(res))
                return
            try:
                self.build_context.logging_module.info(
                    json.dumps(ast.literal_eval(res[0]), indent=4)
                )
            except ValueError:
                self.build_context.logging_module.error(
                    f"unable to parse result for result with value: {res[0]}"
                )
        except ApiException:
            self.build_context.logging_module.exception(
                "Failed to get context, error trying to communicate with demisto server"
            )


class BuildContext:
    def __init__(self, kwargs: dict, logging_module: ParallelLoggingManager):
        global IS_XSIAM
        self.is_xsiam = True if kwargs["server_type"] == XSIAM_SERVER_TYPE else False
        IS_XSIAM = self.is_xsiam
        self.logging_module: ParallelLoggingManager = logging_module
        self.server = kwargs["server"]
        self.xsiam_machine = kwargs.get("xsiam_machine")
        self.xsiam_servers_path = kwargs.get("xsiam_servers_path")
        self.conf, self.secret_conf = self._load_conf_files(
            kwargs["conf"], kwargs["secret"]
        )
        if self.is_xsiam:
            with open(kwargs.get("xsiam_servers_api_keys_path")) as json_file:  # type: ignore[arg-type]
                xsiam_servers_api_keys = json.loads(json_file.read())
            self.xsiam_conf = self._load_xsiam_file(self.xsiam_servers_path)
            self.env_json = [self.xsiam_conf.get(self.xsiam_machine, {})]
            self.api_key = xsiam_servers_api_keys.get(self.xsiam_machine)
            self.auth_id = self.env_json[0].get("x-xdr-auth-id")
            self.xsiam_ui_path = self.env_json[0].get("ui_url")
        else:
            self.api_key = kwargs["api_key"]
            self.env_json = self._load_env_results_json()
            self.xsiam_conf = None
            self.auth_id = None
            self.xsiam_ui_path = None
        self.is_nightly = kwargs["nightly"]
        self.slack_client = SlackClient(kwargs["slack"])
        self.circleci_token = kwargs["circleci"]
        self.build_number = kwargs["build_number"]
        self.build_name = kwargs["branch_name"]
        self.isAMI = kwargs["is_ami"] if not self.is_xsiam else False
        self.memCheck = kwargs["mem_check"]
        self.server_version = kwargs["server_version"]  # AMI Role
        self.is_local_run = self.server is not None
        self.server_numeric_version = self._get_server_numeric_version()
        self.instances_ips = self._get_instances_ips()
        self.filtered_tests = self._extract_filtered_tests()
        self.tests_data_keeper = TestResults(self.conf.unmockable_integrations)
        self.conf_unmockable_tests = self._get_unmockable_tests_from_conf()
        self.unmockable_test_ids: Set[str] = set()
        (
            self.mockable_tests_to_run,
            self.unmockable_tests_to_run,
        ) = self._get_tests_to_run()
        self.test_retries_queue: Queue = Queue()
        self.slack_user_id = self._retrieve_slack_user_id()
        self.all_integrations_configurations = self._get_all_integration_config(
            self.instances_ips
        )

    def _get_all_integration_config(self, instances_ips: list) -> Optional[list]:
        """
        Gets all integration configuration as it exists on the demisto server
        Since in all packs are installed the data returned from this request is very heavy and we want to avoid
        running it in multiple threads.
        Args:
            instances_ips: the instances urls
        Returns:
            A dict containing the configuration for the integration if found, else empty list
        """
        if not self.is_nightly:
            return []
        url = instances_ips[0]
        if IS_XSIAM:
            server_url = url
        else:
            server_url = f"https://{url}"
        return self.get_all_installed_integrations_configurations(server_url)

    def get_all_installed_integrations_configurations(self, server_url: str) -> list:
        """
        Gets all integration configuration as it exists on the demisto server
        Args:
            server_url: The url of the server to create integration in

        Returns:
            A dict containing the configuration for the integration if found else empty list
        """
        if IS_XSIAM:
            # in xsiam we dont use demisto username
            os.environ.pop("DEMISTO_USERNAME", None)
        tmp_client = demisto_client.configure(
            base_url=server_url,
            auth_id=self.auth_id,
            api_key=self.api_key,
            verify_ssl=False,
        )
        self.logging_module.debug("Getting all integrations instances")
        try:
            res_raw = demisto_client.generic_request_func(
                self=tmp_client,
                path="/settings/integration/search",
                method="POST",
                body={},
            )
        except ApiException:
            self.logging_module.exception(
                "failed to get all integrations configuration"
            )
            return []
        res = ast.literal_eval(res_raw[0])
        TIMEOUT = 180
        SLEEP_INTERVAL = 5
        total_sleep = 0
        while "configurations" not in res:
            if total_sleep == TIMEOUT:
                self.logging_module.error(
                    f"Timeout - failed to get all integration configuration. Error: {res}"
                )
                return []

            time.sleep(SLEEP_INTERVAL)
            total_sleep += SLEEP_INTERVAL
        all_configurations = res["configurations"]
        return all_configurations

    def _generate_tests_queue(self, tests_to_run: List[TestConfiguration]) -> Queue:
        """
        Generates a queue containing test playbooks to run
        Args:
            tests_to_run: A list containing playbook names
        """
        queue: Queue = Queue()
        for test in tests_to_run:
            playbook = TestPlaybook(self, test)
            if playbook.should_test_run():
                queue.put(playbook)
        return queue

    def _get_tests_to_run(self) -> Tuple[Queue, Queue]:
        """
        Gets tests to run in the current build and updates the unmockable tests ids set
        Returns:
            - A queue with mockable TestPlaybook instances to run in the current build
            - A queue with unmockable TestPlaybook instances to run in the current build
        """
        unmockable_tests = []
        all_tests = self._get_all_tests()
        # for xsiam, set all tests to unmockable.
        if self.server or not self.isAMI or self.is_xsiam:
            unmockable_tests = all_tests
            self.unmockable_test_ids = {test.playbook_id for test in all_tests}
        elif self.isAMI:
            unmockable_tests = self.conf_unmockable_tests
            self.unmockable_test_ids = {
                test.playbook_id for test in self.conf_unmockable_tests
            }
        self.logging_module.debug(
            f"Unmockable tests selected: {pformat(self.unmockable_test_ids)}"
        )
        mockable_tests = [
            test
            for test in all_tests
            if test.playbook_id not in self.unmockable_test_ids
        ]
        self.logging_module.debug(
            f"Mockable tests selected: {pformat([test.playbook_id for test in mockable_tests])}"
        )
        mockable_tests_queue = self._generate_tests_queue(mockable_tests)
        unmockable_tests_queue = self._generate_tests_queue(unmockable_tests)
        return mockable_tests_queue, unmockable_tests_queue

    @staticmethod
    def _extract_filtered_tests() -> list:
        """
        Reads the content from ./artifacts/filter_file.txt and parses it into a list of test playbook IDs that should be run
        in the current build
        Returns:
            A list of playbook IDs that should be run in the current build
        """
        with open(FILTER_CONF) as filter_file:
            filtered_tests = [line.strip("\n") for line in filter_file.readlines()]

        return filtered_tests

    def _get_unmockable_tests_from_conf(self) -> list:
        """
        Extracts the unmockable test playbook by looking at all playbooks that has unmockable integrations
        Returns:
            A with unmockable playbook names
        """
        unmockable_integrations = self.conf.unmockable_integrations
        tests = self.conf.tests
        unmockable_tests = []
        for test_record in tests:
            test_name = test_record.playbook_id
            unmockable_integrations_used = [
                integration_name
                for integration_name in test_record.test_integrations
                if integration_name in unmockable_integrations
            ]
            if (
                test_name
                and (not test_record.test_integrations or unmockable_integrations_used)
                or not self.isAMI
            ):
                unmockable_tests.append(test_record)
                # In case a test has both - an unmockable integration and is configured with is_mockable=False -
                # it will be added twice if we don't continue.
                continue
            if test_record.is_mockable is False:
                unmockable_tests.append(test_record)
        return unmockable_tests

    @staticmethod
    def _get_used_integrations(test_configuration: dict) -> list:
        """
        Gets the integration used in a test playbook configurations
        Args:
            test_configuration: A test configuration from content conf.json file.

        Returns:
            A list with integration names
        """
        tested_integrations = test_configuration.get("integrations", [])
        if isinstance(tested_integrations, list):
            return tested_integrations
        else:
            return [tested_integrations]

    def _get_all_tests(self) -> List[TestConfiguration]:
        """
        Gets a list of all playbooks configured in content conf.json
        Returns:
            A list with playbook names
        """
        tests_records = self.conf.tests
        return [test for test in tests_records if test.playbook_id]

    def _get_instances_ips(self) -> List[str]:
        """
        Parses the env_results.json and extracts the instance ip from each server configured in it.
        Returns:
            A list contains server internal ips.
        """
        if self.server:
            return [self.server]
        if self.is_xsiam:
            return [env.get("base_url") for env in self.env_json]
        return [
            env.get("InstanceDNS")
            for env in self.env_json
            if env.get("Role") == self.server_version
        ]

    @staticmethod
    def _parse_tests_list_arg(tests_list: str):
        """
        Parses the test list arguments if present.

        :param tests_list: CSV string of tests to run.
        :return: List of tests if there are any, otherwise empty list.
        """
        tests_to_run = tests_list.split(",") if tests_list else []
        return tests_to_run

    @staticmethod
    def _load_env_results_json():
        if not os.path.isfile(ENV_RESULTS_PATH):
            return {}

        with open(ENV_RESULTS_PATH) as json_file:
            return json.load(json_file)

    def _get_server_numeric_version(self) -> str:
        """
        Gets the current server version

        Returns:
            Server numeric version
        """
        default_version = "99.99.98"
        if self.is_local_run:
            self.logging_module.info(
                f"Local run, assuming server version is {default_version}",
                real_time=True,
            )
            return default_version

        if not self.env_json:
            self.logging_module.warning(
                f"Did not find {ENV_RESULTS_PATH} file, assuming server version is {default_version}.",
                real_time=True,
            )
            return default_version

        server_version_mapping = {
            "Server 5.0": "5.0.0",
            "Server 5.5": "5.5.0",
            "Server 6.0": "6.0.0",
            "Server 6.1": "6.1.0",
            "Server 6.2": "6.2.0",
            "Server 6.5": "6.5.0",
            "Server 6.6": "6.6.0",
            "Server 6.8": "6.8.0",
            "Server 6.9": "6.9.0",
            "Server 6.10": "6.10.0",
            "Server 6.11": "6.11.0",
            "Server Master": default_version,
            "XSIAM 1.2": "6.9.0",
            "XSIAM Master": default_version,
        }
        server_numeric_version = server_version_mapping.get(
            self.server_version, default_version
        )
        self.logging_module.info(
            f"Server version: {server_numeric_version}", real_time=True
        )
        return server_numeric_version

    @staticmethod
    def _load_xsiam_file(xsiam_servers_path):
        conf = None
        if xsiam_servers_path:
            with open(xsiam_servers_path) as data_file:
                conf = json.load(data_file)

        return conf

    @staticmethod
    def _load_conf_files(conf_path, secret_conf_path):
        with open(conf_path) as data_file:
            conf = Conf(json.load(data_file))

        secret_conf = None
        if secret_conf_path:
            with open(secret_conf_path) as data_file:
                secret_conf = SecretConf(json.load(data_file))

        return conf, secret_conf

    def _get_user_name_from_circle(self):
        url = f"https://circleci.com/api/v1.1/project/github/demisto/content/{self.build_number}"
        res = self._http_request(url, params_dict={"circle-token": self.circleci_token})

        user_details = res.get("user", {})
        return user_details.get("name", "")

    @staticmethod
    def _http_request(url, params_dict=None):
        res = requests.request(
            "GET",
            url,
            verify=True,
            params=params_dict,
        )
        res.raise_for_status()

        return res.json()

    def _retrieve_slack_user_id(self):
        """
        Gets the user id of the circle user who triggered the current build
        """
        user_id = ""
        try:
            user_name = (
                os.getenv("GITLAB_USER_LOGIN") or self._get_user_name_from_circle()
            )
            res = self.slack_client.api_call("users.list")

            user_list = res.get("members", [])  # type: ignore
            for user in user_list:
                profile = user.get("profile", {})
                name = profile.get("real_name_normalized", "")
                if name == user_name:
                    user_id = user.get("id", "")
        except Exception as exc:
            logging.debug(f"failed to retrieve the slack user ID.\nError: {exc}")

        return user_id


class TestResults:
    def __init__(self, unmockable_integrations):
        self.succeeded_playbooks = []
        self.failed_playbooks = set()
        self.playbook_report = dict()
        self.skipped_tests = dict()
        self.skipped_integrations = dict()
        self.rerecorded_tests = []
        self.empty_files = []
        self.unmockable_integrations = unmockable_integrations
        self.playbook_skipped_integration = set()

    def add_proxy_related_test_data(self, proxy):
        # Using multiple appends and not extend since append is guaranteed to be thread safe
        for playbook_id in proxy.rerecorded_tests:
            self.rerecorded_tests.append(playbook_id)
        for playbook_id in proxy.empty_files:
            self.empty_files.append(playbook_id)

    def create_result_files(self):
        with open("./Tests/succeeded_tests.txt", "w") as succeeded_tests_file:
            succeeded_tests_file.write("\n".join(self.succeeded_playbooks))
        with open("./Tests/failed_tests.txt", "w") as failed_tests_file:
            failed_tests_file.write("\n".join(self.failed_playbooks))
        with open("./Tests/skipped_tests.txt", "w") as skipped_tests_file:
            skipped_tests_file.write("\n".join(self.skipped_tests))
        with open("./Tests/skipped_integrations.txt", "w") as skipped_integrations_file:
            skipped_integrations_file.write("\n".join(self.skipped_integrations))
        with open(
            "./Tests/test_playbooks_report.json", "w"
        ) as test_playbooks_report_file:
            json.dump(self.playbook_report, test_playbooks_report_file, indent=4)

    def print_test_summary(
        self,
        is_ami: bool = True,
        logging_module: Union[Any, ParallelLoggingManager] = logging,
    ) -> None:
        """
        Takes the information stored in the tests_data_keeper and prints it in a human readable way.
        Args:
            is_ami: indicating if the server running the tests is an AMI or not.
            logging_module: Logging module to use for test_summary
        """
        succeed_playbooks = self.succeeded_playbooks
        failed_playbooks = self.failed_playbooks
        skipped_tests = self.skipped_tests
        unmocklable_integrations = self.unmockable_integrations
        skipped_integration = self.skipped_integrations
        rerecorded_tests = self.rerecorded_tests
        empty_files = self.empty_files

        succeed_count = len(succeed_playbooks)
        failed_count = len(failed_playbooks)
        rerecorded_count = len(rerecorded_tests) if is_ami else 0
        empty_mocks_count = len(empty_files) if is_ami else 0
        logging_module.real_time_logs_only = True
        logging_module.info("TEST RESULTS:")
        logging_module.info(
            f"Number of playbooks tested - {succeed_count + failed_count}"
        )
        if failed_count:
            logging_module.error(f"Number of failed tests - {failed_count}:")
            logging_module.error(
                "Failed Tests: {}".format(
                    "".join(
                        [
                            f"\n\t\t\t\t\t\t\t - {playbook_id}"
                            for playbook_id in failed_playbooks
                        ]
                    )
                )
            )
        if succeed_count:
            logging_module.success(f"Number of succeeded tests - {succeed_count}")
            logging_module.success(
                "Successful Tests: {}".format(
                    "".join(
                        [
                            f"\n\t\t\t\t\t\t\t - {playbook_id}"
                            for playbook_id in succeed_playbooks
                        ]
                    )
                )
            )
        if rerecorded_count > 0:
            logging_module.debug(
                f"Number of tests with failed playback and successful re-recording - {rerecorded_count}"
            )
            logging_module.debug(
                "Tests with failed playback and successful re-recording: {}".format(
                    "".join(
                        [
                            f"\n\t\t\t\t\t\t\t - {playbook_id}"
                            for playbook_id in rerecorded_tests
                        ]
                    )
                )
            )

        if empty_mocks_count > 0:
            logging_module.debug(
                f"Successful tests with empty mock files count- {empty_mocks_count}:\n"
            )
            proxy_explanation = (
                "\t\t\t\t\t\t\t (either there were no http requests or no traffic is passed through the proxy.\n"
                "\t\t\t\t\t\t\t Investigate the playbook and the integrations.\n"
                "\t\t\t\t\t\t\t If the integration has no http traffic, add to unmockable_integrations in conf.json)"
            )
            logging_module.debug(proxy_explanation)
            logging_module.debug(
                "Successful tests with empty mock files: {}".format(
                    "".join(
                        [
                            f"\n\t\t\t\t\t\t\t - {playbook_id}"
                            for playbook_id in empty_files
                        ]
                    )
                )
            )

        if skipped_integration:
            self.print_table(
                "Skipped Integrations", skipped_integration, logging_module.debug
            )

        if skipped_tests:
            self.print_table("Skipped Tests", skipped_tests, logging_module.debug)

        if unmocklable_integrations:
            self.print_table(
                "Unmockable Integrations",
                unmocklable_integrations,
                logging_module.debug,
            )

    @staticmethod
    def print_table(
        table_name: str, table_data: dict, logging_method: Callable
    ) -> None:
        table = prettytable.PrettyTable()
        table.field_names = ["Index", "Name", "Reason"]
        for index, record in enumerate(table_data, start=1):
            row = [index, record, table_data[record]]
            table.add_row(row)
        logging_method(f"{table_name}:\n{table}", real_time=True)


class Integration:
    def __init__(
        self,
        build_context: BuildContext,
        integration_name: str,
        potential_integration_instance_names: list,
    ):
        """
        An integration class that should represent the integrations during the build
        Args:
            build_context: The context of the build
            integration_name: The name of the integration
            potential_integration_instance_names: A list of instance names, one of those names should be the actual reason
            but we won't know for sure until we will try to filter it with conf.json from content-test-conf repo
        """
        self.build_context = build_context
        self.name = integration_name
        self.instance_names = potential_integration_instance_names
        self.instance_name = ""
        self.configuration: Optional[
            IntegrationConfiguration
        ] = IntegrationConfiguration({"name": self.name, "params": {}})
        self.docker_image: list = []
        self.integration_configuration_from_server: dict = {}
        self.integration_type: str = ""
        self.module_instance: dict = {}

    @staticmethod
    def _change_placeholders_to_values(
        server_url: str, config_item: IntegrationConfiguration
    ) -> IntegrationConfiguration:
        """Some integration should be configured on the current server as host and has the string '%%SERVER_HOST%%'
        in the content-test-conf conf.json configuration.
        This method replaces these placeholders in the configuration to their real values

        Args:
            server_url: The server url that should be inserted instead of the placeholder in the configuration params
            config_item: Integration configuration object.

        Returns:
            IntegrationConfiguration class with the modified params
        """
        placeholders_map = {"%%SERVER_HOST%%": server_url}
        item_as_string = json.dumps(config_item.params)
        for key, value in placeholders_map.items():
            item_as_string = item_as_string.replace(key, value)
        config_item.params = json.loads(item_as_string)
        return config_item

    def _set_integration_params(
        self, server_url: str, playbook_id: str, is_mockable: bool
    ) -> bool:
        """
        Finds the matching configuration for the integration in content-test-data conf.json file
        in accordance with the configured instance name if exist and configures the proxy parameter if needed
        Args:
            server_url: The url of the demisto server to configure the integration in
            playbook_id: The ID of the playbook for which the integration should be configured
            is_mockable: Should the proxy parameter be set to True or not

        Returns:
            True if found a matching configuration else False if found more that one configuration candidate returns False
        """
        self.build_context.logging_module.debug(
            f"Searching integration configuration for {self}"
        )

        # Finding possible configuration matches
        integration_params: List[IntegrationConfiguration] = [
            deepcopy(conf)
            for conf in self.build_context.secret_conf.integrations
            if conf.name == self.name
        ]
        # Modifying placeholders if exists
        integration_params: List[IntegrationConfiguration] = [
            self._change_placeholders_to_values(server_url, conf)
            for conf in integration_params
        ]
        if self.name == "Core REST API":
            self.configuration.params = {  # type: ignore
                "url": server_url if IS_XSIAM else "https://localhost",
                "creds_apikey": {
                    "identifier": str(self.build_context.auth_id) if IS_XSIAM else "",
                    "password": self.build_context.api_key,
                },
                "auth_method": "Standard",
                "insecure": True,
            }
        elif integration_params:
            # If we have more then one configuration for this integration - we will try to filter by instance name
            if len(integration_params) != 1:
                found_matching_instance = False
                for item in integration_params:
                    if item.instance_name in self.instance_names:
                        self.configuration = item
                        found_matching_instance = True
                        break

                if not found_matching_instance:
                    optional_instance_names = [
                        optional_integration.instance_name
                        for optional_integration in integration_params
                    ]
                    error_msg = FAILED_MATCH_INSTANCE_MSG.format(
                        playbook_id,
                        len(integration_params),
                        self.name,
                        "\n".join(optional_instance_names),
                    )
                    self.build_context.logging_module.error(error_msg)
                    return False
            else:
                self.configuration = integration_params[0]

        if is_mockable:
            self.build_context.logging_module.debug(
                f"configuring {self} with proxy params"
            )
            for param in ("proxy", "useProxy", "useproxy", "insecure", "unsecure"):
                self.configuration.params[param] = True  # type: ignore
        return True

    def _get_integration_config(self, server_url: str) -> Optional[dict]:
        """
        Gets integration configuration as it exists on the demisto server
        Args:
            server_url: The url of the server to create integration in

        Returns:
            A dict containing the configuration for the integration if found, else None
        """
        if self.build_context.all_integrations_configurations:
            match_configurations = [
                x
                for x in self.build_context.all_integrations_configurations
                if x["name"] == self.name
            ]
        else:
            all_configurations = (
                self.build_context.get_all_installed_integrations_configurations(
                    server_url
                )
            )
            match_configurations = [
                x for x in all_configurations if x["name"] == self.name
            ]

        if not match_configurations:
            self.build_context.logging_module.error("integration was not found")
            return None

        return deepcopy(match_configurations[0])

    def _delete_integration_instance_if_determined_by_name(
        self, client: DefaultApi, instance_name: str
    ) -> None:
        """Deletes integration instance by it's name.

        Args:
            client (demisto_client): The configured client to use.
            instance_name (str): The name of the instance to delete.

        Notes:
            This function is needed when the name of the instance is pre-defined in the tests configuration, and the test
            itself depends on the instance to be called as the `instance name`.
            In case we need to configure another instance with the same name, the client will throw an error, so we
            will call this function first, to delete the instance with this name.

        """
        try:
            int_resp = demisto_client.generic_request_func(
                self=client,
                method="POST",
                path="/settings/integration/search",
                body={"size": 1000},
            )
            int_instances = ast.literal_eval(int_resp[0])
        except ApiException:
            self.build_context.logging_module.exception(
                f"Failed to delete integration {self} instance, error trying to communicate with demisto server"
            )
            return
        if int(int_resp[1]) != 200:
            self.build_context.logging_module.error(
                f"Get integration {self} instance failed with status code: {int_resp[1]}"
            )
            return
        if "instances" not in int_instances:
            self.build_context.logging_module.info(
                f"No integrations instances found to delete for {self}"
            )
            return

        for instance in int_instances["instances"]:
            if instance.get("name") == instance_name:
                self.build_context.logging_module.info(
                    f"Deleting integration instance {instance_name} since it is defined by name"
                )
                self.delete_integration_instance(client, instance.get("id"))

    def _set_server_keys(
        self, client: DefaultApi, server_context: "ServerContext"
    ) -> None:
        """In case the the params of the test in the content-test-conf repo has 'server_keys' key:
            Resets containers
            Adds server configuration keys using the demisto_client.

        Args:
            client (demisto_client): The configured client to use.
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in
        """
        if "server_keys" not in self.configuration.params:  # type: ignore
            return

        server_context._reset_containers()

        self.build_context.logging_module.debug(
            f"Setting server keys for integration: {self}"
        )

        data = {"data": {}, "version": -1}

        for key, value in self.configuration.params.get("server_keys").items():  # type: ignore
            data["data"][key] = value  # type: ignore

        _, _, prev_system_conf = update_server_configuration(
            client=client,
            server_configuration=self.configuration.params.get("server_keys"),  # type: ignore
            error_msg="Failed to set server keys",
            logging_manager=self.build_context.logging_module,
        )
        server_context.prev_system_conf = prev_system_conf

    def create_module(
        self,
        instance_name: str,
        configuration: dict,
        incident_configuration: dict = None,
    ):
        module_configuration = configuration["configuration"]

        # If incident_type is given in Test Playbook configuration on test-conf, we change the default configuration.
        if incident_configuration and incident_configuration.get("incident_type"):
            incident_type_configuration = list(
                filter(
                    lambda config: config.get("name") == "incidentType",
                    module_configuration,
                )
            )

            incident_type_configuration[0]["value"] = incident_configuration.get(
                "incident_type"
            )

        module_instance = {
            "brand": configuration["name"],
            "category": configuration["category"],
            "configuration": configuration,
            "data": [],
            "enabled": "true",
            "engine": "",
            "id": "",
            "isIntegrationScript": self.configuration.is_byoi,  # type: ignore
            "name": instance_name,
            "passwordProtected": False,
            "version": 0,
            "incomingMapperId": configuration.get("defaultMapperIn", ""),
            "mappingId": configuration.get("defaultClassifier", ""),
            "outgoingMapperId": configuration.get("defaultMapperOut", ""),
        }

        # If default mapper or classifier are given in test-conf we ignore defaultMapperIn or defaultClassifier from yml.
        if incident_configuration and incident_configuration.get("classifier_id"):
            module_instance["mappingId"] = incident_configuration.get("classifier_id")
        if incident_configuration and incident_configuration.get("incoming_mapper_id"):
            module_instance["incomingMapperId"] = incident_configuration.get(
                "incoming_mapper_id"
            )

        return module_instance

    def create_integration_instance(
        self,
        client: DefaultApi,
        playbook_id: str,
        is_mockable: bool,
        server_context: "ServerContext",
        instance_configuration: dict,
    ) -> bool:
        """
        Create an instance of the integration in the server specified in the demisto client instance.
        Args:
            client: The demisto_client instance to use
            playbook_id: The playbook id for which the instance should be created
            is_mockable: Indicates whether the integration should be configured with proxy=True or not
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in

        Returns:
            The integration configuration as it exists on the server after it was configured
        """
        server_url = client.api_client.configuration.host
        self._set_integration_params(server_url, playbook_id, is_mockable)
        configuration = self._get_integration_config(
            client.api_client.configuration.host
        )
        if not configuration:
            self.build_context.logging_module.error(
                f"Could not find configuration for integration {self}"
            )
            return False

        module_configuration = configuration["configuration"]
        if not module_configuration:
            module_configuration = []

        if "integrationInstanceName" in self.configuration.params:  # type: ignore
            instance_name = self.configuration.params["integrationInstanceName"]  # type: ignore
            self._delete_integration_instance_if_determined_by_name(
                client, instance_name
            )
        else:
            instance_name = f'{self.configuration.instance_name.replace(" ", "_")}_test_{uuid.uuid4()}'  # type: ignore

        self.build_context.logging_module.info(
            f"Configuring instance for {self} (instance name: {instance_name}, "  # type: ignore
            f'validate "test-module": {self.configuration.should_validate_test_module})'
        )

        # define module instance:
        params = self.configuration.params  # type: ignore

        module_instance = self.create_module(
            instance_name, configuration, instance_configuration
        )

        # set server keys
        if not IS_XSIAM:
            self._set_server_keys(client, server_context)

        # set module params
        for param_conf in module_configuration:
            if param_conf["display"] in params or param_conf["name"] in params:
                # param defined in conf
                key = (
                    param_conf["display"]
                    if param_conf["display"] in params
                    else param_conf["name"]
                )
                if key in {"credentials", "creds_apikey"}:
                    credentials = params[key]
                    param_value = {
                        "credential": "",
                        "identifier": credentials.get("identifier", ""),
                        "password": credentials["password"],
                        "passwordChanged": False,
                    }
                else:
                    param_value = params[key]

                param_conf["value"] = param_value
                param_conf["hasvalue"] = True
            elif param_conf["defaultValue"]:
                # param is required - take default value
                param_conf["value"] = param_conf["defaultValue"]
            module_instance["data"].append(param_conf)
        try:
            res = demisto_client.generic_request_func(
                self=client,
                method="PUT",
                path="/settings/integration",
                body=module_instance,
            )
        except ApiException:
            self.build_context.logging_module.exception(
                f"Error trying to create instance for integration: {self}"
            )
            return False

        if res[1] != 200:
            self.build_context.logging_module.error(
                f"create instance failed with status code  {res[1]}"
            )
            self.build_context.logging_module.error(pformat(res[0]))
            return False

        integration_config = ast.literal_eval(res[0])
        self.integration_configuration_from_server = integration_config
        self.module_instance = module_instance
        integration_script = (
            integration_config.get("configuration", {}).get("integrationScript", {})
            or {}
        )
        self.integration_type = integration_script.get("type", "")
        return True

    def delete_integration_instance(
        self, client, instance_id: Optional[str] = None
    ) -> bool:
        """
        Deletes an integration with the given ID
        Args:
            client: Demisto client instance
            instance_id: The instance ID to Delete

        Returns:
            True if integration was deleted else False
        """
        self.build_context.logging_module.debug(f"Deleting {self} instance")
        instance_id = instance_id or self.integration_configuration_from_server.get(
            "id"
        )
        if not instance_id:
            self.build_context.logging_module.debug(
                f"no instance ID for integration {self} was supplied"
            )
            return True
        try:
            res = demisto_client.generic_request_func(
                self=client,
                method="DELETE",
                path="/settings/integration/" + urllib.parse.quote(instance_id),
            )
        except ApiException:
            self.build_context.logging_module.exception(
                "Failed to delete integration instance, error trying to communicate with demisto."
            )
            return False
        if int(res[1]) != 200:
            self.build_context.logging_module.error(
                f"delete integration instance failed\nStatus code {res[1]}"
            )
            self.build_context.logging_module.error(pformat(res))
            return False
        if self.module_instance:
            self.module_instance = {}
        return True

    def test_integration_instance(self, client: DefaultApi) -> bool:
        """Runs test module on the integration instance
        Args:
            client: The demisto_client instance to use

        Returns:
            The integration configuration as it exists on the server after it was configured
        """
        if not self.configuration.should_validate_test_module:  # type: ignore
            self.build_context.logging_module.debug(
                f'Skipping test-module on {self} because the "validate_test" flag is set to False'
            )
            return True

        connection_retries = 3
        response_code = 0
        integration_of_instance = self.integration_configuration_from_server.get(
            "brand", ""
        )
        instance_name = self.integration_configuration_from_server.get("name", "")
        self.build_context.logging_module.info(
            f'Running "test-module" for instance "{instance_name}" of integration "{integration_of_instance}".'
        )
        for i in range(connection_retries):
            try:
                response_data, response_code, _ = demisto_client.generic_request_func(
                    self=client,
                    method="POST",
                    path="/settings/integration/test",
                    body=self.module_instance,
                    _request_timeout=120,
                )
                break
            except ApiException:
                self.build_context.logging_module.exception(
                    f"Failed to test integration {self} instance, error trying to communicate with demisto server: "
                    f"{get_ui_url(client.api_client.configuration.host)}"
                )
                return False
            except urllib3.exceptions.ReadTimeoutError:
                self.build_context.logging_module.warning(
                    f"Could not connect. Trying to connect for the {i + 1} time"
                )

        if int(response_code) != 200:
            self.build_context.logging_module.error(
                f"Integration-instance test-module failed. Bad status code: {response_code}.\n"
                f"Sever URL: {get_ui_url(client.api_client.configuration.host)}"
            )
            return False

        result_object = ast.literal_eval(response_data)
        success, failure_message = bool(
            result_object.get("success")
        ), result_object.get("message")
        if not success:
            server_url = get_ui_url(client.api_client.configuration.host)
            test_failed_msg = (
                f"Test integration failed - server: {server_url}.\n"
                f"Failure message: {failure_message}"
                if failure_message
                else " No failure message."
            )
            self.build_context.logging_module.error(test_failed_msg)
        return success

    def disable_integration_instance(self, client) -> None:
        """Disables the integration
        Args:
            client: The demisto_client instance to use

        Returns:
            The integration configuration as it exists on the server after it was configured
        """
        # tested with POSTMAN, this is the minimum required fields for the request.
        module_instance = {
            key: self.integration_configuration_from_server[key]
            for key in [
                "id",
                "brand",
                "name",
                "data",
                "isIntegrationScript",
            ]
        }
        module_instance["enable"] = "false"
        module_instance["version"] = -1
        self.build_context.logging_module.debug(
            f'Disabling integration instance "{module_instance.get("name")}"'
        )
        try:
            res = demisto_client.generic_request_func(
                self=client,
                method="PUT",
                path="/settings/integration",
                body=module_instance,
            )
        except ApiException:
            self.build_context.logging_module.exception(
                "Failed to disable integration instance"
            )
            return

        if res[1] != 200:
            self.build_context.logging_module.error(
                f"disable instance failed, Error: {pformat(res)}"
            )

    def get_docker_images(self) -> List[str]:
        """
        Gets the docker image name from the configured integration instance's body if such body exists
        Returns:

        """
        if self.integration_configuration_from_server:
            return Docker.get_integration_image(
                self.integration_configuration_from_server
            )
        else:
            raise Exception(
                "Cannot get docker image - integration instance was not created yet"
            )

    def __str__(self):
        return f'"{self.name}"'

    def __repr__(self):
        return str(self)


class TestContext:
    def __init__(
        self,
        build_context: BuildContext,
        playbook: TestPlaybook,
        client: DefaultApi,
        server_context: "ServerContext",
    ):
        """
        Initializes the TestContext class
        Args:
            build_context: The context of the current build
            playbook: The TestPlaybook instance to run in the current test execution
            client: A demisto client instance to use for communication with the server
            server_context (ServerContext): The ServerContext instance in which the TestContext instance is created in
        """
        self.build_context = build_context
        self.server_context = server_context
        self.playbook = playbook
        self.incident_id: Optional[str] = None
        self.test_docker_images: Set[str] = set()
        self.client: DefaultApi = client

    def _get_investigation_playbook_state(self) -> str:
        """
        Queried the server for the current status of the test's investigation
        Returns:
            A string representing the status of the playbook
        """
        try:
            investigation_playbook_raw = demisto_client.generic_request_func(
                self=self.client, method="GET", path=f"/inv-playbook/{self.incident_id}"
            )
            investigation_playbook = ast.literal_eval(investigation_playbook_raw[0])
        except ApiException as err:
            self.build_context.logging_module.debug(f"{err=}", real_time=True)
            self.build_context.logging_module.debug(f"{err.status=}", real_time=True)
            if err.status == 401:
                # resetting client due to possible session timeouts
                self.server_context._configure_new_client()
                self.build_context.logging_module.debug(
                    f"new demisto_client created because of err: {err}", real_time=True
                )
                # after resetting client, playbook's state should still be in progress
                return PB_Status.IN_PROGRESS
            # if a different error other than 401 was returned, we log the exception and fail the test playbook.
            else:
                self.build_context.logging_module.exception(
                    f"Failed to get investigation playbook state, "
                    f"error trying to communicate with demisto server: {err}",
                    real_time=True,
                )
                return PB_Status.FAILED

        try:
            state = investigation_playbook["state"]
            return state
        except Exception:
            # setting state to `in progress` in XSIAM build,
            # Because `investigation_playbook` returned empty if xsiam investigation is still in progress.
            if IS_XSIAM:
                return PB_Status.IN_PROGRESS
            return PB_Status.NOT_SUPPORTED_VERSION

    def _collect_docker_images(self) -> None:
        """
        Collects docker images of the playbook's integration.
        This method can be called only after the integrations were configured in the server.
        """
        for integration in self.playbook.integrations:
            docker_images = integration.get_docker_images()
            if docker_images:
                self.test_docker_images.update(docker_images)

    def _print_investigation_error(self):
        try:
            res = demisto_client.generic_request_func(
                self=self.client,
                method="POST",
                path="/investigation/" + urllib.parse.quote(self.incident_id),  # type: ignore
                body={"pageSize": 1000},
            )
            if res and int(res[1]) == 200:
                resp_json = ast.literal_eval(res[0])
                entries = resp_json["entries"]

                self.build_context.logging_module.error(
                    f"Playbook {self.playbook} has failed:"
                )
                for entry in entries:
                    if entry["type"] == ENTRY_TYPE_ERROR and entry["parentContent"]:
                        self.build_context.logging_module.error(
                            f'- Task ID: {entry["taskId"]}'
                        )
                        # Checks for passwords and replaces them with "******"
                        parent_content = re.sub(
                            r' (P|p)assword="[^";]*"',
                            " password=******",
                            entry["parentContent"],
                        )
                        self.build_context.logging_module.error(
                            f"  Command: {parent_content}"
                        )
                        self.build_context.logging_module.error(
                            f'  Body:\n{entry["contents"]}'
                        )
            else:
                self.build_context.logging_module.error(
                    f"Failed getting entries for investigation: {self.incident_id}. Res: {res}"
                )
        except ApiException:
            self.build_context.logging_module.exception(
                "Failed to print investigation error, error trying to communicate with demisto server"
            )

    def _poll_for_playbook_state(self) -> str:
        """
        Polls for the playbook execution in the incident and return it's state.
        Returns:
            A string representing the status of the playbook
        """
        timeout = time.time() + self.playbook.configuration.timeout
        number_of_attempts = 1
        # wait for playbook to finish run
        while True:
            # give playbook time to run
            time.sleep(5)
            try:
                # fetch status
                playbook_state = self._get_investigation_playbook_state()
            except demisto_client.demisto_api.rest.ApiException:
                playbook_state = "Pending"
                self.build_context.logging_module.exception(
                    "Error when trying to get investigation playbook state"
                )

            if playbook_state in (PB_Status.COMPLETED, PB_Status.NOT_SUPPORTED_VERSION):
                break
            if playbook_state == PB_Status.FAILED:
                self.build_context.logging_module.error(
                    f"{self.playbook} failed with error/s"
                )
                self._print_investigation_error()
                break
            if time.time() > timeout:
                self.build_context.logging_module.error(
                    f"{self.playbook} failed on timeout"
                )
                break

            if number_of_attempts % DEFAULT_INTERVAL == 0:
                self.build_context.logging_module.info(
                    f"{self.playbook} loop no. {number_of_attempts / DEFAULT_INTERVAL}, {playbook_state=}"
                )
            number_of_attempts = number_of_attempts + 1
        return playbook_state

    def _run_incident_test(self) -> str:
        """
        Creates an incident in demisto server and return its status
        Returns:
            Empty string or
        """
        try:
            instance_configuration = self.playbook.configuration.instance_configuration

            if not self.playbook.configure_integrations(
                self.client, self.server_context, instance_configuration
            ):
                return PB_Status.CONFIGURATION_FAILED

            test_module_result = self.playbook.run_test_module_on_integrations(
                self.client
            )
            if not test_module_result:
                self.playbook.disable_integrations(self.client, self.server_context)
                return PB_Status.FAILED

            external_playbook_configuration = (
                self.playbook.configuration.external_playbook_config
            )

            # Change Configuration for external configuration if needed
            (
                restore_needed,
                default_vals,
                restore_path,
            ) = replace_external_playbook_configuration(
                self.client, external_playbook_configuration
            )

            incident = self.playbook.create_incident(self.client)
            if not incident:
                return ""

            self.incident_id = incident.id if IS_XSIAM else incident.investigation_id
            investigation_id = self.incident_id
            if investigation_id is None:
                self.build_context.logging_module.error(
                    f"Failed to get investigation id of incident: {incident}"
                )
                return ""

            self.build_context.logging_module.info(
                f"Found incident with incident ID: {investigation_id}."
            )

            server_url = get_ui_url(self.client.api_client.configuration.host)
            if IS_XSIAM:
                self.build_context.logging_module.info(
                    f"Investigation URL: {self.build_context.xsiam_ui_path}incident-view/alerts_and_insights?caseId="
                    f"{investigation_id}&action:openAlertDetails={investigation_id}-work_plan"
                )
            else:
                self.build_context.logging_module.info(
                    f"Investigation URL: {server_url}/#/WorkPlan/{investigation_id}"
                )
            playbook_state = self._poll_for_playbook_state()
            self.build_context.logging_module.info(
                f"Got incident: {investigation_id} status: {playbook_state}."
            )
            if self.playbook.configuration.context_print_dt:
                self.playbook.print_context_to_log(self.client, investigation_id)

            # restore Configuration for external playbook
            if restore_needed:
                restore_external_playbook_configuration(
                    self.client, restore_path=restore_path, restore_values=default_vals
                )

            self.playbook.disable_integrations(self.client, self.server_context)
            self._clean_incident_if_successful(playbook_state)
            return playbook_state
        except Exception:
            self.build_context.logging_module.exception(
                f"Failed to run incident test for {self.playbook}"
            )
            return PB_Status.FAILED

    def _clean_incident_if_successful(self, playbook_state: str) -> None:
        """
        Deletes the integration instances and the incident if the test was successful or failed on docker rate limit
        Args:
            playbook_state: The state of the playbook with which we can check if the test was successful
        """
        test_passed = playbook_state in (
            PB_Status.COMPLETED,
            PB_Status.NOT_SUPPORTED_VERSION,
        )
        # batchDelete is not supported in XSIAM, only close.
        # in XSAIAM we are closing both successful and failed incidents
        if IS_XSIAM and self.incident_id:
            self.playbook.close_incident(self.client, self.incident_id)
            self.playbook.delete_integration_instances(self.client)
        elif self.incident_id and test_passed:
            self.playbook.delete_incident(self.client, self.incident_id)
            self.playbook.delete_integration_instances(self.client)

    def _run_docker_threshold_test(self):
        self._collect_docker_images()
        if self.test_docker_images:
            memory_threshold, pid_threshold = self.get_threshold_values()
            error_message = Docker.check_resource_usage(
                server_url=self.server_context.server_ip,
                docker_images=self.test_docker_images,
                def_memory_threshold=memory_threshold,
                def_pid_threshold=pid_threshold,
                docker_thresholds=self.build_context.conf.docker_thresholds,
                logging_module=self.build_context.logging_module,
            )
            if error_message:
                self.build_context.logging_module.error(error_message)
                return False
        return True

    def get_threshold_values(self) -> Tuple[int, int]:
        """
        Gets the memory and pid threshold values to enforce on the current test.
        If one of the playbook's integrations is a powershell integration - the threshold have to be equals or
        higher than the default powershell threshold value.
        Returns:
            - The memory threshold value
            - The pid threshold value
        """
        memory_threshold = self.playbook.configuration.memory_threshold
        pid_threshold = self.playbook.configuration.pid_threshold
        has_pwsh_integration = any(
            integration
            for integration in self.playbook.integrations
            if integration.integration_type == Docker.POWERSHELL_INTEGRATION_TYPE
        )
        if has_pwsh_integration:
            memory_threshold = max(
                Docker.DEFAULT_PWSH_CONTAINER_MEMORY_USAGE, memory_threshold
            )
            pid_threshold = max(Docker.DEFAULT_PWSH_CONTAINER_PIDS_USAGE, pid_threshold)
        return memory_threshold, pid_threshold

    def _send_slack_message(self, channel, text, user_name, as_user):
        self.build_context.slack_client.api_call(
            "chat.postMessage",
            json={
                "channel": channel,
                "username": user_name,
                "as_user": as_user,
                "text": text,
                "mrkdwn": "true",
            },
        )

    def _notify_failed_test(self):
        text = (
            f"{self.build_context.build_name} - {self.playbook} Failed\n"
            f"for more details browse into the following link\n"
            f"{get_ui_url(self.client.api_client.configuration.host)}"
        )
        text += f"/#/WorkPlan/{self.incident_id}" if self.incident_id else ""
        if self.build_context.slack_user_id:
            self.build_context.slack_client.api_call(
                "chat.postMessage",
                json={
                    "channel": self.build_context.slack_user_id,
                    "username": "Content CircleCI",
                    "as_user": "False",
                    "text": text,
                },
            )

    def _add_to_succeeded_playbooks(self) -> None:
        """
        Adds the playbook to the succeeded playbooks list
        """
        self.build_context.tests_data_keeper.succeeded_playbooks.append(
            self.playbook.configuration.playbook_id
        )

    def _add_details_to_failed_tests_report(
        self, playbook_name: str, failed_stage: str
    ) -> None:
        """
        Adds the relevant details to the failed tests report.

        Args:
            playbook_name: The test's name.
            failed_stage: The stage where the test failed.
        """
        self.build_context.tests_data_keeper.playbook_report.setdefault(
            playbook_name, []
        ).append(
            {
                "number_of_executions": self.playbook.configuration.number_of_executions,
                "number_of_successful_runs": self.playbook.configuration.number_of_successful_runs,
                "failed_stage": failed_stage,
            }
        )

    @staticmethod
    def _get_failed_stage(
        status: Optional[str], is_second_playback_run: bool = False
    ) -> str:
        """
        Gets the test failed stage.

        Args:
            status: what is the test status.
            is_second_playback_run: Is The playbook run on a second playback after a freshly created record.
        """
        if is_second_playback_run:
            return "Second playback"
        if status == PB_Status.FAILED_DOCKER_TEST:
            return "Docker test"
        if status == PB_Status.CONFIGURATION_FAILED:
            return "Configuration"
        return "Execution"

    def _add_to_failed_playbooks(
        self, is_second_playback_run: bool = False, status: Optional[str] = None
    ) -> None:
        """
        Adds the playbook to the failed playbooks list

        Args:
            is_second_playback_run: Is The playbook run on a second playback after a freshly created record
        """
        failed_stage = self._get_failed_stage(status, is_second_playback_run)
        playbook_name_to_add = self.playbook.configuration.playbook_id
        if not self.playbook.is_mockable:
            playbook_name_to_add += " (Mock Disabled)"
        if is_second_playback_run:
            self.build_context.logging_module.error(
                "Playback on newly created record has failed, see the following confluence page for help:\n"
                "https://confluence.paloaltonetworks.com/display/DemistoContent/Debug+Proxy-Related+Test+Failures"
            )
            playbook_name_to_add += " (Second Playback)"

        self._add_details_to_failed_tests_report(
            self.playbook.configuration.playbook_id, failed_stage
        )
        self.build_context.logging_module.error(f"Test failed: {self}")
        self.build_context.tests_data_keeper.failed_playbooks.add(playbook_name_to_add)

    @staticmethod
    def _get_circle_memory_data() -> Tuple[str, str]:
        """
        Checks how many bytes are currently in use in the circle build instance
        Returns:
            The number of bytes in use
        """
        process = subprocess.Popen(
            ["cat", "/sys/fs/cgroup/memory/memory.usage_in_bytes"],
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )
        stdout, stderr = process.communicate()
        return stdout.decode(), stderr.decode()

    @staticmethod
    def _get_circle_processes_data() -> Tuple[str, str]:
        """
        Returns some data about the processes currently running in the circle build instance
        """
        process = subprocess.Popen(
            ["ps", "aux"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
        )
        stdout, stderr = process.communicate()
        return stdout.decode(), stderr.decode()

    def _send_circle_memory_and_pid_stats_on_slack(self):
        """
        Sends a slack messages with the number of bytes currently in use and the number of processes currently in use
        """
        if (
            self.build_context.is_nightly
            and self.build_context.memCheck
            and not self.build_context.is_local_run
        ):
            stdout, stderr = self._get_circle_memory_data()
            text = f"Memory Usage: {stdout}" if not stderr else stderr
            self._send_slack_message(
                SLACK_MEM_CHANNEL_ID, text, "Content CircleCI", "False"
            )
            stdout, stderr = self._get_circle_processes_data()
            text = stdout if not stderr else stderr
            self._send_slack_message(
                SLACK_MEM_CHANNEL_ID, text, "Content CircleCI", "False"
            )

    def _incident_and_docker_test(self) -> str:
        """
        Runs incident and docker tests (the docker test is optional)
        Returns:
            The result of the test.
        """
        playbook_state = self._run_incident_test()
        # We don't want to run docker tests on redhat instance because it does not use docker and it does not support
        # the threshold configurations.
        if (
            playbook_state == PB_Status.COMPLETED
            and self.server_context.is_instance_using_docker
        ):
            #  currently not supported on XSIAM (etc/#47908)
            if not IS_XSIAM:
                docker_test_results = self._run_docker_threshold_test()
                if not docker_test_results:
                    playbook_state = PB_Status.FAILED_DOCKER_TEST
                self._clean_incident_if_successful(playbook_state)
        return playbook_state

    def _execute_unmockable_test(self) -> bool:
        """
        Executes an unmockable test
        In case the test failed to execute because one of the test's integrations were locked will return False
        as indication that the test was not done.
        Returns:
            True if test has finished it's execution else False
        """
        self.build_context.logging_module.info(
            f"------ Test {self} start ------ (Mock: Disabled)"
        )
        with acquire_test_lock(self.playbook) as lock:
            if lock:
                status = self._incident_and_docker_test()
                self.playbook.configuration.number_of_executions += 1
                self._update_playbook_status(status)
            else:
                return False
        return True

    def _update_complete_status(
        self,
        is_first_execution: bool,
        is_record_run: bool,
        is_first_playback_run: bool,
        is_second_playback_run: bool,
        use_retries_mechanism: bool,
        number_of_executions: int,
    ):
        """
        Updates (if necessary) the playbook status in case the original status is complete.
        Args:
            is_first_execution: whether it is the first execution or not.
            is_record_run: whether it is a record execution or not.
            is_first_playback_run: whether it is the first playback execution or not.
            is_second_playback_run: whether it is the second playback execution or not.
            use_retries_mechanism: whether to use the retries mechanism or not.
            number_of_executions: the number of times the test was executed.
        Returns:
            PB_Status.COMPLETED if the Test-Playbook passed successfully and was added to succeeded playbooks.
            PB_Status.FAILED if the Test-Playbook failed and was added to failed playbooks.
            PB_Status.SECOND_PLAYBACK_REQUIRED if second playback is needed.
            PB_Status.IN_PROGRESS if more executions are needed in order to determine whether the playbook is successful or not.
        """

        self.build_context.logging_module.success(f"PASS: {self} succeed")

        # count successful run only when recording mockable or executing unmockable test.
        if not is_first_playback_run and not is_second_playback_run:
            self.playbook.configuration.number_of_successful_runs += 1

        # mockable test, first record passed, still need to check second playback.
        if is_first_execution and is_record_run:
            return PB_Status.SECOND_PLAYBACK_REQUIRED

        # first/last playback passed (mockable tests) or first execution passed (unmockable tests). adding to succeeded
        # playbooks, no more executions are needed.
        if is_first_execution or is_first_playback_run or is_second_playback_run:
            return PB_Status.COMPLETED

        if use_retries_mechanism:
            return self._update_status_based_on_retries_mechanism(
                number_of_executions, is_record_run
            )

        return PB_Status.COMPLETED

    def _update_failed_status(
        self,
        is_record_run: bool,
        is_first_playback_run: bool,
        is_second_playback_run: bool,
        use_retries_mechanism: bool,
        number_of_executions: int,
    ):
        """
        Handles the playbook failed status
        - Logs according to the results
        Args:
            is_record_run: whether it is a record execution.
            is_first_playback_run: whether it is the first playback execution.
            is_second_playback_run: whether it is the second playback execution.
            use_retries_mechanism: whether to use the retries mechanism.
            number_of_executions: how many times the test was executed.
        Returns:
            PB_Status.COMPLETED if the Test-Playbook passed successfully and was added to succeeded playbooks.
            PB_Status.FAILED if the Test-Playbook failed and was added to failed playbooks.
            PB_Status.SECOND_PLAYBACK_REQUIRED if second playback is needed.
            PB_Status.IN_PROGRESS if more executions are needed in order to determine whether the playbook is successful or not.
        """
        # in case the first playback run fails, the code should continue to record the playback.
        if is_first_playback_run:
            return PB_Status.IN_PROGRESS

        # in case the second playback run fails, test-playbook is considered a failure.
        if is_second_playback_run:
            return PB_Status.FAILED

        # in case of using the retries mechanism, the function should determine whether the test-playbook is considered
        # a failure or give it another try.
        if use_retries_mechanism:
            return self._update_status_based_on_retries_mechanism(
                number_of_executions, is_record_run
            )

        # the test-playbook is considered a failed playbook.
        return PB_Status.FAILED

    def _update_status_based_on_retries_mechanism(
        self, number_of_executions, is_record_run
    ):
        """
        Updates the status of a test-playbook when using the retries mechanism.
        Args:
            number_of_executions: how many times the test was executed.
            is_record_run: whether it is a record execution.
        Returns:
            PB_Status.COMPLETED if the Test-Playbook passed successfully and was added to succeeded playbooks.
            PB_Status.FAILED if the Test-Playbook failed and was added to failed playbooks.
            PB_Status.SECOND_PLAYBACK_REQUIRED if second playback is needed.
            PB_Status.IN_PROGRESS if more executions are needed in order to determine whether the playbook is successful or not.
        """
        if number_of_executions < MAX_RETRIES:
            self.build_context.logging_module.info(
                f"Using the retries mechanism for test {self}.\n"
                f"Test-Playbook was executed {number_of_executions} times, more executions are needed."
            )
            self.build_context.test_retries_queue.put(self.playbook)
            return PB_Status.IN_PROGRESS

        else:  # number_of_executions == MAX_RETRIES:
            # check if in most executions, the test passed.
            if (
                self.playbook.configuration.number_of_successful_runs
                >= RETRIES_THRESHOLD
            ):
                # It's not enough that the record run will pass to declare the test as successful,
                # we need the second playback to pass as well.
                if is_record_run:
                    self.build_context.logging_module.info(
                        f"Test-Playbook recording was executed {MAX_RETRIES} times, and passed {self.playbook.configuration.number_of_successful_runs} times."
                        f" Running second playback."
                    )
                    return PB_Status.SECOND_PLAYBACK_REQUIRED
                self.build_context.logging_module.info(
                    f"Test-Playbook was executed {MAX_RETRIES} times, and passed {self.playbook.configuration.number_of_successful_runs} times."
                    f" Adding to succeeded playbooks."
                )
                return PB_Status.COMPLETED
            else:
                self.build_context.logging_module.info(
                    f"Test-Playbook was executed {MAX_RETRIES} times, and passed only {self.playbook.configuration.number_of_successful_runs} times."
                    f" Adding to failed playbooks."
                )
                return PB_Status.FAILED

    def _update_playbook_status(
        self,
        status: str,
        is_first_playback_run: bool = False,
        is_second_playback_run: bool = False,
        is_record_run: bool = False,
    ) -> str:
        """
        Updates the playbook status if necessary and adds the test to the right set (succeeded/failed) if test is done.
        Args:
            status: The string representation of the playbook execution
            is_first_playback_run: Is the playbook runs in playback mode
            is_second_playback_run: Is The playbook run on a second playback after a freshly created record
        Returns:
            PB_Status.COMPLETED if the Test-Playbook passed successfully and was added to succeeded playbooks.
            PB_Status.FAILED if the Test-Playbook failed and was added to failed playbooks.
            PB_Status.SECOND_PLAYBACK_REQUIRED if second playback is needed.
            PB_Status.IN_PROGRESS if more executions are needed in order to determine whether the playbook is successful or not.
        """
        use_retries_mechanism = self.server_context.use_retries_mechanism
        number_of_executions = self.playbook.configuration.number_of_executions
        is_first_execution = number_of_executions == 1

        if status == PB_Status.COMPLETED:
            updated_status = self._update_complete_status(
                is_first_execution,
                is_record_run,
                is_first_playback_run,
                is_second_playback_run,
                use_retries_mechanism,
                number_of_executions,
            )

        elif status in (PB_Status.FAILED_DOCKER_TEST, PB_Status.CONFIGURATION_FAILED):
            self._add_to_failed_playbooks(status=status)
            return status

        else:  # test-playbook failed
            updated_status = self._update_failed_status(
                is_record_run,
                is_first_playback_run,
                is_second_playback_run,
                use_retries_mechanism,
                number_of_executions,
            )

        if updated_status == PB_Status.COMPLETED:
            self._add_to_succeeded_playbooks()
        elif updated_status == PB_Status.FAILED:
            self._notify_failed_test()
            self._add_to_failed_playbooks(is_second_playback_run=is_second_playback_run)
        return updated_status

    def _execute_mockable_test(self, proxy: MITMProxy):
        """
        Executes a mockable test
        In case the test failed to execute because one of the test's integrations were locked will return False
        as indication that the test was not done.
        Returns:
            True if test has finished it's execution else False
        """
        # we want to test first playback only once (we want to skip it when using retries mechanism)
        if not self.playbook.configuration.is_first_playback_failed:
            if proxy.has_mock_file(self.playbook.configuration.playbook_id):
                # Running first playback run on mock file
                self.build_context.logging_module.info(
                    f"------ Test {self} start ------ (Mock: Playback)"
                )
                with run_with_mock(
                    proxy, self.playbook.configuration.playbook_id
                ) as result_holder:
                    status = self._incident_and_docker_test()
                    status = self._update_playbook_status(
                        status, is_first_playback_run=True
                    )
                    result_holder[RESULT] = status == PB_Status.COMPLETED
                if status in (PB_Status.COMPLETED, PB_Status.FAILED_DOCKER_TEST):
                    return True
                self.build_context.logging_module.warning(
                    "Test failed with mock, recording new mock file. (Mock: Recording)"
                )
                self.playbook.configuration.is_first_playback_failed = True

        # Running on record mode since playback has failed or mock file was not found
        self.build_context.logging_module.info(
            f"------ Test {self} start ------ (Mock: Recording)"
        )
        with acquire_test_lock(self.playbook) as lock:
            if lock:
                with run_with_mock(
                    proxy, self.playbook.configuration.playbook_id, record=True
                ) as result_holder:
                    status = self._incident_and_docker_test()
                    self.playbook.configuration.number_of_executions += 1
                    status = self._update_playbook_status(status, is_record_run=True)
                    result_holder[RESULT] = status == PB_Status.SECOND_PLAYBACK_REQUIRED
            else:
                # If the integrations were not locked - the test has not finished it's execution
                return False

        # Running playback after successful record to verify the record is valid for future runs
        if status == PB_Status.SECOND_PLAYBACK_REQUIRED:
            self.build_context.logging_module.info(
                f"------ Test {self} start ------ (Mock: Second playback)"
            )
            with run_with_mock(
                proxy, self.playbook.configuration.playbook_id
            ) as result_holder:
                status = self._run_incident_test()
                self._update_playbook_status(status, is_second_playback_run=True)
                result_holder[RESULT] = status == PB_Status.COMPLETED
        return True

    def _is_runnable_on_current_server_instance(self) -> bool:
        """
        Nightly builds can have RHEL instances that uses podman instead of docker as well as the regular LINUX instance.
        In such case - if the test in runnable on docker instances **only** and the current instance uses podman -
        we will not execute the test under this instance and instead will will return it to the queue in order to run
        it under some other instance
        Returns:
            True if this instance can be run on the current instance else False
        """
        if (
            self.playbook.configuration.runnable_on_docker_only
            and not self.server_context.is_instance_using_docker
        ):
            self.build_context.logging_module.debug(
                f"Skipping test {self.playbook} since it's not runnable on podman instances"
            )
            return False
        return True

    def execute_test(self, proxy: Optional[MITMProxy] = None) -> bool:
        """
        Executes the test and return a boolean that indicates whether the test was executed or not.
        In case the test was not executed - it will be returned to the queue and will be collected later in the future
        by some other ServerContext instance.
        Args:
            proxy: The MITMProxy instance to use in the current test

        Returns:
            True if the test was executed by the instance else False
        """
        try:
            if not self._is_runnable_on_current_server_instance():
                return False
            self._send_circle_memory_and_pid_stats_on_slack()
            if self.playbook.is_mockable:
                test_executed = self._execute_mockable_test(proxy)  # type: ignore
            else:
                test_executed = self._execute_unmockable_test()
            return test_executed
        except Exception:
            self.build_context.logging_module.exception(
                f"Unexpected error while running test on playbook {self.playbook}"
            )
            self._add_to_failed_playbooks()
            return True
        finally:
            self.build_context.logging_module.info(f"------ Test {self} end ------ \n")
            self.build_context.logging_module.execute_logs()

    def __str__(self):
        test_message = f"playbook: {self.playbook}"
        if self.playbook.integrations:
            test_message += f" with integration(s): {self.playbook.integrations}"
        else:
            test_message += " with no integrations"
        if not self.server_context.is_instance_using_docker:
            test_message += ", RedHat instance"
        return test_message

    def __repr__(self):
        return str(self)


class ServerContext:
    def __init__(
        self,
        build_context: BuildContext,
        server_private_ip: str,
        use_retries_mechanism: bool = True,
    ):
        self.build_context = build_context
        self.server_ip = server_private_ip
        if IS_XSIAM:
            self.server_url = server_private_ip
            # we use client without demisto username
            os.environ.pop("DEMISTO_USERNAME", None)
        else:
            self.server_url = f"https://{self.server_ip}"
        self.client: Optional[DefaultApi] = None
        self._configure_new_client()
        # currently not supported on XSIAM (etc/#47851)
        if IS_XSIAM:
            self.proxy = None
        else:
            self.proxy = MITMProxy(
                server_private_ip,
                self.build_context.logging_module,
                build_number=self.build_context.build_number,
                branch_name=self.build_context.build_name,
            )
        self.is_instance_using_docker = not is_redhat_instance(self.server_ip)
        self.executed_tests: Set[str] = set()
        self.executed_in_current_round: Set[str] = set()
        self.prev_system_conf: dict = {}
        self.use_retries_mechanism: bool = use_retries_mechanism
        if IS_XSIAM:
            self.check_if_can_create_manual_alerts()

    def _execute_unmockable_tests(self):
        """
        Iterates the mockable tests queue and executes them as long as there are tests to execute
        """
        self._execute_tests(self.build_context.unmockable_tests_to_run)

    def _execute_tests(self, queue: Queue) -> None:
        """
        Iterates the tests queue and executes them as long as there are tests to execute.
        Before the tests execution starts we will reset the containers to make sure the proxy configuration is correct
        - We need it before the mockable tests because the server starts the python2 default container when it starts
            and it has no proxy configurations.
        - We need it before the unmockable tests because at that point all containers will have the proxy configured
            and we want to clean those configurations when testing unmockable playbooks
        Args:
            queue: The queue to fetch tests to execute from
        """
        self._reset_containers()
        while queue.unfinished_tasks:
            try:
                test_playbook: TestPlaybook = queue.get(block=False)
                self._reset_tests_round_if_necessary(test_playbook, queue)
            except Empty:
                continue
            self._configure_new_client()
            test_executed = TestContext(
                self.build_context, test_playbook, self.client, self
            ).execute_test(self.proxy)
            if test_executed:
                self.executed_tests.add(test_playbook.configuration.playbook_id)
            else:
                queue.put(test_playbook)
            queue.task_done()

    def _execute_mockable_tests(self):
        """
        Iterates the mockable tests queue and executes them as long as there are tests to execute
        """
        # we running XSIAM without proxy. This code wont be executed on xsiam servers
        if not IS_XSIAM:
            self.proxy.configure_proxy_in_demisto(  # type: ignore[union-attr]
                proxy=self.proxy.ami.internal_ip + ":" + self.proxy.PROXY_PORT,  # type: ignore[union-attr]
                username=self.build_context.secret_conf.server_username,
                password=self.build_context.secret_conf.server_password,
                server=self.server_url,
            )
        self._execute_tests(self.build_context.mockable_tests_to_run)
        if not IS_XSIAM:
            self.proxy.configure_proxy_in_demisto(  # type: ignore[union-attr]
                proxy="",
                username=self.build_context.secret_conf.server_username,
                password=self.build_context.secret_conf.server_password,
                server=self.server_url,
            )

    def _execute_failed_tests(self):
        self._execute_tests(self.build_context.test_retries_queue)

    def _reset_tests_round_if_necessary(
        self, test_playbook: TestPlaybook, queue_: Queue
    ) -> None:
        """
        Checks if the string representation of the current test configuration is already in
        the executed_in_current_round set.
        If it is- it means we have already executed this test and the we have reached a round and there are tests that
        were not able to be locked by this execution..
        In that case we want to start a new round monitoring by emptying the 'executed_in_current_round' set and sleep
        in order to let the tests be unlocked
        Since this operation can be performed by multiple threads - this operaion is protected by the queue's lock
        Args:
            test_playbook: Test playbook to check if has been executed in the current round
            queue_: The queue from which the current tests are executed
        """
        queue_.mutex.acquire()
        if str(test_playbook.configuration) in self.executed_in_current_round:
            self.build_context.logging_module.info(
                "all tests in the queue were executed, sleeping for 30 seconds to let locked tests get unlocked."
            )
            self.executed_in_current_round = {str(test_playbook.configuration)}
            queue_.mutex.release()
            time.sleep(30)
            return
        else:
            self.executed_in_current_round.add(str(test_playbook.configuration))
        queue_.mutex.release()

    def _configure_new_client(self):
        if self.client:
            self.client.api_client.pool.close()
            self.client.api_client.pool.terminate()
            del self.client
        self.client = demisto_client.configure(
            base_url=self.server_url,
            api_key=self.build_context.api_key,
            auth_id=self.build_context.auth_id,
            verify_ssl=False,
        )

    def _reset_containers(self):
        if self.build_context.is_xsiam:
            self.build_context.logging_module.info(
                "Skip reset containers - this API is not supported.", real_time=True
            )
            return

        self.build_context.logging_module.info("Resetting containers\n", real_time=True)

        body, status_code, _ = demisto_client.generic_request_func(
            self=self.client, method="POST", path="/containers/reset"
        )
        if status_code != 200:
            self.build_context.logging_module.critical(
                f'Request to reset containers failed with status code "{status_code}"\n{body}',
                real_time=True,
            )
            sys.exit(1)
        time.sleep(10)

    def execute_tests(self):
        try:
            self.build_context.logging_module.info(
                f"Starts tests with server url - {get_ui_url(self.server_url)}",
                real_time=True,
            )
            self._execute_mockable_tests()
            self.build_context.logging_module.info(
                "Running mock-disabled tests", real_time=True
            )
            self._configure_new_client()
            self._execute_unmockable_tests()
            if self.use_retries_mechanism:
                self.build_context.logging_module.info(
                    "Running failed tests", real_time=True
                )
                self._execute_failed_tests()
            self.build_context.logging_module.info(
                f"Finished tests with server url - " f"{get_ui_url(self.server_url)}",
                real_time=True,
            )
            # no need in xsiam, no proxy
            if not IS_XSIAM:
                self.build_context.tests_data_keeper.add_proxy_related_test_data(
                    self.proxy
                )

            if self.build_context.isAMI and not IS_XSIAM:
                if self.proxy.should_update_mock_repo:  # type: ignore[union-attr]
                    self.proxy.push_mock_files()  # type: ignore[union-attr]
            self.build_context.logging_module.debug(
                f"Tests executed on server {self.server_ip}:\n"
                f"{pformat(self.executed_tests)}"
            )
        except Exception:
            self.build_context.logging_module.exception("~~ Thread failed ~~")
            raise
        finally:
            self.build_context.logging_module.execute_logs()

    def check_if_can_create_manual_alerts(self):
        """
        In XSIAM we can't create a new incident/alert using API call.
        We need a correlation rule in order to create an alert.
        We want to create an alert manually, when we send an API call to XSIAM server to create a new alert.
        Server check which integration sent a new alert, if the request was sent manually and not from integration it
        sets "sourceBrand" header to be "Manual". XSIAM Server looks for a correlation rule for such sourceBrand,
        and if there is no such correlation rule, no alert will be created.
        If there is a correlation rule for "Manual" integration the alert will be created.

        If this step fails please create an integration with id and name "Manual", set isFetch: true for such
        integration and make sure that the corresponding correlation rule is created.
        """
        body = {
            "query": 'id:"Manual"',
        }
        try:
            res_raw = demisto_client.generic_request_func(
                self=self.client,
                method="POST",
                path="/settings/integration/search",
                body=body,
            )
            res = ast.literal_eval(res_raw[0])
            if int(res_raw[1]) != 200:
                self.build_context.logging_module.error(
                    f"Failed to get integrations configuration with status code: {res_raw[1]}"
                )
                return

            all_configurations = res["configurations"]
            for instance in all_configurations:
                if instance.get("id") == "Manual":
                    self.build_context.logging_module.info(
                        "Server is able to create manual alerts "
                        '("Manual" integration exists).'
                    )
                    return
        except ApiException:
            self.build_context.logging_module.exception(
                "Failed to get integrations configuration."
            )

        self.build_context.logging_module.warning(
            'No "Manual" integration found in XSIAM instance. '
            "Adding it in order to create Manual Correlation Rule."
        )
        self.create_manual_integration()

    def create_manual_integration(self):
        manual_integration = {
            "name": "Manual",
            "version": 1,
            "display": "Manual",
            "category": "Utilities",
            "description": "This integration creates Manual Correlation Rule.",
            "configuration": [],
            "integrationScript": {
                "script": "",
                "commands": [],
                "type": "python",
                "isFetch": True,
                "subtype": "python3",
            },
        }

        try:
            res_raw_integration = demisto_client.generic_request_func(
                self=self.client,
                method="PUT",
                path="/settings/integration-conf",
                body=manual_integration,
            )
            if int(res_raw_integration[1]) != 200:
                self.build_context.logging_module.error(
                    f"Failed to get integrations configuration with status code: {res_raw_integration[1]}"
                )

        except ApiException:
            self.build_context.logging_module.exception(
                'No "Manual" integration found in XSIAM instance. '
                "Please add it in order to create Manual Correlation Rule."
            )


def replace_external_playbook_configuration(
    client: DefaultApi,
    external_playbook_configuration: dict,
    logger_module: logging.Logger = logging.getLogger("demisto-sdk"),
):
    """takes external configuration of shape {"playbookID": "Isolate Endpoint - Generic V2",
                                           "input_parameters":{"Endpoint_hostname": {"simple", "test"}}}
    and changes the specified playbook configuration to the mentioned one.
    If playbook's inputs had changed, revert will be needed.
    Returns (Whether the Playbook changed, The values to restore, the path to use when restoring)
    Only to be used with server version 6.2 and above."""

    # Checking configuration
    if not external_playbook_configuration:
        logger_module.info(
            "External Playbook Configuration not provided, skipping re-configuration."
        )
        return False, {}, ""

    # Checking server version
    server_version = get_demisto_version(client)

    if Version(server_version.base_version) < Version("6.2.0"):  # type: ignore
        logger_module.info(
            "External Playbook not supported in versions previous to 6.2.0, skipping re-configuration."
        )
        return False, {}, ""

    logger_module.info("External Playbook in use, starting re-configuration.")

    # Getting current configuration
    external_playbook_id = external_playbook_configuration["playbookID"]
    external_playbook_path = f"/playbook/{external_playbook_id}"
    res, _, _ = demisto_client.generic_request_func(
        client, method="GET", path=external_playbook_path, response_type="object"
    )

    inputs = res.get("inputs", [])
    if not inputs:
        raise Exception("External Playbook was not found or has no inputs.")

    # Save current for default Configuration.
    inputs_default = deepcopy(inputs)
    logger_module.info("Saved current configuration.")

    changed_keys = []
    failed_keys = []

    # Change Configuration for external pb.
    for input_ in external_playbook_configuration["input_parameters"]:
        matching_record = list(filter(lambda x: x.get("key") == input_, inputs))
        if matching_record:
            existing_val = matching_record[0]
            simple = external_playbook_configuration["input_parameters"][input_].get(
                "simple"
            )
            complex = external_playbook_configuration["input_parameters"][input_].get(
                "complex"
            )

            # If no value (simple or complex) was found, It is a typo
            if complex is None and simple is None:
                raise Exception(
                    f"Could not found neither a `simple` nor `complex` value for field: {input_}. "
                    "A valid configuration should be of the followng format: "
                    '{<param name>: {"simple", <required value>}}'
                )

            existing_val["value"]["simple"] = simple
            existing_val["value"]["complex"] = complex
            changed_keys.append(input_)

        else:
            failed_keys.append(input_)

    if failed_keys:
        raise Exception(
            f'Some input keys was not found in playbook: {",".join(failed_keys)}.'
        )

    logger_module.info(f"Changing keys: {changed_keys}.")
    saving_inputs_path = f"/playbook/inputs/{external_playbook_id}"

    try:
        if changed_keys:
            demisto_client.generic_request_func(
                client, method="POST", path=saving_inputs_path, body=inputs
            )

    except Exception as e:
        raise Exception(
            f"Could not change inputs in playbook configuration. Error: {e}"
        )

    logger_module.info(
        f"Re-configured {external_playbook_id} successfully with {len(changed_keys)} new values."
    )

    return True, inputs_default, saving_inputs_path


def restore_external_playbook_configuration(
    client: DefaultApi,
    restore_path: str,
    restore_values: dict,
    logger_module: logging.Logger = logging.getLogger("demisto-sdk"),
):
    logger_module.info("Restoring External Playbook parameters.")

    res, _, _ = demisto_client.generic_request_func(
        client, method="POST", path=restore_path, body=restore_values
    )

    logger_module.info("Restored External Playbook successfully.")


# demisto_sdk/commands/test_content/constants.py

SSH_USER = "gcp-user"


# demisto_sdk/commands/test_content/execute_test_content.py

import os
import sys
from threading import Thread

import requests

from demisto_sdk.commands.test_content.ParallelLoggingManager import (
    ParallelLoggingManager,
)
from demisto_sdk.commands.test_content.TestContentClasses import (
    BuildContext,
    ServerContext,
)

SKIPPED_CONTENT_COMMENT = (
    "The following integrations/tests were collected by the CI build but are currently skipped. "
    "The collected tests are related to this pull request and might be critical."
)
COVERAGE_REPORT_COMMENT = "Link to the unit tests coverage report"


def _handle_github_response(response, logging_module) -> dict:
    res_dict = response.json()
    if not response.ok:
        logging_module.error(
            f'Add pull request comment failed: {res_dict.get("message")}',
            real_time=True,
        )
    return res_dict


def _add_pr_comment(comment, logging_module):
    token = os.environ["XSOAR_BOT_TEST_CONTENT"]
    branch_name = os.environ["CI_COMMIT_BRANCH"]
    sha1 = os.environ["CI_COMMIT_SHA"]

    query = f"?q={sha1}+repo:demisto/content+org:demisto+is:pr+is:open+head:{branch_name}+is:open"
    url = "https://api.github.com/search/issues"
    headers = {"Authorization": "Bearer " + token}

    is_skipped_tests_flow = "The following integrations/tests were collected" in comment

    try:
        response = requests.get(url + query, headers=headers, verify=False)
        res = _handle_github_response(response, logging_module)

        if res and res.get("total_count", 0) == 1:
            issue_url = (
                res["items"][0].get("comments_url") if res.get("items", []) else None
            )
            if issue_url:
                # Check if a comment about skipped tests already exists. If there is delete it first and then post a
                # new comment:
                response = requests.get(issue_url, headers=headers, verify=False)
                issue_comments = _handle_github_response(response, logging_module)
                for existing_comment in issue_comments:
                    if (
                        is_skipped_tests_flow
                        and SKIPPED_CONTENT_COMMENT in existing_comment.get("body", "")
                    ) or (
                        not is_skipped_tests_flow
                        and COVERAGE_REPORT_COMMENT in existing_comment.get("body", "")
                    ):
                        comment_url = existing_comment.get("url")
                        requests.delete(comment_url, headers=headers, verify=False)
                response = requests.post(
                    issue_url, json={"body": comment}, headers=headers, verify=False
                )
                _handle_github_response(response, logging_module)
        else:
            logging_module.warning(
                "Add pull request comment failed: There is more then one open pull "
                f"request for branch {branch_name}.",
                real_time=True,
            )
    except Exception:
        logging_module.exception("Add pull request comment failed")


def execute_test_content(**kwargs):
    logging_manager = ParallelLoggingManager(
        "Run_Tests.log", real_time_logs_only=not kwargs["nightly"]
    )
    build_context = BuildContext(kwargs, logging_manager)
    use_retries_mechanism = kwargs.get("use_retries", False)
    threads_list = []
    for server_ip in build_context.instances_ips:
        tests_execution_instance = ServerContext(
            build_context,
            server_private_ip=server_ip,
            use_retries_mechanism=use_retries_mechanism,
        )
        threads_list.append(Thread(target=tests_execution_instance.execute_tests))

    logging_manager.info("Finished creating configurations, starting to run tests.")
    for thread in threads_list:
        thread.start()

    for t in threads_list:
        t.join()

    if (
        not build_context.unmockable_tests_to_run.empty()
        or not build_context.mockable_tests_to_run.empty()
    ):
        raise Exception("Not all tests have been executed")
    if (
        build_context.tests_data_keeper.playbook_skipped_integration
        and build_context.build_name != "master"
        and not build_context.is_nightly
    ):
        skipped_integrations = "\n- ".join(
            build_context.tests_data_keeper.playbook_skipped_integration
        )
        comment = f"{SKIPPED_CONTENT_COMMENT}:\n- {skipped_integrations}"
        _add_pr_comment(comment, logging_manager)
    build_context.tests_data_keeper.print_test_summary(
        build_context.isAMI, logging_manager
    )
    build_context.tests_data_keeper.create_result_files()
    if build_context.tests_data_keeper.failed_playbooks:
        logging_manager.critical(
            "Some tests have failed. Not destroying instances.", real_time=True
        )
        sys.exit(1)


# demisto_sdk/commands/test_content/mock_server.py

import ast
import os
import string
import time
import unicodedata
from contextlib import contextmanager
from pprint import pformat
from subprocess import STDOUT, CalledProcessError, call, check_call, check_output
from threading import Lock
from typing import Dict, Iterator

import demisto_client.demisto_api
import urllib3

from demisto_sdk.commands.test_content.constants import SSH_USER

VALID_FILENAME_CHARS = f"-_.() {string.ascii_letters}{string.digits}"
PROXY_PROCESS_INIT_TIMEOUT = 20
PROXY_PROCESS_INIT_INTERVAL = 1
RESULT = "result"
# Disable insecure warnings
urllib3.disable_warnings()


def clean_filename(
    playbook_or_integration_id, whitelist=VALID_FILENAME_CHARS, replace=" ()"
):
    filename = playbook_or_integration_id

    # replace spaces
    for r in replace:
        filename = filename.replace(r, "_")

    # keep only valid ascii chars
    cleaned_filename = (
        unicodedata.normalize("NFKD", filename).encode("ASCII", "ignore").decode()
    )

    # keep only whitelisted chars
    cleaned_filename = "".join(c for c in cleaned_filename if c in whitelist)
    return cleaned_filename


def silence_output(cmd_method, *args, **kwargs):
    """Redirect linux command output(s) to /dev/null
    To redirect to /dev/null: pass 'null' string in 'stdout' or 'stderr' keyword args

    Args:
        cmd_method (PyFunctionObject): the "subprocess" (or wrapper) method to run
        *args: additional parameters for cmd_method
        **kwargs: additional parameters for cmd_method

    Returns:
        string. output of cmd_method
    """
    with open(os.devnull, "w") as fnull:
        for k in ("stdout", "stderr"):
            if kwargs.get(k) == "null":
                kwargs[k] = fnull

        return cmd_method(*args, **kwargs)


def get_mock_file_path(playbook_or_integration_id):
    clean = clean_filename(playbook_or_integration_id)
    return os.path.join(clean + "/", clean + ".mock")


def get_log_file_path(playbook_or_integration_id, record=False):
    clean = clean_filename(playbook_or_integration_id)
    suffix = "_record" if record else "_playback"
    return os.path.join(clean + "/", clean + suffix + ".log")


def get_folder_path(playbook_or_integration_id):
    return clean_filename(playbook_or_integration_id) + "/"


class AMIConnection:
    """Wrapper for AMI communication.

    Attributes:
        internal_ip (string): The  internal IP of the instance.
    """

    REMOTE_HOME = f"/home/{SSH_USER}/"
    LOCAL_SCRIPTS_DIR = "/home/circleci/project/Tests/scripts/"

    def __init__(self, internal_ip):
        self.internal_ip = internal_ip

    def add_ssh_prefix(self, command, ssh_options=""):
        """Add necessary text before a command in order to run it on the AMI instance via SSH.

        Args:
            command (list): Command to run on the AMI machine (according to "subprocess" interface).
            ssh_options (string): optional parameters for ssh connection to AMI.

        Returns:
            string: ssh command that will run the desired command on the AMI.
        """
        if ssh_options and not isinstance(ssh_options, str):
            raise TypeError("options must be string")
        if not isinstance(command, list):
            raise TypeError("command must be list")
        prefix = f"ssh {ssh_options} {SSH_USER}@{self.internal_ip}".split()
        return prefix + command

    def call(self, command, **kwargs):
        return call(self.add_ssh_prefix(command), **kwargs)

    def check_call(self, command, **kwargs):
        return check_call(self.add_ssh_prefix(command), **kwargs)

    def check_output(self, command, **kwargs):
        return check_output(self.add_ssh_prefix(command), **kwargs)

    def copy_file(self, src, dst=REMOTE_HOME, **kwargs):
        silence_output(
            check_call,
            ["scp", src, f"{SSH_USER}@{self.internal_ip}:{dst}"],
            stdout="null",
            **kwargs,
        )
        return os.path.join(dst, os.path.basename(src))

    def run_script(self, script, *args):
        """Copy a script to the AMI and run it.

        Args:
            script (string): Name of the script file in the LOCAL_SCRIPTS_DIR.
            *args: arguments to be passed to the script.
        """
        remote_script_path = self.copy_file(
            os.path.join(self.LOCAL_SCRIPTS_DIR, script)
        )

        silence_output(
            self.check_call, ["chmod", "+x", remote_script_path], stdout="null"
        )
        silence_output(
            self.check_call, [remote_script_path] + list(args), stdout="null"
        )


class MITMProxy:
    """Manager for MITM Proxy and the mock file structure.

    Attributes:
        logging_module: Logging module to use
        public_ip (string): The IP of the AMI instance.
        repo_folder (string): path to the local clone of the content-test-data git repo.
        tmp_folder (string): path to a temporary folder for log/mock files before pushing to git.
        current_folder (string): the current folder to use for mock/log files.
        ami (AMIConnection): Wrapper for AMI communication.
        empty_files (list): List of playbooks that have empty mock files (indicating no usage of mock mechanism).
        rerecorded_tests (list): List of playbook ids that failed on mock playback but succeeded on new recording.
        build_number (str): The number of the circleci build.
        branch_name (str): The name of the content branch in which the current job works on.
    """

    PROXY_PORT = "9997"
    MOCKS_TMP_PATH = "/tmp/Mocks/"
    MOCKS_GIT_PATH = f"{AMIConnection.REMOTE_HOME}content-test-data/"
    TIME_TO_WAIT_FOR_PROXY_SECONDS = 30
    content_data_lock = Lock()

    def __init__(
        self,
        internal_ip,
        logging_module,
        build_number,
        branch_name,
        repo_folder=MOCKS_GIT_PATH,
        tmp_folder=MOCKS_TMP_PATH,
    ):
        is_branch_master = branch_name == "master"
        self.internal_ip = internal_ip
        self.current_folder = self.repo_folder = repo_folder
        self.tmp_folder = tmp_folder
        self.logging_module = logging_module
        self.build_number = build_number
        self.ami = AMIConnection(self.internal_ip)
        self.should_update_mock_repo = is_branch_master
        self.should_validate_playback = not is_branch_master
        self.empty_files = []
        self.failed_tests_count = 0
        self.successful_tests_count = 0
        self.successful_rerecord_count = 0
        self.failed_rerecord_count = 0
        self.failed_rerecord_tests = []
        self.rerecorded_tests = []
        silence_output(self.ami.call, ["mkdir", "-p", tmp_folder], stderr="null")
        script_filepath = os.path.join(
            os.path.dirname(os.path.abspath(__file__)), "timestamp_replacer.py"
        )
        self.ami.copy_file(script_filepath)

    def commit_mock_file(self, folder_name):
        self.logging_module.debug("Committing mock files")
        try:
            output = self.ami.check_output(
                "cd content-test-data && "
                "git add * -v && "
                f"git commit -m \"Updated mock files for '{folder_name}' from build number - {self.build_number}\" -v".split(),
                stderr=STDOUT,
            )
            self.logging_module.debug(
                f"Committing mock files output:\n{output.decode()}"
            )
        except CalledProcessError as exc:
            self.logging_module.debug(
                f"Committing mock files output:\n{exc.output.decode()}"
            )

    def reset_mock_files(self):
        self.logging_module.debug("Resetting mock files")
        try:
            output = self.ami.check_output(
                "cd content-test-data && git reset --hard".split(), stderr=STDOUT
            )
            self.logging_module.debug(
                f"Resetting mock files output:\n{output.decode()}"
            )
        except CalledProcessError as exc:
            self.logging_module.debug(
                f"Resetting mock files output:\n{exc.output.decode()}"
            )

    def push_mock_files(self):
        self.logging_module.debug(
            "Pushing new/updated mock files to mock git repo.", real_time=True
        )
        self.content_data_lock.acquire()
        try:
            output = self.ami.check_output(
                "cd content-test-data && git reset --hard && git pull -r -Xtheirs && git push -f".split(),
                stderr=STDOUT,
            )
            self.logging_module.debug(
                f"Pushing mock files output:\n{output.decode()}", real_time=True
            )
        except CalledProcessError as exc:
            self.logging_module.debug(
                f"Pushing mock files output:\n{exc.output.decode()}", real_time=True
            )
        except Exception as exc:
            self.logging_module.debug(
                f"Failed pushing mock files with error: {exc}", real_time=True
            )
        finally:
            self.content_data_lock.release()

    def configure_proxy_in_demisto(
        self, username, password, server, api_key=None, auth_id=None, proxy=""
    ):
        client = demisto_client.configure(
            base_url=server,
            username=username,
            password=password,
            api_key=api_key,
            auth_id=auth_id,
            verify_ssl=False,
        )
        self.logging_module.debug("Adding proxy server configurations")
        system_conf_response = demisto_client.generic_request_func(
            self=client, path="/system/config", method="GET"
        )
        system_conf = ast.literal_eval(system_conf_response[0]).get("sysConf", {})
        self.logging_module.debug(
            f"Server configurations before proxy server configurations:\n{pformat(system_conf)}"
        )
        http_proxy = https_proxy = proxy
        if proxy:
            http_proxy = "http://" + proxy
            https_proxy = "http://" + proxy
        system_conf.update({"http_proxy": http_proxy, "https_proxy": https_proxy})
        data = {"data": system_conf, "version": -1}
        response = demisto_client.generic_request_func(
            self=client, path="/system/config", method="POST", body=data
        )
        self.logging_module.debug(
            f"Server configurations response:\n{pformat(ast.literal_eval(response[0]))}"
        )

        return response

    def get_mock_file_size(self, filepath):
        return self.ami.check_output(["stat", "-c", "%s", filepath]).strip()

    @staticmethod
    def get_script_mode(is_record: bool) -> str:
        """
        Returns the string describing script mode for the SCRIPT_MODE env variable needed for the mitmdump initialization
        Args:
            is_record: A boolean indicating this is record mode or not

        Returns:
            'record' if is_record is True else 'playback'
        """
        return "record" if is_record else "playback"

    def has_mock_file(self, playbook_or_integration_id):
        command = [
            "[",
            "-f",
            os.path.join(
                self.current_folder, get_mock_file_path(playbook_or_integration_id)
            ),
            "]",
        ]
        return self.ami.call(command) == 0

    def has_mock_folder(self, playbook_or_integration_id):
        command = [
            "[",
            "-d",
            os.path.join(
                self.current_folder, get_folder_path(playbook_or_integration_id)
            ),
            "]",
        ]
        return self.ami.call(command) == 0

    def set_repo_folder(self):
        """Set the repo folder as the current folder (the one used to store mock and log files)."""
        self.current_folder = self.repo_folder

    def set_tmp_folder(self):
        """Set the temp folder as the current folder (the one used to store mock and log files)."""
        self.current_folder = self.tmp_folder

    def move_mock_file_to_repo(self, playbook_or_integration_id):
        """Move the mock and log files of a (successful) test playbook run from the temp folder to the repo folder

        Args:
            playbook_or_integration_id (string): ID of the test playbook or integration of which the files should be moved.
        """
        src_filepath = os.path.join(
            self.tmp_folder, get_mock_file_path(playbook_or_integration_id)
        )
        src_files = os.path.join(
            self.tmp_folder, get_folder_path(playbook_or_integration_id) + "*"
        )
        dst_folder = os.path.join(
            self.repo_folder, get_folder_path(playbook_or_integration_id)
        )

        if not self.has_mock_file(playbook_or_integration_id):
            self.logging_module.debug("Mock file not created!")
        elif self.get_mock_file_size(src_filepath) == "0":
            self.logging_module.debug("Mock file is empty, ignoring.")
            self.empty_files.append(playbook_or_integration_id)
        else:
            # Move to repo folder
            self.logging_module.debug(
                f'Moving "{src_files}" files to "{dst_folder}" directory'
            )
            self.ami.call(["mkdir", "--parents", dst_folder])
            self.ami.call(["mv", src_files, dst_folder])

    def normalize_mock_file(self, playbook_or_integration_id: str, path: str = None):
        """Normalize the mock file of a test playbook

        Normalizes a mock file by doing the following:
        1. Replaces the timestamp/variable data in a mock file as identified by
        the keys provided in the mock file's associated 'problematic_keys.json'
        file, with constant values.
        2. Standardizes the query parameter data order for all requests.

        Args:
            playbook_or_integration_id: ID of the test playbook or
                integration for which the associated mock file will be
                normalized.
            path: Path to the directory in which to search
                for the required data (temp directory or repo). If not
                provided, method will use the current directory.
        """
        self.logging_module.debug(
            f'normalize_mock_file was called for test "{playbook_or_integration_id}"'
        )
        path = path or self.current_folder
        problem_keys_filepath = os.path.join(
            path, get_folder_path(playbook_or_integration_id), "problematic_keys.json"
        )
        self.logging_module.debug(f'problem_keys_filepath="{problem_keys_filepath}"')
        problem_key_file_exists = ["[", "-f", problem_keys_filepath, "]"]
        if not self.ami.call(problem_key_file_exists) == 0:
            self.logging_module.debug(
                "Error: The problematic_keys.json file was not written to the file path"
                f' "{problem_keys_filepath}" when recording '
                f'the "{playbook_or_integration_id}" test playbook'
            )
            return

        mock_file_path = os.path.join(
            path, get_mock_file_path(playbook_or_integration_id)
        )
        cleaned_mock_filepath = mock_file_path.strip(".mock") + "_cleaned.mock"
        log_file = os.path.join(
            path, get_log_file_path(playbook_or_integration_id, record=True)
        )
        command = (
            "/home/ec2-user/.local/bin/mitmdump -ns ~/timestamp_replacer.py "
            f"--set script_mode=clean --set keys_filepath={problem_keys_filepath}"
            f" -r {mock_file_path} -w {cleaned_mock_filepath} | sudo tee -a {log_file}"
        )
        self.logging_module.debug(f"command to normalize mockfile:\n\t{command}")
        self.logging_module.debug("Let's try and normalize the mockfile")
        try:
            check_output(
                self.ami.add_ssh_prefix(command.split(), ssh_options="-t"),
                stderr=STDOUT,
            )
        except CalledProcessError as e:
            self.logging_module.debug(
                "There may have been a problem while normalizing the mock file."
            )
            err_msg = f"command `{command}` exited with return code [{e.returncode}]"
            err_msg = (
                f'{err_msg} and the output of "{e.output}"' if e.output else err_msg
            )
            if e.stderr:
                err_msg += f"STDERR: {e.stderr}"
            self.logging_module.debug(err_msg)
        else:
            self.logging_module.debug("Success!")

        # verify cleaned mock is different than original
        diff_cmd = f"diff -sq {cleaned_mock_filepath} {mock_file_path}"
        try:
            diff_cmd_output = self.ami.check_output(diff_cmd.split()).decode().strip()
            self.logging_module.debug(f"diff_cmd_output={diff_cmd_output}")
            if diff_cmd_output.endswith("are identical"):
                self.logging_module.debug(
                    "normalized mock file and original mock file are identical"
                )
            else:
                self.logging_module.debug(
                    "the normalized mock file differs from the original"
                )

        except CalledProcessError:
            self.logging_module.debug(
                "the normalized mock file differs from the original"
            )

        self.logging_module.debug(
            "Replacing original mock file with the normalized one."
        )
        mv_cmd = f"mv {cleaned_mock_filepath} {mock_file_path}"
        self.ami.call(mv_cmd.split())

    def start(self, playbook_or_integration_id, path=None, record=False) -> None:
        """Start the proxy process and direct traffic through it.

        Args:
            playbook_or_integration_id (string): ID of the test playbook to run.
            path (string): path override for the mock/log files.
            record (bool): Select proxy mode (record/playback)
        """
        if self.is_proxy_listening():
            self.logging_module.debug("proxy service is already running, stopping it")
            self.ami.call(["sudo", "systemctl", "stop", "mitmdump"])
        self.logging_module.debug(
            f"Attempting to start proxy in {self.get_script_mode(record)} mode"
        )
        self.prepare_proxy_start(path, playbook_or_integration_id, record)
        # Start proxy server
        self._start_proxy_and_wait_until_its_up(is_record=record)

    def _start_proxy_and_wait_until_its_up(self, is_record: bool) -> None:
        """
        Starts mitmdump service and wait for it to listen to port 9997 with timeout of 5 seconds
        Args:
            is_record (bool):  Indicates whether this is a record run or not
        """
        self._start_mitmdump_service()
        was_proxy_up = self.wait_until_proxy_is_listening()
        if was_proxy_up:
            self.logging_module.debug(
                f"Proxy service started in {self.get_script_mode(is_record)} mode"
            )
        else:
            self.logging_module.error(
                f"Proxy failed to start after {self.TIME_TO_WAIT_FOR_PROXY_SECONDS} seconds"
            )

    def _start_mitmdump_service(self) -> None:
        """
        Starts mitmdump service on the remote service
        """
        self.ami.call(["sudo", "systemctl", "start", "mitmdump"])

    def prepare_proxy_start(
        self, path: str, playbook_or_integration_id: str, record: bool
    ) -> bool:
        """
        Writes proxy server run configuration options to the remote host, the details of which include:
        - Creating new tmp directory on remote machine if in record mode and moves the problematic keys file in to it
        - Creating the mitmdump_rc file that includes the script mode, keys file path, mock file path and log file path
          for the mitmdump service and puts it in '/home/ec2-user/mitmdump_rc' in the remote machine.
        - starts the systemd mitmdump service

        Args:
            path: the path to the temp folder in which the record files should be created
            playbook_or_integration_id: The ID of the playbook or integration that is tested
            record: Indicates whether this is a record run or not
        """
        path = path or self.current_folder
        folder_path = get_folder_path(playbook_or_integration_id)

        repo_problem_keys_path = os.path.join(
            self.repo_folder, folder_path, "problematic_keys.json"
        )
        current_problem_keys_path = os.path.join(
            path, folder_path, "problematic_keys.json"
        )
        log_file_path = os.path.join(
            path, get_log_file_path(playbook_or_integration_id, record)
        )
        mock_file_path = os.path.join(
            path, get_mock_file_path(playbook_or_integration_id)
        )

        file_content = f'export KEYS_FILE_PATH="{current_problem_keys_path if record else repo_problem_keys_path}"\n'
        file_content += f"export SCRIPT_MODE={self.get_script_mode(record)}\n"
        file_content += f'export MOCK_FILE_PATH="{mock_file_path}"\n'
        file_content += f'export LOG_FILE_PATH="{log_file_path}"\n'

        # Create mock files directory
        silence_output(
            self.ami.call, ["mkdir", os.path.join(path, folder_path)], stderr="null"
        )
        # when recording, copy the `problematic_keys.json` for the test to current temporary directory if it exists
        # that way previously recorded or manually added keys will only be added upon and not wiped with an overwrite
        if record:
            try:
                silence_output(
                    self.ami.call,
                    ["mv", repo_problem_keys_path, current_problem_keys_path],
                    stdout="null",
                    stderr="null",
                )
            except CalledProcessError as e:
                self.logging_module.debug(
                    f"Failed to move problematic_keys.json with exit code {e.returncode}"
                )

        return self._write_mitmdump_rc_file_to_host(file_content)

    def _write_mitmdump_rc_file_to_host(self, file_content: str) -> bool:
        """
        Does all needed preparation for starting the proxy service which include:
        - Creating the mitmdump_rc file that includes the script mode, keys file path, mock file path and log file path
          for the mitmdump service and puts it in '/home/ec2-user/mitmdump_rc' in the remote machine.

        Args:
            file_content: The content of the mitmdump_rc file that includes the script mode, keys file path,
            mock file path and log file path

        Returns:
            True if file was successfully copied to the server, else False
        """
        try:
            self.ami.call(
                [
                    "echo",
                    f"'{file_content}'",
                    ">",
                    os.path.join(AMIConnection.REMOTE_HOME, "mitmdump_rc"),
                ]
            )
            return True
        except CalledProcessError:
            self.logging_module.exception(
                f"Could not copy arg file for mitmdump service to server {self.ami.internal_ip},"
            )
        return False

    def wait_until_proxy_is_listening(self):
        """
        Checks if the mitmdump service is listening, and raises an exception if 30 seconds pass without positive answer
        """
        for i in range(self.TIME_TO_WAIT_FOR_PROXY_SECONDS):
            proxy_is_listening = self.is_proxy_listening()
            if proxy_is_listening:
                return True
            time.sleep(1)
        return False

    def is_proxy_listening(self) -> bool:
        """
        Runs 'sudo lsof -iTCP:9997 -sTCP:LISTEN' on the remote machine and returns answer according to the results
        Returns:
            True if the ssh command return exit code 0 and False otherwise
        """
        try:
            self.ami.check_output(["sudo", "lsof", "-iTCP:9997", "-sTCP:LISTEN"])
            return True
        except CalledProcessError:
            return False

    def stop(self):
        self.logging_module.debug("Stopping mitmdump service")
        if not self.is_proxy_listening():
            self.logging_module.debug("proxy service was already down.")
        else:
            self.ami.call(["sudo", "systemctl", "stop", "mitmdump"])
        self.ami.call(["rm", "-rf", "/tmp/_MEI*"])  # Clean up temp files


@contextmanager
def run_with_mock(
    proxy_instance: MITMProxy, playbook_or_integration_id: str, record: bool = False
) -> Iterator[dict]:
    """
    Runs proxy in a context
    If it's a record mode:
        - Setting the current folder of the proxy to the tmp folder before mitmdump starts
        - Starts the mitmdump service in record mode
        - Handles the newly created mock files
        - Setting the current folder of the proxy to the repo folder after mitmdump stops
    If it's a playback mode:
        - Starts the mitmdump service in playback mode
        - In case the playback failed - will show the status of the mitmdump service which will include the last 10
        lines of the service's log.
    Args:
        proxy_instance: The instance of the proxy to use
        playbook_or_integration_id: The ID of the playbook or integration that is tested
        record: A boolean indicating this is record mode or not
    Yields: A result holder dict in which the calling method can add the result of the proxy run under the key 'result'
    """
    if record:
        proxy_instance.set_tmp_folder()
        # If the record files should be committed - clean the content-test-data repo first
        if proxy_instance.should_update_mock_repo:
            proxy_instance.logging_module.debug("Cleaning content-test-data repo")
            proxy_instance.reset_mock_files()
    proxy_instance.start(playbook_or_integration_id, record=record)
    result_holder: Dict[str, bool] = {}
    try:
        yield result_holder
    except Exception:
        proxy_instance.logging_module.exception(
            "Unexpected failure in proxy context manager"
        )
    finally:
        proxy_instance.stop()
        if record:
            if result_holder.get(RESULT):
                proxy_instance.normalize_mock_file(playbook_or_integration_id)
                proxy_instance.move_mock_file_to_repo(playbook_or_integration_id)
                proxy_instance.successful_rerecord_count += 1
                proxy_instance.rerecorded_tests.append(playbook_or_integration_id)
                if proxy_instance.should_update_mock_repo:
                    proxy_instance.logging_module.debug(
                        "committing new/updated mock files to mock git repo."
                    )
                    proxy_instance.commit_mock_file(playbook_or_integration_id)
            else:
                proxy_instance.failed_rerecord_count += 1
                proxy_instance.failed_rerecord_tests.append(playbook_or_integration_id)
            proxy_instance.set_repo_folder()

        else:
            if result_holder.get(RESULT):
                proxy_instance.successful_tests_count += 1
            else:
                proxy_instance.failed_tests_count += 1


# demisto_sdk/commands/test_content/timestamp_replacer.py

"""BEWARE:

This file is used outside of the demisto-sdk.
DO NOT import anything from custom packages, only use builtins.
"""

import functools
import json  # noqa: TID251
import logging
import urllib.parse
from ast import literal_eval
from collections import OrderedDict
from copy import deepcopy
from os import path
from time import ctime
from typing import List, Union

from dateparser import parse
from mitmproxy import ctx
from mitmproxy.addonmanager import Loader
from mitmproxy.addons.serverplayback import ServerPlayback
from mitmproxy.http import HTTPFlow, Request
from mitmproxy.script import concurrent

logging.basicConfig(
    level=logging.DEBUG, format="[%(asctime)s] - [%(funcName)s] - %(message)s"
)


def record_concurrently(replaying: bool = False):
    """
    A decorator to return a decorator that just executes the function it decorates normally if 'replaying' is true,
    (AKA mitmdump is executing in server-replay mode or reading in a mock file and cleaning it and saving the cleaned
    mock to a new file), otherwise pass the 'concurrent' decorator so that when mitmdump is executing in recording
    mode, that requests will be processed concurrently and not be blocking which can cause proxy errors during
    recording if multiple requests are made in a short timespan.

    Arguments:
        replaying (bool): True if timestamp replacer script is running in server playback mode or cleaning a mock file

    Returns:
        (function): decorator
    """
    logging.info(f"replaying={replaying}")
    if replaying:

        def nonconcurrent_decorator(func):
            @functools.wraps(func)
            def passthrough_wrapper(*args, **kwargs):
                value = func(*args, **kwargs)
                return value

            return passthrough_wrapper

        return nonconcurrent_decorator
    else:
        return concurrent


class TimestampReplacer:
    def __init__(self):
        self.count = 0
        self.constant = "constant_value"
        self.json_keys = set()
        self.form_keys = set()
        self.query_keys = set()
        self.bad_keys_filepath = ""
        self.detect_timestamps = False

    def load(self, loader: Loader):
        loader.add_option(
            name="detect_timestamps",
            typespec=bool,
            default=False,
            help="""
            Set to True only if recording a mock file. Used to determine which keys need to be replaced in incoming
            request bodies during a mock playback.
            """,
        )
        loader.add_option(
            name="keys_filepath",
            typespec=str,
            default="problematic_keys.json",  # disable-secrets-detection
            help="""
            The path to the file that contains the problematic keys for the test playbook recording that resides
            in the same directory.
            """,
        )
        loader.add_option(
            name="script_mode",
            typespec=str,
            default="playback",
            help="""
            The mode that timestamp_replacer.py is being executed in. The options are 'record', 'clean', and
            'playback'. If no option is provided, defaults to 'playback'.
            """,
        )
        loader.add_option(
            name="debug",
            typespec=bool,
            default=False,
            help="""
            Set to True to print out additional information for each request that comes in.
            """,
        )

    def running(self):
        if ctx.options.debug:
            logging.info(f"ctx.options={ctx.options}")

        self.bad_keys_filepath = ctx.options.keys_filepath
        if ctx.options.detect_timestamps:
            logging.info("Detecting Timestamp Fields")
            self.detect_timestamps = True
        self.load_problematic_keys()

    def _debug_request(self, flow: HTTPFlow) -> None:
        """Print details of the request"""
        req = flow.request
        logging.info(f"{req.method} {req.pretty_url}")
        _, _, _, _, query, _ = urllib.parse.urlparse(req.url)
        queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
        logging.info(f"queriesArray={queriesArray}")
        if req.multipart_form:
            logging.info(f"multipart_form data = {req.multipart_form.items()}")
        if req.urlencoded_form:
            logging.info(f"urlencoded_form data = {req.urlencoded_form.items()}")
        logging.info(f"hashed_data={ServerPlayback._hash(self, flow)}")  # type: ignore

    # @record_concurrently(
    #     replaying=bool(
    #         ctx.options.server_replay or (ctx.options.rfile and ctx.options.save_stream_file)
    #     )
    # )
    def request(self, flow: HTTPFlow) -> None:
        self.count += 1
        if ctx.options.debug:
            self._debug_request(flow)
        req = flow.request
        if req.multipart_form and "boundary" in req.headers.get("Content-Type", ""):
            self.replace_boundary(req)
        if ctx.options.script_mode == "record":
            if ctx.options.detect_timestamps:
                self.run_all_key_detections(req)
                logging.info(
                    f'updating problem_keys file at "{self.bad_keys_filepath}"'
                )
                self.update_problem_keys_file()
        elif ctx.options.script_mode in {"clean", "playback"}:
            logging.info(f"flow.live is: {flow.live}")
            flow.live = False
            logging.info(
                f"mode={ctx.options.script_mode} cleaning problematic key values from the request"
            )
            self.clean_bad_keys(req)

    @staticmethod
    def replace_boundary(req: Request):
        fixed_boundary = "fixed_boundary"
        content_type_header, old_boundary = req.headers["Content-Type"].split(
            "boundary="
        )
        req.headers["Content-Type"] = "boundary=".join(
            [content_type_header, fixed_boundary]
        )
        req.content = req.content.replace(old_boundary.encode(), fixed_boundary.encode())  # type: ignore

    def clean_bad_keys(self, req: Request) -> None:
        """Modify the request so that values of problematic keys are constant data

        Args:
            req (Request): The request to modify
        """
        self.clean_url_query(req)
        self.clean_urlencoded_form(req)
        self.clean_multipart_form(req)
        self.clean_json_body(req)

    def clean_url_query(self, req: Request) -> None:
        """Replace any problematic values of query parameters with constant data

        Args:
            req (Request): The request to modify
        """
        query_data = sorted(req._get_query())
        logging.info(f"fetched query_data: {query_data}")
        updated_query_data = []
        if query_data and self.query_keys:
            for key, val in query_data:
                if key in self.query_keys:
                    updated_query_data.append((key, self.constant))
                else:
                    updated_query_data.append((key, val))
        req._set_query(updated_query_data or query_data)
        logging.info(f"updated query_data: {req._get_query()}")

    def clean_urlencoded_form(self, req: Request) -> None:
        """Replace any problematic values of urlencoded form keys with constant data

        Args:
            req (Request): The request to modify
        """
        if req.urlencoded_form and self.form_keys:
            updated_urlencoded_form_data = []
            for key, val in req.urlencoded_form.items(multi=True):
                if key in self.form_keys:
                    updated_urlencoded_form_data.append((key, self.constant))
                else:
                    updated_urlencoded_form_data.append((key, val))
            req._set_urlencoded_form(updated_urlencoded_form_data)

    def clean_multipart_form(self, req: Request) -> None:
        """Replace any problematic values of multipart form keys with constant data

        Args:
            req (Request): The request to modify
        """
        if req.multipart_form and self.form_keys:
            updated_multipart_form_data = []
            for key, val in req.multipart_form.items(multi=True):
                if key in self.form_keys:
                    updated_multipart_form_data.append((key, self.constant))
                else:
                    updated_multipart_form_data.append((key, val))
            req._set_multipart_form(updated_multipart_form_data)

    def clean_json_body(self, req: Request) -> None:
        """Replace any problematic values of keys in the request's json body (if it has one)

        Args:
            req (Request): The request to modify
        """
        if req.method == "POST":
            raw_content = req.raw_content
            if raw_content is not None:
                try:
                    content = raw_content.decode()
                except UnicodeDecodeError:
                    logging.error("Failed to decode request content")
                    content = ""
            else:
                content = ""
            logging.info(f"cleaning json body: content={content}")
            json_data = content.startswith("{")
            if json_data:
                try:
                    content = OrderedDict(literal_eval(content))  # type: ignore
                    self.modify_json_body(req, content)  # type: ignore
                    return
                except Exception:
                    logging.exception(
                        f"failed to run literal_eval on content {content}"
                    )
                try:
                    logging.info(
                        'parsing the request body with "literal_eval" failed - trying with "json.loads"'
                    )
                    content = json.loads(content)
                    self.modify_json_body(req, content)  # type: ignore
                except Exception:
                    logging.exception(f"failed to run json.loads on content {content}")

    def modify_json_body(self, req: Request, json_body: dict) -> None:
        """Modify the json body of a request by replacing any timestamp data with constant data

        Args:
            req (Request): The request whose json body will be modified.
            json_body (dict): The request body to modify.
        """
        original_content = deepcopy(json_body)
        modified = False
        keys_to_replace = self.json_keys
        logging.info(f"{keys_to_replace}")
        for key_path in keys_to_replace:
            body = json_body
            keys = key_path.split(".")
            logging.info(f"keypath parts: {keys}")
            lastkey = keys[-1]
            logging.info(f"lastkey: {lastkey}")
            skip_key = False
            for k in keys[:-1]:
                if k in body:
                    body = body[k]
                elif isinstance(body, list) and k.isdigit():
                    if int(k) > len(body) - 1:
                        skip_key = True
                        break
                    body = body[int(k)]
                else:
                    skip_key = True
                    break
            if not skip_key:
                if lastkey in body:
                    logging.info(f'modifying request to "{req.pretty_url}"')
                    body[lastkey] = self.constant
                    modified = True
                elif (
                    isinstance(body, list)
                    and lastkey.isdigit()
                    and int(lastkey) <= len(body) - 1
                ):
                    logging.info(f'modifying request to "{req.pretty_url}"')
                    body[int(lastkey)] = self.constant
                    modified = True
        if modified:
            logging.info(
                f"original request body:\n{json.dumps(original_content, indent=4)}"
            )
            logging.info(f"modified request body:\n{json.dumps(json_body, indent=4)}")
            req.set_content(json.dumps(json_body).encode())

    def run_all_key_detections(self, req: Request) -> None:
        """Used to detect problematic keys in
        1. request query parameters
        2. urlencoded forms parameters and multipart forms parameters
        3. json request body

        Args:
            req (Request): The request to inspect for problematic keys
        """
        self.handle_url_query(req)
        self.handle_urlencoded_form(req)
        self.handle_multipart_form(req)
        self.handle_json_body(req)

    def handle_url_query(self, req: Request) -> None:
        query_data = req._get_query()
        logging.info(f"query_data: {query_data}")
        for key, val in query_data:
            # don't bother trying to interpret an argument less than 4 characters as some type of timestamp
            if len(val) > 4:
                if self.safely_parse(val):
                    self.query_keys.add(key)

    def handle_multipart_form(self, req: Request) -> None:
        """Used when detecting what keys in a multipart form to replace with constants.

        Args:
            req (Request): The request to inspect
        """
        if req.multipart_form:
            for key, val in req.multipart_form.items(multi=True):
                # don't bother trying to interpret an argument less than 4 characters as some type of timestamp
                if len(val) > 4:
                    if self.safely_parse(val):
                        self.form_keys.add(key)

    def handle_urlencoded_form(self, req: Request) -> None:
        """Used when detecting what keys in an url encoded parameters to replace with constants.

        Args:
            req (Request): The request to inspect
        """
        if req.urlencoded_form:
            for key, val in req.urlencoded_form.items(multi=True):
                # don't bother trying to interpret an argument less than 4 characters as some type of timestamp
                if len(val) > 4:
                    if self.safely_parse(val):
                        self.form_keys.add(key)

    def handle_json_body(self, req: Request) -> None:
        """Used when detecting what keys in a request's json body to replace with constants.

        Args:
            req (Request): The request to inspect
        """
        if req.method == "POST":
            raw_content = req.raw_content
            if raw_content is not None:
                try:
                    content = raw_content.decode()
                except UnicodeDecodeError:
                    logging.error("Failed to decode request content")
                    content = ""
            else:
                content = ""
            logging.info(f"handling json body: content={content}")
            json_data = content.startswith("{")
            if json_data:
                try:
                    content = OrderedDict(literal_eval(content))  # type: ignore
                    json_keys = self.determine_problematic_keys(content)  # type: ignore
                    self.json_keys.update(json_keys)
                    return
                except Exception:
                    logging.exception(f"failed to run literal_eval content: {content}")
                try:
                    logging.info(
                        'parsing the request body with "literal_eval" failed - trying with "json.loads"'
                    )
                    content = json.loads(content)
                    json_keys = self.determine_problematic_keys(content)  # type: ignore
                    self.json_keys.update(json_keys)
                except Exception:
                    logging.exception(f"failed to run json.loads on content {content}")

    def determine_problematic_keys(self, content: dict) -> List[str]:
        """Given a json request body, return the keys (in dot notation) whose values are potentially timestamp data.

        Args:
            content (dict): The json request body to iterate through and find problematic timestamp data.

        Returns:
            List[str]: A list of keys (in dot notation, e.g. 'query.filter.time' is an example of what could be one
                problematic key) whose values are potentially timestamp data.
        """

        def travel_dict(obj: Union[dict, list], key_path="") -> List[str]:
            bad_key_paths = []
            if isinstance(obj, dict):
                for key, val in obj.items():
                    sub_key_path = f"{key_path}.{key}" if key_path else key
                    if isinstance(val, (list, dict)):
                        bad_key_paths.extend(travel_dict(val, sub_key_path))
                    else:
                        is_string = isinstance(val, str) and len(val) > 4
                        possible_timestamp = (
                            isinstance(val, (int, float)) and len(str(val)) >= 8
                        )
                        if is_string or possible_timestamp:
                            for_eval = val
                            if possible_timestamp:
                                if isinstance(for_eval, float):
                                    digits = str(val).split(".")
                                    for_eval = digits[0]
                                if len(str(for_eval)) < 13:
                                    parsed_date = self.safely_parse(ctime(val))
                                else:
                                    parsed_date = self.safely_parse(ctime(val / 1000.0))
                            else:
                                parsed_date = self.safely_parse(val)
                            # if parsed_date is not None then successfully interpreted value as some sort of
                            # time related thingieding
                            if parsed_date:
                                bad_key_paths.append(sub_key_path)
            elif isinstance(obj, list):
                for i, val in enumerate(obj):
                    sub_key_path = f"{key_path}.{i}" if key_path else i
                    if isinstance(val, (list, dict)):
                        bad_key_paths.extend(travel_dict(val, sub_key_path))
                    else:
                        is_string = isinstance(val, str) and len(val) > 4
                        possible_timestamp = (
                            isinstance(val, (int, float)) and len(str(val)) >= 8
                        )
                        if is_string or possible_timestamp:
                            for_eval = val
                            if possible_timestamp:
                                if isinstance(for_eval, float):
                                    digits = str(val).split(".")
                                    for_eval = digits[0]
                                if len(str(for_eval)) < 13:
                                    parsed_date = self.safely_parse(ctime(val))
                                else:
                                    parsed_date = self.safely_parse(ctime(val / 1000.0))
                            else:
                                parsed_date = self.safely_parse(val)
                            # if parsed_date is not None then successfully interpreted value as some sort of
                            # time related thingieding
                            if parsed_date:
                                bad_key_paths.append(sub_key_path)
            return bad_key_paths

        bad_keys = travel_dict(content)
        return bad_keys

    def update_problem_keys_file(self):
        """Update the problem keys dictionary at the keys_filepath with new problematic keys"""
        existing_problem_keys = self.read_in_problematic_keys()
        for key, val in existing_problem_keys.items():
            if key == "keys_to_replace":
                existing_problem_keys[key] = " ".join(
                    set(val.split()).union(self.json_keys)
                )
            elif key == "server_replay_ignore_payload_params":
                existing_problem_keys[key] = " ".join(
                    set(val.split()).union(self.form_keys)
                )
            elif key == "server_replay_ignore_params":
                existing_problem_keys[key] = " ".join(
                    set(val.split()).union(self.query_keys)
                )
        self.write_out_problematic_keys(existing_problem_keys)

    def read_in_problematic_keys(self):
        """Load problematic keys dictionary from the keys_filepath argument filepath in content-test-data repo
        if it exists. Otherwise, return the dictionary with empty values.
        """
        logging.info('executing "read_in_problematic_keys" method')
        repo_bad_keys_filepath = self.bad_keys_filepath.replace(
            "/tmp/Mocks", "content-test-data"
        )
        logging.info(
            f'reading in problematic keys data from "{repo_bad_keys_filepath}"'
        )
        if not path.exists(self.bad_keys_filepath) and path.exists(
            repo_bad_keys_filepath
        ):
            with open(repo_bad_keys_filepath) as fp:
                problem_keys = json.load(fp)
        elif path.exists(self.bad_keys_filepath):
            with open(self.bad_keys_filepath) as fp:
                problem_keys = json.load(fp)
        else:
            problem_keys = {
                "keys_to_replace": "",
                "server_replay_ignore_params": "",
                "server_replay_ignore_payload_params": "",
            }
        return problem_keys

    def write_out_problematic_keys(self, problem_keys: dict):
        """Write updated problematic keys dictionary back to the file at the keys_filepath argument

        Args:
            problem_keys (dict): Updated dictionary of problematic keys
        """
        with open(self.bad_keys_filepath, "w") as bad_keys_file:
            bad_keys_file.write(json.dumps(problem_keys, indent=4))

    def load_problematic_keys(self):
        """Load problematic keys from the keys_filepath argument filepath if it exists. Only necessary when running
        mitmdump in playback mode. Resets command line options with the key value pairs from the loaded dictionary.
        """
        logging.info('executing "load_problematic_keys" method')
        if path.exists(self.bad_keys_filepath):
            logging.info(f'"{self.bad_keys_filepath}" path exists - loading bad keys')

            problem_keys = json.load(open(self.bad_keys_filepath))

            query_keys = problem_keys.get("server_replay_ignore_params")
            self.query_keys.update(
                query_keys.split() if isinstance(query_keys, str) else query_keys
            )
            form_keys = problem_keys.get("server_replay_ignore_payload_params")
            self.form_keys.update(
                form_keys.split() if isinstance(form_keys, str) else form_keys
            )
            json_keys = problem_keys.get("keys_to_replace")
            self.json_keys.update(
                json_keys.split() if isinstance(json_keys, str) else json_keys
            )

            logging.info("bad keys loaded\n---------------")
            logging.info(f"self.query_keys={self.query_keys}")
            logging.info(f"self.form_keys={self.form_keys}")
            logging.info(f"self.json_keys={self.json_keys}")
        else:
            logging.info(
                f'"{self.bad_keys_filepath}" path doesn\'t exist - no bad keys to set'
            )
            logging.info("not setting bad keys from file")

    @staticmethod
    def safely_parse(val):
        """
        Safely tries to parse a value as a datetime object.
        If it fails - logs the error and the malformed output.
        Args:
            val: The val to parse

        Returns:
            True if the val was parsed to datetime object and False otherwise
        """
        try:
            if parse(val):
                return True
        except Exception as exc:
            logging.exception(f"Failed to parse as date object: {val}", exc)
        return False


# mitmproxy picks up the contents of the addons global list and loads what it finds into the addons mechanism.
addons = [TimestampReplacer()]


# demisto_sdk/commands/test_content/tools.py

import ast
import logging
from copy import deepcopy
from pprint import pformat
from subprocess import STDOUT, CalledProcessError, check_output
from typing import Dict, Optional, Set

import demisto_client

from demisto_sdk.commands.test_content.constants import SSH_USER

logger = logging.getLogger("demisto-sdk")


def update_server_configuration(
    client: demisto_client,
    server_configuration: Optional[Dict],
    error_msg: str,
    logging_manager=logger,
    config_keys_to_delete: Optional[Set[str]] = None,
):
    """updates server configuration

    Args:
        client (demisto_client): The configured client to use.
        server_configuration (dict): The server configuration to be added
        error_msg (str): The error message
        config_keys_to_delete (set): The server configuration keys to be deleted

    Returns:
        response_data: The response data
        status_code: The response status code
        prev_system_conf: Previous stored system conf
    """
    system_conf_response = demisto_client.generic_request_func(
        self=client, path="/system/config", method="GET"
    )
    system_conf = ast.literal_eval(system_conf_response[0]).get("sysConf", {})
    logging_manager.debug(f"Current server configurations are {pformat(system_conf)}")

    prev_system_conf = deepcopy(system_conf)

    if config_keys_to_delete:
        for key in config_keys_to_delete:
            system_conf.pop(key, None)

    if server_configuration:
        system_conf.update(server_configuration)

    data = {"data": system_conf, "version": -1}
    response_data, status_code, _ = demisto_client.generic_request_func(
        self=client, path="/system/config", method="POST", body=data
    )

    logging_manager.debug(f"Updating server configurations with {pformat(system_conf)}")

    try:
        result_object = ast.literal_eval(response_data)
        logging_manager.debug(
            f"Updated server configurations with response: {pformat(result_object)}"
        )
    except ValueError as err:
        logging_manager.exception(
            f"failed to parse response from demisto. response is {response_data}.\nError:\n{err}"
        )
        return

    if status_code >= 300 or status_code < 200:
        message = result_object.get("message", "")
        logging_manager.error(f"{error_msg} {status_code}\n{message}")
    return response_data, status_code, prev_system_conf


def is_redhat_instance(instance_ip: str) -> bool:
    """
    As part of the AMI creation - in case the AMI is RHEL a file named '/home/ec2-user/rhel_ami' is created as
    an indication.
    If not
    Args:
        instance_ip: The instance IP to check.

    Returns:
        True if the file '/home/ec2-user/rhel_ami' exists on the instance, else False
    """
    try:
        check_output(
            f"ssh {SSH_USER}@{instance_ip} ls -l /home/ec2-user/rhel_ami".split(),
            stderr=STDOUT,
        )
        return True
    except CalledProcessError:
        return False


def get_ui_url(client_host):
    """

    Args:
        client_host: the client host

    Returns: the UI URL of the server. For XSIAM we remove the 'api-' prefix to get the UI URL of the server,
    whereas for XSOAR it will remain the same (the UI URL is the same).

    """
    return client_host.replace("https://api-", "https://")


# demisto_sdk/commands/update_release_notes/update_rn.py

"""
This script is used to create a release notes template
"""
import copy
import errno
import os
import re
from pathlib import Path
from typing import Any, Iterable, Optional, Tuple, Union

from packaging.version import Version

from demisto_sdk.commands.common.constants import (
    ALL_FILES_VALIDATION_IGNORE_WHITELIST,
    DEPRECATED_DESC_REGEX,
    DEPRECATED_NO_REPLACE_DESC_REGEX,
    IGNORED_PACK_NAMES,
    RN_CONTENT_ENTITY_WITH_STARS,
    RN_HEADER_BY_FILE_TYPE,
    SIEM_ONLY_ENTITIES,
    XSIAM_DASHBOARDS_DIR,
    XSIAM_REPORTS_DIR,
    FileType,
)
from demisto_sdk.commands.common.content import Content
from demisto_sdk.commands.common.content.objects.pack_objects import (
    Integration,
    Playbook,
    Script,
)
from demisto_sdk.commands.common.content.objects.pack_objects.abstract_pack_objects.yaml_content_object import (
    YAMLContentObject,
)
from demisto_sdk.commands.common.git_util import GitUtil
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import (
    find_type,
    get_api_module_dependencies_from_graph,
    get_api_module_ids,
    get_content_path,
    get_display_name,
    get_from_version,
    get_json,
    get_latest_release_notes_text,
    get_pack_name,
    get_remote_file,
    get_yaml,
    pack_name_to_path,
    run_command,
)
from demisto_sdk.commands.content_graph.commands.update import update_content_graph
from demisto_sdk.commands.content_graph.interface import (
    ContentGraphInterface,
)

CLASS_BY_FILE_TYPE = {
    FileType.INTEGRATION: Integration,
    FileType.SCRIPT: Script,
    FileType.PLAYBOOK: Playbook,
}


def get_deprecated_comment_from_desc(description: str) -> str:
    """
    find deprecated comment from description
    Args:
        description: The yml description

    Returns:
        If a deprecated description is found return it for rn.
    """
    deprecate_line_with_replacement = re.findall(DEPRECATED_DESC_REGEX, description)
    deprecate_line_no_replacement = re.findall(
        DEPRECATED_NO_REPLACE_DESC_REGEX, description
    )

    deprecate_line = deprecate_line_with_replacement + deprecate_line_no_replacement
    return deprecate_line[0] if deprecate_line else ""


def deprecated_commands(commands: list) -> set:
    """return a set of the deprecated commands only"""
    return {command.get("name") for command in commands if command.get("deprecated")}


def get_yml_objects(path: str, file_type) -> Tuple[Any, YAMLContentObject]:
    """
    Generate yml files from master and from the current branch
    Args:
        path: The requested file path
        file_type: the file type

    Returns:
        Two YML objects, the first of the yml at master (old yml) and the second from the current branch (new yml)
    """
    old_yml_obj = get_remote_file(path)
    new_yml_obj = CLASS_BY_FILE_TYPE[file_type](path)

    return old_yml_obj, new_yml_obj


def get_deprecated_rn(path: str, file_type):
    """Generate rn for deprecated items"""
    old_yml, new_yml = get_yml_objects(path, file_type)

    if not old_yml.get("deprecated") and new_yml.is_deprecated:
        description = (
            new_yml.get("comment")
            if file_type == file_type.SCRIPT
            else new_yml.get("description")
        )
        rn_from_description = get_deprecated_comment_from_desc(description)
        return f'- Deprecated. {rn_from_description or "Use %%% instead"}.\n'

    if file_type != FileType.INTEGRATION:
        return ""

    # look for deprecated commands
    rn = ""
    old_commands = deprecated_commands(old_yml.get("script", {}).get("commands") or [])
    new_commands = deprecated_commands(new_yml.script.get("commands") or [])

    for command_name in new_commands:
        # if command is deprecated in new yml, and not in old yml
        if command_name not in old_commands:
            rn += f"- Command ***{command_name}*** is deprecated. Use %%% instead.\n"
    return rn


class UpdateRN:
    CONTENT_PATH = Path(get_content_path())  # type: ignore[arg-type]

    def __init__(
        self,
        pack_path: str,
        update_type: Union[str, None],
        modified_files_in_pack: set,
        added_files: set,
        specific_version: str = None,
        pre_release: bool = False,
        pack: str = None,
        pack_metadata_only: bool = False,
        text: str = "",
        existing_rn_version_path: str = "",
        is_force: bool = False,
        is_bc: bool = False,
    ):
        self.pack = pack if pack else get_pack_name(pack_path)
        self.update_type = update_type
        self.pack_path = pack_path
        # renamed files will appear in the modified list as a tuple: (old path, new path)
        modified_files_in_pack = {
            file_[1] if isinstance(file_, tuple) else file_
            for file_ in modified_files_in_pack
        }
        self.modified_files_in_pack = set()
        for file_path in modified_files_in_pack:
            self.modified_files_in_pack.add(
                self.change_image_or_desc_file_path(
                    (self.CONTENT_PATH / file_path).as_posix()
                )
            )

        self.added_files = added_files
        self.pre_release = pre_release
        self.specific_version = specific_version
        self.existing_rn_changed = False
        self.text = text
        self.existing_rn_version_path = existing_rn_version_path
        self.should_delete_existing_rn = False
        self.pack_metadata_only = pack_metadata_only
        self.is_force = is_force
        git_util = GitUtil(repo=Content.git())
        self.main_branch = git_util.handle_prev_ver()[1]
        self.metadata_path = os.path.join(self.pack_path, "pack_metadata.json")
        self.master_version = self.get_master_version()
        self.rn_path = ""
        self.is_bc = is_bc
        self.bc_path = ""

    @staticmethod
    def change_image_or_desc_file_path(file_path: str) -> str:
        """Changes image and description file paths to the corresponding yml file path.
        if a non-image or description file path is given, it remains unchanged.

        :param file_path: The file path to check

        :rtype: ``str``
        :return
            The new file path if was changed
        """

        def validate_new_path(expected_path: str):
            if not Path(expected_path).exists():
                logger.info(
                    f"[yellow]file {file_path} implies the existence of {str(expected_path)}, which is missing. "
                    f"Did you mistype {file_path}?[/yellow]"
                )

        if file_path.endswith("_image.png"):
            if Path(file_path).parent.name in (XSIAM_DASHBOARDS_DIR, XSIAM_REPORTS_DIR):
                new_path = file_path.replace("_image.png", ".json")
            else:
                new_path = file_path.replace("_image.png", ".yml")
            validate_new_path(new_path)
            return new_path

        elif file_path.endswith("_description.md"):
            new_path = file_path.replace("_description.md", ".yml")
            validate_new_path(new_path)
            return new_path

        return file_path

    def handle_existing_rn_version_path(self, rn_path: str) -> str:
        """Checks whether the existing RN version path exists and return it's content.

        :param rn_path: The rn path to check

        :rtype: ``str``
        :return
            The content of the rn
        """
        if self.existing_rn_version_path:
            existing_rn_abs_path = self.CONTENT_PATH / self.existing_rn_version_path
            rn_path_abs_path = self.CONTENT_PATH / rn_path
            self.should_delete_existing_rn = str(existing_rn_abs_path) != str(
                rn_path_abs_path
            )
            try:
                return existing_rn_abs_path.read_text()
            except Exception as e:
                logger.info(
                    f"[red]Failed to load the previous release notes file content: {e}[/red]"
                )
        return ""

    def execute_update(self) -> bool:
        """Obtains the information needed in order to update the pack and executes the update.

        :rtype: ``bool``
        :return
            Whether the RN was updated successfully or not
        """
        if self.pack in IGNORED_PACK_NAMES:
            logger.info(
                f"[yellow]Release notes are not required for the {self.pack} pack since this pack"
                f" is not versioned.[/yellow]"
            )
            return False

        new_version, new_metadata = self.get_new_version_and_metadata()
        rn_path = self.get_release_notes_path(new_version)
        self.check_rn_dir(rn_path)
        self.rn_path = rn_path
        self.find_added_pack_files()
        changed_files = {}
        for packfile in self.modified_files_in_pack:
            file_name, file_type = self.get_changed_file_name_and_type(packfile)
            if (
                "yml" in packfile
                and file_type
                in [FileType.INTEGRATION, FileType.BETA_INTEGRATION, FileType.SCRIPT]
                and packfile not in self.added_files
            ):
                docker_image_name: Optional[str] = check_docker_image_changed(
                    main_branch=self.main_branch, packfile=packfile
                )
            else:
                docker_image_name = None
            changed_files[(file_name, file_type)] = {
                "description": get_file_description(packfile, file_type),
                "is_new_file": packfile in self.added_files,
                "fromversion": get_from_version_at_update_rn(packfile),
                "dockerimage": docker_image_name,
                "path": packfile,
            }
        return self.create_pack_rn(rn_path, changed_files, new_metadata, new_version)

    def create_pack_rn(
        self, rn_path: str, changed_files: dict, new_metadata: dict, new_version: str
    ) -> bool:
        """Checks whether the pack requires a new rn and if so, creates it.

        :param
            rn_path (str): The rn path
            changed_files (dict): The changed files details
            new_metadata (dict): The new pack metadata
            new_version (str): The new version str representation, e.g 1.0.2, 1.11.2 etc.


        :rtype: ``bool``
        :return
            Whether the RN was updated successfully or not
        """
        rn_string = self.handle_existing_rn_version_path(rn_path)
        if not rn_string:
            rn_string = self.build_rn_template(changed_files)
        if len(rn_string) > 0 or self.is_force:
            if self.is_bump_required():
                self.write_metadata_to_file(new_metadata)
            self.create_markdown(rn_path, rn_string, changed_files)
            self.build_rn_config_file(new_version)
            if self.existing_rn_changed:
                logger.info(
                    f"[green]Finished updating release notes for {self.pack}.[/green]"
                )
                if not self.text:
                    logger.info(
                        f"\n[green]Next Steps:\n - Please review the "
                        f"created release notes found at {rn_path} and document any changes you "
                        f"made by replacing '%%UPDATE_RN%%'.\n - Commit "
                        f"the new release notes to your branch.\nFor information regarding proper"
                        f" format of the release notes, please refer to "
                        f"https://xsoar.pan.dev/docs/integrations/changelog[/green]"
                    )
                return True
            else:
                logger.info(
                    f"[green]No changes to {self.pack} pack files were detected from the previous time "
                    "this command was run. The release notes have not been "
                    "changed.[/green]"
                )
        else:
            logger.info(
                "[yellow]No changes which would belong in release notes were detected.[/yellow]"
            )
        return False

    def build_rn_config_file(self, new_version: str) -> None:
        """
        Builds RN config file if needed. Currently, we use RN config file only for cases where version has breaking
        changes.
        Args:
            new_version (str): The new version number representation, e.g 1.2.1, 1.22.1, etc.

        Returns:
            (None): Creates/updates config file with BC entries, if -bc flag was given.
        """
        # Currently, we only use config file if version is BC. If version is not BC no need to create config file.
        if not self.is_bc:
            return
        bc_file_path: str = (
            f"""{self.pack_path}/ReleaseNotes/{new_version.replace('.', '_')}.json"""
        )
        self.bc_path = bc_file_path
        bc_file_data: dict = dict()
        if os.path.exists(bc_file_path):
            with open(bc_file_path) as f:
                bc_file_data = json.loads(f.read())
        bc_file_data["breakingChanges"] = True
        bc_file_data["breakingChangesNotes"] = bc_file_data.get("breakingChangesNotes")
        with open(bc_file_path, "w") as f:
            f.write(json.dumps(bc_file_data, indent=4))
        logger.info(
            f"[green]Finished creating config file for RN version {new_version}.\n"
            "If you wish only specific text to be shown as breaking changes, please fill the "
            "`breakingChangesNotes` field with the appropriate breaking changes text.[/green]"
        )

    def get_new_version_and_metadata(self) -> Tuple[str, dict]:
        """
        Gets the new version and the new metadata after version bump or by getting it from the pack metadata if
        bump is not required.

        :rtype: ``(str, dict)``
        :return: The new version and new metadata dictionary
        """
        if self.is_bump_required():
            if self.update_type is None:
                self.update_type = "revision"
            new_version, new_metadata = self.bump_version_number(
                self.specific_version, self.pre_release
            )
            if self.is_force:
                logger.info(
                    f"Bumping {self.pack} to version: {new_version}",
                )
            else:
                logger.info(
                    f"Changes were detected. Bumping {self.pack} to version: {new_version}",
                )
        else:
            new_metadata = self.get_pack_metadata()
            new_version = new_metadata.get("currentVersion", "99.99.99")

        if self.master_version == "0.0.0" and new_version == "1.0.0":
            raise ValueError(
                "Release notes do not need to be updated for version '1.0.0'."
            )

        return new_version, new_metadata

    def _does_pack_metadata_exist(self) -> bool:
        """Check if pack_metadata.json exists

        :rtype: ``bool``
        :return
            Whether the pack metadata exists
        """
        if not os.path.isfile(self.metadata_path):
            logger.info(
                f'[red]"{self.metadata_path}" file does not exist, create one in the root of the pack[/red]'
            )
            return False

        return True

    def get_master_version(self) -> str:
        """
        Gets the current version from origin/master or origin/main if available, otherwise return '0.0.0'.

        :rtype: ``str``
        :return
            The master version

        """
        master_current_version = "0.0.0"
        master_metadata = None
        try:
            master_metadata = get_remote_file(self.metadata_path, tag=self.main_branch)
        except Exception:
            logger.exception(
                f"[red]Failed fetching {self.metadata_path} from remote master branch."
                "Using the local version (if exists), instead[/red]",
            )
        if master_metadata:
            master_current_version = master_metadata.get("currentVersion", "0.0.0")
        return master_current_version

    def is_bump_required(self) -> bool:
        """
        Checks if the currentVersion in the pack metadata has been changed or not. Additionally, it will verify
        that there is no conflict with the currentVersion in then Master branch.

        :rtype: ``bool``
        :return
            Whether a version bump is required
        """
        try:
            if self.only_docs_changed() and not self.is_force:
                return False
            new_metadata = self.get_pack_metadata()
            new_version = new_metadata.get("currentVersion", "99.99.99")
            if Version(self.master_version) >= Version(new_version):
                return True
            return False
        except RuntimeError as e:
            raise RuntimeError(
                f"Unable to locate a pack with the name {self.pack} in the git diff.\n"
                f"Please verify the pack exists and the pack name is correct."
            ) from e

    def only_docs_changed(self) -> bool:
        """
        Checks if the only files that were changed are documentation files.

        :rtype: ``bool``
        :return
            Whether only the docs were changed
        """
        changed_files = self.added_files.union(self.modified_files_in_pack)
        changed_files_copy = copy.deepcopy(
            changed_files
        )  # copying as pop will leave the file out of the set
        if (len(changed_files) == 1 and "README" in changed_files_copy.pop()) or (
            all(
                "README" in file or (".png" in file and "_image.png" not in file)
                for file in changed_files
            )
        ):
            return True
        return False

    def find_added_pack_files(self):
        """
        Checks if the added files in the given pack require RN and if so, adds them to the modified files in the
        pack.
        """
        for a_file in self.added_files:
            if self.pack in a_file:
                if any(
                    item in a_file for item in ALL_FILES_VALIDATION_IGNORE_WHITELIST
                ):
                    continue
                else:
                    self.modified_files_in_pack.add(
                        self.change_image_or_desc_file_path(a_file)
                    )

    def get_release_notes_path(self, input_version: str) -> str:
        """Gets the release notes path.

        :param input_version: The new rn version

        :rtype: ``bool``
        :return
        Whether the RN was updated successfully or not
        """
        _new_version = input_version.replace(".", "_")
        new_version = _new_version.replace("_prerelease", "")
        return os.path.join(self.pack_path, "ReleaseNotes", f"{new_version}.md")

    @staticmethod
    def find_corresponding_yml(file_path) -> str:
        """Gets the pack's corresponding yml file from the python/yml file.

        :param file_path: The pack python/yml file

        :rtype: ``str``
        :return
        The path to the pack's yml file
        """
        if file_path.endswith(".py"):
            yml_filepath = file_path.replace(".py", ".yml")
        else:
            yml_filepath = file_path
        return yml_filepath

    def get_changed_file_name_and_type(
        self, file_path
    ) -> Tuple[str, Optional[FileType]]:
        """Gets the changed file name and type.

        :param file_path: The file path

        :rtype: ``str, FileType``
        :return
        The changed file name and type
        """
        _file_type = None
        file_name = "N/A"

        if self.pack + "/" in file_path and ("README" not in file_path):
            _file_path = self.find_corresponding_yml(file_path)
            file_name = get_display_name(_file_path)
            _file_type = find_type(_file_path)

        return file_name, _file_type

    def get_pack_metadata(self) -> dict:
        """Gets the pack metadata.

        :rtype: ``dict``
        :return
        The pack metadata dictionary
        """
        try:
            data_dictionary = get_json(self.metadata_path, cache_clear=True)
        except FileNotFoundError as e:
            raise FileNotFoundError(
                f"The metadata file of pack {self.pack} was not found. Please verify the pack name is correct, and that the file exists."
            ) from e
        return data_dictionary

    def bump_version_number(
        self, specific_version: str = None, pre_release: bool = False
    ) -> Tuple[str, dict]:
        """Increases the version number by user input or update type.

        :param
            specific_version: The specific version to change the version to
            pre_release: Indicates that the change should be designated a pre-release version

        :rtype: ``str, dict``
        :return
        The new version number (for example: 1.0.3) and the new pack metadata after version bump
        """
        if self.update_type is None and specific_version is None:
            raise ValueError("Received no update type when one was expected.")
        new_version = ""  # This will never happen since we pre-validate the argument
        data_dictionary = self.get_pack_metadata()
        current_version = (
            self.master_version
            if self.master_version != "0.0.0"
            else self.get_pack_metadata().get("currentVersion", "99.99.99")
        )
        if specific_version:
            logger.info(
                f"Bumping {self.pack} to the version {specific_version}. If you need to update"
                f" the release notes a second time, please remove the -v flag.",
            )
            data_dictionary["currentVersion"] = specific_version
            return specific_version, data_dictionary
        elif self.update_type == "major":
            version = current_version.split(".")
            version[0] = str(int(version[0]) + 1)
            if int(version[0]) > 99:
                raise ValueError(
                    f"Version number is greater than 99 for the {self.pack} pack. "
                    f"Please verify the currentVersion is correct."
                )
            version[1] = "0"
            version[2] = "0"
            new_version = ".".join(version)
        elif self.update_type == "minor":
            version = current_version.split(".")
            version[1] = str(int(version[1]) + 1)
            if int(version[1]) > 99:
                raise ValueError(
                    f"Version number is greater than 99 for the {self.pack} pack. "
                    f"Please verify the currentVersion is correct. If it is, "
                    f"then consider bumping to a new Major version."
                )
            version[2] = "0"
            new_version = ".".join(version)
        # We validate the input via click

        elif self.update_type in ["revision", "documentation"]:
            version = current_version.split(".")
            version[2] = str(int(version[2]) + 1)
            if int(version[2]) > 99:
                raise ValueError(
                    f"Version number is greater than 99 for the {self.pack} pack. "
                    f"Please verify the currentVersion is correct. If it is, "
                    f"then consider bumping to a new Minor version."
                )
            new_version = ".".join(version)
        elif self.update_type == "maintenance":
            raise ValueError(
                "The *maintenance* option is no longer supported."
                ' Please use the "revision" option and make sure to provide informative release notes.'
            )
        if pre_release:
            new_version = new_version + "_prerelease"
        data_dictionary["currentVersion"] = new_version
        return new_version, data_dictionary

    def write_metadata_to_file(self, metadata_dict: dict):
        """Writes the new metadata to the pack metadata file.

        :param
            metadata_dict: The new metadata to write

        """
        if self._does_pack_metadata_exist():
            with open(self.metadata_path, "w") as file_path:
                json.dump(metadata_dict, file_path, indent=4)
                logger.info(
                    f"[green]Updated pack metadata version at path : {self.metadata_path}[/green]"
                )
            try:
                run_command(f"git add {self.metadata_path}", exit_on_error=False)
            except RuntimeError:
                logger.error(f"[red]Failed git-adding {self.metadata_path}[/red]")

    @staticmethod
    def check_rn_dir(rn_path: str):
        """Checks whether the release notes folder exists and if not creates it.

        :param rn_path: The RN path to check/create

        """
        if not os.path.exists(os.path.dirname(rn_path)):
            try:
                os.makedirs(os.path.dirname(rn_path))
            except OSError as exc:  # Guard against race condition
                if exc.errno != errno.EEXIST:
                    raise

    def build_rn_template(self, changed_items: dict) -> str:
        """Builds the new release notes template.

        :param
            changed_items: The changed items data dictionary

        :rtype: ``str``
        :return
        The new release notes template
        """
        rn_string = ""

        if self.pack_metadata_only:
            rn_string += f"\n#### Integrations\n\n##### {self.pack}\n\n- Documentation and metadata improvements.\n"
            return rn_string
        rn_template_as_dict: dict = {}
        if self.is_force:
            rn_string = self.build_rn_desc(content_name=self.pack, text=self.text)
        # changed_items.items() looks like that: [((name, type), {...}), (name, type), {...}] and we want to sort
        # them by type (x[0][1])

        for (content_name, _type), data in sorted(
            changed_items.items(),
            key=lambda x: RN_HEADER_BY_FILE_TYPE[x[0][1]] if x[0] and x[0][1] else "",
        ):  # Sort RN by header
            desc = data.get("description", "")
            is_new_file = data.get("is_new_file", False)
            from_version = data.get("fromversion", "")
            docker_image = data.get("dockerimage")
            path = data.get("path")
            # Skipping the invalid files
            if not _type or content_name == "N/A":
                continue
            rn_desc = self.build_rn_desc(
                _type=_type,
                content_name=content_name,
                desc=desc,
                is_new_file=is_new_file,
                text=self.text,
                docker_image=docker_image,
                from_version=from_version,
                path=path,
            )

            header = f"\n#### {RN_HEADER_BY_FILE_TYPE[_type]}\n\n"
            rn_template_as_dict[header] = rn_template_as_dict.get(header, "") + rn_desc

        for key, val in rn_template_as_dict.items():
            rn_string = f"{rn_string}{key}{val}"

        return rn_string

    def build_rn_desc(
        self,
        _type: FileType = None,
        content_name: str = "",
        desc: str = "",
        is_new_file: bool = False,
        text: str = "",
        docker_image: Optional[str] = "",
        from_version: str = "",
        path: str = "",
    ) -> str:
        """Builds the release notes description.

        :param
            _type: The file type
            content_name: The pack name
            desc: The pack description
            is_new_file: True if the file is new
            text: Text to add to the release notes files
            from_version: From version

        :rtype: ``str``
        :return
        The release notes description
        """
        if _type in RN_CONTENT_ENTITY_WITH_STARS:
            if is_new_file:
                rn_desc = f"- New: **{content_name}**\n"
            else:
                rn_desc = f"- **{content_name}**\n"
        else:
            if is_new_file:
                rn_desc = f"##### New: {content_name}\n\n"
                if desc:
                    rn_desc += f"- New: {desc}"
                if from_version and _type not in SIEM_ONLY_ENTITIES:
                    rn_desc += f" (Available from Cortex XSOAR {from_version})."
                elif _type in SIEM_ONLY_ENTITIES:
                    rn_desc += "(Available from Cortex XSIAM %%XSIAM_VERSION%%)."
                rn_desc += "\n"
            else:
                rn_desc = f"##### {content_name}\n\n"
                if self.update_type == "documentation":
                    rn_desc += "- Documentation and metadata improvements.\n"
                else:
                    deprecate_rn = ""
                    if _type in (
                        FileType.INTEGRATION,
                        FileType.SCRIPT,
                        FileType.PLAYBOOK,
                    ):
                        deprecate_rn = get_deprecated_rn(path, _type)
                    if deprecate_rn:
                        if text:
                            rn_desc += f"- {text}\n"
                        rn_desc += deprecate_rn
                    else:
                        rn_desc += f'- {text or "%%UPDATE_RN%%"}\n'

        if docker_image:
            rn_desc += f"- Updated the Docker image to: *{docker_image}*.\n"
        return rn_desc

    def does_content_item_header_exist_in_rns(
        self, current_rn: str, content_name: str
    ) -> bool:
        """
        Checks whether the content item header exists in the release notes.

        Args:
            current_rn: The current release notes.
            content_name: The content item name.
        Returns:
            True if the content item header exists in the release notes, False otherwise.
        """
        for line in current_rn.replace("#####", "").replace("**", "").split("\n"):
            if content_name == line.replace("-", "", 1).strip():
                return True
        return False

    def update_existing_rn(self, current_rn, changed_files) -> str:
        """Update the existing release notes.

        :param
            current_rn: The existing rn
            changed_files: The new data to add

        :rtype: ``str``
        :return
        The updated release notes
        """
        update_docker_image_regex = r"- Updated the Docker image to: \*.*\*\."
        # Deleting old entry for docker images, will re-write later, this allows easier generating of updated rn.
        current_rn_without_docker_images = re.sub(
            update_docker_image_regex, "", current_rn
        )
        new_rn = current_rn_without_docker_images
        # changed_files.items() looks like that: [((name, type), {...}), (name, type), {...}] and we want to sort
        # them by name (x[0][0])
        for (content_name, _type), data in sorted(
            changed_files.items(),
            key=lambda x: x[0][0] if x[0][0] else "",
            reverse=True,
        ):
            is_new_file = data.get("is_new_file")
            desc = data.get("description", "")
            docker_image = data.get("dockerimage")
            rn_desc = ""
            path = data.get("path")
            if _type is None:
                continue

            _header_by_type = RN_HEADER_BY_FILE_TYPE.get(_type)
            rn_desc += "\n\n" + self.build_rn_desc(
                _type=_type,
                content_name=content_name,
                desc=desc,
                is_new_file=is_new_file,
                docker_image=docker_image,
                path=path,
            )
            if _header_by_type and _header_by_type in current_rn_without_docker_images:
                if self.does_content_item_header_exist_in_rns(
                    current_rn_without_docker_images, content_name
                ):
                    if docker_image:
                        new_rn = self.handle_existing_rn_with_docker_image(
                            new_rn, _header_by_type, docker_image, content_name
                        )
                else:
                    self.existing_rn_changed = True
                    rn_parts = new_rn.split(_header_by_type)
                    new_rn_part = rn_desc
                    if len(rn_parts) > 1:
                        new_rn = f"{rn_parts[0]}{_header_by_type}{new_rn_part[:-1]}{rn_parts[1]}"
                    else:
                        new_rn = "".join(rn_parts) + new_rn_part
            else:
                self.existing_rn_changed = True
                if _header_by_type and _header_by_type in new_rn:
                    rn_parts = new_rn.split(_header_by_type)
                    new_rn_part = rn_desc
                    if len(rn_parts) > 1:
                        new_rn = f"{rn_parts[0]}{_header_by_type}{new_rn_part[:-1]}{rn_parts[1]}"
                else:
                    new_rn_part = f"\n#### {_header_by_type}{rn_desc}\n"
                    new_rn += new_rn_part
        if new_rn != current_rn:
            self.existing_rn_changed = True
        return new_rn

    @staticmethod
    def handle_existing_rn_with_docker_image(
        new_rn: str, header_by_type: str, docker_image: str, content_name: str
    ) -> str:
        """
        Receives the new RN to be written, performs operations to add the docker image to the given RN.
        Args:
            new_rn (str): new RN.
            header_by_type (str): Header of the RN to add docker image to, e.g 'Integrations', 'Scripts'
            docker_image (str): Docker image to add
            content_name (str): The content name to add the docker image entry to, e.g integration name, script name.

        Returns:
            (str): Updated RN
        """
        # Writing or re-writing docker image to release notes.
        rn_parts = new_rn.split(header_by_type)
        new_rn_part = f"- Updated the Docker image to: *{docker_image}*."
        if len(rn_parts) > 1:
            # Splitting again by content name to append the docker image release note to corresponding
            # content entry only
            content_parts = rn_parts[1].split(f"{content_name}\n")
            new_rn = (
                f"{rn_parts[0]}{header_by_type}{content_parts[0]}{content_name}\n{new_rn_part}\n"
                f"{content_parts[1]}"
            )
        else:
            logger.info(
                f"[yellow]Could not parse release notes {new_rn} by header type: {header_by_type}[/yellow]"
            )
        return new_rn

    def create_markdown(
        self, release_notes_path: str, rn_string: str, changed_files: dict
    ):
        """Creates the new markdown and writes it to the release notes file.

        :param
            release_notes_path: The release notes file path
            rn_string: The rn data (if exists)
            changed_files: The changed files details
            docker_image_name: The docker image name

        """
        if os.path.exists(release_notes_path) and self.update_type is not None:
            logger.info(
                f"[yellow]Release notes were found at {release_notes_path}. Skipping[/yellow]"
            )
        elif self.update_type is None and self.specific_version is None:
            current_rn = get_latest_release_notes_text(release_notes_path)
            updated_rn = self.update_existing_rn(current_rn, changed_files)
            with open(release_notes_path, "w") as fp:
                fp.write(updated_rn)
        else:
            self.existing_rn_changed = True
            with open(release_notes_path, "w") as fp:
                fp.write(rn_string)

    def rn_with_docker_image(self, rn_string: str, docker_image: Optional[str]) -> str:
        """
        Receives existing release notes, if docker image was updated, adds docker_image to release notes.
        Taking care of cases s.t:
        1) no docker image update have occurred ('docker_image' is None).
        2) Release notes did not contain updated docker image note.
        3) Release notes contained updated docker image notes, with the newest updated docker image.
        4) Release notes contained updated docker image notes, but docker image was updated again since last time
           release notes have been updated.

        param:
            rn_string (str): The current text contained in the release note
            docker_image (Optional[str]): The docker image str, if given

        :rtype: ``str``
        :return
            The release notes, with the most updated docker image release note, if given
        """
        if not docker_image:
            return rn_string
        docker_image_str = f"- Updated the Docker image to: *{docker_image}*."
        if docker_image_str in rn_string:
            return rn_string
        self.existing_rn_changed = True
        if "- Updated the Docker image to" not in rn_string:
            return rn_string + f"{docker_image_str}\n"
        update_docker_image_regex = r"- Updated the Docker image to: \*.*\*\."
        updated_rn = re.sub(update_docker_image_regex, docker_image_str, rn_string)
        self.existing_rn_changed = True
        return updated_rn


def get_file_description(path, file_type) -> str:
    """Gets the file description.

    :param
        path: The file path
        file_type: The file type

    :rtype: ``str``
    :return
    The file description if exists otherwise returns %%UPDATE_RN%%
    """
    if not os.path.isfile(path):
        logger.info(
            f'[yellow]Cannot get file description: "{path}" file does not exist[/yellow]'
        )
        return ""

    elif file_type in (
        FileType.PLAYBOOK,
        FileType.INTEGRATION,
        FileType.CORRELATION_RULE,
        FileType.MODELING_RULE,
        FileType.PARSING_RULE,
    ):
        yml_file = get_yaml(path)
        return yml_file.get("description", "")

    elif file_type == FileType.SCRIPT:
        yml_file = get_yaml(path)
        return yml_file.get("comment", "")

    elif file_type in (
        FileType.CLASSIFIER,
        FileType.REPORT,
        FileType.WIDGET,
        FileType.DASHBOARD,
        FileType.JOB,
        FileType.TRIGGER,
        FileType.WIZARD,
    ):
        json_file = get_json(path)
        return json_file.get("description", "")

    return "%%UPDATE_RN%%"


def update_api_modules_dependents_rn(
    pre_release: bool,
    update_type: Union[str, None],
    added: Iterable[str],
    modified: Iterable[str],
    text: str = "",
) -> set:
    """Updates release notes for any pack that depends on API module that has changed.
    :param
        pre_release: Indicates whether the change should be designated as a pre-release version
        update_type: The update type
        added: The added files
        modified: The modified files
        id_set_path: The id set path
        text: Text to add to the release notes files

    :rtype: ``set``
    :return
    A set of updated packs
    """
    total_updated_packs: set = set()
    api_module_set = get_api_module_ids(added)
    api_module_set = api_module_set.union(get_api_module_ids(modified))
    logger.info(
        f"[yellow]Changes were found in the following APIModules : {api_module_set}, updating all dependent "
        f"integrations.[/yellow]"
    )
    with ContentGraphInterface() as graph:
        update_content_graph(graph, use_git=True, dependencies=True)
        integrations = get_api_module_dependencies_from_graph(api_module_set, graph)
        if integrations:
            logger.info("Executing update-release-notes on those as well.")
        for integration in integrations:
            integration_pack_name = integration.pack_id
            integration_path = integration.path
            integration_pack_path = pack_name_to_path(integration_pack_name)
            update_pack_rn = UpdateRN(
                pack_path=integration_pack_path,
                update_type=update_type,
                modified_files_in_pack={integration_path},
                pre_release=pre_release,
                added_files=set(),
                pack=integration_pack_name,
                text=text,
            )
            updated = update_pack_rn.execute_update()
            if updated:
                total_updated_packs.add(integration_pack_name)
        return total_updated_packs


def check_docker_image_changed(main_branch: str, packfile: str) -> Optional[str]:
    """Checks whether the docker image was changed in master.

    :param
        main_branch: The git main branch
        packfile: The added or modified yml path

    :rtype: ``Optional[str]``
    :return
    The latest docker image
    """
    try:
        diff = run_command(f"git diff {main_branch} -- {packfile}", exit_on_error=False)
    except RuntimeError as e:
        if any(["is outside repository" in exp for exp in e.args]):
            return None
        else:
            logger.info(
                f"[yellow]skipping docker image check, Encountered the following error:\n{e.args[0]}[/yellow]"
            )
            return None
    else:
        diff_lines = diff.splitlines()
        for diff_line in diff_lines:
            if (
                "dockerimage:" in diff_line
            ):  # search whether exists a line that notes that the Docker image was
                # changed.
                split_line = diff_line.split()
                if split_line[0].startswith("+"):
                    return split_line[-1]
        return None


def get_from_version_at_update_rn(path: str) -> Optional[str]:
    """
    param:
        path (str): path to yml file, if exists

    :rtype: ``Optional[str]``
    :return:
        Fromversion if there is a fromversion key in the yml file

    """
    if not os.path.isfile(path):
        logger.info(
            f'[yellow]Cannot get file fromversion: "{path}" file does not exist[/yellow]'
        )
        return None
    return get_from_version(path)


# demisto_sdk/commands/update_release_notes/update_rn_manager.py

import os
from pathlib import Path
from typing import Optional, Tuple

import git

from demisto_sdk.commands.common.constants import (
    API_MODULES_PACK,
    SKIP_RELEASE_NOTES_FOR_TYPES,
)
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import (
    filter_files_by_type,
    filter_files_on_pack,
    get_pack_name,
    get_pack_names_from_files,
    pack_name_to_path,
    suppress_stdout,
)
from demisto_sdk.commands.update_release_notes.update_rn import (
    UpdateRN,
    update_api_modules_dependents_rn,
)
from demisto_sdk.commands.validate.validate_manager import ValidateManager


class UpdateReleaseNotesManager:
    def __init__(
        self,
        user_input: Optional[str] = None,
        update_type: Optional[str] = None,
        pre_release: bool = False,
        is_all: Optional[bool] = False,
        text: Optional[str] = None,
        specific_version: Optional[str] = None,
        id_set_path: Optional[Path] = None,
        prev_ver: Optional[str] = None,
        is_force: bool = False,
        is_bc: bool = False,
    ):
        self.given_pack = user_input
        self.changed_packs_from_git: set = set()
        self.update_type = update_type
        self.pre_release: bool = False if pre_release is None else pre_release
        # update release notes to every required pack if not specified.
        self.is_all = True if not self.given_pack else is_all
        self.text: str = "" if text is None else text
        self.is_force = is_force
        self.specific_version = specific_version
        self.id_set_path = id_set_path
        self.prev_ver = prev_ver
        self.packs_existing_rn: dict = {}
        self.total_updated_packs: set = set()
        # When a user choose a specific pack to update rn, the -g flag should not be passed
        if self.given_pack and self.is_all:
            raise ValueError("Please remove the -g flag when specifying only one pack.")
        self.rn_path: list = list()
        self.is_bc = is_bc

    def manage_rn_update(self):
        """
        Manages the entire update release notes process.
        """
        logger.info("Starting to update release notes.")
        # The given_pack can be both path or pack name thus, we extract the pack name from the path if needed.
        if self.given_pack and "/" in self.given_pack:
            self.given_pack = get_pack_name(self.given_pack)  # extract pack from path

        # Find which files were changed from git
        modified_files, added_files, old_format_files = self.get_git_changed_files()
        self.changed_packs_from_git = (
            get_pack_names_from_files(modified_files)
            .union(get_pack_names_from_files(added_files))
            .union(get_pack_names_from_files(old_format_files))
        )
        # Check whether the packs have some existing RNs already (created manually or by the command)
        self.check_existing_rn(added_files)

        self.handle_api_module_change(modified_files, added_files)
        self.create_release_notes(modified_files, added_files, old_format_files)
        if len(self.total_updated_packs) > 1:
            logger.info(
                "\n[green]Successfully updated the following packs:\n"
                + "\n".join(self.total_updated_packs)
                + "[/green]"
            )

    def filter_to_relevant_files(
        self, file_set: set, validate_manager: ValidateManager
    ) -> Tuple[set, set, bool]:
        """
        Given a file set, filter it to only files which require RN and if given, from a specific pack
        """
        filtered_set = set()
        for file in file_set:
            if isinstance(file, tuple):
                file_path = str(file[1])

            else:
                file_path = str(file)

            if self.given_pack:
                file_pack_name = get_pack_name(file_path)
                if not file_pack_name or file_pack_name not in self.given_pack:
                    continue

            filtered_set.add(file)

        return validate_manager.filter_to_relevant_files(filtered_set)

    def filter_files_from_git(
        self,
        modified_files: set,
        added_files: set,
        renamed_files: set,
        validate_manager: ValidateManager,
    ):
        """
        Filter the raw file sets to only the relevant files for RN
        """
        filtered_modified, old_format_files, _ = self.filter_to_relevant_files(
            modified_files, validate_manager
        )
        filtered_renamed, _, _ = self.filter_to_relevant_files(
            renamed_files, validate_manager
        )
        filtered_modified = filtered_modified.union(filtered_renamed)
        filtered_added, new_files_in_old_format, _ = self.filter_to_relevant_files(
            added_files, validate_manager
        )
        old_format_files = old_format_files.union(new_files_in_old_format)
        return filtered_modified, filtered_added, old_format_files

    def setup_validate_manager(self):
        return ValidateManager(
            skip_pack_rn_validation=True,
            prev_ver=self.prev_ver,
            silence_init_prints=True,
            skip_conf_json=True,
            check_is_unskipped=False,
            file_path=self.given_pack,
        )

    def get_git_changed_files(self) -> Tuple[set, set, set]:
        """Get the changed files from git (added, modified, old format, metadata).

        :return:
            - The filtered modified files (including the renamed files)
            - The filtered added files
            - The modified old-format files (legacy unified python files)
        """
        try:
            validate_manager = self.setup_validate_manager()
            if not validate_manager.git_util:  # in case git utils can't be initialized.
                raise git.InvalidGitRepositoryError("unable to connect to git.")
            validate_manager.setup_git_params()
            if self.given_pack:
                with suppress_stdout():
                    # The Validator prints errors which are related to all changed files that
                    # were changed against prev version. When the user is giving a specific pack to update,
                    # we want to suppress the error messages which are related to other packs.
                    (
                        modified_files,
                        added_files,
                        renamed_files,
                    ) = validate_manager.get_unfiltered_changed_files_from_git()
                    return self.filter_files_from_git(
                        modified_files, added_files, renamed_files, validate_manager
                    )

            (
                modified_files,
                added_files,
                renamed_files,
            ) = validate_manager.get_unfiltered_changed_files_from_git()
            return self.filter_files_from_git(
                modified_files, added_files, renamed_files, validate_manager
            )

        except (
            git.InvalidGitRepositoryError,
            git.NoSuchPathError,
            FileNotFoundError,
        ) as e:
            raise FileNotFoundError(
                "You are not running `demisto-sdk update-release-notes` command in the content repository.\n"
                "Please run `cd content` from your terminal and run the command again"
            ) from e

    def check_existing_rn(self, added_files: set):
        """
        Checks whether the packs already have an existing release notes files and adds
        them to the packs_existing_rn dictionary.

        :param added_files: A set of new added files
        """
        for file_path in added_files:
            if "ReleaseNotes" in file_path:
                self.packs_existing_rn[get_pack_name(file_path)] = file_path

    def handle_api_module_change(self, modified_files: set, added_files: set):
        """Checks whether the modified file is in the API modules pack and if so, updates every pack which depends
        on that API module.

        :param
            added_files: A set of new added files
            modified_files: A set of modified files
        """
        # We want to handle ApiModules changes when:
        # (1) The user gave a path to the api module which was changed.
        # (2) The user did not give a specific path at all (is_all = True) but some ApiModules were changed.
        api_module_was_given = self.given_pack and API_MODULES_PACK in self.given_pack
        api_module_changed_in_git = (
            self.changed_packs_from_git
            and API_MODULES_PACK in self.changed_packs_from_git
        )

        if api_module_was_given or (api_module_changed_in_git and self.is_all):
            updated_packs = update_api_modules_dependents_rn(
                self.pre_release,
                self.update_type,
                added_files,
                modified_files,
                self.text,
            )
            self.total_updated_packs = self.total_updated_packs.union(updated_packs)

    def create_release_notes(
        self, modified_files: set, added_files: set, old_format_files: set
    ):
        """Iterates over the packs which needs an update and creates a release notes for them.

        :param
            modified_files: A set of modified files
            added_files: A set of added files
            old_format_files: A set of old formatted files
        """
        # Certain file types do not require release notes update
        filtered_modified_files = filter_files_by_type(
            modified_files, skip_file_types=SKIP_RELEASE_NOTES_FOR_TYPES
        )
        filtered_added_files = filter_files_by_type(
            added_files, skip_file_types=SKIP_RELEASE_NOTES_FOR_TYPES
        )
        if self.given_pack:  # A specific pack was chosen to update
            self.create_pack_release_notes(
                self.given_pack,
                filtered_modified_files,
                filtered_added_files,
                old_format_files,
            )

        elif self.changed_packs_from_git:  # update all changed packs
            for pack in self.changed_packs_from_git:
                if (
                    API_MODULES_PACK in pack
                ):  # We already handled Api Modules so we can skip it.
                    continue
                self.create_pack_release_notes(
                    pack,
                    filtered_modified_files,
                    filtered_added_files,
                    old_format_files,
                )
        else:
            logger.info(
                "[yellow]No changes that require release notes were detected. If such changes were made, "
                "please commit the changes and rerun the command.[/yellow]"
            )

    def create_pack_release_notes(
        self,
        pack: str,
        filtered_modified_files: set,
        filtered_added_files: set,
        old_format_files: set,
    ):
        """Creates the release notes for a given pack if was changed.

        :param
            pack: The pack to create release notes for
            filtered_modified_files: A set of filtered modified files
            filtered_added_files: A set of filtered added files
            old_format_files: A set of old formatted files
        """
        existing_rn_version = self.get_existing_rn(pack)
        if (
            existing_rn_version is None
        ):  # New release notes file already found for the pack
            raise RuntimeError(
                f"New release notes file already found for {pack}. "
                f"Please update manually or run `demisto-sdk update-release-notes "
                f"-i {pack}` without specifying the update_type."
            )
        pack_modified = filter_files_on_pack(pack, filtered_modified_files)
        pack_added = filter_files_on_pack(pack, filtered_added_files)
        pack_old = filter_files_on_pack(pack, old_format_files)

        # Checks if update is required
        if pack_modified or pack_added or pack_old or self.is_force:
            pack_path = pack_name_to_path(pack)
            update_pack_rn = UpdateRN(
                pack_path=pack_path,
                update_type=self.update_type,
                modified_files_in_pack=pack_modified.union(pack_old),
                pre_release=self.pre_release,
                added_files=pack_added,
                specific_version=self.specific_version,
                text=self.text,
                is_force=self.is_force,
                existing_rn_version_path=existing_rn_version,
                is_bc=self.is_bc,
            )
            updated = update_pack_rn.execute_update()
            self.rn_path.append(update_pack_rn.rn_path)

            # If new release notes were created add it to the total number of packs that were updated.
            if updated:
                self.total_updated_packs.add(pack)
                # If there is an outdated previous release notes, remove it (for example: User updated his version to
                # 1.0.4 and meanwhile the master version changed to 1.0.4, so we want to remove the user's 1_0_4 file
                # and add a 1_0_5 file.)
                if update_pack_rn.should_delete_existing_rn:
                    os.unlink(self.packs_existing_rn[pack])
        else:
            logger.info(
                f"[yellow]Either no changes were found in {pack} pack "
                f"or the changes found should not be documented in the release notes file.\n"
                f"If relevant changes were made, please commit the changes and rerun the command.[/yellow]"
            )

    def get_existing_rn(self, pack) -> Optional[str]:
        """Gets the existing rn of the pack is exists.

        :param
            pack: The pack to check
        :return
            The existing rn version if exists, otherwise an empty string
            None on error when pack has rn already and update type was given
        """
        if pack not in self.packs_existing_rn:
            return ""

        if self.update_type is None:
            return self.packs_existing_rn[pack]
        else:
            return None


# demisto_sdk/commands/update_xsoar_config_file/update_xsoar_config_file.py

import os
from typing import Dict, List

import demisto_client

from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.logger import logger

XSOAR_CONFIG_FILE_JSON = "xsoar_config.json"
MARKETPLACE_PACKS_SECTION = "marketplace_packs"
CUSTOM_PACKS_SECTION = "custom_packs"


class XSOARConfigFileUpdater:
    """
    XSOARConfigFileUpdater is a class that's designed to update and edit the XSOAR Config File.

    Attributes:
        pack_id (str): The Pack ID to add to XSOAR Configuration File
        pack_data (str): The Pack Data to add to XSOAR Configuration File -
        Pack URL for Custom Pack and Pack Version for OOTB Pack
        add_marketplace_pack: (bool) Add the Pack to the MarketPlace Packs section in the Configuration File
        add_custom_pack (bool): Add the Pack to the Custom Packs section in the Configuration File
        add_all_marketplace_packs (bool): Add all the installed MarketPlace Packs to the XSOAR Configuration File
        file_path (str): XSOAR Configuration File path, the default value is in the repo level
    """

    def __init__(
        self,
        pack_id: str,
        pack_data: str,
        add_marketplace_pack: bool = False,
        add_custom_pack: bool = False,
        add_all_marketplace_packs: bool = False,
        insecure: bool = False,
        file_path: str = XSOAR_CONFIG_FILE_JSON,
        **kwargs,
    ):
        self.pack_id = pack_id
        self.pack_data = pack_data
        self.add_marketplace_pack = add_marketplace_pack
        self.add_custom_pack = add_custom_pack
        self.add_all_marketplace_packs = add_all_marketplace_packs
        self.insecure = insecure
        self.file_path = file_path
        self.client = None

    def update(self) -> int:
        """
        Update the Configuration File according the usr input.
        :return: The exit code
        """
        exit_code: int = self.update_config_file_manager()
        return exit_code

    def update_config_file_manager(self) -> int:
        """
        Manages all XSOAR Configuration File command flows
        :return The exit code of each flow
        """
        if not self.verify_flags():
            return 1
        elif self.add_all_marketplace_packs:
            self.add_all_installed_packs_to_config_file()
        elif self.add_marketplace_pack:
            self.update_marketplace_pack()
        elif self.add_custom_pack:
            self.update_custom_pack()
        return 0

    def verify_flags(self) -> bool:
        """
        Verifies that the flags configuration given by the user is correct
        :return: The verification result
        """
        is_valid_pack_structure = True

        if self.add_marketplace_pack or self.add_custom_pack:
            if not self.pack_id:
                is_valid_pack_structure = False
                logger.info("[red]Error: Missing option '-pi' / '--pack-id'.[/red]")
            if not self.pack_data:
                is_valid_pack_structure = False
                logger.info("[red]Error: Missing option '-pd' / '--pack-data'.[/red]")
        return is_valid_pack_structure

    def add_all_installed_packs_to_config_file(self):
        """
        Update the MarketPlace Packs Section in the Configuration File with all the installed packs on the machine.
        """
        marketplace_packs = self.get_installed_packs()
        self.update_xsoar_config_data(
            section_name=MARKETPLACE_PACKS_SECTION, data_to_update=marketplace_packs
        )

    def get_installed_packs(self) -> List[Dict[str, str]]:
        """
        Gets the current installed packs on the machine.
        """
        client = demisto_client.configure(verify_ssl=self.insecure)
        res = client.generic_request("/contentpacks/metadata/installed", "GET")
        installed_packs_data = eval(res[0])

        installed_packs = [
            {"id": pack["id"], "version": pack["currentVersion"]}
            for pack in installed_packs_data
        ]
        return installed_packs

    def update_marketplace_pack(self):
        """
        Add / Update the MarketPlace Packs Section in the Configuration File with the new pack data.
        """
        new_pack = {"id": self.pack_id, "version": self.pack_data}
        self.update_xsoar_config_data(
            section_name=MARKETPLACE_PACKS_SECTION, data_to_update=new_pack
        )

    def update_custom_pack(self):
        """
        Add / Update the Custom Packs Section in the Configuration File with the new pack data.
        """
        new_pack = {"id": self.pack_id, "url": self.pack_data}
        self.update_xsoar_config_data(
            section_name=CUSTOM_PACKS_SECTION, data_to_update=new_pack
        )

    def update_xsoar_config_data(self, section_name, data_to_update):
        """
        Add / Update the Configuration File according to the section and data that provided.
        """
        config_file_info = self.get_xsoar_config_data()
        if config_file_info.get(section_name):
            if isinstance(data_to_update, list):
                config_file_info[section_name].extend(data_to_update)
            if isinstance(data_to_update, dict):
                config_file_info[section_name].append(data_to_update)
        else:
            config_file_info[section_name] = (
                [data_to_update] if isinstance(data_to_update, dict) else data_to_update
            )
        self.set_xsoar_config_data(config_file_info=config_file_info)

    def get_xsoar_config_data(self):
        config_file_info = {}
        if os.path.exists(self.file_path):
            with open(self.file_path) as config_file:
                try:
                    config_file_info = json.load(config_file)
                except json.JSONDecodeError:  # In case that the file exits but empty
                    pass
        return config_file_info

    def set_xsoar_config_data(self, config_file_info):
        with open(self.file_path, "w") as config_file:
            json.dump(config_file_info, config_file, indent=4)


# demisto_sdk/commands/upload/constants.py

from demisto_sdk.commands.content_graph.common import ContentType

MULTIPLE_ZIPPED_PACKS_FILE_STEM = "uploadable_packs"
MULTIPLE_ZIPPED_PACKS_FILE_NAME = f"{MULTIPLE_ZIPPED_PACKS_FILE_STEM}.zip"

CONTENT_TYPES_EXCLUDED_FROM_UPLOAD = {ContentType.TEST_PLAYBOOK}


# demisto_sdk/commands/upload/exceptions.py

from typing import TYPE_CHECKING, Optional

from packaging.version import Version

if TYPE_CHECKING:
    from demisto_sdk.commands.content_graph.objects.base_content import BaseContent
    from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class NotUploadableException(NotImplementedError):
    def __init__(self, item: "BaseContent", description: Optional[str] = None) -> None:
        description_suffix = f" {description}" if description else ""
        super().__init__(
            f"Object ({item.content_type} {item.object_id}) cannot be uploaded{description_suffix}"
        )


class NotIndivitudallyUploadableException(NotUploadableException):
    """
    Some content items must be uploaded as part of a pack.
    """

    def __init__(self, item: "BaseContent"):
        super().__init__(
            item,
            description=" independently. Use the -z flag to upload the whole pack, zipped.",
        )


class IncompatibleUploadVersionException(NotUploadableException):
    def __init__(self, item: "ContentItem", target: Version) -> None:
        self.item = item
        if target > Version(item.toversion):
            message = f"to_version={item.toversion}"
        elif target < Version(item.fromversion):
            message = f"from_version={item.fromversion}"
        else:
            raise RuntimeError(
                f"Invalid version comparison for {item.path} ({item.fromversion=}, {item.toversion=}, {target=})"
            )

        super().__init__(
            item,
            f". Target version {target} mismatch: "
            f"{item.content_type} {item.normalize_name} has {message}",
        )


# demisto_sdk/commands/upload/tools.py

import contextlib
from pathlib import Path
from typing import TYPE_CHECKING, Any

from demisto_client.demisto_api.rest import ApiException

from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.content_graph.objects.exceptions import FailedUploadException

if TYPE_CHECKING:
    from demisto_sdk.commands.content_graph.common import ContentType


def parse_upload_response(response: Any, path: Path, content_type: "ContentType"):
    if (
        isinstance(response, dict) and len(response) == 2 and "error" in response
    ):  # response format: {"response": {...}, "error": <empty string if ok>}
        if response["error"]:
            raise FailedUploadException(path=path, response_body=response)
    elif (isinstance(response, tuple) and len(response)) == 3:
        (
            data,
            status_code,
            _,
        ) = response  # third output is headers, not used here
        if status_code > 299:
            raise FailedUploadException(
                path=path,
                response_body=data,
                status_code=status_code,
            )
    else:
        response_str = (
            f"(could not convert response of type {type(response)} to string)"
        )
        with contextlib.suppress(Exception):
            response_str = str(response)
        logger.debug(
            f"got the following response when uploading {content_type} {path}: {response_str}"
        )


def parse_error_response(error: ApiException) -> str:
    """
    Parses error message from exception raised in call to client to upload a file

    error (ApiException): The exception which was raised in call in to client
    """
    if isinstance(error, KeyboardInterrupt):
        return "Aborted due to keyboard interrupt."

    if hasattr(error, "reason"):
        reason = str(error.reason)
        if "[SSL: CERTIFICATE_VERIFY_FAILED]" in reason:
            return (
                "[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate.\n"
                "Run the command with the --insecure flag."
            )

        elif "Failed to establish a new connection:" in reason:
            return (
                "Failed to establish a new connection: Connection refused.\n"
                "Check the BASE url configuration."
            )

        elif reason in ("Bad Request", "Forbidden"):
            error_body = json.loads(error.body)
            message = error_body.get("error", "")
            if message.startswith("[") and message.endswith("]"):
                message = message[1:-1]

            if error_body.get("status") == 403:
                message += "\nTry checking your API key configuration."
            return message
        return reason
    return str(error)


# demisto_sdk/commands/upload/upload.py

import logging
import shutil
import tempfile
from contextlib import suppress
from pathlib import Path
from typing import Iterable, List, Sequence
from zipfile import ZipFile

from pydantic import DirectoryPath

from demisto_sdk.commands.common.constants import (
    MarketplaceVersions,
)
from demisto_sdk.commands.common.tools import (
    parse_marketplace_kwargs,
    parse_multiple_path_inputs,
)
from demisto_sdk.commands.content_graph.objects.base_content import BaseContent
from demisto_sdk.commands.content_graph.objects.pack import Pack
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
from demisto_sdk.commands.upload.constants import (
    MULTIPLE_ZIPPED_PACKS_FILE_NAME,
)
from demisto_sdk.commands.upload.uploader import (
    ABORTED_RETURN_CODE,
    ERROR_RETURN_CODE,
    SUCCESS_RETURN_CODE,
)
from demisto_sdk.utils.utils import check_configuration_file

logger = logging.getLogger("demisto-sdk")


def upload_content_entity(**kwargs):
    from demisto_sdk.commands.upload.uploader import ConfigFileParser, Uploader

    inputs = parse_multiple_path_inputs(kwargs.get("input"))

    keep_zip = kwargs.pop("keep_zip", None)
    destination_zip_path = Path(keep_zip or tempfile.mkdtemp())
    marketplace = parse_marketplace_kwargs(kwargs)

    if config_file_path := kwargs.pop("input_config_file", None):
        logger.info("Uploading files from config file")
        if input_ := kwargs.get("input"):
            logger.warning(f"[orange]The input ({input_}) will NOT be used[/orange]")

        paths = ConfigFileParser(Path(config_file_path)).custom_packs_paths

        if not kwargs.get("zip") and are_all_packs_unzipped(paths=paths):
            inputs = paths
        else:
            pack_names = zip_multiple_packs(
                paths=paths,
                marketplace=marketplace,
                dir=destination_zip_path,
            )
            kwargs["detached_files"] = True
            kwargs["pack_names"] = pack_names
            inputs = tuple(
                [Path(destination_zip_path, MULTIPLE_ZIPPED_PACKS_FILE_NAME)]
            )

    check_configuration_file("upload", kwargs)

    if not inputs:
        logger.error("[red]No input provided for uploading[/red]")
        return ERROR_RETURN_CODE

    kwargs.pop("input")
    # Here the magic happens
    upload_result = SUCCESS_RETURN_CODE
    for input in inputs:
        result = Uploader(
            input=input,
            marketplace=marketplace,
            destination_zip_dir=destination_zip_path,
            **kwargs,
        ).upload()
        if result == ABORTED_RETURN_CODE:
            return result
        elif result == ERROR_RETURN_CODE:
            upload_result = ERROR_RETURN_CODE

    # Clean up
    if not keep_zip:
        shutil.rmtree(destination_zip_path, ignore_errors=True)

    return upload_result


def zip_multiple_packs(
    paths: Iterable[Path],
    marketplace: MarketplaceVersions,
    dir: DirectoryPath,
) -> Sequence[str]:
    packs: List[Pack] = []
    were_zipped: List[Path] = []

    for path in paths:
        if not path.exists():
            logger.error(f"[red]{path} does not exist, skipping[/red]")
            continue

        if path.is_file() and path.suffix == ".zip":
            were_zipped.append(path)
            continue

        pack = None
        with suppress(Exception):
            pack = BaseContent.from_path(path)
        if (pack is None) or (not isinstance(pack, Pack)):
            logger.error(f"[red]could not parse pack from {path}, skipping[/red]")
            continue
        packs.append(pack)

    result_zip_path = dir / MULTIPLE_ZIPPED_PACKS_FILE_NAME
    with tempfile.TemporaryDirectory() as tmp_dir:
        tmp_dir_path = Path(tmp_dir)
        ContentDTO(packs=packs).dump(tmp_dir_path, marketplace=marketplace, zip=False)
        with ZipFile(result_zip_path, "w") as zip_file:
            # copy files that were already zipped into the result
            for was_zipped in were_zipped:
                zip_file.write(was_zipped, was_zipped.name)
            for pack_path in tmp_dir_path.iterdir():
                shutil.make_archive(str(pack_path), "zip", pack_path)
                zip_file.write(pack_path.with_suffix(".zip"), f"{pack_path.name}.zip")

    return [pack.name for pack in packs] + [path.name for path in were_zipped]


def are_all_packs_unzipped(paths: Iterable[Path]) -> bool:
    """
    Checks whether all the packs intended to be uploaded are not zip files.
    """
    return not tuple(filter(lambda path: path.suffix == ".zip", paths))


# demisto_sdk/commands/upload/uploader.py

import ast
import glob
import itertools
import os
import zipfile
from pathlib import Path
from typing import Iterable, List, Optional, Tuple, Union

import demisto_client
from demisto_client.demisto_api.rest import ApiException
from packaging.version import Version
from tabulate import tabulate

from demisto_sdk.commands.common.constants import (
    CONTENT_ENTITIES_DIRS,
    INTEGRATIONS_DIR,
    SCRIPTS_DIR,
    FileType,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import (
    find_type,
    get_demisto_version,
    get_file,
    string_to_bool,
)
from demisto_sdk.commands.content_graph.objects.base_content import (
    BaseContent,
)
from demisto_sdk.commands.content_graph.objects.content_item import (
    ContentItem,
)
from demisto_sdk.commands.content_graph.objects.exceptions import (
    FailedUploadException,
    FailedUploadMultipleException,
)
from demisto_sdk.commands.content_graph.objects.pack import Pack, upload_zip
from demisto_sdk.commands.upload.constants import CONTENT_TYPES_EXCLUDED_FROM_UPLOAD
from demisto_sdk.commands.upload.exceptions import (
    IncompatibleUploadVersionException,
    NotUploadableException,
)
from demisto_sdk.commands.upload.tools import parse_error_response

SUCCESS_RETURN_CODE = 0
ERROR_RETURN_CODE = 1
ABORTED_RETURN_CODE = 2


class Uploader:
    """Upload a pack specified in self.infile to a remote Cortex XSOAR instance.
    Attributes:
        path (str): The path of a pack / directory / file to upload.
        client (DefaultApi): Demisto-SDK client object.
    """

    def __init__(
        self,
        input: Optional[Path],
        insecure: bool = False,
        pack_names: Optional[List[str]] = None,
        skip_validation: bool = False,
        detached_files: bool = False,
        reattach: bool = False,
        override_existing: bool = False,
        marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
        zip: bool = False,
        destination_zip_dir: Optional[Path] = None,
        **kwargs,
    ):
        self.path = None if input is None else Path(input)
        verify = (
            (not insecure) if insecure else None
        )  # set to None so demisto_client will use env var DEMISTO_VERIFY_SSL
        self.client = demisto_client.configure(verify_ssl=verify)

        self._successfully_uploaded_content_items: List[Union[ContentItem, Pack]] = []
        self._successfully_uploaded_zipped_packs: List[str] = []

        self._failed_upload_content_items: List[
            Tuple[Union[ContentItem, Pack], str]
        ] = []
        self._failed_upload_version_mismatch: List[ContentItem] = []
        self._failed_upload_zips: List[str] = []
        self.failed_parsing: List[Tuple[Path, str]] = []

        self.demisto_version = get_demisto_version(self.client)
        self.pack_names: List[str] = pack_names or []
        self.skip_upload_packs_validation = skip_validation
        self.should_detach_files = detached_files
        self.should_reattach_files = reattach
        self.override_existing = override_existing
        self.marketplace = marketplace
        self.zip = zip  # -z flag
        self.destination_zip_dir = destination_zip_dir

    def _upload_zipped(self, path: Path) -> bool:
        """
        Upload a zipped pack to the remote Cortex XSOAR instance.
        Args:
            path (Path): The path of the zipped pack to upload.
        Returns:
            bool: True if the upload was successful, False otherwise.
        """

        def notify_user_should_override_packs():
            """Notify the user about possible overridden packs."""

            if (
                response := self.client.generic_request(
                    "/contentpacks/metadata/installed", "GET", response_type="object"
                )
            ) and response[0]:
                installed_packs = {pack["name"] for pack in response[0]}
                if common_packs := installed_packs.intersection(self.pack_names):
                    pack_names = "\n".join(sorted(common_packs))
                    product = (
                        self.marketplace.lower()
                        .replace(MarketplaceVersions.MarketplaceV2, "XSIAM")
                        .upper()
                    )
                    logger.info(
                        "\n".join(
                            (
                                "[red]This command will overwrite the following packs:",
                                pack_names,
                                f"All changes made in these content items on {product} will be lost.[red]",
                            )
                        )
                    )
                    if not self.override_existing:
                        logger.info(
                            "[red]Are you sure you want to continue? y/[N][/red]"
                        )
                        return string_to_bool(str(input()), default_when_empty=False)
            return True

        def _parse_internal_pack_names(zip_path: Path) -> Optional[Tuple[str, ...]]:
            """
            A zip can be
            1. A single pack (just the pack folder zipped); in this case, we parse the name from the pack_metatada.json file
            2. Multiple zipped packs (each in its own .zip file); in this case, we use the names of the inner zips as pack names

            On failure to parse, returns None.
            """
            try:
                with zipfile.ZipFile(zip_path) as zip_file:
                    file_names = zip_file.namelist()

                    if "pack_metadata.json" in file_names:  # single pack
                        with zip_file.open("pack_metadata.json") as pack_metadata:
                            return (json.load(pack_metadata).get("name"),)

                # multiple packs, zipped
                return (
                    tuple(item[:-4] for item in file_names if item.endswith(".zip"))
                    or None
                )

            except Exception:
                logger.debug(f"failed extracting pack names from {zip_path}")
                return None

        pack_names = _parse_internal_pack_names(path) or (path.name,)
        if not notify_user_should_override_packs():
            return False

        try:
            if upload_zip(
                path=path,
                client=self.client,
                target_demisto_version=Version(str(self.demisto_version)),
                skip_validations=True,
                marketplace=self.marketplace,
            ):
                self._successfully_uploaded_zipped_packs.extend(pack_names)
                return True

        except Exception:
            logger.exception(f"Failed uploading {pack_names}")
            self._failed_upload_zips.extend(pack_names)

        return False

    def upload(self):
        """Upload the pack / directory / file to the remote Cortex XSOAR instance."""
        if self.demisto_version.base_version == "0":
            logger.info(
                "[red]Could not connect to the server. Try checking your connection configurations.[/red]"
            )
            return ERROR_RETURN_CODE

        if not self.path or not self.path.exists():
            logger.error(f"[red]input path: {self.path} does not exist[/red]")
            return ERROR_RETURN_CODE

        if self.should_detach_files:
            item_detacher = ItemDetacher(
                client=self.client, marketplace=self.marketplace
            )
            detached_items_ids = item_detacher.detach(upload_file=True)

            if self.should_reattach_files:
                ItemReattacher(client=self.client).reattach(
                    detached_files_ids=detached_items_ids
                )

        logger.info(
            f"Uploading {self.path} to {self.client.api_client.configuration.host}..."
        )

        try:
            if self.path.suffix == ".zip":
                success = self._upload_zipped(self.path)
            elif self.path.is_dir() and is_uploadable_dir(self.path):
                success = self._upload_entity_dir(self.path)
            else:
                success = self._upload_single(self.path)
        except KeyboardInterrupt:
            return ABORTED_RETURN_CODE

        if self.failed_parsing and not any(
            (
                self._successfully_uploaded_content_items,
                self._successfully_uploaded_zipped_packs,
                self._failed_upload_content_items,
                self._failed_upload_version_mismatch,
                self._failed_upload_zips,
            )
        ):
            # Nothing was uploaded, nor collected as error
            logger.error(
                "\n".join(
                    (
                        "[red]Nothing to upload: the input path should point to one of the following:",
                        "\t1. A Pack",
                        "\t2. A content entity directory that is inside a pack, e.g. Integrations",
                        "\t3. A valid content item file, that can be imported to Cortex XSOAR manually.[/red]",
                    )
                )
            )
            return ERROR_RETURN_CODE

        self.print_summary()
        return SUCCESS_RETURN_CODE if success else ERROR_RETURN_CODE

    def _upload_single(self, path: Path) -> bool:
        """
        Upload a content item, a pack, or a zip containing packs.

        Returns:
            bool: whether the item is uploaded succesfully.

        Raises:
            NotIndivitudallyUploadedException (see exception class)
            NotUploadableException
        """
        content_item: Union[ContentItem, Pack] = BaseContent.from_path(
            path
        )  # type:ignore[assignment]
        if content_item is None:
            reason = (
                "Deprecated type - use LayoutContainer instead"
                if find_type(str(path)) == FileType.LAYOUT
                else "unknown"
            )
            self.failed_parsing.append((path, reason))
            return False

        try:
            content_item.upload(
                client=self.client,
                marketplace=self.marketplace,
                target_demisto_version=Version(str(self.demisto_version)),
                zip=self.zip,  # only used for Packs
                destination_zip_dir=self.destination_zip_dir,  # only used for Packs
            )

            # upon reaching this line, the upload is surely successful
            uploaded_successfully = parse_uploaded_successfully(
                content_item=content_item, zip=self.zip
            )
            self._successfully_uploaded_content_items.extend(uploaded_successfully)
            for item_uploaded_successfully in uploaded_successfully:
                logger.debug(
                    f"Uploaded {item_uploaded_successfully.content_type} {item_uploaded_successfully.normalize_name} successfully"
                )
            return True

        except KeyboardInterrupt:
            raise  # the functinos calling this one have a special return code for manual interruption

        except IncompatibleUploadVersionException:
            assert isinstance(
                content_item, ContentItem
            ), "This exception should only be raised for content items"
            self._failed_upload_version_mismatch.append(content_item)
            return False

        except ApiException as e:
            self._failed_upload_content_items.append(
                (content_item, parse_error_response(e))
            )
            return False

        except FailedUploadMultipleException as e:
            for failure in e.upload_failures:
                failure_str = failure.additional_info or str(failure)

                _failed_content_item: Union[
                    Pack, ContentItem, None
                ] = BaseContent.from_path(
                    failure.path
                )  # type:ignore[assignment]

                if _failed_content_item is None:
                    self.failed_parsing.append((failure.path, failure_str))
                else:
                    self._failed_upload_content_items.append(
                        (_failed_content_item, failure_str)
                    )
            for failure_mismatch in e.incompatible_versions_items:
                self._failed_upload_version_mismatch.append(failure_mismatch.item)

            self._successfully_uploaded_content_items.extend(e.uploaded_successfully)
            return False

        except (FailedUploadException, NotUploadableException, Exception) as e:
            self._failed_upload_content_items.append((content_item, str(e)))
            return False

    def _upload_entity_dir(self, path: Path) -> bool:
        """
        Uploads an entity path directory
        Args:
            path: an entity path in the following formats:
                `.../Packs/{Pack_Name}/{Entity_Type}/[optional: entity name]`

        Returns:
            Whether the upload succeeded.

        """
        if not set(path.parts[-2:]).intersection(CONTENT_ENTITIES_DIRS):
            # neither the last, nor second-last are a content entity dir
            raise ValueError(f"Invalid entity dir path: {path}")

        to_upload: Iterable[Path]
        if path.name in {SCRIPTS_DIR, INTEGRATIONS_DIR}:
            # These folders have another level of content
            to_upload = filter(lambda p: p.is_dir(), path.iterdir())
        else:
            to_upload = itertools.chain(path.glob("*.yml"), path.glob("*.json"))

        return all(self._upload_single(item) for item in to_upload)

    def notify_user_should_override_packs(self):
        """Notify the user about possible overridden packs."""

        response = self.client.generic_request(
            "/contentpacks/metadata/installed", "GET"
        )
        if installed_packs := json.loads(response[0]):
            installed_packs = {pack["name"] for pack in installed_packs}
            if common_packs := installed_packs.intersection(self.pack_names):
                pack_names = "\n".join(common_packs)
                product = (
                    self.marketplace.lower()
                    .replace(MarketplaceVersions.MarketplaceV2, "XSIAM")
                    .upper()
                )
                logger.debug(
                    f"[red]This command will overwrite the following packs:\n{pack_names}.\n"
                    f"Any changes made on {product} will be lost.[red]"
                )
                if not self.override_existing:
                    logger.info("[red]Are you sure you want to continue? y/[N][/red]")
                    answer = str(input())
                    return answer in {"y", "Y", "yes"}

        return True

    def print_summary(self) -> None:
        """Prints uploaded files summary
        Successful uploads grid based on `successfully_uploaded_files` attribute in green color
        Failed uploads grid based on `failed_uploaded_files` attribute in red color
        """
        logger.info("UPLOAD SUMMARY:\n")

        if (
            self._successfully_uploaded_content_items
            or self._successfully_uploaded_zipped_packs
        ):
            uploaded_str = tabulate(
                (
                    itertools.chain(
                        (
                            (
                                item.path.name,
                                item.content_type,
                                item.pack_name,
                                item.pack_version,
                            )
                            for item in self._successfully_uploaded_content_items
                        ),
                        (
                            (
                                item,
                                "Pack",
                                item,
                                "",  # When uploading zips we are not aware of the version
                            )
                            for item in self._successfully_uploaded_zipped_packs
                        ),
                    )
                ),
                headers=["NAME", "TYPE", "PACK NAME", "PACK VERSION"],
                tablefmt="fancy_grid",
            )

            logger.info(f"[green]SUCCESSFUL UPLOADS:\n{uploaded_str}\n[/green]")

        if self._failed_upload_version_mismatch:
            version_mismatch_str = tabulate(
                (
                    (
                        item.path.name,
                        item.content_type,
                        self.demisto_version,
                        item.fromversion,
                        item.toversion,
                    )
                    for item in self._failed_upload_version_mismatch
                ),
                headers=[
                    "NAME",
                    "TYPE",
                    "XSOAR Version",
                    "FILE_FROM_VERSION",
                    "FILE_TO_VERSION",
                ],
                tablefmt="fancy_grid",
            )
            logger.info(
                f"[yellow]NOT UPLOADED DUE TO VERSION MISMATCH:\n{version_mismatch_str}\n[/yellow]"
            )

        if self.failed_parsing:
            failed_parsing_str = tabulate(
                ((path.name, path, reason) for path, reason in self.failed_parsing),
                headers=("FILE_NAME", "PATH", "REASON"),
                tablefmt="fancy_grid",
            )
            logger.info(f"[red]FAILED PARSING CONTENT:\n{failed_parsing_str}[/red]")

        if self._failed_upload_content_items or self._failed_upload_zips:
            failed_upload_str = tabulate(
                (
                    itertools.chain(
                        (
                            (item.path.name, item.content_type, error)
                            for item, error in self._failed_upload_content_items
                        ),
                        (
                            (
                                (item, "Pack", "see logs above")
                                for item in self._failed_upload_zips
                            )
                        ),
                    )
                ),
                headers=["NAME", "TYPE", "ERROR"],
                tablefmt="fancy_grid",
            )
            logger.info(f"[red]FAILED UPLOADS:\n{failed_upload_str}\n[/red]")


class ConfigFileParser:
    def __init__(self, path: Path):
        self.path = path

        with self.path.open() as f:
            self.content = json.load(f)

        self.custom_packs_paths: Tuple[Path, ...] = tuple(
            Path(pack["url"]) for pack in self.content.get("custom_packs", ())
        )


class ItemDetacher:
    def __init__(
        self, client, marketplace: MarketplaceVersions, file_path: str = "SystemPacks"
    ):
        self.file_path = file_path
        self.client = client
        self.marketplace = marketplace

    DETACH_ITEM_TYPE_TO_ENDPOINT: dict = {
        "IncidentTypes": "/incidenttype/detach/:id/",
        "Layouts": "/layout/:id/detach/",
        "Playbooks": "/playbook/detach/:id/",
        "Scripts": "/automation/detach/:id/",
    }

    VALID_FILES_FOR_DETACH = ["Playbooks", "Scripts", "IncidentTypes", "Layouts"]

    def detach_item(self, file_id, file_path):
        endpoint: str = ""
        for file_type, file_endpoint in self.DETACH_ITEM_TYPE_TO_ENDPOINT.items():
            if file_type in file_path:
                endpoint = file_endpoint
                break
        endpoint = endpoint.replace(":id", file_id)

        try:
            self.client.generic_request(endpoint, "POST")
            logger.info(f"\n[green]File: {file_id} was detached[/green]")
        except Exception as e:
            raise Exception(f"Exception raised when fetching custom content:\n{e}")

    def extract_items_from_dir(self):
        detach_files_list: list = []

        all_files = glob.glob(f"{self.file_path}/**/*", recursive=True)
        for file_path in all_files:
            if os.path.isfile(file_path) and self.is_valid_file_for_detach(file_path):
                file_type = self.find_item_type_to_detach(file_path)
                file_data = get_file(file_path)
                file_id = file_data.get("id", "")
                if file_id:
                    detach_files_list.append(
                        {
                            "file_id": file_id,
                            "file_type": file_type,
                            "file_path": file_path,
                        }
                    )
        return detach_files_list

    def is_valid_file_for_detach(self, file_path: str) -> bool:
        for file in self.VALID_FILES_FOR_DETACH:
            if file in file_path and (
                file_path.endswith("yml") or file_path.endswith("json")
            ):
                return True
        return False

    def find_item_type_to_detach(self, file_path) -> str:
        return "yml" if "Playbooks" in file_path or "Scripts" in file_path else "json"

    def find_item_id_to_detach(self):
        self.find_item_type_to_detach(self.file_path)
        file_data = get_file(self.file_path)
        return file_data.get("id")

    def detach(self, upload_file: bool = False) -> List[str]:
        detach_files_list: list = []
        if os.path.isdir(self.file_path):
            detach_files_list = self.extract_items_from_dir()
            for file in detach_files_list:
                self.detach_item(file.get("file_id"), file_path=file.get("file_path"))
                if upload_file:
                    Uploader(
                        input=Path(raw_file_path)
                        if (raw_file_path := file.get("file_path")) is not None
                        else None,
                        marketplace=self.marketplace,
                    ).upload()

        elif os.path.isfile(self.file_path):
            file_id = self.find_item_id_to_detach()
            detach_files_list.append({"file_id": file_id, "file_path": self.file_path})
            self.detach_item(file_id=file_id, file_path=self.file_path)
            if upload_file:
                Uploader(
                    input=Path(self.file_path) if self.file_path is not None else None,
                    marketplace=self.marketplace,
                ).upload()

        return [file["file_id"] for file in detach_files_list]


class ItemReattacher:
    def __init__(self, client, file_path: str = ""):
        self.file_path = file_path
        self.client = client

    REATTACH_ITEM_TYPE_TO_ENDPOINT: dict = {
        "IncidentType": "/incidenttype/attach/:id",
        "Layouts": "/layout/:id/attach",
        "Playbooks": "/playbook/attach/:id",
        "Automations": "/automation/attach/:id",
    }

    def download_all_detach_supported_items(self) -> dict:
        all_detach_supported_items: dict = {}
        yml_req_body = {"query": "system:T"}

        for endpoint in ["/playbook/search", "/automation/search"]:
            res = self.client.generic_request(endpoint, "POST", body=yml_req_body)
            res_result = ast.literal_eval(res[0])
            if "playbook" in endpoint:
                all_detach_supported_items["Playbooks"] = res_result.get("playbooks")
            else:
                all_detach_supported_items["Automations"] = res_result.get("scripts")

        for item_type in ["IncidentType", "Layouts"]:
            endpoint = item_type.lower()
            res = self.client.generic_request(endpoint, "GET")
            all_detach_supported_items[item_type] = ast.literal_eval(res[0])

        return all_detach_supported_items

    def reattach_item(self, item_id, item_type):
        endpoint: str = self.REATTACH_ITEM_TYPE_TO_ENDPOINT[item_type]
        endpoint = endpoint.replace(":id", item_id)
        try:
            self.client.generic_request(endpoint, "POST")
            logger.debug(f"\n[green]{item_type}: {item_id} was reattached[/green]")
        except Exception as e:
            raise Exception(f"Exception raised when fetching custom content:\n{e}")

    def reattach(self, detached_files_ids=None):
        if not self.file_path and detached_files_ids:
            all_files: dict = self.download_all_detach_supported_items()
            for item_type, item_list in all_files.items():
                for item in item_list:
                    detached = item.get("detached", "")
                    if not detached or detached == "false":
                        continue
                    item_id = item.get("id")
                    if item_id and item_id not in detached_files_ids:
                        self.reattach_item(item_id, item_type)


def is_uploadable_dir(path: Path) -> bool:
    if not path.is_dir():
        raise ValueError(f"{path} is not a directory")

    return path.name in CONTENT_ENTITIES_DIRS or (
        path.name in {INTEGRATIONS_DIR, SCRIPTS_DIR}
        and path.parent.name in CONTENT_ENTITIES_DIRS
    )


def parse_uploaded_successfully(
    content_item: Union[Pack, ContentItem], zip: bool
) -> Iterable[Union[Pack, ContentItem]]:
    # packs uploaded unzipped are uploaded item by item, we have to extract the item details here
    if isinstance(content_item, Pack) and not zip:
        return iter(
            filter(
                lambda content_item: content_item.content_type
                not in CONTENT_TYPES_EXCLUDED_FROM_UPLOAD,
                content_item.content_items,
            )
        )

    return (content_item,)


# demisto_sdk/commands/validate/validate_manager.py

import os
from concurrent.futures._base import Future, as_completed
from pathlib import Path
from typing import Callable, List, Optional, Set, Tuple

import pebble
from git import InvalidGitRepositoryError
from packaging import version

from demisto_sdk.commands.common import tools
from demisto_sdk.commands.common.configuration import Configuration
from demisto_sdk.commands.common.constants import (
    API_MODULES_PACK,
    AUTHOR_IMAGE_FILE_NAME,
    CONTENT_ENTITIES_DIRS,
    DEFAULT_CONTENT_ITEM_TO_VERSION,
    GENERIC_FIELDS_DIR,
    GENERIC_TYPES_DIR,
    IGNORED_PACK_NAMES,
    OLDEST_SUPPORTED_VERSION,
    PACKS_DIR,
    PACKS_PACK_META_FILE_NAME,
    SKIP_RELEASE_NOTES_FOR_TYPES,
    VALIDATION_USING_GIT_IGNORABLE_DATA,
    XSIAM_DASHBOARDS_DIR,
    FileType,
    FileType_ALLOWED_TO_DELETE,
    PathLevel,
)
from demisto_sdk.commands.common.content import Content
from demisto_sdk.commands.common.content_constant_paths import (
    CONTENT_PATH,
    DEFAULT_ID_SET_PATH,
)
from demisto_sdk.commands.common.cpu_count import cpu_count
from demisto_sdk.commands.common.errors import (
    FOUND_FILES_AND_ERRORS,
    FOUND_FILES_AND_IGNORED_ERRORS,
    PRESET_ERROR_TO_CHECK,
    PRESET_ERROR_TO_IGNORE,
    Errors,
    get_all_error_codes,
)
from demisto_sdk.commands.common.git_util import GitUtil
from demisto_sdk.commands.common.hook_validations.author_image import (
    AuthorImageValidator,
)
from demisto_sdk.commands.common.hook_validations.base_validator import (
    BaseValidator,
    error_codes,
)
from demisto_sdk.commands.common.hook_validations.classifier import ClassifierValidator
from demisto_sdk.commands.common.hook_validations.conf_json import ConfJsonValidator
from demisto_sdk.commands.common.hook_validations.correlation_rule import (
    CorrelationRuleValidator,
)
from demisto_sdk.commands.common.hook_validations.dashboard import DashboardValidator
from demisto_sdk.commands.common.hook_validations.deprecation import (
    DeprecationValidator,
)
from demisto_sdk.commands.common.hook_validations.description import (
    DescriptionValidator,
)
from demisto_sdk.commands.common.hook_validations.generic_definition import (
    GenericDefinitionValidator,
)
from demisto_sdk.commands.common.hook_validations.generic_field import (
    GenericFieldValidator,
)
from demisto_sdk.commands.common.hook_validations.generic_module import (
    GenericModuleValidator,
)
from demisto_sdk.commands.common.hook_validations.generic_type import (
    GenericTypeValidator,
)
from demisto_sdk.commands.common.hook_validations.graph_validator import GraphValidator
from demisto_sdk.commands.common.hook_validations.id import IDSetValidations
from demisto_sdk.commands.common.hook_validations.image import ImageValidator
from demisto_sdk.commands.common.hook_validations.incident_field import (
    IncidentFieldValidator,
)
from demisto_sdk.commands.common.hook_validations.incident_type import (
    IncidentTypeValidator,
)
from demisto_sdk.commands.common.hook_validations.indicator_field import (
    IndicatorFieldValidator,
)
from demisto_sdk.commands.common.hook_validations.integration import (
    IntegrationValidator,
)
from demisto_sdk.commands.common.hook_validations.job import JobValidator
from demisto_sdk.commands.common.hook_validations.layout import (
    LayoutsContainerValidator,
    LayoutValidator,
)
from demisto_sdk.commands.common.hook_validations.layout_rule import LayoutRuleValidator
from demisto_sdk.commands.common.hook_validations.lists import ListsValidator
from demisto_sdk.commands.common.hook_validations.mapper import MapperValidator
from demisto_sdk.commands.common.hook_validations.modeling_rule import (
    ModelingRuleValidator,
)
from demisto_sdk.commands.common.hook_validations.pack_unique_files import (
    PackUniqueFilesValidator,
)
from demisto_sdk.commands.common.hook_validations.parsing_rule import (
    ParsingRuleValidator,
)
from demisto_sdk.commands.common.hook_validations.playbook import PlaybookValidator
from demisto_sdk.commands.common.hook_validations.pre_process_rule import (
    PreProcessRuleValidator,
)
from demisto_sdk.commands.common.hook_validations.python_file import PythonFileValidator
from demisto_sdk.commands.common.hook_validations.readme import ReadMeValidator
from demisto_sdk.commands.common.hook_validations.release_notes import (
    ReleaseNotesValidator,
)
from demisto_sdk.commands.common.hook_validations.release_notes_config import (
    ReleaseNotesConfigValidator,
)
from demisto_sdk.commands.common.hook_validations.report import ReportValidator
from demisto_sdk.commands.common.hook_validations.reputation import ReputationValidator
from demisto_sdk.commands.common.hook_validations.script import ScriptValidator
from demisto_sdk.commands.common.hook_validations.structure import StructureValidator
from demisto_sdk.commands.common.hook_validations.test_playbook import (
    TestPlaybookValidator,
)
from demisto_sdk.commands.common.hook_validations.triggers import TriggersValidator
from demisto_sdk.commands.common.hook_validations.widget import WidgetValidator
from demisto_sdk.commands.common.hook_validations.wizard import WizardValidator
from demisto_sdk.commands.common.hook_validations.xdrc_templates import (
    XDRCTemplatesValidator,
)
from demisto_sdk.commands.common.hook_validations.xsiam_dashboard import (
    XSIAMDashboardValidator,
)
from demisto_sdk.commands.common.hook_validations.xsiam_report import (
    XSIAMReportValidator,
)
from demisto_sdk.commands.common.hook_validations.xsoar_config_json import (
    XSOARConfigJsonValidator,
)
from demisto_sdk.commands.common.logger import get_log_file, logger
from demisto_sdk.commands.common.tools import (
    _get_file_id,
    find_type,
    get_api_module_dependencies_from_graph,
    get_api_module_ids,
    get_file,
    get_pack_ignore_content,
    get_pack_name,
    get_pack_names_from_files,
    get_relative_path_from_packs_dir,
    get_remote_file,
    get_yaml,
    open_id_set_file,
    run_command_os,
)
from demisto_sdk.commands.create_id_set.create_id_set import IDSetCreator

SKIPPED_FILES = [
    "CommonServerUserPython.py",
    "demistomock.py",
    "DemistoClassApiModule.py",
]


class ValidateManager:
    def __init__(
        self,
        is_backward_check=True,
        prev_ver=None,
        use_git=False,
        only_committed_files=False,
        print_ignored_files=False,
        skip_conf_json=True,
        validate_graph=False,
        validate_id_set=False,
        file_path=None,
        validate_all=False,
        is_external_repo=False,
        skip_pack_rn_validation=False,
        print_ignored_errors=False,
        silence_init_prints=False,
        no_docker_checks=False,
        skip_dependencies=False,
        id_set_path=None,
        staged=False,
        create_id_set=False,
        json_file_path=None,
        skip_schema_check=False,
        debug_git=False,
        include_untracked=False,
        pykwalify_logs=False,
        check_is_unskipped=True,
        quiet_bc=False,
        multiprocessing=True,
        specific_validations=None,
    ):
        # General configuration
        self.skip_docker_checks = False
        self.no_configuration_prints = silence_init_prints
        self.skip_conf_json = skip_conf_json
        self.is_backward_check = is_backward_check
        self.is_circle = only_committed_files
        self.validate_all = validate_all
        self.use_git = use_git
        self.skip_pack_rn_validation = skip_pack_rn_validation
        self.print_ignored_files = print_ignored_files
        self.print_ignored_errors = print_ignored_errors
        self.skip_dependencies = skip_dependencies or not use_git
        self.skip_id_set_creation = not create_id_set or skip_dependencies
        self.validate_graph = validate_graph
        self.compare_type = "..."
        self.staged = staged
        self.skip_schema_check = skip_schema_check
        self.debug_git = debug_git
        self.include_untracked = include_untracked
        self.pykwalify_logs = pykwalify_logs
        self.quiet_bc = quiet_bc
        self.check_is_unskipped = check_is_unskipped
        self.conf_json_data = {}
        self.run_with_multiprocessing = multiprocessing
        self.packs_with_mp_change = set()
        self.is_possible_validate_readme = (
            self.is_node_exist() or ReadMeValidator.is_docker_available()
        )

        if json_file_path:
            self.json_file_path = (
                os.path.join(json_file_path, "validate_outputs.json")
                if os.path.isdir(json_file_path)
                else json_file_path
            )
        else:
            self.json_file_path = ""

        self.specific_validations = specific_validations
        if specific_validations:
            self.specific_validations = specific_validations.split(",")

        # Class constants
        self.handle_error = BaseValidator(json_file_path=json_file_path).handle_error
        self.should_run_validation = BaseValidator(
            specific_validations=specific_validations
        ).should_run_validation
        self.file_path = file_path
        self.id_set_path = id_set_path or DEFAULT_ID_SET_PATH
        # create the id_set only once per run.
        self.id_set_file = self.get_id_set_file(
            self.skip_id_set_creation, self.id_set_path
        )

        self.id_set_validations = (
            IDSetValidations(
                is_circle=self.is_circle,
                configuration=Configuration(),
                ignored_errors=None,
                id_set_file=self.id_set_file,
                json_file_path=json_file_path,
                specific_validations=self.specific_validations,
            )
            if validate_id_set
            else None
        )

        self.deprecation_validator = DeprecationValidator(id_set_file=self.id_set_file)

        try:
            self.git_util = GitUtil(repo=Content.git())
            self.branch_name = self.git_util.get_current_git_branch_or_hash()
        except (InvalidGitRepositoryError, TypeError):
            # if we are using git - fail the validation by raising the exception.
            if self.use_git:
                raise
            # if we are not using git - simply move on.
            else:
                logger.info("Unable to connect to git")
                self.git_util = None  # type: ignore[assignment]
                self.branch_name = ""

        if prev_ver and not prev_ver.startswith("origin"):
            self.prev_ver = self.setup_prev_ver("origin/" + prev_ver)
        else:
            self.prev_ver = self.setup_prev_ver(prev_ver)

        self.check_only_schema = False
        self.always_valid = False
        self.ignored_files = set()
        self.new_packs = set()
        self.skipped_file_types = (
            FileType.CHANGELOG,
            FileType.DOC_IMAGE,
            FileType.MODELING_RULE_SCHEMA,
            FileType.XSIAM_REPORT_IMAGE,
            FileType.PIPFILE,
            FileType.PIPFILE_LOCK,
            FileType.TXT,
            FileType.JAVASCRIPT_FILE,
            FileType.POWERSHELL_FILE,
            FileType.PYLINTRC,
            FileType.SECRET_IGNORE,
            FileType.LICENSE,
            FileType.UNIFIED_YML,
            FileType.PACK_IGNORE,
            FileType.INI,
            FileType.PEM,
            FileType.METADATA,
        )

        self.is_external_repo = is_external_repo
        if is_external_repo:
            if not self.no_configuration_prints:
                logger.info("Running in a private repository")
            self.skip_conf_json = True

        self.print_percent = False
        self.completion_percentage = 0

        if validate_all:
            # No need to check docker images on build branch hence we do not check on -a mode
            # also do not skip id set creation unless the flag is up
            self.skip_docker_checks = True
            self.skip_pack_rn_validation = True
            self.print_percent = (
                not self.run_with_multiprocessing
            )  # the Multiprocessing will mismatch the percent
            self.check_is_unskipped = False

        if no_docker_checks:
            self.skip_docker_checks = True

        if self.skip_conf_json:
            self.check_is_unskipped = False

        if not self.skip_conf_json:
            self.conf_json_validator = ConfJsonValidator(
                specific_validations=self.specific_validations
            )
            self.conf_json_data = self.conf_json_validator.conf_data

    def is_node_exist(self) -> bool:
        """Check if node interpreter exists.
        Returns:
            bool: True if node exist, else False
        """
        # Check node exist
        content_path = CONTENT_PATH
        stdout, stderr, exit_code = run_command_os("node -v", cwd=content_path)  # type: ignore
        if exit_code:
            return False
        return True

    def print_final_report(self, valid):
        self.print_ignored_files_report(self.print_ignored_files)
        self.print_ignored_errors_report(self.print_ignored_errors)

        if valid:
            logger.info("\n[green]The files are valid[/green]")
            return 0

        else:
            all_failing_files = "\n".join(set(FOUND_FILES_AND_ERRORS))
            logger.info(
                f"\n[red]=========== Found errors in the following files ===========\n\n{all_failing_files}[/red]\n"
            )

            if self.always_valid:
                logger.info(
                    "[yellow]Found the errors above, but not failing build[/yellow]"
                )
                return 0

            logger.info(
                "[red]The files were found as invalid, the exact error message can be located above[/red]"
            )
            return 1

    def run_validation(self):
        """Initiates validation in accordance with mode (i,g,a)"""
        if self.validate_all:
            is_valid = self.run_validation_on_all_packs()
        elif self.use_git:
            is_valid = self.run_validation_using_git()
        elif self.file_path:
            is_valid = self.run_validation_on_specific_files()
        else:
            # default validate to -g --post-commit
            self.use_git = True
            self.is_circle = True
            is_valid = self.run_validation_using_git()
        return self.print_final_report(is_valid)

    @staticmethod
    def detect_file_level(file_path: str) -> PathLevel:
        """
        Detect the whether the path points to a file, a content entity dir, a content generic entity dir
        (i.e GenericFields or GenericTypes), a pack dir or package dir

        Args:
             file_path(str): the path to check.

        Returns:
            PathLevel. File, ContentDir, ContentGenericDir, Pack or Package - depending on the file path level.
        """
        if os.path.isfile(file_path):
            return PathLevel.FILE

        file_path = file_path.rstrip("/")
        dir_name = os.path.basename(file_path)
        if dir_name in CONTENT_ENTITIES_DIRS:
            return PathLevel.CONTENT_ENTITY_DIR

        if str(os.path.dirname(file_path)).endswith(GENERIC_TYPES_DIR) or str(
            os.path.dirname(file_path)
        ).endswith(GENERIC_FIELDS_DIR):
            return PathLevel.CONTENT_GENERIC_ENTITY_DIR

        if os.path.basename(os.path.dirname(file_path)) == PACKS_DIR:
            return PathLevel.PACK

        else:
            return PathLevel.PACKAGE

    def run_validation_on_specific_files(self):
        """Run validations only on specific files"""
        files_validation_result = set()
        if self.use_git:
            self.setup_git_params()
        files_to_validate = self.file_path.split(",")

        for path in files_to_validate:
            error_ignore_list = self.get_error_ignore_list(get_pack_name(path))
            file_level = self.detect_file_level(path)

            if file_level == PathLevel.FILE:
                logger.info(
                    f"\n[cyan]================= Validating file {path} =================[/cyan]"
                )
                files_validation_result.add(
                    self.run_validations_on_file(path, error_ignore_list)
                )

            elif file_level == PathLevel.CONTENT_ENTITY_DIR:
                logger.info(
                    f"\n[cyan]================= Validating content directory {path} =================[/cyan]"
                )
                files_validation_result.add(
                    self.run_validation_on_content_entities(path, error_ignore_list)
                )

            elif file_level == PathLevel.CONTENT_GENERIC_ENTITY_DIR:
                logger.info(
                    f"\n[cyan]================= Validating content directory {path} =================[/cyan]"
                )
                files_validation_result.add(
                    self.run_validation_on_generic_entities(path, error_ignore_list)
                )

            elif file_level == PathLevel.PACK:
                logger.info(
                    f"\n[cyan]================= Validating pack {path} =================[/cyan]"
                )
                files_validation_result.add(self.run_validations_on_pack(path)[0])

            else:
                logger.info(
                    f"\n[cyan]================= Validating package {path} =================[/cyan]"
                )
                files_validation_result.add(
                    self.run_validation_on_package(path, error_ignore_list)
                )

        if self.validate_graph:
            logger.info(
                "\n[cyan]================= Validating graph =================[/cyan]"
            )
            with GraphValidator(
                specific_validations=self.specific_validations,
                input_files=files_to_validate,
                include_optional_deps=True,
            ) as graph_validator:
                files_validation_result.add(graph_validator.is_valid_content_graph())

        return all(files_validation_result)

    def wait_futures_complete(self, futures_list: List[Future], done_fn: Callable):
        """Wait for all futures to complete, Raise exception if occurred.
        Args:
            futures_list: futures to wait for.
            done_fn: Function to run on result.
        Raises:
            Exception: Raise caught exception for further cleanups.
        """
        for future in as_completed(futures_list):
            try:
                result = future.result()
                done_fn(result[0], result[1])
            except Exception as e:
                logger.info(
                    f"[red]An error occurred while tried to collect result, Error: {e}[/red]"
                )
                raise

    def run_validation_on_all_packs(self):
        """Runs validations on all files in all packs in repo (-a option)

        Returns:
            bool. true if all files are valid, false otherwise.
        """
        logger.info(
            "\n[cyan]================= Validating all files =================[/cyan]"
        )
        all_packs_valid = set()

        if not self.skip_conf_json:
            all_packs_valid.add(self.conf_json_validator.is_valid_conf_json())

        count = 1
        # Filter non-pack files that might exist locally (e.g, .DS_STORE on MacOS)
        all_packs = list(
            filter(
                os.path.isdir,
                [os.path.join(PACKS_DIR, p) for p in os.listdir(PACKS_DIR)],
            )
        )
        num_of_packs = len(all_packs)
        all_packs.sort(key=str.lower)

        ReadMeValidator.add_node_env_vars()
        if self.is_possible_validate_readme:
            with ReadMeValidator.start_mdx_server(handle_error=self.handle_error):
                return self.validate_packs(
                    all_packs, all_packs_valid, count, num_of_packs
                )
        else:
            return self.validate_packs(all_packs, all_packs_valid, count, num_of_packs)

    def validate_packs(
        self, all_packs: list, all_packs_valid: set, count: int, num_of_packs: int
    ) -> bool:

        if self.run_with_multiprocessing:
            with pebble.ProcessPool(max_workers=cpu_count()) as executor:
                futures = []
                for pack_path in all_packs:
                    futures.append(
                        executor.schedule(
                            self.run_validations_on_pack, args=(pack_path,)
                        )
                    )
                self.wait_futures_complete(
                    futures_list=futures,
                    done_fn=lambda x, y: (
                        all_packs_valid.add(x),  # type: ignore
                        FOUND_FILES_AND_ERRORS.extend(y),  # type: ignore[func-returns-value]
                    ),
                )
        else:
            for pack_path in all_packs:
                self.completion_percentage = format((count / num_of_packs) * 100, ".2f")  # type: ignore
                all_packs_valid.add(self.run_validations_on_pack(pack_path)[0])
                count += 1
        if self.validate_graph:
            logger.info(
                "\n[cyan]================= Validating graph =================[/cyan]"
            )
            specific_validations_list = (
                self.specific_validations if self.specific_validations else []
            )
            with GraphValidator(
                specific_validations=self.specific_validations,
                include_optional_deps=(
                    True if "GR103" in specific_validations_list else False
                ),
            ) as graph_validator:
                all_packs_valid.add(graph_validator.is_valid_content_graph())

        return all(all_packs_valid)

    def run_validations_on_pack(self, pack_path, skip_files: Optional[Set[str]] = None):
        """Runs validation on all files in given pack. (i,g,a)

        Args:
            pack_path: the path to the pack.
            skip_files: a list of files to skip.

        Returns:
            bool. true if all files in pack are valid, false otherwise.
        """
        if not skip_files:
            skip_files = set()

        pack_entities_validation_results = set()
        pack_error_ignore_list = self.get_error_ignore_list(os.path.basename(pack_path))

        pack_entities_validation_results.add(
            self.validate_pack_unique_files(pack_path, pack_error_ignore_list)
        )

        for content_dir in os.listdir(pack_path):
            content_entity_path = os.path.join(pack_path, content_dir)
            if content_entity_path not in skip_files:
                if content_dir in CONTENT_ENTITIES_DIRS:
                    pack_entities_validation_results.add(
                        self.run_validation_on_content_entities(
                            content_entity_path, pack_error_ignore_list
                        )
                    )
                else:
                    self.ignored_files.add(content_entity_path)

        return all(pack_entities_validation_results), FOUND_FILES_AND_ERRORS

    def run_validation_on_content_entities(
        self, content_entity_dir_path, pack_error_ignore_list
    ):
        """Gets non-pack folder and runs validation within it (Scripts, Integrations...)

        Returns:
            bool. true if all files in directory are valid, false otherwise.
        """
        content_entities_validation_results = set()
        if content_entity_dir_path.endswith(
            GENERIC_FIELDS_DIR
        ) or content_entity_dir_path.endswith(GENERIC_TYPES_DIR):
            for dir_name in os.listdir(content_entity_dir_path):
                dir_path = os.path.join(content_entity_dir_path, dir_name)
                if not os.path.isfile(dir_path):
                    # should be only directories (not files) in generic types/fields directory
                    content_entities_validation_results.add(
                        self.run_validation_on_generic_entities(
                            dir_path, pack_error_ignore_list
                        )
                    )
                else:
                    self.ignored_files.add(dir_path)
        else:
            for file_name in os.listdir(content_entity_dir_path):
                file_path = os.path.join(content_entity_dir_path, file_name)
                if os.path.isfile(file_path):
                    if (
                        file_path.endswith(".json")
                        or file_path.endswith(".yml")
                        or file_path.endswith(".md")
                        or (
                            content_entity_dir_path.endswith(XSIAM_DASHBOARDS_DIR)
                            and file_path.endswith(".png")
                        )
                    ):
                        content_entities_validation_results.add(
                            self.run_validations_on_file(
                                file_path, pack_error_ignore_list
                            )
                        )
                    else:
                        self.ignored_files.add(file_path)

                else:
                    content_entities_validation_results.add(
                        self.run_validation_on_package(
                            file_path, pack_error_ignore_list
                        )
                    )

        return all(content_entities_validation_results)

    @staticmethod
    def should_validate_xsiam_content(package_path):
        parent_name = Path(package_path).stem
        dir_name = Path(package_path).parent.stem
        return (
            parent_name in {"XSIAMDashboards", "XSIAMReports"}
            or dir_name == "XDRCTemplates"
        )

    def run_validation_on_package(self, package_path, pack_error_ignore_list):
        package_entities_validation_results = set()
        for file_name in os.listdir(package_path):
            file_path = os.path.join(package_path, file_name)
            package_entities_validation_results.add(
                self.run_validations_on_file(file_path, pack_error_ignore_list)
            )

        return all(package_entities_validation_results)

    def run_validation_on_generic_entities(self, dir_path, pack_error_ignore_list):
        """
        Gets a generic content entity directory (i.e a sub-directory of GenericTypes or GenericFields)
        and runs validation within it.

        Returns:
            bool. true if all files in directory are valid, false otherwise.
        """
        package_entities_validation_results = set()

        for file_name in os.listdir(dir_path):
            file_path = os.path.join(dir_path, file_name)
            if file_path.endswith(".json"):  # generic types/fields are jsons
                package_entities_validation_results.add(
                    self.run_validations_on_file(file_path, pack_error_ignore_list)
                )
            else:
                self.ignored_files.add(file_path)

        return all(package_entities_validation_results)

    @error_codes("BA114")
    def is_valid_pack_name(self, file_path, old_file_path, ignored_errors):
        """
        Valid pack name is currently considered to be a new pack name or an existing pack.
        If pack name is changed, will return `False`.
        """
        if not old_file_path:
            return True
        original_pack_name = get_pack_name(old_file_path)
        new_pack_name = get_pack_name(file_path)
        if original_pack_name != new_pack_name:
            error_message, error_code = Errors.changed_pack_name(original_pack_name)
            if self.handle_error(
                error_message=error_message,
                error_code=error_code,
                file_path=file_path,
                drop_line=True,
                ignored_errors=ignored_errors,
            ):
                return False
        return True

    @error_codes("BA102,IM110")
    def is_valid_file_type(
        self, file_type: FileType, file_path: str, ignored_errors: dict
    ):
        """
        If a file_type is unsupported, will return `False`.
        """
        if not file_type:
            error_message, error_code = Errors.file_type_not_supported(
                file_type, file_path
            )
            if str(file_path).endswith(".png"):
                error_message, error_code = Errors.invalid_image_name_or_location()
            if self.handle_error(
                error_message=error_message,
                error_code=error_code,
                file_path=file_path,
                drop_line=True,
                ignored_errors=ignored_errors,
            ):
                return False

        return True

    def is_skipped_file(self, file_path: str) -> bool:
        """check wether the file in the given file_path is in the SKIPPED_FILES list.

        Args:
            file_path: the file on which to run.

        Returns:
            bool. true if file is in SKIPPED_FILES list, false otherwise.
        """
        path = Path(file_path)
        if get_log_file() == path:
            return True
        return path.name in SKIPPED_FILES or (
            path.name == "CommonServerPython.py" and path.parent.parent.name != "Base"
        )

    # flake8: noqa: C901
    def run_validations_on_file(
        self,
        file_path,
        pack_error_ignore_list,
        is_modified=False,
        old_file_path=None,
        modified_files=None,
        added_files=None,
    ):
        """Choose a validator to run for a single file. (i)

        Args:
            modified_files: A set of modified files - used for RN validation
            added_files: A set of added files - used for RN validation
            old_file_path: The old file path for renamed files
            pack_error_ignore_list: A dictionary of all pack ignored errors
            file_path: the file on which to run.
            is_modified: whether the file is modified or added.

        Returns:
            bool. true if file is valid, false otherwise.
        """
        if not self.is_valid_pack_name(
            file_path, old_file_path, pack_error_ignore_list
        ):
            return False
        file_type = find_type(file_path)

        is_added_file = file_path in added_files if added_files else False
        if file_type == FileType.MODELING_RULE_TEST_DATA:
            file_path = file_path.replace("_testdata.json", ".yml")
        if file_path.endswith(".xif"):
            file_path = file_path.replace(".xif", ".yml")
        if (
            file_type in self.skipped_file_types
            or self.is_skipped_file(file_path)
            or (self.git_util and self.git_util._is_file_git_ignored(file_path))
            or self.detect_file_level(file_path)
            in (PathLevel.PACKAGE, PathLevel.CONTENT_ENTITY_DIR)
        ):
            self.ignored_files.add(file_path)
            return True
        elif not self.is_valid_file_type(file_type, file_path, pack_error_ignore_list):
            return False

        if file_type == FileType.XSOAR_CONFIG:
            xsoar_config_validator = XSOARConfigJsonValidator(
                file_path,
                specific_validations=self.specific_validations,
                ignored_errors=pack_error_ignore_list,
            )
            return xsoar_config_validator.is_valid_xsoar_config_file()

        if not self.check_only_schema:
            validation_print = f"\nValidating {file_path} as {file_type.value}"
            if self.print_percent:
                if FOUND_FILES_AND_ERRORS:
                    validation_print += f" [red][{self.completion_percentage}%][/red]"
                else:
                    validation_print += (
                        f" [green][{self.completion_percentage}%][/green]"
                    )

            logger.info(validation_print)

        structure_validator = StructureValidator(
            file_path,
            predefined_scheme=file_type,
            ignored_errors=pack_error_ignore_list,
            tag=self.prev_ver,
            old_file_path=old_file_path,
            branch_name=self.branch_name,
            is_new_file=not is_modified,
            json_file_path=self.json_file_path,
            skip_schema_check=self.skip_schema_check,
            pykwalify_logs=self.pykwalify_logs,
            quiet_bc=self.quiet_bc,
            specific_validations=self.specific_validations,
        )

        # schema validation
        if file_type not in {
            FileType.TEST_PLAYBOOK,
            FileType.TEST_SCRIPT,
            FileType.DESCRIPTION,
        }:
            if not structure_validator.is_valid_file():
                return False

        # Passed schema validation
        # if only schema validation is required - stop check here
        if self.check_only_schema:
            return True

        # id_set validation
        if (
            self.id_set_validations
            and not self.id_set_validations.is_file_valid_in_set(
                file_path, file_type, pack_error_ignore_list
            )
        ):
            return False

        # conf.json validation
        valid_in_conf = True
        if self.check_is_unskipped and file_type in {
            FileType.INTEGRATION,
            FileType.SCRIPT,
            FileType.BETA_INTEGRATION,
        }:
            if not self.conf_json_validator.is_valid_file_in_conf_json(
                structure_validator.current_file,
                file_type,
                file_path,
                pack_error_ignore_list,
            ):
                valid_in_conf = False

        # Note: these file are not ignored but there are no additional validators for connections
        if file_type == FileType.CONNECTION:
            return True

        # test playbooks and test scripts are using the same validation.
        elif file_type in {FileType.TEST_PLAYBOOK, FileType.TEST_SCRIPT}:
            return self.validate_test_playbook(
                structure_validator, pack_error_ignore_list
            )

        elif file_type == FileType.RELEASE_NOTES:
            if not self.skip_pack_rn_validation:
                return self.validate_release_notes(
                    file_path,
                    added_files,
                    modified_files,
                    pack_error_ignore_list,
                )
            else:
                logger.info("[yellow]Skipping release notes validation[/yellow]")

        elif file_type == FileType.RELEASE_NOTES_CONFIG:
            return self.validate_release_notes_config(file_path, pack_error_ignore_list)

        elif file_type == FileType.DESCRIPTION:
            return self.validate_description(file_path, pack_error_ignore_list)

        elif file_type == FileType.README:
            if not self.is_possible_validate_readme:
                error_message, error_code = Errors.error_uninstall_node()
                if self.handle_error(
                    error_message=error_message,
                    error_code=error_code,
                    file_path=file_path,
                    ignored_errors=pack_error_ignore_list,
                ):
                    return False
            if not self.validate_all:
                ReadMeValidator.add_node_env_vars()
                if (
                    not ReadMeValidator.are_modules_installed_for_verify(
                        CONTENT_PATH  # type: ignore
                    )
                    and not ReadMeValidator.is_docker_available()
                ):  # shows warning message
                    return True
                with ReadMeValidator.start_mdx_server(handle_error=self.handle_error):
                    return self.validate_readme(file_path, pack_error_ignore_list)
            return self.validate_readme(file_path, pack_error_ignore_list)

        elif file_type == FileType.REPORT:
            return self.validate_report(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.PLAYBOOK:
            return self.validate_playbook(
                structure_validator, pack_error_ignore_list, file_type, is_modified
            )

        elif file_type == FileType.INTEGRATION:
            return all(
                [
                    self.validate_integration(
                        structure_validator,
                        pack_error_ignore_list,
                        is_modified,
                        file_type,
                    ),
                    valid_in_conf,
                ]
            )

        elif file_type == FileType.SCRIPT:
            return all(
                [
                    self.validate_script(
                        structure_validator,
                        pack_error_ignore_list,
                        is_modified,
                        file_type,
                    ),
                    valid_in_conf,
                ]
            )
        elif file_type == FileType.PYTHON_FILE:
            return self.validate_python_file(file_path, pack_error_ignore_list)

        elif file_type == FileType.BETA_INTEGRATION:
            return self.validate_beta_integration(
                structure_validator, pack_error_ignore_list
            )

        # Validate only images of packs
        elif file_type == FileType.IMAGE:
            return self.validate_image(file_path, pack_error_ignore_list)

        elif file_type == FileType.AUTHOR_IMAGE:
            return self.validate_author_image(file_path, pack_error_ignore_list)

        elif file_type == FileType.INCIDENT_FIELD:
            return self.validate_incident_field(
                structure_validator, pack_error_ignore_list, is_modified, is_added_file
            )

        elif file_type == FileType.INDICATOR_FIELD:
            return self.validate_indicator_field(
                structure_validator, pack_error_ignore_list, is_modified, is_added_file
            )

        elif file_type == FileType.REPUTATION:
            return self.validate_reputation(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.LAYOUT:
            return self.validate_layout(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.LAYOUTS_CONTAINER:
            return self.validate_layoutscontainer(
                structure_validator, pack_error_ignore_list
            )

        elif file_type == FileType.PRE_PROCESS_RULES:
            return self.validate_pre_process_rule(
                structure_validator, pack_error_ignore_list
            )

        elif file_type == FileType.LISTS:
            return self.validate_lists(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.DASHBOARD:
            return self.validate_dashboard(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.INCIDENT_TYPE:
            return self.validate_incident_type(
                structure_validator, pack_error_ignore_list, is_modified
            )

        elif file_type == FileType.MAPPER:
            return self.validate_mapper(
                structure_validator, pack_error_ignore_list, is_modified
            )

        elif file_type in (FileType.OLD_CLASSIFIER, FileType.CLASSIFIER):
            return self.validate_classifier(
                structure_validator, pack_error_ignore_list, file_type
            )

        elif file_type == FileType.WIDGET:
            return self.validate_widget(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.TRIGGER:
            return self.validate_triggers(structure_validator, pack_error_ignore_list)

        elif file_type in (
            FileType.PARSING_RULE,
            FileType.PARSING_RULE_XIF,
        ):
            return self.validate_parsing_rule(
                structure_validator, pack_error_ignore_list
            )

        elif file_type in (
            FileType.MODELING_RULE,
            FileType.MODELING_RULE_XIF,
            FileType.MODELING_RULE_TEST_DATA,
        ):
            logger.info(f"Validating {file_type.value} file: {file_path}")
            if self.validate_all:
                error_ignore_list = pack_error_ignore_list.copy()
                error_ignore_list.setdefault(os.path.basename(file_path), [])
                error_ignore_list.get(os.path.basename(file_path)).append("MR104")
                return self.validate_modeling_rule(
                    structure_validator, error_ignore_list
                )
            return self.validate_modeling_rule(
                structure_validator, pack_error_ignore_list
            )

        elif file_type == FileType.CORRELATION_RULE:
            return self.validate_correlation_rule(
                structure_validator, pack_error_ignore_list
            )

        elif file_type in {FileType.XSIAM_DASHBOARD, FileType.XSIAM_DASHBOARD_IMAGE}:
            return self.validate_xsiam_dashboard(
                structure_validator, pack_error_ignore_list
            )

        elif file_type in {FileType.XDRC_TEMPLATE, FileType.XDRC_TEMPLATE_YML}:
            return self.validate_xdrc_templates(
                structure_validator, pack_error_ignore_list
            )

        elif file_type == FileType.XSIAM_REPORT:
            return self.validate_xsiam_report(
                structure_validator, pack_error_ignore_list
            )

        elif file_type == FileType.WIZARD:
            return self.validate_wizard(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.GENERIC_FIELD:
            return self.validate_generic_field(
                structure_validator, pack_error_ignore_list, is_added_file
            )

        elif file_type == FileType.GENERIC_TYPE:
            return self.validate_generic_type(
                structure_validator, pack_error_ignore_list
            )

        elif file_type == FileType.GENERIC_MODULE:
            return self.validate_generic_module(
                structure_validator, pack_error_ignore_list
            )

        elif file_type == FileType.GENERIC_DEFINITION:
            return self.validate_generic_definition(
                structure_validator, pack_error_ignore_list
            )

        elif file_type == FileType.JOB:
            return self.validate_job(structure_validator, pack_error_ignore_list)

        elif file_type == FileType.LAYOUT_RULE:
            return self.validate_layout_rules(
                structure_validator, pack_error_ignore_list
            )

        elif file_type == FileType.CONTRIBUTORS:
            # This is temporarily - need to add a proper contributors validations
            return True

        elif file_type == FileType.CONF_JSON and not self.skip_conf_json:
            return self.validate_conf_json()

        else:
            return self.file_type_not_supported(
                file_type, file_path, pack_error_ignore_list
            )
        return True

    @error_codes("BA102")
    def file_type_not_supported(
        self, file_type: FileType, file_path: str, ignored_errors: dict
    ):
        error_message, error_code = Errors.file_type_not_supported(file_type, file_path)
        if self.handle_error(
            error_message=error_message,
            error_code=error_code,
            file_path=file_path,
            ignored_errors=ignored_errors,
        ):
            return False
        return True

    @staticmethod
    def get_file_by_status(
        modified_files: Set, old_format_files: Set, file_path: str
    ) -> Tuple[Set, Set, Set]:
        """Given a specific file path identify in which git status set
        it exists and return a set containing that file and 2 additional empty sets.

        Args:
            modified_files(Set): A set of modified and renamed files.
            old_format_files(Set): A set of old format files.
            file_path(str): The file path to check.

        Returns:
            Tuple[Set, Set, Set]. 3 sets representing modified, added or old format files respectively
            where the file path is in the appropriate set
        """
        filtered_modified_files: Set = set()
        filtered_added_files: Set = set()
        filtered_old_format: Set = set()

        # go through modified files and try to identify if the file is there
        for file in modified_files:
            if isinstance(file, str) and file == file_path:
                filtered_modified_files.add(file_path)
                return (
                    filtered_modified_files,
                    filtered_added_files,
                    filtered_old_format,
                )

            # handle renamed files which are in tuples
            elif file_path in file:
                filtered_modified_files.add(file)
                return (
                    filtered_modified_files,
                    filtered_added_files,
                    filtered_old_format,
                )

        # if the file is not modified check if it is in old format files
        if file_path in old_format_files:
            filtered_old_format.add(file_path)

        else:
            # if not found in either modified or old format consider the file newly added
            filtered_added_files.add(file_path)

        return filtered_modified_files, filtered_added_files, filtered_old_format

    @staticmethod
    def specify_files_from_directory(file_set: Set, directory_path: str) -> Set:
        """Filter a set of file paths to only include ones which are from a specified directory.

        Args:
            file_set(Set): A set of file paths - could be stings or tuples for rename files.
            directory_path(str): the directory path in which to check for the files.

        Returns:
            Set. A set of all the paths of files that appear in the given directory.
        """
        filtered_set: Set = set()
        for file in file_set:
            if isinstance(file, str) and directory_path in file:
                filtered_set.add(file)

            # handle renamed files
            elif isinstance(file, tuple) and directory_path in file[1]:
                filtered_set.add(file)

        return filtered_set

    def specify_files_by_status(
        self, modified_files: Set, added_files: Set, old_format_files: Set
    ) -> Tuple[Set, Set, Set]:
        """Filter the files identified from git to only specified files.

        Args:
            modified_files(Set): A set of modified and renamed files.
            added_files(Set): A set of added files.
            old_format_files(Set): A set of old format files.

        Returns:
            Tuple[Set, Set, Set]. 3 sets for modified, added an old format files where the only files that
            appear are the ones specified by the 'file_path' ValidateManager parameter
        """
        filtered_modified_files: Set = set()
        filtered_added_files: Set = set()
        filtered_old_format: Set = set()

        for path in self.file_path.split(","):
            path = get_relative_path_from_packs_dir(path)
            file_level = self.detect_file_level(path)
            if file_level == PathLevel.FILE:
                temp_modified, temp_added, temp_old_format = self.get_file_by_status(
                    modified_files, old_format_files, path
                )
                filtered_modified_files = filtered_modified_files.union(temp_modified)
                filtered_added_files = filtered_added_files.union(temp_added)
                filtered_old_format = filtered_old_format.union(temp_old_format)

            else:
                filtered_modified_files = filtered_modified_files.union(
                    self.specify_files_from_directory(modified_files, path)
                )
                filtered_added_files = filtered_added_files.union(
                    self.specify_files_from_directory(added_files, path)
                )
                filtered_old_format = filtered_old_format.union(
                    self.specify_files_from_directory(old_format_files, path)
                )

        return filtered_modified_files, filtered_added_files, filtered_old_format

    def run_validation_using_git(self):
        """Runs validation on only changed packs/files (g)"""
        valid_git_setup = self.setup_git_params()
        if not self.no_configuration_prints:
            self.print_git_config()

        (
            modified_files,
            added_files,
            changed_meta_files,
            old_format_files,
            valid_types,
        ) = self.get_changed_files_from_git()

        # filter to only specified paths if given
        if self.file_path:
            (
                modified_files,
                added_files,
                old_format_files,
            ) = self.specify_files_by_status(
                modified_files, added_files, old_format_files
            )
        deleted_files = self.git_util.deleted_files(
            prev_ver=self.prev_ver,
            committed_only=self.is_circle,
            staged_only=self.staged,
            include_untracked=self.include_untracked,
        )

        validation_results = {valid_git_setup, valid_types}

        validation_results.add(
            self.validate_modified_files(modified_files | old_format_files)
        )
        validation_results.add(self.validate_added_files(added_files, modified_files))
        validation_results.add(
            self.validate_changed_packs_unique_files(
                modified_files, added_files, old_format_files, changed_meta_files
            )
        )
        validation_results.add(self.validate_deleted_files(deleted_files, added_files))
        logger.debug("*** after adding validate_deleted_files")

        logger.debug(
            f"*** Before ifs, {old_format_files=}, {self.skip_pack_rn_validation=}"
        )
        if old_format_files:
            logger.info(
                "\n[cyan]================= Running validation on old format files =================[/cyan]"
            )
            validation_results.add(self.validate_no_old_format(old_format_files))
            logger.info("*** after adding validate_no_old_format")

        if not self.skip_pack_rn_validation:
            logger.info("*** adding validate_no_duplicated_release_notes")
            validation_results.add(
                self.validate_no_duplicated_release_notes(added_files)
            )
            logger.info("*** after adding validate_no_duplicated_release_notes")

        all_files_set = list(
            set().union(
                modified_files, added_files, old_format_files, changed_meta_files
            )
        )
        if self.validate_graph:
            logger.info(
                "\n[cyan]================= Validating graph =================[/cyan]"
            )
            if all_files_set:
                with GraphValidator(
                    specific_validations=self.specific_validations,
                    git_files=all_files_set,
                    include_optional_deps=True,
                ) as graph_validator:
                    validation_results.add(graph_validator.is_valid_content_graph())
                    validation_results.add(
                        self.validate_no_missing_release_notes(
                            modified_files,
                            old_format_files,
                            added_files,
                            graph_validator,
                        )
                    )

        if self.packs_with_mp_change:
            logger.info(
                "\n[cyan]================= Running validation on Marketplace Changed Packs =================[/cyan]"
            )
            logger.debug(
                f"Found marketplace change in the following packs: {self.packs_with_mp_change}"
            )

            for mp_changed_metadata_pack in self.packs_with_mp_change:
                # Running validation on the whole pack, excluding files that were already checked.
                validation_results.add(
                    self.run_validations_on_pack(
                        mp_changed_metadata_pack, skip_files=set(all_files_set)
                    )[0]
                )

            logger.debug("Finished validating marketplace changed packs.")

        return all(validation_results)

    """ ######################################## Unique Validations ####################################### """

    def validate_description(self, file_path, pack_error_ignore_list):
        description_validator = DescriptionValidator(
            file_path,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
            specific_validations=self.specific_validations,
        )
        return description_validator.is_valid_file()

    def validate_readme(self, file_path, pack_error_ignore_list):
        readme_validator = ReadMeValidator(
            file_path,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
            specific_validations=self.specific_validations,
        )
        return readme_validator.is_valid_file()

    def validate_python_file(self, file_path, pack_error_ignore_list):
        python_file_validator = PythonFileValidator(
            file_path,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
            specific_validations=self.specific_validations,
        )
        return python_file_validator.is_valid_file()

    def validate_test_playbook(self, structure_validator, pack_error_ignore_list):
        test_playbook_validator = TestPlaybookValidator(
            structure_validator=structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return test_playbook_validator.is_valid_test_playbook(validate_rn=False)

    @error_codes("RN108")
    def validate_no_release_notes_for_new_pack(
        self, pack_name, file_path, ignored_errors
    ):
        if pack_name in self.new_packs:
            error_message, error_code = Errors.added_release_notes_for_new_pack(
                pack_name
            )
            if self.handle_error(
                error_message=error_message,
                error_code=error_code,
                file_path=file_path,
                ignored_errors=ignored_errors,
            ):
                return False
        return True

    def validate_release_notes(
        self,
        file_path,
        added_files,
        modified_files,
        pack_error_ignore_list,
    ):
        pack_name = get_pack_name(file_path)

        # added new RN to a new pack
        if not self.validate_no_release_notes_for_new_pack(
            pack_name, file_path, pack_error_ignore_list
        ):
            return False

        if pack_name != "NonSupported":
            if not added_files:
                added_files = {file_path}

            release_notes_validator = ReleaseNotesValidator(
                file_path,
                pack_name=pack_name,
                modified_files=modified_files,
                added_files=added_files,
                ignored_errors=pack_error_ignore_list,
                json_file_path=self.json_file_path,
                specific_validations=self.specific_validations,
            )
            return release_notes_validator.is_file_valid()

        return True

    def validate_release_notes_config(
        self, file_path: str, pack_error_ignore_list: list
    ) -> bool:
        """
        Builds validator for RN config file and returns its validation results.
        Args:
            file_path (str): Path to RN config file.
            pack_error_ignore_list (list): Pack error ignore list.

        Returns:
            (bool): Whether RN config file is valid.
        """
        pack_name = get_pack_name(file_path)
        if pack_name == "NonSupported":
            return True
        release_notes_config_validator = ReleaseNotesConfigValidator(
            file_path,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
            specific_validations=self.specific_validations,
        )
        return release_notes_config_validator.is_file_valid()

    def validate_playbook(
        self, structure_validator, pack_error_ignore_list, file_type, is_modified
    ):
        playbook_validator = PlaybookValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
            validate_all=self.validate_all,
            deprecation_validator=self.deprecation_validator,
        )

        deprecated_result = self.check_and_validate_deprecated(
            file_type=file_type,
            file_path=structure_validator.file_path,
            current_file=playbook_validator.current_file,
            is_modified=True,
            is_backward_check=False,
            validator=playbook_validator,
        )
        if deprecated_result is not None:
            return deprecated_result

        return playbook_validator.is_valid_playbook(
            validate_rn=False, id_set_file=self.id_set_file, is_modified=is_modified
        )

    def validate_integration(
        self, structure_validator, pack_error_ignore_list, is_modified, file_type
    ):
        integration_validator = IntegrationValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            skip_docker_check=self.skip_docker_checks,
            json_file_path=self.json_file_path,
            validate_all=self.validate_all,
            deprecation_validator=self.deprecation_validator,
        )

        deprecated_result = self.check_and_validate_deprecated(
            file_type=file_type,
            file_path=structure_validator.file_path,
            current_file=integration_validator.current_file,
            is_modified=is_modified,
            is_backward_check=self.is_backward_check,
            validator=integration_validator,
        )
        if deprecated_result is not None:
            return deprecated_result
        if is_modified and self.is_backward_check:
            return all(
                [
                    integration_validator.is_valid_file(
                        validate_rn=False,
                        skip_test_conf=self.skip_conf_json,
                        check_is_unskipped=self.check_is_unskipped,
                        conf_json_data=self.conf_json_data,
                        is_modified=is_modified,
                    ),
                    integration_validator.is_backward_compatible(),
                ]
            )
        else:
            return integration_validator.is_valid_file(
                validate_rn=False,
                skip_test_conf=self.skip_conf_json,
                check_is_unskipped=self.check_is_unskipped,
                conf_json_data=self.conf_json_data,
            )

    def validate_script(
        self, structure_validator, pack_error_ignore_list, is_modified, file_type
    ):
        script_validator = ScriptValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            skip_docker_check=self.skip_docker_checks,
            json_file_path=self.json_file_path,
            validate_all=self.validate_all,
            deprecation_validator=self.deprecation_validator,
        )

        deprecated_result = self.check_and_validate_deprecated(
            file_type=file_type,
            file_path=structure_validator.file_path,
            current_file=script_validator.current_file,
            is_modified=is_modified,
            is_backward_check=self.is_backward_check,
            validator=script_validator,
        )
        if deprecated_result is not None:
            return deprecated_result

        if is_modified and self.is_backward_check:
            return all(
                [
                    script_validator.is_valid_file(validate_rn=False),
                    script_validator.is_backward_compatible(),
                ]
            )
        else:
            return script_validator.is_valid_file(validate_rn=False)

    def validate_beta_integration(self, structure_validator, pack_error_ignore_list):
        integration_validator = IntegrationValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            skip_docker_check=self.skip_docker_checks,
            json_file_path=self.json_file_path,
            validate_all=self.validate_all,
        )
        return integration_validator.is_valid_beta_integration()

    def validate_image(self, file_path, pack_error_ignore_list):
        pack_name = get_pack_name(file_path)
        if pack_name == "NonSupported":
            return True
        image_validator = ImageValidator(
            file_path,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
            specific_validations=self.specific_validations,
        )
        return image_validator.is_valid()

    def validate_author_image(self, file_path, pack_error_ignore_list):
        author_image_validator: AuthorImageValidator = AuthorImageValidator(
            file_path,
            ignored_errors=pack_error_ignore_list,
            specific_validations=self.specific_validations,
        )
        return author_image_validator.is_valid()

    def validate_report(self, structure_validator, pack_error_ignore_list):
        report_validator = ReportValidator(
            structure_validator=structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return report_validator.is_valid_file(validate_rn=False)

    def validate_incident_field(
        self, structure_validator, pack_error_ignore_list, is_modified, is_added_file
    ):
        incident_field_validator = IncidentFieldValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
            id_set_file=self.id_set_file,
        )
        if is_modified and self.is_backward_check:
            return all(
                [
                    incident_field_validator.is_valid_file(
                        validate_rn=False,
                        is_new_file=not is_modified,
                        use_git=self.use_git,
                        is_added_file=is_added_file,
                    ),
                    incident_field_validator.is_backward_compatible(),
                ]
            )
        else:
            return incident_field_validator.is_valid_file(
                validate_rn=False,
                is_new_file=not is_modified,
                use_git=self.use_git,
                is_added_file=is_added_file,
            )

    def validate_indicator_field(
        self, structure_validator, pack_error_ignore_list, is_modified, is_added_file
    ):
        indicator_field_validator = IndicatorFieldValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        if is_modified and self.is_backward_check:
            return all(
                [
                    indicator_field_validator.is_valid_file(
                        validate_rn=False,
                        is_new_file=not is_modified,
                        use_git=self.use_git,
                        is_added_file=is_added_file,
                    ),
                    indicator_field_validator.is_backward_compatible(),
                ]
            )
        else:
            return indicator_field_validator.is_valid_file(
                validate_rn=False,
                is_new_file=not is_modified,
                use_git=self.use_git,
                is_added_file=is_added_file,
            )

    def validate_reputation(self, structure_validator, pack_error_ignore_list):
        reputation_validator = ReputationValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return reputation_validator.is_valid_file(validate_rn=False)

    def validate_layout(self, structure_validator, pack_error_ignore_list):
        layout_validator = LayoutValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return layout_validator.is_valid_layout(
            validate_rn=False, id_set_file=self.id_set_file, is_circle=self.is_circle
        )

    def validate_layoutscontainer(self, structure_validator, pack_error_ignore_list):
        layout_validator = LayoutsContainerValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return layout_validator.is_valid_layout(
            validate_rn=False, id_set_file=self.id_set_file, is_circle=self.is_circle
        )

    def validate_pre_process_rule(self, structure_validator, pack_error_ignore_list):
        pre_process_rules_validator = PreProcessRuleValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return pre_process_rules_validator.is_valid_pre_process_rule(
            validate_rn=False, id_set_file=self.id_set_file, is_ci=self.is_circle
        )

    def validate_lists(self, structure_validator, pack_error_ignore_list):
        lists_validator = ListsValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return lists_validator.is_valid_list()

    def validate_dashboard(self, structure_validator, pack_error_ignore_list):
        dashboard_validator = DashboardValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return dashboard_validator.is_valid_dashboard(validate_rn=False)

    def validate_incident_type(
        self, structure_validator, pack_error_ignore_list, is_modified
    ):
        incident_type_validator = IncidentTypeValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        if is_modified and self.is_backward_check:
            return all(
                [
                    incident_type_validator.is_valid_incident_type(validate_rn=False),
                    incident_type_validator.is_backward_compatible(),
                ]
            )
        else:
            return incident_type_validator.is_valid_incident_type(validate_rn=False)

    def validate_mapper(self, structure_validator, pack_error_ignore_list, is_modified):
        mapper_validator = MapperValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        if is_modified and self.is_backward_check:
            return all(
                [
                    mapper_validator.is_valid_mapper(
                        validate_rn=False,
                        id_set_file=self.id_set_file,
                        is_circle=self.is_circle,
                    ),
                    mapper_validator.is_backward_compatible(),
                ]
            )

        return mapper_validator.is_valid_mapper(
            validate_rn=False, id_set_file=self.id_set_file, is_circle=self.is_circle
        )

    def validate_classifier(
        self, structure_validator, pack_error_ignore_list, file_type
    ):
        if file_type == FileType.CLASSIFIER:
            new_classifier_version = True

        else:
            new_classifier_version = False

        classifier_validator = ClassifierValidator(
            structure_validator,
            new_classifier_version=new_classifier_version,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return classifier_validator.is_valid_classifier(
            validate_rn=False, id_set_file=self.id_set_file, is_circle=self.is_circle
        )

    def validate_widget(self, structure_validator, pack_error_ignore_list):
        widget_validator = WidgetValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return widget_validator.is_valid_file(validate_rn=False)

    def validate_triggers(self, structure_validator, pack_error_ignore_list):
        triggers_validator = TriggersValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return triggers_validator.is_valid_file(validate_rn=False)

    def validate_layout_rules(self, structure_validator, pack_error_ignore_list):
        layout_rules_validator = LayoutRuleValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return layout_rules_validator.is_valid_file(validate_rn=False)

    def validate_conf_json(self):
        conf_json_validator = ConfJsonValidator()
        return conf_json_validator.is_valid_conf_json()

    def validate_xsiam_report(self, structure_validator, pack_error_ignore_list):
        xsiam_report_validator = XSIAMReportValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return xsiam_report_validator.is_valid_file(validate_rn=False)

    def validate_xsiam_dashboard(self, structure_validator, pack_error_ignore_list):
        xsiam_dashboard_validator = XSIAMDashboardValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return xsiam_dashboard_validator.is_valid_file(validate_rn=False)

    def validate_xdrc_templates(self, structure_validator, pack_error_ignore_list):
        xdrc_templates_validator = XDRCTemplatesValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return xdrc_templates_validator.is_valid_file(validate_rn=False)

    def validate_parsing_rule(self, structure_validator, pack_error_ignore_list):
        parsing_rule_validator = ParsingRuleValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return parsing_rule_validator.is_valid_file(validate_rn=False)

    def validate_correlation_rule(self, structure_validator, pack_error_ignore_list):
        correlation_rule_validator = CorrelationRuleValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return correlation_rule_validator.is_valid_file(validate_rn=False)

    def validate_modeling_rule(self, structure_validator, pack_error_ignore_list):
        modeling_rule_validator = ModelingRuleValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return modeling_rule_validator.is_valid_file(validate_rn=False)

    def validate_wizard(self, structure_validator, pack_error_ignore_list):
        wizard_validator = WizardValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        return wizard_validator.is_valid_file(
            validate_rn=False, id_set_file=self.id_set_file
        )

    def validate_generic_field(
        self, structure_validator, pack_error_ignore_list, is_added_file
    ):
        generic_field_validator = GenericFieldValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )

        return generic_field_validator.is_valid_file(
            validate_rn=False, is_added_file=is_added_file
        )

    def validate_generic_type(self, structure_validator, pack_error_ignore_list):
        generic_type_validator = GenericTypeValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )

        return generic_type_validator.is_valid_file(validate_rn=False)

    def validate_generic_module(self, structure_validator, pack_error_ignore_list):
        generic_module_validator = GenericModuleValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )

        return generic_module_validator.is_valid_file(validate_rn=False)

    def validate_generic_definition(self, structure_validator, pack_error_ignore_list):
        generic_definition_validator = GenericDefinitionValidator(
            structure_validator,
            ignored_errors=pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )

        return generic_definition_validator.is_valid_file(validate_rn=False)

    def validate_pack_unique_files(
        self, pack_path: str, pack_error_ignore_list: dict, should_version_raise=False
    ) -> bool:
        """
        Runs validations on the following pack files:
        * .secret-ignore: Validates that the file exist and that the file's secrets can be parsed as a list delimited by '\n'
        * .pack-ignore: Validates that the file exists and that all regexes in it can be compiled
        * README.md file: Validates that the file exists and image links are valid
        * 2.pack_metadata.json: Validates that the file exists and that it has a valid structure
        * Author_image.png: Validates that the file isn't empty, dimension of 120*50 and that image size is up to 4kb
        Runs validation on the pack dependencies
        Args:
            should_version_raise: Whether we should check if the version of the metadata was raised
            pack_error_ignore_list: A dictionary of all pack ignored errors
            pack_path: A path to a pack
        """
        files_valid = True
        author_valid = True

        logger.info(f"\nValidating {pack_path} unique pack files")
        pack_unique_files_validator = PackUniqueFilesValidator(
            pack=os.path.basename(pack_path),
            pack_path=pack_path,
            ignored_errors=pack_error_ignore_list,
            should_version_raise=should_version_raise,
            validate_dependencies=not self.skip_dependencies,
            id_set_path=self.id_set_path,
            private_repo=self.is_external_repo,
            skip_id_set_creation=self.skip_id_set_creation,
            prev_ver=self.prev_ver,
            json_file_path=self.json_file_path,
            specific_validations=self.specific_validations,
        )
        pack_errors = pack_unique_files_validator.are_valid_files(
            self.id_set_validations
        )
        if pack_errors:
            files_valid = False

        # check author image
        author_image_path = os.path.join(pack_path, AUTHOR_IMAGE_FILE_NAME)
        if os.path.exists(author_image_path):
            logger.info("Validating pack author image")
            author_valid = self.validate_author_image(
                author_image_path, pack_error_ignore_list
            )

        if pack_unique_files_validator.check_metadata_for_marketplace_change():
            self.packs_with_mp_change = self.packs_with_mp_change.union({pack_path})

        return files_valid and author_valid

    def validate_job(self, structure_validator, pack_error_ignore_list):
        job_validator = JobValidator(
            structure_validator,
            pack_error_ignore_list,
            json_file_path=self.json_file_path,
        )
        is_valid = job_validator.is_valid_file()
        if not is_valid:
            job_validator_errors = job_validator.get_errors()
            logger.info(f"[red]{job_validator_errors}[/red]")

        return is_valid

    def validate_modified_files(self, modified_files):
        logger.info(
            "\n[cyan]================= Running validation on modified files =================[/cyan]"
        )
        valid_files = set()
        for file_path in modified_files:
            # handle renamed files
            if isinstance(file_path, tuple):
                old_file_path = file_path[0]
                file_path = file_path[1]

            else:
                old_file_path = None

            pack_name = get_pack_name(file_path)
            valid_files.add(
                self.run_validations_on_file(
                    file_path,
                    self.get_error_ignore_list(pack_name),
                    is_modified=True,
                    old_file_path=old_file_path,
                )
            )
        return all(valid_files)

    def validate_added_files(self, added_files, modified_files):
        logger.info(
            "\n[cyan]================= Running validation on newly added files =================[/cyan]"
        )

        valid_files = set()
        for file_path in added_files:
            pack_name = get_pack_name(file_path)
            valid_files.add(
                self.run_validations_on_file(
                    file_path,
                    self.get_error_ignore_list(pack_name),
                    is_modified=False,
                    modified_files=modified_files,
                    added_files=added_files,
                )
            )
        return all(valid_files)

    @staticmethod
    def should_raise_pack_version(pack: str) -> bool:
        """
        Args:
            pack: The pack name.

        Returns: False if pack is in IGNORED_PACK_NAMES else True.

        """
        return pack not in IGNORED_PACK_NAMES

    @staticmethod
    def is_file_allowed_to_be_deleted(file_path):
        """
        Args:
            file_path: The file path.

        Returns: True if the file allowed to be deleted, else False.

        """
        file_path = str(file_path)
        file_dict = get_remote_file(file_path, tag="master")
        file_type = find_type(file_path, file_dict)
        return file_type in FileType_ALLOWED_TO_DELETE or not file_type

    @staticmethod
    def was_file_renamed_but_labeled_as_deleted(deleted_file_path, added_files):
        """Check if a file was renamed and not deleted (git false label the file as deleted)
        Args:
            file_path: The file path.

        Returns: True if the file was renamed and not deleted, else False.

        """
        if added_files:
            deleted_file_path = str(deleted_file_path)
            deleted_file_dict = get_remote_file(
                deleted_file_path, tag="master"
            )  # for detecting deleted files
            if deleted_file_type := find_type(deleted_file_path, deleted_file_dict):
                deleted_file_id = _get_file_id(
                    deleted_file_type.value, deleted_file_dict
                )
                if deleted_file_id:
                    for file in added_files:
                        file = str(file)
                        file_type = find_type(file)
                        if file_type == deleted_file_type:
                            file_dict = get_file(file)
                            if deleted_file_id == _get_file_id(
                                file_type.value, file_dict
                            ):
                                return True
        return False

    @error_codes("BA115")
    def validate_deleted_files(self, deleted_files: set, added_files: set) -> bool:
        logger.info(
            "\n[cyan]================= Checking for prohibited deleted files =================[/cyan]"
        )

        is_valid = True
        for file_path in deleted_files:
            file_path = Path(file_path)
            ignored_errors = self.get_error_ignore_list(get_pack_name(str(file_path)))
            if "Packs" not in file_path.absolute().parts:
                # not allowed to delete non-content files
                file_path = str(file_path)
                error_message, error_code = Errors.file_cannot_be_deleted(file_path)
                if self.handle_error(
                    error_message, error_code, file_path, ignored_errors=ignored_errors
                ):
                    is_valid = False

            else:
                if not self.was_file_renamed_but_labeled_as_deleted(
                    file_path, added_files
                ):
                    if not self.is_file_allowed_to_be_deleted(file_path):
                        file_path = str(file_path)
                        error_message, error_code = Errors.file_cannot_be_deleted(
                            file_path
                        )
                        if self.handle_error(
                            error_message,
                            error_code,
                            file_path,
                            ignored_errors=ignored_errors,
                        ):
                            is_valid = False

        return is_valid

    def validate_changed_packs_unique_files(
        self, modified_files, added_files, old_format_files, changed_meta_files
    ):
        logger.info(
            "\n[cyan]================= Running validation on changed pack unique files =================[/cyan]"
        )
        valid_pack_files = set()

        added_packs = get_pack_names_from_files(added_files)
        modified_packs = get_pack_names_from_files(modified_files).union(
            get_pack_names_from_files(old_format_files)
        )
        changed_meta_packs = get_pack_names_from_files(changed_meta_files)

        packs_that_should_have_version_raised = (
            self.get_packs_that_should_have_version_raised(
                modified_files, added_files, old_format_files
            )
        )

        changed_packs = modified_packs.union(added_packs).union(changed_meta_packs)

        for pack in changed_packs:
            raise_version = False
            pack_path = tools.pack_name_to_path(pack)
            if pack in packs_that_should_have_version_raised:
                raise_version = self.should_raise_pack_version(pack)
            valid_pack_files.add(
                self.validate_pack_unique_files(
                    pack_path,
                    self.get_error_ignore_list(pack),
                    should_version_raise=raise_version,
                )
            )

        return all(valid_pack_files)

    @error_codes("ST106")
    def validate_no_old_format(self, old_format_files):
        """Validate there are no files in the old format (unified yml file for the code and configuration
        for python integration).

        Args:
            old_format_files(set): file names which are in the old format.
        """
        handle_error = True
        for file_path in old_format_files:
            logger.info(f"Validating old-format file {file_path}")
            yaml_data = get_yaml(file_path)
            # we only fail on old format if no toversion (meaning it is latest) or if the ynl is not deprecated.
            if "toversion" not in yaml_data and not yaml_data.get("deprecated"):
                error_message, error_code = Errors.invalid_package_structure()
                ignored_errors = self.get_error_ignore_list(get_pack_name(file_path))
                if self.handle_error(
                    error_message,
                    error_code,
                    file_path=file_path,
                    ignored_errors=ignored_errors,
                ):
                    handle_error = False
        return handle_error

    @error_codes("RN105")
    def validate_no_duplicated_release_notes(self, added_files):
        """Validated that among the added files - there are no duplicated RN for the same pack.

        Args:
            added_files(set): The added files

        Returns:
            bool. True if no duplications found, false otherwise
        """
        logger.info(
            "\n[cyan]================= Verifying no duplicated release notes =================[/cyan]"
        )
        added_rn = set()
        for file in added_files:
            if find_type(file) == FileType.RELEASE_NOTES:
                pack_name = get_pack_name(file)
                if pack_name not in added_rn:
                    added_rn.add(pack_name)
                else:
                    error_message, error_code = Errors.multiple_release_notes_files()
                    ignored_errors = self.get_error_ignore_list(pack_name)
                    if self.handle_error(
                        error_message,
                        error_code,
                        file_path=pack_name,
                        ignored_errors=ignored_errors,
                    ):
                        return False

        logger.info("\n[green]No duplicated release notes found.[/green]\n")
        return True

    @error_codes("RN106")
    def validate_no_missing_release_notes(
        self, modified_files, old_format_files, added_files, graph_validator
    ):
        """Validate that there are no missing RN for changed files

        Args:
            modified_files (set): a set of modified files.
            old_format_files (set): a set of old format files that were changed.
            added_files (set): a set of files that were added.
            graph_validator : Content graph

        Returns:
            bool. True if no missing RN found, False otherwise
        """
        logger.info(
            "\n[cyan]================= Checking for missing release notes =================[/cyan]\n"
        )
        packs_that_should_have_new_rn_api_module_related: set = set()
        # existing packs that have files changed (which are not RN, README nor test files) - should have new RN
        changed_files = modified_files.union(old_format_files).union(added_files)
        packs_that_should_have_new_rn = get_pack_names_from_files(
            changed_files, skip_file_types=SKIP_RELEASE_NOTES_FOR_TYPES
        )
        if API_MODULES_PACK in packs_that_should_have_new_rn:
            api_module_set = get_api_module_ids(changed_files)
            integrations = get_api_module_dependencies_from_graph(
                api_module_set, graph_validator.graph
            )
            packs_that_should_have_new_rn_api_module_related = set(
                map(lambda integration: integration.pack_id, integrations)
            )
            packs_that_should_have_new_rn = packs_that_should_have_new_rn.union(
                packs_that_should_have_new_rn_api_module_related
            )

            # APIModules pack is without a version and should not have RN
            packs_that_should_have_new_rn.remove(API_MODULES_PACK)

        # new packs should not have RN
        packs_that_should_have_new_rn = packs_that_should_have_new_rn - self.new_packs

        packs_that_have_new_rn = self.get_packs_with_added_release_notes(added_files)

        packs_that_have_missing_rn = packs_that_should_have_new_rn.difference(
            packs_that_have_new_rn
        )

        if packs_that_have_missing_rn:
            is_valid = set()
            for pack in packs_that_have_missing_rn:
                # # ignore RN in NonSupported pack
                if "NonSupported" in pack:
                    continue
                ignored_errors_list = self.get_error_ignore_list(pack)
                if pack in packs_that_should_have_new_rn_api_module_related:
                    error_message, error_code = Errors.missing_release_notes_for_pack(
                        API_MODULES_PACK
                    )
                else:
                    error_message, error_code = Errors.missing_release_notes_for_pack(
                        pack
                    )
                if not BaseValidator(
                    ignored_errors=ignored_errors_list,
                    json_file_path=self.json_file_path,
                    specific_validations=self.specific_validations,
                ).handle_error(
                    error_message,
                    error_code,
                    file_path=os.path.join(
                        os.getcwd(), PACKS_DIR, pack, PACKS_PACK_META_FILE_NAME
                    ),
                ):
                    is_valid.add(True)

                else:
                    is_valid.add(False)

            return all(is_valid)

        else:
            logger.info("[green]No missing release notes found.[/green]\n")
            return True

    """ ######################################## Git Tools and filtering ####################################### """

    def setup_prev_ver(self, prev_ver: Optional[str]):
        """Setting up the prev_ver parameter"""
        # if prev_ver parameter is set, use it
        if prev_ver:
            return prev_ver

        # If git is connected - Use it to get prev_ver
        if self.git_util:
            # If demisto exists in remotes if so set prev_ver as 'demisto/master'
            if self.git_util.check_if_remote_exists("demisto"):
                return "demisto/master"

            # Otherwise, use git to get the primary branch
            _, branch = self.git_util.handle_prev_ver()
            return "origin/" + branch

        # Default to 'origin/master'
        return "origin/master"

    def setup_git_params(self):
        """Setting up the git relevant params"""
        self.branch_name = (
            self.git_util.get_current_git_branch_or_hash()
            if (self.git_util and not self.branch_name)
            else self.branch_name
        )

        # check remote validity
        if "/" in self.prev_ver and not self.git_util.check_if_remote_exists(
            self.prev_ver
        ):
            non_existing_remote = self.prev_ver.split("/")[0]
            logger.info(
                f"[red]Could not find remote {non_existing_remote} reverting to "
                f"{str(self.git_util.repo.remote())}[/red]"
            )
            self.prev_ver = self.prev_ver.replace(
                non_existing_remote, str(self.git_util.repo.remote())
            )

        # if running on release branch check against last release.
        if self.branch_name.startswith("21.") or self.branch_name.startswith("22."):
            self.skip_pack_rn_validation = True
            self.prev_ver = os.environ.get("GIT_SHA1")
            self.is_circle = True

            # when running against git while on release branch - show errors but don't fail the validation
            self.always_valid = True

        # On main or master don't check RN
        elif self.branch_name in ["master", "main"]:
            self.skip_pack_rn_validation = True
            error_message, error_code = Errors.running_on_master_with_git()
            if self.handle_error(
                error_message,
                error_code,
                file_path="General",
                warning=(not self.is_external_repo or self.is_circle),
                drop_line=True,
            ):
                return False
        return True

    def print_git_config(self):
        logger.info(
            f"\n[cyan]================= Running validation on branch {self.branch_name} =================[/cyan]"
        )
        if not self.no_configuration_prints:
            logger.info(f"Validating against {self.prev_ver}")

            if self.branch_name in [
                self.prev_ver,
                self.prev_ver.replace("origin/", ""),
            ]:  # pragma: no cover
                logger.info("Running only on last commit")

            elif self.is_circle:
                logger.info("Running only on committed files")

            elif self.staged:
                logger.info("Running only on staged files")

            else:
                logger.info("Running on committed and staged files")

            if self.skip_pack_rn_validation:
                logger.info("Skipping release notes validation")

            if self.skip_docker_checks:
                logger.info("Skipping Docker checks")

            if not self.is_backward_check:
                logger.info("Skipping backwards compatibility checks")

            if self.skip_dependencies:
                logger.info("Skipping pack dependencies check")

    def get_unfiltered_changed_files_from_git(self) -> Tuple[Set, Set, Set]:
        """
        Get the added and modified before file filtration to only relevant files

        Returns:
            3 sets:
            - The unfiltered modified files
            - The unfiltered added files
            - The unfiltered renamed files
        """
        # get files from git by status identification against prev-ver
        modified_files = self.git_util.modified_files(
            prev_ver=self.prev_ver,
            committed_only=self.is_circle,
            staged_only=self.staged,
            debug=self.debug_git,
            include_untracked=self.include_untracked,
        )
        added_files = self.git_util.added_files(
            prev_ver=self.prev_ver,
            committed_only=self.is_circle,
            staged_only=self.staged,
            debug=self.debug_git,
            include_untracked=self.include_untracked,
        )
        renamed_files = self.git_util.renamed_files(
            prev_ver=self.prev_ver,
            committed_only=self.is_circle,
            staged_only=self.staged,
            debug=self.debug_git,
            include_untracked=self.include_untracked,
            get_only_current_file_names=True,
        )

        return modified_files, added_files, renamed_files

    def get_changed_files_from_git(self) -> Tuple[Set, Set, Set, Set, bool]:
        """Get the added and modified after file filtration to only relevant files for validate

        Returns:
            - The filtered modified files (including the renamed files)
            - The filtered added files
            - The changed metadata files
            - The modified old-format files (legacy unified python files)
            - Boolean flag that indicates whether all file types are supported
        """

        (
            modified_files,
            added_files,
            renamed_files,
        ) = self.get_unfiltered_changed_files_from_git()

        # filter files only to relevant files
        filtered_modified, old_format_files, _ = self.filter_to_relevant_files(
            modified_files
        )
        filtered_renamed, _, valid_types_renamed = self.filter_to_relevant_files(
            renamed_files
        )
        filtered_modified = filtered_modified.union(filtered_renamed)

        (
            filtered_added,
            old_format_added,
            valid_types_added,
        ) = self.filter_to_relevant_files(added_files)
        old_format_files |= old_format_added

        valid_types = all((valid_types_added, valid_types_renamed))

        # extract metadata files from the recognised changes
        changed_meta = self.pack_metadata_extraction(
            modified_files, added_files, renamed_files
        )
        filtered_changed_meta, old_format_changed, _ = self.filter_to_relevant_files(
            changed_meta, check_metadata_files=True
        )
        old_format_files |= old_format_changed
        return (
            filtered_modified,
            filtered_added,
            filtered_changed_meta,
            old_format_files,
            valid_types,
        )

    def pack_metadata_extraction(self, modified_files, added_files, renamed_files):
        """Extract pack metadata files from the modified and added files

        Return all modified metadata file paths
        and get all newly added packs from the added metadata files."""
        changed_metadata_files = set()
        for path in modified_files.union(renamed_files):
            file_path = str(path[1]) if isinstance(path, tuple) else str(path)

            if file_path.endswith(PACKS_PACK_META_FILE_NAME):
                changed_metadata_files.add(file_path)

        for path in added_files:
            if str(path).endswith(PACKS_PACK_META_FILE_NAME):
                self.new_packs.add(get_pack_name(str(path)))

        return changed_metadata_files

    def filter_to_relevant_files(self, file_set, check_metadata_files=False):
        """Goes over file set and returns only a filtered set of only files relevant for validation"""
        filtered_set: set = set()
        old_format_files: set = set()
        valid_types: set = set()
        for path in file_set:
            old_path = None
            if isinstance(path, tuple):
                file_path = str(path[1])
                old_path = str(path[0])

            else:
                file_path = str(path)

            try:
                (
                    formatted_path,
                    old_path,
                    valid_file_extension,
                ) = self.check_file_relevance_and_format_path(
                    file_path,
                    old_path,
                    old_format_files,
                    check_metadata_files=check_metadata_files,
                )
                valid_types.add(valid_file_extension)
                if formatted_path:
                    if old_path:
                        filtered_set.add((old_path, formatted_path))
                    else:
                        filtered_set.add(formatted_path)

            # handle a case where a file was deleted locally though recognised as added against master.
            except FileNotFoundError:
                if file_path not in self.ignored_files:
                    if self.print_ignored_files:
                        logger.info(f"[yellow]ignoring file {file_path}[/yellow]")
                    self.ignored_files.add(file_path)

        return filtered_set, old_format_files, all(valid_types)

    def check_file_relevance_and_format_path(
        self, file_path, old_path, old_format_files, check_metadata_files=False
    ):
        """
        Determines if a file is relevant for validation and create any modification to the file_path if needed
        :returns a tuple(string, string, bool) where
            - the first element is the path of the file that should be returned, if the file isn't relevant then returns an empty string
            - the second element is the old path in case the file was renamed, if the file wasn't renamed then return an empty string
            - true if the file type is supported, false otherwise
        """
        irrelevant_file_output = "", "", True
        if file_path.split(os.path.sep)[0] in (
            ".gitlab",
            ".circleci",
            ".github",
            ".devcontainer",
            ".vscode",
        ):
            return irrelevant_file_output

        file_type = find_type(file_path)

        if file_type == FileType.CONF_JSON:
            return file_path, "", True

        if self.ignore_files_irrelevant_for_validation(
            file_path, check_metadata_files=check_metadata_files
        ):
            return irrelevant_file_output

        if not file_type:
            if str(file_path).endswith(".png"):
                error_message, error_code = Errors.invalid_image_name_or_location()
            else:
                error_message, error_code = Errors.file_type_not_supported(
                    None, file_path
                )

            self.handle_error(error_message, error_code, file_path=file_path)
            return "", "", False

        # redirect non-test code files to the associated yml file
        if file_type in [
            FileType.PYTHON_FILE,
            FileType.POWERSHELL_FILE,
            FileType.JAVASCRIPT_FILE,
            FileType.XIF_FILE,
            FileType.MODELING_RULE_XIF,
            FileType.PARSING_RULE_XIF,
        ]:
            if not (
                str(file_path).endswith("_test.py")
                or str(file_path).endswith(".Tests.ps1")
                or str(file_path).endswith("_test.js")
            ):
                file_path = (
                    file_path.replace(".py", ".yml")
                    .replace(".ps1", ".yml")
                    .replace(".js", ".yml")
                    .replace(".xif", ".yml")
                )

                if old_path:
                    old_path = (
                        old_path.replace(".py", ".yml")
                        .replace(".ps1", ".yml")
                        .replace(".js", ".yml")
                        .replace(".xif", ".yml")
                    )
            else:
                return irrelevant_file_output

        if file_type == FileType.XDRC_TEMPLATE_YML:
            file_path = file_path.replace(".yml", ".json")

            if old_path:
                old_path = old_path.replace(".yml", ".json")

        # redirect schema file when updating release notes
        if file_type == FileType.MODELING_RULE_SCHEMA:
            file_path = file_path.replace("_schema", "").replace(".json", ".yml")

            if old_path:
                old_path = old_path.replace("_schema", "").replace(".json", ".yml")

        # redirect _testdata.json file to the associated yml file
        if file_type == FileType.MODELING_RULE_TEST_DATA:
            file_path = file_path.replace("_testdata.json", ".yml")

            if old_path:
                old_path = old_path.replace("_testdata.json", ".yml")

        # check for old file format
        if self.is_old_file_format(file_path, file_type):
            old_format_files.add(file_path)
            return irrelevant_file_output

        # if renamed file - return a tuple
        if old_path:
            return file_path, old_path, True

        # else return the file path
        else:
            return file_path, "", True

    def ignore_files_irrelevant_for_validation(
        self, file_path: str, check_metadata_files: bool = False
    ) -> bool:
        """
        Will ignore files that are not in the packs directory, are .txt files or are in the
        VALIDATION_USING_GIT_IGNORABLE_DATA tuple.

        Args:
            file_path: path of file to check if should be ignored.
            check_metadata_files: If True will not ignore metadata files.
        Returns: True if file is ignored, false otherwise
        """

        if PACKS_DIR not in file_path:
            self.ignore_file(file_path)
            return True

        if check_metadata_files and find_type(file_path) == FileType.METADATA:
            return False

        if file_path.endswith(".txt"):
            self.ignore_file(file_path)
            return True

        if any(name in str(file_path) for name in VALIDATION_USING_GIT_IGNORABLE_DATA):
            self.ignore_file(file_path)
            return True
        return False

    def ignore_file(self, file_path: str) -> None:
        if self.print_ignored_files:
            logger.info(f"[yellow]ignoring file {file_path}[/yellow]")
        self.ignored_files.add(file_path)

    """ ######################################## Validate Tools ############################################### """

    @staticmethod
    def create_ignored_errors_list(errors_to_check):
        """Creating a list of errors without the errors in the errors_to_check list"""
        ignored_error_list = []
        all_errors = get_all_error_codes()
        for error_code in all_errors:
            error_type = error_code[:2]
            if error_code not in errors_to_check and error_type not in errors_to_check:
                ignored_error_list.append(error_code)

        return ignored_error_list

    def add_ignored_errors_to_list(self, config, section, key, ignored_errors_list):
        if key == "ignore":
            ignored_errors_list.extend(str(config[section][key]).split(","))

        if key in PRESET_ERROR_TO_IGNORE:
            ignored_errors_list.extend(PRESET_ERROR_TO_IGNORE.get(key))

        if key in PRESET_ERROR_TO_CHECK:
            ignored_errors_list.extend(
                self.create_ignored_errors_list(PRESET_ERROR_TO_CHECK.get(key))
            )

    def get_error_ignore_list(self, pack_name):
        ignored_errors_list: dict = {}
        if pack_name:
            if config := get_pack_ignore_content(pack_name):
                # create file specific ignored errors list
                for section in filter(
                    lambda section: section.startswith("file:"), config.sections()
                ):
                    file_name = section[5:]
                    ignored_errors_list[file_name] = []
                    for key in config[section]:
                        self.add_ignored_errors_to_list(
                            config, section, key, ignored_errors_list[file_name]
                        )

        return ignored_errors_list

    @staticmethod
    def is_old_file_format(file_path: str, file_type: FileType):
        if file_type not in {FileType.INTEGRATION, FileType.SCRIPT}:
            return False
        file_yml = get_file(file_path)
        # check for unified integration
        if file_type == FileType.INTEGRATION and file_yml.get("script", {}).get(
            "script", "-"
        ) not in ["-", ""]:
            if file_yml.get("script", {}).get("type", "javascript") != "python":
                return False
            return True

        # check for unified script
        if file_type == FileType.SCRIPT and file_yml.get("script", "-") not in [
            "-",
            "",
        ]:
            if file_yml.get("type", "javascript") != "python":
                return False
            return True
        return False

    @staticmethod
    def get_packs_with_added_release_notes(added_files):
        added_rn = set()
        for file in added_files:
            if find_type(path=file) == FileType.RELEASE_NOTES:
                added_rn.add(get_pack_name(file))

        return added_rn

    def print_ignored_errors_report(self, print_ignored_errors):
        if print_ignored_errors:
            all_ignored_errors = "\n".join(FOUND_FILES_AND_IGNORED_ERRORS)
            logger.info(
                f"\n[yellow]=========== Found ignored errors "
                f"in the following files ===========\n\n{all_ignored_errors}[/yellow]"
            )

    def print_ignored_files_report(self, print_ignored_files):
        if print_ignored_files:
            all_ignored_files = "\n".join(list(self.ignored_files))
            logger.info(
                f"\n[yellow]=========== Ignored the following files ===========\n\n{all_ignored_files}[/yellow]"
            )

    def get_packs_that_should_have_version_raised(
        self, modified_files, added_files, old_format_files
    ):
        # modified packs (where the change is not test-playbook, test-script, readme, metadata file, release notes or
        # doc/author images)
        all_modified_files = modified_files.union(old_format_files)
        modified_packs_that_should_have_version_raised = get_pack_names_from_files(
            all_modified_files,
            skip_file_types={
                FileType.RELEASE_NOTES,
                FileType.README,
                FileType.TEST_PLAYBOOK,
                FileType.TEST_SCRIPT,
                FileType.DOC_IMAGE,
                FileType.AUTHOR_IMAGE,
                FileType.CONTRIBUTORS,
            },
        )

        # also existing packs with added files which are not test-playbook, test-script readme or release notes
        # should have their version raised
        modified_packs_that_should_have_version_raised = (
            modified_packs_that_should_have_version_raised.union(
                get_pack_names_from_files(
                    added_files,
                    skip_file_types={
                        FileType.RELEASE_NOTES,
                        FileType.README,
                        FileType.TEST_PLAYBOOK,
                        FileType.TEST_SCRIPT,
                        FileType.DOC_IMAGE,
                        FileType.AUTHOR_IMAGE,
                        FileType.CONTRIBUTORS,
                    },
                )
                - self.new_packs
            )
        )

        return modified_packs_that_should_have_version_raised

    @staticmethod
    def get_packs(changed_files):
        packs = set()
        for changed_file in changed_files:
            if isinstance(changed_file, tuple):
                changed_file = changed_file[1]
            pack = get_pack_name(changed_file)
            if pack:
                packs.add(pack)

        return packs

    def get_id_set_file(self, skip_id_set_creation, id_set_path):
        """

        Args:
            skip_id_set_creation (bool): whether should skip id set validation or not
            this will also determine whether a new id_set can be created by validate.
            id_set_path (str): id_set.json path file

        Returns:
            str: is_set file path
        """
        id_set = {}
        if not os.path.isfile(id_set_path):
            if not skip_id_set_creation:
                id_set, _, _ = IDSetCreator(print_logs=False).create_id_set()

        else:
            id_set = open_id_set_file(id_set_path)

        if not id_set and not self.no_configuration_prints:
            error_message, error_code = Errors.no_id_set_file()
            self.handle_error(
                error_message,
                error_code,
                file_path=os.path.join(os.getcwd(), id_set_path),
                warning=True,
            )

        return id_set

    def check_and_validate_deprecated(
        self,
        file_type,
        file_path,
        current_file,
        is_modified,
        is_backward_check,
        validator,
    ):
        """If file is deprecated, validate it. Return None otherwise.

        Files with 'deprecated: true' or 'toversion < OLDEST_SUPPORTED_VERSION' fields are considered deprecated.

        Args:
            file_type: (FileType) Type of file to validate.
            file_path: (str) file path to validate.
            current_file: (dict) file in json format to validate.
            is_modified: (boolean) for whether the file was modified.
            is_backward_check: (boolean) for whether to preform backwards compatibility validation.
            validator: (ContentEntityValidator) validator object to run backwards compatibility validation from.

        Returns:
            True if current_file is deprecated and valid.
            False if current_file is deprecated and invalid.
            None if current_file is not deprecated.
        """
        is_deprecated = current_file.get("deprecated")

        toversion_is_old = "toversion" in current_file and version.parse(
            current_file.get("toversion", DEFAULT_CONTENT_ITEM_TO_VERSION)
        ) < version.parse(OLDEST_SUPPORTED_VERSION)

        if is_deprecated or toversion_is_old:
            logger.info(f"Validating deprecated file: {file_path}")

            is_valid_as_deprecated = True
            if hasattr(validator, "is_valid_as_deprecated"):
                is_valid_as_deprecated = validator.is_valid_as_deprecated()

            if is_modified and is_backward_check:
                return all([is_valid_as_deprecated, validator.is_backward_compatible()])

            self.ignored_files.add(file_path)
            if self.print_ignored_files:
                logger.info(f"Skipping validation for: {file_path}")

            return is_valid_as_deprecated
        return None


# demisto_sdk/commands/zip_packs/packs_zipper.py

import logging
import os
import sys
from contextlib import contextmanager
from pathlib import Path
from shutil import make_archive

from demisto_sdk.commands.common.constants import PACKS_DIR, MarketplaceVersions
from demisto_sdk.commands.common.content.objects.pack_objects.pack import Pack
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import arg_to_list
from demisto_sdk.commands.create_artifacts.content_artifacts_creator import (
    IGNORED_PACKS,
    ArtifactsManager,
    ContentObject,
    create_dirs,
    delete_dirs,
    dump_pack,
    zip_packs,
)

EX_SUCCESS = 0
EX_FAIL = 1


class PacksZipper:
    def __init__(
        self,
        pack_paths: str,
        output: str,
        content_version: str,
        zip_all: bool,
        marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
        quiet_mode: bool = False,
        **kwargs,
    ):
        self.artifacts_manager = PacksManager(
            pack_paths=pack_paths,
            artifacts_path=output,
            content_version=content_version,
            all_in_one_zip=zip_all,
            quiet_mode=quiet_mode,
            marketplace=marketplace.value,
        )

    def zip_packs(self):
        """Compress the given packs to one zip file or zip for each pack (depended on the zip_all flag).

        Returns:
            tuple: (the created zip path or the directory of the pack zips, list of the pack names)
                    or (None, None) if the pack names are empty

        """
        if self.artifacts_manager.pack_names:
            self.artifacts_manager.dump_packs()
            return self.artifacts_manager.output_path, self.artifacts_manager.pack_names

        else:
            return None, None


class PacksManager(ArtifactsManager):
    """Manages the work with packs in zip-packs command.

    Attributes:
        zip_all (bool): Flag indicating whether to zip all the packs in one zip or not.
        quiet_mode (bool): Flag indicating is in quiet mode or not.
        output_path (str): The target of the created zip

    """

    def __init__(
        self,
        pack_paths: str,
        all_in_one_zip: bool,
        quiet_mode: bool,
        marketplace: str = MarketplaceVersions.XSOAR.value,
        **kwargs,
    ):
        super().__init__(
            packs=True, zip=True, cpus=1, suffix="", marketplace=marketplace, **kwargs
        )
        self.init_packs(pack_paths)
        self.zip_all = all_in_one_zip
        self.quiet_mode = quiet_mode
        self.output_path = (
            f"{self.content_uploadable_zips_path}.zip"
            if self.zip_all
            else str(self.content_uploadable_zips_path)
        )

    def init_packs(self, pack_paths):
        """Init dict that map pack name to Pack object and init also the pack_names property.

        Args:
            pack_paths (str): CSV str with the pack paths.

        """
        self.packs = {}
        for path_str in arg_to_list(pack_paths):
            path = Path(path_str)
            if (
                len(path.parts) == 2 and path.parts[0] == PACKS_DIR
            ):  # relative path from Packs/...
                path = self.content.path / path

            if not os.path.exists(path):
                logger.info(
                    f"[red]Error: Given input path: {path} does not exist, ignored[/red]"
                )
                continue

            self.packs.update({path.name: Pack(path)})

        self.pack_names = [*self.packs]

    def get_relative_pack_path(self, content_object: ContentObject):
        """Get the path of the given object relatively from the Packs directory, for example HelloWorld/Scripts.

        Args:
            content_object (ContentObject): The object to get the relative path for.

        Returns:
            Path: The relative path.
        """
        for part in content_object.path.parts:
            if part in self.packs:
                return content_object.path.relative_to(self.packs[part].path.parent)

    def get_dir_to_delete(self):
        """Get list of directories to delete after the unify process finished.

        Returns:
            list: The value from ArtifactsManager.get_dir_to_delete, if zip_all is True - the uploadable_packsalso dir also
        """

        result = super().get_dir_to_delete()
        if self.zip_all:
            result.append(self.content_uploadable_zips_path)
        return result

    def dump_packs(self):
        """Dump all the packs stored in the packs field
        and will print summery according to the quit_mode field

        """
        reports = []
        # we quiet the outputs and in case we want the output - a summery will be printed
        with QuietModeController(
            quiet_logger=True, quiet_output=True
        ), PacksDirsHandler(self):
            for pack_name in self.pack_names:
                if pack_name not in IGNORED_PACKS:
                    reports.append(dump_pack(self, self.packs[pack_name]))

        if not self.quiet_mode:
            for report in reports:
                logger.info(report.to_str(src_relative_to=None))

            created_zip_path = (
                self.output_path
                if self.zip_all
                else "\n".join(
                    [
                        f"{self.output_path}/{pack_name}.zip"
                        for pack_name in self.pack_names
                    ]
                )
            )
            logger.info(f"\nCreated zips:\n{created_zip_path}")


class QuietModeController:
    """Control a quiet mode for loggers and stdout.

    Attributes:
        quiet_modes (dict): Dict with bool flags for the various outputs (logger, stdout).
        old_stdout (TextIO): The previous target of the system stdout.
        logger (logging.Logger): The active logger.
        prev_logger_level (int): The previous log level before quiet mode was activated.
    """

    def __init__(self, quiet_logger: bool, quiet_output):
        self.quiet_modes = dict(quiet_logger=quiet_logger, quiet_output=quiet_output)
        self.old_stdout = sys.stdout
        self.prev_logger_level = logger.getEffectiveLevel()

    def __enter__(self):
        if self.quiet_modes["quiet_output"]:
            sys.stdout = open(os.devnull, "w")
        if self.quiet_modes["quiet_logger"]:
            logger.setLevel(logging.ERROR)

    def __exit__(self, exc_type, exc_val, exc_tb):
        logger.debug(f"{exc_type} {exc_val} {exc_tb}")
        sys.stdout = self.old_stdout
        logger.setLevel(self.prev_logger_level)


def zip_uploadable_packs(artifact_manager: ArtifactsManager):
    """Zip the zipped packs directory"""
    pack_zips_dir = artifact_manager.content_uploadable_zips_path
    make_archive(pack_zips_dir, "zip", pack_zips_dir)


@contextmanager
def PacksDirsHandler(artifact_manager: PacksManager):
    """Artifacts Directories handler.
    Logic by time line:
        1. Delete artifacts directories if exists.
        2. Create directories.
        3. If any error occurred -> Delete artifacts directories -> Exit.
        4. If finish successfully:
            a. If zip all:
                 Zip artifacts zip.
               Else
                 Zip packs for uploading.
            b. Delete artifacts directories.

    Args:
        artifact_manager: Packs manager object.
    """
    try:
        delete_dirs(artifact_manager)
        create_dirs(artifact_manager)
        yield
    except (Exception, KeyboardInterrupt) as e:
        delete_dirs(artifact_manager)
        artifact_manager.exit_code = EX_FAIL
        logger.error(e)
        exit(EX_FAIL)
    else:
        if artifact_manager.zip_all:
            zip_packs(artifact_manager)
            zip_uploadable_packs(artifact_manager)
        else:
            zip_packs(artifact_manager)

        delete_dirs(artifact_manager)


# demisto_sdk/__main__.py

# Site packages
import copy
import functools
import logging
import os
import sys
from pathlib import Path
from typing import IO, Any, Dict, Iterable, Tuple, Union

import click
import git
import typer
from pkg_resources import DistributionNotFound, get_distribution

from demisto_sdk.commands.common.configuration import Configuration
from demisto_sdk.commands.common.constants import (
    ENV_DEMISTO_SDK_MARKETPLACE,
    FileType,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.content_constant_paths import (
    ALL_PACKS_DEPENDENCIES_DEFAULT_PATH,
    CONTENT_PATH,
)
from demisto_sdk.commands.common.cpu_count import cpu_count
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.hook_validations.readme import ReadMeValidator
from demisto_sdk.commands.common.logger import handle_deprecated_args, logging_setup
from demisto_sdk.commands.common.tools import (
    find_type,
    get_last_remote_release_version,
    get_release_note_entries,
    is_external_repository,
    is_sdk_defined_working_offline,
    parse_marketplace_kwargs,
)
from demisto_sdk.commands.content_graph.commands.create import create
from demisto_sdk.commands.content_graph.commands.get_relationships import (
    get_relationships,
)
from demisto_sdk.commands.content_graph.commands.update import update
from demisto_sdk.commands.content_graph.objects.repository import all_content_repo
from demisto_sdk.commands.generate_modeling_rules import generate_modeling_rules
from demisto_sdk.commands.prepare_content.prepare_upload_manager import (
    PrepareUploadManager,
)
from demisto_sdk.commands.split.ymlsplitter import YmlSplitter
from demisto_sdk.commands.test_content.test_modeling_rule import (
    init_test_data,
    test_modeling_rule,
)
from demisto_sdk.commands.upload.upload import upload_content_entity
from demisto_sdk.utils.utils import check_configuration_file

SDK_OFFLINE_ERROR_MESSAGE = (
    "[red]An internet connection is required for this command. If connected to the "
    "internet, un-set the DEMISTO_SDK_OFFLINE_ENV environment variable.[/red]"
)

logger = logging.getLogger("demisto-sdk")


# Third party packages

# Common tools


class PathsParamType(click.Path):
    """
    Defines a click options type for use with the @click.option decorator

    The type accepts a string of comma-separated values where each individual value adheres
    to the definition for the click.Path type. The class accepts the same parameters as the
    click.Path type, applying those arguments for each comma-separated value in the list.
    See https://click.palletsprojects.com/en/8.0.x/parameters/#implementing-custom-types for
    more details.
    """

    def convert(self, value, param, ctx):
        if "," not in value:
            return super().convert(value, param, ctx)

        split_paths = value.split(",")
        # check the validity of each of the paths
        _ = [
            super(PathsParamType, self).convert(path, param, ctx)
            for path in split_paths
        ]
        return value


class VersionParamType(click.ParamType):
    """
    Defines a click options type for use with the @click.option decorator

    The type accepts a string represents a version number.
    """

    name = "version"

    def convert(self, value, param, ctx):
        version_sections = value.split(".")
        if len(version_sections) == 3 and all(
            version_section.isdigit() for version_section in version_sections
        ):
            return value
        else:
            self.fail(
                f"Version {value} is not according to the expected format. "
                f"The format of version should be in x.y.z format, e.g: <2.1.3>",
                param,
                ctx,
            )


class DemistoSDK:
    """
    The core class for the SDK.
    """

    def __init__(self):
        self.configuration = None


pass_config = click.make_pass_decorator(DemistoSDK, ensure=True)


def logging_setup_decorator(func, *args, **kwargs):
    def get_context_arg(args):
        for arg in args:
            if type(arg) == click.core.Context:
                return arg
        print(  # noqa: T201
            "Error: Cannot find the Context arg. Is the command configured correctly?"
        )
        return None

    @click.option(
        "--console_log_threshold",
        help="Minimum logging threshold for the console logger."
        " Possible values: DEBUG, INFO, WARNING, ERROR.",
    )
    @click.option(
        "--file_log_threshold",
        help="Minimum logging threshold for the file logger."
        " Possible values: DEBUG, INFO, WARNING, ERROR.",
    )
    @click.option(
        "--log_file_path",
        help="Path to the log file. Default: Content root path.",
    )
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        logging_setup(
            console_log_threshold=kwargs.get("console_log_threshold") or logging.INFO,
            file_log_threshold=kwargs.get("file_log_threshold") or logging.DEBUG,
            log_file_path=kwargs.get("log_file_path") or None,
        )

        handle_deprecated_args(get_context_arg(args).args)
        return func(*args, **kwargs)

    return wrapper


@click.group(
    invoke_without_command=True,
    no_args_is_help=True,
    context_settings=dict(max_content_width=100),
)
@click.help_option("-h", "--help")
@click.option(
    "-v",
    "--version",
    help="Get the demisto-sdk version.",
    is_flag=True,
    default=False,
    show_default=True,
)
@click.option(
    "-rn",
    "--release-notes",
    help="Get the release notes of the current demisto-sdk version.",
    is_flag=True,
    default=False,
    show_default=True,
)
@pass_config
@click.pass_context
def main(ctx, config, version, release_notes, **kwargs):
    logging_setup(
        console_log_threshold=kwargs.get("console_log_threshold", logging.INFO),
        file_log_threshold=kwargs.get("file_log_threshold", logging.DEBUG),
        log_file_path=kwargs.get("log_file_path"),
    )
    global logger
    logger = logging.getLogger("demisto-sdk")
    handle_deprecated_args(ctx.args)

    config.configuration = Configuration()
    import dotenv

    if sys.version_info[:2] == (3, 8):
        logger.info(
            "[red]Demisto-SDK will soon stop supporting Python 3.8. Please update your python environment.[/red]"
        )

    dotenv.load_dotenv(CONTENT_PATH / ".env", override=True)  # type: ignore # load .env file from the cwd
    if (
        (not os.getenv("DEMISTO_SDK_SKIP_VERSION_CHECK")) or version
    ) and not is_sdk_defined_working_offline():  # If the key exists/called to version
        try:
            __version__ = get_distribution("demisto-sdk").version
        except DistributionNotFound:
            __version__ = "dev"
            logger.info(
                "[yellow]Cound not find the version of the demisto-sdk. This usually happens when running in a development environment.[/yellow]"
            )
        else:
            last_release = ""
            if not os.environ.get(
                "CI"
            ):  # Check only when not running in CI (e.g running locally).
                last_release = get_last_remote_release_version()
            logger.info(f"[yellow]You are using demisto-sdk {__version__}.[/yellow]")
            if last_release and __version__ != last_release:
                logger.info(
                    f"[yellow]however version {last_release} is available.\n"
                    f"To update, run pip3 install --upgrade demisto-sdk[/yellow]"
                )
            if release_notes:
                rn_entries = get_release_note_entries(__version__)

                if not rn_entries:
                    logger.warning(
                        "\n[yellow]Could not get the release notes for this version.[/yellow]"
                    )
                else:
                    logger.info(
                        "\nThe following are the release note entries for the current version:\n"
                    )
                    for rn in rn_entries:
                        logger.info(rn)
                    logger.info("")


# ====================== split ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option("-i", "--input", help="The yml/json file to extract from", required=True)
@click.option(
    "-o",
    "--output",
    help="The output dir to write the extracted code/description/image/json to.",
)
@click.option(
    "--no-demisto-mock",
    help="Don't add an import for demisto mock. (only for yml files)",
    is_flag=True,
    show_default=True,
)
@click.option(
    "--no-common-server",
    help="Don't add an import for CommonServerPython. (only for yml files)",
    is_flag=True,
    show_default=True,
)
@click.option(
    "--no-auto-create-dir",
    help="Don't auto create the directory if the target directory ends with *Integrations/*Scripts/*Dashboards"
    "/*GenericModules.",
    is_flag=True,
    show_default=True,
)
@click.option(
    "--new-module-file",
    help="Create a new module file instead of editing the existing file. (only for json files)",
    is_flag=True,
    show_default=True,
)
@pass_config
@click.pass_context
@logging_setup_decorator
def split(ctx, config, **kwargs):
    """Split the code, image and description files from a Demisto integration or script yaml file
    to multiple files(To a package format - https://demisto.pan.dev/docs/package-dir).
    """
    from demisto_sdk.commands.split.jsonsplitter import JsonSplitter

    check_configuration_file("split", kwargs)
    file_type: FileType = find_type(kwargs.get("input", ""), ignore_sub_categories=True)
    if file_type not in [
        FileType.INTEGRATION,
        FileType.SCRIPT,
        FileType.GENERIC_MODULE,
        FileType.MODELING_RULE,
        FileType.PARSING_RULE,
    ]:
        logger.info(
            "[red]File is not an Integration, Script, Generic Module, Modeling Rule or Parsing Rule.[/red]"
        )
        return 1

    if file_type in [
        FileType.INTEGRATION,
        FileType.SCRIPT,
        FileType.MODELING_RULE,
        FileType.PARSING_RULE,
    ]:
        yml_splitter = YmlSplitter(
            configuration=config.configuration, file_type=file_type.value, **kwargs
        )
        return yml_splitter.extract_to_package_format()

    else:
        json_splitter = JsonSplitter(
            input=kwargs.get("input"),  # type: ignore[arg-type]
            output=kwargs.get("output"),  # type: ignore[arg-type]
            no_auto_create_dir=kwargs.get("no_auto_create_dir"),  # type: ignore[arg-type]
            new_module_file=kwargs.get("new_module_file"),  # type: ignore[arg-type]
        )
        return json_splitter.split_json()


# ====================== extract-code ====================== #
@main.command(hidden=True)
@click.help_option("-h", "--help")
@click.option("--input", "-i", help="The yml file to extract from", required=True)
@click.option(
    "--output", "-o", required=True, help="The output file to write the code to"
)
@click.option(
    "--no-demisto-mock",
    help="Don't add an import for demisto mock, false by default",
    is_flag=True,
    show_default=True,
)
@click.option(
    "--no-common-server",
    help="Don't add an import for CommonServerPython."
    "If not specified will import unless this is CommonServerPython",
    is_flag=True,
    show_default=True,
)
@pass_config
@click.pass_context
@logging_setup_decorator
def extract_code(ctx, config, **kwargs):
    """Extract code from a Demisto integration or script yaml file."""
    from demisto_sdk.commands.split.ymlsplitter import YmlSplitter

    check_configuration_file("extract-code", kwargs)
    file_type: FileType = find_type(kwargs.get("input", ""), ignore_sub_categories=True)
    if file_type not in [FileType.INTEGRATION, FileType.SCRIPT]:
        logger.info("[red]File is not an Integration or Script.[/red]")
        return 1
    extractor = YmlSplitter(
        configuration=config.configuration, file_type=file_type.value, **kwargs
    )
    return extractor.extract_code(kwargs["outfile"])


# ====================== prepare-content ====================== #
@main.command(name="prepare-content")
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="Comma-separated list of paths to directories or files to unify.",
    required=False,
    type=PathsParamType(dir_okay=True, exists=True),
)
@click.option(
    "-a",
    "--all",
    is_flag=True,
    help="Run prepare-content on all content packs. If no output path is given, will dump the result in the current working path.",
)
@click.option(
    "-g",
    "--graph",
    help="Whether use the content graph",
    is_flag=True,
    default=False,
)
@click.option(
    "--skip-update",
    help="Whether to skip updating the content graph (used only when graph is true)",
    is_flag=True,
    default=False,
)
@click.option(
    "-o", "--output", help="The output dir to write the unified yml to", required=False
)
@click.option(
    "-c",
    "--custom",
    help="Add test label to unified yml id/name/display",
    required=False,
)
@click.option(
    "-f",
    "--force",
    help="Forcefully overwrites the preexisting yml if one exists",
    is_flag=True,
    show_default=False,
)
@click.option(
    "-ini",
    "--ignore-native-image",
    help="Whether to ignore the addition of the nativeimage key to the yml of a script/integration",
    is_flag=True,
    show_default=False,
    default=False,
)
@click.option(
    "-mp",
    "--marketplace",
    help="The marketplace the content items are created for, that determines usage of marketplace "
    "unique text. Default is the XSOAR marketplace.",
    default="xsoar",
    type=click.Choice(["xsoar", "marketplacev2", "v2"]),
)
@click.pass_context
@logging_setup_decorator
def prepare_content(ctx, **kwargs):
    """
    This command is used to prepare the content to be used in the platform.
    """
    assert (
        sum([bool(kwargs["all"]), bool(kwargs["input"])]) == 1
    ), "Exactly one of the '-a' or '-i' parameters must be provided."

    if kwargs["all"]:
        content_DTO = all_content_repo()
        output_path = kwargs.get("output", ".") or "."
        content_DTO.dump(
            dir=Path(output_path, "prepare-content-tmp"),
            marketplace=parse_marketplace_kwargs(kwargs),
        )
        return 0

    inputs = []
    if input_ := kwargs["input"]:
        inputs = input_.split(",")

    if output_path := kwargs["output"]:
        if "." in Path(output_path).name:  # check if the output path is a file
            if len(inputs) > 1:
                raise ValueError(
                    "When passing multiple inputs, the output path should be a directory and not a file."
                )
        else:
            dest_path = Path(output_path)
            dest_path.mkdir(exist_ok=True)

    for input_content in inputs:
        if output_path and len(inputs) > 1:
            path_name = Path(input_content).name
            kwargs["output"] = str(Path(output_path, path_name))

        if click.get_current_context().info_name == "unify":
            kwargs["unify_only"] = True

        check_configuration_file("unify", kwargs)
        # Input is of type Path.
        kwargs["input"] = str(input_content)
        file_type = find_type(kwargs["input"])
        if marketplace := kwargs.get("marketplace"):
            os.environ[ENV_DEMISTO_SDK_MARKETPLACE] = marketplace.lower()
        if file_type == FileType.GENERIC_MODULE:
            from demisto_sdk.commands.prepare_content.generic_module_unifier import (
                GenericModuleUnifier,
            )

            # pass arguments to GenericModule unifier and call the command
            generic_module_unifier = GenericModuleUnifier(**kwargs)
            generic_module_unifier.merge_generic_module_with_its_dashboards()
        else:
            PrepareUploadManager.prepare_for_upload(**kwargs)
    return 0


main.add_command(prepare_content, name="unify")


# ====================== zip-packs ====================== #


@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="The packs to be zipped as csv list of pack paths.",
    type=PathsParamType(exists=True, resolve_path=True),
    required=True,
)
@click.option(
    "-o",
    "--output",
    help="The destination directory to create the packs.",
    type=click.Path(file_okay=False, resolve_path=True),
    required=True,
)
@click.option(
    "-v",
    "--content-version",
    help="The content version in CommonServerPython.",
    default="0.0.0",
)
@click.option(
    "-u",
    "--upload",
    is_flag=True,
    help="Upload the unified packs to the marketplace.",
    default=False,
)
@click.option(
    "--zip-all", is_flag=True, help="Zip all the packs in one zip file.", default=False
)
@click.pass_context
@logging_setup_decorator
def zip_packs(ctx, **kwargs) -> int:
    """Generating zipped packs that are ready to be uploaded to Cortex XSOAR machine."""
    from demisto_sdk.commands.upload.uploader import Uploader
    from demisto_sdk.commands.zip_packs.packs_zipper import (
        EX_FAIL,
        EX_SUCCESS,
        PacksZipper,
    )

    check_configuration_file("zip-packs", kwargs)

    # if upload is true - all zip packs will be compressed to one zip file
    should_upload = kwargs.pop("upload", False)
    zip_all = kwargs.pop("zip_all", False) or should_upload
    marketplace = parse_marketplace_kwargs(kwargs)

    packs_zipper = PacksZipper(
        zip_all=zip_all, pack_paths=kwargs.pop("input"), quiet_mode=zip_all, **kwargs
    )
    zip_path, unified_pack_names = packs_zipper.zip_packs()

    if should_upload and zip_path:
        return Uploader(
            input=Path(zip_path), pack_names=unified_pack_names, marketplace=marketplace
        ).upload()

    return EX_SUCCESS if zip_path is not None else EX_FAIL


# ====================== validate ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "--no-conf-json",
    is_flag=True,
    default=False,
    show_default=True,
    help="Skip conf.json validation.",
)
@click.option(
    "-s",
    "--id-set",
    is_flag=True,
    default=False,
    show_default=True,
    help="Perform validations using the id_set file.",
)
@click.option(
    "-idp",
    "--id-set-path",
    help="The path of the id-set.json used for validations.",
    type=click.Path(resolve_path=True),
)
@click.option(
    "-gr",
    "--graph",
    is_flag=True,
    default=False,
    show_default=True,
    help="Perform validations on content graph.",
)
@click.option(
    "--prev-ver", help="Previous branch or SHA1 commit to run checks against."
)
@click.option(
    "--no-backward-comp",
    is_flag=True,
    show_default=True,
    help="Whether to check backward compatibility or not.",
)
@click.option(
    "-g",
    "--use-git",
    is_flag=True,
    show_default=True,
    default=False,
    help="Validate changes using git - this will check current branch's changes against origin/master or origin/main. "
    "If the --post-commit flag is supplied: validation will run only on the current branch's changed files "
    "that have been committed. "
    "If the --post-commit flag is not supplied: validation will run on all changed files in the current branch, "
    "both committed and not committed. ",
)
@click.option(
    "-pc",
    "--post-commit",
    is_flag=True,
    help="Whether the validation should run only on the current branch's committed changed files. "
    "This applies only when the -g flag is supplied.",
)
@click.option(
    "-st",
    "--staged",
    is_flag=True,
    help="Whether the validation should ignore unstaged files."
    "This applies only when the -g flag is supplied.",
)
@click.option(
    "-iu",
    "--include-untracked",
    is_flag=True,
    help="Whether to include untracked files in the validation. "
    "This applies only when the -g flag is supplied.",
)
@click.option(
    "-a",
    "--validate-all",
    is_flag=True,
    show_default=True,
    default=False,
    help="Whether to run all validation on all files or not.",
)
@click.option(
    "-i",
    "--input",
    type=PathsParamType(
        exists=True, resolve_path=True
    ),  # PathsParamType allows passing a list of paths
    help="The path of the content pack/file to validate specifically.",
)
@click.option(
    "--skip-pack-release-notes",
    is_flag=True,
    help="Skip validation of pack release notes.",
)
@click.option(
    "--print-ignored-errors", is_flag=True, help="Print ignored errors as warnings."
)
@click.option(
    "--print-ignored-files",
    is_flag=True,
    help="Print which files were ignored by the command.",
)
@click.option(
    "--no-docker-checks", is_flag=True, help="Whether to run docker image validation."
)
@click.option(
    "--silence-init-prints",
    is_flag=True,
    help="Whether to skip the initialization prints.",
)
@click.option(
    "--skip-pack-dependencies",
    is_flag=True,
    help="Skip validation of pack dependencies.",
)
@click.option(
    "--create-id-set", is_flag=True, help="Whether to create the id_set.json file."
)
@click.option(
    "-j",
    "--json-file",
    help="The JSON file path to which to output the command results.",
)
@click.option(
    "--skip-schema-check", is_flag=True, help="Whether to skip the file schema check."
)
@click.option(
    "--debug-git", is_flag=True, help="Whether to print debug logs for git statuses."
)
@click.option(
    "--print-pykwalify", is_flag=True, help="Whether to print the pykwalify log errors."
)
@click.option(
    "--quiet-bc-validation",
    help="Set backwards compatibility validation's errors as warnings.",
    is_flag=True,
)
@click.option(
    "--allow-skipped",
    help="Don't fail on skipped integrations or when all test playbooks are skipped.",
    is_flag=True,
)
@click.option(
    "--no-multiprocessing",
    help="run validate all without multiprocessing, for debugging purposes.",
    is_flag=True,
    default=False,
)
@click.option(
    "-sv",
    "--run-specific-validations",
    help="Run specific validations by stating the error codes.",
    is_flag=False,
)
@click.argument("file_paths", nargs=-1, type=click.Path(exists=True, resolve_path=True))
@pass_config
@click.pass_context
@logging_setup_decorator
def validate(ctx, config, file_paths: str, **kwargs):
    """Validate your content files. If no additional flags are given, will validated only committed files."""
    from demisto_sdk.commands.validate.validate_manager import ValidateManager

    if is_sdk_defined_working_offline():
        logger.error(SDK_OFFLINE_ERROR_MESSAGE)
        sys.exit(1)

    if file_paths and not kwargs["input"]:
        # If file_paths is given as an argument, use it as the file_paths input (instead of the -i flag). If both, input wins.
        kwargs["input"] = ",".join(file_paths)
    run_with_mp = not kwargs.pop("no_multiprocessing")
    check_configuration_file("validate", kwargs)
    sys.path.append(config.configuration.env_dir)

    file_path = kwargs["input"]

    if kwargs["post_commit"] and kwargs["staged"]:
        logger.info(
            "[red]Could not supply the staged flag with the post-commit flag[/red]"
        )
        sys.exit(1)
    try:
        is_external_repo = is_external_repository()
        # default validate to -g --post-commit
        if not kwargs.get("validate_all") and not kwargs["use_git"] and not file_path:
            kwargs["use_git"] = True
            kwargs["post_commit"] = True
        validator = ValidateManager(
            is_backward_check=not kwargs["no_backward_comp"],
            only_committed_files=kwargs["post_commit"],
            prev_ver=kwargs["prev_ver"],
            skip_conf_json=kwargs["no_conf_json"],
            use_git=kwargs["use_git"],
            file_path=file_path,
            validate_all=kwargs.get("validate_all"),
            validate_id_set=kwargs["id_set"],
            validate_graph=kwargs.get("graph"),
            skip_pack_rn_validation=kwargs["skip_pack_release_notes"],
            print_ignored_errors=kwargs["print_ignored_errors"],
            is_external_repo=is_external_repo,
            print_ignored_files=kwargs["print_ignored_files"],
            no_docker_checks=kwargs["no_docker_checks"],
            silence_init_prints=kwargs["silence_init_prints"],
            skip_dependencies=kwargs["skip_pack_dependencies"],
            id_set_path=kwargs.get("id_set_path"),
            staged=kwargs["staged"],
            create_id_set=kwargs.get("create_id_set"),
            json_file_path=kwargs.get("json_file"),
            skip_schema_check=kwargs.get("skip_schema_check"),
            debug_git=kwargs.get("debug_git"),
            include_untracked=kwargs.get("include_untracked"),
            quiet_bc=kwargs.get("quiet_bc_validation"),
            multiprocessing=run_with_mp,
            check_is_unskipped=not kwargs.get("allow_skipped", False),
            specific_validations=kwargs.get("run_specific_validations"),
        )
        return validator.run_validation()
    except (git.InvalidGitRepositoryError, git.NoSuchPathError, FileNotFoundError) as e:
        logger.info(f"[red]{e}[/red]")
        logger.info(
            "\n[red]You may not be running `demisto-sdk validate` command in the content directory.\n"
            "Please run the command from content directory[red]"
        )
        sys.exit(1)


# ====================== create-content-artifacts ====================== #
@main.command(hidden=True)
@click.help_option("-h", "--help")
@click.option(
    "-a",
    "--artifacts_path",
    help="Destination directory to create the artifacts.",
    type=click.Path(file_okay=False, resolve_path=True),
    required=True,
)
@click.option("--zip/--no-zip", help="Zip content artifacts folders", default=True)
@click.option(
    "--packs",
    help="Create only content_packs artifacts. "
    "Used for server version 5.5.0 and earlier.",
    is_flag=True,
)
@click.option(
    "-v",
    "--content_version",
    help="The content version in CommonServerPython.",
    default="0.0.0",
)
@click.option(
    "-s",
    "--suffix",
    help="Suffix to add all yaml/json/yml files in the created artifacts.",
)
@click.option(
    "--cpus",
    help="Number of cpus/vcpus available - only required when os not reflect number of cpus (CircleCI"
    "always show 32, but medium has 3.",
    hidden=True,
    default=cpu_count(),
)
@click.option(
    "-idp",
    "--id-set-path",
    help="The full path of id_set.json",
    hidden=True,
    type=click.Path(exists=True, resolve_path=True),
)
@click.option(
    "-p",
    "--pack-names",
    help=(
        "Packs to create artifacts for. Optional values are: `all` or "
        "csv list of packs. "
        "Default is set to `all`"
    ),
    default="all",
    hidden=True,
)
@click.option(
    "-sk",
    "--signature-key",
    help="Base64 encoded signature key used for signing packs.",
    hidden=True,
)
@click.option(
    "-sd",
    "--sign-directory",
    help="Path to the signDirectory executable file.",
    type=click.Path(exists=True, resolve_path=True),
    hidden=True,
)
@click.option(
    "-rt",
    "--remove-test-playbooks",
    is_flag=True,
    help="Should remove test playbooks from content packs or not.",
    default=True,
    hidden=True,
)
@click.option(
    "-mp",
    "--marketplace",
    help="The marketplace the artifacts are created for, that "
    "determines which artifacts are created for each pack. "
    "Default is the XSOAR marketplace, that has all of the packs "
    "artifacts.",
    default="xsoar",
    type=click.Choice(["xsoar", "marketplacev2", "v2", "xpanse"]),
)
@click.option(
    "-fbi",
    "--filter-by-id-set",
    is_flag=True,
    help="Whether to use the id set as content items guide, meaning only include in the packs the "
    "content items that appear in the id set.",
    default=False,
    hidden=True,
)
@click.option(
    "-af",
    "--alternate-fields",
    is_flag=True,
    help="Use the alternative fields if such are present in the yml or json of the content item.",
    default=False,
    hidden=True,
)
@click.pass_context
@logging_setup_decorator
def create_content_artifacts(ctx, **kwargs) -> int:
    """Generating the following artifacts:
    1. content_new - Contains all content objects of type json,yaml (from_version < 6.0.0)
    2. content_packs - Contains all packs from Packs - Ignoring internal files (to_version >= 6.0.0).
    3. content_test - Contains all test scripts/playbooks (from_version < 6.0.0)
    4. content_all - Contains all from content_new and content_test.
    5. uploadable_packs - Contains zipped packs that are ready to be uploaded to Cortex XSOAR machine.
    """
    from demisto_sdk.commands.create_artifacts.content_artifacts_creator import (
        ArtifactsManager,
    )

    check_configuration_file("create-content-artifacts", kwargs)
    if marketplace := kwargs.get("marketplace"):
        os.environ[ENV_DEMISTO_SDK_MARKETPLACE] = marketplace.lower()
    artifacts_conf = ArtifactsManager(**kwargs)
    return artifacts_conf.create_content_artifacts()


# ====================== secrets ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option("-i", "--input", help="Specify file of to check secret on.")
@click.option(
    "--post-commit",
    is_flag=True,
    show_default=True,
    help="Whether the secretes is done after you committed your files, "
    "this will help the command to determine which files it should check in its "
    "run. Before you commit the files it should not be used. Mostly for build "
    "validations.",
)
@click.option(
    "-ie",
    "--ignore-entropy",
    is_flag=True,
    help="Ignore entropy algorithm that finds secret strings (passwords/api keys).",
)
@click.option(
    "-wl",
    "--whitelist",
    default="./Tests/secrets_white_list.json",
    show_default=True,
    help='Full path to whitelist file, file name should be "secrets_white_list.json"',
)
@click.option("--prev-ver", help="The branch against which to run secrets validation.")
@click.argument("file_paths", nargs=-1, type=click.Path(exists=True, resolve_path=True))
@pass_config
@click.pass_context
@logging_setup_decorator
def secrets(ctx, config, file_paths: str, **kwargs):
    """Run Secrets validator to catch sensitive data before exposing your code to public repository.
    Attach path to whitelist to allow manual whitelists.
    """
    if file_paths and not kwargs["input"]:
        # If file_paths is given as an argument, use it as the file_paths input (instead of the -i flag). If both, input wins.
        kwargs["input"] = ",".join(file_paths)

    from demisto_sdk.commands.secrets.secrets import SecretsValidator

    check_configuration_file("secrets", kwargs)
    sys.path.append(config.configuration.env_dir)
    secrets_validator = SecretsValidator(
        configuration=config.configuration,
        is_circle=kwargs["post_commit"],
        ignore_entropy=kwargs["ignore_entropy"],
        white_list_path=kwargs["whitelist"],
        input_path=kwargs.get("input"),
    )
    return secrets_validator.run()


# ====================== lint ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="Specify directory(s) of integration/script",
    type=PathsParamType(exists=True, resolve_path=True),
)
@click.option("-g", "--git", is_flag=True, help="Will run only on changed packages")
@click.option(
    "-a",
    "--all-packs",
    is_flag=True,
    help="Run lint on all directories in content repo",
)
@click.option(
    "-p",
    "--parallel",
    default=1,
    help="Run tests in parallel",
    type=click.IntRange(0, 15, clamp=True),
    show_default=True,
)
@click.option("--no-flake8", is_flag=True, help="Do NOT run flake8 linter")
@click.option("--no-bandit", is_flag=True, help="Do NOT run bandit linter")
@click.option("--no-xsoar-linter", is_flag=True, help="Do NOT run XSOAR linter")
@click.option("--no-mypy", is_flag=True, help="Do NOT run mypy static type checking")
@click.option("--no-vulture", is_flag=True, help="Do NOT run vulture linter")
@click.option("--no-pylint", is_flag=True, help="Do NOT run pylint linter")
@click.option("--no-test", is_flag=True, help="Do NOT test (skip pytest)")
@click.option("--no-pwsh-analyze", is_flag=True, help="Do NOT run powershell analyze")
@click.option("--no-pwsh-test", is_flag=True, help="Do NOT run powershell test")
@click.option("-kc", "--keep-container", is_flag=True, help="Keep the test container")
@click.option(
    "--prev-ver",
    help="Previous branch or SHA1 commit to run checks against",
    default="master",
)
@click.option(
    "--test-xml",
    help="Path to store pytest xml results",
    type=click.Path(exists=True, resolve_path=True),
)
@click.option(
    "--failure-report",
    help="Path to store failed packs report",
    type=click.Path(exists=True, resolve_path=True),
)
@click.option(
    "-j",
    "--json-file",
    help="The JSON file path to which to output the command results.",
    type=click.Path(resolve_path=True),
)
@click.option("--no-coverage", is_flag=True, help="Do NOT run coverage report.")
@click.option(
    "--coverage-report",
    help="Specify directory for the coverage report files",
    type=PathsParamType(),
)
@click.option(
    "-dt",
    "--docker-timeout",
    default=60,
    help="The timeout (in seconds) for requests done by the docker client.",
    type=int,
)
@click.option(
    "-di",
    "--docker-image",
    default="from-yml",
    help="The docker image to check package on. Can be a comma separated list of Possible values: 'native:maintenance', 'native:ga', 'native:dev',"
    " 'all', a specific docker image from Docker Hub (e.g devdemisto/python3:3.10.9.12345) or the default"
    " 'from-yml', 'native:target'. To run lint only on native supported content with a specific image,"
    " use 'native:target' with --docker-image-target <specific-image>.",
)
@click.option(
    "-dit",
    "--docker-image-target",
    default="",
    help="The docker image to lint native supported content with. Should only be used with "
    "--docker-image native:target. An error will be raised otherwise.",
)
@click.option(
    "-cdam",
    "--check-dependent-api-module",
    is_flag=True,
    help="Run unit tests and lint on all packages that "
    "are dependent on the found "
    "modified api modules.",
    default=False,
)
@click.option(
    "--time-measurements-dir",
    help="Specify directory for the time measurements report file",
    type=PathsParamType(),
)
@click.pass_context
@logging_setup_decorator
def lint(ctx, **kwargs):
    """Lint command will perform:
    1. Package in host checks - flake8, bandit, mypy, vulture.
    2. Package in docker image checks -  pylint, pytest, powershell - test, powershell - analyze.
    Meant to be used with integrations/scripts that use the folder (package) structure.
    Will lookup up what docker image to use and will setup the dev dependencies and file in the target folder.
    If no additional flags specifying the packs are given, will lint only changed files.
    """
    from demisto_sdk.commands.lint.lint_manager import LintManager

    check_configuration_file("lint", kwargs)
    lint_manager = LintManager(
        input=kwargs.get("input"),  # type: ignore[arg-type]
        git=kwargs.get("git"),  # type: ignore[arg-type]
        all_packs=kwargs.get("all_packs"),  # type: ignore[arg-type]
        prev_ver=kwargs.get("prev_ver"),  # type: ignore[arg-type]
        json_file_path=kwargs.get("json_file"),  # type: ignore[arg-type]
        check_dependent_api_module=kwargs.get("check_dependent_api_module"),  # type: ignore[arg-type]
    )
    return lint_manager.run(
        parallel=kwargs.get("parallel"),  # type: ignore[arg-type]
        no_flake8=kwargs.get("no_flake8"),  # type: ignore[arg-type]
        no_bandit=kwargs.get("no_bandit"),  # type: ignore[arg-type]
        no_mypy=kwargs.get("no_mypy"),  # type: ignore[arg-type]
        no_vulture=kwargs.get("no_vulture"),  # type: ignore[arg-type]
        no_xsoar_linter=kwargs.get("no_xsoar_linter"),  # type: ignore[arg-type]
        no_pylint=kwargs.get("no_pylint"),  # type: ignore[arg-type]
        no_test=kwargs.get("no_test"),  # type: ignore[arg-type]
        no_pwsh_analyze=kwargs.get("no_pwsh_analyze"),  # type: ignore[arg-type]
        no_pwsh_test=kwargs.get("no_pwsh_test"),  # type: ignore[arg-type]
        keep_container=kwargs.get("keep_container"),  # type: ignore[arg-type]
        test_xml=kwargs.get("test_xml"),  # type: ignore[arg-type]
        failure_report=kwargs.get("failure_report"),  # type: ignore[arg-type]
        no_coverage=kwargs.get("no_coverage"),  # type: ignore[arg-type]
        coverage_report=kwargs.get("coverage_report"),  # type: ignore[arg-type]
        docker_timeout=kwargs.get("docker_timeout"),  # type: ignore[arg-type]
        docker_image_flag=kwargs.get("docker_image"),  # type: ignore[arg-type]
        docker_image_target=kwargs.get("docker_image_target"),  # type: ignore[arg-type]
        time_measurements_dir=kwargs.get("time_measurements_dir"),  # type: ignore[arg-type]
    )


# ====================== coverage-analyze ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="The .coverage file to analyze.",
    default=os.path.join("coverage_report", ".coverage"),
    type=PathsParamType(resolve_path=True),
)
@click.option(
    "--default-min-coverage",
    help="Default minimum coverage (for new files).",
    default=70.0,
    type=click.FloatRange(0.0, 100.0),
)
@click.option(
    "--allowed-coverage-degradation-percentage",
    help="Allowed coverage degradation percentage (for modified files).",
    default=1.0,
    type=click.FloatRange(0.0, 100.0),
)
@click.option(
    "--no-cache",
    help="Force download of the previous coverage report file.",
    is_flag=True,
    type=bool,
)
@click.option(
    "--report-dir",
    help="Directory of the coverage report files.",
    default="coverage_report",
    type=PathsParamType(resolve_path=True),
)
@click.option(
    "--report-type",
    help="The type of coverage report (posible values: 'text', 'html', 'xml', 'json' or 'all').",
    type=str,
)
@click.option(
    "--no-min-coverage-enforcement",
    help="Do not enforce minimum coverage.",
    is_flag=True,
)
@click.option(
    "--previous-coverage-report-url",
    help="URL of the previous coverage report.",
    default="https://storage.googleapis.com/marketplace-dist-dev/code-coverage-reports/coverage-min.json",
    type=str,
)
@click.pass_context
def coverage_analyze(ctx, **kwargs):
    logger = logging_setup(
        console_log_threshold=kwargs.get("console_log_threshold") or logging.INFO,
        file_log_threshold=kwargs.get("file_log_threshold") or logging.DEBUG,
        log_file_path=kwargs.get("log_file_path") or None,
    )
    from demisto_sdk.commands.coverage_analyze.coverage_report import CoverageReport

    try:
        no_degradation_check = (
            kwargs["allowed_coverage_degradation_percentage"] == 100.0
        )
        no_min_coverage_enforcement = kwargs["no_min_coverage_enforcement"]

        cov_report = CoverageReport(
            default_min_coverage=kwargs["default_min_coverage"],
            allowed_coverage_degradation_percentage=kwargs[
                "allowed_coverage_degradation_percentage"
            ],
            coverage_file=kwargs["input"],
            no_cache=kwargs.get("no_cache", False),
            report_dir=kwargs["report_dir"],
            report_type=kwargs["report_type"],
            no_degradation_check=no_degradation_check,
            previous_coverage_report_url=kwargs["previous_coverage_report_url"],
        )
        cov_report.coverage_report()
        # if no_degradation_check=True we will suppress the minimum coverage check
        if (
            no_degradation_check
            or cov_report.coverage_diff_report()
            or no_min_coverage_enforcement
        ):
            return 0
    except Exception as error:
        logger.error(error)

    return 1


# ====================== format ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="The path of the script yml file or a comma separated list\n"
    "If no input is specified, the format will be executed on all new/changed files.",
    type=PathsParamType(
        exists=True, resolve_path=True
    ),  # PathsParamType allows passing a list of paths
)
@click.option(
    "-o",
    "--output",
    help="The path where the formatted file will be saved to",
    type=click.Path(resolve_path=True),
)
@click.option("-fv", "--from-version", help="Specify fromversion of the pack")
@click.option(
    "-nv", "--no-validate", help="Set when validate on file is not wanted", is_flag=True
)
@click.option(
    "-ud",
    "--update-docker",
    help="Set if you want to update the docker image of the integration/script",
    is_flag=True,
)
@click.option(
    "-y/-n",
    "--assume-yes/--assume-no",
    help="Automatic yes/no to prompts; assume 'yes'/'no' as answer to all prompts and run non-interactively",
    is_flag=True,
    default=None,
)
@click.option(
    "-d",
    "--deprecate",
    help="Set if you want to deprecate the integration/script/playbook",
    is_flag=True,
)
@click.option(
    "-g",
    "--use-git",
    help="Use git to automatically recognize which files changed and run format on them.",
    is_flag=True,
)
@click.option(
    "--prev-ver", help="Previous branch or SHA1 commit to run checks against."
)
@click.option(
    "-iu",
    "--include-untracked",
    is_flag=True,
    help="Whether to include untracked files in the formatting.",
)
@click.option(
    "-at",
    "--add-tests",
    is_flag=True,
    help="Whether to answer manually to add tests configuration prompt when running interactively.",
)
@click.option(
    "-s",
    "--id-set-path",
    help="Deprecated. The path of the id_set json file.",
    type=click.Path(exists=True, resolve_path=True),
)
@click.option(
    "-gr/-ngr",
    "--graph/--no-graph",
    help="Whether to use the content graph or not.",
    is_flag=True,
    default=True,
)
@click.argument("file_paths", nargs=-1, type=click.Path(exists=True, resolve_path=True))
@click.pass_context
@logging_setup_decorator
def format(
    ctx,
    input: str,
    output: Path,
    from_version: str,
    no_validate: bool,
    update_docker: bool,
    assume_yes: Union[None, bool],
    deprecate: bool,
    use_git: bool,
    prev_ver: str,
    include_untracked: bool,
    add_tests: bool,
    id_set_path: str,
    file_paths: Tuple[str, ...],
    **kwargs,
):
    """Run formatter on a given script/playbook/integration/incidentfield/indicatorfield/
    incidenttype/indicatortype/layout/dashboard/classifier/mapper/widget/report file/genericfield/generictype/
    genericmodule/genericdefinition.
    """
    from demisto_sdk.commands.format.format_module import format_manager

    if is_sdk_defined_working_offline():
        logger.error(SDK_OFFLINE_ERROR_MESSAGE)
        sys.exit(1)

    if file_paths and not input:
        input = ",".join(file_paths)

    with ReadMeValidator.start_mdx_server():
        return format_manager(
            str(input) if input else None,
            str(output) if output else None,
            from_version=from_version,
            no_validate=no_validate,
            update_docker=update_docker,
            assume_answer=assume_yes,
            deprecate=deprecate,
            use_git=use_git,
            prev_ver=prev_ver,
            include_untracked=include_untracked,
            add_tests=add_tests,
            id_set_path=id_set_path,
            use_graph=kwargs.get("graph", True),
        )


# ====================== upload ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    type=PathsParamType(exists=True, resolve_path=True),
    help="The path of file or a directory to upload. The following are supported:\n"
    "- Pack\n"
    "- A content entity directory that is inside a pack. For example: an Integrations "
    "directory or a Layouts directory.\n"
    "- Valid file that can be imported to Cortex XSOAR manually. For example a playbook: "
    "helloWorld.yml",
    required=False,
)
@click.option(
    "--input-config-file",
    type=PathsParamType(exists=True, resolve_path=True),
    help="The path to the config file to download all the custom packs from",
    required=False,
)
@click.option(
    "-z/-nz",
    "--zip/--no-zip",
    help="Compress the pack to zip before upload, this flag is relevant only for packs.",
    is_flag=True,
    default=True,
)
@click.option(
    "-x",
    "--xsiam",
    help="Upload the pack to XSIAM server. Must be used together with -z",
    is_flag=True,
)
@click.option(
    "-mp",
    "--marketplace",
    help="The marketplace to which the content will be uploaded.",
)
@click.option(
    "--keep-zip",
    help="Directory where to store the zip after creation, this argument is relevant only for packs "
    "and in case the --zip flag is used.",
    required=False,
    type=click.Path(exists=True),
)
@click.option("--insecure", help="Skip certificate validation", is_flag=True)
@click.option(
    "--skip_validation",
    is_flag=True,
    help="Only for upload zipped packs, "
    "if true will skip upload packs validation, use just when migrate existing custom content to packs.",
)
@click.option(
    "--reattach",
    help="Reattach the detached files in the XSOAR instance"
    "for the CI/CD Flow. If you set the --input-config-file flag, "
    "any detached item in your XSOAR instance that isn't currently in the repo's SystemPacks folder "
    "will be re-attached.)",
    is_flag=True,
)
@click.option(
    "--override-existing",
    is_flag=True,
    help="This value (True/False) determines if the user should be presented with a confirmation prompt when "
    "attempting to upload a content pack that is already installed on the Cortex XSOAR server. This allows the upload "
    "command to be used within non-interactive shells.",
)
@click.pass_context
@logging_setup_decorator
def upload(ctx, **kwargs):
    """Upload integration or pack to Demisto instance.
    DEMISTO_BASE_URL environment variable should contain the Demisto server base URL.
    DEMISTO_API_KEY environment variable should contain a valid Demisto API Key.
    * Note: Uploading classifiers to Cortex XSOAR is available from version 6.0.0 and up. *
    """
    return upload_content_entity(**kwargs)


# ====================== download ====================== #


@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-o",
    "--output",
    help="The path of a package directory to download custom content to",
    required=False,
    multiple=False,
)
@click.option(
    "-i",
    "--input",
    help="Custom content file name to be downloaded. Can be provided multiple times",
    required=False,
    multiple=True,
)
@click.option(
    "-r",
    "--regex",
    help="Regex Pattern, download all the custom content files that match this regex pattern.",
    required=False,
)
@click.option("--insecure", help="Skip certificate validation", is_flag=True)
@click.option(
    "-f", "--force", help="Whether to override existing files or not", is_flag=True
)
@click.option(
    "-lf",
    "--list-files",
    help="Prints a list of all custom content files available to be downloaded",
    is_flag=True,
)
@click.option(
    "-a",
    "--all-custom-content",
    help="Download all available custom content files",
    is_flag=True,
)
@click.option(
    "-fmt",
    "--run-format",
    help="Whether to run demisto-sdk format on downloaded files or not",
    is_flag=True,
)
@click.option("--system", help="Download system items", is_flag=True, default=False)
@click.option(
    "-it",
    "--item-type",
    help="The items type to download, use just when downloading system items, should be one "
    "form the following list: [IncidentType, IndicatorType, Field, Layout, Playbook, "
    "Automation, Classifier, Mapper]",
    type=click.Choice(
        [
            "IncidentType",
            "IndicatorType",
            "Field",
            "Layout",
            "Playbook",
            "Automation",
            "Classifier",
            "Mapper",
        ],
        case_sensitive=False,
    ),
)
@click.option(
    "--init",
    help="Create a directory structure and download the items to it",
    is_flag=True,
    default=False,
)
@click.option(
    "--keep-empty-folders",
    help="Keep empty folders when using the --init flag",
    is_flag=True,
    default=False,
)
@click.pass_context
@logging_setup_decorator
def download(ctx, **kwargs):
    """Download custom content from Demisto instance.
    DEMISTO_BASE_URL environment variable should contain the Demisto server base URL.
    DEMISTO_API_KEY environment variable should contain a valid Demisto API Key.
    """
    from demisto_sdk.commands.download.downloader import Downloader

    check_configuration_file("download", kwargs)
    downloader: Downloader = Downloader(**kwargs)
    return downloader.download()


# ====================== update-xsoar-config-file ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-pi",
    "--pack-id",
    help="The Pack ID to add to XSOAR Configuration File",
    required=False,
    multiple=False,
)
@click.option(
    "-pd",
    "--pack-data",
    help="The Pack Data to add to XSOAR Configuration File - "
    "Pack URL for Custom Pack and Pack Version for OOTB Pack",
    required=False,
    multiple=False,
)
@click.option(
    "-mp",
    "--add-marketplace-pack",
    help="Add a Pack to the MarketPlace Packs section in the Configuration File",
    required=False,
    is_flag=True,
)
@click.option(
    "-cp",
    "--add-custom-pack",
    help="Add the Pack to the Custom Packs section in the Configuration File",
    is_flag=True,
)
@click.option(
    "-all",
    "--add-all-marketplace-packs",
    help="Add all the installed MarketPlace Packs to the marketplace_packs in XSOAR Configuration File",
    is_flag=True,
)
@click.option("--insecure", help="Skip certificate validation", is_flag=True)
@click.option(
    "--file-path",
    help="XSOAR Configuration File path, the default value is in the repo level",
    is_flag=False,
)
@click.pass_context
@logging_setup_decorator
def xsoar_config_file_update(ctx, **kwargs):
    """Handle your XSOAR Configuration File.
    Add automatically all the installed MarketPlace Packs to the marketplace_packs section in XSOAR Configuration File.
    Add a Pack to both marketplace_packs and custom_packs sections in the Configuration File.
    """
    from demisto_sdk.commands.update_xsoar_config_file.update_xsoar_config_file import (
        XSOARConfigFileUpdater,
    )

    file_updater: XSOARConfigFileUpdater = XSOARConfigFileUpdater(**kwargs)
    return file_updater.update()


# ====================== run ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option("-q", "--query", help="The query to run", required=True)
@click.option("--insecure", help="Skip certificate validation", is_flag=True)
@click.option(
    "-id",
    "--incident-id",
    help="The incident to run the query on, if not specified the playground will be used.",
)
@click.option(
    "-D",
    "--debug",
    help="Whether to enable the debug-mode feature or not, if you want to save the output file "
    "please use the --debug-path option",
    is_flag=True,
)
@click.option(
    "--debug-path",
    help="The path to save the debug file at, if not specified the debug file will be printed to the "
    "terminal",
)
@click.option(
    "--json-to-outputs",
    help="Whether to run json_to_outputs command on the context output of the query. If the "
    "context output does not exists or the `-r` flag is used, will use the raw"
    " response of the query",
    is_flag=True,
)
@click.option(
    "-p",
    "--prefix",
    help="Used with `json-to-outputs` flag. Output prefix e.g. Jira.Ticket, VirusTotal.IP, "
    "the base path for the outputs that the script generates",
)
@click.option(
    "-r",
    "--raw-response",
    help="Used with `json-to-outputs` flag. Use the raw response of the query for"
    " `json-to-outputs`",
    is_flag=True,
)
@click.pass_context
@logging_setup_decorator
def run(ctx, **kwargs):
    """Run integration command on remote Demisto instance in the playground.
    DEMISTO_BASE_URL environment variable should contain the Demisto base URL.
    DEMISTO_API_KEY environment variable should contain a valid Demisto API Key.
    """
    from demisto_sdk.commands.run_cmd.runner import Runner

    check_configuration_file("run", kwargs)
    runner = Runner(**kwargs)
    return runner.run()


# ====================== run-playbook ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "--url",
    "-u",
    help="URL to a Demisto instance. You can also specify the URL as an environment variable named: DEMISTO_BASE_URL",
)
@click.option("--playbook_id", "-p", help="The playbook ID to run.", required=True)
@click.option(
    "--wait",
    "-w",
    is_flag=True,
    help="Wait until the playbook run is finished and get a response.",
)
@click.option(
    "--timeout",
    "-t",
    default=90,
    show_default=True,
    help="Timeout for the command. The playbook will continue to run in Demisto",
)
@click.option("--insecure", help="Skip certificate validation.", is_flag=True)
@click.pass_context
@logging_setup_decorator
def run_playbook(ctx, **kwargs):
    """Run a playbook in Demisto.
    DEMISTO_API_KEY environment variable should contain a valid Demisto API Key.
    Example: DEMISTO_API_KEY=<API KEY> demisto-sdk run-playbook -p 'p_name' -u
    'https://demisto.local'.
    """
    from demisto_sdk.commands.run_playbook.playbook_runner import PlaybookRunner

    check_configuration_file("run-playbook", kwargs)
    playbook_runner = PlaybookRunner(**kwargs)
    return playbook_runner.run_playbook()


# ====================== run-test-playbook ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-tpb",
    "--test-playbook-path",
    help="Path to test playbook to run, "
    "can be a path to specific test playbook or path to pack name for example: Packs/GitHub.",
    required=False,
)
@click.option(
    "--all", is_flag=True, help="Run all the test playbooks from this repository."
)
@click.option(
    "--wait",
    "-w",
    is_flag=True,
    default=True,
    help="Wait until the test-playbook run is finished and get a response.",
)
@click.option(
    "--timeout",
    "-t",
    default=90,
    show_default=True,
    help="Timeout for the command. The test-playbook will continue to run in your instance",
)
@click.option("--insecure", help="Skip certificate validation.", is_flag=True)
@click.pass_context
@logging_setup_decorator
def run_test_playbook(ctx, **kwargs):
    """Run a test playbooks in your instance."""
    from demisto_sdk.commands.run_test_playbook.test_playbook_runner import (
        TestPlaybookRunner,
    )

    check_configuration_file("run-test-playbook", kwargs)
    test_playbook_runner = TestPlaybookRunner(**kwargs)
    return test_playbook_runner.manage_and_run_test_playbooks()


# ====================== generate-outputs ====================== #
@main.command(short_help="""Generates outputs (from json or examples).""")
@click.help_option("-h", "--help")
@click.option(
    "-c",
    "--command",
    help="Specific command name (e.g. xdr-get-incidents)",
    required=False,
)
@click.option(
    "-j",
    "--json",
    help="Valid JSON file path. If not specified, the script will wait for user input in the terminal. "
    "The response can be obtained by running the command with `raw-response=true` argument.",
    required=False,
)
@click.option(
    "-p",
    "--prefix",
    help="Output prefix like Jira.Ticket, VirusTotal.IP, the base path for the outputs that the "
    "script generates",
    required=False,
)
@click.option(
    "-o",
    "--output",
    help="Output file path, if not specified then will print to stdout",
    required=False,
)
@click.option(
    "--ai",
    is_flag=True,
    help="**Experimental** - Help generate context descriptions via AI transformers (must have a valid AI21 key at ai21.com)",
)
@click.option(
    "--interactive",
    help="If passed, then for each output field will ask user interactively to enter the "
    "description. By default is interactive mode is disabled. No need to use with --ai (it is already interactive)",
    is_flag=True,
)
@click.option(
    "-d",
    "--descriptions",
    help="A JSON or a path to a JSON file, mapping field names to their descriptions. "
    "If not specified, the script prompt the user to input the JSON content.",
    is_flag=True,
)
@click.option("-i", "--input", help="Valid YAML integration file path.", required=False)
@click.option(
    "-e",
    "--examples",
    help="Integrations: path for file containing command examples."
    " Each command should be in a separate line."
    " Scripts: the script example surrounded by quotes."
    " For example: -e '!ConvertFile entry_id=<entry_id>'",
)
@click.option(
    "--insecure",
    help="Skip certificate validation to run the commands in order to generate the docs.",
    is_flag=True,
)
@click.pass_context
@logging_setup_decorator
def generate_outputs(ctx, **kwargs):
    """Demisto integrations/scripts have a YAML file that defines them.
    Creating the YAML file is a tedious and error-prone task of manually copying outputs from the API result to the
    file/UI/PyCharm. This script auto generates the YAML for a command from the JSON result of the relevant API call
    In addition you can supply examples files and generate the context description directly in the YML from those examples.
    """
    from demisto_sdk.commands.generate_outputs.generate_outputs import (
        run_generate_outputs,
    )

    check_configuration_file("generate-outputs", kwargs)
    return run_generate_outputs(**kwargs)


# ====================== generate-test-playbook ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i", "--input", required=True, help="Specify integration/script yml path"
)
@click.option(
    "-o",
    "--output",
    required=False,
    help="Specify output directory or path to an output yml file. "
    "If a path to a yml file is specified - it will be the output path.\n"
    "If a folder path is specified - a yml output will be saved in the folder.\n"
    "If not specified, and the input is located at `.../Packs/<pack_name>/Integrations`, "
    "the output will be saved under `.../Packs/<pack_name>/TestPlaybooks`.\n"
    "Otherwise (no folder in the input hierarchy is named `Packs`), "
    "the output will be saved in the current directory.",
)
@click.option(
    "-n",
    "--name",
    required=True,
    help="Specify test playbook name. The output file name will be `playbook-<name>_Test.yml",
)
@click.option(
    "--no-outputs",
    is_flag=True,
    help="Skip generating verification conditions for each output contextPath. Use when you want to decide which "
    "outputs to verify and which not",
)
@click.option(
    "-ab",
    "--all-brands",
    "use_all_brands",
    help="Generate a test-playbook which calls commands using integrations of all available brands. "
    "When not used, the generated playbook calls commands using instances of the provided integration brand.",
    is_flag=True,
)
@click.option(
    "-c",
    "--commands",
    help="A comma-separated command names to generate playbook tasks for, "
    "will ignore the rest of the commands."
    "e.g xdr-get-incidents,xdr-update-incident",
    required=False,
)
@click.option(
    "-e",
    "--examples",
    help="For integrations: path for file containing command examples."
    " Each command should be in a separate line."
    " For scripts: the script example surrounded by quotes."
    " For example: -e '!ConvertFile entry_id=<entry_id>'",
)
@click.option(
    "-u",
    "--upload",
    help="Whether to upload the test playbook after the generation.",
    is_flag=True,
)
@click.pass_context
@logging_setup_decorator
def generate_test_playbook(ctx, **kwargs):
    """Generate test playbook from integration or script"""
    from demisto_sdk.commands.generate_test_playbook.test_playbook_generator import (
        PlaybookTestsGenerator,
    )

    check_configuration_file("generate-test-playbook", kwargs)
    file_type: FileType = find_type(kwargs.get("input", ""), ignore_sub_categories=True)
    if file_type not in [FileType.INTEGRATION, FileType.SCRIPT]:
        logger.info(
            "[red]Generating test playbook is possible only for an Integration or a Script.[/red]"
        )
        return 1

    try:
        generator = PlaybookTestsGenerator(file_type=file_type.value, **kwargs)
        if generator.run():
            sys.exit(0)
        sys.exit(1)
    except PlaybookTestsGenerator.InvalidOutputPathError as e:
        logger.info(f"[red]{e}[/red]")
        return 1


# ====================== init ====================== #


@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-n", "--name", help="The name of the directory and file you want to create"
)
@click.option("--id", help="The id used in the yml file of the integration or script")
@click.option(
    "-o",
    "--output",
    help="The output dir to write the object into. The default one is the current working "
    "directory.",
)
@click.option(
    "--integration",
    is_flag=True,
    help="Create an Integration based on BaseIntegration template",
)
@click.option(
    "--script", is_flag=True, help="Create a Script based on BaseScript example"
)
@click.option(
    "--xsiam",
    is_flag=True,
    help="Create an Event Collector based on a template, and create matching sub directories",
)
@click.option("--pack", is_flag=True, help="Create pack and its sub directories")
@click.option(
    "-t",
    "--template",
    help="Create an Integration/Script based on a specific template.\n"
    "Integration template options: HelloWorld, HelloIAMWorld, FeedHelloWorld.\n"
    "Script template options: HelloWorldScript",
)
@click.option(
    "-a",
    "--author-image",
    help="Path of the file 'Author_image.png'. \n "
    "Image will be presented in marketplace under PUBLISHER section. File should be up to 4kb and dimensions of 120x50",
)
@click.option(
    "--demisto_mock",
    is_flag=True,
    help="Copy the demistomock. Relevant for initialization of Scripts and Integrations within a Pack.",
)
@click.option(
    "--common-server",
    is_flag=True,
    help="Copy the CommonServerPython. Relevant for initialization of Scripts and Integrations within a Pack.",
)
@click.pass_context
@logging_setup_decorator
def init(ctx, **kwargs):
    """Initialize a new Pack, Integration or Script.
    If the script/integration flags are not present, we will create a pack with the given name.
    Otherwise when using the flags we will generate a script/integration based on your selection.
    """
    from demisto_sdk.commands.init.initiator import Initiator

    check_configuration_file("init", kwargs)
    marketplace = parse_marketplace_kwargs(kwargs)
    initiator = Initiator(marketplace=marketplace, **kwargs)
    initiator.init()
    return 0


# ====================== generate-docs ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option("-i", "--input", help="Path of the yml file.", required=True)
@click.option(
    "-o",
    "--output",
    help="The output dir to write the documentation file into,"
    " documentation file name is README.md. If not specified, will be in the yml dir.",
    required=False,
)
@click.option(
    "-uc",
    "--use_cases",
    help="For integration - Top use-cases. Number the steps by '*' (i.e. '* foo. * bar.')",
    required=False,
)
@click.option(
    "-c",
    "--command",
    help="A comma-separated command names to generate doc for, will ignore the rest of the commands."
    "e.g xdr-get-incidents,xdr-update-incident",
    required=False,
)
@click.option(
    "-e",
    "--examples",
    help="Integrations: path for file containing command examples."
    " Each command should be in a separate line."
    " Scripts: the script example surrounded by quotes."
    " For example: -e '!ConvertFile entry_id=<entry_id>'",
)
@click.option(
    "-p",
    "--permissions",
    type=click.Choice(["none", "general", "per-command"]),
    help="Permissions needed.",
    required=True,
    default="none",
)
@click.option(
    "-cp",
    "--command-permissions",
    help="Path for file containing commands permissions"
    " Each command permissions should be in a separate line."
    " (i.e. '<command-name> Administrator READ-WRITE')",
    required=False,
)
@click.option(
    "-l",
    "--limitations",
    help="Known limitations. Number the steps by '*' (i.e. '* foo. * bar.')",
    required=False,
)
@click.option(
    "--insecure",
    help="Skip certificate validation to run the commands in order to generate the docs.",
    is_flag=True,
)
@click.option("--old-version", help="Path of the old integration version yml file.")
@click.option(
    "--skip-breaking-changes",
    is_flag=True,
    help="Skip generating of breaking changes section.",
)
@click.option(
    "--custom-image-path",
    help="A custom path to a playbook image. If not stated, a default link will be added to the file.",
)
@click.option(
    "-rt",
    "--readme-template",
    help="The readme template that should be appended to the given README.md file",
    type=click.Choice(["syslog", "xdrc", "http-collector"]),
)
@click.option(
    "-gr/-ngr",
    "--graph/--no-graph",
    help="Whether to use the content graph or not.",
    is_flag=True,
    default=True,
)
@click.pass_context
@logging_setup_decorator
def generate_docs(ctx, **kwargs):
    """Generate documentation for integration, playbook or script from yaml file."""
    try:
        check_configuration_file("generate-docs", kwargs)
        input_path_str: str = kwargs.get("input", "")
        if not (input_path := Path(input_path_str)).exists():
            raise Exception(f"[red]input {input_path_str} does not exist[/red]")

        if (output_path := kwargs.get("output")) and not Path(output_path).is_dir():
            raise Exception(
                f"[red]Output directory {output_path} is not a directory.[/red]"
            )

        if input_path.is_file():
            if input_path.suffix.lower() not in {".yml", ".md"}:
                raise Exception(
                    f"[red]input {input_path} is not a valid yml or readme file.[/red]"
                )

            _generate_docs_for_file(kwargs)

        # Add support for input which is a Playbooks directory and not a single yml file
        elif input_path.is_dir() and input_path.name == "Playbooks":
            for yml in input_path.glob("*.yml"):
                file_kwargs = copy.deepcopy(kwargs)
                file_kwargs["input"] = str(yml)
                _generate_docs_for_file(file_kwargs)

        else:
            raise Exception(
                f"[red]Input {input_path} is neither a valid yml file, nor a folder named Playbooks, nor a readme file."
            )

        return 0

    except Exception:
        logger.exception("Failed generating docs")
        sys.exit(1)


def _generate_docs_for_file(kwargs: Dict[str, Any]):
    """Helper function for supporting Playbooks directory as an input and not only a single yml file."""

    from demisto_sdk.commands.generate_docs.generate_integration_doc import (
        generate_integration_doc,
    )
    from demisto_sdk.commands.generate_docs.generate_playbook_doc import (
        generate_playbook_doc,
    )
    from demisto_sdk.commands.generate_docs.generate_readme_template import (
        generate_readme_template,
    )
    from demisto_sdk.commands.generate_docs.generate_script_doc import (
        generate_script_doc,
    )

    # Extract all the necessary arguments
    input_path: str = kwargs.get("input", "")
    output_path = kwargs.get("output")
    command = kwargs.get("command")
    examples: str = kwargs.get("examples", "")
    permissions = kwargs.get("permissions")
    limitations = kwargs.get("limitations")
    insecure: bool = kwargs.get("insecure", False)
    old_version: str = kwargs.get("old_version", "")
    skip_breaking_changes: bool = kwargs.get("skip_breaking_changes", False)
    custom_image_path: str = kwargs.get("custom_image_path", "")
    readme_template: str = kwargs.get("readme_template", "")
    use_graph = kwargs.get("graph", True)

    try:
        if command:
            if (
                output_path
                and (not os.path.isfile(os.path.join(output_path, "README.md")))
                or (not output_path)
                and (
                    not os.path.isfile(
                        os.path.join(
                            os.path.dirname(os.path.realpath(input_path)), "README.md"
                        )
                    )
                )
            ):
                raise Exception(
                    "[red]The `command` argument must be presented with existing `README.md` docs."
                )

        file_type = find_type(kwargs.get("input", ""), ignore_sub_categories=True)
        if file_type not in {
            FileType.INTEGRATION,
            FileType.SCRIPT,
            FileType.PLAYBOOK,
            FileType.README,
        }:
            raise Exception(
                "[red]File is not an Integration, Script, Playbook or a README.[/red]"
            )

        if old_version and not os.path.isfile(old_version):
            raise Exception(
                f"[red]Input old version file {old_version} was not found.[/red]"
            )

        if old_version and not old_version.lower().endswith(".yml"):
            raise Exception(
                f"[red]Input old version {old_version} is not a valid yml file.[/red]"
            )

        if file_type == FileType.INTEGRATION:
            logger.info(f"Generating {file_type.value.lower()} documentation")
            use_cases = kwargs.get("use_cases")
            command_permissions = kwargs.get("command_permissions")
            return generate_integration_doc(
                input_path=input_path,
                output=output_path,
                use_cases=use_cases,
                examples=examples,
                permissions=permissions,
                command_permissions=command_permissions,
                limitations=limitations,
                insecure=insecure,
                command=command,
                old_version=old_version,
                skip_breaking_changes=skip_breaking_changes,
            )
        elif file_type == FileType.SCRIPT:
            logger.info(f"Generating {file_type.value.lower()} documentation")
            return generate_script_doc(
                input_path=input_path,
                output=output_path,
                examples=examples,
                permissions=permissions,
                limitations=limitations,
                insecure=insecure,
                use_graph=use_graph,
            )
        elif file_type == FileType.PLAYBOOK:
            logger.info(f"Generating {file_type.value.lower()} documentation")
            return generate_playbook_doc(
                input_path=input_path,
                output=output_path,
                permissions=permissions,
                limitations=limitations,
                custom_image_path=custom_image_path,
            )

        elif file_type == FileType.README:
            logger.info(f"Adding template to {file_type.value.lower()} file")
            return generate_readme_template(
                input_path=Path(input_path), readme_template=readme_template
            )

        else:
            raise Exception(f"[red]File type {file_type.value} is not supported.[/red]")

    except Exception:
        logger.exception(f"Failed generating docs for {input_path}")
        sys.exit(1)


# ====================== create-id-set ====================== #
@main.command(hidden=True)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="Input file path, the default is the content repo.",
    default="",
)
@click.option(
    "-o",
    "--output",
    help="Output file path, the default is the Tests directory.",
    default="",
)
@click.option(
    "-fd",
    "--fail-duplicates",
    help="Fails the process if any duplicates are found.",
    is_flag=True,
)
@click.option(
    "-mp",
    "--marketplace",
    help="The marketplace the id set are created for, that determines which packs are"
    " inserted to the id set, and which items are present in the id set for "
    "each pack. Default is all packs exists in the content repository.",
    default="",
)
@click.pass_context
@logging_setup_decorator
def create_id_set(ctx, **kwargs):
    """Create the content dependency tree by ids."""
    from demisto_sdk.commands.create_id_set.create_id_set import IDSetCreator
    from demisto_sdk.commands.find_dependencies.find_dependencies import (
        remove_dependencies_from_id_set,
    )

    check_configuration_file("create-id-set", kwargs)
    id_set_creator = IDSetCreator(**kwargs)
    (
        id_set,
        excluded_items_by_pack,
        excluded_items_by_type,
    ) = id_set_creator.create_id_set()

    if excluded_items_by_pack:
        remove_dependencies_from_id_set(
            id_set,
            excluded_items_by_pack,
            excluded_items_by_type,
            kwargs.get("marketplace", ""),
        )
        id_set_creator.save_id_set()


# ====================== merge-id-sets ====================== #
@main.command(hidden=True)
@click.help_option("-h", "--help")
@click.option("-i1", "--id-set1", help="First id_set.json file path", required=True)
@click.option("-i2", "--id-set2", help="Second id_set.json file path", required=True)
@click.option("-o", "--output", help="File path of the united id_set", required=True)
@click.option(
    "-fd",
    "--fail-duplicates",
    help="Fails the process if any duplicates are found.",
    is_flag=True,
)
@click.pass_context
@logging_setup_decorator
def merge_id_sets(ctx, **kwargs):
    """Merge two id_sets"""
    from demisto_sdk.commands.common.update_id_set import merge_id_sets_from_files

    check_configuration_file("merge-id-sets", kwargs)
    first = kwargs["id_set1"]
    second = kwargs["id_set2"]
    output = kwargs["output"]
    fail_duplicates = kwargs["fail_duplicates"]

    _, duplicates = merge_id_sets_from_files(
        first_id_set_path=first, second_id_set_path=second, output_id_set_path=output
    )
    if duplicates:
        logger.info(
            f"[red]Failed to merge ID sets: {first} with {second}, "
            f"there are entities with ID: {duplicates} that exist in both ID sets"
        )
        if fail_duplicates:
            sys.exit(1)


# ====================== update-release-notes =================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="The relative path of the content pack. For example Packs/Pack_Name",
)
@click.option(
    "-u",
    "--update-type",
    help="The type of update being done. [major, minor, revision, documentation]",
    type=click.Choice(["major", "minor", "revision", "documentation"]),
)
@click.option(
    "-v", "--version", help="Bump to a specific version.", type=VersionParamType()
)
@click.option(
    "-g",
    "--use-git",
    help="Use git to identify the relevant changed files, will be used by default if '-i' is not set",
    is_flag=True,
)
@click.option(
    "-f",
    "--force",
    help="Force update release notes for a pack (even if not required).",
    is_flag=True,
)
@click.option(
    "--text",
    help="Text to add to all of the release notes files.",
)
@click.option(
    "--prev-ver", help="Previous branch or SHA1 commit to run checks against."
)
@click.option(
    "--pre_release",
    help="Indicates that this change should be designated a pre-release version.",
    is_flag=True,
)
@click.option(
    "-idp",
    "--id-set-path",
    help="The path of the id-set.json used for APIModule updates.",
    type=click.Path(resolve_path=True),
)
@click.option(
    "-bc",
    "--breaking-changes",
    help="If new version contains breaking changes.",
    is_flag=True,
)
@click.pass_context
@logging_setup_decorator
def update_release_notes(ctx, **kwargs):
    """Auto-increment pack version and generate release notes template."""
    from demisto_sdk.commands.update_release_notes.update_rn_manager import (
        UpdateReleaseNotesManager,
    )

    if is_sdk_defined_working_offline():
        logger.error(SDK_OFFLINE_ERROR_MESSAGE)
        sys.exit(1)

    check_configuration_file("update-release-notes", kwargs)
    if kwargs.get("force") and not kwargs.get("input"):
        logger.info(
            "[red]Please add a specific pack in order to force a release notes update."
        )
        sys.exit(0)

    if not kwargs.get("use_git") and not kwargs.get("input"):
        click.confirm(
            "No specific pack was given, do you want to update all changed packs?",
            abort=True,
        )

    try:
        rn_mng = UpdateReleaseNotesManager(
            user_input=kwargs.get("input"),
            update_type=kwargs.get("update_type"),
            pre_release=kwargs.get("pre_release", False),
            is_all=kwargs.get("use_git"),
            text=kwargs.get("text"),
            specific_version=kwargs.get("version"),
            id_set_path=kwargs.get("id_set_path"),
            prev_ver=kwargs.get("prev_ver"),
            is_force=kwargs.get("force", False),
            is_bc=kwargs.get("breaking_changes", False),
        )
        rn_mng.manage_rn_update()
        sys.exit(0)
    except Exception as e:
        logger.info(
            f"[red]An error occurred while updating the release notes: {str(e)}[/red]"
        )
        sys.exit(1)


# ====================== find-dependencies ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="Pack path to find dependencies. For example: Pack/HelloWorld. When using the"
    " --get-dependent-on flag, this argument can be used multiple times.",
    required=False,
    type=click.Path(exists=True, dir_okay=True),
    multiple=True,
)
@click.option(
    "-idp",
    "--id-set-path",
    help="Path to id set json file.",
    required=False,
    default="",
)
@click.option(
    "--no-update",
    help="Use to find the pack dependencies without updating the pack metadata.",
    required=False,
    is_flag=True,
)
@click.option(
    "--use-pack-metadata",
    help="Whether to update the dependencies from the pack metadata.",
    required=False,
    is_flag=True,
)
@click.option(
    "--all-packs-dependencies",
    help="Return a json file with ALL content packs dependencies. "
    "The json file will be saved under the path given in the "
    "'--output-path' argument",
    required=False,
    is_flag=True,
)
@click.option(
    "-o",
    "--output-path",
    help="The destination path for the packs dependencies json file. This argument is "
    "only relevant for when using the '--all-packs-dependecies' flag.",
    required=False,
)
@click.option(
    "--get-dependent-on",
    help="Get only the packs dependent ON the given pack. Note: this flag can not be"
    " used for the packs ApiModules and Base",
    required=False,
    is_flag=True,
)
@click.option(
    "-d",
    "--dependency",
    help="Find which items in a specific content pack appears as a mandatory "
    "dependency of the searched pack ",
    required=False,
)
@click.pass_context
@logging_setup_decorator
def find_dependencies(ctx, **kwargs):
    """Find pack dependencies and update pack metadata."""
    from demisto_sdk.commands.find_dependencies.find_dependencies import (
        PackDependencies,
    )

    check_configuration_file("find-dependencies", kwargs)
    update_pack_metadata = not kwargs.get("no_update")
    input_paths = kwargs.get("input")  # since it can be multiple, received as a tuple
    id_set_path = kwargs.get("id_set_path", "")
    use_pack_metadata = kwargs.get("use_pack_metadata", False)
    all_packs_dependencies = kwargs.get("all_packs_dependencies", False)
    get_dependent_on = kwargs.get("get_dependent_on", False)
    output_path = kwargs.get("output_path", ALL_PACKS_DEPENDENCIES_DEFAULT_PATH)
    dependency = kwargs.get("dependency", "")
    try:

        PackDependencies.find_dependencies_manager(
            id_set_path=str(id_set_path),
            update_pack_metadata=update_pack_metadata,
            use_pack_metadata=use_pack_metadata,
            input_paths=input_paths,
            all_packs_dependencies=all_packs_dependencies,
            get_dependent_on=get_dependent_on,
            output_path=output_path,
            dependency=dependency,
        )

    except ValueError as exp:
        logger.info(f"[red]{exp}[/red]")


# ====================== postman-codegen ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="The Postman collection 2.1 JSON file",
    required=True,
    type=click.File(),
)
@click.option(
    "-o",
    "--output",
    help="The output directory to save the config file or the integration",
    type=click.Path(dir_okay=True, exists=True),
    default=Path("."),
    show_default=True,
)
@click.option("-n", "--name", help="The output integration name")
@click.option(
    "-op",
    "--output-prefix",
    help="The global integration output prefix. By default it is the product name.",
)
@click.option(
    "-cp",
    "--command-prefix",
    help="The prefix for each command in the integration. By default is the product name in lower case",
)
@click.option(
    "--config-out",
    help="Used for advanced integration customisation. Generates a config json file instead of integration.",
    is_flag=True,
)
@click.option(
    "-p",
    "--package",
    help="Generated integration will be split to package format instead of a yml file.",
    is_flag=True,
)
@pass_config
@click.pass_context
def postman_codegen(
    ctx,
    config,
    input: IO,
    output: Path,
    name: str,
    output_prefix: str,
    command_prefix: str,
    config_out: bool,
    package: bool,
    **kwargs,
):
    """Generates a Cortex XSOAR integration given a Postman collection 2.1 JSON file."""
    logger = logging_setup(
        console_log_threshold=kwargs.get("console_log_threshold") or logging.INFO,
        file_log_threshold=kwargs.get("file_log_threshold") or logging.DEBUG,
        log_file_path=kwargs.get("log_file_path") or None,
    )
    from demisto_sdk.commands.postman_codegen.postman_codegen import (
        postman_to_autogen_configuration,
    )
    from demisto_sdk.commands.split.ymlsplitter import YmlSplitter

    postman_config = postman_to_autogen_configuration(
        collection=json.load(input),
        name=name,
        command_prefix=command_prefix,
        context_path_prefix=output_prefix,
    )

    if config_out:
        path = Path(output) / f"config-{postman_config.name}.json"
        path.write_text(json.dumps(postman_config.to_dict(), indent=4))
        logger.info(f"Config file generated at:\n{str(path.absolute())}")
    else:
        # generate integration yml
        yml_path = postman_config.generate_integration_package(output, is_unified=True)
        if package:
            yml_splitter = YmlSplitter(
                configuration=config.configuration,
                file_type=FileType.INTEGRATION,
                input=str(yml_path),
                output=str(output),
            )
            yml_splitter.extract_to_package_format()
            logger.info(
                f"[green]Package generated at {str(Path(output).absolute())} successfully[/green]"
            )
        else:
            logger.info(
                f"[green]Integration generated at {str(yml_path.absolute())} successfully[/green]"
            )


# ====================== generate-integration ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="config json file produced by commands like postman-codegen and openapi-codegen",
    required=True,
    type=click.File(),
)
@click.option(
    "-o",
    "--output",
    help="The output directory to save the integration package",
    type=click.Path(dir_okay=True, exists=True),
    default=Path("."),
)
@click.pass_context
@logging_setup_decorator
def generate_integration(ctx, input: IO, output: Path, **kwargs):
    """Generates a Cortex XSOAR integration from a config json file,
    which is generated by commands like postman-codegen
    """
    from demisto_sdk.commands.generate_integration.code_generator import (
        IntegrationGeneratorConfig,
    )

    config_dict = json.load(input)
    config = IntegrationGeneratorConfig(**config_dict)

    config.generate_integration_package(output, True)


# ====================== openapi-codegen ====================== #
@main.command(
    short_help="""Generates a Cortex XSOAR integration given an OpenAPI specification file."""
)
@click.help_option("-h", "--help")
@click.option(
    "-i", "--input_file", help="The swagger file to load in JSON format", required=True
)
@click.option(
    "-cf",
    "--config_file",
    help="The integration configuration file. It is created in the first run of the command",
    required=False,
)
@click.option(
    "-n",
    "--base_name",
    help="The base filename to use for the generated files",
    required=False,
)
@click.option(
    "-o",
    "--output_dir",
    help="Directory to store the output in (default is current working directory)",
    required=False,
)
@click.option(
    "-pr",
    "--command_prefix",
    help="Add a prefix to each command in the code",
    required=False,
)
@click.option("-c", "--context_path", help="Context output path", required=False)
@click.option(
    "-u",
    "--unique_keys",
    help="Comma separated unique keys to use in context paths (case sensitive)",
    required=False,
)
@click.option(
    "-r",
    "--root_objects",
    help="Comma separated JSON root objects to use in command outputs (case sensitive)",
    required=False,
)
@click.option(
    "-f", "--fix_code", is_flag=True, help="Fix the python code using autopep8"
)
@click.option(
    "-a",
    "--use_default",
    is_flag=True,
    help="Use the automatically generated integration configuration"
    " (Skip the second run).",
)
@click.pass_context
@logging_setup_decorator
def openapi_codegen(ctx, **kwargs):
    """Generates a Cortex XSOAR integration given an OpenAPI specification file.
    In the first run of the command, an integration configuration file is created, which can be modified.
    Then, the command is run a second time with the integration configuration to generate the actual integration files.
    """
    from demisto_sdk.commands.openapi_codegen.openapi_codegen import OpenAPIIntegration

    check_configuration_file("openapi-codegen", kwargs)
    if not kwargs.get("output_dir"):
        output_dir = os.getcwd()
    else:
        output_dir = kwargs["output_dir"]

    # Check the directory exists and if not, try to create it
    if not os.path.exists(output_dir):
        try:
            os.mkdir(output_dir)
        except Exception as err:
            logger.info(f"[red]Error creating directory {output_dir} - {err}[/red]")
            sys.exit(1)
    if not os.path.isdir(output_dir):
        logger.info(f'[red]The directory provided "{output_dir}" is not a directory')
        sys.exit(1)

    input_file = kwargs["input_file"]
    base_name = kwargs.get("base_name")
    if base_name is None:
        base_name = "GeneratedIntegration"

    command_prefix = kwargs.get("command_prefix")
    if command_prefix is None:
        command_prefix = "-".join(base_name.split(" ")).lower()

    context_path = kwargs.get("context_path")
    if context_path is None:
        context_path = base_name.replace(" ", "")

    unique_keys = kwargs.get("unique_keys", "")
    if unique_keys is None:
        unique_keys = ""

    root_objects = kwargs.get("root_objects", "")
    if root_objects is None:
        root_objects = ""

    fix_code = kwargs.get("fix_code", False)

    configuration = None
    if kwargs.get("config_file"):
        try:
            with open(kwargs["config_file"]) as config_file:
                configuration = json.load(config_file)
        except Exception as e:
            logger.info(f"[red]Failed to load configuration file: {e}[/red]")

    logger.info("Processing swagger file...")
    integration = OpenAPIIntegration(
        input_file,
        base_name,
        command_prefix,
        context_path,
        unique_keys=unique_keys,
        root_objects=root_objects,
        fix_code=fix_code,
        configuration=configuration,
    )

    integration.load_file()
    if not kwargs.get("config_file"):
        integration.save_config(integration.configuration, output_dir)
        logger.info(f"[green]Created configuration file in {output_dir}[/green]")
        if not kwargs.get("use_default", False):
            config_path = os.path.join(output_dir, f"{base_name}_config.json")
            command_to_run = (
                f'demisto-sdk openapi-codegen -i "{input_file}" -cf "{config_path}" -n "{base_name}" '
                f'-o "{output_dir}" -pr "{command_prefix}" -c "{context_path}"'
            )
            if unique_keys:
                command_to_run = command_to_run + f' -u "{unique_keys}"'
            if root_objects:
                command_to_run = command_to_run + f' -r "{root_objects}"'
            if (
                kwargs.get("console_log_threshold")
                and int(kwargs.get("console_log_threshold", logging.INFO))
                >= logging.DEBUG
            ):
                command_to_run = command_to_run + " -v"
            if fix_code:
                command_to_run = command_to_run + " -f"

            logger.info(
                f"Run the command again with the created configuration file(after a review): {command_to_run}"
            )
            sys.exit(0)

    if integration.save_package(output_dir):
        logger.info(
            f"Successfully finished generating integration code and saved it in {output_dir}",
            "green",
        )
    else:
        logger.info(
            f"[red]There was an error creating the package in {output_dir}[/red]"
        )
        sys.exit(1)


# ====================== test-content command ====================== #
@main.command(
    short_help="""Created incidents for selected test-playbooks and gives a report about the results""",
    hidden=True,
)
@click.help_option("-h", "--help")
@click.option(
    "-k", "--api-key", help="The Demisto API key for the server", required=True
)
@click.option("-s", "--server", help="The server URL to connect to")
@click.option("-c", "--conf", help="Path to content conf.json file", required=True)
@click.option("-e", "--secret", help="Path to content-test-conf conf.json file")
@click.option("-n", "--nightly", type=bool, help="Run nightly tests")
@click.option("-t", "--slack", help="The token for slack", required=True)
@click.option("-a", "--circleci", help="The token for circleci", required=True)
@click.option("-b", "--build-number", help="The build number", required=True)
@click.option(
    "-g", "--branch-name", help="The current content branch name", required=True
)
@click.option("-i", "--is-ami", type=bool, help="is AMI build or not", default=False)
@click.option(
    "-m",
    "--mem-check",
    type=bool,
    help="Should trigger memory checks or not. The slack channel to check the data is: "
    "dmst_content_nightly_memory_data",
    default=False,
)
@click.option(
    "-d",
    "--server-version",
    help="Which server version to run the tests on(Valid only when using AMI)",
    default="NonAMI",
)
@click.option(
    "-u",
    "--use-retries",
    is_flag=True,
    help="Should use retries mechanism or not (if test-playbook fails, it will execute it again few times and "
    "determine success according to most of the runs",
    default=False,
)
@click.option(
    "--server-type", help="Which server runs the tests? XSIAM or XSOAR", default="XSOAR"
)
@click.option(
    "-x", "--xsiam-machine", help="XSIAM machine to use, if it is XSIAM build."
)
@click.option("--xsiam-servers-path", help="Path to secret xsiam server metadata file.")
@click.option(
    "--xsiam-servers-api-keys-path", help="Path to file with XSIAM Servers api keys."
)
@click.pass_context
@logging_setup_decorator
def test_content(ctx, **kwargs):
    """Configure instances for the integration needed to run tests_to_run tests.
    Run test module on each integration.
    create an investigation for each test.
    run test playbook on the created investigation using mock if possible.
    Collect the result and give a report.
    """
    from demisto_sdk.commands.test_content.execute_test_content import (
        execute_test_content,
    )

    check_configuration_file("test-content", kwargs)
    execute_test_content(**kwargs)


# ====================== doc-review ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i", "--input", type=str, help="The path to the file to check", multiple=True
)
@click.option(
    "--no-camel-case",
    is_flag=True,
    help="Whether to check CamelCase words",
    default=False,
)
@click.option(
    "--known-words",
    type=str,
    help="The path to a file containing additional known words",
    multiple=True,
)
@click.option(
    "--always-true",
    is_flag=True,
    help="Whether to fail the command if misspelled words are found",
)
@click.option(
    "--expand-dictionary",
    is_flag=True,
    help="Whether to expand the base dictionary to include more words - "
    "will download 'brown' corpus from nltk package",
)
@click.option(
    "--templates", is_flag=True, help="Whether to print release notes templates"
)
@click.option(
    "-g",
    "--use-git",
    is_flag=True,
    help="Use git to identify the relevant changed files, "
    "will be used by default if '-i' and '--templates' are not set",
)
@click.option(
    "--prev-ver",
    type=str,
    help="The branch against which changes will be detected "
    "if '-g' flag is set. Default is 'demisto/master'",
)
@click.option(
    "-rn", "--release-notes", is_flag=True, help="Will run only on release notes files"
)
@click.option(
    "-xs",
    "--xsoar-only",
    is_flag=True,
    help="Run only on files from XSOAR-supported Packs.",
    default=False,
)
@click.option(
    "-pkw/-spkw",
    "--use-packs-known-words/--skip-packs-known-words",
    is_flag=True,
    help="Will find and load the known_words file from the pack. "
    "To use this option make sure you are running from the "
    "content directory.",
    default=True,
)
@click.pass_context
@logging_setup_decorator
def doc_review(ctx, **kwargs):
    """Check the spelling in .md and .yml files as well as review release notes"""
    from demisto_sdk.commands.doc_reviewer.doc_reviewer import DocReviewer

    doc_reviewer = DocReviewer(
        file_paths=kwargs.get("input", []),
        known_words_file_paths=kwargs.get("known_words", []),
        no_camel_case=kwargs.get("no_camel_case"),
        no_failure=kwargs.get("always_true"),
        expand_dictionary=kwargs.get("expand_dictionary"),
        templates=kwargs.get("templates"),
        use_git=kwargs.get("use_git"),
        prev_ver=kwargs.get("prev_ver"),
        release_notes_only=kwargs.get("release_notes"),
        xsoar_only=kwargs.get("xsoar_only"),
        load_known_words_from_pack=kwargs.get("use_packs_known_words"),
    )
    result = doc_reviewer.run_doc_review()
    if result:
        sys.exit(0)

    sys.exit(1)


# ====================== integration-diff ====================== #
@main.command(
    name="integration-diff",
    help="""Given two versions of an integration, Check that everything in the old integration is covered in
              the new integration""",
)
@click.help_option("-h", "--help")
@click.option(
    "-n",
    "--new",
    type=str,
    help="The path to the new version of the integration",
    required=True,
)
@click.option(
    "-o",
    "--old",
    type=str,
    help="The path to the old version of the integration",
    required=True,
)
@click.option(
    "--docs-format",
    is_flag=True,
    help="Whether output should be in the format for the version differences section in README.",
)
@click.pass_context
@logging_setup_decorator
def integration_diff(ctx, **kwargs):
    """
    Checks for differences between two versions of an integration, and verified that the new version covered the old version.
    """
    from demisto_sdk.commands.integration_diff.integration_diff_detector import (
        IntegrationDiffDetector,
    )

    integration_diff_detector = IntegrationDiffDetector(
        new=kwargs.get("new", ""),
        old=kwargs.get("old", ""),
        docs_format=kwargs.get("docs_format", False),
    )
    result = integration_diff_detector.check_different()

    if result:
        sys.exit(0)

    sys.exit(1)


# ====================== generate_yml_from_python ====================== #
@main.command(
    name="generate-yml-from-python",
    help="""Generate YML file from Python code that includes special syntax.\n
                      The output file name will be the same as the Python code with the `.yml` extension instead of `.py`.\n
                      The generation currently supports integrations only.\n
                      For more information on usage and installation visit the command's README.md file.""",
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    type=click.Path(exists=True),
    help="The path to the python code to generate from",
    required=True,
)
@click.option(
    "-f",
    "--force",
    is_flag=True,
    type=bool,
    help="Override existing yml file.",
    required=False,
)
@click.pass_context
@logging_setup_decorator
def generate_yml_from_python(ctx, **kwargs):
    """
    Checks for differences between two versions of an integration, and verified that the new version covered the old version.
    """
    from demisto_sdk.commands.generate_yml_from_python.generate_yml import YMLGenerator

    yml_generator = YMLGenerator(
        filename=kwargs.get("input", ""),
        force=kwargs.get("force", False),
    )
    yml_generator.generate()
    yml_generator.save_to_yml_file()


# ====================== convert ====================== #
@main.command(
    context_settings=dict(
        ignore_unknown_options=True,
        allow_extra_args=True,
    )
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    type=click.Path(exists=True),
    required=True,
    help="The path of the content pack/directory/file to convert.",
)
@click.option(
    "-v", "--version", required=True, help="Version the input to be compatible with."
)
@pass_config
@click.pass_context
@logging_setup_decorator
def convert(ctx, config, **kwargs):
    """
    Convert the content of the pack/directory in the given input to be compatible with the version given by
    version command.
    """
    from demisto_sdk.commands.convert.convert_manager import ConvertManager

    check_configuration_file("convert", kwargs)
    sys.path.append(config.configuration.env_dir)

    input_path = kwargs["input"]
    server_version = kwargs["version"]
    convert_manager = ConvertManager(input_path, server_version)
    result = convert_manager.convert()

    if result:
        sys.exit(1)

    sys.exit(0)


# ====================== generate-unit-tests ====================== #


@main.command(short_help="""Generates unit tests for integration code.""")
@click.help_option("-h", "--help")
@click.option(
    "-c",
    "--commands",
    help="Specific commands name to generate unit test for (e.g. xdr-get-incidents)",
    required=False,
)
@click.option(
    "-o",
    "--output_dir",
    help="Directory to store the output in (default is the input integration directory)",
    required=False,
)
@click.option("-i", "--input_path", help="Valid integration file path.", required=True)
@click.option(
    "-d", "--use_demisto", help="Run commands at Demisto automatically.", is_flag=True
)
@click.option("--insecure", help="Skip certificate validation", is_flag=True)
@click.option(
    "-e",
    "--examples",
    help="Integrations: path for file containing command examples."
    " Each command should be in a separate line.",
)
@click.option(
    "-a",
    "--append",
    help="Append generated test file to the existing <integration_name>_test.py. Else, overwriting existing UT",
    is_flag=True,
)
@click.pass_context
@logging_setup_decorator
def generate_unit_tests(
    ctx,
    input_path: str = "",
    commands: list = [],
    output_dir: str = "",
    examples: str = "",
    insecure: bool = False,
    use_demisto: bool = False,
    append: bool = False,
    **kwargs,
):
    """
    This command is used to generate unit tests automatically from an  integration python code.
    Also supports generating unit tests for specific commands.
    """
    logging.getLogger("PYSCA").propagate = False
    from demisto_sdk.commands.generate_unit_tests.generate_unit_tests import (
        run_generate_unit_tests,
    )

    return run_generate_unit_tests(
        input_path, commands, output_dir, examples, insecure, use_demisto, append
    )


@main.command(
    name="error-code",
    help="Quickly find relevant information regarding an error code.",
)
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    required=True,
    help="The error code to search for.",
)
@pass_config
@click.pass_context
@logging_setup_decorator
def error_code(ctx, config, **kwargs):
    from demisto_sdk.commands.error_code_info.error_code_info import (
        generate_error_code_information,
    )

    check_configuration_file("error-code-info", kwargs)
    sys.path.append(config.configuration.env_dir)

    result = generate_error_code_information(kwargs.get("input"))

    sys.exit(result)


# ====================== create-content-graph ====================== #
@main.command(
    hidden=True,
)
@click.help_option("-h", "--help")
@click.option(
    "-o",
    "--output-path",
    type=click.Path(resolve_path=True, path_type=Path, dir_okay=True, file_okay=False),
    default=None,
    help="Output folder to place the zip file of the graph exported CSVs files",
)
@click.option(
    "-mp",
    "--marketplace",
    help="The marketplace to generate the graph for.",
    default="xsoar",
    type=click.Choice(list(MarketplaceVersions)),
)
@click.option(
    "-nd",
    "--no-dependencies",
    is_flag=True,
    help="Whether or not to include dependencies.",
    default=False,
)
@click.pass_context
@logging_setup_decorator
def create_content_graph(
    ctx,
    marketplace: str = MarketplaceVersions.XSOAR,
    no_dependencies: bool = False,
    output_path: Path = None,
    **kwargs,
):
    ctx.invoke(
        create,
        ctx,
        marketplace=marketplace,
        no_dependencies=no_dependencies,
        output_path=output_path,
        **kwargs,
    )


# ====================== update-content-graph ====================== #
@main.command(
    hidden=True,
)
@click.help_option("-h", "--help")
@click.option(
    "-mp",
    "--marketplace",
    help="The marketplace the artifacts are created for, that "
    "determines which artifacts are created for each pack. "
    "Default is the XSOAR marketplace, that has all of the packs "
    "artifacts.",
    default="xsoar",
    type=click.Choice(list(MarketplaceVersions)),
)
@click.option(
    "-g",
    "--use-git",
    is_flag=True,
    show_default=True,
    default=False,
    help="Whether to use git to determine the packs to update",
)
@click.option(
    "-i",
    "--imported-path",
    type=click.Path(
        path_type=Path, resolve_path=True, exists=True, file_okay=True, dir_okay=False
    ),
    default=None,
    help="Path to content graph zip file to import",
)
@click.option(
    "-uc",
    "--use-current",
    is_flag=True,
    help="Whether to use the current content graph to update",
    default=False,
)
@click.option(
    "-p",
    "--packs",
    help="A comma-separated list of packs to update",
    multiple=True,
    default=None,
)
@click.option(
    "-nd",
    "--no-dependencies",
    is_flag=True,
    help="Whether dependencies should be included in the graph",
    default=False,
)
@click.option(
    "-o",
    "--output-path",
    type=click.Path(resolve_path=True, path_type=Path, dir_okay=True, file_okay=False),
    default=None,
    help="Output folder to place the zip file of the graph exported CSVs files",
)
@click.pass_context
@logging_setup_decorator
def update_content_graph(
    ctx,
    use_git: bool = False,
    marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
    use_current: bool = False,
    imported_path: Path = None,
    packs: list = None,
    no_dependencies: bool = False,
    output_path: Path = None,
    **kwargs,
):
    ctx.invoke(
        update,
        ctx,
        use_git=use_git,
        marketplace=marketplace,
        use_current=use_current,
        imported_path=imported_path,
        packs_to_update=packs,
        no_dependencies=no_dependencies,
        output_path=output_path,
        **kwargs,
    )


@main.command(short_help="Runs pre-commit hooks on the files in the repository")
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    help="The path to the input file to run the command on.",
    type=PathsParamType(
        exists=True, resolve_path=True
    ),  # PathsParamType allows passing a list of paths
)
@click.option(
    "-s",
    "--staged-only",
    help="Whether to run only on staged files",
    is_flag=True,
    default=False,
)
@click.option(
    "-g",
    "--git-diff",
    help="Whether to use git to determine which files to run on",
    is_flag=True,
    default=False,
)
@click.option(
    "-a",
    "--all-files",
    help="Whether to run on all files",
    is_flag=True,
    default=False,
)
@click.option(
    "-ut/--no-ut",
    "--unit-test/--no-unit-test",
    help="Whether to run unit tests for content items",
    default=False,
)
@click.option(
    "--skip",
    help="A comma separated list of precommit hooks to skip",
)
@click.option(
    "--validate/--no-validate",
    help="Whether to run demisto-sdk validate",
    default=True,
)
@click.option(
    "--format/--no-format",
    help="Whether to run demisto-sdk format",
    default=False,
)
@click.option(
    "--secrets/--no-secrets",
    help="Whether to run demisto-sdk secrets",
    default=True,
)
@click.option(
    "-v",
    "--verbose",
    help="Verbose output of pre-commit",
    is_flag=True,
    default=False,
)
@click.option(
    "--show-diff-on-failure",
    help="Show diff on failure",
    is_flag=True,
    default=False,
)
@click.option(
    "--sdk-ref",
    help="The demisto-sdk ref to use for the pre-commit hooks",
    default="",
)
@click.argument(
    "file_paths",
    nargs=-1,
    type=click.Path(exists=True, resolve_path=True, path_type=Path),
)
@click.pass_context
@logging_setup_decorator
def pre_commit(
    ctx,
    input: str,
    staged_only: bool,
    git_diff: bool,
    all_files: bool,
    unit_test: bool,
    skip: str,
    validate: bool,
    format: bool,
    secrets: bool,
    verbose: bool,
    show_diff_on_failure: bool,
    sdk_ref: str,
    file_paths: Iterable[Path],
    **kwargs,
):
    from demisto_sdk.commands.pre_commit.pre_commit_command import pre_commit_manager

    if file_paths and input:
        logger.info(
            "Both `--input` parameter and `file_paths` arguments were provided. Will use the `--input` parameter."
        )
    input_files = []
    if input:
        input_files = [Path(i) for i in input.split(",")]
    elif file_paths:
        input_files = list(file_paths)
    if skip:
        skip = skip.split(",")  # type: ignore[assignment]

    sys.exit(
        pre_commit_manager(
            input_files,
            staged_only,
            git_diff,
            all_files,
            unit_test,
            skip,
            validate,
            format,
            secrets,
            verbose,
            show_diff_on_failure,
            sdk_ref=sdk_ref,
        )
    )


@main.command(short_help="Run unit tests in a docker for integrations and scripts")
@click.help_option("-h", "--help")
@click.option(
    "-i",
    "--input",
    type=PathsParamType(
        exists=True, resolve_path=True
    ),  # PathsParamType allows passing a list of paths
    help="The path of the content pack/file to validate specifically.",
)
@click.option(
    "-v", "--verbose", is_flag=True, default=False, help="Verbose output of unit tests"
)
@click.argument("file_paths", nargs=-1, type=click.Path(exists=True, resolve_path=True))
@click.pass_context
@logging_setup_decorator
def run_unit_tests(
    ctx, input: str, file_paths: Tuple[str, ...], verbose: bool, **kwargs
):
    if input:
        file_paths = tuple(input.split(","))
    from demisto_sdk.commands.run_unit_tests.unit_tests_runner import unit_test_runner

    sys.exit(unit_test_runner(file_paths, verbose))


@main.result_callback()
def exit_from_program(result=0, **kwargs):
    sys.exit(result)


# ====================== modeling-rules command group ====================== #

app = typer.Typer(name="modeling-rules", hidden=True, no_args_is_help=True)
app.command("test", no_args_is_help=True)(test_modeling_rule.test_modeling_rule)
app.command("init-test-data", no_args_is_help=True)(init_test_data.init_test_data)
typer_click_object = typer.main.get_command(app)
main.add_command(typer_click_object, "modeling-rules")

app_generate_modeling_rules = typer.Typer(
    name="generate-modeling-rules", no_args_is_help=True
)
app_generate_modeling_rules.command("generate-modeling-rules", no_args_is_help=True)(
    generate_modeling_rules.generate_modeling_rules
)

typer_click_object2 = typer.main.get_command(app_generate_modeling_rules)
main.add_command(typer_click_object2, "generate-modeling-rules")


# ====================== graph command group ====================== #

graph_cmd_group = typer.Typer(name="graph", hidden=True, no_args_is_help=True)
graph_cmd_group.command("create", no_args_is_help=False)(create)
graph_cmd_group.command("update", no_args_is_help=False)(update)
graph_cmd_group.command("get-relationships", no_args_is_help=True)(get_relationships)
main.add_command(typer.main.get_command(graph_cmd_group), "graph")


if __name__ == "__main__":
    main()


# demisto_sdk/commands/content_graph/commands/common.py

import demisto_sdk.commands.content_graph.neo4j_service as neo4j_service
from demisto_sdk.commands.common.logger import logger


def recover_if_fails(func):
    def func_wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception:
            if not neo4j_service.is_running_on_docker():

                logger.error(
                    "Either start the Docker service or install Neo4j locally with this guide: https://github.com/demisto/demisto-sdk/blob/master/demisto_sdk/commands/content_graph/README.md",
                    exc_info=True,
                )
                raise
            logger.warning(
                "Failed to build content graph, retrying with a clean environment.",
                exc_info=True,
            )
            neo4j_service.stop(force=True, clean=True)
            neo4j_service.start()
            return func(*args, **kwargs)

    return func_wrapper


# demisto_sdk/commands/content_graph/commands/create.py

from pathlib import Path
from typing import Optional

import typer

import demisto_sdk.commands.content_graph.neo4j_service as neo4j_service
from demisto_sdk.commands.common.constants import (
    MarketplaceVersions,
)
from demisto_sdk.commands.common.logger import (
    logger,
    logging_setup,
)
from demisto_sdk.commands.content_graph.commands.common import recover_if_fails
from demisto_sdk.commands.content_graph.common import (
    NEO4J_DATABASE_HTTP,
    NEO4J_PASSWORD,
    NEO4J_USERNAME,
)
from demisto_sdk.commands.content_graph.content_graph_builder import (
    ContentGraphBuilder,
)
from demisto_sdk.commands.content_graph.interface import ContentGraphInterface

app = typer.Typer()


@recover_if_fails
def create_content_graph(
    content_graph_interface: ContentGraphInterface,
    marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
    dependencies: bool = True,
    output_path: Optional[Path] = None,
) -> None:
    """This function creates a new content graph database in neo4j from the content path

    Args:
        content_graph_interface (ContentGraphInterface): The content graph interface.
        marketplace (MarketplaceVersions): The marketplace to update.
        dependencies (bool): Whether to create the dependencies.
        output_path (Path): The path to export the graph zip to.
    """
    ContentGraphBuilder(content_graph_interface).create_graph()
    if dependencies:
        content_graph_interface.create_pack_dependencies()
    if output_path:
        output_path = output_path / marketplace.value
    content_graph_interface.export_graph(output_path)
    logger.info(
        f"Successfully created the content graph. UI representation "
        f"is available at {NEO4J_DATABASE_HTTP} "
        f"(username: {NEO4J_USERNAME}, password: {NEO4J_PASSWORD})"
    )


@app.command(
    no_args_is_help=True,
    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
)
def create(
    ctx: typer.Context,
    marketplace: MarketplaceVersions = typer.Option(
        MarketplaceVersions.XSOAR,
        "-mp",
        "--marketplace",
        help="The marketplace to generate the graph for.",
    ),
    no_dependencies: bool = typer.Option(
        False,
        "-nd",
        "--no-dependencies",
        is_flag=True,
        help="Whether or not to include dependencies in the graph.",
    ),
    output_path: Path = typer.Option(
        None,
        exists=True,
        dir_okay=True,
        file_okay=False,
        resolve_path=True,
        help="Output folder to locate the zip file of the graph exported file.",
    ),
    console_log_threshold: str = typer.Option(
        "INFO",
        "-clt",
        "--console_log_threshold",
        help=("Minimum logging threshold for the console logger."),
    ),
    file_log_threshold: str = typer.Option(
        "DEBUG",
        "-flt",
        "--file_log_threshold",
        help=("Minimum logging threshold for the file logger."),
    ),
    log_file_path: str = typer.Option(
        "demisto_sdk_debug.log",
        "-lp",
        "--log_file_path",
        help=("Path to the log file. Default: ./demisto_sdk_debug.log."),
    ),
) -> None:
    """
    Parses all content packs under the repository, including their
    relationships. Then, the parsed content objects are mapped to
    a Repository model and uploaded to the graph database.
    """
    logging_setup(
        console_log_threshold=console_log_threshold,
        file_log_threshold=file_log_threshold,
        log_file_path=log_file_path,
    )
    with ContentGraphInterface() as content_graph_interface:
        create_content_graph(
            content_graph_interface=content_graph_interface,
            marketplace=marketplace,
            dependencies=not no_dependencies,
            output_path=output_path,
        )


def stop_content_graph() -> None:
    """
    This function stops the neo4j service if it is running.
    """
    neo4j_service.stop()


# demisto_sdk/commands/content_graph/commands/get_relationships.py

from enum import Enum
from pathlib import Path
from typing import Any, Dict, Optional

import typer
from tabulate import tabulate

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.logger import (
    logger,
    logging_setup,
)
from demisto_sdk.commands.content_graph.commands.update import update_content_graph
from demisto_sdk.commands.content_graph.common import (
    ContentType,
    RelationshipType,
)
from demisto_sdk.commands.content_graph.interface import ContentGraphInterface

app = typer.Typer()


COMMAND_OUTPUTS_FILENAME = "get_relationships_outputs.json"


class Direction(str, Enum):
    SOURCES = "sources"
    TARGETS = "targets"
    BOTH = "both"


@app.command(
    no_args_is_help=True,
    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
)
def get_relationships(
    ctx: typer.Context,
    input: Path = typer.Argument(
        ...,
        exists=True,
        dir_okay=True,
        resolve_path=True,
        show_default=False,
        help="The path to a content item or a pack.",
    ),
    relationship: RelationshipType = typer.Option(
        RelationshipType.USES,
        "-r",
        "--relationship",
        show_default=True,
        case_sensitive=False,
        help="The type of relationships to inspect.",
    ),
    content_type: ContentType = typer.Option(
        ContentType.BASE_CONTENT,
        "-ct",
        "--content-type",
        show_default=True,
        case_sensitive=False,
        help="The content type of the related object.",
    ),
    depth: int = typer.Option(
        1,
        "-d",
        "--depth",
        min=1,
        max=5,
        show_default=True,
        help="Maximum depth (length) of the relationships paths.",
    ),
    update_graph: bool = typer.Option(
        True,
        "-u/-nu",
        "--update-graph/--no-update-graph",
        is_flag=True,
        help="If true, runs an update on the graph before querying.",
    ),
    marketplace: MarketplaceVersions = typer.Option(
        MarketplaceVersions.XSOAR,
        "-mp",
        "--marketplace",
        show_default=True,
        case_sensitive=False,
        help="The marketplace version.",
    ),
    mandatory_only: bool = typer.Option(
        False,
        "--mandatory-only",
        is_flag=True,
        help="If true, returns only mandatory relationships (relevant only for DEPENDS_ON/USES relationships).",
    ),
    include_tests: bool = typer.Option(
        False,
        "--include-tests",
        is_flag=True,
        help="If true, includes tests in outputs (relevant only for DEPENDS_ON/USES relationships).",
    ),
    include_deprecated: bool = typer.Option(
        False,
        "--include-deprecated",
        is_flag=True,
        help="If true, includes deprecated in outputs.",
    ),
    include_hidden: bool = typer.Option(
        False,
        "--include-hidden",
        is_flag=True,
        help="If true, includes hidden packs in outputs (relevant only for DEPENDS_ON relationships).",
    ),
    direction: Direction = typer.Option(
        Direction.BOTH,
        "-dir",
        "--direction",
        show_default=True,
        case_sensitive=False,
        help="Specifies whether to return only sources, only targets or both.",
    ),
    output: Optional[Path] = typer.Option(
        None,
        "-o",
        "--output",
        exists=True,
        dir_okay=True,
        resolve_path=True,
        show_default=False,
        help="A path to a directory in which to dump the outputs to.",
    ),
    console_log_threshold: str = typer.Option(
        "INFO",
        "-clt",
        "--console_log_threshold",
        help=("Minimum logging threshold for the console logger."),
    ),
    file_log_threshold: str = typer.Option(
        "DEBUG",
        "-flt",
        "--file_log_threshold",
        help=("Minimum logging threshold for the file logger."),
    ),
    log_file_path: str = typer.Option(
        "demisto_sdk_debug.log",
        "-lp",
        "--log_file_path",
        help=("Path to the log file. Default: ./demisto_sdk_debug.log."),
    ),
) -> None:
    """
    Returns relationships of a given content object.
    """
    logging_setup(
        console_log_threshold=console_log_threshold,
        file_log_threshold=file_log_threshold,
        log_file_path=log_file_path,
    )
    if relationship == RelationshipType.HAS_COMMAND:
        raise ValueError(
            f"Searching relationships of type {relationship} is not supported for this command."
            "To find which integrations implement specific commands, please run "
            "`demisto-sdk graph get-command-usage <COMMAND_NAME>`"
        )
    with ContentGraphInterface() as graph:
        if update_graph:
            update_content_graph(graph)
        result = get_relationships_by_path(
            graph,
            input.relative_to(graph.repo_path),
            relationship,
            content_type,
            depth,
            marketplace,
            direction,
            mandatory_only,
            include_tests,
            include_deprecated,
            include_hidden,
        )
        if output:
            (output / COMMAND_OUTPUTS_FILENAME).write_text(
                json.dumps(result, indent=4),
            )


def get_relationships_by_path(
    graph: ContentGraphInterface,
    input_filepath: Path,
    relationship: RelationshipType,
    content_type: ContentType,
    depth: int,
    marketplace: MarketplaceVersions,
    direction: Direction,
    mandatory_only: bool,
    include_tests: bool,
    include_deprecated: bool,
    include_hidden: bool,
) -> Dict[str, Any]:
    retrieve_sources: bool = direction != Direction.TARGETS
    retrieve_targets: bool = direction != Direction.SOURCES

    sources, targets = graph.get_relationships_by_path(
        input_filepath,
        relationship,
        content_type,
        depth,
        marketplace,
        retrieve_sources,
        retrieve_targets,
        mandatory_only,
        include_tests,
        include_deprecated,
        include_hidden,
    )
    for record in sources + targets:
        log_record(record, relationship)
        format_record_for_outputs(record, relationship)
    logger.info("[cyan]====== SUMMARY ======[/cyan]")
    if retrieve_sources:
        logger.info(f"Sources:\n{to_tabulate(sources)}\n")
    if retrieve_targets:
        logger.info(f"Targets:\n{to_tabulate(targets)}\n")
    return {"sources": sources, "targets": targets}


def log_record(
    record: Dict[str, Any],
    relationship: RelationshipType,
) -> None:
    is_source = record["is_source"]
    for path in record["paths"]:
        mandatorily = (
            f" (mandatory: {path['mandatorily']})"
            if path["mandatorily"] is not None
            else ""
        )
        logger.debug(
            f"[yellow]Found a {relationship} path{mandatorily}"
            f"{' from ' if is_source else ' to '}"
            f"{record['filepath']}[/yellow]\n"
            f"{path_to_str(relationship, path['path'])}\n"
        )


def path_to_str(
    relationship: RelationshipType,
    path: list,
) -> str:
    def node_to_str(node_data: dict) -> str:
        name = f"[cyan]{node_data['name']}[/cyan]"
        content_type = f"[lightblue]{node_data['content_type']}[/lightblue]"
        path = node_data["path"]
        return f"• ({name}:{content_type} {{path: {path}}})\n"

    def rel_to_str(rel: RelationshipType, props: dict) -> str:
        return f"   └─ [[purple]{rel}[/purple]]{props or ''} ↴\n"

    path_str = ""
    for idx, path_element in enumerate(path):
        if idx % 2 == 0:
            path_str += node_to_str(path_element)
        else:
            path_str += rel_to_str(relationship, path_element)
    return path_str


def format_record_for_outputs(
    record: Dict[str, Any],
    relationship: RelationshipType,
) -> None:
    del record["is_source"]  # unnecessary field in output
    for path_data in record["paths"]:
        formatted_path = []
        for idx, path_element in enumerate(path_data["path"]):
            if idx % 2 == 0:
                formatted_path.append({f"node_{idx // 2}": path_element})
            else:
                formatted_path.append({relationship: path_element})
        path_data["path"] = formatted_path


def to_tabulate(
    data: list,
) -> str:
    if not data:
        return "No results."

    headers = ["Name", "Type", "Path", "Depth"]
    fieldnames_to_collect = ["name", "content_type", "filepath", "minDepth"]
    maxcolwidths = [50] * len(headers)

    tabulated_data = []
    for record in data:
        tabulated_data.append([record[f] for f in fieldnames_to_collect])

    return tabulate(
        tabulated_data,
        headers=headers,
        tablefmt="fancy_grid",
        maxcolwidths=maxcolwidths,
    )


# demisto_sdk/commands/content_graph/commands/update.py

import os
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import List, Optional

import typer

from demisto_sdk.commands.common.constants import (
    MarketplaceVersions,
)
from demisto_sdk.commands.common.git_util import GitUtil
from demisto_sdk.commands.common.logger import (
    logger,
    logging_setup,
)
from demisto_sdk.commands.common.tools import download_content_graph
from demisto_sdk.commands.content_graph.commands.common import recover_if_fails
from demisto_sdk.commands.content_graph.commands.create import (
    create,
    create_content_graph,
)
from demisto_sdk.commands.content_graph.common import (
    NEO4J_DATABASE_HTTP,
    NEO4J_PASSWORD,
    NEO4J_USERNAME,
)
from demisto_sdk.commands.content_graph.content_graph_builder import (
    ContentGraphBuilder,
)
from demisto_sdk.commands.content_graph.interface import ContentGraphInterface

app = typer.Typer()


@recover_if_fails
def update_content_graph(
    content_graph_interface: ContentGraphInterface,
    marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
    use_git: bool = False,
    imported_path: Optional[Path] = None,
    use_current: bool = False,
    packs_to_update: Optional[List[str]] = None,
    dependencies: bool = True,
    output_path: Optional[Path] = None,
) -> None:
    """This function updates a new content graph database in neo4j from the content path
    Args:
        content_graph_interface (ContentGraphInterface): The content graph interface.
        marketplace (MarketplaceVersions): The marketplace to update.
        use_git (bool): Whether to use git to get the packs to update.
        imported_path (Path): The path to the imported graph.
        use_current (bool): Whether to use the current graph.
        packs_to_update (List[str]): The packs to update.
        dependencies (bool): Whether to create the dependencies.
        output_path (Path): The path to export the graph zip to.
    """
    packs_to_update = list(packs_to_update) if packs_to_update else []
    builder = ContentGraphBuilder(content_graph_interface)
    if not use_current:
        content_graph_interface.clean_import_dir()
        if not imported_path:
            # getting the graph from remote, so we need to clean the import dir
            try:
                extract_remote_import_files(content_graph_interface)
            except RuntimeError as e:
                logger.warning(
                    "Failed to download the content graph, recreating it instead"
                )
                logger.debug(f"Runtime Error: {e}", exc_info=True)
                create_content_graph(
                    content_graph_interface, marketplace, dependencies, output_path
                )
                return
    if not content_graph_interface.import_graph(imported_path):
        logger.warning("Failed to import the content graph, will create a new graph")
        create_content_graph(
            content_graph_interface, marketplace, dependencies, output_path
        )
        return

    if use_git and (commit := content_graph_interface.commit):
        packs_to_update.extend(GitUtil().get_all_changed_pack_ids(commit))

    packs_str = "\n".join([f"- {p}" for p in packs_to_update])
    logger.info(f"Updating the following packs:\n{packs_str}")
    builder.update_graph(packs_to_update)

    if dependencies:
        content_graph_interface.create_pack_dependencies()
    if output_path:
        output_path = output_path / marketplace.value
    content_graph_interface.export_graph(output_path)
    logger.info(
        f"Successfully updated the content graph. UI representation is available at {NEO4J_DATABASE_HTTP} "
        f"(username: {NEO4J_USERNAME}, password: {NEO4J_PASSWORD})"
    )


@app.command(
    no_args_is_help=True,
    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
)
def update(
    ctx: typer.Context,
    use_git: bool = typer.Option(
        False,
        "-g",
        "--use-git",
        is_flag=True,
        help="If true, uses git to determine the packs to update.",
    ),
    marketplace: MarketplaceVersions = typer.Option(
        MarketplaceVersions.XSOAR,
        "-mp",
        "--marketplace",
        help="The marketplace to generate the graph for.",
    ),
    use_current: bool = typer.Option(
        False,
        "-uc",
        "--use-current",
        is_flag=True,
        help="Whether to use the current content graph to update.",
    ),
    imported_path: Path = typer.Option(
        None,
        "-i",
        "--imported-path",
        exists=True,
        file_okay=True,
        dir_okay=False,
        resolve_path=True,
        help="Path to content graph zip file to import.",
    ),
    packs_to_update: Optional[List[str]] = typer.Option(
        None,
        "-p",
        "--packs",
        help="A comma-separated list of packs to update.",
    ),
    no_dependencies: bool = typer.Option(
        False,
        "-nd",
        "--no-dependencies",
        is_flag=True,
        help="Whether or not to include dependencies in the graph.",
    ),
    output_path: Path = typer.Option(
        None,
        exists=True,
        dir_okay=True,
        file_okay=False,
        resolve_path=True,
        help="Output folder to locate the zip file of the graph exported file.",
    ),
    console_log_threshold: str = typer.Option(
        "INFO",
        "-clt",
        "--console_log_threshold",
        help=("Minimum logging threshold for the console logger."),
    ),
    file_log_threshold: str = typer.Option(
        "DEBUG",
        "-flt",
        "--file_log_threshold",
        help=("Minimum logging threshold for the file logger."),
    ),
    log_file_path: str = typer.Option(
        "demisto_sdk_debug.log",
        "-lp",
        "--log_file_path",
        help=("Path to the log file. Default: ./demisto_sdk_debug.log."),
    ),
) -> None:
    """
    Downloads the official content graph, imports it locally,
    and updates it with the changes in the given repository
    or by an argument of packs to update with.
    """
    if os.getenv("DEMISTO_SDK_GRAPH_FORCE_CREATE"):
        logger.info("DEMISTO_SDK_GRAPH_FORCE_CREATE is set. Will create a new graph")
        ctx.invoke(
            create,
            ctx,
            marketplace=marketplace,
            no_dependencies=no_dependencies,
            output_path=output_path,
            console_log_threshold=console_log_threshold,
            file_log_threshold=file_log_threshold,
            log_file_path=log_file_path,
        )
        return
    logging_setup(
        console_log_threshold=console_log_threshold,
        file_log_threshold=file_log_threshold,
        log_file_path=log_file_path,
    )
    with ContentGraphInterface() as content_graph_interface:
        update_content_graph(
            content_graph_interface,
            marketplace=marketplace,
            use_git=use_git,
            imported_path=imported_path,
            use_current=use_current,
            packs_to_update=list(packs_to_update) if packs_to_update else [],
            dependencies=not no_dependencies,
            output_path=output_path,
        )


def extract_remote_import_files(
    content_graph_interface: ContentGraphInterface,
) -> None:
    """Get or create a content graph.
    If the graph is not in the bucket or there are network issues,
    it will create a new one.

    Args:
        content_graph_interface (ContentGraphInterface)
        builder (ContentGraphBuilder)

    """
    try:
        with NamedTemporaryFile() as temp_file:
            official_content_graph = download_content_graph(
                Path(temp_file.name),
            )
            content_graph_interface.move_to_import_dir(official_content_graph)
    except Exception as e:
        raise RuntimeError("Failed to download the content graph") from e


# demisto_sdk/commands/content_graph/interface/__init__.py

from demisto_sdk.commands.content_graph.interface.neo4j.neo4j_graph import (
    Neo4jContentGraphInterface as ContentGraphInterface,
)

__all__ = ["ContentGraphInterface"]


# demisto_sdk/commands/content_graph/interface/graph.py

import shutil
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH
from demisto_sdk.commands.common.git_util import GitUtil
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import get_file, sha1_dir
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.objects.base_content import BaseContent
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem
from demisto_sdk.commands.content_graph.objects.pack import Pack
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO


class ContentGraphInterface(ABC):
    repo_path = CONTENT_PATH  # type: ignore
    METADATA_FILE_NAME = "metadata.json"

    @property
    @abstractmethod
    def import_path(self) -> Path:
        pass

    @abstractmethod
    def clean_import_dir(self) -> None:
        pass

    @abstractmethod
    def move_to_import_dir(self, imported_path: Path) -> None:
        pass

    @property
    def metadata(self) -> Optional[dict]:
        try:
            return get_file(self.import_path / self.METADATA_FILE_NAME)
        except FileNotFoundError:
            return None

    @property
    def commit(self) -> Optional[str]:
        if self.metadata:
            return self.metadata.get("commit")
        return None

    @property
    def content_parser_latest_hash(self) -> Optional[str]:
        if self.metadata:
            return self.metadata.get("content_parser_latest_hash")
        return None

    def dump_metadata(self) -> None:
        """Adds metadata to the graph."""
        metadata = {
            "commit": GitUtil().get_current_commit_hash(),
            "content_parser_latest_hash": self._get_latest_content_parser_hash(),
        }
        with open(self.import_path / self.METADATA_FILE_NAME, "w") as f:
            json.dump(metadata, f)

    def _get_latest_content_parser_hash(self) -> Optional[str]:
        parsers_path = Path(__file__).parent.parent / "parsers"
        parsers_sha1 = sha1_dir(parsers_path)
        logger.debug(f"Content parser hash: {parsers_sha1}")
        return parsers_sha1

    def _has_infra_graph_been_changed(self) -> bool:
        if not self.content_parser_latest_hash:
            logger.warning("The content parser hash is missing.")
        elif self.content_parser_latest_hash != self._get_latest_content_parser_hash():
            logger.warning("The content parser has been changed.")
            return True
        try:
            self.marshal_graph(MarketplaceVersions.XSOAR)
        except Exception as e:
            logger.warning("Failed to load the content graph.")
            logger.debug(f"Validation Error: {e}")
            return True

        return False

    def zip_import_dir(self, output_file: Path) -> None:
        shutil.make_archive(str(output_file), "zip", self.import_path)

    @abstractmethod
    def create_indexes_and_constraints(self) -> None:
        pass

    @abstractmethod
    def create_nodes(self, nodes: Dict[ContentType, List[Dict[str, Any]]]) -> None:
        pass

    @abstractmethod
    def create_relationships(
        self, relationships: Dict[RelationshipType, List[Dict[str, Any]]]
    ) -> None:
        pass

    @abstractmethod
    def remove_non_repo_items(self) -> None:
        pass

    @abstractmethod
    def import_graph(self, imported_path: Optional[Path] = None) -> bool:
        pass

    @abstractmethod
    def export_graph(self, output_path: Optional[Path] = None) -> None:
        pass

    @abstractmethod
    def get_unknown_content_uses(
        self, file_paths: List[str], raises_error: bool
    ) -> List[BaseContent]:
        pass

    @abstractmethod
    def get_duplicate_pack_display_name(
        self, file_paths: List[str]
    ) -> List[Tuple[str, List[str]]]:
        pass

    @abstractmethod
    def find_uses_paths_with_invalid_fromversion(
        self, file_paths: List[str], for_supported_versions=False
    ) -> List[BaseContent]:
        pass

    @abstractmethod
    def find_uses_paths_with_invalid_toversion(
        self, file_paths: List[str], for_supported_versions=False
    ) -> List[BaseContent]:
        pass

    @abstractmethod
    def find_uses_paths_with_invalid_marketplaces(
        self, pack_ids: List[str]
    ) -> List[BaseContent]:
        pass

    @abstractmethod
    def find_core_packs_depend_on_non_core_packs(
        self,
        pack_ids: List[str],
        marketplace: MarketplaceVersions,
        core_pack_list: List[str],
    ) -> List[BaseContent]:
        pass

    @abstractmethod
    def validate_duplicate_ids(
        self, file_paths: List[str]
    ) -> List[Tuple[BaseContent, List[BaseContent]]]:
        pass

    @abstractmethod
    def clean_graph(self):
        ...

    @abstractmethod
    def find_items_using_deprecated_items(self, file_paths: List[str]) -> List[dict]:
        pass

    @abstractmethod
    def get_relationships_by_path(
        self,
        path: Path,
        relationship_type: RelationshipType,
        content_type: ContentType,
        depth: int,
        marketplace: MarketplaceVersions,
        retrieve_sources: bool,
        retrieve_targets: bool,
        mandatory_only: bool,
        include_tests: bool,
        include_deprecated: bool,
        include_hidden: bool,
    ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
        pass

    @abstractmethod
    def search(
        self,
        marketplace: MarketplaceVersions = None,
        content_type: Optional[ContentType] = None,
        ids_list: Optional[Iterable[int]] = None,
        all_level_dependencies: bool = False,
        **properties,
    ) -> List[BaseContent]:
        """
        This searches the database for content items and returns a list of them, including their relationships

        Args:
            marketplace (MarketplaceVersions, optional): Marketplace to search by. Defaults to None.
            content_type (Optional[ContentType], optional): The content_type to filter. Defaults to None.
            ids_list (Optional[Iterable[int]], optional): A list of unique IDs to filter. Defaults to None.
            all_level_dependencies (bool, optional): Whether to return all level dependencies. Defaults to False.
            **properties: A key, value filter for the search. For example: `search(object_id="QRadar")`.

        Returns:
            List[BaseContent]: The search results
        """
        if not marketplace and all_level_dependencies:
            raise ValueError(
                "Cannot search for all level dependencies without a marketplace"
            )
        return []

    def from_path(
        self, path: Path, marketplace: Optional[MarketplaceVersions] = None
    ) -> Union[Pack, ContentItem]:
        """This will return a pack or content item from a path with the local changes, enriched with relationships from the graph

        Args:
            path (Path): The path from a content item
            marketplace (Optional[MarketplaceVersions], optional): The marketplace to use. Defaults to None.

        Raises:
            ValueError: If the path cannot be parsed locally
            ValueError: If the path cannot be found in the graph

        Returns:
            Union[Pack, ContentItem]: The content item found
        """
        content_item = BaseContent.from_path(path)
        if not isinstance(content_item, (ContentItem, Pack)):
            raise ValueError(f"Could not parse content_item from {path}")
        # enrich the content_item with the graph
        result = self.search(
            path=content_item.path.relative_to(self.repo_path), marketplace=marketplace
        )
        if not result or not isinstance(result[0], (Pack, ContentItem)):
            raise ValueError(f"Could not find content item in graph from {path}")
        return result[0]

    def marshal_graph(
        self, marketplace: MarketplaceVersions, all_level_dependencies: bool = False
    ) -> ContentDTO:
        """
        This marshals the graph into a ContentDTO object

        Args:
            marketplace (MarketplaceVersions): The marketplace to filter on
            all_level_dependencies (bool, optional): Whether to marshal all level dependencies. Defaults to False.

        Returns:
            ContentDTO: Marshalled object.
        """
        packs = self.search(
            marketplace,
            content_type=ContentType.PACK,
            all_level_dependencies=all_level_dependencies,
        )
        return ContentDTO(packs=packs)

    @abstractmethod
    def create_pack_dependencies(self):
        ...

    @abstractmethod
    def run_single_query(self, query: str, **kwargs) -> Any:
        pass

    @abstractmethod
    def find_mandatory_hidden_packs_dependencies(
        self, pack_ids: List[str]
    ) -> List[BaseContent]:
        pass

    @abstractmethod
    def get_content_items_by_identifier(
        self,
        identifier_values_list: List[str],
        content_type: ContentType,
        identifier: str,
    ) -> Any:
        pass


# demisto_sdk/commands/content_graph/objects/__init__.py

__all__ = [
    "Pack",
    "Classifier",
    "CorrelationRule",
    "Dashboard",
    "GenericDefinition",
    "GenericField",
    "GenericModule",
    "GenericType",
    "GenericType",
    "IncidentField",
    "IncidentType",
    "IndicatorField",
    "IndicatorType",
    "Integration",
    "Job",
    "Layout",
    "List",
    "Mapper",
    "ModelingRule",
    "ParsingRule",
    "Playbook",
    "Report",
    "Script",
    "TestPlaybook",
    "Trigger",
    "Widget",
    "Wizard",
    "XSIAMDashboard",
    "XSIAMReport",
    "XDRCTemplate",
    "RelationshipData",
    "LayoutRule",
    "PreProcessRule",
]

from demisto_sdk.commands.content_graph.objects.classifier import Classifier
from demisto_sdk.commands.content_graph.objects.correlation_rule import CorrelationRule
from demisto_sdk.commands.content_graph.objects.dashboard import Dashboard
from demisto_sdk.commands.content_graph.objects.generic_definition import (
    GenericDefinition,
)
from demisto_sdk.commands.content_graph.objects.generic_field import GenericField
from demisto_sdk.commands.content_graph.objects.generic_module import GenericModule
from demisto_sdk.commands.content_graph.objects.generic_type import GenericType
from demisto_sdk.commands.content_graph.objects.incident_field import IncidentField
from demisto_sdk.commands.content_graph.objects.incident_type import IncidentType
from demisto_sdk.commands.content_graph.objects.indicator_field import IndicatorField
from demisto_sdk.commands.content_graph.objects.indicator_type import IndicatorType
from demisto_sdk.commands.content_graph.objects.integration import Integration
from demisto_sdk.commands.content_graph.objects.job import Job
from demisto_sdk.commands.content_graph.objects.layout import Layout
from demisto_sdk.commands.content_graph.objects.layout_rule import LayoutRule
from demisto_sdk.commands.content_graph.objects.list import List
from demisto_sdk.commands.content_graph.objects.mapper import Mapper
from demisto_sdk.commands.content_graph.objects.modeling_rule import ModelingRule
from demisto_sdk.commands.content_graph.objects.pack import Pack
from demisto_sdk.commands.content_graph.objects.parsing_rule import ParsingRule
from demisto_sdk.commands.content_graph.objects.playbook import Playbook
from demisto_sdk.commands.content_graph.objects.pre_process_rule import PreProcessRule
from demisto_sdk.commands.content_graph.objects.relationship import RelationshipData
from demisto_sdk.commands.content_graph.objects.report import Report
from demisto_sdk.commands.content_graph.objects.script import Script
from demisto_sdk.commands.content_graph.objects.test_playbook import TestPlaybook
from demisto_sdk.commands.content_graph.objects.trigger import Trigger
from demisto_sdk.commands.content_graph.objects.widget import Widget
from demisto_sdk.commands.content_graph.objects.wizard import Wizard
from demisto_sdk.commands.content_graph.objects.xdrc_template import XDRCTemplate
from demisto_sdk.commands.content_graph.objects.xsiam_dashboard import XSIAMDashboard
from demisto_sdk.commands.content_graph.objects.xsiam_report import XSIAMReport


# demisto_sdk/commands/content_graph/objects/base_content.py

from abc import ABC, abstractmethod
from collections import defaultdict
from functools import lru_cache
from pathlib import Path
from typing import (
    TYPE_CHECKING,
    Any,
    ClassVar,
    Dict,
    List,
    Optional,
    Set,
    Type,
    cast,
)

import demisto_client
from packaging.version import Version
from pydantic import BaseModel, DirectoryPath, Field
from pydantic.main import ModelMetaclass

import demisto_sdk.commands.content_graph.parsers.content_item
from demisto_sdk.commands.common.constants import (
    MARKETPLACE_MIN_VERSION,
    PACKS_FOLDER,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.parsers.content_item import (
    ContentItemParser,
    InvalidContentItemException,
    NotAContentItemException,
)
from demisto_sdk.commands.content_graph.parsers.pack import PackParser

if TYPE_CHECKING:
    from demisto_sdk.commands.content_graph.objects.relationship import RelationshipData

content_type_to_model: Dict[ContentType, Type["BaseContent"]] = {}
json = JSON_Handler()


class BaseContentMetaclass(ModelMetaclass):
    def __new__(
        cls, name, bases, namespace, content_type: ContentType = None, **kwargs
    ):
        """This method is called before every creation of a ContentItem *class* (NOT class instances!).
        If `content_type` is passed as an argument of the class, we add a mapping between the content type
        and the model class object.

        After all the model classes are created, `content_type_to_model` has a full mapping between content types
        and models, and only then we are ready to determine which model class to use based on a content item's type.

        Args:
            name: The class object name (e.g., Integration)
            bases: The bases of the class object (e.g., [YAMLContentItem, ContentItem, BaseContent])
            namespace: The namespaces of the class object.
            content_type (ContentType, optional): The type corresponds to the class (e.g., ContentType.INTEGRATIONS)

        Returns:
            BaseContent: The model class.
        """
        super_cls: BaseContentMetaclass = super().__new__(cls, name, bases, namespace)
        # for type checking
        model_cls: Type["BaseContent"] = cast(Type["BaseContent"], super_cls)
        if content_type:
            content_type_to_model[content_type] = model_cls
            model_cls.content_type = content_type
        return model_cls


class BaseContent(ABC, BaseModel, metaclass=BaseContentMetaclass):
    database_id: Optional[str] = Field(None)  # used for the database
    object_id: str = Field(alias="id")
    content_type: ClassVar[ContentType] = Field(include=True)
    node_id: str
    marketplaces: List[MarketplaceVersions] = list(MarketplaceVersions)

    relationships_data: Dict[RelationshipType, Set["RelationshipData"]] = Field(
        defaultdict(set), exclude=True, repr=False
    )

    class Config:
        arbitrary_types_allowed = (
            True  # allows having custom classes for properties in model
        )
        orm_mode = True  # allows using from_orm() method
        allow_population_by_field_name = True  # when loading from orm, ignores the aliases and uses the property name

    def __getstate__(self):
        """Needed to for the object to be pickled correctly (to use multiprocessing)"""
        if "relationships_data" not in self.__dict__:
            # if we don't have relationships, we can use the default __getstate__ method
            return super().__getstate__()

        dict_copy = self.__dict__.copy()
        # This avoids circular references when pickling store only the first level relationships.
        relationships_data_copy = dict_copy["relationships_data"].copy()
        dict_copy["relationships_data"] = defaultdict(set)
        for _, relationship_data in relationships_data_copy.items():
            for r in relationship_data:
                # override the relationships_data of the content item to avoid circular references
                r: RelationshipData  # type: ignore[no-redef]
                r_copy = r.copy()
                content_item_to_copy = r_copy.content_item_to.copy()
                r_copy.content_item_to = content_item_to_copy
                content_item_to_copy.relationships_data = defaultdict(set)
                dict_copy["relationships_data"][r.relationship_type].add(r_copy)

        return {
            "__dict__": dict_copy,
            "__fields_set__": self.__fields_set__,
        }

    @property
    def normalize_name(self) -> str:
        # if has name attribute, return it, otherwise return the object id
        return self.object_id

    def to_dict(self) -> Dict[str, Any]:
        """
        This returns a JSON dictionary representation of the class.
        We use it instead of `self.dict()` because sometimes we need only the primitive values.

        Returns:
            Dict[str, Any]: _description_
        """

        json_dct = json.loads(self.json(exclude={"commands", "database_id"}))
        if "path" in json_dct and Path(json_dct["path"]).is_absolute():
            json_dct["path"] = (Path(json_dct["path"]).relative_to(CONTENT_PATH)).as_posix()  # type: ignore
        json_dct["content_type"] = self.content_type
        return json_dct

    @staticmethod
    @lru_cache
    def from_path(path: Path) -> Optional["BaseContent"]:
        logger.debug(f"Loading content item from path: {path}")
        if (
            path.is_dir() and path.parent.name == PACKS_FOLDER
        ):  # if the path given is a pack
            try:
                return content_type_to_model[ContentType.PACK].from_orm(
                    PackParser(path)
                )
            except InvalidContentItemException:
                logger.error(f"Could not parse content from {str(path)}")
                return None
        try:
            content_item_parser = ContentItemParser.from_path(path)
        except NotAContentItemException:
            # This is a workaround because `create-content-artifacts` still creates deprecated content items
            demisto_sdk.commands.content_graph.parsers.content_item.MARKETPLACE_MIN_VERSION = (
                "0.0.0"
            )
            try:
                content_item_parser = ContentItemParser.from_path(path)
            except NotAContentItemException:
                logger.error(
                    f"Invalid content path provided: {str(path)}. Please provide a valid content item or pack path."
                )
                return None
            demisto_sdk.commands.content_graph.parsers.content_item.MARKETPLACE_MIN_VERSION = (
                MARKETPLACE_MIN_VERSION
            )
        except InvalidContentItemException:
            logger.error(
                f"Invalid content path provided: {str(path)}. Please provide a valid content item or pack path."
            )
            return None

        model = content_type_to_model.get(content_item_parser.content_type)
        logger.debug(f"Loading content item from path: {path} as {model}")
        if not model:
            logger.error(f"Could not parse content item from path: {path}")
            return None
        try:
            return model.from_orm(content_item_parser)
        except Exception as e:
            logger.error(
                f"Could not parse content item from path: {path}: {e}. Parser class: {content_item_parser}"
            )
            return None

    @abstractmethod
    def dump(
        self,
        path: DirectoryPath,
        marketplace: MarketplaceVersions,
    ) -> None:
        pass

    def upload(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
        target_demisto_version: Version,
        **kwargs,
    ) -> None:
        # Implemented at the ContentItem/Pack level rather than here
        raise NotImplementedError()

    def add_relationship(
        self, relationship_type: RelationshipType, relationship: "RelationshipData"
    ) -> None:
        if relationship.content_item_to == self:
            # skip adding circular dependency
            return
        self.relationships_data[relationship_type].add(relationship)


class UnknownContent(BaseContent):
    """A model for non-existing content items used by existing content items."""

    not_in_repository: bool = True
    node_id: str = ""  # just because it's missing from the db
    object_id: str = ""
    name: str = ""

    def dump(self, _, __):
        ...

    @property
    def identifier(self):
        return self.object_id or self.name


# demisto_sdk/commands/content_graph/objects/classifier.py

from typing import Callable, Optional, Set

import demisto_client
from pydantic import Field

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class Classifier(ContentItem, content_type=ContentType.CLASSIFIER):  # type: ignore[call-arg]
    type: Optional[str]
    definition_id: Optional[str] = Field(alias="definitionId")

    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        return client.import_classifier


# demisto_sdk/commands/content_graph/objects/content_item.py

from abc import abstractmethod
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import TYPE_CHECKING, Callable, List, Optional, Set

import demisto_client
from packaging.version import Version

from demisto_sdk.commands.common.handlers import (
    JSON_Handler,
    XSOAR_Handler,
    YAML_Handler,
)
from demisto_sdk.commands.content_graph.parsers.content_item import (
    InvalidContentItemException,
)
from demisto_sdk.commands.upload.exceptions import IncompatibleUploadVersionException
from demisto_sdk.commands.upload.tools import parse_upload_response

if TYPE_CHECKING:
    from demisto_sdk.commands.content_graph.objects.pack import Pack
    from demisto_sdk.commands.content_graph.objects.relationship import RelationshipData
    from demisto_sdk.commands.content_graph.objects.test_playbook import TestPlaybook

from pydantic import DirectoryPath, validator

from demisto_sdk.commands.common.constants import PACKS_FOLDER, MarketplaceVersions
from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import (
    get_file,
    get_pack_name,
    replace_incident_to_alert,
)
from demisto_sdk.commands.content_graph.common import (
    ContentType,
    RelationshipType,
)
from demisto_sdk.commands.content_graph.objects.base_content import (
    BaseContent,
)
from demisto_sdk.commands.prepare_content.preparers.marketplace_suffix_preparer import (
    MarketplaceSuffixPreparer,
)


class ContentItem(BaseContent):
    path: Path
    marketplaces: List[MarketplaceVersions]
    name: str
    fromversion: str
    toversion: str
    display_name: str
    deprecated: bool
    description: Optional[str]
    is_test: bool = False

    @validator("path", always=True)
    def validate_path(cls, v: Path) -> Path:
        if v.is_absolute():
            return v
        return CONTENT_PATH / v

    @property
    def pack_id(self) -> str:
        return self.in_pack.pack_id if self.in_pack else ""

    @property
    def pack_name(self) -> str:
        return self.in_pack.name if self.in_pack else ""

    @property
    def pack_version(self) -> Optional[Version]:
        return self.in_pack.pack_version if self.in_pack else None

    @property
    def in_pack(self) -> Optional["Pack"]:
        """
        This returns the Pack which the content item is in.

        Returns:
            Pack: Pack model.
        """
        if in_pack := self.relationships_data[RelationshipType.IN_PACK]:
            return next(iter(in_pack)).content_item_to  # type: ignore[return-value]
        if pack_name := get_pack_name(self.path):
            try:
                return BaseContent.from_path(
                    CONTENT_PATH / PACKS_FOLDER / pack_name
                )  # type: ignore[return-value]
            except InvalidContentItemException:
                logger.warning(
                    f"Could not parse pack {pack_name} for content item {self.path}"
                )
                return None
        logger.warning(f"Could not find pack for content item {self.path}")
        return None

    @property
    def uses(self) -> List["RelationshipData"]:
        """
        This returns the content items which this content item uses.
        In addition, we can tell if it's a mandatorily use or not.

        Returns:
            List[RelationshipData]:
                RelationshipData:
                    relationship_type: RelationshipType
                    source: BaseContent
                    target: BaseContent

                    # this is the attribute we're interested in when querying
                    content_item: BaseContent

                    # Whether the relationship between items is direct or not
                    is_direct: bool

                    # Whether using the command mandatorily (or optional)
                    mandatorily: bool = False

        """
        return [
            r
            for r in self.relationships_data[RelationshipType.USES]
            if r.content_item_to.database_id == r.target_id
        ]

    @property
    def tested_by(self) -> List["TestPlaybook"]:
        """
        This returns the test playbooks which the content item is tested by.

        Returns:
            List[TestPlaybook]: List of TestPlaybook models.
        """
        return [
            r.content_item_to  # type: ignore[misc]
            for r in self.relationships_data[RelationshipType.TESTED_BY]
            if r.content_item_to.database_id == r.target_id
        ]

    @property
    def used_by(self) -> List["RelationshipData"]:
        """
        This returns the content items which this content item used by.
        In addition, we can tell if it's a mandatorily use or not.

        Returns:
            List[RelationshipData]:
                RelationshipData:
                    relationship_type: RelationshipType
                    source: BaseContent
                    target: BaseContent

                    # this is the attribute we're interested in when querying
                    content_item: BaseContent

                    # Whether the relationship between items is direct or not
                    is_direct: bool

                    # Whether using the command mandatorily (or optional)
                    mandatorily: bool = False

        """
        return [
            r
            for r in self.relationships_data[RelationshipType.USES]
            if r.content_item_to.database_id == r.source_id
        ]

    @property
    def handler(self) -> XSOAR_Handler:
        # we use a high value so the code lines will not break
        return (
            JSON_Handler()
            if self.path.suffix.lower() == ".json"
            else YAML_Handler(width=50_000)
        )

    @property
    def data(self) -> dict:
        return get_file(self.path, keep_order=False)

    def prepare_for_upload(
        self,
        current_marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
        **kwargs,
    ) -> dict:
        if not self.path.exists():
            raise FileNotFoundError(f"Could not find file {self.path}")
        data = self.data
        logger.debug(f"preparing {self.path}")
        return MarketplaceSuffixPreparer.prepare(
            data, current_marketplace, self.marketplaces
        )

    def summary(
        self,
        marketplace: Optional[MarketplaceVersions] = None,
        incident_to_alert: bool = False,
    ) -> dict:
        """Summary of a content item (the most important metadata fields)

        Args:
            marketplace: The marketplace to get the summary for.
        Returns:
            dict: Dictionary representation of the summary content item.
        """
        summary_res = self.dict(include=self.metadata_fields(), by_alias=True)
        if marketplace and marketplace != MarketplaceVersions.XSOAR:
            data = self.data
            if "id" in summary_res:
                summary_res["id"] = (
                    data.get("commonfields", {}).get("id_x2") or self.object_id
                )
            if "name" in summary_res:
                summary_res["name"] = data.get("name_x2") or self.name

            if incident_to_alert:
                if "name" in summary_res:
                    summary_res["name"] = replace_incident_to_alert(summary_res["name"])
                if "description" in summary_res:
                    summary_res["description"] = replace_incident_to_alert(
                        summary_res["description"]
                    )

        return summary_res

    @abstractmethod
    def metadata_fields(self) -> Set[str]:
        raise NotImplementedError("Should be implemented in subclasses")

    @property
    def normalize_name(self) -> str:
        """
        This will add the server prefix of the content item to its name
        In addition it will remove the existing server_names of the name.

        Args:
            name (str): content item name.
        Returns:
            str: The normalized name.
        """
        name = self.path.name
        server_names = ContentType.server_names()
        for _ in range(2):
            # we iterate twice to handle cases of doubled prefixes like `classifier-mapper-`
            for prefix in server_names:
                try:
                    name = name.removeprefix(f"{prefix}-")  # type: ignore[attr-defined]
                except AttributeError:
                    # not supported in python 3.8
                    name = (
                        name[len(prefix) + 1 :]
                        if name.startswith(f"{prefix}-")
                        else name
                    )
        normalized = f"{self.content_type.server_name}-{name}"
        logger.debug(f"Normalized file name from {name} to {normalized}")
        return normalized

    def dump(
        self,
        dir: DirectoryPath,
        marketplace: MarketplaceVersions,
    ) -> None:
        if not self.path.exists():
            logger.warning(f"Could not find file {self.path}, skipping dump")
            return
        dir.mkdir(exist_ok=True, parents=True)
        try:
            with (dir / self.normalize_name).open("w") as f:
                self.handler.dump(
                    self.prepare_for_upload(current_marketplace=marketplace),
                    f,
                )
        except FileNotFoundError as e:
            logger.warning(f"Failed to dump {self.path} to {dir}: {e}")

    def to_id_set_entity(self) -> dict:
        """
        Transform the model to content item id_set.
        This is temporarily until the content graph is fully merged.

        Returns:
            dict: id_set entiity
        """
        id_set_entity = self.dict()
        id_set_entity["file_path"] = str(self.path)
        id_set_entity["pack"] = self.in_pack.object_id  # type: ignore[union-attr]
        return id_set_entity

    def is_incident_to_alert(self, marketplace: MarketplaceVersions) -> bool:
        """
        As long as the content item does not have an implementation of the `is_incident_to_alert` function,
        the return value will always be false,
        When there is, please override this method in the inheriting class and return `True`.
        Namely, there is no need for special preparation of an incident to alert for the content item.

        Args:
            marketplace (MarketplaceVersions): the destination marketplace.

        Returns:
            bool: False
        """
        return False

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        """
        This attribute sets the method when the upload flow is only of the following form
            >   with TemporaryDirectory("w") as f:
            >       dir_path = Path(f)
            >       self.dump(dir_path, marketplace=marketplace)
            >       client.<<SOME_METHOD>>(file=dir_path / self.normalize_name)

        When the flow is different, return None (default).
        """
        raise NotImplementedError

    def _upload(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
    ) -> None:
        """
        Called once the version is validated.
        Implementation may differ between content items.
        Most items use _client_upload_method, refer to its docstrings.
        """
        try:
            upload_method = self._client_upload_method(client=client)
        except NotImplementedError as e:
            raise NotImplementedError(
                f"missing overriding upload method for {self.content_type}"
            ) from e

        with TemporaryDirectory() as f:
            dir_path = Path(f)
            self.dump(
                dir_path,
                marketplace=marketplace,
            )
            response = upload_method(dir_path / self.normalize_name)
            parse_upload_response(
                response, path=self.path, content_type=self.content_type
            )  # raises on error

    def upload(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
        target_demisto_version: Version,
        **kwargs,
    ) -> None:
        """
        The only upload-related function to be used - the rest are abstract.
        This one checks for version compatibility and then calls _upload.
        """
        if not (
            Version(self.fromversion)
            <= target_demisto_version
            <= Version(self.toversion)
        ):
            raise IncompatibleUploadVersionException(self, target_demisto_version)
        self._upload(client, marketplace)


# demisto_sdk/commands/content_graph/objects/content_item_xsiam.py

from abc import ABC
from pathlib import Path
from typing import List

import demisto_client
from packaging.version import Version
from pydantic import DirectoryPath

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.objects.content_item import (
    ContentItem,
)
from demisto_sdk.commands.upload.exceptions import (
    NotIndivitudallyUploadableException,
)


class ContentItemXSIAM(ContentItem, ABC):
    def dump(
        self,
        dir: DirectoryPath,
        marketplace: MarketplaceVersions,
    ) -> None:
        dir.mkdir(exist_ok=True, parents=True)

        output_paths: List[Path] = []
        if Version(self.fromversion) >= Version("6.10.0"):
            # export XSIAM 1.3 items only with the external prefix
            output_paths.append(dir / f"external-{self.normalize_name}")

        elif Version(self.toversion) < Version("6.10.0"):
            # export XSIAM 1.2 items only without the external prefix
            output_paths.append(dir / self.normalize_name)
        else:
            # export 2 versions of the file, with/without the external prefix.
            output_paths.append(dir / f"external-{self.normalize_name}")
            output_paths.append(dir / self.normalize_name)

        data = self.prepare_for_upload(
            marketplace,
        )

        for file in output_paths:
            with open(file, "w") as f:
                self.handler.dump(
                    data,
                    f,
                )

    def _upload(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
    ) -> None:
        """
        Uploadable XSIAM items should override this method.
        The rest will raise as default.
        """
        raise NotIndivitudallyUploadableException(self)


# demisto_sdk/commands/content_graph/objects/correlation_rule.py

from typing import Set

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item_xsiam import (
    ContentItemXSIAM,
)


class CorrelationRule(ContentItemXSIAM, content_type=ContentType.CORRELATION_RULE):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}


# demisto_sdk/commands/content_graph/objects/dashboard.py

from typing import Callable, Set

import demisto_client

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class Dashboard(ContentItem, content_type=ContentType.DASHBOARD):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"name"}

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        return client.import_dashboard


# demisto_sdk/commands/content_graph/objects/exceptions.py

from pathlib import Path
from typing import TYPE_CHECKING, Optional, Sequence

from demisto_sdk.commands.upload.exceptions import IncompatibleUploadVersionException

if TYPE_CHECKING:
    from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class FailedUploadException(RuntimeError):
    def __init__(
        self,
        path: Path,
        response_body: dict,
        status_code: Optional[int] = None,
        additional_info: Optional[str] = None,
    ) -> None:
        self.path = path
        self.response_body = response_body
        self.status_code = status_code
        self.additional_info = additional_info

        super().__init__(
            "\n".join(
                (
                    additional_info or "",
                    f"{status_code=}" if status_code else "",
                    f"{response_body=}" if response_body else "",
                )
            )
        )


class FailedUploadMultipleException(RuntimeError):
    def __init__(
        self,
        uploaded_successfully: Sequence["ContentItem"],
        upload_failures: Sequence[FailedUploadException],
        incompatible_versions_items: Sequence[IncompatibleUploadVersionException],
    ) -> None:
        self.uploaded_successfully = uploaded_successfully
        self.upload_failures = upload_failures
        self.incompatible_versions_items = incompatible_versions_items
        super().__init__((self.upload_failures, self.incompatible_versions_items))

    def __str__(self) -> str:
        new_line = "\n"
        return f"""Upload Failures: {new_line.join(str(failure) for failure in self.upload_failures)}{new_line}Incompatible versions: {new_line.join(str(item) for item in self.incompatible_versions_items)}"""


# demisto_sdk/commands/content_graph/objects/generic_definition.py

from typing import Set

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class GenericDefinition(ContentItem, content_type=ContentType.GENERIC_DEFINITION):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}


# demisto_sdk/commands/content_graph/objects/generic_field.py

from typing import Optional, Set

from pydantic import DirectoryPath, Field

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class GenericField(ContentItem, content_type=ContentType.GENERIC_FIELD):  # type: ignore[call-arg]
    definition_id: Optional[str] = Field(alias="definitionId")

    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    def dump(
        self,
        dir: DirectoryPath,
        marketplace: MarketplaceVersions,
    ) -> None:
        super().dump(
            dir=dir / self.path.parent.name,
            marketplace=marketplace,
        )


# demisto_sdk/commands/content_graph/objects/generic_module.py

from typing import List, Optional, Set

from pydantic import Field

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class GenericModule(ContentItem, content_type=ContentType.GENERIC_MODULE):  # type: ignore[call-arg]
    definition_ids: Optional[List[str]] = Field(alias="definitionIds")

    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}


# demisto_sdk/commands/content_graph/objects/generic_type.py

from typing import Optional, Set

from pydantic import DirectoryPath, Field

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class GenericType(ContentItem, content_type=ContentType.GENERIC_TYPE):  # type: ignore[call-arg]
    definition_id: Optional[str] = Field(alias="definitionId")

    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    def dump(
        self,
        dir: DirectoryPath,
        marketplace: MarketplaceVersions,
    ) -> None:
        super().dump(
            dir=dir / self.path.parent.name,
            marketplace=marketplace,
        )


# demisto_sdk/commands/content_graph/objects/incident_field.py

from tempfile import NamedTemporaryFile
from typing import Set

import demisto_client
from pydantic import Field

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem

json = JSON_Handler()


class IncidentField(ContentItem, content_type=ContentType.INCIDENT_FIELD):  # type: ignore[call-arg]
    cli_name: str = Field(alias="cliName")
    field_type: str = Field(alias="type")
    associated_to_all: bool = Field(False, alias="associatedToAll")

    def metadata_fields(self) -> Set[str]:
        return {"name", "field_type", "description"}

    def _upload(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
    ) -> None:
        with NamedTemporaryFile(suffix=".json", mode="r+") as file:
            json.dump(
                # Wrapping the data as the server expects to receive it
                {"incidentFields": [self.prepare_for_upload(marketplace=marketplace)]},
                file,
            )
            file.flush()
            file.seek(0)
            return client.import_incident_fields(file=file.name)


# demisto_sdk/commands/content_graph/objects/incident_type.py

from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Optional, Set

import demisto_client
from pydantic import Field

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class IncidentType(ContentItem, content_type=ContentType.INCIDENT_TYPE):  # type: ignore[call-arg]
    playbook: Optional[str]
    hours: int
    days: int
    weeks: int
    closure_script: Optional[str] = Field(alias="closureScript")

    def metadata_fields(self) -> Set[str]:
        return {"name", "playbook", "closure_script", "hours", "days", "week"}

    def _upload(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
    ) -> None:
        with TemporaryDirectory() as dir:
            file_path = Path(dir, self.normalize_name)
            with open(file_path, "w") as f:
                # Wrapping the dictionary with a list, as that's what the server expects
                json.dump([self.prepare_for_upload(marketplace=marketplace)], f)
            client.import_incident_types_handler(str(file_path))


# demisto_sdk/commands/content_graph/objects/indicator_field.py

from typing import Callable, Set

import demisto_client
from pydantic import Field

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class IndicatorField(ContentItem, content_type=ContentType.INDICATOR_FIELD):  # type: ignore[call-arg]
    cli_name: str = Field(alias="cliName")
    type: str
    associated_to_all: bool = Field(alias="associatedToAll")

    def metadata_fields(self) -> Set[str]:
        return {"name", "type", "description"}

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        return client.import_incident_fields


# demisto_sdk/commands/content_graph/objects/indicator_type.py

from pathlib import Path
from tempfile import TemporaryDirectory
from typing import List, Optional, Set

import demisto_client
from pydantic import Field

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.handlers import JSON_Handler
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem

json = JSON_Handler()


class IndicatorType(ContentItem, content_type=ContentType.INDICATOR_TYPE):  # type: ignore[call-arg]
    description: str = Field(alias="details")
    regex: Optional[str]
    reputation_script_name: Optional[str] = Field(alias="reputationScriptName")
    enhancement_script_names: Optional[List[str]] = Field(
        alias="enhancementScriptNames"
    )

    def metadata_fields(self) -> Set[str]:
        return {"details", "reputation_script_name", "enhancement_script_names"}

    def _upload(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
    ) -> None:
        with TemporaryDirectory() as dir:
            file_path = Path(dir, self.normalize_name)
            with open(file_path, "w") as f:
                # Wrapping the dictionary with a list, as that's what the server expects
                json.dump([self.prepare_for_upload(marketplace=marketplace)], f)
            client.import_reputation_handler(str(file_path))


# demisto_sdk/commands/content_graph/objects/integration.py

from typing import TYPE_CHECKING, Callable, List, Optional

import demisto_client

from demisto_sdk.commands.content_graph.objects.base_content import BaseContent

if TYPE_CHECKING:
    # avoid circular imports
    from demisto_sdk.commands.content_graph.objects.script import Script

from pydantic import Field

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.objects.integration_script import (
    IntegrationScript,
)


class Command(BaseContent, content_type=ContentType.COMMAND):  # type: ignore[call-arg]
    name: str

    # From HAS_COMMAND relationship
    deprecated: bool = False
    description: Optional[str] = ""

    # missing attributes in DB
    node_id: str = ""
    object_id: str = Field("", alias="id")

    @property
    def integrations(self) -> List["Integration"]:
        return [
            r.content_item_to  # type: ignore[misc]
            for r in self.relationships_data[RelationshipType.HAS_COMMAND]
            if r.content_item_to.database_id == r.source_id
        ]

    def dump(self, *args) -> None:
        raise NotImplementedError()


class Integration(IntegrationScript, content_type=ContentType.INTEGRATION):  # type: ignore[call-arg]
    is_fetch: bool = False
    is_fetch_events: bool = False
    is_fetch_events_and_assets: bool = False
    is_feed: bool = False
    category: str
    commands: List[Command] = []

    @property
    def imports(self) -> List["Script"]:
        return [
            r.content_item_to  # type: ignore[misc]
            for r in self.relationships_data[RelationshipType.IMPORTS]
            if r.content_item_to.database_id == r.target_id
        ]

    def set_commands(self):
        commands = [
            Command(
                # the related to has to be a command
                name=r.content_item_to.name,  # type: ignore[union-attr,attr-defined]
                marketplaces=self.marketplaces,
                deprecated=r.deprecated,
                description=r.description,
            )
            for r in self.relationships_data[RelationshipType.HAS_COMMAND]
        ]
        self.commands = commands

    def metadata_fields(self):
        return {
            "name": True,
            "description": True,
            "category": True,
            "commands": {
                "__all__": {"name": True, "description": True}
            },  # for all commands, keep the name and description
        }

    def prepare_for_upload(
        self,
        current_marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
        **kwargs,
    ) -> dict:
        data = super().prepare_for_upload(current_marketplace, **kwargs)

        if supported_native_images := self.get_supported_native_images(
            marketplace=current_marketplace,
            ignore_native_image=kwargs.get("ignore_native_image") or False,
        ):
            logger.debug(
                f"Adding the following native images {supported_native_images} to integration {self.object_id}"
            )
            data["script"]["nativeimage"] = supported_native_images

        return data

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        return client.integration_upload


# demisto_sdk/commands/content_graph/objects/integration_script.py

from pathlib import Path
from typing import List, Optional

from pydantic import Field

from demisto_sdk.commands.common.constants import (
    NATIVE_IMAGE_FILE_NAME,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.native_image import (
    ScriptIntegrationSupportedNativeImages,
    file_to_native_image_config,
)
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem
from demisto_sdk.commands.prepare_content.integration_script_unifier import (
    IntegrationScriptUnifier,
)


class IntegrationScript(ContentItem):
    type: str
    docker_image: Optional[str]
    description: Optional[str]
    is_unified: bool = Field(False, exclude=True)
    code: Optional[str] = Field(None, exclude=True)

    def prepare_for_upload(
        self,
        current_marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
        **kwargs,
    ) -> dict:
        data = (
            self.data
            if kwargs.get("unify_only")
            else super().prepare_for_upload(current_marketplace)
        )
        data = IntegrationScriptUnifier.unify(
            self.path, data, current_marketplace, **kwargs
        )
        return data

    def get_supported_native_images(
        self, marketplace: MarketplaceVersions, ignore_native_image: bool = False
    ) -> List[str]:
        if marketplace == MarketplaceVersions.XSOAR and not ignore_native_image:
            if not Path(f"Tests/{NATIVE_IMAGE_FILE_NAME}").exists():
                logger.debug(f"The {NATIVE_IMAGE_FILE_NAME} file could not be found.")
                return []
            return ScriptIntegrationSupportedNativeImages(
                _id=self.object_id,
                docker_image=self.docker_image,
                native_image_config=file_to_native_image_config(),
            ).get_supported_native_image_versions(get_raw_version=True)
        return []


# demisto_sdk/commands/content_graph/objects/job.py

import logging
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Set

import demisto_client
from pydantic import Field

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem

logger = logging.getLogger("demisto-sdk")


class Job(ContentItem, content_type=ContentType.JOB):  # type: ignore[call-arg]
    description: str = Field(alias="details")

    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    def _upload(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
    ) -> None:
        with TemporaryDirectory("w") as f:
            dir_path = Path(f)
            self.dump(dir_path, marketplace=marketplace)

            return client.generic_request(
                method="POST",
                path="jobs/import",
                files={"file": str(self.path)},
                content_type="multipart/form-data",
            )


# demisto_sdk/commands/content_graph/objects/layout.py

import logging
from typing import Callable, List, Optional, Set, Union

import demisto_client
from pydantic import Field

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem

logger = logging.getLogger("demisto-sdk")


class Layout(ContentItem, content_type=ContentType.LAYOUT):  # type: ignore[call-arg]
    kind: Optional[str]
    tabs: Optional[List[str]]
    definition_id: Optional[str] = Field(alias="definitionId")
    group: str
    edit: bool
    indicators_details: bool
    indicators_quick_view: bool
    quick_view: bool
    close: bool
    details: bool
    details_v2: bool
    mobile: bool

    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    def prepare_for_upload(
        self,
        current_marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
        **kwargs,
    ) -> dict:
        # marketplace is the marketplace for which the content is prepared.
        data = super().prepare_for_upload(current_marketplace, **kwargs)
        data = self._fix_from_and_to_server_version(data)

        if (
            current_marketplace == MarketplaceVersions.MarketplaceV2
            and self.group == "indicator"
        ):
            data = replace_layout_incident_alert(data)

        return data

    def _fix_from_and_to_server_version(self, data: dict) -> dict:
        # On Layouts, we manually add the `fromServerVersion`, `toServerVersion` fields, see CIAC-5195.
        data["fromServerVersion"] = self.fromversion
        data["toServerVersion"] = self.toversion
        return data

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        return client.import_layout


def replace_layout_incident_alert(data: dict) -> dict:
    """
    Changes {"name": "Related/Linked/Chiled Incidents", ... }
         to {"name": "Related/Linked/Chiled Alerts", ... }
    """

    if not isinstance(data, dict):
        raise TypeError(f"expected dictionary, got {type(data)}")

    def fix_recursively(datum: Union[list, dict]) -> Union[list, dict]:
        if isinstance(datum, dict):
            if datum.get("name_x2") is not None:
                # already has a xsiam name, then we have nothing to do
                return datum
            if (name := datum.get("name", ""), datum.get("type")) in {
                ("Child Incidents", "childInv"),
                ("Linked Incidents", "linkedIncidents"),
                ("Related Incidents", "relatedIncidents"),
            }:
                datum["name"] = name.replace("Incident", "Alert")
                return datum
            else:  # not the atomic dictionary that we intend to fix, use recursion instead.
                return {key: fix_recursively(value) for key, value in datum.items()}

        elif isinstance(datum, list):
            return [fix_recursively(item) for item in datum]

        else:
            return datum  # nothing to change

    if not isinstance(result := fix_recursively(data), dict):
        """
        the inner function returns a value of the same type as its input,
        so a dict input should never return a non-dict. this part is just for safety (mypy).
        """
        raise ValueError(f"unexpected type for a fixed-dictionary output {type(data)}")

    return result


# demisto_sdk/commands/content_graph/objects/layout_rule.py

from typing import Set

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item_xsiam import (
    ContentItemXSIAM,
)


class LayoutRule(ContentItemXSIAM, content_type=ContentType.LAYOUT_RULE):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"rule_name", "description"}


# demisto_sdk/commands/content_graph/objects/list.py

import logging
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Set

import demisto_client

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem

logger = logging.getLogger("demisto-sdk")


class List(ContentItem, content_type=ContentType.LIST):  # type: ignore[call-arg]
    type: str

    def metadata_fields(self) -> Set[str]:
        return {"name"}

    def _upload(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
    ) -> None:
        with TemporaryDirectory("w") as f:
            dir_path = Path(f)
            self.dump(dir_path, marketplace=marketplace)

            client.generic_request(
                method="POST",
                path="lists/save",
                body=dir_path / self.normalize_name,
                response_type="object",
            )


# demisto_sdk/commands/content_graph/objects/mapper.py

from typing import Callable, Optional, Set

import demisto_client
from pydantic import Field

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class Mapper(ContentItem, content_type=ContentType.MAPPER):  # type: ignore[call-arg]
    type: Optional[str]
    definition_id: Optional[str] = Field(
        alias="definitionId"
    )  # TODO decide if this should be optional or not

    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        return client.import_classifier


# demisto_sdk/commands/content_graph/objects/modeling_rule.py

from typing import Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item_xsiam import (
    ContentItemXSIAM,
)
from demisto_sdk.commands.prepare_content.rule_unifier import RuleUnifier


class ModelingRule(ContentItemXSIAM, content_type=ContentType.MODELING_RULE):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    def prepare_for_upload(
        self,
        current_marketplace: MarketplaceVersions = MarketplaceVersions.MarketplaceV2,
        **kwargs
    ) -> dict:
        if not kwargs.get("unify_only"):
            data = super().prepare_for_upload(current_marketplace)
        else:
            data = self.data
        data = RuleUnifier.unify(self.path, data, current_marketplace)
        return data


# demisto_sdk/commands/content_graph/objects/pack.py

import shutil
from collections import defaultdict
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import TYPE_CHECKING, Any, Generator, List, Optional

import demisto_client
from demisto_client.demisto_api.rest import ApiException
from packaging.version import Version, parse
from pydantic import BaseModel, DirectoryPath, Field, validator

from demisto_sdk.commands.common.constants import (
    BASE_PACK,
    CONTRIBUTORS_README_TEMPLATE,
    MARKETPLACE_MIN_VERSION,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH
from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import MarketplaceTagParser
from demisto_sdk.commands.content_graph.common import (
    PACK_METADATA_FILENAME,
    ContentType,
    Nodes,
    Relationships,
    RelationshipType,
)
from demisto_sdk.commands.content_graph.objects.base_content import (
    BaseContent,
)
from demisto_sdk.commands.content_graph.objects.classifier import Classifier
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem
from demisto_sdk.commands.content_graph.objects.content_item_xsiam import (
    NotIndivitudallyUploadableException,
)
from demisto_sdk.commands.content_graph.objects.correlation_rule import CorrelationRule
from demisto_sdk.commands.content_graph.objects.dashboard import Dashboard
from demisto_sdk.commands.content_graph.objects.exceptions import (
    FailedUploadException,
    FailedUploadMultipleException,
)
from demisto_sdk.commands.content_graph.objects.generic_definition import (
    GenericDefinition,
)
from demisto_sdk.commands.content_graph.objects.generic_field import GenericField
from demisto_sdk.commands.content_graph.objects.generic_module import GenericModule
from demisto_sdk.commands.content_graph.objects.generic_type import GenericType
from demisto_sdk.commands.content_graph.objects.incident_field import IncidentField
from demisto_sdk.commands.content_graph.objects.incident_type import IncidentType
from demisto_sdk.commands.content_graph.objects.indicator_field import IndicatorField
from demisto_sdk.commands.content_graph.objects.indicator_type import IndicatorType
from demisto_sdk.commands.content_graph.objects.integration import Integration
from demisto_sdk.commands.content_graph.objects.job import Job
from demisto_sdk.commands.content_graph.objects.layout import Layout
from demisto_sdk.commands.content_graph.objects.layout_rule import LayoutRule
from demisto_sdk.commands.content_graph.objects.list import List as ListObject
from demisto_sdk.commands.content_graph.objects.mapper import Mapper
from demisto_sdk.commands.content_graph.objects.modeling_rule import ModelingRule
from demisto_sdk.commands.content_graph.objects.parsing_rule import ParsingRule
from demisto_sdk.commands.content_graph.objects.playbook import Playbook
from demisto_sdk.commands.content_graph.objects.pre_process_rule import PreProcessRule
from demisto_sdk.commands.content_graph.objects.report import Report
from demisto_sdk.commands.content_graph.objects.script import Script
from demisto_sdk.commands.content_graph.objects.test_playbook import TestPlaybook
from demisto_sdk.commands.content_graph.objects.trigger import Trigger
from demisto_sdk.commands.content_graph.objects.widget import Widget
from demisto_sdk.commands.content_graph.objects.wizard import Wizard
from demisto_sdk.commands.content_graph.objects.xdrc_template import XDRCTemplate
from demisto_sdk.commands.content_graph.objects.xsiam_dashboard import XSIAMDashboard
from demisto_sdk.commands.content_graph.objects.xsiam_report import XSIAMReport
from demisto_sdk.commands.upload.constants import (
    CONTENT_TYPES_EXCLUDED_FROM_UPLOAD,
    MULTIPLE_ZIPPED_PACKS_FILE_NAME,
    MULTIPLE_ZIPPED_PACKS_FILE_STEM,
)
from demisto_sdk.commands.upload.exceptions import IncompatibleUploadVersionException
from demisto_sdk.commands.upload.tools import (
    parse_error_response,
    parse_upload_response,
)

if TYPE_CHECKING:
    from demisto_sdk.commands.content_graph.objects.relationship import RelationshipData


MINIMAL_UPLOAD_SUPPORTED_VERSION = Version("6.5.0")
MINIMAL_ALLOWED_SKIP_VALIDATION_VERSION = Version("6.6.0")


def upload_zip(
    path: Path,
    client: demisto_client,
    skip_validations: bool,
    target_demisto_version: Version,
    marketplace: MarketplaceVersions,
) -> bool:
    """
    Used to upload an existing zip file
    """
    if path.suffix != ".zip":
        raise RuntimeError(f"cannot upload {path} as zip")
    if (
        marketplace == MarketplaceVersions.XSOAR
        and target_demisto_version < MINIMAL_UPLOAD_SUPPORTED_VERSION
    ):
        raise RuntimeError(
            f"Uploading packs to XSOAR versions earlier than {MINIMAL_UPLOAD_SUPPORTED_VERSION} is no longer supported."
            "Use older versions of the Demisto-SDK for that (<=1.13.0)"
        )
    server_kwargs = {"skip_verify": "true"}

    if (
        skip_validations
        and target_demisto_version >= MINIMAL_ALLOWED_SKIP_VALIDATION_VERSION
    ):
        server_kwargs["skip_validation"] = "true"

    response = client.upload_content_packs(
        file=str(path),
        **server_kwargs,
    )
    if response is None:  # uploaded successfully
        return True

    parse_upload_response(
        response, path=path, content_type=ContentType.PACK
    )  # raises on error
    return True


class PackContentItems(BaseModel):
    # The alias is for marshalling purposes
    classifier: List[Classifier] = Field([], alias=ContentType.CLASSIFIER.value)
    correlation_rule: List[CorrelationRule] = Field(
        [], alias=ContentType.CORRELATION_RULE.value
    )
    dashboard: List[Dashboard] = Field([], alias=ContentType.DASHBOARD.value)
    generic_definition: List[GenericDefinition] = Field(
        [], alias=ContentType.GENERIC_DEFINITION.value
    )
    generic_field: List[GenericField] = Field([], alias=ContentType.GENERIC_FIELD.value)
    generic_module: List[GenericModule] = Field(
        [], alias=ContentType.GENERIC_MODULE.value
    )
    generic_type: List[GenericType] = Field([], alias=ContentType.GENERIC_TYPE.value)
    incident_field: List[IncidentField] = Field(
        [], alias=ContentType.INCIDENT_FIELD.value
    )
    incident_type: List[IncidentType] = Field([], alias=ContentType.INCIDENT_TYPE.value)
    indicator_field: List[IndicatorField] = Field(
        [], alias=ContentType.INDICATOR_FIELD.value
    )
    indicator_type: List[IndicatorType] = Field(
        [], alias=ContentType.INDICATOR_TYPE.value
    )
    integration: List[Integration] = Field([], alias=ContentType.INTEGRATION.value)
    job: List[Job] = Field([], alias=ContentType.JOB.value)
    layout: List[Layout] = Field([], alias=ContentType.LAYOUT.value)
    list: List[ListObject] = Field([], alias=ContentType.LIST.value)
    mapper: List[Mapper] = Field([], alias=ContentType.MAPPER.value)
    modeling_rule: List[ModelingRule] = Field([], alias=ContentType.MODELING_RULE.value)
    parsing_rule: List[ParsingRule] = Field([], alias=ContentType.PARSING_RULE.value)
    playbook: List[Playbook] = Field([], alias=ContentType.PLAYBOOK.value)
    report: List[Report] = Field([], alias=ContentType.REPORT.value)
    script: List[Script] = Field([], alias=ContentType.SCRIPT.value)
    test_playbook: List[TestPlaybook] = Field([], alias=ContentType.TEST_PLAYBOOK.value)
    trigger: List[Trigger] = Field([], alias=ContentType.TRIGGER.value)
    widget: List[Widget] = Field([], alias=ContentType.WIDGET.value)
    wizard: List[Wizard] = Field([], alias=ContentType.WIZARD.value)
    xsiam_dashboard: List[XSIAMDashboard] = Field(
        [], alias=ContentType.XSIAM_DASHBOARD.value
    )
    xsiam_report: List[XSIAMReport] = Field([], alias=ContentType.XSIAM_REPORT.value)
    xdrc_template: List[XDRCTemplate] = Field([], alias=ContentType.XDRC_TEMPLATE.value)
    layout_rule: List[LayoutRule] = Field([], alias=ContentType.LAYOUT_RULE.value)
    preprocess_rule: List[PreProcessRule] = Field(
        [], alias=ContentType.PREPROCESS_RULE.value
    )

    def __iter__(self) -> Generator[ContentItem, Any, Any]:  # type: ignore
        """Defines the iteration of the object. Each iteration yields a single content item."""
        for content_items in vars(self).values():
            yield from content_items

    def __bool__(self) -> bool:
        """Used for easier determination of content items existence in a pack."""
        return bool(list(self))

    class Config:
        arbitrary_types_allowed = True
        orm_mode = True
        allow_population_by_field_name = True


class PackMetadata(BaseModel):
    name: str
    description: Optional[str]
    created: Optional[str]
    updated: Optional[str]
    support: Optional[str]
    email: Optional[str]
    url: Optional[str]
    author: Optional[str]
    certification: Optional[str]
    hidden: Optional[bool]
    server_min_version: Optional[str] = Field(alias="serverMinVersion")
    current_version: Optional[str] = Field(alias="currentVersion")
    tags: Optional[List[str]]
    categories: Optional[List[str]]
    use_cases: Optional[List[str]] = Field(alias="useCases")
    keywords: Optional[List[str]]
    price: Optional[int] = None
    premium: Optional[bool] = None
    vendor_id: Optional[str] = Field(None, alias="vendorId")
    vendor_name: Optional[str] = Field(None, alias="vendorName")
    preview_only: Optional[bool] = Field(None, alias="previewOnly")
    excluded_dependencies: Optional[List[str]]


class Pack(BaseContent, PackMetadata, content_type=ContentType.PACK):  # type: ignore[call-arg]
    path: Path
    contributors: Optional[List[str]] = None
    relationships: Relationships = Field(Relationships(), exclude=True)
    deprecated: bool = False
    content_items: PackContentItems = Field(
        PackContentItems(), alias="contentItems", exclude=True
    )

    @validator("path", always=True)
    def validate_path(cls, v: Path) -> Path:
        if v.is_absolute():
            return v
        return CONTENT_PATH / v

    @property
    def pack_id(self) -> str:
        return self.object_id

    @property
    def pack_name(self) -> str:
        return self.name

    @property
    def pack_version(self) -> Optional[Version]:
        return Version(self.current_version) if self.current_version else None

    @property
    def depends_on(self) -> List["RelationshipData"]:
        """
        This returns the packs which this content item depends on.
        In addition, we can tell if it's a mandatorily dependency or not.

        Returns:
            List[RelationshipData]:
                RelationshipData:
                    relationship_type: RelationshipType
                    source: BaseContent
                    target: BaseContent

                    # this is the attribute we're interested in when querying
                    content_item: BaseContent

                    # Whether the relationship between items is direct or not
                    is_direct: bool

                    # Whether using the command mandatorily (or optional)
                    mandatorily: bool = False

        """
        return [
            r
            for r in self.relationships_data[RelationshipType.DEPENDS_ON]
            if r.content_item_to.database_id == r.target_id
        ]

    def set_content_items(self):
        content_items: List[ContentItem] = [
            r.content_item_to  # type: ignore[misc]
            for r in self.relationships_data[RelationshipType.IN_PACK]
            if r.content_item_to.database_id == r.source_id
        ]
        content_item_dct = defaultdict(list)
        for c in content_items:
            content_item_dct[c.content_type.value].append(c)

        # If there is no server_min_version, set it to the minimum of its content items fromversion
        min_content_items_version = MARKETPLACE_MIN_VERSION
        if content_items:
            min_content_items_version = str(
                min(parse(content_item.fromversion) for content_item in content_items)
            )
        self.server_min_version = self.server_min_version or min_content_items_version
        self.content_items = PackContentItems(**content_item_dct)

    def dump_metadata(self, path: Path, marketplace: MarketplaceVersions) -> None:
        self.server_min_version = self.server_min_version or MARKETPLACE_MIN_VERSION
        metadata = self.dict(
            exclude={"path", "node_id", "content_type", "excluded_dependencies"},
            by_alias=True,
        )
        metadata["contentItems"] = {}
        metadata["id"] = self.object_id
        for content_item in self.content_items:

            if content_item.content_type == ContentType.TEST_PLAYBOOK:
                logger.debug(
                    f"Skip loading the {content_item.name} test playbook into metadata.json"
                )
                continue
            if content_item.is_incident_to_alert(marketplace):
                metadata["contentItems"].setdefault(
                    content_item.content_type.metadata_name, []
                ).append(content_item.summary(marketplace, incident_to_alert=True))

            metadata["contentItems"].setdefault(
                content_item.content_type.metadata_name, []
            ).append(content_item.summary(marketplace))

        metadata["contentDisplays"] = self.get_content_display(metadata["contentItems"])

        with open(path, "w") as f:
            json.dump(metadata, f, indent=4, sort_keys=True)

    def get_content_display(self, content_items):

        content_displays: dict = {}
        for content_item in self.content_items:
            try:
                content_displays[
                    content_item.content_type.metadata_name
                ] = (
                    content_item.content_type.metadata_display_name
                )  # type: ignore[index]
            except TypeError as e:
                raise Exception(
                    f"Could not set metadata_name of type {content_item.content_type.metadata_name} - "
                    f"{content_item.content_type.metadata_display_name} in {content_displays}\n{e}"
                )
        content_displays = {
            content_type: content_type_display
            if (
                content_items.get(content_type)
                and len(content_items.get(content_type)) == 1
            )
            else f"{content_type_display}s"
            for content_type, content_type_display in content_displays.items()
        }

        return content_displays

    def dump_readme(self, path: Path, marketplace: MarketplaceVersions) -> None:
        shutil.copyfile(self.path / "README.md", path)
        if self.contributors:
            fixed_contributor_names = [
                f" - {contrib_name}\n" for contrib_name in self.contributors
            ]
            contribution_data = CONTRIBUTORS_README_TEMPLATE.format(
                contributors_names="".join(fixed_contributor_names)
            )
            with open(path, "a+") as f:
                f.write(contribution_data)
        with open(path, "r+") as f:
            try:
                text = f.read()
                parsed_text = MarketplaceTagParser(marketplace).parse_text(text)
                if len(text) != len(parsed_text):
                    f.seek(0)
                    f.write(parsed_text)
                    f.truncate()
            except Exception as e:
                logger.error(f"Failed dumping readme: {e}")

    def dump(
        self,
        path: Path,
        marketplace: MarketplaceVersions,
    ):
        if not self.path.exists():
            logger.warning(f"Pack {self.name} does not exist in {self.path}")
            return

        try:
            path.mkdir(exist_ok=True, parents=True)

            for content_item in self.content_items:
                if content_item.content_type in CONTENT_TYPES_EXCLUDED_FROM_UPLOAD:
                    logger.debug(
                        f"SKIPPING dump {content_item.content_type} {content_item.normalize_name}"
                        "whose type was passed in `exclude_content_types`"
                    )
                    continue

                if marketplace not in content_item.marketplaces:
                    logger.debug(
                        f"SKIPPING dump {content_item.content_type} {content_item.normalize_name}"
                        f"to destination {marketplace=}"
                        f" - content item has marketplaces {content_item.marketplaces}"
                    )
                    continue

                folder = content_item.content_type.as_folder
                if (
                    content_item.content_type == ContentType.SCRIPT
                    and content_item.is_test
                ):
                    folder = ContentType.TEST_PLAYBOOK.as_folder

                content_item.dump(
                    dir=path / folder,
                    marketplace=marketplace,
                )
            self.dump_metadata(path / "metadata.json", marketplace)
            self.dump_readme(path / "README.md", marketplace)
            shutil.copy(
                self.path / PACK_METADATA_FILENAME, path / PACK_METADATA_FILENAME
            )
            try:
                shutil.copytree(self.path / "ReleaseNotes", path / "ReleaseNotes")
            except FileNotFoundError:
                logger.debug(f'No such file {self.path / "ReleaseNotes"}')

            try:
                shutil.copy(self.path / "Author_image.png", path / "Author_image.png")
            except FileNotFoundError:
                logger.debug(f'No such file {self.path / "Author_image.png"}')

            if self.object_id == BASE_PACK:
                self._copy_base_pack_docs(path, marketplace)

            pack_files = "\n".join([str(f) for f in path.iterdir()])
            logger.info(f"Dumped pack {self.name}.")
            logger.debug(f"Pack {self.name} files:\n{pack_files}")

        except Exception:
            logger.exception(f"Failed dumping pack {self.name}")
            raise

    def upload(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
        target_demisto_version: Version,
        destination_zip_dir: Optional[Path] = None,
        zip: bool = False,
        **kwargs,
    ):
        if destination_zip_dir is None:
            raise ValueError("invalid destination_zip_dir=None")

        if zip:
            self._zip_and_upload(
                client=client,
                marketplace=marketplace,
                target_demisto_version=target_demisto_version,
                skip_validations=kwargs.get("skip_validations", False),
                destination_dir=destination_zip_dir,
            )
        else:
            self._upload_item_by_item(
                client=client,
                marketplace=marketplace,
                target_demisto_version=target_demisto_version,
            )

    def _zip_and_upload(
        self,
        client: demisto_client,
        target_demisto_version: Version,
        skip_validations: bool,
        marketplace: MarketplaceVersions,
        destination_dir: DirectoryPath,
    ) -> bool:
        # this should only be called from Pack.upload
        logger.debug(f"Uploading zipped pack {self.object_id}")

        # 1) dump the pack into a temporary file
        with TemporaryDirectory() as temp_dump_dir:
            temp_dir_path = Path(temp_dump_dir)
            self.dump(temp_dir_path, marketplace=marketplace)

            # 2) zip the dumped pack
            with TemporaryDirectory() as pack_zips_dir:
                pack_zip_path = Path(
                    shutil.make_archive(
                        str(Path(pack_zips_dir, self.name)), "zip", temp_dir_path
                    )
                )
                str(pack_zip_path)

                # 3) zip the zipped pack into uploadable_packs.zip under the result directory
                try:
                    shutil.make_archive(
                        str(destination_dir / MULTIPLE_ZIPPED_PACKS_FILE_STEM),
                        "zip",
                        pack_zips_dir,
                    )
                except Exception:
                    logger.exception(
                        f"Cannot write to {str(destination_dir / MULTIPLE_ZIPPED_PACKS_FILE_NAME)}"
                    )

                # upload the pack zip (not the result)
                return upload_zip(
                    path=pack_zip_path,
                    client=client,
                    target_demisto_version=target_demisto_version,
                    skip_validations=skip_validations,
                    marketplace=marketplace,
                )

    def _upload_item_by_item(
        self,
        client: demisto_client,
        marketplace: MarketplaceVersions,
        target_demisto_version: Version,
    ) -> bool:
        # this should only be called from Pack.upload
        logger.debug(
            f"Uploading pack {self.object_id} element-by-element, as -z was not specified"
        )
        upload_failures: List[FailedUploadException] = []
        uploaded_successfully: List[ContentItem] = []
        incompatible_content_items = []

        for item in self.content_items:
            if item.content_type in CONTENT_TYPES_EXCLUDED_FROM_UPLOAD:
                logger.debug(
                    f"SKIPPING upload of {item.content_type} {item.object_id}: type is skipped"
                )
                continue

            try:
                logger.debug(
                    f"uploading pack {self.object_id}: {item.content_type} {item.object_id}"
                )
                item.upload(
                    client=client,
                    marketplace=marketplace,
                    target_demisto_version=target_demisto_version,
                )
                uploaded_successfully.append(item)
            except NotIndivitudallyUploadableException:
                if marketplace == MarketplaceVersions.MarketplaceV2:
                    raise  # many XSIAM content types must be uploaded zipped.
                logger.warning(
                    f"Not uploading pack {self.object_id}: {item.content_type} {item.object_id} as it was not indivudally uploaded"
                )
            except ApiException as e:
                upload_failures.append(
                    FailedUploadException(
                        item.path,
                        response_body={},
                        additional_info=parse_error_response(e),
                    )
                )
            except IncompatibleUploadVersionException as e:
                incompatible_content_items.append(e)

            except FailedUploadException as e:
                upload_failures.append(e)

        if upload_failures or incompatible_content_items:
            raise FailedUploadMultipleException(
                uploaded_successfully, upload_failures, incompatible_content_items
            )

        return True

    def _copy_base_pack_docs(
        self, destination_path: Path, marketplace: MarketplaceVersions
    ):

        documentation_path = CONTENT_PATH / "Documentation"
        documentation_output = destination_path / "Documentation"
        documentation_output.mkdir(exist_ok=True, parents=True)
        if (
            marketplace.value
            and (documentation_path / f"doc-howto-{marketplace.value}.json").exists()
        ):
            shutil.copy(
                documentation_path / f"doc-howto-{marketplace.value}.json",
                documentation_output / "doc-howto.json",
            )
        elif (documentation_path / "doc-howto-xsoar.json").exists():
            shutil.copy(
                documentation_path / "doc-howto-xsoar.json",
                documentation_output / "doc-howto.json",
            )
        else:
            shutil.copy(
                documentation_path / "doc-howto.json",
                documentation_output / "doc-howto.json",
            )
        if (documentation_path / "doc-CommonServer.json").exists():
            shutil.copy(
                documentation_path / "doc-CommonServer.json",
                documentation_output / "doc-CommonServer.json",
            )

    def to_nodes(self) -> Nodes:
        return Nodes(
            self.to_dict(),
            *[content_item.to_dict() for content_item in self.content_items],
        )


# demisto_sdk/commands/content_graph/objects/parsing_rule.py

from typing import Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item_xsiam import (
    ContentItemXSIAM,
)
from demisto_sdk.commands.prepare_content.rule_unifier import RuleUnifier


class ParsingRule(ContentItemXSIAM, content_type=ContentType.PARSING_RULE):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    def prepare_for_upload(
        self,
        current_marketplace: MarketplaceVersions = MarketplaceVersions.MarketplaceV2,
        **kwargs
    ) -> dict:
        if not kwargs.get("unify_only"):
            data = super().prepare_for_upload(current_marketplace)
        else:
            data = self.data
        data = RuleUnifier.unify(self.path, data, current_marketplace)
        return data


# demisto_sdk/commands/content_graph/objects/playbook.py

from typing import Callable, Set

import demisto_client

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem
from demisto_sdk.commands.prepare_content.preparers.marketplace_incident_to_alert_playbooks_prepare import (
    MarketplaceIncidentToAlertPlaybooksPreparer,
)


class Playbook(ContentItem, content_type=ContentType.PLAYBOOK):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    def prepare_for_upload(
        self,
        current_marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
        **kwargs
    ) -> dict:
        data = super().prepare_for_upload(current_marketplace, **kwargs)
        return MarketplaceIncidentToAlertPlaybooksPreparer.prepare(
            self,
            data,
            current_marketplace=current_marketplace,
            supported_marketplaces=self.marketplaces,
        )

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        return client.import_playbook


# demisto_sdk/commands/content_graph/objects/pre_process_rule.py

from typing import Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem
from demisto_sdk.commands.upload.exceptions import NotIndivitudallyUploadableException


class PreProcessRule(ContentItem, content_type=ContentType.PREPROCESS_RULE):
    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    def _upload(self, client, marketplace: MarketplaceVersions) -> None:
        raise NotIndivitudallyUploadableException(self)


# demisto_sdk/commands/content_graph/objects/relationship.py

from typing import Optional

from pydantic import BaseModel

from demisto_sdk.commands.content_graph.common import RelationshipType
from demisto_sdk.commands.content_graph.objects.base_content import BaseContent


class RelationshipData(BaseModel):
    relationship_type: RelationshipType

    # These are the database ids of the relationships
    source_id: str
    target_id: str

    # this is the attribute we're interested in when querying
    content_item_to: BaseContent

    is_direct: bool = True

    # USES, DEPENDS_ON relationship properties
    mandatorily: bool = False

    # DEPENDS_ON relationship properties
    is_test: bool = False

    # HAS_COMMAND relationship properties
    description: Optional[str] = None
    deprecated: bool = False

    def __hash__(self):
        """This is the unique identifier of the relationship"""

        return hash(
            (
                self.source_id,
                self.target_id,
                self.relationship_type,
            )
        )

    def __eq__(self, __o: object) -> bool:
        """This is needed to check if the relationship already exists"""
        return hash(self) == hash(__o)


# demisto_sdk/commands/content_graph/objects/report.py

from typing import Callable, Set

import demisto_client

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class Report(ContentItem, content_type=ContentType.REPORT):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        return client.upload_report


# demisto_sdk/commands/content_graph/objects/repository.py

import shutil
import time
from multiprocessing.pool import Pool
from pathlib import Path
from typing import List

from pydantic import BaseModel, DirectoryPath

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH
from demisto_sdk.commands.common.cpu_count import cpu_count
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.content_graph.objects.pack import Pack
from demisto_sdk.commands.content_graph.parsers.repository import RepositoryParser

USE_MULTIPROCESSING = False  # toggle this for better debugging


def all_content_repo():
    """
    Returns a ContentDTO object with all the packs of the content repository.
    """
    repo_parser = RepositoryParser(CONTENT_PATH)
    repo_parser.parse()
    return ContentDTO.from_orm(repo_parser)


class ContentDTO(BaseModel):
    path: DirectoryPath = Path(CONTENT_PATH)  # type: ignore
    packs: List[Pack]

    def dump(
        self,
        dir: DirectoryPath,
        marketplace: MarketplaceVersions,
        zip: bool = True,
        output_stem: str = "content_packs",  # without extension
    ):
        dir.mkdir(parents=True, exist_ok=True)
        logger.debug("Starting repository dump")
        start_time = time.time()
        if USE_MULTIPROCESSING:
            with Pool(processes=cpu_count()) as pool:
                pool.starmap(
                    Pack.dump,
                    ((pack, dir / pack.path.name, marketplace) for pack in self.packs),
                )

        else:
            for pack in self.packs:
                pack.dump(dir / pack.path.name, marketplace)

        time_taken = time.time() - start_time
        logger.debug(f"Repository dump ended. Took {time_taken} seconds")

        if zip:
            shutil.make_archive(str(dir.parent / output_stem), "zip", dir)
            shutil.rmtree(dir)

    class Config:
        orm_mode = True
        allow_population_by_field_name = True


# demisto_sdk/commands/content_graph/objects/script.py

from typing import Callable, List, Set

import demisto_client
from pydantic import DirectoryPath

from demisto_sdk.commands.common.constants import (
    SKIP_PREPARE_SCRIPT_NAME,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.objects.base_content import BaseContent
from demisto_sdk.commands.content_graph.objects.integration_script import (
    IntegrationScript,
)
from demisto_sdk.commands.prepare_content.preparers.marketplace_incident_to_alert_scripts_prepare import (
    MarketplaceIncidentToAlertScriptsPreparer,
)


class Script(IntegrationScript, content_type=ContentType.SCRIPT):  # type: ignore[call-arg]
    tags: List[str]
    skip_prepare: List[str]

    def metadata_fields(self) -> Set[str]:
        return {"name", "description", "tags"}

    def prepare_for_upload(
        self,
        current_marketplace: MarketplaceVersions = MarketplaceVersions.XSOAR,
        **kwargs,
    ) -> dict:
        data = super().prepare_for_upload(current_marketplace, **kwargs)

        if supported_native_images := self.get_supported_native_images(
            marketplace=current_marketplace,
            ignore_native_image=kwargs.get("ignore_native_image") or False,
        ):
            logger.debug(
                f"Adding the following native images {supported_native_images} to script {self.object_id}"
            )
            data["nativeimage"] = supported_native_images

        return data

    @property
    def imported_by(self) -> List[BaseContent]:
        return [
            r.content_item_to
            for r in self.relationships_data[RelationshipType.IMPORTS]
            if r.content_item_to.database_id == r.source_id
        ]

    def dump(self, dir: DirectoryPath, marketplace: MarketplaceVersions) -> None:
        dir.mkdir(exist_ok=True, parents=True)
        data = self.prepare_for_upload(current_marketplace=marketplace)

        for data in MarketplaceIncidentToAlertScriptsPreparer.prepare(
            data, marketplace, self.is_incident_to_alert(marketplace)
        ):
            # Two scripts return from the preparation, one the original, and other the new script,
            # in order to normalize the name of the new script, make a copy of the original object
            # in case it is a new script with an update of the name and path.
            script_name = data.get("name")

            if script_name == self.name:  # the original script
                obj = self

            else:  # a modified script, replaced incidents->alerts
                obj = self.copy(
                    update={
                        "name": script_name,
                        "path": self.path.with_name(f"{script_name}.yml"),
                    }
                )
            try:
                with (dir / obj.normalize_name).open("w") as f:
                    obj.handler.dump(data, f)
            except FileNotFoundError as e:
                logger.warning(f"Failed to dump {obj.path} to {dir}: {e}")

    def is_incident_to_alert(self, marketplace: MarketplaceVersions) -> bool:
        """
        Checks whether the script needs the preparation
        of an `incident to alert`,
        and this affects the `metadata.json` and the `dump` process of the script.

        Args:
            marketplace (MarketplaceVersions): the destination marketplace.

        Returns:
            bool: True if all conditions are true otherwise False
        """
        return all(
            (
                marketplace == MarketplaceVersions.MarketplaceV2,
                "incident" in self.name.lower(),
                SKIP_PREPARE_SCRIPT_NAME not in self.skip_prepare,
                not self.deprecated,
            )
        )

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        return client.import_script


# demisto_sdk/commands/content_graph/objects/test_playbook.py

from typing import Set

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.playbook import Playbook


class TestPlaybook(Playbook, content_type=ContentType.TEST_PLAYBOOK):  # type: ignore[call-arg]
    is_test: bool = True

    def metadata_fields(self) -> Set[str]:
        raise NotImplementedError("TestPlaybooks not included in metadata")


# demisto_sdk/commands/content_graph/objects/trigger.py

from typing import Set

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item_xsiam import (
    ContentItemXSIAM,
)


class Trigger(ContentItemXSIAM, content_type=ContentType.TRIGGER):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}


# demisto_sdk/commands/content_graph/objects/widget.py

from typing import Callable, Optional, Set

import demisto_client
from pydantic import Field

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class Widget(ContentItem, content_type=ContentType.WIDGET):  # type: ignore[call-arg]
    widget_type: str = Field(alias="widgetType")
    data_type: Optional[str] = Field(alias="dataType")

    def metadata_fields(self) -> Set[str]:
        return {"name", "data_type", "widget_type"}

    @classmethod
    def _client_upload_method(cls, client: demisto_client) -> Callable:
        return client.import_widget


# demisto_sdk/commands/content_graph/objects/wizard.py

from typing import List, Set

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem


class Wizard(ContentItem, content_type=ContentType.WIZARD):  # type: ignore[call-arg]
    packs: List[str]
    integrations: List[str]
    playbooks: List[str]

    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}


# demisto_sdk/commands/content_graph/objects/xdrc_template.py

from typing import Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item_xsiam import (
    ContentItemXSIAM,
)
from demisto_sdk.commands.prepare_content.xdrc_template_unifier import (
    XDRCTemplateUnifier,
)


class XDRCTemplate(ContentItemXSIAM, content_type=ContentType.XDRC_TEMPLATE):
    content_global_id: str
    os_type: str
    profile_type: str

    def metadata_fields(self) -> Set[str]:
        return {"name", "os_type", "profile_type"}

    def prepare_for_upload(
        self,
        current_marketplace: MarketplaceVersions = MarketplaceVersions.MarketplaceV2,
        **kwargs
    ) -> dict:
        data = super().prepare_for_upload(current_marketplace)
        data = XDRCTemplateUnifier.unify(self.path, data, current_marketplace)
        return data


# demisto_sdk/commands/content_graph/objects/xsiam_dashboard.py

import shutil
from typing import Set

from pydantic import DirectoryPath

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item_xsiam import (
    ContentItemXSIAM,
)


class XSIAMDashboard(ContentItemXSIAM, content_type=ContentType.XSIAM_DASHBOARD):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}

    def dump(
        self,
        dir: DirectoryPath,
        marketplace: MarketplaceVersions,
    ) -> None:
        super().dump(dir, marketplace)
        if (self.path.parent / f"{self.path.stem}_image.png").exists():
            shutil.copy(
                self.path.parent / f"{self.path.stem}_image.png",
                dir / f"{self.path.stem}_image.png",
            )


# demisto_sdk/commands/content_graph/objects/xsiam_report.py

from typing import Set

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.objects.content_item_xsiam import (
    ContentItemXSIAM,
)


class XSIAMReport(ContentItemXSIAM, content_type=ContentType.XSIAM_REPORT):  # type: ignore[call-arg]
    def metadata_fields(self) -> Set[str]:
        return {"name", "description"}


# demisto_sdk/commands/content_graph/parsers/__init__.py

__all__ = [
    "ClassifierParser",
    "CorrelationRuleParser",
    "DashboardParser",
    "GenericDefinitionParser",
    "GenericFieldParser",
    "GenericModuleParser",
    "GenericTypeParser",
    "GenericTypeParser",
    "IncidentFieldParser",
    "IncidentTypeParser",
    "IndicatorFieldParser",
    "IndicatorTypeParser",
    "IntegrationParser",
    "JobParser",
    "LayoutParser",
    "ListParser",
    "MapperParser",
    "ModelingRuleParser",
    "ParsingRuleParser",
    "PlaybookParser",
    "ReportParser",
    "ScriptParser",
    "TestPlaybookParser",
    "TriggerParser",
    "WidgetParser",
    "WizardParser",
    "XSIAMDashboardParser",
    "XSIAMReportParser",
    "XDRCTemplateParser",
    "LayoutRuleParser",
    "PreProcessRuleParser",
]

from demisto_sdk.commands.content_graph.parsers.classifier import ClassifierParser
from demisto_sdk.commands.content_graph.parsers.correlation_rule import (
    CorrelationRuleParser,
)
from demisto_sdk.commands.content_graph.parsers.dashboard import DashboardParser
from demisto_sdk.commands.content_graph.parsers.generic_definition import (
    GenericDefinitionParser,
)
from demisto_sdk.commands.content_graph.parsers.generic_field import GenericFieldParser
from demisto_sdk.commands.content_graph.parsers.generic_module import (
    GenericModuleParser,
)
from demisto_sdk.commands.content_graph.parsers.generic_type import GenericTypeParser
from demisto_sdk.commands.content_graph.parsers.incident_field import (
    IncidentFieldParser,
)
from demisto_sdk.commands.content_graph.parsers.incident_type import IncidentTypeParser
from demisto_sdk.commands.content_graph.parsers.indicator_field import (
    IndicatorFieldParser,
)
from demisto_sdk.commands.content_graph.parsers.indicator_type import (
    IndicatorTypeParser,
)
from demisto_sdk.commands.content_graph.parsers.integration import IntegrationParser
from demisto_sdk.commands.content_graph.parsers.job import JobParser
from demisto_sdk.commands.content_graph.parsers.layout import LayoutParser
from demisto_sdk.commands.content_graph.parsers.layout_rule import LayoutRuleParser
from demisto_sdk.commands.content_graph.parsers.list import ListParser
from demisto_sdk.commands.content_graph.parsers.mapper import MapperParser
from demisto_sdk.commands.content_graph.parsers.modeling_rule import ModelingRuleParser
from demisto_sdk.commands.content_graph.parsers.parsing_rule import ParsingRuleParser
from demisto_sdk.commands.content_graph.parsers.playbook import PlaybookParser
from demisto_sdk.commands.content_graph.parsers.pre_process_rule import (
    PreProcessRuleParser,
)
from demisto_sdk.commands.content_graph.parsers.report import ReportParser
from demisto_sdk.commands.content_graph.parsers.script import ScriptParser
from demisto_sdk.commands.content_graph.parsers.test_playbook import TestPlaybookParser
from demisto_sdk.commands.content_graph.parsers.trigger import TriggerParser
from demisto_sdk.commands.content_graph.parsers.widget import WidgetParser
from demisto_sdk.commands.content_graph.parsers.wizard import WizardParser
from demisto_sdk.commands.content_graph.parsers.xdrc_template import XDRCTemplateParser
from demisto_sdk.commands.content_graph.parsers.xsiam_dashboard import (
    XSIAMDashboardParser,
)
from demisto_sdk.commands.content_graph.parsers.xsiam_report import XSIAMReportParser


# demisto_sdk/commands/content_graph/parsers/base_content.py

from abc import ABC, abstractmethod
from pathlib import Path
from typing import Optional

from demisto_sdk.commands.content_graph.common import ContentType


class BaseContentParser(ABC):
    """An abstract class for all content types.

    Attributes:
        object_id (str): The content object ID.
        content_type (ContentType): The content object type.
        node_id (str): The content object node ID.
    """

    content_type: ContentType

    def __init__(self, path: Path) -> None:
        self.path: Path = path

    @property
    @abstractmethod
    def object_id(self) -> Optional[str]:
        pass

    @property
    def node_id(self) -> str:
        return f"{self.content_type}:{self.object_id}"


# demisto_sdk/commands/content_graph/parsers/classifier.py

from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.content_item import (
    IncorrectParserException,
)
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)
from demisto_sdk.commands.content_graph.parsers.mapper import MapperParser


class ClassifierParser(JSONContentItemParser, content_type=ContentType.CLASSIFIER):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        """Parses the classifier.

        Args:
            path (Path): The classifier's path.

        Raises:
            IncorrectParserException: When detecting this content item is a mapper.
        """
        super().__init__(path, pack_marketplaces)

        self.type = self.json_data.get("type")
        if self.type != "classification":
            raise IncorrectParserException(correct_parser=MapperParser)

        self.definition_id = self.json_data.get("definitionId")
        self.connect_to_dependencies()

    @property
    def name(self) -> Optional[str]:
        return self.json_data.get("name") or self.json_data.get("brandName")

    def get_filters_and_transformers_from_complex_value(
        self, complex_value: dict
    ) -> None:
        for filter in complex_value.get("filters", []):
            if filter:
                filter_script = filter[0].get("operator").split(".")[-1]
                self.add_dependency_by_id(filter_script, ContentType.SCRIPT)

        for transformer in complex_value.get("transformers", []):
            if transformer:
                transformer_script = transformer.get("operator").split(".")[-1]
                self.add_dependency_by_id(transformer_script, ContentType.SCRIPT)

    def connect_to_dependencies(self) -> None:
        """Collects the incident types, filters and transformers used in the classifier as required dependencies."""
        if self.json_data.get("feed"):
            content_type_to_map = ContentType.INDICATOR_TYPE
        else:
            content_type_to_map = ContentType.INCIDENT_TYPE

        if default_incident_type := self.json_data.get("defaultIncidentType"):
            self.add_dependency_by_id(
                default_incident_type, content_type_to_map, is_mandatory=False
            )

        for incident_type in self.json_data.get("keyTypeMap", {}).values():
            self.add_dependency_by_id(
                incident_type, content_type_to_map, is_mandatory=False
            )

        if transformer_complex_value := self.json_data.get("transformer", {}).get(
            "complex", {}
        ):
            self.get_filters_and_transformers_from_complex_value(
                transformer_complex_value
            )

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2}


# demisto_sdk/commands/content_graph/parsers/content_item.py

from abc import ABCMeta, abstractmethod
from pathlib import Path
from typing import Dict, List, Optional, Set, Type, cast

from packaging.version import Version

from demisto_sdk.commands.common.constants import (
    MARKETPLACE_MIN_VERSION,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.content_graph.common import (
    UNIFIED_FILES_SUFFIXES,
    ContentType,
    Relationships,
    RelationshipType,
)
from demisto_sdk.commands.content_graph.parsers.base_content import BaseContentParser


class NotAContentItemException(Exception):
    pass


class InvalidContentItemException(Exception):
    pass


class IncorrectParserException(Exception):
    def __init__(self, correct_parser: Type["ContentItemParser"], **kwargs) -> None:
        self.correct_parser = correct_parser
        self.kwargs = kwargs
        super().__init__()


class ParserMetaclass(ABCMeta):
    def __new__(
        cls, name, bases, namespace, content_type: ContentType = None, **kwargs
    ):
        """This method is called before every creation of a ContentItemParser *class* (NOT class instances!).
        If `content_type` is passed as an argument of the class, we add a mapping between the content type
        and the parser class object.
        After all the parser classes are created, `content_type_to_parser` has a full mapping between content types
        and parsers, and only then we are ready to determine which parser class to use based on a content item's type.

        Args:
            name: The class object name (e.g., IntegrationParser)
            bases: The bases of the class object (e.g., [YAMLContentItemParser, ContentItemParser, BaseContentParser])
            namespace: The namespaces of the class object.
            content_type (ContentType, optional): The type corresponds to the class (e.g., ContentType.INTEGRATIONS)

        Returns:
            ContentItemParser: The parser class.
        """
        super_cls: ParserMetaclass = super().__new__(cls, name, bases, namespace)
        # for type checking
        parser_cls: Type["ContentItemParser"] = cast(
            Type["ContentItemParser"], super_cls
        )
        if content_type:
            ContentItemParser.content_type_to_parser[content_type] = parser_cls
            parser_cls.content_type = content_type
        return parser_cls


class ContentItemParser(BaseContentParser, metaclass=ParserMetaclass):
    """A content item parser.

    Static Attributes:
        content_type_to_parser (Dict[ContentType, Type[ContentItemParser]]):
            A mapping between content types and parsers.
    Attributes:
        relationships (Relationships): The relationships collections of the content item.
    """

    content_type_to_parser: Dict[ContentType, Type["ContentItemParser"]] = {}

    def __init__(
        self,
        path: Path,
        pack_marketplaces: List[MarketplaceVersions] = list(MarketplaceVersions),
    ) -> None:
        self.pack_marketplaces: List[MarketplaceVersions] = pack_marketplaces
        super().__init__(path)
        self.relationships: Relationships = Relationships()

    @staticmethod
    def from_path(
        path: Path,
        pack_marketplaces: List[MarketplaceVersions] = list(MarketplaceVersions),
    ) -> "ContentItemParser":
        """Tries to parse a content item by its path.
        If during the attempt we detected the file is not a content item, `None` is returned.

        Returns:
            Optional[ContentItemParser]: The parsed content item.
        """
        logger.debug(f"Parsing content item {path}")
        if not ContentItemParser.is_content_item(path):
            if ContentItemParser.is_content_item(path.parent):
                path = path.parent
            else:
                raise NotAContentItemException
        try:
            content_type: ContentType = ContentType.by_path(path)
        except ValueError as e:
            logger.error(f"Could not determine content type for {path}: {e}")
            raise InvalidContentItemException from e
        if parser_cls := ContentItemParser.content_type_to_parser.get(content_type):
            try:
                return ContentItemParser.parse(
                    parser_cls,
                    path,
                    pack_marketplaces,
                )
            except IncorrectParserException as e:
                return ContentItemParser.parse(
                    e.correct_parser, path, pack_marketplaces, **e.kwargs
                )
            except NotAContentItemException:
                logger.debug(f"{path} is not a content item, skipping")
                raise
            except Exception as e:
                logger.error(f"Failed to parse {path}: {e}")
                raise InvalidContentItemException from e
        logger.warning(f"Could not find parser for {content_type} of {path}")
        raise NotAContentItemException

    @staticmethod
    def parse(
        parser_cls: Type["ContentItemParser"],
        path: Path,
        pack_marketplaces: List[MarketplaceVersions],
        **kwargs,
    ) -> "ContentItemParser":
        parser = parser_cls(path, pack_marketplaces, **kwargs)
        logger.debug(f"Parsed {parser.node_id}")
        return parser

    @property
    @abstractmethod
    def name(self) -> Optional[str]:
        pass

    @property
    @abstractmethod
    def display_name(self) -> Optional[str]:
        pass

    @property
    @abstractmethod
    def deprecated(self) -> bool:
        pass

    @property
    @abstractmethod
    def description(self) -> Optional[str]:
        pass

    @property
    @abstractmethod
    def marketplaces(self) -> List[MarketplaceVersions]:
        pass

    @property
    @abstractmethod
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        pass

    @property
    @abstractmethod
    def fromversion(self) -> str:
        pass

    @property
    @abstractmethod
    def toversion(self) -> str:
        pass

    @staticmethod
    def is_package(path: Path) -> bool:
        return path.is_dir() and path.parent.name in ContentType.folders()

    @staticmethod
    def is_unified_file(path: Path) -> bool:
        if path.suffix in UNIFIED_FILES_SUFFIXES:
            if path.parent.name in ContentType.folders():
                return path.parent.parent.name not in ContentType.folders()
            if path.parent.parent.name in ContentType.folders():
                return (
                    ContentType.by_path(path) in ContentType.threat_intel_report_types()
                )
        return False

    @staticmethod
    def is_content_item(path: Path) -> bool:
        """Determines whether a file path is of a content item, by one of the following conditions:
        1. If this is a directory
        2. If it's a unified file

        Args:
            path (Path): The file path

        Returns:
            bool: True iff the file path is of a content item.
        """
        return ContentItemParser.is_package(path) or ContentItemParser.is_unified_file(
            path
        )

    def should_skip_parsing(self) -> bool:
        """Returns true if any of the minimal conditions for parsing is not met.

        Returns:
            bool: Whether or not this content item should be parsed.
        """
        return not all(
            [
                self.is_above_marketplace_min_version(),
            ]
        )

    def is_above_marketplace_min_version(self) -> bool:
        return Version(self.toversion) >= Version(MARKETPLACE_MIN_VERSION)

    def add_relationship(
        self,
        relationship: RelationshipType,
        target: str,
        target_type: ContentType,
        **kwargs,
    ) -> None:
        """Adds a single relationship to the collection of the content item relationships.

        Args:
            relationship (Relationship): The relationship type.
            target (str): The identifier of the target content object (e.g, its node_id).
            kwargs: Additional information about the relationship.
        """
        self.relationships.add(
            relationship,
            source_id=self.object_id,
            source_type=self.content_type,
            source_fromversion=self.fromversion,
            source_marketplaces=self.marketplaces,
            target=target,
            target_type=target_type,
            **kwargs,
        )

    def add_to_pack(self, pack_id: Optional[str]) -> None:
        """Creates an IN_PACK relationship between the content item and its pack.

        Args:
            pack_id (Optional[str]): The pack id.
        """
        if not pack_id:
            raise ValueError(f"{self.node_id}: pack ID must have a value.")
        self.add_relationship(RelationshipType.IN_PACK, pack_id, ContentType.PACK)

    def add_dependency_by_id(
        self,
        dependency_id: str,
        dependency_type: ContentType,
        is_mandatory: bool = True,
    ) -> None:
        """Creates a USES_BY_ID relationship between the content item and a given dependency.

        Args:
            dependency_id (str): The dependency id.
            dependency_type (ContentType): The dependency content type.
            is_mandatory (bool, optional): Whether or not the dependency is mandatory. Defaults to True.
        """
        self.add_relationship(
            RelationshipType.USES_BY_ID,
            target=dependency_id,
            target_type=dependency_type,
            mandatorily=is_mandatory,
        )

    def add_dependency_by_name(
        self,
        dependency_name: str,
        dependency_type: ContentType,
        is_mandatory: bool = True,
    ) -> None:
        """Creates a USES_BY_NAME relationship between the content item and a given dependency.

        Args:
            dependency_name (str): The dependency name.
            dependency_type (ContentType): The dependency content type.
            is_mandatory (bool, optional): Whether or not the dependency is mandatory. Defaults to True.
        """
        self.add_relationship(
            RelationshipType.USES_BY_NAME,
            target=dependency_name,
            target_type=dependency_type,
            mandatorily=is_mandatory,
        )

    def add_command_or_script_dependency(
        self, dependency_id: str, is_mandatory: bool = True
    ) -> None:
        """Creates a USES_COMMAND_OR_SCRIPT relationship between the content item and a given dependency.

        Args:
            dependency_id (str): The dependency id.
            is_mandatory (bool, optional): Whether or not the dependency is mandatory. Defaults to True.
        """
        self.add_relationship(
            RelationshipType.USES_COMMAND_OR_SCRIPT,
            target=dependency_id,
            target_type=ContentType.COMMAND_OR_SCRIPT,
            mandatorily=is_mandatory,
        )


# demisto_sdk/commands/content_graph/parsers/content_items_list.py

from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.content_item import ContentItemParser


class ContentItemsList(list):
    """An extension for list - a list of a specific content type.

    Attributes:
        content_type (ContentType): The content types allowed to be included in this list.
    """

    def __init__(self, content_type: ContentType):
        self.content_type: ContentType = content_type
        super().__init__()

    def __eq__(self, __o: object) -> bool:
        return super().__eq__(__o)

    def append(self, content_item: ContentItemParser) -> None:
        """Appends if the content item is in the correct type.

        Args:
            content_item (ContentItemParser): The content item.
        """
        if content_item.content_type != self.content_type:
            raise TypeError(
                f"{content_item.node_id}: Expected a ContentItemParser of type {self.content_type}"
            )
        super().append(content_item)


# demisto_sdk/commands/content_graph/parsers/correlation_rule.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.yaml_content_item import (
    YAMLContentItemParser,
)


class CorrelationRuleParser(
    YAMLContentItemParser, content_type=ContentType.CORRELATION_RULE
):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)

    @property
    def object_id(self) -> str:
        return self.yml_data["global_rule_id"]

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.MarketplaceV2}


# demisto_sdk/commands/content_graph/parsers/dashboard.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class DashboardParser(JSONContentItemParser, content_type=ContentType.DASHBOARD):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)

        self.connect_to_dependencies()

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR}

    def connect_to_dependencies(self) -> None:
        """Collects the scripts used in the dashboard as optional dependencies."""
        for layout in self.json_data.get("layout", []):
            widget_data = layout.get("widget")
            if widget_data.get("dataType") == "scripts":
                if script_name := widget_data.get("query"):
                    self.add_dependency_by_id(
                        script_name, ContentType.SCRIPT, is_mandatory=False
                    )


# demisto_sdk/commands/content_graph/parsers/generic_definition.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class GenericDefinitionParser(
    JSONContentItemParser, content_type=ContentType.GENERIC_DEFINITION
):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR}


# demisto_sdk/commands/content_graph/parsers/generic_field.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class GenericFieldParser(JSONContentItemParser, content_type=ContentType.GENERIC_FIELD):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.definition_id = self.json_data.get("definitionId")

        self.connect_to_dependencies()

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR}

    def connect_to_dependencies(self) -> None:
        """Collects the generic types associated to the generic field as optional dependencies."""
        for associated_type in set(self.json_data.get("associatedTypes") or []):
            self.add_dependency_by_name(
                associated_type, ContentType.GENERIC_TYPE, is_mandatory=False
            )

        for system_associated_type in set(
            self.json_data.get("systemAssociatedTypes") or []
        ):
            self.add_dependency_by_name(
                system_associated_type, ContentType.GENERIC_TYPE, is_mandatory=False
            )


# demisto_sdk/commands/content_graph/parsers/generic_module.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class GenericModuleParser(
    JSONContentItemParser, content_type=ContentType.GENERIC_MODULE
):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.definition_ids = self.json_data.get("definitionIds")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR}


# demisto_sdk/commands/content_graph/parsers/generic_type.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class GenericTypeParser(JSONContentItemParser, content_type=ContentType.GENERIC_TYPE):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.definition_id = self.json_data.get("definitionId")

        self.connect_to_dependencies()

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR}

    def connect_to_dependencies(self) -> None:
        """Collects the layouts used in the generic type as mandatory dependencies."""
        if layout := self.json_data.get("layout"):
            self.add_dependency_by_id(layout, ContentType.LAYOUT)


# demisto_sdk/commands/content_graph/parsers/incident_field.py

from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class IncidentFieldParser(
    JSONContentItemParser, content_type=ContentType.INCIDENT_FIELD
):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.cli_name = self.json_data.get("cliName")
        self.field_type = self.json_data.get("type")
        self.associated_to_all = self.json_data.get("associatedToAll")

        self.connect_to_dependencies()

    @property
    def object_id(self) -> Optional[str]:
        return self.json_data.get("cliName")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {
            MarketplaceVersions.XSOAR,
            MarketplaceVersions.MarketplaceV2,
            MarketplaceVersions.XPANSE,
        }

    def connect_to_dependencies(self) -> None:
        """Collects incident types used as optional dependencies, and scripts as mandatory dependencies."""
        for associated_type in set(self.json_data.get("associatedTypes") or []):
            if associated_type:
                self.add_dependency_by_name(
                    associated_type, ContentType.INCIDENT_TYPE, is_mandatory=False
                )

        for system_associated_type in set(
            self.json_data.get("systemAssociatedTypes") or []
        ):
            if system_associated_type:
                self.add_dependency_by_name(
                    system_associated_type,
                    ContentType.INCIDENT_TYPE,
                    is_mandatory=False,
                )

        if script := self.json_data.get("script"):
            self.add_dependency_by_id(script, ContentType.SCRIPT)

        if field_calc_script := self.json_data.get("fieldCalcScript"):
            self.add_dependency_by_id(field_calc_script, ContentType.SCRIPT)


# demisto_sdk/commands/content_graph/parsers/incident_type.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class IncidentTypeParser(JSONContentItemParser, content_type=ContentType.INCIDENT_TYPE):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.playbook = self.json_data.get("playbookId")
        self.hours = self.json_data.get("hours")
        self.days = self.json_data.get("days")
        self.weeks = self.json_data.get("weeks")
        self.closure_script = self.json_data.get("closureScript") or None

        self.connect_to_dependencies()

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {
            MarketplaceVersions.XSOAR,
            MarketplaceVersions.MarketplaceV2,
            MarketplaceVersions.XPANSE,
        }

    def connect_to_dependencies(self) -> None:
        """Collects the script, playbook and layout used by the incident type as mandatory dependencies."""
        if pre_processing_script := self.json_data.get("preProcessingScript"):
            self.add_dependency_by_id(pre_processing_script, ContentType.SCRIPT)

        if playbook := self.json_data.get("playbookId"):
            self.add_dependency_by_id(playbook, ContentType.PLAYBOOK)

        if layout := self.json_data.get("layout"):
            self.add_dependency_by_id(layout, ContentType.LAYOUT, is_mandatory=False)


# demisto_sdk/commands/content_graph/parsers/indicator_field.py

from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class IndicatorFieldParser(
    JSONContentItemParser, content_type=ContentType.INDICATOR_FIELD
):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.cli_name = self.json_data.get("cliName")
        self.type = self.json_data.get("type")
        self.associated_to_all = self.json_data.get("associatedToAll")

        self.connect_to_dependencies()

    @property
    def object_id(self) -> Optional[str]:
        return self.json_data.get("cliName")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2}

    def connect_to_dependencies(self) -> None:
        """Collects indicator types used by the field as optional dependencies, and scripts as mandatory dependencies."""
        for associated_type in set(self.json_data.get("associatedTypes") or []):
            if associated_type:
                self.add_dependency_by_name(
                    associated_type, ContentType.INDICATOR_TYPE, is_mandatory=False
                )

        for system_associated_type in set(
            self.json_data.get("systemAssociatedTypes") or []
        ):
            if system_associated_type:
                self.add_dependency_by_name(
                    system_associated_type,
                    ContentType.INDICATOR_TYPE,
                    is_mandatory=False,
                )

        if script := self.json_data.get("script"):
            self.add_dependency_by_id(script, ContentType.SCRIPT)

        if field_calc_script := self.json_data.get("fieldCalcScript"):
            self.add_dependency_by_id(field_calc_script, ContentType.SCRIPT)


# demisto_sdk/commands/content_graph/parsers/indicator_type.py

from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class IndicatorTypeParser(
    JSONContentItemParser, content_type=ContentType.INDICATOR_TYPE
):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.connect_to_dependencies()
        self.regex = self.json_data.get("regex")
        self.reputation_script_name = self.json_data.get("reputationScriptName") or None
        self.enhancement_script_names = self.json_data.get("enhancementScriptNames")

    @property
    def name(self) -> Optional[str]:
        return self.json_data.get("details")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2}

    @property
    def description(self) -> Optional[str]:
        return self.json_data.get("details")

    def connect_to_dependencies(self) -> None:
        """Collects scripts and the reputation command used as optional dependencies,
        and the layout as a mandatory dependency.
        """
        for field in ["reputationScriptName", "enhancementScriptNames"]:
            associated_scripts = self.json_data.get(field)
            if associated_scripts and associated_scripts != "null":
                associated_scripts = (
                    [associated_scripts]
                    if not isinstance(associated_scripts, list)
                    else associated_scripts
                )
                for script in associated_scripts:
                    self.add_dependency_by_id(
                        script, ContentType.SCRIPT, is_mandatory=False
                    )

        if reputation_command := self.json_data.get("reputationCommand"):
            self.add_dependency_by_id(
                reputation_command, ContentType.COMMAND, is_mandatory=False
            )

        if layout := self.json_data.get("layout"):
            self.add_dependency_by_id(layout, ContentType.LAYOUT, is_mandatory=False)


# demisto_sdk/commands/content_graph/parsers/integration.py

from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, List, Optional

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.parsers.integration_script import (
    IntegrationScriptParser,
)
from demisto_sdk.commands.prepare_content.integration_script_unifier import (
    IntegrationScriptUnifier,
)


@dataclass
class CommandParser:
    name: str
    deprecated: bool
    description: str


class IntegrationParser(IntegrationScriptParser, content_type=ContentType.INTEGRATION):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.script_info: Dict[str, Any] = self.yml_data.get("script", {})
        self.category = self.yml_data["category"]
        self.is_fetch = self.script_info.get("isfetch", False)
        self.is_fetch_events_and_assets = self.script_info.get(
            "isfetcheventsandassets", False
        )
        # if the integration fetches assets, it fetches events as well
        self.is_fetch_events = (
            self.script_info.get("isfetchevents", False)
            or self.is_fetch_events_and_assets
        )
        self.is_feed = self.script_info.get("feed", False)
        self.type = self.script_info.get("subtype") or self.script_info.get("type")
        if self.type == "python":
            self.type += "2"
        self.commands: List[CommandParser] = []
        self.connect_to_commands()
        self.connect_to_dependencies()
        self.connect_to_tests()

    @property
    def display_name(self) -> Optional[str]:
        return self.yml_data.get("display")

    @property
    def docker_image(self) -> str:
        return self.script_info.get("dockerimage", "")

    def connect_to_commands(self) -> None:
        """Creates HAS_COMMAND relationships with the integration commands.
        Command's properties are stored in the relationship's data,
        since there will be a single node for all commands with the same name.
        """
        for command_data in self.script_info.get("commands", []):
            name = command_data.get("name")
            deprecated = command_data.get("deprecated", False) or self.deprecated
            description = command_data.get("description")
            self.add_relationship(
                RelationshipType.HAS_COMMAND,
                target=name,
                target_type=ContentType.COMMAND,
                name=name,
                deprecated=deprecated,
                description=description,
            )
            self.commands.append(
                CommandParser(name=name, description=description, deprecated=deprecated)
            )

    def connect_to_dependencies(self) -> None:
        """Collects the default classifier, mappers and incident type used as mandatory dependencies."""
        if default_classifier := self.yml_data.get("defaultclassifier"):
            if default_classifier != "null":
                self.add_dependency_by_id(
                    default_classifier, ContentType.CLASSIFIER, is_mandatory=False
                )

        if default_mapper_in := self.yml_data.get("defaultmapperin"):
            if default_mapper_in != "null":
                self.add_dependency_by_id(
                    default_mapper_in, ContentType.MAPPER, is_mandatory=False
                )

        if default_mapper_out := self.yml_data.get("defaultmapperout"):
            if default_mapper_out != "null":
                self.add_dependency_by_id(
                    default_mapper_out, ContentType.MAPPER, is_mandatory=False
                )

        if default_incident_type := self.yml_data.get("defaultIncidentType"):
            if default_incident_type != "null":
                self.add_dependency_by_id(
                    default_incident_type, ContentType.INCIDENT_TYPE, is_mandatory=False
                )

    @property
    def code(self) -> Optional[str]:
        """Gets the integration code.
        If the integration is unified, then it is taken from the yml file.
        Otherwise, uses the Unifier object to get it.

        Returns:
            str: The integration code.
        """
        if self.is_unified or self.script_info.get("script") not in ("-", "", None):
            return self.script_info.get("script")
        return IntegrationScriptUnifier.get_script_or_integration_package_data(
            self.path.parent
        )[1]


# demisto_sdk/commands/content_graph/parsers/integration_script.py

from abc import abstractmethod
from pathlib import Path
from typing import Any, Dict, List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.parsers.yaml_content_item import (
    YAMLContentItemParser,
)
from demisto_sdk.commands.prepare_content.integration_script_unifier import (
    IntegrationScriptUnifier,
)


class IntegrationScriptParser(YAMLContentItemParser):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        self.is_unified = YAMLContentItemParser.is_unified_file(path)
        super().__init__(path, pack_marketplaces)
        self.script_info: Dict[str, Any] = self.yml_data.get("script", {})
        self.connect_to_api_modules()

    @property
    def object_id(self) -> Optional[str]:
        return self.yml_data.get("commonfields", {}).get("id")

    @property
    @abstractmethod
    def docker_image(self) -> str:
        ...

    @property
    @abstractmethod
    def code(self) -> Optional[str]:
        pass

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {
            MarketplaceVersions.XSOAR,
            MarketplaceVersions.MarketplaceV2,
            MarketplaceVersions.XPANSE,
        }

    def connect_to_api_modules(self) -> None:
        """Creates IMPORTS relationships with the API modules used in the integration."""
        code = self.code
        if not code:
            raise ValueError("Integration code is not available")
        api_modules = IntegrationScriptUnifier.check_api_module_imports(code).values()
        for api_module in api_modules:
            self.add_relationship(
                RelationshipType.IMPORTS, api_module, ContentType.SCRIPT
            )


# demisto_sdk/commands/content_graph/parsers/job.py

from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class JobParser(JSONContentItemParser, content_type=ContentType.JOB):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)

        self.connect_to_dependencies()

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR}

    @property
    def description(self) -> Optional[str]:
        return self.json_data.get("details")

    def connect_to_dependencies(self) -> None:
        if playbook := self.json_data.get("selectedFeeds"):
            raise Exception(
                "When supported, need to make sure selectedFeeds is a list of integrations "
                "on server side, because currently it's a list of instances."
            )
        if playbook := self.json_data.get("playbookId"):
            self.add_dependency_by_id(playbook, ContentType.PLAYBOOK)


# demisto_sdk/commands/content_graph/parsers/json_content_item.py

from pathlib import Path
from typing import Any, Dict, List, Optional

from demisto_sdk.commands.common.constants import (
    DEFAULT_CONTENT_ITEM_FROM_VERSION,
    DEFAULT_CONTENT_ITEM_TO_VERSION,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.tools import get_files_in_dir, get_json
from demisto_sdk.commands.content_graph.parsers.content_item import (
    ContentItemParser,
    InvalidContentItemException,
    NotAContentItemException,
)


class JSONContentItemParser(ContentItemParser):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.json_data: Dict[str, Any] = self.get_json()
        self.original_json_data: Dict[str, Any] = self.get_json()
        if not isinstance(self.json_data, dict):
            raise InvalidContentItemException(
                f"The content of {self.path} must be in a JSON dictionary format"
            )

        if self.should_skip_parsing():
            raise NotAContentItemException

    @property
    def object_id(self) -> Optional[str]:
        return self.json_data.get("id")

    @property
    def name(self) -> Optional[str]:
        return self.json_data.get("name")

    @property
    def display_name(self) -> Optional[str]:
        return self.name or self.object_id

    @property
    def deprecated(self) -> bool:
        return self.json_data.get("deprecated", False)

    @property
    def description(self) -> Optional[str]:
        return self.json_data.get("description")

    @property
    def fromversion(self) -> str:
        return self.json_data.get("fromVersion") or DEFAULT_CONTENT_ITEM_FROM_VERSION

    @property
    def toversion(self) -> str:
        return self.json_data.get("toVersion") or DEFAULT_CONTENT_ITEM_TO_VERSION

    @property
    def marketplaces(self) -> List[MarketplaceVersions]:
        if file_marketplaces := [
            MarketplaceVersions(mp) for mp in self.json_data.get("marketplaces", [])
        ]:
            return file_marketplaces
        return sorted(set(self.pack_marketplaces) & self.supported_marketplaces)

    def get_json(self) -> Dict[str, Any]:
        if self.path.is_dir():
            json_files_in_dir = get_files_in_dir(self.path.as_posix(), ["json"], False)
            if len(json_files_in_dir) != 1:
                raise NotAContentItemException(
                    f"Directory {self.path} must have a single JSON file."
                )
            self.path = Path(json_files_in_dir[0])
        return get_json(self.path.as_posix())


# demisto_sdk/commands/content_graph/parsers/layout.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.content_item import (
    NotAContentItemException,
)
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class LayoutParser(JSONContentItemParser, content_type=ContentType.LAYOUT):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        if "group" not in self.json_data:
            logger.debug(f"{path}: Not a layout container, skipping.")
            raise NotAContentItemException

        self.kind = self.json_data.get("kind")
        self.tabs = self.json_data.get("tabs")
        self.definition_id = self.json_data.get("definitionId")
        self.group = self.json_data.get("group")

        self.edit: bool = bool(self.json_data.get("edit"))
        self.indicators_details: bool = bool(self.json_data.get("indicatorsDetails"))
        self.indicators_quick_view: bool = bool(
            self.json_data.get("indicatorsQuickView")
        )
        self.quick_view: bool = bool(self.json_data.get("quickView"))
        self.close: bool = bool(self.json_data.get("close"))
        self.details: bool = bool(self.json_data.get("details"))
        self.details_v2: bool = bool(self.json_data.get("detailsV2"))
        self.mobile: bool = bool(self.json_data.get("mobile"))

        self.connect_to_dependencies()

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2}

    def connect_to_dependencies(self) -> None:
        """Collects the incident/indicator fields used as optional dependencies."""
        if self.group == "incident":
            dependency_field_type = ContentType.INCIDENT_FIELD
        elif self.group == "indicator":
            dependency_field_type = ContentType.INDICATOR_FIELD
        else:
            raise ValueError(
                f'{self.node_id}: Unknown group "{self.group}" - Expected "incident" or "indicator".'
            )

        for field in self.get_field_ids_recursively():
            if field:
                self.add_dependency_by_id(
                    field, dependency_field_type, is_mandatory=False
                )

    def get_field_ids_recursively(self) -> Set[str]:
        """Recursively iterates over the layout json data to extract all fieldId items.

        Returns:
            A set of the field IDs.
        """
        values: Set[str] = set()

        def get_values(current_object):
            if isinstance(current_object, list):
                for item in current_object:
                    get_values(item)

            elif isinstance(current_object, dict):
                for key, value in current_object.items():
                    if key == "fieldId" and isinstance(value, str):
                        values.add(
                            value.replace("incident_", "").replace("indicator_", "")
                        )
                    else:
                        get_values(value)

        get_values(self.json_data)
        return values


# demisto_sdk/commands/content_graph/parsers/layout_rule.py

from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class LayoutRuleParser(JSONContentItemParser, content_type=ContentType.LAYOUT_RULE):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)

        self.connect_to_dependencies()

    @property
    def object_id(self) -> Optional[str]:
        return self.json_data.get("rule_id")

    @property
    def name(self) -> Optional[str]:
        return self.json_data.get("rule_name")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.MarketplaceV2}

    def connect_to_dependencies(self) -> None:
        """Collects t he playbook used in the trigger as a mandatory dependency."""
        if layout := self.json_data.get("layout_id"):
            self.add_dependency_by_id(layout, ContentType.LAYOUT)


# demisto_sdk/commands/content_graph/parsers/list.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class ListParser(JSONContentItemParser, content_type=ContentType.LIST):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.type = self.json_data.get("type")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {
            MarketplaceVersions.XSOAR,
            MarketplaceVersions.MarketplaceV2,
            MarketplaceVersions.XPANSE,
        }


# demisto_sdk/commands/content_graph/parsers/mapper.py

from pathlib import Path
from typing import Any, Dict, List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.tools import extract_field_from_mapping
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)

IGNORED_INCIDENT_TYPES = ["dbot_classification_incident_type_all"]


class MapperParser(JSONContentItemParser, content_type=ContentType.MAPPER):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.type = self.json_data.get("type")
        self.definition_id = self.json_data.get("definitionId")
        self.connect_to_dependencies()

    @property
    def name(self) -> Optional[str]:
        return self.json_data.get("name") or self.json_data.get("brandName")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2}

    def get_filters_and_transformers_from_complex_value(
        self, complex_value: dict
    ) -> None:
        for filter in complex_value.get("filters", []):
            if filter:
                filter_script = filter[0].get("operator").split(".")[-1]
                self.add_dependency_by_id(filter_script, ContentType.SCRIPT)

        for transformer in complex_value.get("transformers", []):
            if transformer:
                transformer_script = transformer.get("operator").split(".")[-1]
                self.add_dependency_by_id(transformer_script, ContentType.SCRIPT)

    def connect_to_dependencies(self) -> None:
        """Collects the incident types, incident fields, filters and transformers
        used in the mapper as required dependencies.
        """
        if self.json_data.get("feed"):
            content_type_to_map = ContentType.INDICATOR_TYPE
            fields_content_type = ContentType.INDICATOR_FIELD
            add_dependency_func = self.add_dependency_by_id
        else:
            content_type_to_map = ContentType.INCIDENT_TYPE
            fields_content_type = ContentType.INCIDENT_FIELD
            add_dependency_func = self.add_dependency_by_name  # type: ignore

        if default_incident_type := self.json_data.get("defaultIncidentType"):
            add_dependency_func(
                default_incident_type, content_type_to_map, is_mandatory=False
            )

        for incident_type, mapping_data in (
            self.json_data.get("mapping") or {}
        ).items():
            if incident_type not in IGNORED_INCIDENT_TYPES:
                add_dependency_func(
                    incident_type, content_type_to_map, is_mandatory=False
                )
            internal_mapping: Dict[str, Any] = mapping_data.get("internalMapping") or {}

            if self.type == "mapping-outgoing":
                # incident fields are in the simple / complex.root key of each key
                for fields_mapper in internal_mapping.values():
                    if isinstance(fields_mapper, dict):
                        mapping_value = fields_mapper.get(
                            "simple"
                        ) or fields_mapper.get("complex", {}).get("root")
                        if field := extract_field_from_mapping(mapping_value):
                            self.add_dependency_by_id(
                                field,
                                fields_content_type,
                                is_mandatory=False,
                            )

            elif self.type == "mapping-incoming":
                # all the incident fields are the keys of the mapping
                for incident_field in internal_mapping.keys():
                    self.add_dependency_by_name(
                        incident_field,
                        fields_content_type,
                        is_mandatory=False,
                    )
            else:
                raise ValueError(
                    f'{self.node_id}: Unknown type "{self.type}" - expected "mapping-outgoing" or "mapping-incoming".'
                )

            for internal_mapping in internal_mapping.values():
                if incident_field_complex := internal_mapping.get("complex", {}):
                    self.get_filters_and_transformers_from_complex_value(
                        incident_field_complex
                    )


# demisto_sdk/commands/content_graph/parsers/modeling_rule.py

from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.yaml_content_item import (
    YAMLContentItemParser,
)


class ModelingRuleParser(YAMLContentItemParser, content_type=ContentType.MODELING_RULE):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)

    @property
    def object_id(self) -> Optional[str]:
        return self.yml_data.get("id")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.MarketplaceV2}


# demisto_sdk/commands/content_graph/parsers/pack.py

from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional

import regex

from demisto_sdk.commands.common.constants import (
    DEPRECATED_DESC_REGEX,
    DEPRECATED_NO_REPLACE_DESC_REGEX,
    PACK_NAME_DEPRECATED_REGEX,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import get_json
from demisto_sdk.commands.content_graph.common import (
    PACK_CONTRIBUTORS_FILENAME,
    PACK_METADATA_FILENAME,
    ContentType,
    Relationships,
    RelationshipType,
)
from demisto_sdk.commands.content_graph.parsers.base_content import BaseContentParser
from demisto_sdk.commands.content_graph.parsers.content_item import (
    ContentItemParser,
    InvalidContentItemException,
    NotAContentItemException,
)
from demisto_sdk.commands.content_graph.parsers.content_items_list import (
    ContentItemsList,
)

DEFAULT_MARKETPLACES = [
    MarketplaceVersions.XSOAR,
    MarketplaceVersions.MarketplaceV2,
]


class PackContentItems:
    """A class that holds all pack's content items in lists by their types."""

    def __init__(self) -> None:
        self.classifier = ContentItemsList(content_type=ContentType.CLASSIFIER)
        self.correlation_rule = ContentItemsList(
            content_type=ContentType.CORRELATION_RULE
        )
        self.dashboard = ContentItemsList(content_type=ContentType.DASHBOARD)
        self.generic_definition = ContentItemsList(
            content_type=ContentType.GENERIC_DEFINITION
        )
        self.generic_field = ContentItemsList(content_type=ContentType.GENERIC_FIELD)
        self.generic_module = ContentItemsList(content_type=ContentType.GENERIC_MODULE)
        self.generic_type = ContentItemsList(content_type=ContentType.GENERIC_TYPE)
        self.incident_field = ContentItemsList(content_type=ContentType.INCIDENT_FIELD)
        self.incident_type = ContentItemsList(content_type=ContentType.INCIDENT_TYPE)
        self.indicator_field = ContentItemsList(
            content_type=ContentType.INDICATOR_FIELD
        )
        self.indicator_type = ContentItemsList(content_type=ContentType.INDICATOR_TYPE)
        self.integration = ContentItemsList(content_type=ContentType.INTEGRATION)
        self.job = ContentItemsList(content_type=ContentType.JOB)
        self.layout = ContentItemsList(content_type=ContentType.LAYOUT)
        self.list = ContentItemsList(content_type=ContentType.LIST)
        self.mapper = ContentItemsList(content_type=ContentType.MAPPER)
        self.modeling_rule = ContentItemsList(content_type=ContentType.MODELING_RULE)
        self.parsing_rule = ContentItemsList(content_type=ContentType.PARSING_RULE)
        self.playbook = ContentItemsList(content_type=ContentType.PLAYBOOK)
        self.report = ContentItemsList(content_type=ContentType.REPORT)
        self.script = ContentItemsList(content_type=ContentType.SCRIPT)
        self.test_playbook = ContentItemsList(content_type=ContentType.TEST_PLAYBOOK)
        self.trigger = ContentItemsList(content_type=ContentType.TRIGGER)
        self.widget = ContentItemsList(content_type=ContentType.WIDGET)
        self.wizard = ContentItemsList(content_type=ContentType.WIZARD)
        self.xsiam_dashboard = ContentItemsList(
            content_type=ContentType.XSIAM_DASHBOARD
        )
        self.xsiam_report = ContentItemsList(content_type=ContentType.XSIAM_REPORT)
        self.xdrc_template = ContentItemsList(content_type=ContentType.XDRC_TEMPLATE)
        self.layout_rule = ContentItemsList(content_type=ContentType.LAYOUT_RULE)
        self.preprocess_rule = ContentItemsList(
            content_type=ContentType.PREPROCESS_RULE
        )

    def iter_lists(self) -> Iterator[ContentItemsList]:
        yield from vars(self).values()

    def append(self, obj: ContentItemParser) -> None:
        """
        Appends the object to the list with the same content_type.

        Args:
            obj (ContentItemParser): The content item to append.

        Raises:
            NotAContentItemException: If did not find any matching content item list.
        """
        for item_list in self.iter_lists():
            if item_list.content_type == obj.content_type:
                item_list.append(obj)
                return

        raise NotAContentItemException(
            f"Could not find list of {obj.content_type} items"
        )


NOW = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")


class PackMetadataParser:
    """A pack metadata parser."""

    def __init__(self, metadata: Dict[str, Any]) -> None:
        self.name: str = metadata["name"]
        self.description: str = metadata["description"]
        self.created: str = metadata.get("created", NOW)
        self.updated: str = metadata.get("updated", NOW)
        self.support: str = metadata["support"]
        self.email: str = metadata.get("email", "")
        self.url: str = metadata["url"]
        self.author: str = metadata["author"]
        self.certification: str = (
            "certified" if self.support.lower() in ["xsoar", "partner"] else ""
        )
        self.hidden: bool = metadata.get("hidden", False)
        self.server_min_version: str = metadata.get("serverMinVersion", "")
        self.current_version: str = metadata["currentVersion"]
        self.tags: List[str] = metadata["tags"]
        self.categories: List[str] = metadata["categories"]
        self.use_cases: List[str] = metadata["useCases"]
        self.keywords: List[str] = metadata["keywords"]
        self.price: Optional[int] = metadata.get("price")
        self.premium: Optional[bool] = metadata.get("premium")
        self.vendor_id: Optional[str] = metadata.get("vendorId")
        self.vendor_name: Optional[str] = metadata.get("vendorName")
        self.preview_only: Optional[bool] = metadata.get("previewOnly")
        self.marketplaces: List[MarketplaceVersions] = (
            metadata.get("marketplaces") or DEFAULT_MARKETPLACES
        )
        self.excluded_dependencies: List[str] = metadata.get("excludedDependencies", [])


class PackParser(BaseContentParser, PackMetadataParser):
    """A parsed representation of a pack.

    Attributes:
        marketplaces (List[MarketplaceVersions]): The marketplaces supporting this pack.
        content_items (PackContentItems): A collection of this pack's content item parsers.
        relationships (Relationships): A collection of the relationships in this pack.
    """

    content_type = ContentType.PACK

    def __init__(self, path: Path) -> None:
        """Parses a pack and its content items.

        Args:
            path (Path): The pack path.
        """
        BaseContentParser.__init__(self, path)
        try:
            metadata = get_json(path / PACK_METADATA_FILENAME)
        except FileNotFoundError:
            raise InvalidContentItemException(
                f"{PACK_METADATA_FILENAME} not found in pack in {path=}"
            )
        PackMetadataParser.__init__(self, metadata)
        self.content_items: PackContentItems = PackContentItems()
        self.relationships: Relationships = Relationships()
        self.connect_pack_dependencies(metadata)
        try:
            self.contributors: List[str] = get_json(path / PACK_CONTRIBUTORS_FILENAME)
        except FileNotFoundError:
            logger.debug(f"No contributors file found in {path}")
        logger.info(f"Parsing {self.node_id}")
        self.parse_pack_folders()
        logger.debug(f"Successfully parsed {self.node_id}")

    @property
    def object_id(self) -> Optional[str]:
        return self.path.name

    def connect_pack_dependencies(self, metadata: Dict[str, Any]) -> None:
        dependency: Dict[str, Dict[str, Any]]
        for pack_id, dependency in metadata.get("dependencies", {}).items():
            self.relationships.add(
                RelationshipType.DEPENDS_ON,
                source=self.object_id,
                target=pack_id,
                mandatorily=dependency.get("mandatory"),
            )

    def parse_pack_folders(self) -> None:
        """Parses all pack content items by iterating its folders."""
        for folder_path in ContentType.pack_folders(self.path):
            for (
                content_item_path
            ) in folder_path.iterdir():  # todo: consider multiprocessing
                self.parse_content_item(content_item_path)

    def parse_content_item(self, content_item_path: Path) -> None:
        """Potentially parses a single content item.

        Args:
            content_item_path (Path): The content item path.
        """
        try:
            content_item = ContentItemParser.from_path(
                content_item_path, self.marketplaces
            )
            content_item.add_to_pack(self.object_id)
            self.content_items.append(content_item)
            self.relationships.update(content_item.relationships)
        except NotAContentItemException:
            logger.debug(f"Skipping {content_item_path} - not a content item")
        except InvalidContentItemException:
            logger.error(f"{content_item_path} - invalid content item")
            raise

    @property
    def deprecated(self) -> bool:
        if regex.match(PACK_NAME_DEPRECATED_REGEX, self.name) and (
            regex.match(DEPRECATED_NO_REPLACE_DESC_REGEX, self.description)
            or regex.match(DEPRECATED_DESC_REGEX, self.description)
        ):
            return True
        return False


# demisto_sdk/commands/content_graph/parsers/parsing_rule.py

from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.yaml_content_item import (
    YAMLContentItemParser,
)


class ParsingRuleParser(YAMLContentItemParser, content_type=ContentType.PARSING_RULE):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)

    @property
    def object_id(self) -> Optional[str]:
        return self.yml_data.get("id")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.MarketplaceV2}


# demisto_sdk/commands/content_graph/parsers/playbook.py

from pathlib import Path
from typing import Any, Dict, List, Optional, Set

import networkx

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.update_id_set import (
    BUILT_IN_FIELDS,
    build_tasks_graph,
    get_fields_by_script_argument,
)
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.parsers.yaml_content_item import (
    YAMLContentItemParser,
)

LIST_COMMANDS = ["Builtin|||setList", "Builtin|||getList"]
IGNORED_FIELDS = [
    "appendTags",
    "addLabels",
    "appendMultiSelect",
    "retry-count",
    "retry-interval",
]


class PlaybookParser(YAMLContentItemParser, content_type=ContentType.PLAYBOOK):
    def __init__(
        self,
        path: Path,
        pack_marketplaces: List[MarketplaceVersions],
        is_test_playbook: bool = False,
    ) -> None:
        """Builds a directed graph representing the playbook and parses it.

        Args:
            path (Path): The playbook path.
            is_test_playbook (bool, optional): Whether this is a test playbook or not. Defaults to False.
        """
        super().__init__(path, pack_marketplaces)
        self.is_test: bool = is_test_playbook
        self.graph: networkx.DiGraph = build_tasks_graph(self.yml_data)
        self.connect_to_dependencies()
        self.connect_to_tests()

    @property
    def object_id(self) -> Optional[str]:
        return self.yml_data.get("id")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {
            MarketplaceVersions.XSOAR,
            MarketplaceVersions.MarketplaceV2,
            MarketplaceVersions.XPANSE,
        }

    def is_mandatory_dependency(self, task_id: str) -> bool:
        try:
            return self.graph.nodes[task_id]["mandatory"]
        except KeyError:
            # task is not connected to a branch
            return False

    def handle_playbook_task(self, task: Dict[str, Any], is_mandatory: bool) -> None:
        """Collects a playbook dependency.

        Args:
            task (Dict[str, Any]): The task details.
            is_mandatory (bool): Whether or not the dependency is mandatory.
        """
        if playbook := task.get("task", {}).get("playbookName"):
            self.add_relationship(
                RelationshipType.USES_PLAYBOOK,
                target=playbook,
                target_type=ContentType.PLAYBOOK,
                mandatorily=is_mandatory,
            )

    def handle_script_task(self, task: Dict[str, Any], is_mandatory: bool) -> None:
        """Collects a script dependency.

        Args:
            task (Dict[str, Any]): The task details.
            is_mandatory (bool): Whether or not the dependency is mandatory.
        """
        if script := task.get("task", {}).get("scriptName"):
            self.add_dependency_by_id(script, ContentType.SCRIPT, is_mandatory)

    def handle_command_task(self, task: Dict[str, Any], is_mandatory: bool) -> None:
        """Collects dependencies in a commands task.

        Args:
            task (Dict[str, Any]): The task details.
            is_mandatory (bool): Whether or not the dependency is mandatory.
        """
        if command := task.get("task", {}).get("script"):
            if "setIncident" in command:
                for indicator_field in get_fields_by_script_argument(task):
                    if indicator_field and indicator_field not in IGNORED_FIELDS:
                        self.add_dependency_by_id(
                            indicator_field,
                            ContentType.INCIDENT_FIELD,
                            is_mandatory=False,
                        )

            elif "setIndicator" in command:
                for indicator_field in get_fields_by_script_argument(task):
                    if indicator_field and indicator_field not in IGNORED_FIELDS:
                        self.add_dependency_by_id(
                            indicator_field,
                            ContentType.INDICATOR_FIELD,
                            is_mandatory=False,
                        )

            elif command in LIST_COMMANDS:
                # if list := task.get('scriptarguments', {}).get('listName', {}).get('simple'):
                #     self.add_dependency_by_id(list, ContentType.LIST, is_mandatory)
                pass  # TODO: CIAC-4017

            elif "Builtin" not in command:
                if "|" not in command:
                    self.add_command_or_script_dependency(command, is_mandatory)
                else:
                    integration, *_, command = command.split("|")
                    if integration:
                        self.add_dependency_by_id(
                            integration, ContentType.INTEGRATION, is_mandatory
                        )
                    else:
                        self.add_dependency_by_id(
                            command, ContentType.COMMAND, is_mandatory
                        )

    def add_complex_input_filters_and_transformers(
        self, complex_input: Dict[str, Any], is_mandatory: bool
    ) -> None:
        for filter in complex_input.get("filters", []):
            if filter:
                operator = filter[0].get("operator").split(".")[-1]
                self.add_dependency_by_id(operator, ContentType.SCRIPT, is_mandatory)

        for transformer in complex_input.get("transformers", []):
            if transformer:
                operator = transformer.get("operator").split(".")[-1]
                self.add_dependency_by_id(operator, ContentType.SCRIPT, is_mandatory)

    def handle_task_filter_and_transformer_scripts(
        self, task: Dict[str, Any], is_mandatory: bool
    ) -> None:
        """Collects filters/transformers in a task as dependencies.

        Args:
            task (Dict[str, Any]): The task details.
            is_mandatory (bool): Whether or not the dependency is mandatory.
        """
        if task.get("type") == "condition":
            for condition_entry in task.get("conditions", []):
                for inner_condition in condition_entry.get("condition", []):
                    for condition in inner_condition:
                        if (
                            condition_lhs := condition.get("left", {})
                            .get("value", {})
                            .get("complex", {})
                        ):
                            self.add_complex_input_filters_and_transformers(
                                condition_lhs, is_mandatory
                            )
                        if (
                            condition_rhs := condition.get("right", {})
                            .get("value", {})
                            .get("complex", {})
                        ):
                            self.add_complex_input_filters_and_transformers(
                                condition_rhs, is_mandatory
                            )
        else:
            for script_argument in task.get("scriptarguments", {}).values():
                if arg_value := script_argument.get("complex", {}):
                    self.add_complex_input_filters_and_transformers(
                        arg_value, is_mandatory
                    )

    def handle_field_mapping(self, task: Dict[str, Any], is_mandatory: bool) -> None:
        if field_mapping := task.get("task", {}).get("fieldMapping"):
            for incident_field in field_mapping:
                if incident_field not in BUILT_IN_FIELDS:
                    self.add_dependency_by_id(
                        incident_field, ContentType.INCIDENT_FIELD, is_mandatory
                    )

    def connect_to_dependencies(self) -> None:
        """Collects content items used by the playbook as dependencies.
        Whether or not they are mandatory is determined by their "mandatory" task node fields
        in the graph representation.
        """
        for task_id, task in self.yml_data.get("tasks", {}).items():
            is_mandatory = self.is_mandatory_dependency(task_id)
            self.handle_task_filter_and_transformer_scripts(task, is_mandatory)
            self.handle_playbook_task(task, is_mandatory)
            self.handle_script_task(task, is_mandatory)
            self.handle_command_task(task, is_mandatory)
            self.handle_field_mapping(task, is_mandatory)


# demisto_sdk/commands/content_graph/parsers/pre_process_rule.py

from typing import Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class PreProcessRuleParser(
    JSONContentItemParser, content_type=ContentType.PREPROCESS_RULE
):
    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR}


# demisto_sdk/commands/content_graph/parsers/report.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class ReportParser(JSONContentItemParser, content_type=ContentType.REPORT):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)

        self.connect_to_dependencies()

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR}

    def connect_to_dependencies(self) -> None:
        """Collects scripts used in the report as optional dependencies."""
        for layout in self.json_data.get("dashboard", {}).get("layout", []):
            widget_data = layout.get("widget")
            if widget_data.get("dataType") == "scripts":
                if script_name := widget_data.get("query"):
                    self.add_dependency_by_id(
                        script_name, ContentType.SCRIPT, is_mandatory=False
                    )


# demisto_sdk/commands/content_graph/parsers/repository.py

import multiprocessing
import traceback
from pathlib import Path
from typing import Iterator, List, Optional

from demisto_sdk.commands.common.constants import PACKS_FOLDER
from demisto_sdk.commands.common.cpu_count import cpu_count
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.content_graph.parsers.pack import PackParser

IGNORED_PACKS_FOR_PARSING = ["NonSupported"]


class RepositoryParser:
    """
    Attributes:
        path (Path): The repository path.
        packs_to_parse (Optional[List[str]]): A list of packs to parse. If not provided, parses all packs.
        packs (List[PackParser]): A list of the repository's packs parser objects.
    """

    def __init__(self, path: Path) -> None:
        """Parsing all repository packs.

        Args:
            path (Path): The repository path.
            packs_to_parse (Optional[List[str]]): A list of packs to parse. If not provided, parses all packs.
        """
        self.path: Path = path
        self.packs: List[PackParser] = []

    def parse(self, packs_to_parse: Optional[List[Path]] = None):
        if not packs_to_parse:
            # if no packs to parse were provided, parse all packs
            packs_to_parse = list(self.iter_packs())
        try:
            logger.info("Parsing packs...")
            with multiprocessing.Pool(processes=cpu_count()) as pool:
                self.packs = list(pool.map(PackParser, packs_to_parse))
        except Exception:
            logger.error(traceback.format_exc())
            raise

    @staticmethod
    def should_parse_pack(path: Path) -> bool:
        return (
            path.is_dir()
            and not path.name.startswith(".")
            and path.name not in IGNORED_PACKS_FOR_PARSING
        )

    def iter_packs(self, packs_to_parse: Optional[List[str]] = None) -> Iterator[Path]:
        """Iterates all packs in the repository.

        Yields:
            Iterator[Path]: A pack path.
        """
        packs_folder: Path = self.path / PACKS_FOLDER
        if packs_to_parse:
            for pack in packs_to_parse:
                path = packs_folder / pack
                if not path.is_dir():
                    raise FileNotFoundError(f"Pack {pack} does not exist.")
                if self.should_parse_pack(path):
                    yield path

        else:
            for path in packs_folder.iterdir():
                if self.should_parse_pack(path):
                    yield path

    def clear(self):
        self.packs = []


# demisto_sdk/commands/content_graph/parsers/script.py

import re
from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.integration_script import (
    IntegrationScriptParser,
)
from demisto_sdk.commands.prepare_content.integration_script_unifier import (
    IntegrationScriptUnifier,
)

EXECUTE_CMD_PATTERN = re.compile(
    r"execute_?command\(['\"]([a-zA-Z-_]+)['\"].*", re.IGNORECASE
)


class ScriptParser(IntegrationScriptParser, content_type=ContentType.SCRIPT):
    def __init__(
        self,
        path: Path,
        pack_marketplaces: List[MarketplaceVersions],
        is_test_script: bool = False,
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.is_test: bool = is_test_script
        self.type = self.yml_data.get("subtype") or self.yml_data.get("type")
        self.tags: List[str] = self.yml_data.get("tags", [])
        self.skip_prepare: List[str] = self.yml_data.get("skipprepare", [])
        if self.type == "python":
            self.type += "2"

        self.connect_to_dependencies()
        self.connect_to_tests()

    @property
    def description(self) -> Optional[str]:
        return self.yml_data.get("comment")

    @property
    def docker_image(self) -> str:
        return self.yml_data.get("dockerimage", "")

    def connect_to_dependencies(self) -> None:
        """Creates USES_COMMAND_OR_SCRIPT mandatory relationships with the commands/scripts used.
        At this stage, we can't determine whether the dependencies are commands or scripts.
        """
        for cmd in self.get_depends_on():
            self.add_command_or_script_dependency(cmd)

        for cmd in self.get_command_executions():
            self.add_command_or_script_dependency(cmd)

    @property
    def code(self) -> Optional[str]:
        """Gets the script code.
        If the script is unified, it is taken from the yml file.
        Otherwise, uses the Unifier object to get it.

        Returns:
            str: The script code.
        """
        if self.is_unified or self.yml_data.get("script") not in ["-", ""]:
            return self.yml_data.get("script")
        return IntegrationScriptUnifier.get_script_or_integration_package_data(
            self.path.parent
        )[1]

    def get_depends_on(self) -> Set[str]:
        depends_on: List[str] = self.yml_data.get("dependson", {}).get("must", [])
        return {cmd.split("|")[-1] for cmd in depends_on}

    def get_command_executions(self) -> Set[str]:
        code = self.code
        if not code:
            raise ValueError("Script code is not available")
        return set(EXECUTE_CMD_PATTERN.findall(code))


# demisto_sdk/commands/content_graph/parsers/test_playbook.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.content_item import (
    IncorrectParserException,
    NotAContentItemException,
)
from demisto_sdk.commands.content_graph.parsers.playbook import PlaybookParser
from demisto_sdk.commands.content_graph.parsers.script import ScriptParser

NON_CIRCLE_TESTS_DIRECTORY = "NonCircleTests"


class TestPlaybookParser(PlaybookParser, content_type=ContentType.TEST_PLAYBOOK):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        """Parses the test playbook.

        Args:
            path (Path): The test playbook's path.

        Raises:
            NotAContentItemException: Indicating this test playbook should not be included in the graph.
            IncorrectParserException: When detecting this content item is a test script.
        """
        if NON_CIRCLE_TESTS_DIRECTORY in path.name:
            raise NotAContentItemException

        super().__init__(path, pack_marketplaces, is_test_playbook=True)

        if self.yml_data.get("script"):
            raise IncorrectParserException(
                correct_parser=ScriptParser, is_test_script=True
            )

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2}


# demisto_sdk/commands/content_graph/parsers/trigger.py

from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class TriggerParser(JSONContentItemParser, content_type=ContentType.TRIGGER):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)

        self.connect_to_dependencies()

    @property
    def object_id(self) -> Optional[str]:
        return self.json_data.get("trigger_id")

    @property
    def name(self) -> Optional[str]:
        return self.json_data.get("trigger_name")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.MarketplaceV2}

    def connect_to_dependencies(self) -> None:
        """Collects the playbook used in the trigger as a mandatory dependency."""
        if playbook := self.json_data.get("playbook_id"):
            self.add_dependency_by_id(playbook, ContentType.PLAYBOOK)


# demisto_sdk/commands/content_graph/parsers/widget.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class WidgetParser(JSONContentItemParser, content_type=ContentType.WIDGET):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.data_type = self.json_data.get("dataType")
        self.widget_type = self.json_data.get("widgetType")

        self.connect_to_dependencies()

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR}

    def connect_to_dependencies(self) -> None:
        """Collects the playbook used in the widget as a mandatory dependency."""
        if self.data_type == "scripts":
            if script := self.json_data.get("query"):
                self.add_dependency_by_id(script, ContentType.SCRIPT)


# demisto_sdk/commands/content_graph/parsers/wizard.py

from pathlib import Path
from typing import List, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class WizardParser(JSONContentItemParser, content_type=ContentType.WIZARD):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.packs: List[str] = self.get_packs()
        self.integrations: List[str] = self.get_integrations()
        self.playbooks: List[str] = self.get_playbooks()

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.XSOAR}

    def get_packs(self) -> List[str]:
        packs: List[str] = []
        for packs_bundle in self.json_data.get("dependency_packs", []):
            for pack in packs_bundle.get("packs", []):
                packs.append(pack.get("name"))
        return packs

    def get_integrations(self) -> List[str]:
        integrations: List[str] = []
        for integration_type in ["fetching_integrations", "supporting_integrations"]:
            for integration in self.json_data.get("wizard", {}).get(
                integration_type, []
            ):
                integrations.append(integration.get("name"))
        return integrations

    def get_playbooks(self) -> List[str]:
        playbooks: List[str] = []
        for playbook in self.json_data.get("wizard", {}).get("set_playbook", []):
            playbooks.append(playbook.get("name"))
        return playbooks


# demisto_sdk/commands/content_graph/parsers/xdrc_template.py

from pathlib import Path
from typing import List, Optional, Set

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class XDRCTemplateParser(JSONContentItemParser, content_type=ContentType.XDRC_TEMPLATE):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.content_global_id = self.json_data.get("content_global_id")
        self.os_type = self.json_data.get("os_type")
        self.profile_type = self.json_data.get("profile_type")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.MarketplaceV2}

    @property
    def object_id(self) -> Optional[str]:
        return self.json_data.get("content_global_id")


# demisto_sdk/commands/content_graph/parsers/xsiam_dashboard.py

from pathlib import Path
from typing import Any, Dict, List, Optional, Set

from demisto_sdk.commands.common.constants import (
    DEFAULT_CONTENT_ITEM_FROM_VERSION,
    DEFAULT_CONTENT_ITEM_TO_VERSION,
    MarketplaceVersions,
)
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class XSIAMDashboardParser(
    JSONContentItemParser, content_type=ContentType.XSIAM_DASHBOARD
):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.json_data: Dict[str, Any] = self.json_data.get("dashboards_data", [{}])[0]

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.MarketplaceV2}

    @property
    def object_id(self) -> Optional[str]:
        return self.json_data.get("global_id")

    @property
    def fromversion(self) -> str:
        return (
            self.original_json_data.get("fromVersion")
            or DEFAULT_CONTENT_ITEM_FROM_VERSION
        )

    @property
    def toversion(self) -> str:
        return (
            self.original_json_data.get("toVersion") or DEFAULT_CONTENT_ITEM_TO_VERSION
        )


# demisto_sdk/commands/content_graph/parsers/xsiam_report.py

from pathlib import Path
from typing import Any, Dict, List, Optional, Set

from demisto_sdk.commands.common.constants import (
    DEFAULT_CONTENT_ITEM_FROM_VERSION,
    DEFAULT_CONTENT_ITEM_TO_VERSION,
    MarketplaceVersions,
)
from demisto_sdk.commands.content_graph.common import ContentType
from demisto_sdk.commands.content_graph.parsers.json_content_item import (
    JSONContentItemParser,
)


class XSIAMReportParser(JSONContentItemParser, content_type=ContentType.XSIAM_REPORT):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.json_data: Dict[str, Any] = self.json_data.get("templates_data", [{}])[0]

    @property
    def name(self) -> Optional[str]:
        return self.json_data.get("report_name")

    @property
    def object_id(self) -> Optional[str]:
        return self.json_data.get("global_id")

    @property
    def supported_marketplaces(self) -> Set[MarketplaceVersions]:
        return {MarketplaceVersions.MarketplaceV2}

    @property
    def fromversion(self) -> str:
        return (
            self.original_json_data.get("fromVersion")
            or DEFAULT_CONTENT_ITEM_FROM_VERSION
        )

    @property
    def toversion(self) -> str:
        return (
            self.original_json_data.get("toVersion") or DEFAULT_CONTENT_ITEM_TO_VERSION
        )


# demisto_sdk/commands/content_graph/parsers/yaml_content_item.py

from pathlib import Path
from typing import Any, Dict, List, Optional, Union

from demisto_sdk.commands.common.constants import (
    DEFAULT_CONTENT_ITEM_FROM_VERSION,
    DEFAULT_CONTENT_ITEM_TO_VERSION,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.tools import get_yaml, get_yml_paths_in_dir
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.parsers.content_item import (
    ContentItemParser,
    InvalidContentItemException,
    NotAContentItemException,
)


class YAMLContentItemParser(ContentItemParser):
    def __init__(
        self, path: Path, pack_marketplaces: List[MarketplaceVersions]
    ) -> None:
        super().__init__(path, pack_marketplaces)
        self.yml_data: Dict[str, Any] = self.get_yaml()

        if not isinstance(self.yml_data, dict):
            raise InvalidContentItemException(
                f"The content of {self.path} must be in a JSON dictionary format"
            )

        if self.should_skip_parsing():
            raise NotAContentItemException

    @property
    def name(self) -> Optional[str]:
        return self.yml_data.get("name")

    @property
    def display_name(self) -> Optional[str]:
        return self.name or self.object_id

    @property
    def deprecated(self) -> bool:
        return self.yml_data.get("deprecated", False)

    @property
    def description(self) -> Optional[str]:
        description = self.yml_data.get("description", "")
        description = description.replace("\\ ", " ")  # removes unwanted backslashes
        description = description.replace("\\\n", " ")  # removes unwanted backslashes
        description = " ".join(
            description.split()
        )  # substitutes multiple spaces into one
        return description

    @property
    def fromversion(self) -> str:
        return self.yml_data.get("fromversion") or DEFAULT_CONTENT_ITEM_FROM_VERSION

    @property
    def toversion(self) -> str:
        return self.yml_data.get("toversion") or DEFAULT_CONTENT_ITEM_TO_VERSION

    @property
    def marketplaces(self) -> List[MarketplaceVersions]:
        if file_marketplaces := [
            MarketplaceVersions(mp) for mp in self.yml_data.get("marketplaces", [])
        ]:
            return file_marketplaces
        return sorted(set(self.pack_marketplaces) & self.supported_marketplaces)

    def connect_to_tests(self) -> None:
        """Iterates over the test playbooks registered to this content item,
        and creates a TESTED_BY relationship between the content item to each of them.
        """
        tests_playbooks: List[str] = self.yml_data.get("tests", [])
        for test_playbook_id in tests_playbooks:
            if "no test" not in test_playbook_id.lower():
                self.add_relationship(
                    RelationshipType.TESTED_BY,
                    target=test_playbook_id,
                    target_type=ContentType.TEST_PLAYBOOK,
                )

    def get_yaml(self) -> Dict[str, Union[str, List[str]]]:
        if not self.path.is_dir():
            yaml_path = self.path.as_posix()
        else:
            _, yaml_path = get_yml_paths_in_dir(self.path.as_posix())
        if not yaml_path:
            raise NotAContentItemException

        self.path = Path(yaml_path)
        return get_yaml(self.path.as_posix(), keep_order=False)


# demisto_sdk/commands/content_graph/common.py

import enum
import os
import re
from pathlib import Path
from typing import Any, Dict, Iterator, List, NamedTuple, Set

from neo4j import graph

from demisto_sdk.commands.common.constants import PACKS_FOLDER

NEO4J_ADMIN_DOCKER = ""

NEO4J_DATABASE_HTTP = os.getenv(
    "DEMISTO_SDK_NEO4J_DATABASE_HTTP", "http://127.0.0.1:7474"
)
NEO4J_DATABASE_URL = os.getenv(
    "DEMISTO_SDK_NEO4J_DATABASE_URL", "bolt://127.0.0.1:7687"
)
NEO4J_USERNAME = os.getenv("DEMISTO_SDK_NEO4J_USERNAME", "neo4j")
NEO4J_PASSWORD = os.getenv("DEMISTO_SDK_NEO4J_PASSWORD", "contentgraph")

NEO4J_FOLDER = "neo4j-data"

PACK_METADATA_FILENAME = "pack_metadata.json"
PACK_CONTRIBUTORS_FILENAME = "CONTRIBUTORS.json"
UNIFIED_FILES_SUFFIXES = [".yml", ".json"]


class Neo4jRelationshipResult(NamedTuple):
    node_from: graph.Node
    relationships: List[graph.Relationship]
    nodes_to: List[graph.Node]


class RelationshipType(str, enum.Enum):
    DEPENDS_ON = "DEPENDS_ON"
    HAS_COMMAND = "HAS_COMMAND"
    IMPORTS = "IMPORTS"
    IN_PACK = "IN_PACK"
    TESTED_BY = "TESTED_BY"
    USES = "USES"
    USES_BY_ID = "USES_BY_ID"
    USES_BY_NAME = "USES_BY_NAME"
    USES_COMMAND_OR_SCRIPT = "USES_COMMAND_OR_SCRIPT"
    USES_PLAYBOOK = "USES_PLAYBOOK"


class ContentType(str, enum.Enum):
    BASE_CONTENT = "BaseContent"
    CLASSIFIER = "Classifier"
    COMMAND = "Command"
    COMMAND_OR_SCRIPT = "CommandOrScript"
    CONNECTION = "Connection"
    CORRELATION_RULE = "CorrelationRule"
    DASHBOARD = "Dashboard"
    GENERIC_DEFINITION = "GenericDefinition"
    GENERIC_FIELD = "GenericField"
    GENERIC_MODULE = "GenericModule"
    GENERIC_TYPE = "GenericType"
    INCIDENT_FIELD = "IncidentField"
    INCIDENT_TYPE = "IncidentType"
    INDICATOR_FIELD = "IndicatorField"
    INDICATOR_TYPE = "IndicatorType"
    INTEGRATION = "Integration"
    JOB = "Job"
    LAYOUT = "Layout"
    LIST = "List"
    MAPPER = "Mapper"
    MODELING_RULE = "ModelingRule"
    PACK = "Pack"
    PARSING_RULE = "ParsingRule"
    PLAYBOOK = "Playbook"
    PREPROCESS_RULE = "PreProcessRule"
    REPORT = "Report"
    SCRIPT = "Script"
    TEST_PLAYBOOK = "TestPlaybook"
    TRIGGER = "Trigger"
    WIDGET = "Widget"
    XSIAM_DASHBOARD = "XSIAMDashboard"
    XSIAM_REPORT = "XSIAMReport"
    WIZARD = "Wizard"
    XDRC_TEMPLATE = "XDRCTemplate"
    LAYOUT_RULE = "LayoutRule"

    @property
    def labels(self) -> List[str]:
        labels: Set[str] = {ContentType.BASE_CONTENT.value, self.value}

        if self.value == ContentType.TEST_PLAYBOOK.value:
            labels.add(ContentType.PLAYBOOK.value)

        if self in [ContentType.SCRIPT, ContentType.COMMAND]:
            labels.add(ContentType.COMMAND_OR_SCRIPT.value)

        return list(labels)

    @property
    def server_name(self) -> str:
        if self == ContentType.INDICATOR_TYPE:
            return "reputation"
        elif self == ContentType.INDICATOR_FIELD:
            return "incidentfield-indicatorfield"
        elif self == ContentType.LAYOUT:
            return "layoutscontainer"
        elif self == ContentType.PREPROCESS_RULE:
            return "preprocessrule"
        elif self == ContentType.TEST_PLAYBOOK:
            return ContentType.PLAYBOOK.server_name
        elif self == ContentType.MAPPER:
            return "classifier-mapper"
        return self.lower()

    # def __hash__(self) -> int:
    #     return hash(self.value)

    @property
    def metadata_name(self) -> str:
        if self == ContentType.SCRIPT:
            return "automation"
        elif self == ContentType.INDICATOR_TYPE:
            return "reputation"
        elif self == ContentType.LAYOUT:
            return "layoutscontainer"
        elif self == ContentType.TEST_PLAYBOOK:
            return ContentType.PLAYBOOK.server_name
        elif self == ContentType.MAPPER:
            return "classifier"
        return self.lower()

    @property
    def metadata_display_name(self) -> str:
        if self == ContentType.SCRIPT:
            return "Automation"
        elif self == ContentType.INDICATOR_TYPE:
            return "Reputation"
        elif self == ContentType.MAPPER:
            return "Classifier"
        elif self == ContentType.LAYOUT:
            return "Layouts Container"
        else:
            return re.sub(r"([a-z](?=[A-Z])|[A-Z](?=[A-Z][a-z]))", r"\1 ", self.value)

    @staticmethod
    def server_names() -> List[str]:
        return [c.server_name for c in ContentType] + ["indicatorfield", "mapper"]

    @staticmethod
    def values() -> Iterator[str]:
        return (c.value for c in ContentType)

    @classmethod
    def by_path(cls, path: Path) -> "ContentType":
        for idx, folder in enumerate(path.parts):
            if folder == PACKS_FOLDER:
                if len(path.parts) <= idx + 2:
                    raise ValueError("Invalid content path.")
                content_type_dir = path.parts[idx + 2]
                break
        else:
            # less safe option - will raise an exception if the path
            # is not to the content item directory or file
            if path.parts[-2][:-1] in ContentType.values():
                content_type_dir = path.parts[-2]
            elif path.parts[-3][:-1] in ContentType.values():
                content_type_dir = path.parts[-3]
            else:
                raise ValueError(f"Could not find content type in path {path}")
        return cls(content_type_dir[:-1])  # remove the `s`

    @staticmethod
    def folders() -> List[str]:
        return [c.as_folder for c in ContentType]

    @property
    def as_folder(self) -> str:
        if self == ContentType.MAPPER:
            return f"{ContentType.CLASSIFIER}s"
        return f"{self.value}s"

    @staticmethod
    def abstract_types() -> List["ContentType"]:
        return [ContentType.BASE_CONTENT, ContentType.COMMAND_OR_SCRIPT]

    @staticmethod
    def non_content_items() -> List["ContentType"]:
        return [ContentType.PACK, ContentType.COMMAND]

    @staticmethod
    def non_abstracts(
        include_non_content_items: bool = True,
    ) -> Iterator["ContentType"]:
        for content_type in ContentType:
            if content_type in ContentType.abstract_types():
                continue
            if (
                not include_non_content_items
                and content_type in ContentType.non_content_items()
            ):
                continue
            yield content_type

    @staticmethod
    def content_items() -> Iterator["ContentType"]:
        return ContentType.non_abstracts(include_non_content_items=False)

    @staticmethod
    def threat_intel_report_types() -> List["ContentType"]:
        return [ContentType.GENERIC_FIELD, ContentType.GENERIC_TYPE]

    @staticmethod
    def pack_folders(pack_path: Path) -> Iterator[Path]:
        for content_type in ContentType.content_items():
            if content_type == ContentType.MAPPER:
                continue
            pack_folder = pack_path / content_type.as_folder
            if pack_folder.is_dir() and not pack_folder.name.startswith("."):
                if content_type not in ContentType.threat_intel_report_types():
                    yield pack_folder
                else:
                    for tir_folder in pack_folder.iterdir():
                        if tir_folder.is_dir() and not tir_folder.name.startswith("."):
                            yield tir_folder


class Relationships(dict):
    def add(self, relationship: RelationshipType, **kwargs):
        if relationship not in self.keys():
            self.__setitem__(relationship, [])
        self.__getitem__(relationship).append(kwargs)

    def add_batch(self, relationship: RelationshipType, data: List[Dict[str, Any]]):
        if relationship not in self.keys():
            self.__setitem__(relationship, [])
        self.__getitem__(relationship).extend(data)

    def update(self, other: "Relationships") -> None:  # type: ignore
        for relationship, parsed_data in other.items():
            if relationship not in RelationshipType or not isinstance(
                parsed_data, list
            ):
                raise TypeError
            self.add_batch(relationship, parsed_data)


class Nodes(dict):
    def __init__(self, *args) -> None:
        super().__init__(self)
        for arg in args:
            if not isinstance(arg, dict):
                raise ValueError(f"Expected a dict: {arg}")
        self.add_batch(args)  # type: ignore[arg-type]

    def add(self, **kwargs):
        content_type: ContentType = ContentType(kwargs.get("content_type"))
        if content_type not in self.keys():
            self.__setitem__(content_type, [])
        self.__getitem__(content_type).append(kwargs)

    def add_batch(self, data: Iterator[Dict[str, Any]]):
        for obj in data:
            self.add(**obj)

    def update(self, other: "Nodes") -> None:  # type: ignore[override]
        data: Iterator[Dict[str, Any]]
        for content_type, data in other.items():
            if content_type not in ContentType or not isinstance(data, list):
                raise TypeError
            self.add_batch(data)


SERVER_CONTENT_ITEMS: dict = {
    ContentType.INCIDENT_FIELD: [
        "name",
        "details",
        "severity",
        "owner",
        "created",
        "modified",
        "dbotCreatedBy",
        "type",
        "dbotSource",
        "category",
        "dbotStatus",
        "playbookId",
        "dbotCreated",
        "dbotClosed",
        "closed",
        "occurred",
        "activated",
        "openDuration",
        "lastOpen",
        "dbotDueDate",
        "dueDate",
        "dbotModified",
        "dbotTotalTime",
        "reason",
        "closeReason",
        "closeNotes",
        "closingUserId",
        "activatingingUserId",
        "reminder",
        "notifyTime",
        "lastJobRunTime",
        "sla",
        "phase",
        "rawPhase",
        "rawName",
        "rawType",
        "parent",
        "roles",
        "xsoarReadOnlyRoles",
        "labels",
        "attachment",
        "runStatus",
        "sourceBrand",
        "sourceInstance",
        "CustomFields",
        "droppedCount",
        "linkedCount",
        "linkedIncidents",
        "feedBased",
        "isDebug",
        "dbotMirrorId",
        "dbotMirrorInstance",
        "dbotMirrorDirection",
        "dbotDirtyFields",
        "dbotCurrentDirtyFields",
        "dbotMirrorTags",
        "dbotMirrorLastSync",
        "timestamp",
        "slaField",
    ],
    ContentType.INDICATOR_FIELD: [
        "name",
        "relatedIncCount",
        "timestamp",
        "indicator_type",
        "value",
        "source",
        "investigationIDs",
        "lastSeen",
        "calculatedTime",
        "firstSeen",
        "score",
        "md5",
        "sha1",
        "sha256",
        "sha512",
        "ssdeep",
        "imphash",
        "size",
        "filetype",
        "comment",
        "expiration",
        "manualExpirationTime",
        "expirationStatus",
        "expirationdate",
        "sourceInstances",
        "sourceBrands",
        "modifiedTime",
        "comments",
        "modified",
        "isShared",
        "registrarname",
        "indicatortype",
        "aggregatedReliability",
        "starttime",
        "indicatorIDs",
        "indicatorsValues",
        "verdict",
        "reputation",
    ],
    ContentType.SCRIPT: [
        "getAPIKeyFromLicense",
        "handleIndicatorFormatterCache",
        "dockerImageUpdate",
        "addSystem",
        "getEntries",
        "getContext",
        "getFindings",
        "delContext",
        "getEntry",
        "closeInvestigation",
        "reopenInvestigation",
        "setSeverity",
        "setOwner",
        "setPhase",
        "taskReopen",
        "taskComplete",
        "taskAssign",
        "setTaskDueDate",
        "todoRemove",
        "todoAdd",
        "todoReopen",
        "todoComplete",
        "todoAssign",
        "todoDueDate",
        "addOneTimeEntitlement",
        "addEntitlement",
        "setPlaybook",
        "setIncident",
        "resetDirtyFields",
        "investigate",
        "setIncidentReminder",
        "createEntry",
        "addEntries",
        "createNewIncident",
        "setPlaybookAccordingToType",
        "getUserByEmail",
        "getUserByUsername",
        "getFilePath",
        "getIncidents",
        "addTask",
        "scheduleEntry",
        "cancelScheduledEntry",
        "markAsEvidence",
        "markAsNote",
        "setYourselfAs",
        "appendIndicatorField",
        "removeIndicatorField",
        "enrichIndicators",
        "getList",
        "setList",
        "createList",
        "addToList",
        "removeFromList",
        "setEntriesTags",
        "resetEntriesTags",
        "findIndicators",
        "getIndicator",
        "deleteIndicators",
        "executeCommandAt",
        "getUsers",
        "getRoles",
        "setRoleShifts",
        "setIndicator",
        "setIndicators",
        "createNewIndicator",
        "associateIndicatorToIncident",
        "associateIndicatorsToIncident",
        "unAssociateIndicatorToIncident",
        "unAssociateIndicatorsFromIncident",
        "addChildInvestigation",
        "pauseInvestigation",
        "generateSummaryReport",
        "generateGeneralReport",
        "resumeInvestigation",
        "getOwnerSuggestion",
        "getIndicatorScoreCache",
        "restrictInvestigation",
        "linkIncidents",
        "mdToHtml",
        "relatedIncidents",
        "maliciousRatio",
        "similarSsdeep",
        "extractIndicators",
        "isWhitelisted",
        "invite",
        "startTimer",
        "resetTimer",
        "stopTimer",
        "pauseTimer",
        "createMLModel",
        "deleteMLModel",
        "evaluateMLModel",
        "getMLModel",
        "reevaluateMLModel",
        "shareIndicators",
        "expireIndicators",
        "getWorkersStatistics",
        "excludeIndicators",
        "getMirrorStatistics",
        "getSyncMirrorRecords",
        "purgeClosedSyncMirrorRecords",
        "getInvPlaybookMetaData",
        "getDBStatistics",
        "getInternalData",
        "drawCanvas",
        "deleteRelationships",
        "searchRelationships",
        "getSystemDiagnostics",
        "triggerDebugMirroringRun",
        "stopScheduleEntry",
        # Filters
        "isEqual",
        "isNotEqual",
        "isEqualCase",
        "isNotEqualCase",
        "isEqualNumber",
        "isNotEqualNumber",
        "isEqualString",
        "isNotEqualString",
        "contains",
        "notContains",
        "containsString",
        "notContainsString",
        "startWith",
        "notStartWith",
        "endWith",
        "notEndWith",
        "inList",
        "notInList",
        "match",
        "stringHasLength",
        "isEqual",
        "isNotEqual",
        "greaterThan",
        "greaterThanOrEqual",
        "lessThan",
        "lessThanOrEqual",
        "isSame",
        "isBefore",
        "isAfter",
        "isTrue",
        "isFalse",
        "isExists",
        "isNotExists",
        "isEmpty",
        "isNotEmpty",
        "contains",
        "notContains",
        "in",
        "notIn",
        "hasLength",
        "isIdenticalIncident",
        "isNotIdenticalIncident",
        "containsGeneral",
        "notContainsGeneral",
        # Transformers
        "toUpperCase",
        "toLowerCase",
        "substringFrom",
        "substringTo",
        "substring",
        "split",
        "splitAndTrim",
        "trim",
        "replace",
        "replaceMatch",
        "concat",
        "strLength",
        "round",
        "floor",
        "ceil",
        "addition",
        "subtraction",
        "multiply",
        "division",
        "modulo",
        "toPercent",
        "abs",
        "precision",
        "quadraticEquation",
        "toString",
        "toUnix",
        "getField",
        "sort",
        "count",
        "atIndex",
        "join",
        "uniq",
        "indexOf",
        "slice",
        "sliceByItem",
        "splice",
        "Stringify",
        "append",
        "ConvertKeysToTableFieldFormat",
        # XSIAM scripts aliases
        "setAlert",
        "setAlertReminder",
        "createNewAlert",
        "getAlerts",
        "associateIndicatorToAlert",
        "unAssociateIndicatorToAlert",
        "linkAlerts",
        "relatedAlerts",
        "associateIndicatorsToAlert",
        "unAssociateIndicatorsFromAlert",
    ],
    ContentType.COMMAND: [
        # activedir-login integration commands
        "ad-default-domain",
        "ad-authenticate",
        "ad-authentication-roles",
        "ad-authenticate-and-roles",
        "ad-groups",
        # activedir integration commands
        "ad-search",
        "ad-expire-password",
        "ad-set-new-password",
        "ad-unlock-account",
        "ad-disable-account",
        "ad-enable-account",
        "ad-remove-from-group",
        "ad-add-to-group",
        "ad-create-user",
        "ad-update-user",
        "ad-delete-user",
        "ad-modify-computer-ou",
        "ad-create-contact",
        "ad-update-contact",
        # carbonblackprotection integration commands
        "cbp-fileCatalog-search",
        "cbp-fileInstance-search",
        "cbp-fileRule-search",
        "cbp-fileRule-get",
        "cbp-fileRule-delete",
        "cbp-fileRule-update",
        "cbp-fileAnalysis-get",
        "cbp-fileAnalysis-createOrUpdate",
        "cbp-fileAnalysis-search",
        "cbp-fileUpload-get",
        "cbp-fileUpload-download",
        "cbp-fileUpload-createOrUpdate",
        "cbp-fileUpload-search",
        "cbp-connector-get",
        "cbp-connector-search",
        "cbp-computer-search",
        "cbp-computer-get",
        "cbp-computer-update",
        "cbp-notification-search",
        "cbp-publisher-search",
        "cbp-event-search",
        "cbp-approvalRequest-search",
        "cbp-serverConfig-search",
        "cbp-policy-search",
        # carbonblack integration commands
        "cb-version",
        "cb-process",
        "cb-process-events",
        "cb-binary",
        "cb-binary-get",
        "cb-alert",
        "cb-list-sensors",
        "cb-list-sessions",
        "cb-sensor-info",
        "cb-session-create",
        "cb-session-close",
        "cb-keepalive",
        "cb-session-info",
        "cb-archive",
        "cb-command-create",
        "cb-list-commands",
        "cb-command-info",
        "cb-command-cancel",
        "cb-list-files",
        "cb-file-info",
        "cb-file-delete",
        "cb-file-get",
        "cb-watchlist-get",
        "cb-watchlist-new",
        "cb-watchlist-set",
        "cb-watchlist-del",
        "cb-terminate-process",
        "cb-quarantine-device",
        "cb-unquarantine-device",
        "cb-block-hash",
        "cb-unblock-hash",
        "cb-get-hash-blacklist",
        "cb-get-process",
        "cb-get-processes",
        # cylance integration commands
        "file",
        "cy-upload",
        # duo integration commands
        "duo-authenticate",
        "duo-authenticate-status",
        "duo-check",
        "duo-preauth",
        # elasticsearch integration commands
        "search",
        # fcm integration commands
        "fcm-push",
        # google integration commands
        "googleapps-list-users",
        "googleapps-get-user",
        "googleapps-delete-user",
        "googleapps-get-user-roles",
        "googleapps-revoke-user-role",
        "googleapps-gmail-search",
        "googleapps-gmail-get-mail",
        "googleapps-device-action",
        "googleapps-get-devices-for-user",
        "googleapps-get-tokens-for-user",
        "googleapps-chrome-device-action",
        "googleapps-get-chrome-devices-for-user",
        "googleapps-gmail-get-attachment",
        # kafka integration commands
        "kafka-publish-msg",
        "kafka-print-topics",
        "kafka-consume-msg",
        "kafka-fetch-partitions",
        # mail-sender integration commands
        "send-mail",
        # mattermost integration commands
        "send-notification",
        "mattermost-send",
        "mattermost-send-file",
        "mattermost-close-channel",
        "close-channel",
        "mattermost-mirror-investigation",
        "mirror-investigation",
        # esm integration commands
        "search",
        "esmFetchAllFields",
        # mysql integration commands
        "query",
        # nexpose integration commands
        "vulnerability-list",
        "vulnerability-details",
        "generate-adhoc-report",
        "send-xml",
        # pagerduty integration commands
        "PagerDutyGetUsersOnCall",
        "PagerDutyGetAllSchedules",
        "PagerDutyGetUsersOnCallNow",
        "PagerDutyIncidents",
        "pagerDutySubmitEvent",
        # remoteaccess integration commands
        "ssh",
        "copy-to",
        "copy-from",
        # sharedagent integration commands
        "sharedagent_create",
        "execute",
        "sharedagent_remove",
        "sharedagent_status",
        # slack integration commands
        "send-notification",
        "slack-send",
        "mirror-investigation",
        "slack-mirror-investigation",
        "close-channel",
        "slack-close-channel",
        "slack-send-file",
        # mssql integration commands
        "query",
        # threatcentral integration commands
        "Threat-Central",
    ],
    ContentType.INTEGRATION: [
        "mail-listener",
        "osxcollector",
        "volatility",
        "threatcentral",
        "mattermost",
        "indicators-share",
        "sharedagent",
        "activedir",
        "activedir-login",
        "esm",
        "saml",
        "pagerduty",
        "mail-sender",
        "carbonblack",
        "carbonblackprotection",
        "slack",
        "nexpose",
        "duo",
        "cylance",
        "remoteaccess",
        "elasticsearch",
        "mysql",
        "mssql",
        "google",
        "crowdstrike-streaming-api",
        "kafka",
        "syslog",
        "fcm",
    ],
}


# Used to remove content-private nodes, as a temporary temporary workaround.
# For more details: https://jira-hq.paloaltonetworks.local/browse/CIAC-7149
CONTENT_PRIVATE_ITEMS: dict = {
    ContentType.INCIDENT_FIELD: [
        "Employee ID",
        "employeeid",
        "Employee Number",
        "employeenumber",
        "Employee Type",
        "employeetype",
        "Employment Status",
        "employmentstatus",
        "Hire Date",
        "hiredate",
        "Last Day of Work",
        "lastdayofwork",
        "Prehire Flag",
        "prehireflag",
        "Rehired Employee",
        "rehiredemployee",
        "Termination Date",
        "terminationdate",
        "userprofile",
        "organization",
        "actor",
        "Termination Trigger",
        "terminationtrigger",
        "State Name",
        "statename",
        "profileid",
        "timezonesidkey",
        "localesidkey",
    ],
    ContentType.INCIDENT_TYPE: [
        "IAM - AD User Activation",
        "IAM - AD User Deactivation",
        "IAM - New Hire",
        "IAM - Rehire User",
        "IAM - Sync User",
        "IAM - Terminate User",
        "IAM - Update User",
        "User Profile - Create",
        "User Profile - Update",
        "User Profile",
        "IAM - App Add",
        "IAM - Group Membership Update",
        "IAM - App Remove",
        "IAM - App Update",
    ],
    ContentType.SCRIPT: [
        "IAM-Init-AD-User",
    ],
    ContentType.LAYOUT: [
        "MITRE Layout",
    ],
}


# demisto_sdk/commands/content_graph/content_graph_builder.py

import gc
from typing import List, Optional

import more_itertools

from demisto_sdk.commands.content_graph.common import Nodes, Relationships
from demisto_sdk.commands.content_graph.interface.graph import ContentGraphInterface
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
from demisto_sdk.commands.content_graph.parsers.repository import RepositoryParser

PACKS_PER_BATCH = 50


class ContentGraphBuilder:
    def __init__(self, content_graph: ContentGraphInterface) -> None:
        """Given a graph DB interface:
        1. Creates a repository model
        2. Collects all nodes and relationships from the model

        Args:
            content_graph (ContentGraphInterface): The interface to create the graph with.
        """
        self.content_graph = content_graph
        self.nodes: Nodes = Nodes()
        self.relationships: Relationships = Relationships()
        self._preprepare_database()

    def update_graph(
        self,
        packs_to_update: Optional[List[str]] = None,
    ) -> None:
        """Imports a content graph from files and updates the given pack nodes.

        Args:
            packs_to_update (Optional[List[str]]): A list of packs to update.
        """
        if not packs_to_update:
            return
        self._parse_and_model_content(packs_to_update)
        self._create_or_update_graph()

    def _preprepare_database(self) -> None:
        self.content_graph.clean_graph()
        self.content_graph.create_indexes_and_constraints()

    def _parse_and_model_content(
        self, packs_to_parse: Optional[List[str]] = None
    ) -> None:

        content_dtos: List[ContentDTO] = self._create_content_dtos(packs_to_parse)
        for content_dto in content_dtos:
            self._collect_nodes_and_relationships_from_model(content_dto)

    def _create_content_dtos(
        self, packs_to_parse: Optional[List[str]]
    ) -> List[ContentDTO]:
        """Parses the repository, then creates and returns a repository model.

        Args:
            path (Path): The repository path.
            packs_to_parse (Optional[List[str]]): A list of packs to parse. If not provided, parses all packs.
        """
        content_dtos = []
        repository_parser = RepositoryParser(self.content_graph.repo_path)
        for packs_batch in more_itertools.chunked_even(
            repository_parser.iter_packs(packs_to_parse), PACKS_PER_BATCH
        ):
            repository_parser.parse(packs_batch)
            content_dtos.append(ContentDTO.from_orm(repository_parser))
            repository_parser.clear()
            gc.collect()
        return content_dtos

    def _collect_nodes_and_relationships_from_model(
        self, content_dto: ContentDTO
    ) -> None:
        for pack in content_dto.packs:
            self.nodes.update(pack.to_nodes())
            self.relationships.update(pack.relationships)

    def create_graph(self) -> None:
        self._parse_and_model_content()
        self._create_or_update_graph()

    def _create_or_update_graph(self) -> None:
        """Runs DB queries using the collected nodes and relationships to create or update the content graph."""
        self.content_graph.create_nodes(self.nodes)
        self.content_graph.create_relationships(self.relationships)
        self.content_graph.remove_non_repo_items()


# demisto_sdk/commands/content_graph/neo4j_service.py

import hashlib
import os
import shutil
from pathlib import Path

import docker
import requests

from demisto_sdk.commands.common.content_constant_paths import CONTENT_PATH
from demisto_sdk.commands.common.docker_helper import init_global_docker_client
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.content_graph.common import (
    NEO4J_DATABASE_HTTP,
    NEO4J_FOLDER,
    NEO4J_PASSWORD,
)

REPO_PATH = CONTENT_PATH.absolute()
NEO4J_VERSION = "5.5.0"

NEO4J_SERVICE_IMAGE = f"neo4j:{NEO4J_VERSION}"

LOCAL_NEO4J_PATH = Path("/var/lib/neo4j")
NEO4J_IMPORT_FOLDER = "import"
NEO4J_DATA_FOLDER = "data"
NEO4J_PLUGINS_FOLDER = "plugins"

# When updating the APOC version, make sure to update the checksum as well
APOC_URL_VERSIONS = "https://neo4j.github.io/apoc/versions.json"


class Neo4jServiceException(Exception):
    pass


def _stop_neo4j_service_docker(docker_client: docker.DockerClient):
    """Helper function to stop the neo4j service docker container

    Args:
        docker_client (docker.DockerClient): The docker client to use
    """
    try:
        neo4j_docker = docker_client.containers.get("neo4j-content")
        if not neo4j_docker:
            return
        neo4j_docker.stop()
        neo4j_docker.remove(force=True)
    except Exception as e:
        logger.debug(f"Could not remove neo4j container: {e}")


def _is_apoc_available(plugins_path: Path, sha1: str) -> bool:
    for plugin in plugins_path.iterdir():
        if (
            plugin.name.startswith("apoc")
            and hashlib.sha1(plugin.read_bytes()).hexdigest() == sha1
        ):
            return True
    return False


def _download_apoc():
    apocs = [
        apoc
        for apoc in requests.get(APOC_URL_VERSIONS, verify=False).json()
        if apoc["neo4j"] == NEO4J_VERSION
    ]
    if not apocs:
        logger.debug(f"Could not find APOC for neo4j version {NEO4J_VERSION}")
        return
    download_url = apocs[0].get("downloadUrl")
    sha1 = apocs[0].get("sha1")
    plugins_folder = REPO_PATH / NEO4J_FOLDER / NEO4J_PLUGINS_FOLDER
    plugins_folder.mkdir(parents=True, exist_ok=True)

    if _is_apoc_available(plugins_folder, sha1):
        logger.debug("APOC is already available, skipping installation")
        return
    logger.info("Downloading APOC plugin, please wait...")
    # Download APOC_URL and save it to plugins folder in neo4j
    response = requests.get(download_url, verify=False, stream=True)

    with open(plugins_folder / "apoc.jar", "wb") as f:
        f.write(response.content)


def _docker_start():
    logger.debug("Starting neo4j service")
    docker_client = init_global_docker_client()
    _stop_neo4j_service_docker(docker_client)
    (REPO_PATH / NEO4J_FOLDER / NEO4J_DATA_FOLDER).mkdir(parents=True, exist_ok=True)
    (REPO_PATH / NEO4J_FOLDER / NEO4J_IMPORT_FOLDER).mkdir(parents=True, exist_ok=True)
    (REPO_PATH / NEO4J_FOLDER / NEO4J_PLUGINS_FOLDER).mkdir(parents=True, exist_ok=True)
    docker_client.containers.run(
        image=NEO4J_SERVICE_IMAGE,
        name="neo4j-content",
        ports={"7474/tcp": 7474, "7687/tcp": 7687, "7473/tcp": 7473},
        volumes=[
            f"{REPO_PATH / NEO4J_FOLDER / NEO4J_DATA_FOLDER}:/{NEO4J_DATA_FOLDER}",
            f"{REPO_PATH / NEO4J_FOLDER / NEO4J_IMPORT_FOLDER}:{LOCAL_NEO4J_PATH / NEO4J_IMPORT_FOLDER}",
            f"{REPO_PATH / NEO4J_FOLDER / NEO4J_PLUGINS_FOLDER}:/{NEO4J_PLUGINS_FOLDER}",
        ],
        detach=True,
        environment={
            "NEO4J_AUTH": f"neo4j/{NEO4J_PASSWORD}",
            "NEO4J_apoc_export_file_enabled": "true",
            "NEO4J_apoc_import_file_enabled": "true",
            "NEO4J_apoc_import_file_use__neo4j__config": "true",
            "NEO4J_dbms_security_procedures_unrestricted": "apoc.*",
            "NEO4J_dbms_security_procedures_allowlist": "apoc.*",
            "NEO4J_dbms_connector_http_advertised__address": "127.0.0.1:7474",
            "NEO4J_dbms_connector_bolt_advertised__address": "127.0.0.1:7687",
        },
        healthcheck={
            "test": f"curl --fail {NEO4J_DATABASE_HTTP} || exit 1",
            "interval": 5 * 1000000000,
            "timeout": 15 * 1000000000,
            "retries": 10,
        },
        user=f"{os.getuid()}:{os.getgid()}",
    )

    logger.debug("Neo4j service started successfully")


def start():
    """Starting the neo4j service

    Args:
    """
    if is_alive():
        logger.debug("Neo4j is already running")
        return
    if not is_running_on_docker():
        logger.debug("Neo4j is running locally. Start manually")
        return

    Path.mkdir(REPO_PATH / NEO4J_FOLDER, exist_ok=True, parents=True)
    # we download apoc only if we are running on docker
    # if the user is running locally he needs to setup apoc manually
    _download_apoc()
    try:
        _docker_start()
    except Exception as e:
        logger.debug(
            f"Could not start neo4j container, delete data folder and trying again. {e}"
        )
        shutil.rmtree(REPO_PATH / NEO4J_FOLDER / NEO4J_DATA_FOLDER, ignore_errors=True)
        _docker_start()


def stop(force: bool = False, clean: bool = False):
    """Stop the neo4j service"""
    if not force and not is_alive():
        return
    if not is_running_on_docker():
        logger.debug("Neo4j is running locally. Stop with `neo4j stop`")
        return
    docker_client = init_global_docker_client()
    _stop_neo4j_service_docker(docker_client)
    if clean:
        shutil.rmtree(REPO_PATH / NEO4J_FOLDER / NEO4J_DATA_FOLDER, ignore_errors=True)


def is_alive():
    try:
        return requests.get(NEO4J_DATABASE_HTTP, timeout=10).ok
    except requests.exceptions.RequestException:
        return False


def is_running_on_docker():
    return not LOCAL_NEO4J_PATH.exists()


def get_neo4j_import_path() -> Path:
    if not is_running_on_docker():
        return LOCAL_NEO4J_PATH / NEO4J_IMPORT_FOLDER
    return REPO_PATH / NEO4J_FOLDER / NEO4J_IMPORT_FOLDER


# demisto_sdk/commands/content_graph/tests/common_test.py

from demisto_sdk.commands.content_graph.common import ContentType


def test_content_type_does_not_contain_colon():
    """
    A colon is a special character in neo4j Cypher indicates when the label starts in the query, e.g.:
    `MATCH (n:Integration) return n`
    We want to make sure there are no colons in all content type values.
    """

    for content_type in ContentType:
        assert ":" not in content_type.value


# demisto_sdk/commands/content_graph/tests/create_content_graph_test.py

from pathlib import Path
from typing import Any, Dict, List, Tuple
from zipfile import ZipFile

import pytest

import demisto_sdk.commands.content_graph.neo4j_service as neo4j_service
from demisto_sdk.commands.common.constants import (
    SKIP_PREPARE_SCRIPT_NAME,
    MarketplaceVersions,
)
from demisto_sdk.commands.content_graph.commands.create import (
    create_content_graph,
)
from demisto_sdk.commands.content_graph.common import (
    ContentType,
    RelationshipType,
)
from demisto_sdk.commands.content_graph.interface import (
    ContentGraphInterface,
)
from demisto_sdk.commands.content_graph.objects import IncidentField, Layout, Mapper
from demisto_sdk.commands.content_graph.objects.classifier import Classifier
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem
from demisto_sdk.commands.content_graph.objects.integration import Command, Integration
from demisto_sdk.commands.content_graph.objects.integration_script import (
    IntegrationScript,
)
from demisto_sdk.commands.content_graph.objects.pack import Pack
from demisto_sdk.commands.content_graph.objects.playbook import Playbook
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
from demisto_sdk.commands.content_graph.objects.script import Script
from demisto_sdk.commands.content_graph.objects.test_playbook import TestPlaybook
from demisto_sdk.commands.content_graph.objects.widget import Widget
from demisto_sdk.commands.content_graph.tests.test_tools import load_json
from TestSuite.repo import Repo
from TestSuite.test_tools import ChangeCWD

# Fixtures for mock content object models


@pytest.fixture(autouse=True)
def setup_method(mocker, repo: Repo):
    """Auto-used fixture for setup before every test run"""
    import demisto_sdk.commands.content_graph.objects.base_content as bc

    bc.CONTENT_PATH = Path(repo.path)
    mocker.patch.object(ContentGraphInterface, "repo_path", Path(repo.path))
    mocker.patch.object(neo4j_service, "REPO_PATH", Path(repo.path))
    neo4j_service.stop()


@pytest.fixture
def repository(mocker):
    repository = ContentDTO(
        path=Path(),
        packs=[],
    )
    mocker.patch(
        "demisto_sdk.commands.content_graph.content_graph_builder.ContentGraphBuilder._create_content_dtos",
        return_value=[repository],
    )
    return repository


def mock_pack(
    name: str = "SamplePack",
    path: Path = Path("Packs"),
    repository: ContentDTO = None,
) -> Pack:
    pack = Pack(
        object_id=name,
        content_type=ContentType.PACK,
        node_id=f"{ContentType.PACK}:{name}",
        path=path,
        name=name,
        marketplaces=[MarketplaceVersions.XSOAR],
        hidden=False,
        server_min_version="5.5.0",
        current_version="1.0.0",
        tags=[],
        categories=[],
        useCases=[],
        keywords=[],
        contentItems=[],
        excluded_dependencies=[],
        deprecated=False,
    )
    if repository:
        repository.packs.append(pack)
    return pack


def mock_integration(
    name: str = "SampleIntegration",
    path: Path = Path("Packs"),
    pack: Pack = None,
    uses: List[Tuple[ContentItem, bool]] = None,
) -> Integration:
    integration = Integration(
        id=name,
        content_type=ContentType.INTEGRATION,
        node_id=f"{ContentType.INTEGRATION}:{name}",
        path=path,
        fromversion="5.0.0",
        toversion="99.99.99",
        display_name=name,
        name=name,
        marketplaces=[MarketplaceVersions.XSOAR],
        deprecated=False,
        type="python3",
        docker_image="mock:docker",
        category="blabla",
        commands=[Command(name="test-command", description="")],
    )
    if pack is not None:
        pack.content_items.integration.append(integration)
        build_in_pack_relationship(pack, integration)
        add_uses_relationships(pack, integration, uses)
    return integration


def mock_script(
    name: str = "SampleScript",
    marketplaces: List[MarketplaceVersions] = [MarketplaceVersions.XSOAR],
    skip_prepare: List[str] = [],
    path: Path = Path("Packs"),
    pack: Pack = None,
    uses: List[Tuple[ContentItem, bool]] = None,
    importing_items: List[ContentItem] = None,
) -> Script:
    script = Script(
        id=name,
        content_type=ContentType.SCRIPT,
        node_id=f"{ContentType.SCRIPT}:{name}",
        path=path,
        fromversion="5.0.0",
        display_name=name,
        toversion="99.99.99",
        name=name,
        marketplaces=marketplaces,
        deprecated=False,
        type="python3",
        docker_image="mock:docker",
        tags=[],
        is_test=False,
        skip_prepare=skip_prepare,
    )
    if pack:
        pack.content_items.script.append(script)
        build_in_pack_relationship(pack, script)
        add_uses_relationships(pack, script, uses)
        if importing_items is not None:
            # Note: `importing_items` won't necessarily be actually of `pack`,
            # But for mocking purposes it's enough.
            pack.relationships.add_batch(
                RelationshipType.IMPORTS,
                [
                    mock_relationship(
                        ci.object_id,
                        ci.content_type,
                        script.object_id,
                        script.content_type,
                    )
                    for ci in importing_items
                ],
            )

    return script


def mock_classifier(
    name: str = "SampleClassifier",
    path: Path = Path("Packs"),
    pack: Pack = None,
    uses: List[Tuple[ContentItem, bool]] = None,
) -> Classifier:
    classifier = Classifier(
        id=name,
        content_type=ContentType.CLASSIFIER,
        node_id=f"{ContentType.CLASSIFIER}:{name}",
        path=path,
        fromversion="5.0.0",
        display_name=name,
        toversion="99.99.99",
        name=name,
        marketplaces=[MarketplaceVersions.XSOAR],
        deprecated=False,
        type="python3",
        docker_image="mock:docker",
        tags=[],
        is_test=False,
    )
    if pack:
        pack.content_items.classifier.append(classifier)
        build_in_pack_relationship(pack, classifier)
        add_uses_relationships(pack, classifier, uses)
    return classifier


def mock_playbook(
    name: str = "SamplePlaybook",
    path: Path = Path("Packs"),
    marketplaces: List[MarketplaceVersions] = [MarketplaceVersions.XSOAR],
    pack: Pack = None,
    uses: List[Tuple[ContentItem, bool]] = None,
) -> Playbook:
    playbook = Playbook(
        id=name,
        content_type=ContentType.PLAYBOOK,
        node_id=f"{ContentType.PLAYBOOK}:{name}",
        path=path,
        fromversion="5.0.0",
        toversion="99.99.99",
        display_name=name,
        name=name,
        marketplaces=marketplaces,
        deprecated=False,
        is_test=False,
    )
    if pack:
        pack.content_items.playbook.append(playbook)
        build_in_pack_relationship(pack, playbook)
        add_uses_relationships(pack, playbook, uses)
    return playbook


def mock_test_playbook(
    name: str = "SampleTestPlaybook",
    path: Path = Path("Packs"),
    pack: Pack = None,
    uses: List[Tuple[ContentItem, bool]] = None,
    tested_items: List[ContentItem] = None,
) -> TestPlaybook:
    test_playbook = TestPlaybook(
        id=name,
        # content_type=ContentType.TEST_PLAYBOOK,
        node_id=f"{ContentType.PLAYBOOK}:{name}",
        path=path,
        fromversion="5.0.0",
        toversion="99.99.99",
        display_name=name,
        name=name,
        marketplaces=[MarketplaceVersions.XSOAR],
        deprecated=False,
        is_test=True,
    )
    if pack:
        pack.content_items.test_playbook.append(test_playbook)
        build_in_pack_relationship(pack, test_playbook)
        add_uses_relationships(pack, test_playbook, uses)
        if tested_items is not None:
            pack.relationships.add_batch(
                RelationshipType.TESTED_BY,
                [
                    mock_relationship(
                        ci.object_id,
                        ci.content_type,
                        test_playbook.object_id,
                        test_playbook.content_type,
                    )
                    for ci in tested_items
                ],
            )

    return test_playbook


def mock_widget(
    name: str = "SampleWidget",
    path: Path = Path("Packs"),
    pack: Pack = None,
    uses: List[Tuple[ContentItem, bool]] = None,
) -> Widget:
    widget = Widget(
        id=name,
        content_type=ContentType.WIDGET,
        node_id=f"{ContentType.WIDGET}:{name}",
        path=path,
        fromversion="5.0.0",
        toversion="99.99.99",
        display_name=name,
        name=name,
        marketplaces=[MarketplaceVersions.XSOAR],
        deprecated=False,
        widget_type="number",
        data_type="roi",
    )
    if pack:
        pack.content_items.widget.append(widget)
        build_in_pack_relationship(pack, widget)
        add_uses_relationships(pack, widget, uses)
    return widget


def mock_mapper(path: str, name: str = "SampleMapper", data: Dict = {}):
    return Mapper(
        id=name,
        content_type=ContentType.MAPPER,
        node_id=f"{ContentType.MAPPER}:{name}",
        path=path,
        fromversion="5.0.0",
        display_name=name,
        toversion="99.99.99",
        name=name,
        marketplaces=[MarketplaceVersions.XSOAR],
        deprecated=False,
        type="python3",
        docker_image="mock:docker",
        tags=[],
        is_test=False,
        data=data,
    )


def mock_layout(path: str, name: str = "SampleLayout", data: Dict = {}):
    return Layout(
        id=name,
        content_type=ContentType.LAYOUT,
        node_id=f"{ContentType.LAYOUT}:{name}",
        path=path,
        fromversion="5.0.0",
        display_name=name,
        toversion="99.99.99",
        name=name,
        marketplaces=[MarketplaceVersions.XSOAR],
        deprecated=False,
        type="python3",
        docker_image="mock:docker",
        tags=[],
        is_test=False,
        data=data,
        group="incident",
        edit=False,
        indicators_details=False,
        indicators_quick_view=False,
        quick_view=False,
        close=False,
        details=False,
        details_v2=True,
        mobile=False,
    )


def mock_incident_field(
    cli_name: str,
    path: str,
    marketplaces: List,
    data: Dict = {},
    name: str = "SampleIncidentField",
):
    return IncidentField(
        id=name,
        content_type=ContentType.INCIDENT_FIELD,
        node_id=f"{ContentType.INCIDENT_FIELD}:{name}",
        path=path,
        fromversion="5.0.0",
        display_name=name,
        toversion="99.99.99",
        name=name,
        marketplaces=marketplaces,
        deprecated=False,
        type="python3",
        docker_image="mock:docker",
        tags=[],
        is_test=False,
        data=data,
        cli_name=cli_name,
    )


# HELPERS


def mock_relationship(
    source: str,
    source_type: ContentType,
    target: str,
    target_type: ContentType,
    source_fromversion: str = "5.0.0",
    source_marketplaces: List[str] = [MarketplaceVersions.XSOAR],
    **kwargs,
) -> Dict[str, Any]:
    rel = {
        "source_id": source,
        "source_type": source_type,
        "source_fromversion": source_fromversion,
        "source_marketplaces": source_marketplaces,
        "target": target,
        "target_type": target_type,
    }
    rel.update(kwargs)
    return rel


def build_in_pack_relationship(
    pack: Pack,
    content_item: ContentItem,
) -> None:
    """Given a pack with content items,
    creates the "in pack" and "has command" basic relationships.

    Args:
        pack (Pack): a pack for which to build basic relationships.
    """
    pack.relationships.add(
        RelationshipType.IN_PACK,
        **mock_relationship(
            content_item.object_id,
            content_item.content_type,
            pack.object_id,
            ContentType.PACK,
        ),
    )
    if isinstance(content_item, Integration):
        for command in content_item.commands:
            pack.relationships.add(
                RelationshipType.HAS_COMMAND,
                **mock_relationship(
                    content_item.object_id,
                    content_item.content_type,
                    command.object_id,
                    ContentType.COMMAND,
                    description="",
                    deprecated=False,
                ),
            )


def add_uses_relationships(
    pack: Pack,
    content_item: ContentItem,
    used_content_items: List[Tuple[ContentItem, bool]] = None,
) -> None:
    if used_content_items is not None:
        pack.relationships.add_batch(
            RelationshipType.USES_BY_ID,
            [
                mock_relationship(
                    content_item.object_id,
                    content_item.content_type,
                    ci.object_id,
                    ci.content_type,
                    mandatorily=mandatorily,
                )
                for ci, mandatorily in used_content_items
            ],
        )


def find_model_for_id(packs: List[Pack], source_id: str):
    for pack in packs:
        if pack.object_id == source_id:
            return pack
        for content_item in pack.content_items:
            if content_item.object_id == source_id:
                return content_item
            if isinstance(content_item, Integration):
                for command in content_item.commands:
                    if command.name == source_id:
                        return command
    return None


def create_mini_content(repository: ContentDTO):
    """Created a content repo with three packs and relationships§

    Args:
        repository (ContentDTO): the content dto to populate
    """
    relationships = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SamplePack",
                ContentType.PACK,
            ),
            mock_relationship(
                "SampleScript",
                ContentType.SCRIPT,
                "SamplePack",
                ContentType.PACK,
            ),
        ],
        RelationshipType.HAS_COMMAND: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "test-command",
                ContentType.COMMAND,
                name="test-command",
                description="",
                deprecated=False,
            )
        ],
        RelationshipType.IMPORTS: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "TestApiModule",
                ContentType.SCRIPT,
            )
        ],
        RelationshipType.TESTED_BY: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SampleTestPlaybook",
                ContentType.TEST_PLAYBOOK,
            )
        ],
        RelationshipType.USES_BY_ID: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SampleClassifier",
                ContentType.CLASSIFIER,
                mandatorily=True,
            ),
        ],
    }
    relationship_pack2 = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SampleClassifier",
                ContentType.CLASSIFIER,
                "SamplePack2",
                ContentType.PACK,
            ),
            mock_relationship(
                "SampleTestPlaybook",
                ContentType.TEST_PLAYBOOK,
                "SamplePack2",
                ContentType.PACK,
            ),
            mock_relationship(
                "TestApiModule", ContentType.SCRIPT, "SamplePack2", ContentType.PACK
            ),
        ],
        RelationshipType.USES_BY_ID: [
            mock_relationship(
                "TestApiModule",
                ContentType.SCRIPT,
                "SampleScript2",
                ContentType.SCRIPT,
                mandatorily=True,
            ),
            mock_relationship(
                "SampleTestPlaybook",
                ContentType.TEST_PLAYBOOK,
                "SampleIntegration",
                ContentType.INTEGRATION,
            ),
        ],
    }
    relationship_pack3 = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SamplePlaybook",
                ContentType.PLAYBOOK,
                "SamplePack3",
                ContentType.PACK,
            ),
            mock_relationship(
                "SampleScript2",
                ContentType.SCRIPT,
                "SamplePack3",
                ContentType.PACK,
            ),
        ]
    }
    pack1 = mock_pack()
    pack2 = mock_pack("SamplePack2")
    pack3 = mock_pack("SamplePack3")
    pack1.relationships = relationships
    pack2.relationships = relationship_pack2
    pack3.relationships = relationship_pack3
    pack1.content_items.integration.append(mock_integration())
    pack1.content_items.script.append(mock_script())
    pack2.content_items.script.append(mock_script("TestApiModule"))
    pack2.content_items.classifier.append(mock_classifier())
    pack2.content_items.test_playbook.append(mock_test_playbook())
    pack3.content_items.playbook.append(mock_playbook())
    pack3.content_items.script.append(mock_script("SampleScript2"))
    repository.packs.extend([pack1, pack2, pack3])


class TestCreateContentGraph:
    def test_create_content_graph_end_to_end(self, repo: Repo, tmp_path: Path, mocker):
        """
        Given:
            - A repository with a pack TestPack, containing an integration TestIntegration.
        When:
            - Running create_content_graph()
        Then:
            - Make sure the service remains available by querying for all content items in the graph.
            - Make sure there is a single integration in the query response.
        """
        mocker.patch.object(
            IntegrationScript, "get_supported_native_images", return_value=[]
        )

        pack = repo.create_pack("TestPack")
        pack.pack_metadata.write_json(load_json("pack_metadata.json"))
        integration = pack.create_integration()
        integration.create_default_integration(
            "TestIntegration", ["test-command1", "test-command2"]
        )
        script = pack.create_script()
        api_module = pack.create_script()
        script.create_default_script("SampleScript")
        api_module.create_default_script("TestApiModule")

        pack.create_classifier(
            name="SampleClassifier", content=load_json("classifier.json")
        )

        with ContentGraphInterface() as interface:
            create_content_graph(interface, output_path=tmp_path)
            packs = interface.search(
                marketplace=MarketplaceVersions.XSOAR, content_type=ContentType.PACK
            )
            integrations = interface.search(
                marketplace=MarketplaceVersions.XSOAR,
                content_type=ContentType.INTEGRATION,
            )
            all_content_items = interface.search(marketplace=MarketplaceVersions.XSOAR)
            content_cto = interface.marshal_graph(MarketplaceVersions.XSOAR)
        assert len(packs) == 1
        assert len(integrations) == 1
        assert len(all_content_items) == 8
        returned_pack = packs[0]
        assert returned_pack.object_id == "TestPack"
        # make sure that data from pack_metadata.json updated
        assert returned_pack.name == "HelloWorld"
        assert returned_pack.support == "community"
        assert len(packs[0].content_items.integration) == 1
        returned_integration = packs[0].content_items.integration[0]
        assert returned_integration == integrations[0]
        assert returned_integration.name == "TestIntegration"
        assert {command.name for command in returned_integration.commands} == {
            "test-command",
            "test-command1",
            "test-command2",
        }
        returned_scripts = {
            script.object_id for script in packs[0].content_items.script
        }
        assert returned_scripts == {"SampleScript", "TestApiModule"}
        with ChangeCWD(repo.path):
            content_cto.dump(tmp_path, MarketplaceVersions.XSOAR, zip=False)
        assert (tmp_path / "TestPack").exists()
        assert (tmp_path / "TestPack" / "metadata.json").exists()
        assert (
            tmp_path / "TestPack" / "Integrations" / "integration-integration_0.yml"
        ).exists()
        assert (tmp_path / "TestPack" / "Scripts" / "script-script0.yml").exists()
        assert (tmp_path / "TestPack" / "Scripts" / "script-script1.yml").exists()

        # make sure that the output file zip is created
        assert Path.exists(tmp_path / "xsoar.zip")
        with ZipFile(tmp_path / "xsoar.zip", "r") as zip_obj:
            zip_obj.extractall(tmp_path / "extracted")
            # make sure that the extracted files are all .csv
            extracted_files = list(tmp_path.glob("extracted/*"))
            assert extracted_files
            assert all(
                file.suffix == ".graphml" or file.name == "metadata.json"
                for file in extracted_files
            )

    def test_create_content_graph_relationships(
        self,
        repository: ContentDTO,
    ):
        """
        Given:
            - A mocked model of a repository with a pack TestPack, containing:
              - An integration SampleIntegration, which:
                1. Has a single command test-command.
                2. Imports TestApiModule in the code.
                3. Is tested by SampleTestPlaybook.
                4. A default classifier SampleClassifier.
              - A script SampleScript that uses SampleScript2.
        When:
            - Running create_content_graph().
        Then:
            - Make sure the graph has all the corresponding nodes and relationships.
        """
        create_mini_content(repository)
        with ContentGraphInterface() as interface:
            create_content_graph(interface)
            packs = interface.search(
                marketplace=MarketplaceVersions.XSOAR, content_type=ContentType.PACK
            )
            for pack in repository.packs:
                for relationship_type, relationships in pack.relationships.items():
                    for relationship in relationships:
                        content_item_source = find_model_for_id(
                            packs, relationship.get("source_id")
                        )
                        content_item_target = find_model_for_id(
                            packs, relationship.get("target")
                        )
                        assert content_item_source
                        assert content_item_target
                        if relationship_type == RelationshipType.IN_PACK:
                            assert (
                                content_item_source.in_pack.object_id == pack.object_id
                            )
                        if relationship_type == RelationshipType.IMPORTS:
                            assert (
                                content_item_source.imports[0].object_id
                                == content_item_target.object_id
                            )
                        if relationship_type == RelationshipType.USES_BY_ID:
                            assert (
                                content_item_source.uses[0].content_item_to.object_id
                                == content_item_target.object_id
                            )
                        if relationship_type == RelationshipType.TESTED_BY:
                            assert (
                                content_item_source.tested_by[0].object_id
                                == content_item_target.object_id
                            )

            assert packs[0].depends_on[0].content_item_to == packs[1]
            assert not packs[0].depends_on[0].is_test  # this is not a test dependency

            for p in packs[1].depends_on:
                if p.content_item_to == packs[2]:
                    # regular dependency
                    assert not p.is_test
                elif p.content_item_to == packs[0]:
                    # test dependency
                    assert p.is_test
                else:
                    assert False

            # now with all levels
            packs = interface.search(
                MarketplaceVersions.XSOAR,
                content_type=ContentType.PACK,
                all_level_dependencies=True,
            )
            depends_on_pack1 = [r for r in packs[0].depends_on]
            assert depends_on_pack1
            for depends in depends_on_pack1:
                if depends.content_item_to == packs[1]:
                    assert depends.is_direct
                elif depends.content_item_to == packs[2]:
                    assert not depends.is_direct
                else:
                    assert False

    def test_create_content_graph_two_integrations_with_same_command(
        self,
        repository: ContentDTO,
    ):
        """
        Given:
            - A mocked model of a repository with a pack TestPack, containing two integrations,
              each has a command named test-command.
        When:
            - Running create_content_graph().
        Then:
            - Make sure only one command node was created.
        """
        pack = mock_pack()
        integration1 = mock_integration()
        integration2 = mock_integration("SampleIntegration2")
        integration2.name = integration2.object_id = "SampleIntegration2"
        integration2.node_id = f"{ContentType.INTEGRATION}:{integration2.object_id}"

        relationships = {
            RelationshipType.IN_PACK: [
                mock_relationship(
                    "SampleIntegration",
                    ContentType.INTEGRATION,
                    "SamplePack",
                    ContentType.PACK,
                ),
                mock_relationship(
                    "SampleIntegration2",
                    ContentType.INTEGRATION,
                    "SamplePack",
                    ContentType.PACK,
                ),
            ],
            RelationshipType.HAS_COMMAND: [
                mock_relationship(
                    "SampleIntegration",
                    ContentType.INTEGRATION,
                    "test-command",
                    ContentType.COMMAND,
                    name="test-command",
                    description="",
                    deprecated=False,
                ),
                mock_relationship(
                    "SampleIntegration2",
                    ContentType.INTEGRATION,
                    "test-command",
                    ContentType.COMMAND,
                    name="test-command",
                    description="",
                    deprecated=False,
                ),
            ],
        }
        pack.relationships = relationships
        pack.content_items.integration.append(integration1)
        pack.content_items.integration.append(integration2)
        repository.packs.append(pack)
        with ContentGraphInterface() as interface:
            create_content_graph(interface)
            assert interface.search(
                MarketplaceVersions.XSOAR, object_id="SampleIntegration"
            )
            assert interface.search(
                MarketplaceVersions.XSOAR, object_id="SampleIntegration2"
            )
            assert (
                len(
                    interface.search(
                        MarketplaceVersions.XSOAR, content_type=ContentType.COMMAND
                    )
                )
                == 1
            )

    def test_create_content_graph_playbook_uses_script_not_in_repository(
        self,
        repository: ContentDTO,
    ):
        """
        Given:
            - A mocked model of a repository with a pack TestPack, containing a playbook tha
              wasn't parsed, meaning it's not in the repository.
        When:
            - Running create_content_graph().
        Then:
            - Make sure the script has the boolean property "not_in_repository".
        """
        relationships = {
            RelationshipType.IN_PACK: [
                mock_relationship(
                    "SamplePlaybook",
                    ContentType.PLAYBOOK,
                    "SamplePack",
                    ContentType.PACK,
                ),
            ],
            RelationshipType.USES_BY_ID: [
                mock_relationship(
                    "SamplePlaybook",
                    ContentType.PLAYBOOK,
                    "TestScript",
                    ContentType.SCRIPT,
                ),
            ],
        }
        pack = mock_pack()
        pack.relationships = relationships
        pack.content_items.playbook.append(mock_playbook())
        repository.packs.append(pack)
        with ContentGraphInterface() as interface:
            create_content_graph(interface)
            script = interface.search(object_id="TestScript")[0]
        assert script.not_in_repository

    def test_create_content_graph_duplicate_widgets(
        self,
        repository: ContentDTO,
    ):
        """
        Given:
            - A mocked model of a repository with a pack TestPack, containing two widgets
              with the exact same id, fromversion and marketplaces properties.
        When:
            - Running create_content_graph().
        Then:
            - Make sure both widgets exist in the graph.
        """
        pack = mock_pack()
        widget = mock_widget()
        widget2 = mock_widget()
        relationships = {
            RelationshipType.IN_PACK: [
                mock_relationship(
                    "SampleWidget",
                    ContentType.WIDGET,
                    "SamplePack",
                    ContentType.PACK,
                ),
                mock_relationship(
                    "SampleWidget",
                    ContentType.WIDGET,
                    "SamplePack",
                    ContentType.PACK,
                ),
            ],
        }
        pack.relationships = relationships
        pack.content_items.widget.append(widget)
        pack.content_items.widget.append(widget2)
        repository.packs.append(pack)
        with ContentGraphInterface() as interface:
            create_content_graph(interface)
            assert len(interface.search(object_id="SampleWidget")) == 2

    def test_create_content_graph_duplicate_integrations_different_marketplaces(
        self,
        repository: ContentDTO,
    ):
        """
        Given:
            - A mocked model of a repository with a pack TestPack, containing two integrations
              with the exact same properties but they are from different markletplaces.
        When:
            - Running create_content_graph().
        Then:
            - Make sure the the integrations are not recognized as duplicates and the command succeeds.
        """
        pack = mock_pack()
        integration = mock_integration()
        integration2 = mock_integration()
        integration2.marketplaces = [MarketplaceVersions.MarketplaceV2]
        relationships = {
            RelationshipType.IN_PACK: [
                mock_relationship(
                    "SampleIntegration",
                    ContentType.INTEGRATION,
                    "SamplePack",
                    ContentType.PACK,
                ),
                mock_relationship(
                    "SampleIntegration",
                    ContentType.INTEGRATION,
                    "SamplePack",
                    ContentType.PACK,
                    source_marketplaces=[MarketplaceVersions.MarketplaceV2],
                ),
            ],
            RelationshipType.HAS_COMMAND: [
                mock_relationship(
                    "SampleIntegration",
                    ContentType.INTEGRATION,
                    "test-command",
                    ContentType.COMMAND,
                    name="test-command",
                    description="",
                    deprecated=False,
                ),
                mock_relationship(
                    "SampleIntegration",
                    ContentType.INTEGRATION,
                    "test-command",
                    ContentType.COMMAND,
                    name="test-command",
                    description="",
                    deprecated=False,
                    source_marketplaces=[MarketplaceVersions.MarketplaceV2],
                ),
            ],
        }
        pack.relationships = relationships
        pack.content_items.integration.append(integration)
        pack.content_items.integration.append(integration2)
        repository.packs.append(pack)
        with ContentGraphInterface() as interface:
            create_content_graph(interface)
            assert len(interface.search(object_id="SampleIntegration")) == 2
            assert (
                len(
                    interface.search(
                        MarketplaceVersions.XSOAR, object_id="SampleIntegration"
                    )
                )
                == 1
            )
            assert (
                len(
                    interface.search(
                        MarketplaceVersions.MarketplaceV2, object_id="SampleIntegration"
                    )
                )
                == 1
            )

    def test_create_content_graph_duplicate_integrations_different_fromversion(
        self,
        repository: ContentDTO,
    ):
        """
        Given:
            - A mocked model of a repository with a pack TestPack, containing two integrations
              with the exact same properties but have different version ranges.
        When:
            - Running create_content_graph().
        Then:
            - Make sure the the integrations are not recognized as duplicates and the command succeeds.
        """
        pack = mock_pack()
        integration = mock_integration()
        integration2 = mock_integration()
        integration.toversion = "6.0.0"
        integration2.fromversion = "6.0.2"
        relationships = {
            RelationshipType.IN_PACK: [
                mock_relationship(
                    "SampleIntegration",
                    ContentType.INTEGRATION,
                    "SamplePack",
                    ContentType.PACK,
                ),
                mock_relationship(
                    "SampleIntegration",
                    ContentType.INTEGRATION,
                    "SamplePack",
                    ContentType.PACK,
                    source_fromversion="6.0.2",
                ),
            ],
            RelationshipType.HAS_COMMAND: [
                mock_relationship(
                    "SampleIntegration",
                    ContentType.INTEGRATION,
                    "test-command",
                    ContentType.COMMAND,
                    name="test-command",
                    description="",
                    deprecated=False,
                ),
                mock_relationship(
                    "SampleIntegration",
                    ContentType.INTEGRATION,
                    "test-command",
                    ContentType.COMMAND,
                    name="test-command",
                    description="",
                    deprecated=False,
                    source_fromversion="6.0.2",
                ),
            ],
        }
        pack.relationships = relationships
        pack.content_items.integration.append(integration)
        pack.content_items.integration.append(integration2)
        repository.packs.append(pack)
        with ContentGraphInterface() as interface:
            create_content_graph(interface)
            assert len(interface.search(object_id="SampleIntegration")) == 2

    def test_create_content_graph_empty_repository(
        self,
    ):
        """
        Given:
            - A mocked model of an empty repository.
        When:
            - Running create_content_graph().
        Then:
            - Make sure the graph contains server items.
            - Make sure all nodes in the graph are server items.
        """
        with ContentGraphInterface() as interface:
            create_content_graph(interface)
            assert not interface.search()

    def test_stop_content_graph(self):
        """
        Given:
            - A running content graph service.
        When:
            - Running neo4j_service.stop()
        Then:
            - Make sure no exception is raised.
        """
        neo4j_service.stop()

    def test_create_content_graph_incident_to_alert_scripts(
        self, repo: Repo, tmp_path: Path, mocker
    ):
        """
        Given:
            - A repository with a pack TestPack, containing two scripts,
            one (getIncident) is set with skipping the preparation of incident to alert
            and the other (setIncident) is not.
        When:
            - Running create_content_graph().
        Then:
            - Ensure that `getIncident` script has passed the prepare process as expected.
            - Ensure the 'setIncident' script has not passed incident to alert preparation.
        """
        mocker.patch.object(
            IntegrationScript, "get_supported_native_images", return_value=[]
        )

        pack = repo.create_pack("TestPack")
        pack.pack_metadata.write_json(load_json("pack_metadata.json"))
        pack.create_script(name="getIncident")
        pack.create_script(name="setIncident", skip_prepare=[SKIP_PREPARE_SCRIPT_NAME])

        with ContentGraphInterface() as interface:
            create_content_graph(interface, output_path=tmp_path)
            packs = interface.search(
                marketplace=MarketplaceVersions.MarketplaceV2,
                content_type=ContentType.PACK,
            )
            scripts = interface.search(
                marketplace=MarketplaceVersions.MarketplaceV2,
                content_type=ContentType.SCRIPT,
            )
            all_content_items = interface.search(
                marketplace=MarketplaceVersions.MarketplaceV2
            )
            content_cto = interface.marshal_graph(MarketplaceVersions.MarketplaceV2)

        assert len(packs) == 1
        assert len(scripts) == 2
        assert len(all_content_items) == 3
        with ChangeCWD(repo.path):
            content_cto.dump(tmp_path, MarketplaceVersions.MarketplaceV2, zip=False)
        script_path = tmp_path / "TestPack" / "Scripts"
        assert (script_path / "script-getIncident.yml").exists()
        assert (script_path / "script-getAlert.yml").exists()
        assert (script_path / "script-setIncident.yml").exists()
        assert not (script_path / "script-setAlert.yml").exists()

    def test_create_content_graph_relationships_from_metadata(
        self,
        repo: Repo,
    ):
        """
        Given:
            - A mocked model of a repository with a pack Core, which depends on NonCorePack according to the pack metadata
        When:
            - Running create_content_graph().
        Then:
            - Make sure the relationship's is_test is not null.
        """
        core_metadata = load_json("pack_metadata.json")
        core_metadata["name"] = "Core"
        core_metadata["dependencies"].update(
            {"NonCorePack": {"mandatory": True, "display_name": "Non Core Pack"}}
        )
        pack_core = repo.create_pack("Core")
        repo.create_pack("NonCorePack")
        pack_core.pack_metadata.write_json(core_metadata)

        with ContentGraphInterface() as interface:
            create_content_graph(interface)

            data = interface.run_single_query(
                "MATCH p=()-[r:DEPENDS_ON]->() WHERE r.is_test IS NULL RETURN p"
            )

            assert not data


# demisto_sdk/commands/content_graph/tests/format_with_graph_test.py

import logging
from pathlib import Path
from typing import List

import pytest
from click.testing import CliRunner

from demisto_sdk.__main__ import main
from demisto_sdk.commands.common.constants import (
    MarketplaceVersions,
)
from demisto_sdk.commands.common.tools import get_dict_from_file
from demisto_sdk.commands.content_graph import neo4j_service
from demisto_sdk.commands.content_graph.commands.create import create_content_graph
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.interface.neo4j.neo4j_graph import (
    Neo4jContentGraphInterface as ContentGraphInterface,
)
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
from demisto_sdk.commands.content_graph.tests.create_content_graph_test import (
    mock_incident_field,
    mock_integration,
    mock_layout,
    mock_mapper,
    mock_pack,
    mock_relationship,
)
from demisto_sdk.commands.content_graph.tests.update_content_graph_test import (
    _get_pack_by_id,
)
from TestSuite.test_tools import ChangeCWD, str_in_call_args_list

FORMAT_CMD = "format"


@pytest.fixture(autouse=True)
def setup_method(mocker, repo):
    """Auto-used fixture for setup before every test run"""
    import demisto_sdk.commands.content_graph.objects.base_content as bc

    bc.CONTENT_PATH = Path(repo.path)
    mocker.patch.object(neo4j_service, "REPO_PATH", Path(repo.path))
    mocker.patch.object(ContentGraphInterface, "repo_path", Path(repo.path))
    neo4j_service.stop()


@pytest.fixture
def repository(mocker, repo) -> ContentDTO:
    repository = ContentDTO(
        path=Path(repo.path),
        packs=[],
    )
    relationships = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SamplePack",
                ContentType.PACK,
            ),
            mock_relationship(
                "Mapper - Incoming Mapper",
                ContentType.MAPPER,
                "SamplePack",
                ContentType.PACK,
            ),
            mock_relationship(
                "Unknown Incident Field",
                ContentType.INCIDENT_FIELD,
                "SamplePack",
                ContentType.PACK,
            ),
            mock_relationship(
                "Layout",
                ContentType.LAYOUT,
                "SamplePack",
                ContentType.PACK,
            ),
            mock_relationship(
                "incidentfield",
                ContentType.INCIDENT_FIELD,
                "SamplePack",
                ContentType.PACK,
            ),
        ],
        RelationshipType.USES: [
            mock_relationship(
                "Mapper - Incoming Mapper",
                ContentType.MAPPER,
                "Unknown Incident Field",
                ContentType.INCIDENT_FIELD,
                name="Unknown Incident Field",
                description="",
                deprecated=False,
            ),
            mock_relationship(
                "Layout",
                ContentType.LAYOUT,
                "Unknown Incident Field",
                ContentType.INCIDENT_FIELD,
                name="Unknown Incident Field",
                description="",
                deprecated=False,
            ),
            mock_relationship(
                "Layout",
                ContentType.LAYOUT,
                "incidentfield",
                ContentType.INCIDENT_FIELD,
                name="Known Incident Field",
                description="",
                deprecated=False,
            ),
        ],
    }

    mapper_data = {
        "description": "",
        "feed": False,
        "id": "Mapper - Incoming Mapper",
        "mapping": {
            "Mapper Finding": {
                "dontMapEventToLabels": True,
                "internalMapping": {"Unknown Incident Field": {"simple": "Item"}},
            },
        },
        "name": "Mapper - Incoming Mapper",
        "type": "mapping-incoming",
        "version": -1,
    }
    layout_content = {
        "detailsV2": {
            "tabs": [
                {
                    "id": "caseinfoid",
                    "name": "Incident Info",
                    "sections": [
                        {
                            "displayType": "ROW",
                            "h": 2,
                            "i": "caseinfoid-fce71720-98b0-11e9-97d7-ed26ef9e46c8",
                            "isVisible": True,
                            "items": [
                                {
                                    "endCol": 2,
                                    "fieldId": "Unknown Incident Field",
                                    "height": 22,
                                    "id": "id1",
                                    "index": 0,
                                    "sectionItemType": "field",
                                    "startCol": 0,
                                },
                                {
                                    "endCol": 2,
                                    "fieldId": "incidentfield",
                                    "height": 22,
                                    "id": "id2",
                                    "index": 0,
                                    "sectionItemType": "field",
                                    "startCol": 0,
                                },
                            ],
                        }
                    ],
                }
            ]
        },
        "group": "incident",
        "id": "Layout",
        "name": "Layout",
        "system": False,
        "version": -1,
        "description": "",
        "marketplaces": ["xsoar"],
    }
    incident_field_content = {
        "cliName": "incidentfield",
        "id": "incident_incidentfield",
        "name": "Incident Field",
        "marketplaces": [MarketplaceVersions.XSOAR],
    }
    original_incident_field_content = {
        "cliName": "originalincidentfield",
        "id": "incident_originalincidentfield",
        "name": "Original Incident Field",
        "Aliases": [
            {
                "cliName": "alias1incidentfield",
                "type": "shortText",
                "name": "Alias1 Incident Field",
            },
            {
                "cliName": "alias2incidentfield",
                "type": "shortText",
                "name": "Alias2 Incident Field",
            },
        ],
        "marketplaces": [MarketplaceVersions.MarketplaceV2],
    }
    alias1_incident_field_content = {
        "cliName": "alias1incidentfield",
        "id": "incident_alias1incidentfield",
        "name": "Alias1 Incident Field",
        "marketplaces": [MarketplaceVersions.MarketplaceV2, MarketplaceVersions.XSOAR],
    }
    alias2_incident_field_content = {
        "cliName": "alias2incidentfield",
        "id": "incident_alias2incidentfield",
        "name": "Alias2 Incident Field",
        "marketplaces": [MarketplaceVersions.XSOAR],
    }
    pack_1 = repo.create_pack("SamplePack")
    mapper = pack_1.create_mapper(name="mapper", content=mapper_data)
    layout = pack_1.create_layoutcontainer(name="Layout", content=layout_content)
    incident_field = pack_1.create_incident_field(
        name="incidentfield", content=incident_field_content
    )

    pack_2 = repo.create_pack("SamplePack2")
    original_incident_field = pack_2.create_incident_field(
        name="originalincidentfield", content=original_incident_field_content
    )
    alias1_incident_field = pack_2.create_incident_field(
        name="alias1incidentfield", content=alias1_incident_field_content
    )
    alias2_incident_field = pack_2.create_incident_field(
        name="alias2incidentfield", content=alias2_incident_field_content
    )

    pack1 = mock_pack()
    pack1.relationships = relationships
    pack1.content_items.integration.append(mock_integration())
    pack1.content_items.mapper.append(
        mock_mapper(name="Mapper - Incoming Mapper", data=mapper_data, path=mapper.path)
    )
    pack1.content_items.layout.append(
        mock_layout(name="Layout", data=layout_content, path=layout.path)
    )
    pack1.content_items.incident_field.append(
        mock_incident_field(
            cli_name="incidentfield",
            path=incident_field.path,
            data=incident_field_content,
            name="incidentfield",
            marketplaces=[MarketplaceVersions.XSOAR],
        )
    )
    pack2 = mock_pack("SamplePack2")
    pack2.content_items.incident_field.append(
        mock_incident_field(
            cli_name="originalincidentfield",
            path=original_incident_field.path,
            data=original_incident_field_content,
            name="originalincidentfield",
            marketplaces=[MarketplaceVersions.MarketplaceV2],
        )
    )
    pack2.content_items.incident_field.append(
        mock_incident_field(
            cli_name="alias1incidentfield",
            path=alias1_incident_field.path,
            data=alias1_incident_field_content,
            name="alias1incidentfield",
            marketplaces=[MarketplaceVersions.MarketplaceV2, MarketplaceVersions.XSOAR],
        )
    )
    pack2.content_items.incident_field.append(
        mock_incident_field(
            cli_name="alias2incidentfield",
            path=alias2_incident_field.path,
            data=alias2_incident_field_content,
            name="alias2incidentfield",
            marketplaces=[MarketplaceVersions.XSOAR],
        )
    )
    repository.packs.extend([pack1, pack2])

    def mock__create_content_dto(packs_to_update: List[str]) -> List[ContentDTO]:
        if not packs_to_update:
            return [repository]
        repo_copy = repository.copy()
        repo_copy.packs = [p for p in repo_copy.packs if p.object_id in packs_to_update]
        return [repo_copy]

    mocker.patch(
        "demisto_sdk.commands.content_graph.content_graph_builder.ContentGraphBuilder._create_content_dtos",
        side_effect=mock__create_content_dto,
    )

    return repository


def test_format_mapper_with_graph_remove_unknown_content(
    mocker, monkeypatch, repository, repo
):
    """
    Given
    - A mapper.
    When
    - Running format command on it
    Then
    -  Ensure that the unknown field was removed from the mapper.
    """

    with ContentGraphInterface() as interface:
        create_content_graph(interface)

    logger_info = mocker.patch.object(logging.getLogger("demisto-sdk"), "info")
    monkeypatch.setenv("COLUMNS", "1000")

    pack_graph_object = _get_pack_by_id(repository, "SamplePack")
    mapper_graph_object = pack_graph_object.content_items.mapper[0]
    mapper_path = str(mapper_graph_object.path)
    mocker.patch(
        "demisto_sdk.commands.format.format_module.ContentGraphInterface",
        return_value=interface,
    )
    mocker.patch(
        "demisto_sdk.commands.format.format_module.update_content_graph",
        return_value=interface,
    )
    with ChangeCWD(repo.path):
        runner = CliRunner()
        result = runner.invoke(
            main, [FORMAT_CMD, "-i", mapper_path, "-at", "-y", "-nv"]
        )
    message = (
        "Removing the fields {'Unknown Incident Field'}"
        + f" from the mapper {mapper_path} "
        f"because they aren't in the content repo."
    )
    assert result.exit_code == 0
    assert not result.exception
    assert str_in_call_args_list(logger_info.call_args_list, message)

    # get_dict_from_file returns a tuple of 2 object. The first is the content of the file,
    # the second is the type of the file.
    file_content = get_dict_from_file(mapper_path)[0]
    assert (
        file_content.get("mapping", {}).get("Mapper Finding", {}).get("internalMapping")
        == {}
    )


def test_format_layout_with_graph_remove_unknown_content(
    mocker, monkeypatch, repository, repo
):
    """
    Given
    - A layout.
    When
    - Running format command on it
    Then
    -  Ensure that the unknown field was removed from the layout.
    """

    with ContentGraphInterface() as interface:
        create_content_graph(interface)

    logger_info = mocker.patch.object(logging.getLogger("demisto-sdk"), "info")
    monkeypatch.setenv("COLUMNS", "1000")

    pack_graph_object = _get_pack_by_id(repository, "SamplePack")
    layout_graph_object = pack_graph_object.content_items.layout[0]
    layout_path = str(layout_graph_object.path)
    mocker.patch(
        "demisto_sdk.commands.format.format_module.ContentGraphInterface",
        return_value=interface,
    )
    mocker.patch(
        "demisto_sdk.commands.format.format_module.update_content_graph",
        return_value=interface,
    )
    with ChangeCWD(repo.path):
        runner = CliRunner()
        result = runner.invoke(
            main, [FORMAT_CMD, "-i", layout_path, "-at", "-y", "-nv"]
        )
    message = (
        "Removing the fields {'Unknown Incident Field'}"
        + f" from the layout {layout_path} "
        f"because they aren't in the content repo."
    )
    assert result.exit_code == 0
    assert not result.exception
    assert str_in_call_args_list(logger_info.call_args_list, message)

    # get_dict_from_file returns a tuple of 2 object. The first is the content of the file,
    # the second is the type of the file.
    file_content = get_dict_from_file(layout_path)[0]
    expected_layout_content = [
        {
            "endCol": 2,
            "fieldId": "incidentfield",
            "height": 22,
            "id": "id2",
            "index": 0,
            "sectionItemType": "field",
            "startCol": 0,
        }
    ]
    assert (
        file_content.get("detailsV2", {})
        .get("tabs", [])[0]
        .get("sections", {})[0]
        .get("items", [])
        == expected_layout_content
    )


def test_format_incident_field_graph_fix_aliases_marketplace(
    mocker, monkeypatch, repository, repo
):
    """
    Given
    - An incident field.
    When
    - Running format command on it
    Then
    -  Ensure that the aliases incident fields marketplaces field contains only xsoar.
    """

    with ContentGraphInterface() as interface:
        create_content_graph(interface)

    monkeypatch.setenv("COLUMNS", "1000")

    pack_graph_object = _get_pack_by_id(repository, "SamplePack2")
    original_incident_field_path = str(
        pack_graph_object.content_items.incident_field[0].path
    )  # original incident field
    alias1_incident_field_path = str(
        pack_graph_object.content_items.incident_field[1].path
    )
    alias2_incident_field_path = str(
        pack_graph_object.content_items.incident_field[2].path
    )
    mocker.patch(
        "demisto_sdk.commands.format.format_module.ContentGraphInterface",
        return_value=interface,
    )
    mocker.patch(
        "demisto_sdk.commands.format.format_module.update_content_graph",
        return_value=interface,
    )
    with ChangeCWD(repo.path):
        runner = CliRunner()
        result = runner.invoke(
            main, [FORMAT_CMD, "-i", original_incident_field_path, "-at", "-y", "-nv"]
        )

    assert result.exit_code == 0
    assert not result.exception

    # get_dict_from_file returns a tuple of 2 object. The first is the content of the file,
    # the second is the type of the file.
    alias1_content = get_dict_from_file(alias1_incident_field_path)[0]
    alias2_content = get_dict_from_file(alias2_incident_field_path)[0]
    assert alias1_content.get("marketplaces", []) == ["xsoar"]
    assert alias2_content.get("marketplaces", []) == ["xsoar"]


# demisto_sdk/commands/content_graph/tests/generate_docs_script_test.py

import os
from pathlib import Path
from typing import List

import pytest

from demisto_sdk.commands.common.tests.docker_test import FILES_PATH
from demisto_sdk.commands.content_graph import neo4j_service
from demisto_sdk.commands.content_graph.commands.create import create_content_graph
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.interface import (
    ContentGraphInterface,
)
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
from demisto_sdk.commands.content_graph.tests.create_content_graph_test import (
    mock_pack,
    mock_playbook,
    mock_relationship,
    mock_script,
)
from demisto_sdk.commands.content_graph.tests.update_content_graph_test import (
    _get_pack_by_id,
)
from demisto_sdk.commands.generate_docs.generate_script_doc import generate_script_doc
from demisto_sdk.commands.generate_docs.tests.generate_docs_test import handle_example

INPUT_SCRIPT = "SampleScript"
USES_SCRIPT = "UsesScript"
USED_BY_PLAYBOOK = "SamplePlaybook"

# FIXTURES


@pytest.fixture(autouse=True)
def setup_method(mocker, repo):
    """Auto-used fixture for setup before every test run"""
    import demisto_sdk.commands.content_graph.objects.base_content as bc

    bc.CONTENT_PATH = Path(repo.path)
    mocker.patch.object(neo4j_service, "REPO_PATH", Path(repo.path))
    mocker.patch.object(ContentGraphInterface, "repo_path", Path(repo.path))
    neo4j_service.stop()


@pytest.fixture
def repository(mocker, repo) -> ContentDTO:
    repository = ContentDTO(
        path=Path(repo.path),
        packs=[],
    )
    relationships = {
        RelationshipType.USES: [
            mock_relationship(
                INPUT_SCRIPT, ContentType.SCRIPT, USES_SCRIPT, ContentType.SCRIPT
            ),
            mock_relationship(
                USED_BY_PLAYBOOK, ContentType.PLAYBOOK, INPUT_SCRIPT, ContentType.SCRIPT
            ),
        ]
    }

    repo_pack = repo.create_pack()
    in_script_yml = os.path.join(FILES_PATH, "docs_test", "script-Set.yml")
    script = repo_pack.create_script(name=INPUT_SCRIPT)
    with open(in_script_yml) as original_yml, open(
        f"{script.path}/{INPUT_SCRIPT}.yml", "w"
    ) as new_script_yml:
        for line in original_yml:
            new_script_yml.write(line)

    pack = mock_pack()
    pack.relationships = relationships
    pack.content_items.script.append(
        mock_script(name=INPUT_SCRIPT, path=Path(f"{script.path}/{INPUT_SCRIPT}.yml"))
    )
    pack.content_items.script.append(mock_script(USES_SCRIPT))
    pack.content_items.playbook.append(mock_playbook(USED_BY_PLAYBOOK))
    repository.packs.extend([pack])

    def mock__create_content_dto(packs_to_update: List[str]) -> List[ContentDTO]:
        if not packs_to_update:
            return [repository]
        repo_copy = repository.copy()
        repo_copy.packs = [p for p in repo_copy.packs if p.object_id in packs_to_update]
        return [repo_copy]

    mocker.patch(
        "demisto_sdk.commands.content_graph.content_graph_builder.ContentGraphBuilder._create_content_dtos",
        side_effect=mock__create_content_dto,
    )

    return repository


def test_generate_script_doc_graph(mocker, repository, tmp_path):
    """
    Given
    - A script (SampleScript) that uses another script (UsesScript)and is used by a playbook (SamplePlaybook).
    When
    - Running generate-docs command on the script.
    Then
    -  The generated readme will have:
        1. no markdown errors.
        2. will contain the names of the script that the input script uses, and the name of the playbook the uses
            the script.
    """
    import demisto_sdk.commands.generate_docs.common as common

    with ContentGraphInterface() as interface:
        create_content_graph(interface)

    pack_graph_object = _get_pack_by_id(repository, "SamplePack")
    input_script_object = pack_graph_object.content_items.script[0]  # INPUTSCRIPT
    expected_readme = os.path.join(FILES_PATH, "docs_test", "set_expected-README.md")
    output_dir = tmp_path / "script_doc_out"
    output_dir.mkdir()
    mocker.patch(
        "demisto_sdk.commands.generate_docs.generate_script_doc.ContentGraphInterface",
        return_value=interface,
    )
    mocker.patch(
        "demisto_sdk.commands.generate_docs.generate_script_doc.update_content_graph",
        return_value=interface,
    )
    mocker.patch.object(common, "execute_command", side_effect=handle_example)

    generate_script_doc(
        input_path=str(input_script_object.path),
        examples="!Set key=k1 value=v1,!Set key=k2 value=v2 append=true",
        output=str(output_dir),
    )
    readme = output_dir / "README.md"
    readme_content = readme.read_text()
    # check the readme content
    with open(expected_readme) as expected_readme_file:
        assert readme_content == expected_readme_file.read()

    # Now try the same thing with a txt file
    command_examples = output_dir / "command_examples.txt"
    with command_examples.open("w") as f:
        f.write("!Set key=k1 value=v1\n!Set key=k2 value=v2 append=true")
    generate_script_doc(
        str(input_script_object.path), command_examples, str(output_dir)
    )
    with open(expected_readme) as expected_readme_file:
        assert readme_content == expected_readme_file.read()


# demisto_sdk/commands/content_graph/tests/get_relationships_test.py

from pathlib import Path

import pytest

from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.content_graph.commands.create import (
    create_content_graph,
)
from demisto_sdk.commands.content_graph.commands.get_relationships import (
    Direction,
    get_relationships_by_path,
)
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.interface import (
    ContentGraphInterface,
)
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
from demisto_sdk.commands.content_graph.tests.create_content_graph_test import (
    mock_integration,
    mock_pack,
    mock_script,
    mock_test_playbook,
    repository,  # noqa: F401
    setup_method,  # noqa: F401
)


def create_mini_content(repository: ContentDTO):  # noqa: F811
    """Creates a content repo with three packs and relationships

              +-----+                        +-----+      +-----+
              |Pack1|                        |Pack2|      |Pack3|
              +-----+                        +-----+      +-----+
                 ^                              ^            ^
                 |  IN_PACK             IN_PACK |            |  IN_PACK
             +---------                         |         ---------
             |         |                        |        |         |
             |         |                        |        |         |
          +-----+   +-----+                     |      +-----+   +-----+
      --> |Intg1|   |Scrp1|                     |      |ApiMd|   | TPB | --
     |    +-----+   +-----+                     |      +-----+   +-----+   |
     |    | |  ^       ^                        |         ^       ^   |    |     USES
     |    | |  |       |                        |         |       |   |    | (mandatorily)
     |    | |  |       | USES (mandatorily)  +-----+      |       |   |    |
     |    | |  |        ---------------------|Scrp2|<-----|-------|---|----
     |    | |   ---------------------------- +-----+      |       |   |
     |    | |            USES (optionally)                |       |   |
     |    | |                                             |       |   |
     |    |  ---------------------------------------------        |   |
     |    |                   IMPORTS                             |   |
     |     -------------------------------------------------------    |
     |                        TESTED_BY                               |
      ----------------------------------------------------------------
                            USES (mandatorily)

    Args:
        repository (ContentDTO): the content dto to populate
    """
    pack1 = mock_pack(
        name="SamplePack",
        path=Path("Packs/SamplePack"),
        repository=repository,
    )
    pack2 = mock_pack(
        name="SamplePack2",
        path=Path("Packs/SamplePack2"),
        repository=repository,
    )
    pack3 = mock_pack(
        name="SamplePack3",
        path=Path("Packs/SamplePack3"),
        repository=repository,
    )

    # pack1 content items
    pack1_integration = mock_integration(
        path=Path(
            "Packs/SamplePack/Integrations/SampleIntegration/SampleIntegration.yml"
        ),
        pack=pack1,
    )
    pack1_script = mock_script(
        path=Path("Packs/SamplePack/Scripts/SampleScript/SampleScript.yml"),
        pack=pack1,
    )

    # pack2 content items
    pack2_script = mock_script(
        "SampleScript2",
        path=Path("Packs/SamplePack2/Scripts/SampleScript2/SampleScript2.yml"),
        pack=pack2,
        uses=[(pack1_integration, False), (pack1_script, True)],
    )

    # pack3 content items
    mock_script(
        "TestApiModule",
        path=Path("Packs/SamplePack3/Scripts/TestApiModule/TestApiModule.yml"),
        pack=pack3,
        importing_items=[pack1_integration],
    )
    mock_test_playbook(
        path=Path(
            "Packs/SamplePack3/TestPlaybooks/SampleTestPlaybook/SampleTestPlaybook.yml"
        ),
        pack=pack3,
        uses=[(pack1_integration, True), (pack2_script, True)],
        tested_items=[pack1_integration],
    )


def compare(result: list, expected: list) -> None:
    assert len(result) == len(expected)
    result.sort(key=lambda r: r["filepath"])
    expected.sort(key=lambda r: r["filepath"])
    for actual, expected in zip(result, expected):
        assert actual.get("mandatorily") == expected.get("mandatorily")
        assert len(actual["paths"]) == expected["paths_count"]


class TestGetRelationships:
    @pytest.mark.parametrize(
        "filepath, relationship, content_type, depth, marketplace, direction, mandatory_only, include_tests, expected_sources, expected_targets",
        [
            pytest.param(
                Path("Packs/SamplePack2/Scripts/SampleScript2/SampleScript2.yml"),
                RelationshipType.USES,
                ContentType.BASE_CONTENT,
                2,
                MarketplaceVersions.XSOAR,
                Direction.BOTH,
                False,
                False,
                [
                    {
                        "filepath": "Packs/SamplePack3/TestPlaybooks/SampleTestPlaybook/SampleTestPlaybook.yml",
                        "mandatorily": True,
                        "paths_count": 1,
                    },
                ],
                [
                    {
                        "filepath": "Packs/SamplePack/Integrations/SampleIntegration/SampleIntegration.yml",
                        "mandatorily": False,
                        "paths_count": 1,
                    },
                    {
                        "filepath": "Packs/SamplePack/Scripts/SampleScript/SampleScript.yml",
                        "mandatorily": True,
                        "paths_count": 1,
                    },
                ],
                id="Verify USES relationships, expecting sources and targets",
            ),
            pytest.param(
                Path("Packs/SamplePack2/Scripts/SampleScript2/SampleScript2.yml"),
                RelationshipType.USES,
                ContentType.BASE_CONTENT,
                2,
                MarketplaceVersions.XSOAR,
                Direction.SOURCES,
                False,
                False,
                [
                    {
                        "filepath": "Packs/SamplePack3/TestPlaybooks/SampleTestPlaybook/SampleTestPlaybook.yml",
                        "mandatorily": True,
                        "paths_count": 1,
                    },
                ],
                [],
                id="Verify USES relationships, sources only",
            ),
            pytest.param(
                Path("Packs/SamplePack2/Scripts/SampleScript2/SampleScript2.yml"),
                RelationshipType.USES,
                ContentType.BASE_CONTENT,
                2,
                MarketplaceVersions.XSOAR,
                Direction.TARGETS,
                False,
                False,
                [],
                [
                    {
                        "filepath": "Packs/SamplePack/Integrations/SampleIntegration/SampleIntegration.yml",
                        "mandatorily": False,
                        "paths_count": 1,
                    },
                    {
                        "filepath": "Packs/SamplePack/Scripts/SampleScript/SampleScript.yml",
                        "mandatorily": True,
                        "paths_count": 1,
                    },
                ],
                id="Verify USES relationships, targets only",
            ),
            pytest.param(
                Path("Packs/SamplePack2/Scripts/SampleScript2/SampleScript2.yml"),
                RelationshipType.USES,
                ContentType.BASE_CONTENT,
                2,
                MarketplaceVersions.XSOAR,
                Direction.TARGETS,
                True,
                False,
                [],
                [
                    {
                        "filepath": "Packs/SamplePack/Scripts/SampleScript/SampleScript.yml",
                        "mandatorily": True,
                        "paths_count": 1,
                    },
                ],
                id="Verify USES relationships, targets only, mandatory only",
            ),
            pytest.param(
                Path("Packs/SamplePack2/Scripts/SampleScript2/SampleScript2.yml"),
                RelationshipType.USES,
                ContentType.INTEGRATION,
                2,
                MarketplaceVersions.XSOAR,
                Direction.BOTH,
                False,
                False,
                [],
                [
                    {
                        "filepath": "Packs/SamplePack/Integrations/SampleIntegration/SampleIntegration.yml",
                        "mandatorily": False,
                        "paths_count": 1,
                    },
                ],
                id="Verify USES relationships, integrations only",
            ),
            pytest.param(
                Path(
                    "Packs/SamplePack3/TestPlaybooks/SampleTestPlaybook/SampleTestPlaybook.yml"
                ),
                RelationshipType.USES,
                ContentType.BASE_CONTENT,
                2,
                MarketplaceVersions.XSOAR,
                Direction.BOTH,
                False,
                False,
                [],
                [
                    {
                        "filepath": "Packs/SamplePack/Integrations/SampleIntegration/SampleIntegration.yml",
                        "mandatorily": True,
                        "paths_count": 2,
                    },
                    {
                        "filepath": "Packs/SamplePack/Scripts/SampleScript/SampleScript.yml",
                        "mandatorily": True,
                        "paths_count": 1,
                    },
                    {
                        "filepath": "Packs/SamplePack2/Scripts/SampleScript2/SampleScript2.yml",
                        "mandatorily": True,
                        "paths_count": 1,
                    },
                ],
                id="Verify USES relationships where depth=2 - 2 paths to SampleIntegration",
            ),
            pytest.param(
                Path(
                    "Packs/SamplePack3/TestPlaybooks/SampleTestPlaybook/SampleTestPlaybook.yml"
                ),
                RelationshipType.USES,
                ContentType.BASE_CONTENT,
                1,
                MarketplaceVersions.XSOAR,
                Direction.BOTH,
                False,
                False,
                [],
                [
                    {
                        "filepath": "Packs/SamplePack/Integrations/SampleIntegration/SampleIntegration.yml",
                        "mandatorily": True,
                        "paths_count": 1,
                    },
                    {
                        "filepath": "Packs/SamplePack2/Scripts/SampleScript2/SampleScript2.yml",
                        "mandatorily": True,
                        "paths_count": 1,
                    },
                ],
                id="Verify USES relationships where depth=1 - only 1 path to SampleIntegration,"
                " no path to SampleScript",
            ),
            pytest.param(
                Path(
                    "Packs/SamplePack/Integrations/SampleIntegration/SampleIntegration.yml"
                ),
                RelationshipType.IMPORTS,
                ContentType.BASE_CONTENT,
                1,
                MarketplaceVersions.XSOAR,
                Direction.BOTH,
                False,
                False,
                [],
                [
                    {
                        "filepath": "Packs/SamplePack3/Scripts/TestApiModule/TestApiModule.yml",
                        "paths_count": 1,
                    },
                ],
                id="Verify IMPORTS relationship",
            ),
            pytest.param(
                Path(
                    "Packs/SamplePack/Integrations/SampleIntegration/SampleIntegration.yml"
                ),
                RelationshipType.TESTED_BY,
                ContentType.BASE_CONTENT,
                1,
                MarketplaceVersions.XSOAR,
                Direction.BOTH,
                False,
                False,
                [],
                [
                    {
                        "filepath": "Packs/SamplePack3/TestPlaybooks/SampleTestPlaybook/SampleTestPlaybook.yml",
                        "paths_count": 1,
                    }
                ],
                id="Verify TESTED_BY relationship",
            ),
            pytest.param(
                Path("Packs/SamplePack3"),
                RelationshipType.DEPENDS_ON,
                ContentType.BASE_CONTENT,
                2,
                MarketplaceVersions.XSOAR,
                Direction.BOTH,
                False,
                False,
                [],
                [],
                id="Verify DEPENDS_ON relationships - don't include tests",
            ),
            pytest.param(
                Path("Packs/SamplePack3"),
                RelationshipType.DEPENDS_ON,
                ContentType.BASE_CONTENT,
                2,
                MarketplaceVersions.XSOAR,
                Direction.BOTH,
                False,
                True,
                [],
                [
                    {
                        "filepath": "Packs/SamplePack2",
                        "paths_count": 1,
                        "mandatorily": True,
                    },
                    {
                        "filepath": "Packs/SamplePack",
                        "paths_count": 2,
                        "mandatorily": True,
                    },
                ],
                id="Verify DEPENDS_ON relationships - include tests",
            ),
        ],
    )
    def test_get_relationships(
        self,
        repository: ContentDTO,  # noqa: F811
        filepath: Path,
        relationship: RelationshipType,
        content_type: ContentType,
        depth: int,
        marketplace: MarketplaceVersions,
        direction: Direction,
        mandatory_only: bool,
        include_tests: bool,
        expected_sources: list,
        expected_targets: list,
    ) -> None:
        """
        Given:
            - A mocked model of a repository.
        When:
            - Running get_relationships_by_path() for the above test cases.
        Then:
            - Make sure the resulted sources and targets are as expected.
        """
        create_mini_content(repository)
        with ContentGraphInterface() as interface:
            create_content_graph(interface)
            result = get_relationships_by_path(
                interface,
                input_filepath=filepath,
                relationship=relationship,
                content_type=content_type,
                depth=depth,
                marketplace=marketplace,
                direction=direction,
                mandatory_only=mandatory_only,
                include_tests=include_tests,
                include_deprecated=False,
                include_hidden=False,
            )
            sources, targets = result["sources"], result["targets"]
        compare(sources, expected_sources)
        compare(targets, expected_targets)


# demisto_sdk/commands/content_graph/tests/graph_validator_test.py

import logging
from pathlib import Path
from typing import Any, Callable, Dict, List

import pytest

import demisto_sdk.commands.content_graph.neo4j_service as neo4j_service
from demisto_sdk.commands.common.constants import (
    GENERAL_DEFAULT_FROMVERSION,
    SKIP_PREPARE_SCRIPT_NAME,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.git_util import GitUtil
from demisto_sdk.commands.common.hook_validations.graph_validator import GraphValidator
from demisto_sdk.commands.common.legacy_git_tools import git_path
from demisto_sdk.commands.content_graph.commands.create import (
    create_content_graph,
)
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.interface import (
    ContentGraphInterface,
)
from demisto_sdk.commands.content_graph.objects.classifier import Classifier
from demisto_sdk.commands.content_graph.objects.integration import Command, Integration
from demisto_sdk.commands.content_graph.objects.pack import Pack
from demisto_sdk.commands.content_graph.objects.playbook import Playbook
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
from demisto_sdk.commands.content_graph.objects.script import Script
from demisto_sdk.commands.content_graph.tests.create_content_graph_test import (
    mock_relationship,
    mock_test_playbook,
)
from TestSuite.test_tools import str_in_call_args_list

GIT_PATH = Path(git_path())


# FIXTURES


@pytest.fixture(autouse=True)
def setup_method(mocker):
    """Auto-used fixture for setup before every test run"""
    import demisto_sdk.commands.content_graph.objects.base_content as bc

    bc.CONTENT_PATH = GIT_PATH
    mocker.patch.object(neo4j_service, "REPO_PATH", GIT_PATH)
    mocker.patch.object(ContentGraphInterface, "repo_path", GIT_PATH)


@pytest.fixture
def repository(mocker) -> ContentDTO:
    repository = ContentDTO(
        path=GIT_PATH,
        packs=[],
    )
    relationships = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SamplePack",
                ContentType.PACK,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.MarketplaceV2,
                ],
            ),
            mock_relationship(
                "SampleScript",
                ContentType.SCRIPT,
                "SamplePack",
                ContentType.PACK,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.MarketplaceV2,
                ],
            ),
        ],
        RelationshipType.HAS_COMMAND: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "test-command",
                ContentType.COMMAND,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.MarketplaceV2,
                ],
                name="test-command",
                description="",
                deprecated=False,
            ),
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "deprecated-command",
                ContentType.COMMAND,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.MarketplaceV2,
                ],
                name="deprecated-command",
                description="",
                deprecated=True,
            ),
        ],
        RelationshipType.IMPORTS: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "TestApiModule",
                ContentType.SCRIPT,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.MarketplaceV2,
                ],
            )
        ],
        RelationshipType.TESTED_BY: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SampleTestPlaybook",
                ContentType.TEST_PLAYBOOK,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.MarketplaceV2,
                ],
            )
        ],
        RelationshipType.USES_BY_ID: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SampleClassifier",
                ContentType.CLASSIFIER,
                mandatorily=True,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.MarketplaceV2,
                ],
            ),
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SampleClassifier2",
                ContentType.CLASSIFIER,
                mandatorily=True,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.MarketplaceV2,
                ],
            ),
        ],
        RelationshipType.DEPENDS_ON: [
            mock_relationship(
                "SamplePack",
                ContentType.PACK,
                "SamplePack2",
                ContentType.PACK,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.MarketplaceV2,
                ],
            ),
        ],
        RelationshipType.USES: [
            mock_relationship(
                "SamplePlaybook",
                ContentType.PLAYBOOK,
                "DeprecatedIntegration",
                ContentType.INTEGRATION,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.XPANSE,
                ],
                source_fromversion="6.5.0",
            ),
            mock_relationship(
                "SamplePlaybook",
                ContentType.PLAYBOOK,
                "deprecated-command",
                ContentType.COMMAND,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.XPANSE,
                ],
                source_fromversion="6.5.0",
            ),
        ],
    }
    relationship_pack2 = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SampleClassifier",
                ContentType.CLASSIFIER,
                "SamplePack2",
                ContentType.PACK,
            ),
            mock_relationship(
                "SampleTestPlaybook",
                ContentType.TEST_PLAYBOOK,
                "SamplePack2",
                ContentType.PACK,
            ),
            mock_relationship(
                "TestApiModule",
                ContentType.SCRIPT,
                "SamplePack2",
                ContentType.PACK,
                source_marketplaces=[MarketplaceVersions.XSOAR],
            ),
            mock_relationship(
                "SampleClassifier2",
                ContentType.CLASSIFIER,
                "SamplePack2",
                ContentType.PACK,
            ),
        ],
        RelationshipType.USES_BY_ID: [
            mock_relationship(
                "TestApiModule",
                ContentType.SCRIPT,
                "SampleScript2",
                ContentType.SCRIPT,
                mandatorily=True,
                source_marketplaces=[MarketplaceVersions.XSOAR],
            ),
        ],
    }
    relationship_pack3 = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SamplePlaybook",
                ContentType.PLAYBOOK,
                "SamplePack3",
                ContentType.PACK,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.XPANSE,
                ],
                source_fromversion="6.5.0",
            ),
            mock_relationship(
                "SamplePlaybook2",
                ContentType.PLAYBOOK,
                "SamplePack3",
                ContentType.PACK,
                source_fromversion=GENERAL_DEFAULT_FROMVERSION,
            ),
            mock_relationship(
                "SampleScript2",
                ContentType.SCRIPT,
                "SamplePack3",
                ContentType.PACK,
            ),
        ],
        RelationshipType.USES_BY_ID: [
            mock_relationship(
                "SamplePlaybook",
                ContentType.PLAYBOOK,
                "SamplePlaybook2",
                ContentType.PLAYBOOK,
                mandatorily=True,
                source_marketplaces=[
                    MarketplaceVersions.XSOAR,
                    MarketplaceVersions.XPANSE,
                ],
                source_fromversion="6.5.0",
            ),
        ],
    }
    relationship_pack4 = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SamplePlaybook", ContentType.PLAYBOOK, "SamplePack4", ContentType.PACK
            )
        ]
    }
    pack1 = mock_pack(
        "SamplePack", [MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2]
    )
    pack2 = mock_pack("SamplePack2", [MarketplaceVersions.XSOAR], hidden=True)
    pack3 = mock_pack(
        "SamplePack3",
        [
            MarketplaceVersions.XSOAR,
            MarketplaceVersions.MarketplaceV2,
            MarketplaceVersions.XPANSE,
        ],
    )
    pack4 = mock_pack("SamplePack4", list(MarketplaceVersions))
    pack1.relationships = relationships
    pack2.relationships = relationship_pack2
    pack3.relationships = relationship_pack3
    pack4.relationships = relationship_pack4
    pack1.content_items.integration.append(mock_integration())
    pack1.content_items.integration.append(
        mock_integration(name="DeprecatedIntegration", deprecated=True)
    )
    pack1.content_items.script.append(
        mock_script(
            "SampleScript",
            [MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2],
        )
    )
    pack1.content_items.script.append(
        mock_script(
            "setIncident",
            [MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2],
        )
    )
    pack2.content_items.script.append(mock_script("TestApiModule"))
    pack2.content_items.script.append(
        mock_script(
            "getIncidents",
            marketplaces=[MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2],
            skip_prepare=[SKIP_PREPARE_SCRIPT_NAME],
        )
    )
    pack2.content_items.classifier.append(mock_classifier("SampleClassifier2"))
    pack2.content_items.test_playbook.append(mock_test_playbook())
    pack3.content_items.playbook.append(
        mock_playbook(
            "SamplePlaybook",
            [MarketplaceVersions.XSOAR, MarketplaceVersions.XPANSE],
            "6.5.0",
            GENERAL_DEFAULT_FROMVERSION,
        )
    )
    pack3.content_items.playbook.append(
        mock_playbook(
            "SamplePlaybook2",
            [MarketplaceVersions.XSOAR],
            GENERAL_DEFAULT_FROMVERSION,
            "6.5.0",
        )
    )
    pack3.content_items.script.append(mock_script("SampleScript2"))
    pack3.content_items.script.append(
        mock_script(
            "setAlert", [MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2]
        )
    )
    pack3.content_items.script.append(
        mock_script(
            "getAlert", [MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2]
        )
    )
    pack3.content_items.script.append(
        mock_script(
            "getAlerts", [MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2]
        )
    )
    pack4.content_items.playbook.append(mock_playbook("SamplePlaybook"))
    repository.packs.extend([pack1, pack2, pack3, pack4])
    mocker.patch(
        "demisto_sdk.commands.content_graph.content_graph_builder.ContentGraphBuilder._create_content_dtos",
        return_value=[repository],
    )
    return repository


# HELPERS


def mock_dependency(source: str, target: str, mandatory: bool = True) -> Dict[str, Any]:
    return {
        "source_id": source,
        "source_type": ContentType.PACK,
        "target": target,
        "target_type": ContentType.PACK,
        "mandatorily": mandatory,
    }


def update_repository(
    repository: ContentDTO,
    commit_func: Callable[[ContentDTO], List[Pack]],
) -> List[str]:
    updated_packs = commit_func(repository)
    pack_ids_to_update = [pack.object_id for pack in updated_packs]
    repository.packs = [
        pack for pack in repository.packs if pack.object_id not in pack_ids_to_update
    ]
    repository.packs.extend(updated_packs)
    return pack_ids_to_update


def _get_pack_by_id(repository: ContentDTO, pack_id: str) -> Pack:
    for pack in repository.packs:
        if pack.object_id == pack_id:
            return pack
    raise ValueError(f"Pack {pack_id} does not exist in the repository.")


def mock_pack(name, marketplaces, hidden=False):
    return Pack(
        object_id=name,
        content_type=ContentType.PACK,
        node_id=f"{ContentType.PACK}:{name}",
        path=Path("Packs"),
        name="pack_name",
        marketplaces=marketplaces,
        hidden=hidden,
        server_min_version="5.5.0",
        current_version="1.0.0",
        tags=[],
        categories=[],
        useCases=[],
        keywords=[],
        contentItems=[],
        excluded_dependencies=[],
        deprecated=False,
    )


def mock_playbook(
    name,
    marketplaces=[MarketplaceVersions.XSOAR],
    fromversion="5.0.0",
    toversion="99.99.99",
):
    return Playbook(
        id=name,
        content_type=ContentType.PLAYBOOK,
        node_id=f"{ContentType.PLAYBOOK}:{name}",
        path=Path(name),
        fromversion=fromversion,
        toversion=toversion,
        display_name=name,
        name=name,
        marketplaces=marketplaces,
        deprecated=False,
        is_test=False,
    )


def mock_script(name, marketplaces=[MarketplaceVersions.XSOAR], skip_prepare=[]):
    return Script(
        id=name,
        content_type=ContentType.SCRIPT,
        node_id=f"{ContentType.SCRIPT}:{name}",
        path=Path("Packs"),
        fromversion="5.0.0",
        display_name=name,
        toversion="6.0.0",
        name=name,
        marketplaces=marketplaces,
        deprecated=False,
        type="python3",
        docker_image="mock:docker",
        tags=[],
        is_test=False,
        skip_prepare=skip_prepare,
    )


def mock_integration(name: str = "SampleIntegration", deprecated: bool = False):
    return Integration(
        id=name,
        content_type=ContentType.INTEGRATION,
        node_id=f"{ContentType.INTEGRATION}:{name}",
        path=Path(name),
        fromversion="5.0.0",
        toversion="99.99.99",
        display_name=name,
        name=name,
        marketplaces=[MarketplaceVersions.XSOAR, MarketplaceVersions.MarketplaceV2],
        deprecated=deprecated,
        type="python3",
        docker_image="mock:docker",
        category="blabla",
        commands=[
            Command(name="test-command", description=""),
            Command(name="deprecated-command", description=""),
        ],
    )


def mock_classifier(name: str = "SampleClassifier"):
    return Classifier(
        id=name,
        content_type=ContentType.CLASSIFIER,
        node_id=f"{ContentType.CLASSIFIER}:{name}",
        path=Path("Packs"),
        fromversion="5.0.0",
        display_name=name,
        toversion="99.99.99",
        name=name,
        marketplaces=[MarketplaceVersions.XSOAR],
        deprecated=False,
        type="python3",
        docker_image="mock:docker",
        tags=[],
        is_test=False,
    )


# TESTS


def test_are_toversion_relationships_paths_valid(repository: ContentDTO):
    """
    Given
    - A content repo
    When
    - running the validation "are_toversion_relationships_paths_valid"
    Then
    - Validate the existance of invalid to_version relationships
    """

    with GraphValidator(update_graph=False) as graph_validator:
        create_content_graph(graph_validator.graph)
        is_valid = graph_validator.validate_toversion_fields()

    assert not is_valid


def test_are_fromversion_relationships_paths_valid(repository: ContentDTO, mocker):
    """
    Given
    - A content repo
    When
    - running the vaidation "are_fromversion_relationships_paths_valid"
    Then
    - Validate the existance of invalid from_version relationships
    """
    logger_error = mocker.patch.object(logging.getLogger("demisto-sdk"), "error")
    with GraphValidator(update_graph=False) as graph_validator:
        create_content_graph(graph_validator.graph)
        is_valid = graph_validator.validate_fromversion_fields()

    assert not is_valid
    assert str_in_call_args_list(
        logger_error.call_args_list,
        "Content item 'SamplePlaybook' whose from_version is '6.5.0' uses the content"
        " items: 'SamplePlaybook2' whose from_version is higher",
    )


@pytest.mark.parametrize(
    "include_optional, is_valid",
    [
        pytest.param(
            False,
            True,
            id="Not providing git_files - should be valid (raised a warning)",
        ),
        pytest.param(
            True,
            False,
            id="providing git_files - should be invalid",
        ),
    ],
)
def test_is_file_using_unknown_content(
    mocker,
    repository: ContentDTO,
    include_optional: bool,
    is_valid: bool,
):
    """
    Given
    - A content repo
    - An integration SampleIntegration's default classifier is set to "SampleClassifier" which does not exist
    When
    - running the vaidation "is_file_using_unknown_content"
    Then
    - Check whether the graph is valid or not, based on whether optional content dependencies were included.
    """
    logger_error = mocker.patch.object(logging.getLogger("demisto-sdk"), "error")
    logger_warning = mocker.patch.object(logging.getLogger("demisto-sdk"), "warning")
    with GraphValidator(
        update_graph=False, git_files=[], include_optional_deps=include_optional
    ) as graph_validator:
        create_content_graph(graph_validator.graph)
        assert graph_validator.is_file_using_unknown_content() == is_valid

    logger_to_search = logger_warning if is_valid else logger_error

    assert str_in_call_args_list(
        logger_to_search.call_args_list,
        "Content item 'SampleIntegration' using content items: SampleClassifier which"
        " cannot be found in the repository",
    )


def test_is_file_display_name_already_exists(repository: ContentDTO, mocker):
    """
    Given
    - A content repo
    When
    - running the vaidation "is_file_display_name_already_exists"
    Then
    - Validate the existance of duplicate display names
    """
    logger_error = mocker.patch.object(logging.getLogger("demisto-sdk"), "error")
    with GraphValidator(update_graph=False) as graph_validator:
        create_content_graph(graph_validator.graph)
        is_valid = graph_validator.is_file_display_name_already_exists()

    assert not is_valid
    for i in range(1, 4):
        assert str_in_call_args_list(
            logger_error.call_args_list,
            f"Pack 'SamplePack{i if i != 1 else ''}' has a duplicate display_name",
        )


def test_validate_unique_script_name(repository: ContentDTO, mocker):
    """
    Given
        - A content repo
    When
        - running the vaidation "validate_unique_script_name"
    Then
        - Validate the existance of duplicate script names
    """
    logger_error = mocker.patch.object(logging.getLogger("demisto-sdk"), "error")
    with GraphValidator(update_graph=False) as graph_validator:
        create_content_graph(graph_validator.graph)
        is_valid = graph_validator.validate_unique_script_name()

    assert not is_valid

    assert str_in_call_args_list(
        logger_error.call_args_list,
        "Cannot create a script with the name setAlert, "
        "because a script with the name setIncident already exists.\n",
    )

    assert not str_in_call_args_list(
        logger_error.call_args_list,
        "Cannot create a script with the name getAlert, "
        "because a script with the name getIncident already exists.\n",
    )

    # Ensure that the script-name-incident-to-alert ignore is working
    assert not str_in_call_args_list(
        logger_error.call_args_list,
        "Cannot create a script with the name getAlerts, "
        "because a script with the name getIncidents already exists.\n",
    )


def test_are_marketplaces_relationships_paths_valid(
    repository: ContentDTO, caplog, mocker
):
    """
    Given
    - A content repo
    When
    - running the validation "is_file_display_name_already_exists"
    Then
    - Validate the existence invalid marketplaces uses
    """
    logger_error = mocker.patch.object(logging.getLogger("demisto-sdk"), "error")
    with GraphValidator(update_graph=False) as graph_validator:
        create_content_graph(graph_validator.graph)
        is_valid = graph_validator.validate_marketplaces_fields()

    assert not is_valid
    assert str_in_call_args_list(
        logger_error.call_args_list,
        "Content item 'SamplePlaybook' can be used in the 'xsoar, xpanse' marketplaces"
        ", however it uses content items: 'SamplePlaybook2' which are not supported in"
        " all of the marketplaces of 'SamplePlaybook'",
    )


def test_validate_dependencies(repository: ContentDTO, caplog, mocker):
    """
    Given
    - A content repo
    When
    - running the vaidation "validate_dependencies"
    Then
    - Validate the existance invalid core pack dependency
    """
    logger_error = mocker.patch.object(logging.getLogger("demisto-sdk"), "error")
    mocker.patch(
        "demisto_sdk.commands.common.hook_validations.graph_validator.get_marketplace_to_core_packs",
        return_value={MarketplaceVersions.XSOAR: {"SamplePack"}},
    )
    with GraphValidator(update_graph=False) as graph_validator:
        create_content_graph(graph_validator.graph)
        is_valid = graph_validator.validate_dependencies()

    assert not is_valid
    assert str_in_call_args_list(
        logger_error.call_args_list,
        "The core pack SamplePack cannot depend on non-core packs: ",
    )


def test_validate_duplicate_id(repository: ContentDTO, mocker):
    """
    Given
    - A content repo with duplicate ids "SamplePlaybook" (configured on repository fixture)
    When
    - running the validation "validate_duplicate_id"
    Then
    - Validate the existence of duplicate ids
    """
    logger_error = mocker.patch.object(logging.getLogger("demisto-sdk"), "error")

    with GraphValidator(update_graph=False) as graph_validator:
        create_content_graph(graph_validator.graph)
        is_valid = graph_validator.validate_duplicate_ids()

    assert not is_valid
    assert str_in_call_args_list(
        logger_error.call_args_list,
        "[GR105] - The ID 'SamplePlaybook' already exists in",
    )


def test_pack_ids_collection():
    git_files = [
        "Tests/conf.json",
        "Packs/MicrosoftExchangeOnline/Integrations/EwsExtension/README.md",
    ]
    expected_pack_ids = ["MicrosoftExchangeOnline"]
    with GraphValidator(update_graph=False, git_files=git_files) as graph_validator:
        assert graph_validator.pack_ids == expected_pack_ids


def test_deprecated_usage__existing_content(repository: ContentDTO, mocker):
    """
    Given
    - A content repo with item using deprecated commands in existing content.
    When
    - running the validation validate_deprecated_items_usage
    Then
    - validate warning is display but it's considered as valid
    """

    logger_info = mocker.patch.object(logging.getLogger("demisto-sdk"), "warning")
    with GraphValidator(update_graph=False) as validator:
        create_content_graph(validator.graph)
        is_valid = validator.validate_deprecated_items_usage()

    assert is_valid
    assert str_in_call_args_list(
        logger_info.call_args_list,
        "[GR107] - The Command 'deprecated-command' is deprecated but used in the following content item:",
    )
    assert str_in_call_args_list(
        logger_info.call_args_list,
        "[GR107] - The Integration 'DeprecatedIntegration' is deprecated but used in the following content item:",
    )


def test_deprecated_usage__new_content(repository: ContentDTO, mocker):
    """
    Given
    - A content repo with the new item "SamplePlaybook" using a deprecated command.
    When
    - running the validation validate_deprecated_items_usage
    Then
    - validate the files considered as invalid.
    """
    mocker.patch.object(GitUtil, "added_files", return_value=[Path("SamplePlaybook")])
    with GraphValidator(update_graph=False) as validator:
        create_content_graph(validator.graph)
        is_valid = validator.validate_deprecated_items_usage()

    assert not is_valid


@pytest.mark.parametrize("changed_pack", ["Packs/SamplePack", "Packs/SamplePack2"])
def test_validate_hidden_pack_is_not_mandatory_dependency(
    repository: ContentDTO, mocker, changed_pack: str
):
    """
    Given
    - A content repo which contains SamplePack that is dependent on
      SamplePack2 (which is hidden) as mandatory dependency

    When
    - Case A: the changed file was the hidden pack.
    - Case B: the changed file was the hidden pack's mandatory dependency

    Then
    - validate that an error occurs with a message stating that SamplePack2 is hidden and has mandatory dependencies
    """
    logger_error = mocker.patch.object(logging.getLogger("demisto-sdk"), "error")

    with GraphValidator(
        update_graph=False, git_files=[Path(changed_pack)]
    ) as graph_validator:
        create_content_graph(graph_validator.graph)
        is_valid = (
            graph_validator.validate_hidden_packs_do_not_have_mandatory_dependencies()
        )

    assert not is_valid
    assert str_in_call_args_list(
        logger_error.call_args_list,
        "[GR108] - SamplePack pack(s) cannot have a mandatory dependency on the hidden pack SamplePack2",
    )


# demisto_sdk/commands/content_graph/tests/neo4j_interface_test.py

from typing import List

import pytest

from demisto_sdk.commands.content_graph.common import ContentType


class TestNeo4jQueries:
    @pytest.mark.parametrize(
        "content_type, expected_labels",
        [
            (
                ContentType.INTEGRATION,
                [ContentType.BASE_CONTENT, ContentType.INTEGRATION],
            ),
            (
                ContentType.TEST_PLAYBOOK,
                [
                    ContentType.BASE_CONTENT,
                    ContentType.PLAYBOOK,
                    ContentType.TEST_PLAYBOOK,
                ],
            ),
            (
                ContentType.SCRIPT,
                [
                    ContentType.BASE_CONTENT,
                    ContentType.SCRIPT,
                    ContentType.COMMAND_OR_SCRIPT,
                ],
            ),
        ],
    )
    def test_labels_of(
        self, content_type: ContentType, expected_labels: List[ContentType]
    ):
        """
        Given:
            - A content type enum object.
        When:
            - Calling labels_of() method.
        Then:
            - Make sure the output string has the expected content type labels.
        """
        from demisto_sdk.commands.content_graph.interface.neo4j.queries.common import (
            labels_of,
        )

        integration_labels = labels_of(content_type)
        for content_type in expected_labels:
            assert str(content_type.value) in integration_labels

    def test_node_map(self):
        """
        Given:
            - A dictionary with string keys and values.
        When:
            - Calling node_map() method.
        Then:
            - Make sure the output is a string representation of a map in neo4j format.
        """
        from demisto_sdk.commands.content_graph.interface.neo4j.queries.common import (
            node_map,
        )

        assert (
            node_map(
                {
                    "object_id": "rel_data.source_id",
                    "content_type": "rel_data.source_type",
                }
            )
            == "{object_id: rel_data.source_id, content_type: rel_data.source_type}"
        )


# demisto_sdk/commands/content_graph/tests/parsers_and_models_test.py

from pathlib import Path
from typing import Dict, List, Optional, Set

import pytest

from demisto_sdk.commands.common.constants import (
    DEFAULT_CONTENT_ITEM_FROM_VERSION,
    DEFAULT_CONTENT_ITEM_TO_VERSION,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.legacy_git_tools import git_path
from demisto_sdk.commands.content_graph.common import (
    ContentType,
    Relationships,
    RelationshipType,
)
from demisto_sdk.commands.content_graph.objects.base_content import BaseContent
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem
from demisto_sdk.commands.content_graph.objects.pack import Pack as PackModel
from demisto_sdk.commands.content_graph.objects.pre_process_rule import PreProcessRule
from demisto_sdk.commands.content_graph.parsers.content_item import (
    NotAContentItemException,
)
from demisto_sdk.commands.content_graph.tests.test_tools import load_json, load_yaml
from TestSuite.pack import Pack
from TestSuite.repo import Repo


def content_items_to_node_ids(
    content_items_dict: Dict[ContentType, List[str]]
) -> Set[str]:
    """A helper method that converts a dict of content items to a set of their node ids."""
    return {
        f"{content_type}:{content_item_id}"
        for content_type, content_items in content_items_dict.items()
        for content_item_id in content_items
    }


class RelationshipsVerifier:
    @staticmethod
    def verify_uses(
        relationships: Relationships,
        relationship_type: RelationshipType,
        expected_targets: Dict[str, ContentType],
    ) -> None:
        targets = {
            relationship.get("target"): relationship.get("target_type")
            for relationship in relationships.get(relationship_type, [])
        }
        assert targets == expected_targets

    @staticmethod
    def verify_command_executions(
        relationships: Relationships,
        expected_commands: List[str],
    ) -> None:
        targets = {
            relationship.get("target")
            for relationship in relationships.get(
                RelationshipType.USES_COMMAND_OR_SCRIPT, []
            )
        }
        expected_targets = {command for command in expected_commands}
        assert targets == expected_targets

    @staticmethod
    def verify_playbook_executions(
        relationships: Relationships,
        expected_playbooks: List[str],
    ) -> None:
        targets = {
            relationship.get("target")
            for relationship in relationships.get(RelationshipType.USES_PLAYBOOK, [])
        }
        expected_targets = {playbook for playbook in expected_playbooks}
        assert targets == expected_targets

    @staticmethod
    def verify_integration_commands(
        relationships: Relationships,
        expected_commands: List[str],
    ) -> None:
        targets = {
            relationship.get("target")
            for relationship in relationships.get(RelationshipType.HAS_COMMAND, [])
        }
        expected_targets = set(expected_commands)
        assert targets == expected_targets

    @staticmethod
    def verify_tests(
        relationships: Relationships,
        expected_tests: List[str],
    ) -> None:
        targets = {
            relationship.get("target")
            for relationship in relationships.get(RelationshipType.TESTED_BY, [])
        }
        expected_targets = set(expected_tests)
        assert targets == expected_targets

    @staticmethod
    def verify_imports(
        relationships: Relationships,
        expected_imports: List[str],
    ) -> None:
        targets = {
            relationship.get("target")
            for relationship in relationships.get(RelationshipType.IMPORTS, [])
        }
        expected_targets = set(expected_imports)
        assert targets == expected_targets

    @staticmethod
    def run(
        relationships: Relationships,
        dependency_ids: Dict[str, ContentType] = {},
        dependency_names: Dict[str, ContentType] = {},
        commands_or_scripts_executions: List[str] = [],
        playbook_executions: List[str] = [],
        tests: List[str] = [],
        imports: List[str] = [],
        integration_commands: List[str] = [],
    ) -> None:
        RelationshipsVerifier.verify_uses(
            relationships, RelationshipType.USES_BY_ID, dependency_ids
        )
        RelationshipsVerifier.verify_uses(
            relationships, RelationshipType.USES_BY_NAME, dependency_names
        )
        RelationshipsVerifier.verify_tests(relationships, tests)
        RelationshipsVerifier.verify_imports(relationships, imports)
        RelationshipsVerifier.verify_command_executions(
            relationships, commands_or_scripts_executions
        )
        RelationshipsVerifier.verify_playbook_executions(
            relationships, playbook_executions
        )
        RelationshipsVerifier.verify_integration_commands(
            relationships, integration_commands
        )


class ContentItemModelVerifier:
    @staticmethod
    def run(
        model: ContentItem,
        expected_id: Optional[str] = None,
        expected_name: Optional[str] = None,
        expected_path: Optional[Path] = None,
        expected_content_type: Optional[ContentType] = None,
        expected_description: Optional[str] = None,
        expected_deprecated: Optional[bool] = None,
        expected_fromversion: Optional[str] = None,
        expected_toversion: Optional[str] = None,
    ) -> None:
        assert expected_id is None or model.object_id == expected_id
        assert expected_name is None or model.name == expected_name
        assert expected_path is None or model.path == expected_path
        assert (
            expected_content_type is None or model.content_type == expected_content_type
        )
        assert expected_description is None or model.description == expected_description
        assert expected_deprecated is None or model.deprecated == expected_deprecated
        assert expected_fromversion is None or model.fromversion == expected_fromversion
        assert expected_toversion is None or model.toversion == expected_toversion


class PackModelVerifier:
    @staticmethod
    def run(
        model: PackModel,
        expected_id: Optional[str] = None,
        expected_name: Optional[str] = None,
        expected_path: Optional[Path] = None,
        expected_description: Optional[str] = None,
        expected_created: Optional[str] = None,
        expected_updated: Optional[str] = None,
        expected_support: Optional[str] = None,
        expected_email: Optional[str] = None,
        expected_url: Optional[str] = None,
        expected_author: Optional[str] = None,
        expected_certification: Optional[str] = None,
        expected_hidden: Optional[bool] = None,
        expected_server_min_version: Optional[str] = None,
        expected_current_version: Optional[str] = None,
        expected_tags: Optional[List[str]] = None,
        expected_categories: Optional[List[str]] = None,
        expected_use_cases: Optional[List[str]] = None,
        expected_keywords: Optional[List[str]] = None,
        expected_price: Optional[int] = None,
        expected_premium: Optional[bool] = None,
        expected_vendor_id: Optional[str] = None,
        expected_vendor_name: Optional[str] = None,
        expected_preview_only: Optional[bool] = None,
        expected_marketplaces: Optional[List[MarketplaceVersions]] = None,
        expected_content_items: Dict[str, ContentType] = {},
        expected_deprecated: Optional[bool] = None,
    ) -> None:
        assert model.content_type == ContentType.PACK
        assert expected_id is None or model.object_id == expected_id
        assert expected_name is None or model.name == expected_name
        assert expected_path is None or model.path == expected_path
        assert expected_description is None or model.description == expected_description
        assert expected_created is None or model.created == expected_created
        assert expected_updated is None or model.updated == expected_updated
        assert expected_support is None or model.support == expected_support
        assert expected_email is None or model.email == expected_email
        assert expected_deprecated is None or model.deprecated == expected_deprecated
        assert expected_url is None or model.url == expected_url
        assert expected_author is None or model.author == expected_author
        assert (
            expected_certification is None
            or model.certification == expected_certification
        )
        assert expected_hidden is None or model.hidden == expected_hidden
        assert (
            expected_server_min_version is None
            or model.server_min_version == expected_server_min_version
        )
        assert (
            expected_current_version is None
            or model.current_version == expected_current_version
        )
        assert expected_tags is None or model.tags == expected_tags
        assert expected_categories is None or model.categories == expected_categories
        assert expected_use_cases is None or model.use_cases == expected_use_cases
        assert expected_keywords is None or model.keywords == expected_keywords
        assert expected_price is None or model.price == expected_price
        assert expected_premium is None or model.premium == expected_premium
        assert expected_vendor_id is None or model.vendor_id == expected_vendor_id
        assert expected_vendor_name is None or model.vendor_name == expected_vendor_name
        assert (
            expected_preview_only is None or model.preview_only == expected_preview_only
        )
        assert (
            expected_marketplaces is None or model.marketplaces == expected_marketplaces
        )

        content_items = {
            content_item.object_id: content_item.content_type
            for content_item in model.content_items
        }
        assert content_items == expected_content_items


class PackRelationshipsVerifier:
    @staticmethod
    def run(
        relationships: Relationships,
        expected_content_items: Dict[str, ContentType] = {},
    ) -> None:
        content_items = {
            relationship.get("source_id"): relationship.get("source_type")
            for relationship in relationships.get(RelationshipType.IN_PACK, [])
        }
        assert content_items == expected_content_items


class TestParsersAndModels:
    def test_classifier_parser_below_min_marketplace_version(self, pack: Pack):
        """
        Given:
            - A pack with a classifier.
        When:
            - Classifier's toversion is 5.9.9.
            - Creating the content item's parser.
        Then:
            - Verify NotAContentItemException is raised, meaning we skip parsing the classifier.
        """
        from demisto_sdk.commands.content_graph.parsers.classifier import (
            ClassifierParser,
        )

        classifier = pack.create_classifier(
            "TestClassifier", load_json("classifier.json")
        )
        classifier.update({"toVersion": "5.9.9"})
        classifier_path = Path(classifier.path)
        with pytest.raises(NotAContentItemException):
            ClassifierParser(classifier_path, list(MarketplaceVersions))

    def test_classifier_parser(self, pack: Pack):
        """
        Given:
            - A pack with a classifier.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.classifier import Classifier
        from demisto_sdk.commands.content_graph.parsers.classifier import (
            ClassifierParser,
        )

        classifier = pack.create_classifier(
            "TestClassifier", load_json("classifier.json")
        )
        classifier_path = Path(classifier.path)
        parser = ClassifierParser(classifier_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "Github": ContentType.INCIDENT_TYPE,
                "DevSecOps New Git PR": ContentType.INCIDENT_TYPE,
                "isEqualString": ContentType.SCRIPT,
                "isNotEmpty": ContentType.SCRIPT,
                "getField": ContentType.SCRIPT,
            },
        )
        model = Classifier.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="Github_Classifier_v1",
            expected_name="Github Classifier",
            expected_path=classifier_path,
            expected_description="Github Classifier",
            expected_content_type=ContentType.CLASSIFIER,
            expected_fromversion="6.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.type == "classification"

    def test_correlation_rule_parser(self, pack: Pack):
        """
        Given:
            - A pack with a correlation rule.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify no relationships were collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.correlation_rule import (
            CorrelationRule,
        )
        from demisto_sdk.commands.content_graph.parsers.correlation_rule import (
            CorrelationRuleParser,
        )

        colrrelation_rule = pack.create_correlation_rule(
            "TestCorrelationRule", load_yaml("correlation_rule.yml")
        )
        colrrelation_rule_path = Path(colrrelation_rule.path)
        parser = CorrelationRuleParser(
            colrrelation_rule_path, list(MarketplaceVersions)
        )
        assert not parser.relationships
        model = CorrelationRule.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="correlation_rule_id",
            expected_name="correlation_rule_name",
            expected_path=colrrelation_rule_path,
            expected_content_type=ContentType.CORRELATION_RULE,
            expected_fromversion=DEFAULT_CONTENT_ITEM_FROM_VERSION,
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )

    def test_dashboard_parser(self, pack: Pack):
        """
        Given:
            - A pack with a dashboard.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.dashboard import Dashboard
        from demisto_sdk.commands.content_graph.parsers.dashboard import DashboardParser

        dashboard = pack.create_dashboard("TestDashboard", load_json("dashboard.json"))
        dashboard_path = Path(dashboard.path)
        parser = DashboardParser(dashboard_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "DetectionsCount": ContentType.SCRIPT,
                "DetectionsData": ContentType.SCRIPT,
            },
        )
        model = Dashboard.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="Confluera Dashboard",
            expected_name="Confluera Dashboard",
            expected_path=dashboard_path,
            expected_content_type=ContentType.DASHBOARD,
            expected_fromversion="6.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )

    def test_generic_definition_parser(self, pack: Pack):
        """
        Given:
            - A pack with a generic definition.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify no relationships were collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.generic_definition import (
            GenericDefinition,
        )
        from demisto_sdk.commands.content_graph.parsers.generic_definition import (
            GenericDefinitionParser,
        )

        generic_definition = pack.create_generic_definition(
            "TestGenericDefinition", load_json("generic_definition.json")
        )
        generic_definition_path = Path(generic_definition.path)
        parser = GenericDefinitionParser(
            generic_definition_path, list(MarketplaceVersions)
        )
        assert not parser.relationships
        model = GenericDefinition.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="ThreatIntelReport",
            expected_name="Threat Intel Report",
            expected_path=generic_definition_path,
            expected_content_type=ContentType.GENERIC_DEFINITION,
            expected_fromversion="6.5.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )

    def test_generic_module_parser(self, pack: Pack):
        """
        Given:
            - A pack with a generic module.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify no relationships were collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.generic_module import (
            GenericModule,
        )
        from demisto_sdk.commands.content_graph.parsers.generic_module import (
            GenericModuleParser,
        )

        generic_module = pack.create_generic_module(
            "TestGenericModule", load_json("generic_module.json")
        )
        generic_module_path = Path(generic_module.path)
        parser = GenericModuleParser(generic_module_path, list(MarketplaceVersions))
        assert not parser.relationships
        model = GenericModule.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="threatIntel",
            expected_name="Threat Intel",
            expected_path=generic_module_path,
            expected_content_type=ContentType.GENERIC_MODULE,
            expected_fromversion="6.5.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert parser.definition_ids == ["ThreatIntelReport"]

    def test_generic_type_parser(self, pack: Pack):
        """
        Given:
            - A pack with a generic type.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.generic_type import GenericType
        from demisto_sdk.commands.content_graph.parsers.generic_type import (
            GenericTypeParser,
        )

        generic_type = pack.create_generic_module(
            "TestGenericType", load_json("generic_type.json")
        )
        generic_type_path = Path(generic_type.path)
        parser = GenericTypeParser(generic_type_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={"Malware Report": ContentType.LAYOUT},
        )
        model = GenericType.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="ThreatIntelReport_Malware",
            expected_name="Malware",
            expected_path=generic_type_path,
            expected_content_type=ContentType.GENERIC_TYPE,
            expected_fromversion="6.5.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.definition_id == "ThreatIntelReport"

    def test_incident_field_parser(self, pack: Pack):
        """
        Given:
            - A pack with an incident field.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.incident_field import (
            IncidentField,
        )
        from demisto_sdk.commands.content_graph.parsers.incident_field import (
            IncidentFieldParser,
        )

        incident_field = pack.create_incident_field(
            "TestIncidentField", load_json("incident_field.json")
        )
        incident_field_path = Path(incident_field.path)
        parser = IncidentFieldParser(incident_field_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_names={
                "Vulnerability": ContentType.INCIDENT_TYPE,
                "Malware": ContentType.INCIDENT_TYPE,
            },
        )
        model = IncidentField.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="cve",
            expected_name="CVE",
            expected_path=incident_field_path,
            expected_content_type=ContentType.INCIDENT_FIELD,
            expected_fromversion="5.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.cli_name == "cve"
        assert model.field_type == "shortText"
        assert not model.associated_to_all

    def test_incident_type_parser(self, pack: Pack):
        """
        Given:
            - A pack with an incident type.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.incident_type import (
            IncidentType,
        )
        from demisto_sdk.commands.content_graph.parsers.incident_type import (
            IncidentTypeParser,
        )

        incident_type = pack.create_incident_field(
            "TestIncidentType", load_json("incident_type.json")
        )
        incident_type_path = Path(incident_type.path)
        parser = IncidentTypeParser(incident_type_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "Traps": ContentType.LAYOUT,
                "Palo Alto Networks - Endpoint Malware Investigation": ContentType.PLAYBOOK,
            },
        )
        model = IncidentType.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="Traps",
            expected_name="Traps",
            expected_path=incident_type_path,
            expected_content_type=ContentType.INCIDENT_TYPE,
            expected_fromversion="5.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.playbook == "Palo Alto Networks - Endpoint Malware Investigation"
        assert model.hours == 0
        assert model.days == 0
        assert model.weeks == 0
        assert not model.closure_script

    def test_indicator_field_parser(self, pack: Pack):
        """
        Given:
            - A pack with an indicator field.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.indicator_field import (
            IndicatorField,
        )
        from demisto_sdk.commands.content_graph.parsers.indicator_field import (
            IndicatorFieldParser,
        )

        indicator_field = pack.create_incident_field(
            "TestIndicatorField", load_json("indicator_field.json")
        )
        indicator_field_path = Path(indicator_field.path)
        parser = IndicatorFieldParser(indicator_field_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_names={
                "User Profile": ContentType.INDICATOR_TYPE,
            },
        )
        model = IndicatorField.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="email",
            expected_name="Email",
            expected_path=indicator_field_path,
            expected_content_type=ContentType.INDICATOR_FIELD,
            expected_fromversion="5.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.type == "shortText"
        assert model.cli_name == "email"
        assert not model.associated_to_all

    def test_indicator_type_parser(self, pack: Pack):
        """
        Given:
            - A pack with an indicator type.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.indicator_type import (
            IndicatorType,
        )
        from demisto_sdk.commands.content_graph.parsers.indicator_type import (
            IndicatorTypeParser,
        )

        indicator_type = pack.create_indicator_type(
            "TestIndicatorType", load_json("indicator_type.json")
        )
        indicator_type_path = Path(indicator_type.path)
        parser = IndicatorTypeParser(indicator_type_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "URLReputation": ContentType.SCRIPT,
                "url": ContentType.COMMAND,
                "urlRep": ContentType.LAYOUT,
            },
        )
        model = IndicatorType.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="urlRep",
            expected_name="URL",
            expected_path=indicator_type_path,
            expected_content_type=ContentType.INDICATOR_TYPE,
            expected_fromversion="5.5.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.regex.startswith("(?i)((?:(?:https?")
        assert not model.reputation_script_name
        assert model.enhancement_script_names == ["URLReputation"]

    def test_integration_parser(self, pack: Pack):
        """
        Given:
            - A pack with an integration.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.integration import Integration
        from demisto_sdk.commands.content_graph.parsers.integration import (
            IntegrationParser,
        )

        integration = pack.create_integration(yml=load_yaml("integration.yml"))
        integration.code.write("from MicrosoftApiModule import *")
        integration.yml.update({"tests": ["test_playbook"]})

        integration_path = Path(integration.path)
        parser = IntegrationParser(integration_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            integration_commands=["test-command"],
            imports=["MicrosoftApiModule"],
            tests=["test_playbook"],
        )
        model = Integration.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="TestIntegration",
            expected_name="TestIntegration",
            expected_content_type=ContentType.INTEGRATION,
            expected_fromversion="5.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.is_fetch_events is True
        assert model.is_fetch_events_and_assets is True

    def test_unified_integration_parser(self, pack: Pack):
        """
        Given:
            - A pack with a unified integration.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.integration import Integration
        from demisto_sdk.commands.content_graph.parsers.integration import (
            IntegrationParser,
        )

        integration = pack.create_integration(yml=load_yaml("unified_integration.yml"))
        integration_path = Path(integration.path)
        parser = IntegrationParser(integration_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            integration_commands=[
                "malwr-submit",
                "malwr-status",
                "malwr-result",
                "malwr-detonate",
            ],
        )
        model = Integration.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="malwr",
            expected_name="malwr",
            expected_content_type=ContentType.INTEGRATION,
            expected_fromversion="5.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.category == "Forensics & Malware Analysis"
        assert model.display_name == "malwr"
        assert model.docker_image == "demisto/bs4:1.0.0.7863"
        assert not model.is_fetch
        assert not model.is_feed
        assert model.type == "python2"

    def test_job_parser(self, pack: Pack):
        """
        Given:
            - A pack with a job.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.job import Job
        from demisto_sdk.commands.content_graph.parsers.job import JobParser

        job = pack.create_job(is_feed=False, name="TestJob")
        job_path = Path(job.path)
        parser = JobParser(job_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "job-TestJob_playbook": ContentType.PLAYBOOK,
            },
        )
        model = Job.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="TestJob",
            expected_name="TestJob",
            expected_path=job_path,
            expected_content_type=ContentType.JOB,
            expected_fromversion="6.8.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )

    def test_layout_parser(self, pack: Pack):
        """
        Given:
            - A pack with a layout.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.parsers.layout import LayoutParser

        layout = pack.create_layout("TestLayout")
        layout_path = Path(layout.path)
        with pytest.raises(NotAContentItemException):
            LayoutParser(layout_path, list(MarketplaceVersions))

    def test_layoutscontainer_parser(self, pack: Pack):
        """
        Given:
            - A pack with a layout.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.layout import Layout
        from demisto_sdk.commands.content_graph.parsers.layout import LayoutParser

        layout = pack.create_layoutcontainer(
            "TestLayoutscontainer", load_json("layoutscontainer.json")
        )
        layout_path = Path(layout.path)
        parser = LayoutParser(layout_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "xdrdevicecontrolviolations": ContentType.INCIDENT_FIELD,
                "type": ContentType.INCIDENT_FIELD,
                "dbotsource": ContentType.INCIDENT_FIELD,
                "sourceinstance": ContentType.INCIDENT_FIELD,
                "severity": ContentType.INCIDENT_FIELD,
                "playbookid": ContentType.INCIDENT_FIELD,
                "sourcebrand": ContentType.INCIDENT_FIELD,
                "owner": ContentType.INCIDENT_FIELD,
            },
        )
        model = Layout.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="Cortex XDR Device Control Violations",
            expected_name="Cortex XDR Device Control Violations",
            expected_path=layout_path,
            expected_content_type=ContentType.LAYOUT,
            expected_fromversion="6.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )

    @pytest.mark.parametrize("override_group", ("incident", "indicator"))
    @pytest.mark.parametrize(
        "marketplace",
        (
            MarketplaceVersions.XSOAR,
            MarketplaceVersions.MarketplaceV2,
            MarketplaceVersions.XPANSE,
        ),
    )
    def test_layoutscontainer_parser_fixes(
        self, pack: Pack, marketplace: MarketplaceVersions, override_group: str
    ):
        """
        Given:
            - A pack with a layout.
            - The marketplace for which the item is prepared.
        When:
            - Preparing for upload.
        Then:
            - Verify a `Related Incidents` field's name is changed to `Related Alerts`
                if and only if marketpalce==marketplacev2 and group=="indicator"
        """
        from demisto_sdk.commands.content_graph.objects.layout import Layout
        from demisto_sdk.commands.content_graph.parsers.layout import LayoutParser

        layout = pack.create_layoutcontainer(
            "TestLayoutscontainer", load_json("layoutscontainer.json")
        )
        model = Layout.from_orm(
            LayoutParser(Path(layout.path), list(MarketplaceVersions))
        )
        model.group = override_group
        ready_for_upload = model.prepare_for_upload(current_marketplace=marketplace)
        checked_dict = ready_for_upload["detailsV2"]["tabs"][5]

        # these two are for sanity, to make sure we're checking the right value (and the test file hasn't been changed)
        assert checked_dict["id"] == "relatedIncidents"
        assert checked_dict["type"] == "relatedIncidents"
        assert ("Alerts" in checked_dict["name"]) == (
            override_group == "indicator"
            and marketplace == MarketplaceVersions.MarketplaceV2
        )

    def test_list_parser(self, pack: Pack):
        """
        Given:
            - A pack with a list.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify no relationships were collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.list import List as ListObject
        from demisto_sdk.commands.content_graph.parsers.list import ListParser

        list_ = pack.create_list("TestList", load_json("list.json"))
        list_path = Path(list_.path)
        parser = ListParser(list_path, list(MarketplaceVersions))
        assert not parser.relationships
        model = ListObject.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="checked integrations",
            expected_name="checked integrations",
            expected_path=list_path,
            expected_description="",
            expected_content_type=ContentType.LIST,
            expected_fromversion="6.5.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.type == "plain_text"

    def test_incoming_mapper_parser(self, pack: Pack):
        """
        Given:
            - A pack with an incoming mapper.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.mapper import Mapper
        from demisto_sdk.commands.content_graph.parsers.mapper import MapperParser

        mapper = pack.create_mapper(
            "TestIncomingMapper", load_json("incoming_mapper.json")
        )
        mapper_path = Path(mapper.path)
        parser = MapperParser(mapper_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "substringTo": ContentType.SCRIPT,
            },
            dependency_names={
                "DevSecOps New Git PR": ContentType.INCIDENT_TYPE,
                "DevSecOps Repository Name": ContentType.INCIDENT_FIELD,
                "DevSecOps Repository Organization": ContentType.INCIDENT_FIELD,
            },
        )
        model = Mapper.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="GitHub Mapper",
            expected_name="GitHub Mapper",
            expected_path=mapper_path,
            expected_description="",
            expected_content_type=ContentType.MAPPER,
            expected_fromversion="6.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.type == "mapping-incoming"

    def test_outgoing_mapper_parser(self, pack: Pack):
        """
        Given:
            - A pack with an outgoing mapper.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.mapper import Mapper
        from demisto_sdk.commands.content_graph.parsers.mapper import MapperParser

        mapper = pack.create_mapper(
            "TestOutgoingMapper", load_json("outgoing_mapper.json")
        )
        mapper_path = Path(mapper.path)
        parser = MapperParser(mapper_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "description": ContentType.INCIDENT_FIELD,
                "azuredevopsprojectname": ContentType.INCIDENT_FIELD,
                "MapValuesTransformer": ContentType.SCRIPT,
            },
            dependency_names={
                "Azure DevOps": ContentType.INCIDENT_TYPE,
            },
        )
        model = Mapper.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="Azure DevOps Outgoing Mapper",
            expected_name="Azure DevOps Outgoing Mapper",
            expected_path=mapper_path,
            expected_description="",
            expected_content_type=ContentType.MAPPER,
            expected_fromversion="6.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.type == "mapping-outgoing"

    def test_modeling_rule_parser(self, pack: Pack):
        """
        Given:
            - A pack with a modeling rule.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify no relationships were collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.modeling_rule import (
            ModelingRule,
        )
        from demisto_sdk.commands.content_graph.parsers.modeling_rule import (
            ModelingRuleParser,
        )

        modeling_rule = pack.create_modeling_rule(
            "TestModelingRule", load_yaml("modeling_rule.yml")
        )
        modeling_rule_path = Path(modeling_rule.path)
        parser = ModelingRuleParser(modeling_rule_path, list(MarketplaceVersions))
        assert not parser.relationships
        model = ModelingRule.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="duo_modeling_rule",
            expected_name="Duo Modeling Rule",
            expected_content_type=ContentType.MODELING_RULE,
            expected_fromversion="6.8.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )

    def test_parsing_rule_parser(self, pack: Pack):
        """
        Given:
            - A pack with a parsing rule.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify no relationships were collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.parsing_rule import ParsingRule
        from demisto_sdk.commands.content_graph.parsers.parsing_rule import (
            ParsingRuleParser,
        )

        parsing_rule = pack.create_parsing_rule(
            "TestParsingRule", load_yaml("parsing_rule.yml")
        )
        parsing_rule_path = Path(parsing_rule.path)
        parser = ParsingRuleParser(parsing_rule_path, list(MarketplaceVersions))
        assert not parser.relationships
        model = ParsingRule.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="_parsing_rule_id",
            expected_name="My Rule",
            expected_content_type=ContentType.PARSING_RULE,
            expected_fromversion="6.8.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )

    def test_playbook_parser(self, pack: Pack):
        """
        Given:
            - A pack with a playbook.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
              In particular, make sure redundant backslashes are removed from the playbook's description.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.playbook import Playbook
        from demisto_sdk.commands.content_graph.parsers.playbook import PlaybookParser

        playbook = pack.create_playbook()
        playbook.create_default_playbook(name="sample")
        playbook.yml.update({"description": "test\\ test2\\\n \\ test3"})
        playbook_path = Path(playbook.path)
        parser = PlaybookParser(playbook_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "DeleteContext": ContentType.SCRIPT,
            },
        )
        model = Playbook.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="sample",
            expected_name="sample",
            expected_content_type=ContentType.PLAYBOOK,
            expected_fromversion="5.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
            expected_description="test test2 test3",
        )
        assert not model.is_test

    def test_report_parser(self, pack: Pack):
        """
        Given:
            - A pack with a report.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.report import Report
        from demisto_sdk.commands.content_graph.parsers.report import ReportParser

        report = pack.create_report("TestReport", load_json("report.json"))
        report_path = Path(report.path)
        parser = ReportParser(report_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "ProofpointTAPMostAttackedUsers": ContentType.SCRIPT,
                "ProofpointTapTopClickers": ContentType.SCRIPT,
            },
        )
        model = Report.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="ProofpointTAPWeeklyReport",
            expected_name="Proofpoint TAP Weekly Report",
            expected_path=report_path,
            expected_content_type=ContentType.REPORT,
            expected_fromversion="5.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )

    def test_script_parser(self, pack: Pack):
        """
        Given:
            - A pack with a script.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.script import Script
        from demisto_sdk.commands.content_graph.parsers.script import ScriptParser

        script = pack.create_script()
        script.create_default_script()
        script.code.write('demisto.executeCommand("dummy-command", dArgs)')
        script_path = Path(script.path)
        parser = ScriptParser(script_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            commands_or_scripts_executions=["dummy-command"],
        )
        model = Script.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="sample_script",
            expected_name="sample_script",
            expected_content_type=ContentType.SCRIPT,
            expected_fromversion="5.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.type == "python3"
        assert model.docker_image == "demisto/python3:3.8.3.8715"
        assert model.tags == ["transformer"]
        assert not model.is_test
        assert not model.skip_prepare

    def test_test_playbook_parser(self, pack: Pack):
        """
        Given:
            - A pack with a test playbook.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.test_playbook import (
            TestPlaybook,
        )
        from demisto_sdk.commands.content_graph.parsers.test_playbook import (
            TestPlaybookParser,
        )

        test_playbook = pack.create_test_playbook()
        test_playbook.create_default_test_playbook(name="sample")
        test_playbook_path = Path(test_playbook.path)
        parser = TestPlaybookParser(test_playbook_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "DeleteContext": ContentType.SCRIPT,
            },
        )
        model = TestPlaybook.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="sample",
            expected_name="sample",
            expected_content_type=ContentType.TEST_PLAYBOOK,
            expected_fromversion="5.0.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert model.is_test

    def test_trigger_parser(self, pack: Pack):
        """
        Given:
            - A pack with a trigger.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.trigger import Trigger
        from demisto_sdk.commands.content_graph.parsers.trigger import TriggerParser

        trigger = pack.create_trigger("TestTrigger", load_json("trigger.json"))
        trigger_path = Path(trigger.path)
        parser = TriggerParser(trigger_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "NGFW Scan": ContentType.PLAYBOOK,
            },
        )
        model = Trigger.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="73545719a1bdeba6ba91f6a16044c021",
            expected_name="NGFW Scanning Alerts",
            expected_path=trigger_path,
            expected_content_type=ContentType.TRIGGER,
            expected_fromversion=DEFAULT_CONTENT_ITEM_FROM_VERSION,
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )

    def test_layout_rule_parser(self, pack: Pack):
        """
        Given:
            - A pack with a layout rule.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.layout_rule import LayoutRule
        from demisto_sdk.commands.content_graph.parsers.layout_rule import (
            LayoutRuleParser,
        )

        rule = pack.create_layout_rule("rule_test")
        rule_path = Path(rule.path)
        parser = LayoutRuleParser(rule_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "test_layout": ContentType.LAYOUT,
            },
        )
        model = LayoutRule.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="rule_test",
            expected_name="rule_test.json",
            expected_path=rule_path,
            expected_content_type=ContentType.LAYOUT_RULE,
            expected_fromversion="6.10.0",
        )

    def test_widget_parser(self, pack: Pack):
        """
        Given:
            - A pack with a widget.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify all relationships of the content item are collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.widget import Widget
        from demisto_sdk.commands.content_graph.parsers.widget import WidgetParser

        widget = pack.create_widget("TestWidget", load_json("widget.json"))
        widget_path = Path(widget.path)
        parser = WidgetParser(widget_path, list(MarketplaceVersions))
        RelationshipsVerifier.run(
            parser.relationships,
            dependency_ids={
                "FeedIntegrationErrorWidget": ContentType.SCRIPT,
            },
        )
        model = Widget.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="Feed Integrations Errors",
            expected_name="Feed Integrations Errors",
            expected_path=widget_path,
            expected_content_type=ContentType.WIDGET,
            expected_fromversion="6.1.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )

    def test_wizard_parser(self, pack: Pack):
        """
        Given:
            - A pack with a wizard.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify no relationships were collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.wizard import Wizard
        from demisto_sdk.commands.content_graph.parsers.wizard import WizardParser

        wizard_json = load_json("wizard.json")
        wizard = pack.create_wizard(
            "TestWizard",
            categories_to_packs={
                c["name"]: c["packs"] for c in wizard_json["dependency_packs"]
            },
            fetching_integrations=[
                i["name"] for i in wizard_json["wizard"]["fetching_integrations"]
            ],
            set_playbooks=wizard_json["wizard"]["set_playbook"],
            supporting_integrations=[
                i["name"] for i in wizard_json["wizard"]["supporting_integrations"]
            ],
        )
        wizard_path = Path(wizard.path)
        parser = WizardParser(wizard_path, list(MarketplaceVersions))
        assert not parser.relationships
        model = Wizard.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="TestWizard",
            expected_name="TestWizard",
            expected_path=wizard_path,
            expected_content_type=ContentType.WIZARD,
            expected_fromversion="6.8.0",
            expected_toversion=DEFAULT_CONTENT_ITEM_TO_VERSION,
        )
        assert set(model.packs) == {
            "CrowdStrikeFalcon",
            "MicrosoftDefenderAdvancedThreatProtection",
            "CortexXDR",
        }
        assert set(model.integrations) == {
            "WildFire-v2",
            "Microsoft Defender Advanced Threat Protection",
        }
        assert set(model.playbooks) == {
            "Malware Investigation & Response Incident Handler"
        }

    def test_xsiam_dashboard_parser(self, pack: Pack):
        """
        Given:
            - A pack with an xsiam dashboard.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify no relationships were collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.xsiam_dashboard import (
            XSIAMDashboard,
        )
        from demisto_sdk.commands.content_graph.parsers.xsiam_dashboard import (
            XSIAMDashboardParser,
        )

        xsiam_dashboard = pack.create_xsiam_dashboard(
            "TestXSIAMDashboard", load_json("xsiam_dashboard.json")
        )
        xsiam_dashboard_path = Path(xsiam_dashboard.path)
        parser = XSIAMDashboardParser(xsiam_dashboard_path, list(MarketplaceVersions))
        assert not parser.relationships
        model = XSIAMDashboard.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="ce27311ce69c41b1b4a84c7888b34852",
            expected_name="New Import test ",
            expected_path=xsiam_dashboard_path,
            expected_content_type=ContentType.XSIAM_DASHBOARD,
            expected_fromversion="8.1.0",
            expected_toversion="8.3.0",
        )

    def test_xsiam_report_parser(self, pack: Pack):
        """
        Given:
            - A pack with an xsiam report.
        When:
            - Creating the content item's parser and model.
        Then:
            - Verify no relationships were collected.
            - Verify the generic content item properties are parsed correctly.
            - Verify the specific properties of the content item are parsed correctly.
        """
        from demisto_sdk.commands.content_graph.objects.xsiam_report import XSIAMReport
        from demisto_sdk.commands.content_graph.parsers.xsiam_report import (
            XSIAMReportParser,
        )

        xsiam_report = pack.create_xsiam_report(
            "TestXSIAMReport", load_json("xsiam_report.json")
        )
        xsiam_report_path = Path(xsiam_report.path)
        parser = XSIAMReportParser(xsiam_report_path, list(MarketplaceVersions))
        assert not parser.relationships
        model = XSIAMReport.from_orm(parser)
        ContentItemModelVerifier.run(
            model,
            expected_id="sample",
            expected_name="sample",
            expected_path=xsiam_report_path,
            expected_content_type=ContentType.XSIAM_REPORT,
            expected_fromversion="8.1.0",
            expected_toversion="8.3.0",
        )

    def test_preprocess_parser(self):
        pre_process_rule = BaseContent.from_path(
            Path(
                f"{git_path()}/demisto_sdk/tests/test_files/content_slim/Packs/Sample01/PreProcessRules/preprocessrule-Drop.json"
            )
        )
        assert isinstance(pre_process_rule, PreProcessRule)
        assert pre_process_rule.name == "Drop"
        assert pre_process_rule.object_id == "preprocessrule-Drop-id"

    def test_pack_parser(self, repo: Repo):
        """
        Given:
            - A pack with several content items.
        When:
            - Creating the pack parser and model.
        Then:
            - Verify all pack's relationships from all its content items are collected.
            - Verify the content items are modeled correctly.
            - Verify the pack is modeled correctly.
        """
        from demisto_sdk.commands.content_graph.objects.pack import Pack as PackModel
        from demisto_sdk.commands.content_graph.parsers.pack import PackParser

        pack = repo.create_pack("HelloWorld")
        pack.pack_metadata.write_json(load_json("pack_metadata.json"))
        pack.create_classifier("sample", load_json("classifier.json"))
        pack.create_incident_field("sample", load_json("incident_field.json"))
        pack.create_incident_type("sample", load_json("incident_type.json"))
        pack.create_indicator_field("sample", load_json("indicator_field.json"))
        pack.create_indicator_type("sample", load_json("indicator_type.json"))
        pack_path = Path(pack.path)
        parser = PackParser(pack_path)
        expected_content_items = {
            "Github_Classifier_v1": ContentType.CLASSIFIER,
            "cve": ContentType.INCIDENT_FIELD,
            "Traps": ContentType.INCIDENT_TYPE,
            "email": ContentType.INDICATOR_FIELD,
            "urlRep": ContentType.INDICATOR_TYPE,
        }
        PackRelationshipsVerifier.run(
            parser.relationships,
            expected_content_items=expected_content_items,
        )
        model = PackModel.from_orm(parser)
        PackModelVerifier.run(
            model,
            expected_id="HelloWorld",
            expected_name="HelloWorld",
            expected_path=pack_path,
            expected_description="This is the Hello World integration for getting started.",
            expected_created="2020-03-10T08:37:18Z",
            expected_support="community",
            expected_url="https://www.paloaltonetworks.com/cortex",
            expected_author="Cortex XSOAR",
            expected_certification="",
            expected_hidden=False,
            expected_current_version="1.2.12",
            expected_tags=[],
            expected_categories=["Utilities"],
            expected_use_cases=[],
            expected_keywords=[],
            expected_marketplaces=[
                MarketplaceVersions.XSOAR,
                MarketplaceVersions.MarketplaceV2,
            ],
            expected_content_items=expected_content_items,
            expected_deprecated=False,
        )

    def test_repo_parser(self, repo: Repo):
        """
        Given:
            - A repository with two packs.
        When:
            - Creating the repository parser and model.
        Then:
            - Verify the repository is modeled correctly.
        """
        from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
        from demisto_sdk.commands.content_graph.parsers.repository import (
            RepositoryParser,
        )

        pack1 = repo.create_pack("sample1")
        pack1.pack_metadata.write_json(load_json("pack_metadata.json"))
        pack2 = repo.create_pack("sample2")
        pack2.pack_metadata.write_json(load_json("pack_metadata.json"))
        parser = RepositoryParser(Path(repo.path))
        parser.parse()
        model = ContentDTO.from_orm(parser)
        pack_ids = {pack.object_id for pack in model.packs}
        assert pack_ids == {"sample1", "sample2"}


@pytest.mark.parametrize(
    "name,type_,expected_change",
    (
        ("Child Incidents", "childInv", True),
        ("Child Incident", "childInv", False),  # name should be Child Incidents
        ("", "childInv", False),
        ("Incidents", "", False),
        ("Linked Incidents", "linkedIncidents", True),
        (
            "Related Incidents",
            "linkedIncidents",
            False,
        ),  # type should be relatedIncidents
        ("Related Incidents", "relatedIncidents", True),
        ("Related Incidents", "", False),
        ("Related Incident", "relatedIncidents", False),
    ),
)
def test_fix_layout_incident_to_alert(
    name: str,
    type_: str,
    expected_change: bool,
):
    """
    Given:
        - A layout body
    When:
        - change_incident_to_alert is called
    Then:
        - Make sure it replaces values as expected
    """
    from demisto_sdk.commands.content_graph.objects.layout import (
        replace_layout_incident_alert,
    )

    expected_name = name.replace("Incident", "Alert") if expected_change else name

    assert replace_layout_incident_alert({"name": name, "type": type_}) == {
        "name": expected_name,
        "type": type_,
    }


# demisto_sdk/commands/content_graph/tests/prepare_content_graph_test.py

from pathlib import Path

import pytest

import demisto_sdk.commands.content_graph.neo4j_service as neo4j_service
from demisto_sdk.commands.common.constants import (
    SKIP_PREPARE_SCRIPT_NAME,
    MarketplaceVersions,
)
from demisto_sdk.commands.common.legacy_git_tools import git_path
from demisto_sdk.commands.common.tools import get_yaml
from demisto_sdk.commands.content_graph.commands.create import (
    create_content_graph,
)
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.interface import (
    ContentGraphInterface,
)
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
from demisto_sdk.commands.content_graph.tests.create_content_graph_test import (
    mock_pack,
    mock_playbook,
    mock_relationship,
    mock_script,
)
from demisto_sdk.commands.prepare_content.preparers.marketplace_incident_to_alert_playbooks_prepare import (
    MarketplaceIncidentToAlertPlaybooksPreparer,
)
from TestSuite.repo import Repo

GIT_ROOT = git_path()


@pytest.fixture(autouse=True)
def setup_method(mocker, repo: Repo):
    """Auto-used fixture for setup before every test run"""
    import demisto_sdk.commands.content_graph.objects.base_content as bc

    bc.CONTENT_PATH = Path(repo.path)
    mocker.patch.object(ContentGraphInterface, "repo_path", Path(repo.path))
    mocker.patch.object(neo4j_service, "REPO_PATH", Path(repo.path))
    neo4j_service.stop()


@pytest.fixture
def repository(mocker):
    repository = ContentDTO(
        path=Path(),
        packs=[],
    )
    mocker.patch(
        "demisto_sdk.commands.content_graph.content_graph_builder.ContentGraphBuilder._create_content_dtos",
        return_value=[repository],
    )
    return repository


def create_mini_content(repository: ContentDTO):
    relationships = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "playbook_1",
                ContentType.PLAYBOOK,
                "TestPack",
                ContentType.PACK,
            )
        ],
        RelationshipType.USES_BY_ID: [
            mock_relationship(
                "playbook_1",
                ContentType.PLAYBOOK,
                "getIncident",
                ContentType.SCRIPT,
                mandatorily=True,
            ),
            mock_relationship(
                "playbook_1",
                ContentType.PLAYBOOK,
                "setIncidentByID",
                ContentType.SCRIPT,
                mandatorily=True,
            ),
        ],
    }
    relationships2 = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "setIncidentByID",
                ContentType.PLAYBOOK,
                "TestPack2",
                ContentType.PACK,
            ),
            mock_relationship(
                "getIncident",
                ContentType.PLAYBOOK,
                "TestPack2",
                ContentType.PACK,
            ),
        ]
    }
    pack1 = mock_pack("TestPack")
    pack2 = mock_pack("TestPack2")
    pack1.relationships = relationships
    pack2.relationships = relationships2
    pack1.content_items.playbook.append(
        mock_playbook(
            name="playbook_1",
        )
    )
    pack2.content_items.script.append(
        mock_script(
            "getIncident",
            skip_prepare=[SKIP_PREPARE_SCRIPT_NAME],
        )
    )
    pack2.content_items.script.append(
        mock_script(
            "setIncidentByID",
        )
    )
    repository.packs.extend([pack1, pack2])


def test_marketplace_version_is_xsiam_with_graph(repository: ContentDTO):
    """
    Given:
        - A playbook which contains scripts that names include incident/s.
        - One script is set as `skipprepare` and a second script is not set as skip.
    When:
        - MarketplaceIncidentToAlertPlaybooksPreparer.prepare() command is executed

    Then:
        - Ensure that only a script that is not set as skip has changed from incident to alert.
    """
    create_mini_content(repository)
    with ContentGraphInterface() as interface:
        create_content_graph(interface)
        playbooks = interface.search(content_type=ContentType.PLAYBOOK)
        data = get_yaml(
            f"{GIT_ROOT}/demisto_sdk/commands/prepare_content/test_files/playbook_2.yml"
        )

        data = MarketplaceIncidentToAlertPlaybooksPreparer.prepare(
            playbook=playbooks[0],
            data=data,
            current_marketplace=MarketplaceVersions.MarketplaceV2,
            supported_marketplaces=[
                MarketplaceVersions.XSOAR,
                MarketplaceVersions.MarketplaceV2,
            ],
        )

    assert data["tasks"]["6"]["task"]["scriptName"] == "getIncident"
    assert data["tasks"]["7"]["task"]["scriptName"] == "setAlertByID"


# demisto_sdk/commands/content_graph/tests/test_tools.py

from pathlib import Path

from demisto_sdk.commands.common.handlers import DEFAULT_JSON_HANDLER as json
from demisto_sdk.commands.common.handlers import DEFAULT_YAML_HANDLER as yaml
from demisto_sdk.commands.common.legacy_git_tools import git_path

TEST_DATA_PATH = (
    Path(git_path())
    / "demisto_sdk"
    / "commands"
    / "content_graph"
    / "tests"
    / "test_data"
)


def load_json(file_path: str):
    full_path = (TEST_DATA_PATH / file_path).as_posix()
    with open(full_path) as f:
        return json.load(f)


def load_yaml(file_path: str):
    full_path = (TEST_DATA_PATH / file_path).as_posix()
    with open(full_path) as f:
        return yaml.load(f)


# demisto_sdk/commands/content_graph/tests/update_content_graph_test.py

from pathlib import Path
from typing import Any, Callable, Dict, List
from zipfile import ZipFile

import pytest

import demisto_sdk.commands.content_graph.neo4j_service as neo4j_service
from demisto_sdk.commands.common.constants import MarketplaceVersions
from demisto_sdk.commands.common.legacy_git_tools import git_path
from demisto_sdk.commands.content_graph.commands.create import (
    create_content_graph,
)
from demisto_sdk.commands.content_graph.commands.update import (
    update_content_graph,
)
from demisto_sdk.commands.content_graph.common import ContentType, RelationshipType
from demisto_sdk.commands.content_graph.interface import (
    ContentGraphInterface,
)
from demisto_sdk.commands.content_graph.objects.content_item import ContentItem
from demisto_sdk.commands.content_graph.objects.integration import Integration
from demisto_sdk.commands.content_graph.objects.pack import Pack
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
from demisto_sdk.commands.content_graph.tests.create_content_graph_test import (
    find_model_for_id,
    mock_classifier,
    mock_integration,
    mock_pack,
    mock_playbook,
    mock_relationship,
    mock_script,
    mock_test_playbook,
)
from demisto_sdk.commands.content_graph.tests.test_tools import TEST_DATA_PATH

GIT_PATH = Path(git_path())


# FIXTURES


@pytest.fixture(autouse=True)
def setup_method(mocker):
    """Auto-used fixture for setup before every test run"""
    import demisto_sdk.commands.content_graph.objects.base_content as bc

    bc.CONTENT_PATH = GIT_PATH
    mocker.patch.object(neo4j_service, "REPO_PATH", GIT_PATH)
    mocker.patch.object(ContentGraphInterface, "repo_path", GIT_PATH)
    neo4j_service.stop()


@pytest.fixture
def repository(mocker) -> ContentDTO:
    repository = ContentDTO(
        path=GIT_PATH,
        packs=[],
    )
    relationships = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SamplePack",
                ContentType.PACK,
            ),
            mock_relationship(
                "SampleScript",
                ContentType.SCRIPT,
                "SamplePack",
                ContentType.PACK,
            ),
        ],
        RelationshipType.HAS_COMMAND: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "test-command",
                ContentType.COMMAND,
                name="test-command",
                description="",
                deprecated=False,
            )
        ],
        RelationshipType.IMPORTS: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "TestApiModule",
                ContentType.SCRIPT,
            )
        ],
        RelationshipType.TESTED_BY: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SampleTestPlaybook",
                ContentType.TEST_PLAYBOOK,
            )
        ],
        RelationshipType.USES_BY_ID: [
            mock_relationship(
                "SampleIntegration",
                ContentType.INTEGRATION,
                "SampleClassifier",
                ContentType.CLASSIFIER,
                mandatorily=True,
            ),
        ],
    }
    relationship_pack2 = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SampleClassifier",
                ContentType.CLASSIFIER,
                "SamplePack2",
                ContentType.PACK,
            ),
            mock_relationship(
                "SampleTestPlaybook",
                ContentType.TEST_PLAYBOOK,
                "SamplePack2",
                ContentType.PACK,
            ),
            mock_relationship(
                "TestApiModule", ContentType.SCRIPT, "SamplePack2", ContentType.PACK
            ),
        ],
        RelationshipType.USES_BY_ID: [
            mock_relationship(
                "TestApiModule",
                ContentType.SCRIPT,
                "SampleScript2",
                ContentType.SCRIPT,
                mandatorily=True,
            ),
        ],
    }
    relationship_pack3 = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SamplePlaybook",
                ContentType.PLAYBOOK,
                "SamplePack3",
                ContentType.PACK,
            ),
            mock_relationship(
                "SampleScript2",
                ContentType.SCRIPT,
                "SamplePack3",
                ContentType.PACK,
            ),
        ],
        RelationshipType.USES_BY_ID: [
            mock_relationship(
                "SamplePlaybook",
                ContentType.PLAYBOOK,
                "test-command",
                ContentType.COMMAND,
            ),
        ],
    }
    pack1 = mock_pack()
    pack2 = mock_pack("SamplePack2")
    pack3 = mock_pack("SamplePack3")
    pack1.relationships = relationships
    pack2.relationships = relationship_pack2
    pack3.relationships = relationship_pack3
    pack1.content_items.integration.append(mock_integration())
    pack1.content_items.script.append(mock_script())
    pack2.content_items.script.append(mock_script("TestApiModule"))
    pack2.content_items.classifier.append(mock_classifier())
    pack2.content_items.test_playbook.append(mock_test_playbook())
    pack3.content_items.playbook.append(mock_playbook())
    pack3.content_items.script.append(mock_script("SampleScript2"))
    repository.packs.extend([pack1, pack2, pack3])

    def mock__create_content_dto(packs_to_update: List[str]) -> List[ContentDTO]:
        if not packs_to_update:
            return [repository]
        repo_copy = repository.copy()
        repo_copy.packs = [p for p in repo_copy.packs if p.object_id in packs_to_update]
        return [repo_copy]

    mocker.patch(
        "demisto_sdk.commands.content_graph.content_graph_builder.ContentGraphBuilder._create_content_dtos",
        side_effect=mock__create_content_dto,
    )

    return repository


# HELPERS


def mock_dependency(source: str, target: str, mandatory: bool = True) -> Dict[str, Any]:
    return {
        "source_id": source,
        "source_type": ContentType.PACK,
        "target": target,
        "target_type": ContentType.PACK,
        "mandatorily": mandatory,
    }


def update_repository(
    repository: ContentDTO,
    commit_func: Callable[[ContentDTO], List[Pack]],
) -> List[str]:
    updated_packs = commit_func(repository)
    pack_ids_to_update = [pack.object_id for pack in updated_packs]
    repository.packs = [
        pack for pack in repository.packs if pack.object_id not in pack_ids_to_update
    ]
    repository.packs.extend(updated_packs)
    return pack_ids_to_update


def _get_pack_by_id(repository: ContentDTO, pack_id: str) -> Pack:
    for pack in repository.packs:
        if pack.object_id == pack_id:
            return pack
    raise ValueError(f"Pack {pack_id} does not exist in the repository.")


# COMPARISON HELPER FUNCTIONS


def compare(
    packs_from_content_dto: List[Pack],
    packs_from_graph: List[Pack],
    expected_added_dependencies: List[Dict[str, Any]],
    expected_removed_dependencies: List[Dict[str, Any]],
    after_update: bool,
) -> None:
    packs_from_content_dto.sort(key=lambda pack: pack.object_id)
    packs_from_graph.sort(key=lambda pack: pack.object_id)
    assert len(packs_from_content_dto) == len(packs_from_graph)
    for pack_a, pack_b in zip(packs_from_content_dto, packs_from_graph):
        assert pack_a.to_dict() == pack_b.to_dict()
        _compare_content_items(list(pack_a.content_items), list(pack_b.content_items))
        _compare_relationships(pack_a, pack_b)
    _verify_dependencies_existence(
        packs_from_graph, expected_added_dependencies, should_exist=after_update
    )
    _verify_dependencies_existence(
        packs_from_graph, expected_removed_dependencies, should_exist=not after_update
    )


def _compare_content_items(
    content_items_list_a: List[ContentItem], content_items_list_b: List[ContentItem]
) -> None:
    assert len(content_items_list_a) == len(content_items_list_b)
    content_items_list_a = sorted(content_items_list_a, key=lambda obj: obj.node_id)
    content_items_list_b = sorted(content_items_list_b, key=lambda obj: obj.node_id)
    for ci_a, ci_b in zip(content_items_list_a, content_items_list_b):
        assert ci_a.to_dict() == ci_b.to_dict()


def _compare_relationships(pack_a: Pack, pack_b: Pack) -> None:
    for relationship_type, relationships in pack_a.relationships.items():
        for relationship in relationships:
            content_item_source = find_model_for_id(
                [pack_b], relationship.get("source_id")
            )
            content_item_target_id = relationship.get("target")
            assert content_item_source
            assert content_item_target_id
            if relationship_type == RelationshipType.IN_PACK:
                assert (
                    content_item_source.in_pack
                ), f"{content_item_source.object_id} is not in pack."
                assert content_item_source.in_pack.object_id == content_item_target_id
            if relationship_type == RelationshipType.IMPORTS:
                assert (
                    content_item_source.imports[0].object_id == content_item_target_id
                )
            if relationship_type == RelationshipType.USES_BY_ID:
                assert (
                    content_item_source.uses[0].content_item_to.object_id
                    == content_item_target_id
                ) or any(
                    [
                        isinstance(uses_rel.content_item_to, Integration)
                        and uses_rel.content_item_to.commands[0].name
                        == content_item_target_id
                        for uses_rel in content_item_source.uses
                    ]
                )
            if relationship_type == RelationshipType.TESTED_BY:
                assert (
                    content_item_source.tested_by[0].object_id == content_item_target_id
                )


def _verify_dependencies_existence(
    packs: List[Pack],
    dependencies: List[Dict[str, Any]],
    should_exist: bool,
) -> None:
    for dependency in dependencies:
        for pack in packs:
            if pack.object_id == dependency["source_id"]:
                if should_exist:
                    assert any(
                        r.content_item_to.object_id == dependency["target"]
                        for r in pack.depends_on
                    )
                else:
                    assert all(
                        r.content_item_to.object_id != dependency["target"]
                        for r in pack.depends_on
                    )
                break
        else:
            assert False


# TEST CASES (commit functions)


def _testcase1__pack3_pack4__script2_uses_script4(repository: ContentDTO) -> List[Pack]:
    """Test case for the following updates:
    * New pack: SamplePack4, with a script: SampleScript4
    * New relationship: SampleScript2 (of existing pack SamplePack3) USES SampleScript4 (of new pack SamplePack4)

    Returns:
        A list of the updated packs: pack3, pack4.
    """
    pack4_relationships = {
        RelationshipType.IN_PACK: [
            mock_relationship(
                "SampleScript4",
                ContentType.SCRIPT,
                "SamplePack4",
                ContentType.PACK,
            )
        ],
    }
    pack4 = mock_pack("SamplePack4")
    pack4.relationships = pack4_relationships
    pack4.content_items.script.append(mock_script("SampleScript4"))

    pack3 = _get_pack_by_id(repository, "SamplePack3")
    pack3.relationships.setdefault(RelationshipType.USES_BY_ID, []).append(
        mock_relationship(
            "SampleScript2",
            ContentType.SCRIPT,
            "SampleScript4",
            ContentType.SCRIPT,
        )
    )
    return [pack3, pack4]


def _testcase2__pack3__remove_relationship(repository: ContentDTO) -> List[Pack]:
    """Test case for the following update:
    * Remove relationship: TestApiModule (of pack SamplePack2) USES SampleScript2 (of pack SamplePack3)

    Returns:
        A list of the updated pack: pack2.
    """
    pack2 = _get_pack_by_id(repository, "SamplePack2")
    uses_relationships = pack2.relationships.get(RelationshipType.USES_BY_ID, [])
    for rel in uses_relationships:
        if rel["source_id"] == "TestApiModule" and rel["target"] == "SampleScript2":
            uses_relationships.remove(rel)
            break
    return [pack2]


def _testcase3__no_change(_: ContentDTO) -> List[Pack]:
    """Test case for a commit diff without update."""
    return []


def _testcase4__new_integration_with_existing_command(
    repository: ContentDTO,
) -> List[Pack]:
    """Test case for the following update:
    * New integration: SampleIntegration2 in SamplePack
    * New command: test-command (node already exists for SampleIntegration2)

    Returns:
        A list of the updated pack: pack.
    """
    pack = _get_pack_by_id(repository, "SamplePack")
    pack.content_items.integration.append(mock_integration("SampleIntegration2"))
    pack.relationships.setdefault(RelationshipType.IN_PACK, []).append(
        mock_relationship(
            "SampleIntegration2",
            ContentType.INTEGRATION,
            "SamplePack",
            ContentType.PACK,
        )
    )
    pack.relationships.setdefault(RelationshipType.HAS_COMMAND, []).append(
        mock_relationship(
            "SampleIntegration2",
            ContentType.INTEGRATION,
            "test-command",
            ContentType.COMMAND,
            name="test-command",
            description="",
            deprecated=False,
        )
    )
    return [pack]


def _testcase5__move_script_from_pack3_to_pack1(repository: ContentDTO) -> List[Pack]:
    """Test case for the following update:
    * Moved SampleScript2 from SamplePack3 to SamplePack.

    Returns:
        A list of the updated packs: pack2 and pack3.
    """
    pack1 = _get_pack_by_id(repository, "SamplePack")
    pack3 = _get_pack_by_id(repository, "SamplePack3")
    pack1.content_items.script.append(pack3.content_items.script[0])
    pack3.content_items.script = []
    in_pack_relationships = pack3.relationships.get(RelationshipType.IN_PACK, [])
    for rel in in_pack_relationships:
        if rel["source_id"] == "SampleScript2" and rel["target"] == "SamplePack3":
            in_pack_relationships.remove(rel)
            break
    pack1.relationships.setdefault(RelationshipType.IN_PACK, []).append(
        mock_relationship(
            "SampleScript2",
            ContentType.SCRIPT,
            "SamplePack",
            ContentType.PACK,
        )
    )
    return [pack1, pack3]


def _testcase6__move_script_from_pack3_to_pack2(repository: ContentDTO) -> List[Pack]:
    """Test case for the following update:
    * Moved SampleScript2 from SamplePack3 to SamplePack2.

    Returns:
        A list of the updated packs: pack2 and pack3.
    """
    pack2 = _get_pack_by_id(repository, "SamplePack2")
    pack3 = _get_pack_by_id(repository, "SamplePack3")
    pack2.content_items.script.append(pack3.content_items.script[0])
    pack3.content_items.script = []
    pack2.relationships.setdefault(RelationshipType.IN_PACK, []).append(
        mock_relationship(
            "SampleScript2",
            ContentType.SCRIPT,
            "SamplePack2",
            ContentType.PACK,
        )
    )
    in_pack_relationships = pack3.relationships.get(RelationshipType.IN_PACK, [])
    for rel in in_pack_relationships:
        if rel["source_id"] == "SampleScript2" and rel["target"] == "SamplePack3":
            in_pack_relationships.remove(rel)
            break
    return [pack2, pack3]


def _testcase7__changed_script2_fromversion(repository: ContentDTO) -> List[Pack]:
    """Test case for the following update:
    * SampleScript2 - changed fromversion

    Returns:
        A list of the updated pack: pack3.
    """
    new_fromversion = "7.0.0"
    pack3 = _get_pack_by_id(repository, "SamplePack3")
    pack3.content_items.script[0].fromversion = new_fromversion
    in_pack_relationships = pack3.relationships.get(RelationshipType.IN_PACK, [])
    for rel in in_pack_relationships:
        if rel["source_id"] == "SampleScript2" and rel["target"] == "SamplePack3":
            rel["source_fromversion"] = new_fromversion
            break
    return [pack3]


# Test class


class TestUpdateContentGraph:
    def test_merge_graphs(self):
        """
        Given:
            - A content graph interface.
            - Valid neo4j CSV files of two repositories to import.
              * The first repository (content) has a single pack `SamplePack`, with:
                1. Integration `SampleIntegration` with a single command `test-command`.
                2. Classifier `SampleClassifier`, used by the integration.
              * The second repository (content-private) has a single pack `SamplePack4`, with:
                1. Integration `SampleIntegration4` with a single command `test-command`.
                   The integration uses `SampleClassifier` of SamplePack (from content repository).
        When:
            - Running update_graph() command.
        Then:
            - Make sure both repositories are imported successfully to the graph, i.e., the graph has:
              1. Two packs
              2. Two integrations, using the same (single) command
              3. One classifier
        """

        def get_nodes_count_by_type(
            interface: ContentGraphInterface,
            content_type: ContentType,
        ) -> int:
            return len(
                interface.search(
                    marketplace=MarketplaceVersions.XSOAR,
                    content_type=content_type,
                )
            )

        with ContentGraphInterface() as interface:
            update_content_graph(
                interface,
                packs_to_update=[],
                imported_path=TEST_DATA_PATH
                / "mock_import_files_multiple_repos__valid"
                / "valid_graph.zip",
            )
            assert get_nodes_count_by_type(interface, ContentType.PACK) == 2
            assert get_nodes_count_by_type(interface, ContentType.INTEGRATION) == 2
            assert get_nodes_count_by_type(interface, ContentType.COMMAND) == 1
            assert get_nodes_count_by_type(interface, ContentType.CLASSIFIER) == 1

    data_test_update_content_graph = [
        pytest.param(
            _testcase1__pack3_pack4__script2_uses_script4,
            [mock_dependency("SamplePack3", "SamplePack4")],
            [],
            id="New pack with USES relationship, causing adding a dependency",
        ),
        pytest.param(
            _testcase2__pack3__remove_relationship,
            [],
            [mock_dependency("SamplePack2", "SamplePack3")],
            id="Remove USES relationship, causing removing a dependency",
        ),
        pytest.param(
            _testcase3__no_change,
            [],
            [],
            id="No change in repository",
        ),
        pytest.param(
            _testcase4__new_integration_with_existing_command,
            [],
            [],
            id="New integration with existing command test-command",
        ),
        pytest.param(
            _testcase5__move_script_from_pack3_to_pack1,
            [mock_dependency("SamplePack2", "SamplePack")],
            [mock_dependency("SamplePack2", "SamplePack3")],
            id="Moved script - dependency pack2 -> pack3 changed to pack2 -> pack1",
        ),
        pytest.param(
            _testcase6__move_script_from_pack3_to_pack2,
            [],
            [mock_dependency("SamplePack2", "SamplePack3")],
            id="Moved script - removed dependency between pack2 and pack3",
        ),
        pytest.param(
            _testcase7__changed_script2_fromversion,
            [],
            [],
            id="Changed fromversion of script",
        ),
    ]

    @pytest.mark.parametrize(
        "commit_func, expected_added_dependencies, expected_removed_dependencies",
        data_test_update_content_graph,
    )
    def test_update_content_graph(
        self,
        tmp_path,
        repository: ContentDTO,
        commit_func: Callable[[ContentDTO], List[Pack]],
        expected_added_dependencies: List[Dict[str, Any]],
        expected_removed_dependencies: List[Dict[str, Any]],
    ):
        """
        Given:
            - A ContentDTO model representing the repository state on master branch.
            - A function representing a commit (an update of certain packs in the repository).
            - Lists of the expected added & removed pack dependencies after the update.
        When:
            - Running create_content_graph() on master.
            - Pushing a commit with the pack updates.
            - Running update_content_graph().
        Then:
            - Make sure the pack models from the interface are equal to the pack models from ContentDTO
                before and after the update.
            - Make sure the expected added dependencies actually don't exist before the update
                and exist after the update.
            - Make sure the expected removed dependencies actually exist before the update
                and don't exist after the update.
        """
        with ContentGraphInterface() as interface:
            # create the graph with dependencies
            create_content_graph(interface, dependencies=True, output_path=tmp_path)
            packs_from_graph = interface.search(
                marketplace=MarketplaceVersions.XSOAR,
                content_type=ContentType.PACK,
                all_level_dependencies=True,
            )

            compare(
                repository.packs,
                packs_from_graph,
                expected_added_dependencies,
                expected_removed_dependencies,
                after_update=False,
            )

            # perform the update on ContentDTO
            pack_ids_to_update = update_repository(repository, commit_func)

            # update the graph accordingly
            update_content_graph(
                interface,
                packs_to_update=pack_ids_to_update,
                dependencies=True,
                output_path=tmp_path,
                use_current=True,
            )
            packs_from_graph = interface.search(
                marketplace=MarketplaceVersions.XSOAR,
                content_type=ContentType.PACK,
                all_level_dependencies=True,
            )
            compare(
                repository.packs,
                packs_from_graph,
                expected_added_dependencies,
                expected_removed_dependencies,
                after_update=True,
            )
        # make sure that the output file zip is created
        assert Path.exists(tmp_path / "xsoar.zip")
        with ZipFile(tmp_path / "xsoar.zip", "r") as zip_obj:
            zip_obj.extractall(tmp_path / "extracted")
            # make sure that the extracted files are all .csv
            extracted_files = list(tmp_path.glob("extracted/*"))
            assert extracted_files
            assert all(
                file.suffix == ".graphml" or file.name == "metadata.json"
                for file in extracted_files
            )

    @pytest.mark.parametrize(
        "commit_func, expected_added_dependencies, expected_removed_dependencies",
        data_test_update_content_graph,
    )
    def test_create_content_graph_if_needed(
        self,
        tmp_path,
        repository: ContentDTO,
        mocker,
        commit_func: Callable[[ContentDTO], List[Pack]],
        expected_added_dependencies: List[Dict[str, Any]],
        expected_removed_dependencies: List[Dict[str, Any]],
    ):
        """
        Given:
            - A ContentDTO model representing the repository state on master branch.
            - A function representing a commit (an update of certain packs in the repository).
            - Lists of the expected added & removed pack dependencies after the update.
        When:
            - Running create_content_graph() on master.
            - Pushing a commit with the pack updates.
            - Running update_content_graph().
        Then:
            - Make sure the pack models from the interface are equal to the pack models from ContentDTO
                before and after the update.
            - Make sure the expected added dependencies actually don't exist before the update
                and exist after the update.
            - Make sure the expected removed dependencies actually exist before the update
                and don't exist after the update.
        """
        import demisto_sdk.commands.content_graph.commands.update as update

        create_content_graph_spy = mocker.spy(update, "create_content_graph")

        with ContentGraphInterface() as interface:
            # create the graph with dependencies
            update.create_content_graph(
                interface, dependencies=True, output_path=tmp_path
            )
            packs_from_graph = interface.search(
                marketplace=MarketplaceVersions.XSOAR,
                content_type=ContentType.PACK,
                all_level_dependencies=True,
            )

            file_added = (
                Path(__file__).parent.parent / "parsers" / "content_graph_test.txt"
            )
            file_added.write_text(
                "this file is created by a test in "
                "demisto_sdk/commands/content_graph/tests/update_content_graph_test.py"
                " and should be removed when the test passes"
            )

            compare(
                repository.packs,
                packs_from_graph,
                expected_added_dependencies,
                expected_removed_dependencies,
                after_update=False,
            )

            # perform the update on ContentDTO
            pack_ids_to_update = update_repository(repository, commit_func)

            # update the graph accordingly
            update.update_content_graph(
                interface,
                packs_to_update=pack_ids_to_update,
                dependencies=True,
                output_path=tmp_path,
                use_current=True,
            )
            packs_from_graph = interface.search(
                marketplace=MarketplaceVersions.XSOAR,
                content_type=ContentType.PACK,
                all_level_dependencies=True,
            )
            compare(
                repository.packs,
                packs_from_graph,
                expected_added_dependencies,
                expected_removed_dependencies,
                after_update=True,
            )
        file_added.unlink()
        assert create_content_graph_spy.call_count == 2
        # make sure that the output file zip is created
        assert Path.exists(tmp_path / "xsoar.zip")
        with ZipFile(tmp_path / "xsoar.zip", "r") as zip_obj:
            zip_obj.extractall(tmp_path / "extracted")
            # make sure that the extracted files are all .csv
            extracted_files = list(tmp_path.glob("extracted/*"))
            assert extracted_files
            assert all(
                file.suffix == ".graphml" or file.name == "metadata.json"
                for file in extracted_files
            )


# demisto_sdk/commands/content_graph/tests/pack_metadata_graph_test.py
from pathlib import Path

import pytest

import demisto_sdk.commands.content_graph.neo4j_service as neo4j_service
from demisto_sdk.commands.common.constants import (
    MarketplaceVersions,
)
from demisto_sdk.commands.common.tools import get_json
from demisto_sdk.commands.content_graph.commands.create import (
    create_content_graph,
)
from demisto_sdk.commands.content_graph.interface.neo4j.neo4j_graph import (
    Neo4jContentGraphInterface as ContentGraphInterface,
)
from demisto_sdk.commands.content_graph.objects.integration_script import (
    IntegrationScript,
)
from demisto_sdk.commands.content_graph.objects.pack_metadata import PackMetadata
from demisto_sdk.commands.content_graph.objects.repository import ContentDTO
from demisto_sdk.commands.content_graph.tests.test_tools import load_json
from TestSuite.repo import Repo
from TestSuite.test_tools import ChangeCWD


@pytest.fixture(autouse=True)
def setup_method(mocker, repo: Repo):
    """Auto-used fixture for setup before every test run"""
    import demisto_sdk.commands.content_graph.objects.base_content as bc

    bc.CONTENT_PATH = Path(repo.path)
    mocker.patch.object(ContentGraphInterface, "repo_path", Path(repo.path))
    mocker.patch.object(neo4j_service, "REPO_PATH", Path(repo.path))
    neo4j_service.stop()


@pytest.fixture
def repository(mocker):
    repository = ContentDTO(
        path=Path(),
        packs=[],
    )
    mocker.patch(
        "demisto_sdk.commands.content_graph.content_graph_builder.ContentGraphBuilder._create_content_dtos",
        return_value=[repository],
    )
    return repository


def test_pack_metadata_xsoar(repo: Repo, tmp_path: Path, mocker):
    """
    Given:
        - A repository with a pack TestPack, containing multiple content items.
    When:
        - Dumping the pack and pack metadata.json for marketplace `xsoar`.
    Then:
        - Make sure the the metadata value as expected.
        - Make sure the `serverMinVersion` is the minimum content item fromversion value except the test playbook.
        - Make sure that along 3 versions of a playbook, only the latest is in the metadata.
        - Make sure the script tags are in the pack metadata tags.
    """
    mocker.patch.object(
        IntegrationScript, "get_supported_native_images", return_value=[]
    )
    mocker.patch.object(PackMetadata, "_get_tags_from_landing_page", return_value=set())

    pack = repo.create_pack("TestPack")
    pack.pack_metadata.write_json(load_json("pack_metadata.json"))

    integration = pack.create_integration()
    integration.create_default_integration(
        name="TestIntegration", commands=["test-command1", "test-command2"]
    )
    integration.yml.update(
        {
            "display": "Test Integration",
            "category": "Authentication & Identity Management",
            "description": "integration  description\n  - for  \n    - example  ",
            "fromversion": "6.8.0",
        }
    )
    integration.yml.update(
        {
            "isfetch": True,
            "type": "python3",
        },
        key_dict_to_update="script",
    )

    script = pack.create_script()
    script.create_default_script("TestScript")
    script.yml.update({"tags": ["transformer", "filter"], "fromversion": "6.5.0"})

    playbook1 = pack.create_playbook()
    playbook1.create_default_playbook(name="MyPlaybook")
    playbook1.yml.update({"fromversion": "6.5.0", "toversion": "6.7.9"})
    playbook2 = pack.create_playbook()
    playbook2.create_default_playbook(name="MyPlaybook")
    playbook2.yml.update({"fromversion": "6.8.0", "toversion": "6.9.9"})
    playbook3 = pack.create_playbook()
    playbook3.create_default_playbook(name="MyPlaybook")
    playbook3.yml.update({"fromversion": "6.10.0"})

    test_playbook = pack.create_test_playbook()
    test_playbook.create_default_test_playbook(name="TestTestPlaybook")
    test_playbook.yml.update({"fromversion": "6.2.0"})

    with ContentGraphInterface() as interface:
        create_content_graph(interface, output_path=tmp_path)
        content_cto = interface.marshal_graph(MarketplaceVersions.XSOAR)

    with ChangeCWD(repo.path):
        content_cto.dump(tmp_path, MarketplaceVersions.XSOAR, zip=False)

    assert (tmp_path / "TestPack" / "metadata.json").exists()
    metadata = get_json(tmp_path / "TestPack" / "metadata.json")

    assert metadata.get("id") == "TestPack"
    assert metadata.get("name") == "HelloWorld"
    assert (
        metadata.get("description")
        == "This is the Hello World integration for getting started."
    )
    assert metadata.get("legacy") is True
    assert metadata.get("support") == "community"
    assert (
        metadata.get("supportDetails", {}).get("url")
        == "https://www.paloaltonetworks.com/cortex"
    )
    assert "email" not in metadata.get("supportDetails", {})
    assert (
        metadata.get("eulaLink")
        == "https://github.com/demisto/content/blob/master/LICENSE"
    )
    assert metadata.get("author") == "Cortex XSOAR"
    assert metadata.get("authorImage") == "content/packs/TestPack/Author_image.png"
    assert metadata.get("certification") == "verified"
    assert metadata.get("price") == 0
    assert metadata.get("hidden") is False
    assert metadata.get("serverMinVersion") == "6.5.0"
    assert metadata.get("currentVersion") == "1.2.12"
    assert metadata.get("versionInfo") == ""
    assert metadata.get("downloads") == 0
    assert sorted(metadata.get("tags", [])) == sorted(
        ["TIM", "Transformer", "Use Case", "Filter"]
    )
    assert metadata.get("categories") == ["Utilities"]
    assert metadata.get("useCases") == ["Identity And Access Management"]
    assert metadata.get("keywords") == []
    assert metadata.get("searchRank") == 0
    assert metadata.get("excludedDependencies") == []
    assert metadata.get("videos") == []
    assert metadata.get("modules") == []
    assert metadata.get("integrations") == []

    metadata_integration = metadata.get("contentItems", {}).get("integration", [{}])[0]
    assert (
        metadata_integration.get("category") == "Authentication & Identity Management"
    )
    assert len(metadata_integration.get("commands", [])) == 3
    assert (
        metadata_integration.get("description")
        == "integration description\n  - for \n    - example "
    )
    assert metadata_integration.get("fromversion") == "6.8.0"
    assert metadata_integration.get("id") == "TestIntegration"
    assert metadata_integration.get("isfetch") is True
    assert (
        metadata_integration.get("name") == "Test Integration (Community Contribution)"
    )

    metadata_playbook = metadata.get("contentItems", {}).get("playbook", [{}])[0]
    assert metadata_playbook.get("fromversion") == "6.10.0"
    assert metadata_playbook.get("id") == "MyPlaybook"
    assert metadata_playbook.get("name") == "MyPlaybook"


def test_pack_metadata_marketplacev2(repo: Repo, tmp_path: Path, mocker):
    """
    Given:
        - A repository with a pack TestPack, containing multiple content items.
    When:
        - Dumping the pack and pack metadata.json for marketplace `marketplacev2`.
    Then:
        - Make sure the the metadata value as expected.
        - Make sure that along 3 versions of a playbook, only the latest is in the metadata.
        - Make sure the script tags are in the pack metadata tags.
    """
    mocker.patch.object(
        IntegrationScript, "get_supported_native_images", return_value=[]
    )
    mocker.patch.object(PackMetadata, "_get_tags_from_landing_page", return_value=set())

    pack = repo.create_pack("TestPack")
    pack.pack_metadata.write_json(load_json("pack_metadata2.json"))

    integration = pack.create_integration()
    integration.create_default_integration(name="TestIntegration")
    integration.yml.update(
        {
            "display": "Test Integration",
            "fromversion": "6.8.0",
        }
    )
    integration.yml.update(
        {
            "isfetchevents": True,
            "type": "python3",
        },
        key_dict_to_update="script",
    )

    playbook1 = pack.create_playbook()
    playbook1.create_default_playbook(name="MyPlaybook")
    playbook1.yml.update({"fromversion": "6.5.0", "toversion": "6.7.9"})
    playbook2 = pack.create_playbook()
    playbook2.create_default_playbook(name="MyPlaybook")
    playbook2.yml.update({"fromversion": "6.10.0"})

    pack.create_modeling_rule(
        name="MyModelingRule",
        yml={
            "id": "my_modeling_rule",
            "name": "My Modeling Rule",
            "fromversion": "6.8.0",
            "toversion": "6.9.9",
        },
    )
    pack.create_modeling_rule(
        name="MyModelingRule_1_3",
        yml={
            "id": "my_ModelingRule",
            "name": "My Modeling Rule",
            "fromversion": "6.10.0",
        },
    )

    with ContentGraphInterface() as interface:
        create_content_graph(interface, output_path=tmp_path)
        content_cto = interface.marshal_graph(MarketplaceVersions.MarketplaceV2)

    with ChangeCWD(repo.path):
        content_cto.dump(tmp_path, MarketplaceVersions.MarketplaceV2, zip=False)

    assert (tmp_path / "TestPack" / "metadata.json").exists()
    metadata = get_json(tmp_path / "TestPack" / "metadata.json")

    assert metadata.get("id") == "TestPack"
    assert metadata.get("name") == "HelloWorld2"
    assert (
        metadata.get("description")
        == "This is the Hello World 2 integration for getting started."
    )
    assert metadata.get("legacy") is True
    assert metadata.get("support") == "xsoar"
    assert (
        metadata.get("supportDetails", {}).get("url")
        == "https://www.paloaltonetworks.com/cortex"
    )
    assert "email" not in metadata.get("supportDetails", {})
    assert (
        metadata.get("eulaLink")
        == "https://github.com/demisto/content/blob/master/LICENSE"
    )
    assert metadata.get("author") == "Cortex XSIAM"
    assert metadata.get("authorImage") == "content/packs/TestPack/Author_image.png"
    assert metadata.get("certification") == "certified"
    assert metadata.get("hidden") is False
    assert metadata.get("serverMinVersion") == "6.5.0"
    assert metadata.get("currentVersion") == "1.2.13"
    assert sorted(metadata.get("tags", [])) == sorted(["Data Source", "Collection"])
    assert metadata.get("categories") == ["Utilities"]

    metadata_integration = metadata.get("contentItems", {}).get("integration", [{}])[0]
    assert len(metadata_integration.get("commands", [])) == 1
    assert metadata_integration.get("fromversion") == "6.8.0"
    assert metadata_integration.get("id") == "TestIntegration"
    assert metadata_integration.get("isfetchevents") is True
    assert metadata_integration.get("name") == "Test Integration"

    metadata_playbook = metadata.get("contentItems", {}).get("playbook", [{}])[0]
    assert metadata_playbook.get("fromversion") == "6.10.0"
    assert metadata_playbook.get("id") == "MyPlaybook"
    assert metadata_playbook.get("name") == "MyPlaybook"

    metadata_modeling_rule = metadata.get("contentItems", {}).get("modelingrule", [{}])[
        0
    ]
    assert metadata_modeling_rule.get("fromversion") == "6.10.0"
    assert metadata_modeling_rule.get("id") == "my_ModelingRule"
    assert metadata_modeling_rule.get("name") == "My Modeling Rule"
