# test/GithubTest.py

from scrape_up import github

class UserTest():

    def __init__(self, username):
        self.username = username
        #SetUp
        self.user = github.Users(username=self.username)

    def test_followers(self):
        followers = self.user.followers()
        return followers

    def test_following(self):
        following = self.user.following()
        return following

    def test_get_avatar(self):
        avatar = self.user.get_avatar()
        return avatar  

    def test_get_bio(self):
        bio = self.user.get_bio()
        return bio

    def test_get_repo(self):
        repos = self.user.get_repo()
        return repos

    def test_repo_count(self):
        repo_count = self.user.repo_count()
        return repo_count

    def test_star_count(self):
        star_count = self.user.star_count()
        return star_count

    def test_get_yearly_contributions(self):
        contributions = self.user.get_yearly_contributions()
        return contributions

    def test_get_repositories(self):
        repositories = self.user.get_repositories()
        return repositories

    def test_get_starred_repos(self):
        starred_repos = self.user.get_starred_repos()
        return starred_repos
        
    def test_pull_requests(self):
        pull_requests = self.user.pul_requests()
        return pull_requests

    def test_get_followers(self):
        followers = self.user.get_followers()
        return followers

    def test_get_following_users(self):
        following_users = self.user.get_following_users()
        return following_users

    def test_get_achievements(self):
        achievements = self.user.get_achievements()
        return achievements

    def test_get_status(self):
        status = self.user.get_status()
        return status
        
    def test_get_contribution_streak(self):
        contribution_streak = self.user.get_contribution_streak()
        return contribution_streak
        
    def test_get_repository_details(self):
        repository_details = self.user.get_repository_details()
        return repository_details
        
    def test_get_branch(self):
        branches = self.user.get_branch()
        return branches

class RepositoryTest:

    def __init__(self, username, repo):
        self.username = username
        self.repo = repo
        #SetUp
        self.repository = github.Repository(self.username, repo)
    
    def test_fork_count(self):
        fork_count = self.repository.fork_count()
        return fork_count

    def test_get_contributors(self):
        contributors = self.repository.get_contributors()
        return contributors
    
    def test_topics(self):
        topics = self.repository.topics()
        return topics
    
    def test_pull_requests(self):
        pull_requests = self.repository.pull_requests()
        return pull_requests

    def test_last_updated_at(self):
        last_updated_at = self.repository.last_update_at()
        return last_updated_at

    def test_tags(self):
        tags = self.repository.tags()
        return tags

    def test_releases(self):
        releases = self.repository.releases()
        return releases

    def test_issues_count(self):
        issues_count = self.repository.issues_count()
        return issues_count        

    def test_readme(self):
        readme_path = self.repository.readme()
        return readme_path

    def test_get_pull_requests_ids(self):
        pull_requests_ids = self.repository.get_pull_requests_ids()
        return pull_requests_ids

    def test_get_issues(self):
        issues = self.repository.get_issues()
        return issues

    def test_commits(self):
        commits = self.repository.commits()
        return commits    

    def test_get_readme(self):
        readme = self.repository.get_readme()
        return readme

    def test_get_environment(self):
        environment = self.repository.get_environment()
        return environment    

    def test_watch_count(self):
        watch_count = self.repository.watch_count()
        return watch_count

    def test_all_watchers(self):
        watchers = self.repository.all_watchers()
        return watchers

class IssueTest:

    def __init__(self, username, repo, issue_no):
        self.username = username
        self.repo = repo
        self.issue_no = issue_no
        #SetUp
        self.issue = github.Issue(self.username, self.repo, self.issue_no)

    def test_assignees(self):
        assignees = self.issue.assignees()
        return assignees

    def test_labels(self):
        labels = self.issue.labels()
        return labels

    def test_opened_by(self):
        opened_by = self.issue.opened_by()
        return opened_by

    def test_title(self):
        title = self.issue.title()
        return title

    def test_is_milestone(self):
        milestone = self.issue.is_milestone()
        return milestone

    def test_opened_at(self):
        opened_at = self.issue.opened_at()
        return opened_at

class PullRequestTest:

    def __init__(self, username, repo, pr_no):
        self.username= username
        self.repo = repo
        self.pr_no = pr_no
        #SetUp
        self.pull_request = github.PullRequest(self.username, self.repo, self.pr_no)
    
    def test_commits(self):
        commits = self.pull_request.commits()
        return commits

    def test_title(self):
        title = self.pull_request.title()
        return title

    def test_labels(self):
        labels = self.pull_request.labels()
        return labels    

    def test_files_changed(self):
        files_changed = self.pull_request.files_changed()
        return files_changed

    def test_reviewers(self):
        reviewers = self.pull_request.reviewers()
        return reviewers

class OrganizationTest:

    def __init__(self, org_name):
        self.org_name = org_name
        #SetUp
        self.organization = github.Organization(self.org_name)
    
    def test_top_topics(self):
        top_topics = self.organization.top_topics()
        return top_topics

    def test_followers(self):
        followers = self.organization.followers()
        return followers

    def test_top_languages(self):
        top_languages = self.organization.top_languages()
        return top_languages

    def test_avatar(self):
        avatar = self.organization.avatar()
        return avatar

    def test_repositories(self):
        repositories = self.organization.repositories()
        return repositories

    def test_people(self):
        people = self.organization.people()
        return people

    def test_peoples(self):
        peoples = self.organization.peoples()
        return peoples

    def test_get_location(self):
        location = self.organization.get_location()
        return location

    def test_repository_details(self):
        repository_details = self.organization.repository_details()
        return repository_details

    def test_pinned_repository(self):
        pinned_repository = self.organization.pinned_repository()
        return pinned_repository

    def test_get_organization_links(self):
        organization_links = self.organization.get_organization_links()
        return organization_links

# src/test/leetcode_test.py

from scrape_up.leetcode.leetcode_scraper import LeetCodeScraper

leetcodeScraper = LeetCodeScraper()
print("problems list: ", leetcodeScraper.get_problems(difficulty="medium", tags=["dynamic-programming"], search_key="palindrome"))
print("contests: ", leetcodeScraper.get_contests())
print("daily challenge: ", leetcodeScraper.get_daily_challenge())

leetcodeScraper = LeetCodeScraper(username="test")
print("rank: ", leetcodeScraper.scrape_rank())
print("rating: ", leetcodeScraper.scrape_rating())
print("total problems: ", leetcodeScraper.get_problems_solved())
print("difficulty wise problems: " ,leetcodeScraper.get_solved_by_difficulty())
print("github link: ", leetcodeScraper.get_github_link())
print("linkedin link: ", leetcodeScraper.get_linkedin_link())
print("community stats: ", leetcodeScraper.get_community_stats())

# src/scrape_up/amazon/__init__.py

from products import Product

__all__ = ["Product"]


# src/scrape_up/amazon/products.py

import requests
from bs4 import BeautifulSoup


# scraping amazon product page
class Product:
    def __init__(self, product_name: str):
        self.product_name = product_name

    def get_product(self):
        """
        Class - `Product`\n
        Example -\n
        ```python
        product = Product(product_name="watch")
        product.get_product()
        ```
        Return\n
        ```python
        return
        {
            "data": product_link,
            "message": f"Product data has been fetched",
        }
        ```
        """
        try:
            product_name = self.product_name
            product_name = product_name.replace(" ", "+")
            url = f"https://www.amazon.in/s?k={product_name}"
            headers = {
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \
                    (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
            }
            r = requests.get(url, headers=headers)
            soup = BeautifulSoup(r.content, "html.parser")
            product = soup.find("div", {"class": "s-product-image-container"})
            product_link = product.find("a", {"class": "a-link-normal"})["href"]
            product_link = "https://www.amazon.in" + product_link
            return {
                "data": product_link,
                "message": f"Product data has been fetched",
            }
        except:
            return {
                "data": None,
                "message": f"Unable to fetch product's data",
            }

    # Get product details
    def get_product_details(self):
        """
        Class - `Product`\n
        Example -\n
        ```python
        product = Product(product_name="watch")
        product.get_product_details()
        ```
        Return\n
        ```python
        return
        {
            "data": product_details,
            "message": f"Product detail has been fetched",
        }
        ```
        """
        try:
            product_link = self.get_product()["data"]
            headers = {
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \
                    (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
            }
            r = requests.get(product_link, headers=headers)
            soup = BeautifulSoup(r.content, "html.parser")
            product_name = soup.find("span", {"id": "productTitle"}).text.strip()
            product_price = soup.find("span", {"class": "a-price-whole"}).text.strip()
            product_rating = soup.find(
                "span", {"class": "a-size-base a-color-base"}
            ).text.strip()
            product_details = {
                "product_name": product_name,
                "product_price": product_price,
                "product_rating": product_rating,
                "product_link": product_link,
            }
            return {
                "data": product_details,
                "message": f"Product detail has been fetched",
            }
        except:
            return {
                "data": None,
                "message": f"Unable to fetch product detail",
            }

    # Get product image
    def get_product_image(self):
        """
        Class - `Product`\n
        Example -\n
        ```python
        product = Product(product_name="watch")
        product.get_product_image()
        ```
        Return\n
        ```python
        return
        {
            "data": product_image,
            "message": f"Product image has been fetched",
        }
        ```
        """
        try:
            product_link = self.get_product()["data"]
            headers = {
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \
                    (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
            }
            r = requests.get(product_link, headers=headers)
            soup = BeautifulSoup(r.content, "html.parser")
            product_image = soup.find(
                "img", {"class": "a-dynamic-image a-stretch-horizontal"}
            )["src"]

            return {
                "data": product_image,
                "message": f"Product image has been fetched",
            }
        except:
            return {
                "data": None,
                "message": f"Unable to fetch product image",
            }

    # Get customer reviews
    def customer_review(self):
        """
        Class - `Product`\n
        Example -\n
        ```python
        product = Product(product_name="watch")
        product.customer_review()
        ```
        Return\n
        ```python
        return
        {
            "data": review,
            "message": f"Product review has been fetched",
        }
        ```
        """
        try:
            product_link = self.get_product()["data"]
            headers = {
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \
                    (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
            }
            r = requests.get(product_link, headers=headers)
            soup = BeautifulSoup(r.content, "html.parser")

            review_elements = soup.find_all("div", {"data-hook": "review"})

            for review_element in review_elements:
                reviewer_name = review_element.find(
                    "span", {"class": "a-profile-name"}
                ).text
                rating = (
                    review_element.find("i", {"class": "a-icon-star"})
                    .find("span", {"class": "a-icon-alt"})
                    .text
                )
                review_title = review_element.find(
                    "a", {"data-hook": "review-title"}
                ).text.strip()
                review_date = review_element.find(
                    "span", {"data-hook": "review-date"}
                ).text
                review_text = review_element.find(
                    "span", {"data-hook": "review-body"}
                ).text.strip()
                review = [reviewer_name, rating, review_title, review_date, review_text]
            return {
                "data": review,
                "message": f"Product review has been fetched",
            }
        except:
            return {
                "data": None,
                "message": f"Unable to fetch product review",
            }


# src/scrape_up/askUbuntu/__init__.py

from scrape_up.askUbuntu.questions import AskUbuntu
__all__ = ["AskUbuntu"]

# src/scrape_up/askUbuntu/questions.py

from bs4 import BeautifulSoup
import requests
import json


class AskUbuntu:
    """
    Create an instance of `AskUbuntu` class.

    ```python
    questions = AskUbuntu("topic")
    ```

    | Methods                     | Details                                                                                              |
    | --------------------------- | ---------------------------------------------------------------------------------------------------- |
    | `.getNewQuestions()`        | Returns the new questions, views, votes, answer counts, and descriptions in JSON format              |
    | `.getActiveQuestions()`     | Returns the active questions, views, votes, answer counts, and descriptions in JSON format           |
    | `.getUnansweredQuestions()` | Returns the unanswered questions, views, votes, answer counts, and descriptions in JSON format       |
    | `.getBountiedQuestions()`   | Returns the bountied questions, views, votes, answer counts, and descriptions in JSON format         |
    | `.getFrequentQuestions()`   | Returns the frequently asked questions, views, votes, answer counts, and descriptions in JSON format |
    | `.getHighScoredQuestions()` | Returns the most voted questions, views, votes, answer counts, and descriptions in JSON format       |
    """

    def __init__(self, topic):
        self.topic = topic

    def getNewQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getNewQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Newest"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getActiveQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getActiveQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Active"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getUnansweredQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getUnansweredQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Unanswered"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getBountiedQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getBountiedQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Bountied"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getFrequentQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getFrequentQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Frequent"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getHighScoredQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getHighScoredQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Votes"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

ask = AskUbuntu(topic="Python")
print(ask.getActiveQuestions())

# src/scrape_up/codeforces/__init__.py

from user import USER
__all__ = ['USER']

# src/scrape_up/codeforces/user.py

import requests
from bs4 import BeautifulSoup


class USER:
    """
    Fetches user data from CodeForces Competitive Platform
    """

    user_url = "https://codeforces.com/profile/{}"

    def __init__(self, username):
        self.username = username
        self.userdata = self.user_get_data()

    def user_get_data(self):
        """
        Gets all information of a username from codeforces.

        Class - `codeforces.user()`\n
        Example -\n
        ```python
        user = USER("benq")
        print(user.user_get_data())
        ```
        Return\n
        ```python
        return
        lastName → Qi
        country → United States
        lastOnlineTimeSeconds → 1686275190
        city → Princeton
        rating → 3783
        friendOfCount → 12513
        titlePhoto → https://userpic.codeforces.org/312472/title/7cf0a442d4071e87.jpg
        handle → Benq
        avatar → https://userpic.codeforces.org/312472/avatar/5716ac69aea8159a.jpg
        firstName → Benjamin
        contribution → 50
        organization → MIT
        rank → legendary grandmaster
        maxRating → 3813
        registrationTimeSeconds → 1435099979
        maxRank → legendary grandmaster
        lastName → Qi
        country → United States
        lastOnlineTimeSeconds → 1686275190
        city → Princeton
        rating → 3783
        friendOfCount → 12513
        titlePhoto → https://userpic.codeforces.org/312472/title/7cf0a442d4071e87.jpg
        handle → Benq
        avatar → https://userpic.codeforces.org/312472/avatar/5716ac69aea8159a.jpg
        firstName → Benjamin
        contribution → 50
        organization → MIT
        rank → legendary grandmaster
        maxRating → 3813
        registrationTimeSeconds → 1435099979
        maxRank → legendary grandmaster
        Data Fetching Successful
        ```
        """
        try:
            url = self.user_url.format(self.username)
            info = requests.get(url)
            soup = BeautifulSoup(info.content)
            user_info_set = info.json()
            user_info_data = user_info_set["result"][0]
            for data in user_info_data:
                print(f"{data} → {user_info_data[data]}")
            return (
                "Data Fetching Successful"
                if user_info_set["status"] == "OK"
                else "Failed"
            )
        except IndexError:
            raise Exception("Invalid User Name.")
        except TypeError:
            raise Exception("Invalid User Name")
        except requests.exceptions.ConnectionError:
            raise Exception("Connection Error, Please try again.")


if __name__ == "__main__":
    user = USER("benq")
    print(user.user_get_data())


# src/scrape_up/coursera/__init__.py

from scrape_up.coursera.courses import Courses
__all__=['Courses']

# src/scrape_up/coursera/courses.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options


class Coursera:
    def __init__(self, keyword, page_count):
        self.keyword = keyword
        self.page_count = page_count

    def __scrape_page(self):
        chromedriver_path = ""
        options = Options()
        options.add_argument("--headless")
        driver = webdriver.Chrome(service=Service(chromedriver_path), options=options)
        wait = WebDriverWait(driver, 10)
        driver.get("https://www.coursera.org/search?query=" + self.keyword)
        return wait, driver

    def titles(self):
        """
        Class - `Courses`
        Example:
        ```
        topic = Courses("Machine Learning", 11)
        courses = topic.titles()
        ```
        Returns:
        {
            "data": titles,
            "message": f"Course Titles for {self.keyword}"
        }
        """
        wait, driver = self.__scrape_page()
        titles = []
        try:
            for i in range(self.page_count):
                courses = wait.until(
                    EC.visibility_of_all_elements_located(
                        (By.CSS_SELECTOR, "main ul>li")
                    )
                )
                titles.extend(
                    [
                        driver.execute_script(
                            'return arguments[0].querySelector("h3")?.innerText', course
                        )
                        for course in courses
                    ]
                )
                next_btn = driver.find_element(
                    By.CSS_SELECTOR, 'button[aria-label="Next Page"]'
                )
                if "disabled" in next_btn.get_attribute("class"):
                    break
                else:
                    next_btn.click()
            return {"data": titles, "message": f"Course Titles for {self.keyword}"}
        except:
            return {"data": None, "message": f"No courses found for {self.keyword}"}


# src/scrape_up/eazydiner/__init__.py

from scrape_up.eazydiner.eazydiner import EazyDiner
__all__ = ["EazyDiner"]

# src/scrape_up/eazydiner/eazydiner.py

import requests
from bs4 import BeautifulSoup
import json


class EazyDiner:
    """
    Class - `EazyDiner`
    Example:
    ```
    hotels = EazyDiner(location="Delhi NCR")
    ```\n
    Methods :\n
    1. ``.getRestaurants() | Response - List of restraunts and its details.
    """

    def __init__(self, location):
        self.location = location

    def getRestaurants(self):
        """
        Class - `EazyDiner`
        Example:
        ```
        del = EazyDiner("Delhi NCR") or del = EazyDiner("delhi-ncr")
        del.getRestaurants()
        ```
        Returns:
        {
            "restaurant": restaurant name
            "location": location of restaurant
            "rating": rating
            "cuisine": cuisines provided
            "price": price for two people
        }
        """
        url = (
            "https://www.eazydiner.com/restaurants?location="
            + self.location.replace(" ", "-").replace(",", "").lower()
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            restaurant_data = {"restaurants": []}

            restaurants = soup.select(".restaurant")
            for r in restaurants:
                name = r.find("h3", class_="res_name").getText().strip()
                location = r.find("h3", class_="res_loc").getText().strip()
                rating = r.find("span", class_="critic").getText().strip()
                cuisine = (
                    r.find("div", class_="res_cuisine").getText().replace(",", ", ")
                )
                price = (
                    r.find("span", class_="cost_for_two")
                    .getText()
                    .encode("ascii", "ignore")
                    .decode()
                    .strip()
                )
                restaurant_data["restaurants"].append(
                    {
                        "restaurant": name,
                        "location": location,
                        "rating": rating,
                        "cuisine": cuisine,
                        "price": "Rs. " + price + " for two",
                    }
                )
            res_json = json.dumps(restaurant_data)
            return res_json
        except:
            error_message = {
                "message": "There are no restaurants in the given location."
            }
            ejson = json.dumps(error_message)
            return ejson

    def getBreakfast(self):
        """
        Class - `EazyDiner`
        Example:
        ```
        del = EazyDiner("Delhi NCR") or del = EazyDiner("delhi-ncr")
        del.getBreakfast()
        ```
        Returns:
        {
            "restaurant": restaurant name
            "location": location of restaurant
            "rating": rating
            "cuisine": cuisines provided
            "price": price for two people
        }
        """
        url = (
            "https://www.eazydiner.com/restaurants?location="
            + self.location.replace(" ", "-").replace(",", "").lower()
            + "&meal_period=breakfast"
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            restaurant_data = {"restaurants": []}

            restaurants = soup.select(".restaurant")
            for r in restaurants:
                name = r.find("h3", class_="res_name").getText().strip()
                location = r.find("h3", class_="res_loc").getText().strip()
                rating = r.find("span", class_="critic").getText().strip()
                cuisine = (
                    r.find("div", class_="res_cuisine").getText().replace(",", ", ")
                )
                price = (
                    r.find("span", class_="cost_for_two")
                    .getText()
                    .encode("ascii", "ignore")
                    .decode()
                    .strip()
                )
                restaurant_data["restaurants"].append(
                    {
                        "restaurant": name,
                        "location": location,
                        "rating": rating,
                        "cuisine": cuisine,
                        "price": "Rs. " + price + " for two",
                    }
                )
            res_json = json.dumps(restaurant_data)
            return res_json
        except:
            error_message = {
                "message": "There are no restaurants in the given location."
            }
            ejson = json.dumps(error_message)
            return ejson

    def getLunch(self):
        """
        Class - `EazyDiner`
        Example:
        ```
        del = EazyDiner("Delhi NCR") or del = EazyDiner("delhi-ncr")
        del.getLunch()
        ```
        Returns:
        {
            "restaurant": restaurant name
            "location": location of restaurant
            "rating": rating
            "cuisine": cuisines provided
            "price": price for two people
        }
        """
        url = (
            "https://www.eazydiner.com/restaurants?location="
            + self.location.replace(" ", "-").replace(",", "").lower()
            + "&meal_period=lunch"
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            restaurant_data = {"restaurants": []}

            restaurants = soup.select(".restaurant")
            for r in restaurants:
                name = r.find("h3", class_="res_name").getText().strip()
                location = r.find("h3", class_="res_loc").getText().strip()
                rating = r.find("span", class_="critic").getText().strip()
                cuisine = (
                    r.find("div", class_="res_cuisine").getText().replace(",", ", ")
                )
                price = (
                    r.find("span", class_="cost_for_two")
                    .getText()
                    .encode("ascii", "ignore")
                    .decode()
                    .strip()
                )
                restaurant_data["restaurants"].append(
                    {
                        "restaurant": name,
                        "location": location,
                        "rating": rating,
                        "cuisine": cuisine,
                        "price": "Rs. " + price + " for two",
                    }
                )
            res_json = json.dumps(restaurant_data)
            return res_json
        except:
            error_message = {
                "message": "There are no restaurants in the given location."
            }
            ejson = json.dumps(error_message)
            return ejson

    def getDinner(self):
        """
        Class - `EazyDiner`
        Example:
        ```
        del = EazyDiner("Delhi NCR") or del = EazyDiner("delhi-ncr")
        del.getDinner()
        ```
        Returns:
        {
            "restaurant": restaurant name
            "location": location of restaurant
            "rating": rating
            "cuisine": cuisines provided
            "price": price for two people
        }
        """
        url = (
            "https://www.eazydiner.com/restaurants?location="
            + self.location.replace(" ", "-").replace(",", "").lower()
            + "&meal_period=dinner"
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            restaurant_data = {"restaurants": []}

            restaurants = soup.select(".restaurant")
            for r in restaurants:
                name = r.find("h3", class_="res_name").getText().strip()
                location = r.find("h3", class_="res_loc").getText().strip()
                rating = r.find("span", class_="critic").getText().strip()
                cuisine = (
                    r.find("div", class_="res_cuisine").getText().replace(",", ", ")
                )
                price = (
                    r.find("span", class_="cost_for_two")
                    .getText()
                    .encode("ascii", "ignore")
                    .decode()
                    .strip()
                )
                restaurant_data["restaurants"].append(
                    {
                        "restaurant": name,
                        "location": location,
                        "rating": rating,
                        "cuisine": cuisine,
                        "price": "Rs. " + price + " for two",
                    }
                )
            res_json = json.dumps(restaurant_data)
            return res_json
        except:
            error_message = {
                "message": "There are no restaurants in the given location."
            }
            ejson = json.dumps(error_message)
            return ejson


# src/scrape_up/finance/__init__.py

from scrape_up.finance.stock_price import StockPrice

__all__ = ["StockPrice"]


# src/scrape_up/finance/bse.py

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import json


# All the prices returned from this class are in INR
class BSE:
    """
    Handles all data request for bse stock.
    """

    autocomplete_url = "https://api.bseindia.com/Msource/90D/getQouteSearch.aspx?Type=EQ&text={}&flag=gq"
    latest_price_url = "https://api.bseindia.com/BseIndiaAPI/api/getScripHeaderData/w?Debtflag=&scripcode={code}&seriesid="
    historical_data_url = "https://api.bseindia.com/BseIndiaAPI/api/StockReachGraph/w?scripcode={code}&flag=1&fromdate={from_date}&todate={to_date}&seriesid="  # date format yyyymmdd
    chart_data_url = "https://api.bseindia.com/BseIndiaAPI/api/StockReachGraph/w?scripcode={code}&flag=0&fromdate=&todate=&seriesid="
    currency_type = "INR"

    headers = {
        "Connection": "keep-alive",
        "Cache-Control": "max-age=0",
        "DNT": "1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36",
        "Sec-Fetch-User": "?1",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-Mode": "navigate",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9,hi;q=0.8",
    }

    # Creating request.Session object so cookies persist among requests
    fetcher = requests.Session()
    fetcher.headers.update(headers)

    def __init__(self, stock_name):
        self.stock_name = stock_name
        self.stock_code = self.get_data()["code"]
        self.latest_price_url = self.latest_price_url.format(code=self.stock_code)
        self.chart_data_url = self.chart_data_url.format(code=self.stock_code)

    # gets closest matching name and symbol of stock based given stock_name
    def get_data(self):
        try:
            data_html = self.fetcher.get(self.autocomplete_url.format(self.stock_name))
            soup = BeautifulSoup(data_html.text, "html.parser")
            li_element = soup.find("li", class_="quotemenu")
            stock_code = li_element.find("span").text.split()[-1]
            stock_name = li_element.find("strong").text
            if li_element.find("strong").next_sibling.strip:
                stock_name += str(li_element.find("strong").next_sibling.strip())

        except IndexError:
            raise Exception("Invalid Stock Name.")
        except TypeError:
            raise Exception("Invalid Stock Name")
        except requests.exceptions.ConnectionError:
            raise Exception("Connection Error, Please try again.")
        else:
            self.stock_name = stock_name
            return {"code": stock_code, "symbol_info": stock_name}

    def get_latest_price(self):
        """
        Gets Latest stock price info of given bse stock.
        """
        price_info = self.fetcher.get(self.latest_price_url).json()["CurrRate"]
        return {
            "latestPrice": float(price_info["LTP"]),
            "rateChange": float(price_info["Chg"]),
            "pChange": float(price_info["PcChg"]),
        }

    # Please give dates in DD-MM-YYYY format
    def get_historical_data(self, from_date, to_date):
        """
        Gets historical stock price (vwap) in range from_date to to_date
        """

        from_date = datetime.strptime(from_date, "%d-%m-%Y").strftime("%Y%m%d")
        to_date = datetime.strptime(to_date, "%d-%m-%Y").strftime("%Y%m%d")

        historical_price_data_raw = self.fetcher.get(
            self.historical_data_url.format(
                code=self.stock_code, from_date=from_date, to_date=to_date
            )
        ).json()["Data"]

        historical_price_data_raw = json.loads(historical_price_data_raw)
        historical_price_data = {}

        for item in historical_price_data_raw:
            dttm = datetime.strptime(item["dttm"], "%a %b %d %Y %H:%M:%S").strftime(
                "%d-%m-%Y"
            )
            vale1 = float(item["vale1"])
            historical_price_data[dttm] = vale1

        return historical_price_data


if __name__ == "__main__":
    infosys = BSE(stock_name="infosys ltd")
    print(infosys.get_latest_price())
    print(infosys.get_historical_data("05-06-2023", "10-06-2023"))


# src/scrape_up/finance/nasdaq.py

import requests
from datetime import datetime


# All prices returned form this class are in USD
class NASDAQ:
    """
    Handles all requests for nasdaq stock
    """

    autocomplete_url = "https://api.nasdaq.com/api/search/result?query={}&order=relevence&offset=0&lang=en"
    latest_price_url = (
        "https://api.nasdaq.com/api/quote/{symbol}/info?assetclass=stocks"
    )
    chart_data_url = "https://api.nasdaq.com/api/quote/{symbol}/chart?assetclass=stocks"
    historical_data_url = "https://api.nasdaq.com/api/quote/{symbol}/historical?assetclass=stocks&fromdate={from_date}&limit=0&todate={to_date}"  # date in yyyy-mm-dd format
    currency_type = "USD"

    headers = {
        "Connection": "keep-alive",
        "Cache-Control": "max-age=0",
        "DNT": "1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36",
        "Sec-Fetch-User": "?1",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-Mode": "navigate",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9,hi;q=0.8",
    }

    # Creating request.Session object so cookies persist among requests
    fetcher = requests.Session()
    fetcher.headers.update(headers)

    def __init__(self, stock_name):
        self.stock_name = stock_name
        self.stock_symbol = self.get_data()["symbol"]
        self.latest_price_url = self.latest_price_url.format(symbol=self.stock_symbol)
        self.chart_data_url = self.chart_data_url.format(symbol=self.stock_symbol)

    def get_data(self):
        try:
            data = self.fetcher.get(
                self.autocomplete_url.format(self.stock_name)
            ).json()["data"]["results"][0]
        except IndexError:
            raise Exception("Invalid Stock Name.")
        except TypeError:
            raise Exception("Invalid Stock Name")
        except requests.exceptions.ConnectionError:
            raise Exception("Connection Error, Please try again.")
        else:
            self.stock_name = data["name"][0]
            return {"symbol": data["symbol"][0], "symbol_info": data["name"][0]}

    def get_latest_price(self):
        """
        Gets Latest stock price info of given nse stock.
        Class - `finance.NASDAQ()`\n
        Example -\n
        ```python
        google = NASDAQ("alphabet")
        print(google.get_latest_price())
        ```
        Return\n
        ```python
        return
        {
            "data": data,
            "message": f"Latest price of the stock has been fetched",
        }
        ```
        """
        try:
            price_info = self.fetcher.get(self.latest_price_url).json()["data"][
                "primaryData"
            ]
            data = {
                "latestPrice": float(price_info["lastSalePrice"][1:]),
                "rateChange": float(price_info["netChange"]),
                "pChange": float(price_info["percentageChange"][:-1]),
            }
            return {
                "data": data,
                "message": f"Latest price of the stock has been fetched",
            }
        except:
            return {
                "data": None,
                "message": f"Not able to fetch latest price of the stock",
            }

    # Please give dates in DD-MM-YYYY format
    def get_historical_data(self, from_date, to_date):
        """
        Gets Latest stock price info of given nse stock.
        Class - `finance.NASDAQ()`\n
        Example -\n
        ```python
        google = NASDAQ("alphabet")
        print(google.get_historical_data("05-06-2023", "09-06-2023"))
        ```
        Return\n
        ```python
        return
        {
            "data": historical_price_data,
            "message": f"Sucessfylly scrapped the historic data of the stock.",
        }
        ```
        """
        try:
            from_date = datetime.strptime(from_date, "%d-%m-%Y").strftime("%Y-%m-%d")
            to_date = datetime.strptime(to_date, "%d-%m-%Y").strftime("%Y-%m-%d")
            historical_price_data_raw = self.fetcher.get(
                self.historical_data_url.format(
                    symbol=self.stock_symbol, from_date=from_date, to_date=to_date
                )
            ).json()["data"]["tradesTable"]["rows"]
            historical_price_data = {}
            for i in historical_price_data_raw:
                date_formatted = datetime.strptime(i["date"], "%m/%d/%Y").strftime(
                    "%d-%m-%Y"
                )
                historical_price_data[date_formatted] = float(i["open"][1:])
            return {
                "data": historical_price_data,
                "message": f"Sucessfylly scrapped the historic data of the stock.",
            }
        except:
            return {
                "data": None,
                "message": f"Not able to fetch historical data of the stock.",
            }


if __name__ == "__main__":
    google = NASDAQ("alphabet")
    print(google.get_latest_price())
    print(google.get_historical_data("05-06-2023", "09-06-2023"))


# src/scrape_up/finance/nse.py

import requests


# All the prices returned from this class are in INR
class NSE:
    """
    Handles all data request for nse stock.
    """

    autocomplete_url = "https://www.nseindia.com/api/search/autocomplete?q={}"
    latest_price_url = "https://www.nseindia.com/api/quote-equity?symbol={symbol}"
    chart_data_url = "https://www.nseindia.com/api/chart-databyindex?index={symbol}"
    historical_data_url = 'https://www.nseindia.com/api/historical/cm/equity?symbol={symbol}&series=["EQ"]&from={from_date}&to={to_date}'
    currency_type = 'INR'
    
    headers = {
        "Connection": "keep-alive",
        "Cache-Control": "max-age=0",
        "DNT": "1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36",
        "Sec-Fetch-User": "?1",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-Mode": "navigate",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9,hi;q=0.8",
    }

    # Creating request.Session object so cookies persist among requests
    fetcher = requests.Session()
    fetcher.headers.update(headers)

    def __init__(self, stock_name):
        self.stock_name = stock_name
        self.stock_symbol = self.get_data()["symbol"]
        self.latest_price_url = self.latest_price_url.format(symbol=self.stock_symbol)
        self.chart_data_url = self.chart_data_url.format(symbol=self.stock_symbol)

    # gets closest matching name and symbol of stock based given stock_name
    def get_data(self):
        try:
            """
            The line below is necessary as nseindia.com does no give access to its
            api unless you have cookies of visiting main page.
            Hence visiting mainpage would imitate behaviour of a human user and let us
            access all the nseindia APIs.
            """
            main_page = self.fetcher.get("https://www.nseindia.com")
            data = self.fetcher.get(
                self.autocomplete_url.format(self.stock_name)
            ).json()["symbols"][0]
        except IndexError:
            raise Exception("Invalid Stock Name.")
        except TypeError:
            raise Exception("Invalid Stock Name")
        except requests.exceptions.ConnectionError:
            raise Exception("Connection Error, Please try again.")
        else:
            self.stock_name = data["symbol_info"]
            return {"symbol": data["symbol"], "symbol_info": data["symbol_info"]}

    # please use this return format for other classes (like bse, nasdaq) too if they are created in future
    def get_latest_price(self):
        """
        Gets Latest stock price info of given nse stock.
        """
        price_info = self.fetcher.get(self.latest_price_url).json()["priceInfo"]
        return {
            "latestPrice": price_info["lastPrice"],
            "rateChange": price_info["change"],
            "pChange": price_info["pChange"],
            "open": price_info["open"],
            "close": price_info["close"],
            "vwap": price_info["vwap"],
        }

    # Please give dates in DD-MM-YYYY format
    def get_historical_data(self, from_date, to_date):
        """
        Gets historical stock price (vwap) in range from_date to to_date
        """
        historical_price_data_raw = self.fetcher.get(
            self.historical_data_url.format(
                symbol=self.stock_symbol, from_date=from_date, to_date=to_date
            )
        ).json()["data"]
        historical_price_data = {}
        for i in historical_price_data_raw:
            historical_price_data[i["mTIMESTAMP"]] = i["VWAP"]
        return historical_price_data




# src/scrape_up/finance/stock_price.py

try:
    from nse import NSE
    from nasdaq import NASDAQ
    from bse import BSE
except ModuleNotFoundError:
    from scrape_up.finance.nse import NSE
    from scrape_up.finance.nasdaq import NASDAQ
    from scrape_up.finance.bse import BSE


class StockPrice:
    """
    This is entry point for users to scrape data.
    Usage:
        Create an instance of Stock price class for Infosys
        >>> infosys_stock = StockPrice('infosys','nse')
        >>> infosys_stock = StockPrice('infosys')

        # the stock_index option is optional as it is set to 'nse' by default for now

        To get latest price info of Infosys's Stock
        >>> latest_price_info = infosys_stock.get_latest_price()

        To get stock price of Infosys from 02-05-2023 to 31-05-2023
        >>> historical_price_info = infosys_stock.get_historical_data('02-05-2023', '31-05-2023')

    """

    def __init__(self, stock_name, stock_index):
        self.stock_name = stock_name.strip()
        self.stock_index = stock_index.strip().lower()
        stock_class = None
        if self.stock_index == "nse":
            stock_class = NSE
        elif self.stock_index == "nasdaq":
            stock_class = NASDAQ
        elif self.stock_index == "bse":
            stock_class = BSE
        if stock_class:
            print(f"Searching for {self.stock_name} {self.stock_index} stock...")
            try:
                self.stock = stock_class(stock_name)
                self.stock_name = self.stock.stock_name
                print(
                    f"Found stock based on provided name: {self.stock_name}\nStock instance created"
                )
            except:
                self.stock = None
                print(f"No stock found based on provided name: {self.stock_name}")
        else:
            print(
                f"{self.stock_index} stock scraping code under development.\nPlease wait for Update."
            )

    # these two functions could potentially be modified to
    # also return data as pandas format
    def get_latest_price(self):
        """
        Gets Latest stock price info of given stock
        Gets Latest stock price info of given nse stock.
        Class - `finance.StockPrice()`\n
        Example -\n
        ```python
        infosys_stock = StockPrice("infosys", "nse")
        print(infosys_stock.get_latest_price())
        ```
        Return\n
        ```python
        return
        {
            "data": data,
            "message": f"No latest stock price found for {self.stock_name}"
        }
        ```
        """
        try:
            data = self.stock.get_latest_price()
            message = f"Found latest stock price(in {self.stock.currency_type}) for {self.stock_name}"
        except:
            data = None
            message = f"No latest stock price found for {self.stock_name}"
        finally:
            return {
                "data": data,
                "message": message,
            }

    def get_historical_data(self, from_date, to_date):
        """
        Gets historical stock price (vwap) in range from_date to to_date
        Class - `finance.StockPrice()`\n
        Example -\n
        ```python
        infosys_stock = StockPrice("infosys", "nse")
        print(infosys_stock.get_historical_data('02-05-2023', '31-05-2023'))
        ```
        Return\n
        ```python
        return
        {
            "data": data,
            "message": f"No historial stock price found for {self.stock_name}"
        }
        ```
        """
        try:
            data = self.stock.get_historical_data(from_date, to_date)
            message = f"Historical stock price(in {self.stock.currency_type}) found for {self.stock_name} in range {from_date} to {to_date}"
        except:
            data = None
            message = f"No historial stock price found for {self.stock_name}"
        finally:
            return {
                "data": data,
                "message": message,
            }


# src/scrape_up/flipkart/__init__.py

from flipkart_file import Flipkart

__all__ = ["Flipkart"]


# src/scrape_up/flipkart/flipkart_file.py

import bs4
from bs4 import BeautifulSoup as bs
import requests


class Flipkart:
    """
    Class - `Flipkart`\n
    Instantiate a class\n
    Example -\n
    ```python
    item = Flipkart()
    ```
    \n
    Available Methods\n
    1. TVs - To fetch the list of trending TVs on Flipkart with other details
    2. BestsellersBooks - To fetch the bestselling items listed on FLipkart

    """

    def __init__(self):
        pass

    def TVs(self):
        try:
            """
            Class - `Flipkart`\n
            Example -\n
            ```python
            item = Flipkart()
            item.TVs()
            ```
            Return\n
            ```python
            return {all_items with details in json}
            ```
            """

            link = "https://www.flipkart.com/search?q=tv&as=on&as-show=on&otracker=AS_Query_TrendingAutoSuggest_8_0_na_na_na&otracker1=AS_Query_TrendingAutoSuggest_8_0_na_na_na&as-pos=8&as-type=TRENDING&suggestionId=tv&requestId=9c9fa553-b7e5-454b-a65b-bbb7a9c74a29"
            page = requests.get(link)
            soup = bs(page.content, "html.parser")

            all_items = []

            for data in soup.findAll("div", class_="_3pLy-c row"):
                names = data.find("div", attrs={"class": "_4rR01T"})
                price = data.find("div", attrs={"class": "_30jeq3 _1_WHN1"})
                rating = data.find("div", attrs={"class": "_3LWZlK"})
                specification = data.find("div", attrs={"class": "fMghEO"})

                specs = []
                for spec in specification.find_all("li", attrs={"class": "rgWa7D"}):
                    specs.append(spec.text)

                item_details = {
                    "Item_Name": names.text,
                    "Price": price.text,
                    "Rating": rating.text,
                    "Specifications": ", ".join(specs),
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None

    def BestsellersBooks(self):
        try:
            """
            Class - `Flipkart`\n
            Example -\n
            ```python
            item = Flipkart()
            item.BestsellersBooks()
            ```
            Return\n
            ```python
            return {all_items with details in json}
            ```
            """

            link = "https://www.flipkart.com/search?q=bestsellers&otracker=search&otracker1=search&marketplace=FLIPKART&as-show=on&as=off"
            page = requests.get(link)
            soup = bs(page.content, "html.parser")

            all_items = []

            for data in soup.findAll("div", class_="_4ddWXP"):
                names = data.find("a", attrs={"class": "s1Q9rs"})
                price = data.find("div", attrs={"class": "_30jeq3"})
                rating = data.find("div", attrs={"class": "_3LWZlK"})
                specification = data.find("div", attrs={"class": "_3Djpdu"})

                item_details = {
                    "Item_Name": names.text,
                    "Price": price.text,
                    "Rating": rating.text if rating else None,
                    "Specifications": specification.text if specification else None,
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None

    def SportsShoes(self):
        try:
            """
            Class - `Flipkart`\n
            Example -\n
            ```python
            item = Flipkart()
            item.SportsShoes()
            ```
            Return\n
            ```python
            return {all_items with details in json}
            ```
            """

            url = "https://www.flipkart.com/mens-footwear/sports-shoes/pr?sid=osp,cil,1cu&otracker=nmenu_sub_Men_0_Sports%20Shoes"
            response = requests.get(url)
            html_content = response.content
            soup = bs(html_content, "html.parser")

            all_items = []

            div_elements = soup.find_all("div", class_="_1xHGtK _373qXS")

            for div in div_elements:
                name_element = div.find("a", class_="IRpwTa")
                name = name_element.text.strip() if name_element else ""

                image_element = div.find("img", class_="_2r_T1I")
                image_url = image_element["src"] if image_element else ""

                details_element = div.find("a", class_="_3bPFwb")
                details = details_element.text.strip() if details_element else ""

                item_details = {
                    "Name": name,
                    "Image URL": image_url,
                    "Details": details,
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None

shoes = Flipkart()
print(shoes.SportsShoes())

# src/scrape_up/github/__init__.py

from scrape_up.github.users import Users
from scrape_up.github.repository import Repository
from scrape_up.github.issue import Issue
from scrape_up.github.organization import Organization
from scrape_up.github.pull_request import PullRequest

__all__ = ["Users", "Repository", "Issue", "Organization", "PullRequest"]


# src/scrape_up/github/issue.py

import requests
from bs4 import BeautifulSoup


class Issue:
    def __init__(self, username: str, repository_name: str, issue_number: int):
        self.username = username
        self.repository = repository_name
        self.issue_number = issue_number

    def __scrape_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/issues/{self.issue_number}"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def assignees(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        assignees = repository.assignees()
        ```
        Returns:
        {
            return {
                "data": assignees,
                "message": f"Found assignees for {self.repository}",
            }
        }
        """
        data = self.__scrape_page()
        try:
            assignees_body = data.find("span", class_="css-truncate js-issue-assignees")
            assignees = []
            for assignee in assignees_body.find_all(
                "a", class_="assignee Link--primary css-truncate-target width-fit"
            ):
                assignees.append(assignee.text.replace("\n", "").strip())
            return {
                "data": assignees,
                "message": f"Found assignees for {self.repository}",
            }
        except:
            message = f"No assignees found for {self.repository}"
            return {
                "data": None,
                "message": message,
            }

    def labels(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        labels = repository.labels()
        ```
        Returns:
        {
            return {
                "data": allLabels,
                "message": f"Found labels for {self.repository}", 
            }
        }
        """
        data = self.__scrape_page()
        try:
            labelsDiv = data.find(class_="js-issue-labels d-flex flex-wrap")
            allLabelsHtml = labelsDiv.find_all(
                class_="css-truncate css-truncate-target width-fit"
            )
            allLabels = []
            for label in allLabelsHtml:
                allLabels.append(label.text)
            return {
                "data": allLabels,
                "message": f"Found labels for {self.repository}",
            }
        except:
            message = f"No labels found for {self.repository}"
            return {
                "data": None,
                "message": message,
            }

    def opened_by(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        opened_by = repository.opened_by()
        ```
        Returns:
        {
            return {
                   "data": author_name,
                "message": f"Found author for {self.repository}",
            }
        }
        """
        data = self.__scrape_page()
        try:
            author_name = data.find("a", class_="author text-bold Link--secondary").text
            return {
                "data": author_name,
                "message": f"Found author for {self.repository}",
            }
        except:
            message = f"No author found for {self.repository}"
            return {
                "data": None,
                "message": message,
            }

    def title(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        title = repository.title()
        ```
        Returns:
        {
            return {
               "data": title,
                "message": f"Found title for {self.repository}",
            }
        }
        
        """
        data = self.__scrape_page()
        try:
            title_body = data.find("bdi", class_="js-issue-title markdown-title")
            title = title_body.text.strip()
            return {
                "data": title,
                "message": f"Found title for {self.repository}",
            }
        except:
            message = f"No title found for {self.repository}"
            return {
                "data": None,
                "message": message,
            }

    def opened_at(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        opened_at = repository.opened_at()
        ```
        Returns:
        {
            return {
               "data": data.find("relative-time").text,
                "message": f"Found time for {self.repository}",
            }
        }
        """
        try:
            data = self.__scrape_page()
            return {
                "data": data.find("relative-time").text,
                "message": f"Found time for {self.repository}",
            }
        except:
            message = f"No time found for {self.repository}"
            return {
                "data": None,
                "message": message,
            }

    def is_milestone(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        is_milestone = repository.is_milestone()
        ```
        Returns:
        {
            return {
               "data": milestone,
                "message": f"Found milestone for {self.repository}",
            }
        }
        """
        data = self.__scrape_page()
        try:
            milestone = data.find(
                "a", class_="Link--secondary mt-1 d-block text-bold css-truncate"
            ).text.strip()
            return {
                "data": milestone,
                "message": f"Found milestone for {self.repository}",
            }
        except:
            message = f"No milestone found for {self.repository}"
            return {
                "data": None,
                "message": message,
            }


# src/scrape_up/github/organization.py

import requests
from bs4 import BeautifulSoup
import json


class Organization:
    def __init__(self, organization_name: str):
        self.organization = organization_name

    def __scrape_page(self):
        data = requests.get(f"https://github.com/{self.organization}")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def top_languages(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        top_languages = repository.top_languages()
        ```
        Returns:
        {
            "data": languages,
            "message": f"Found languages for {self.organization}",
            }
            
        """
        try:
            languages = []
            data = self.__scrape_page()
            lang_raw = data.find_all(
                "a", class_="no-wrap color-fg-muted d-inline-block Link--muted mt-2"
            )
            for lang in lang_raw:
                languages.append(lang.get_text().strip())
            return {
                "data": languages,
                "message": f"Found languages for {self.organization}",
            }
        except:
            message = f"No languages found for {self.organization}"
            return {
                "data": languages,
                "message": message,
            }

    def top_topics(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        top_topics = repository.top_topics()
        ```
        Returns:
        {
            "data": topics,
            "message": f"Found topics for {self.organization}",
        }   
        """
        page = self.__scrape_page()
        try:
            all_topics = page.find_all(class_="topic-tag topic-tag-link")
            topics = []
            for topic in all_topics:
                topics.append(topic.text.strip())
            return {
                "data": topics,
                "message": f"Found topics for {self.organization}",
            }
        except:
            message = f"No topics found for {self.organization}"
            return {
                "data": topics,
                "message": message,
            }

    def followers(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        followers = repository.followers()
        ```
        Returns:
        {
            "data": followers,
            "message": f"Found {followers} followers for {self.organization}",
        }   
        """
        page = self.__scrape_page()
        try:
            followers_body = page.find(
                "a", class_="Link--secondary no-underline no-wrap"
            )
            followers = followers_body.span.text.strip()
            return {
                "data": followers,
                "message": f"Found {followers} followers for {self.organization}",
            }
        except:
            message = f"No followers found for {self.organization}"
            return {
                "data": None,
                "message": message,
            }

    def avatar(self):
    
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        avatar = repository.avatar()
        ```
        Returns:
        {
            "data": url,
            "message": f"Found avatar for {self.organization}",
            
        }   
        """
        page = self.__scrape_page()
        try:
            avatar = page.find("a", attrs={"itemprop": "url"})
            url = avatar.text.strip()
            return {
                "data": url,
                "message": f"Found avatar for {self.organization}",
            }
        except:
            message = f"No avatar found for {self.organization}"
            return {
                "data": None,
                "message": message,
            }

    def __scrape_repositories_page(self):
        """
        scrapes the head page of repositories of an organization
        """
        organization = self.organization
        data = requests.get(f"https://github.com/orgs/{organization}/repositories")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_repositories(self, page):
        """
        scrapes the repositories page of an organization
        """
        data = requests.get(page)
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def repositories(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        repositories = repository.repositories()
        ```
        Returns:
        {
           "data": repositories,
            "message": f"Found {len(repositories)} repositories for {organization}",

        }   
        """
        organization = self.organization
        data = self.__scrape_repositories_page()
        try:
            pages_body = data.find("div", class_="paginate-container")
            current_page = pages_body.find("em", class_="current")
            total_pages = 1
            if current_page != None:
                total_pages = (int)(current_page["data-total-pages"])

            pages = []
            if total_pages == 1:
                pages.append(f"https://github.com/orgs/{organization}/repositories")
            else:
                for i in range(1, total_pages + 1):
                    pages.append(
                        f"https://github.com/orgs/{organization}/repositories?page={i}"
                    )

            repositories = []
            for page in pages:
                page_data = self.__scrape_repositories(page)
                repositories_body = page_data.find("div", id="org-repositories")
                for repo in repositories_body.find_all(
                    "a", attrs={"itemprop": "name codeRepository"}
                ):
                    repositories.append(repo.text.strip())

            return {
                "data": repositories,
                "message": f"Found {len(repositories)} repositories for {organization}",
            }
        except:
            message = f"No repositories found for {organization}"
            return {
                "data": repositories,
                "message": message,
            }

    def __scrape_people_page(self):
        """
         scrapes the head page of people of an organization

        """
        organization = self.organization
        data = requests.get(f"https://github.com/orgs/{organization}/people")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_people(self, page):
        """
        scrapes the people page of an organization
        """
        organization = self.organization
        data = requests.get(page)
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def people(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        people = repository.people()
        ```
        Returns:
        {
           "data": people,
           "message": f"Found {len(people)} people for {organization}",
        } 
        """
        organization = self.organization
        data = self.__scrape_people_page()
        try:
            pages_body = data.find("div", class_="paginate-container")
            current_page = pages_body.find("em", class_="current")
            total_pages = 1
            if current_page != None:
                total_pages = (int)(current_page["data-total-pages"])

            pages = []
            if total_pages == 1:
                pages.append(f"https://github.com/orgs/{organization}/people")
            else:
                for i in range(1, total_pages + 1):
                    pages.append(
                        f"https://github.com/orgs/{organization}/people?page={i}"
                    )

            people = []
            for page in pages:
                page_data = self.__scrape_people(page)
                people_body = page_data.find("div", id="org-members-table")
                for person in people_body.find_all("li"):
                    person_username = person.find("a", class_="d-inline-block")
                    people.append(person_username["href"][1:])

            return {
                "data": people,
                "message": f"Found {len(people)} people for {organization}",
            }
        except:
            message = f"No people found for {organization}"
            return {
                "data": people,
                "message": message,
            }

    def peoples(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        peoples = repository.peoples()
        ```
        Returns:
        {
           "data": people_count,
            "message": f"Found {people_count} people for {self.organization}",
        } 
        """
        data = self.__scrape_people_page()
        try:
            body = data.find("div", class_="paginate-container")
            current_page = body.find("em", class_="current")
            page_count = 1
            if current_page != None:
                page_count = int((current_page["data-total-pages"]))

            pages = []

            if page_count == 1:
                pages.append(f"https://github.com/orgs/{self.organization}/people")
            else:
                for i in range(1, page_count + 1):
                    pages.append(
                        f"https://github.com/orgs/{self.organization}/people?page={i}"
                    )

            people_count = 0
            for page in pages:
                page_data = self.__scrape_people(page)
                people_body = page_data.find("div", id="org-members-table")
                people_count = len(people_body.find_all("li"))

            return {
                "data": people_count,
                "message": f"Found {people_count} people for {self.organization}",
            }
        except:
            message = f"No people found for {self.organization}"
            return {
                "data": people_count,
                "message": message,
            }

    def get_location(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        get_location = repository.get_location()
        ```
        Returns:
        {
           "data": lc.text.strip(),
            "message": f"Found location for {self.organization}",
        }
        """
        page = self.__scrape_page()
        try:
            lc = page.find("span", itemprop="location")
            return {
                "data": lc.text.strip(),
                "message": f"Found location for {self.organization}",
            }
        except:
            message = f"No location found for {self.organization}"
            return {
                "data": None,
                "message": message,
            }

    def repository_stats(self, repo_url):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        repository_stats = repository.repository_stats()
        ```
        Returns:
        {
           "data": {
                    "forks": forksCount,
                    "stars": starCount,
                    "issues": issuesCount,
                    "pullRequests": pullRequests,
                },
                "message": f"Found stats for {repo_url}",
        }
        """
        data = self.__scrape_repositories(repo_url)
        try:
            # forks
            forksCount = data.find("span", id="repo-network-counter").text.strip()
            # stars
            starCount = data.find("span", id="repo-stars-counter-star").text.strip()
            # issues
            issuesCount = data.find("span", id="issues-repo-tab-count").text.strip()
            # pull requests
            pullRequests = data.find(
                "span", id="pull-requests-repo-tab-count"
            ).text.strip()

            return {
                "data": {
                    "forks": forksCount,
                    "stars": starCount,
                    "issues": issuesCount,
                    "pullRequests": pullRequests,
                },
                "message": f"Found stats for {repo_url}",
            }
        except:
            message = f"No stats found for {repo_url}"
            return {
                "data": None,
                "message": message,
            }

    def repository_details(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        repository_details = repository.repository_details()
        ```
        Returns:
        {
           "data": repositories,
            "message": f"Found {len(repositories)} repositories for {organization}",
        }
        """
        organization = self.organization
        data = self.__scrape_repositories_page()
        try:
            pages_body = data.find("div", class_="paginate-container")
            current_page = pages_body.find("em", class_="current")
            total_pages = 1
            if current_page != None:
                total_pages = (int)(current_page["data-total-pages"])

            pages = []
            if total_pages == 1:
                pages.append(f"https://github.com/orgs/{organization}/repositories")
            else:
                for i in range(1, total_pages + 1):
                    pages.append(
                        f"https://github.com/orgs/{organization}/repositories?page={i}"
                    )

            repositories = []
            for page in pages:
                page_data = self.__scrape_repositories(page)
                repositories_body = page_data.find("div", id="org-repositories")
                for repo in repositories_body.find_all("li"):
                    repo_name = repo.find(
                        "a", attrs={"itemprop": "name codeRepository"}
                    ).text.strip()
                    repo_url = f"https://github.com{repo.find('a', attrs = {'itemprop': 'name codeRepository'})['href']}"
                    repo_description_body = repo.find(
                        "p", attrs={"itemprop": "description"}
                    )
                    repo_description = (
                        repo_description_body.text.strip()
                        if repo_description_body != None
                        else "No description"
                    )
                    repo_language_body = repo.find(
                        "span", attrs={"itemprop": "programmingLanguage"}
                    )
                    repo_language = (
                        repo_language_body.text.strip()
                        if repo_language_body != None
                        else "No language"
                    )
                    (
                        repo_forks,
                        repo_stars,
                        repo_issues,
                        repo_pull_requests,
                    ) = self.repository_stats(repo_url)
                    repositories.append(
                        {
                            "name": repo_name,
                            "url": repo_url,
                            "description": repo_description,
                            "language": repo_language,
                            "forks": repo_forks,
                            "stars": repo_stars,
                            "issues": repo_issues,
                            "pull_requests": repo_pull_requests,
                        }
                    )

            return {
                "data": repositories,
                "message": f"Found {len(repositories)} repositories for {organization}",
            }
        except:
            message = f"No repositories found for {organization}"
            return {
                "data": None,
                "message": message,
            }

    def pinned_repository(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        pinned_repository = repository.pinned_repository()
        ```
        Returns:
        {
           "data": json_data,
            "message": f"Found pinned repositories for {organization}",
        }
        """
        organization = self.organization

        data = self.__scrape_page()

        try:
            pinned_repos = data.find(
                "ol",
                class_="d-flex flex-wrap list-style-none gutter-condensed mb-2 js-pinned-items-reorder-list",
            )

            repo_info_list = []

            for repo in pinned_repos.find_all("li"):
                name = repo.find("span", class_="repo").text.strip()

                desc = repo.find(
                    "p", class_="pinned-item-desc color-fg-muted text-small mt-2 mb-0"
                ).text.strip()

                top_tech = repo.find(
                    "span", itemprop="programmingLanguage"
                ).text.strip()

                url = "https://github.com" + repo.find("a", href=True)["href"]
                response = requests.get(url)

                url_parts = url.split("/")

                organization = url_parts[3]

                repository = url_parts[4]

                soup = BeautifulSoup(response.content, "html.parser")

                star_count_elem = soup.find(
                    "a", href=f"/{organization}/{repository}/stargazers"
                ).find("span")
                star_count = int(star_count_elem.text.strip())

                stats_body = soup.find(
                    "ul", class_="pagehead-actions flex-shrink-0 d-none d-md-inline"
                )
                forks = stats_body.find("span", id="repo-network-counter")
                fork_count = forks.text.strip()

                repo_info = {
                    "name": name,
                    "link": url,
                    "detail": desc,
                    "top_lang": top_tech,
                    "stars": star_count,
                    "forks": fork_count,
                }
                repo_info_list.append(repo_info)

            json_data = json.dumps(repo_info_list)

            return {
                "data": json_data,
                "message": f"Found pinned repositories for {organization}",
            }
        except:
            message = f"No pinned repositories found for {organization}"
            return {
                "data": None,
                "message": message,
            }

    def get_organization_links(self):
        """"
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        get_organization_links = repository.get_organization_links()
        ```
        Returns:
        {
            if name != self.organization or name.find(self.organization) == -1:
                    if not name in links:
                        links[name] = o["href"]
            return links;
        }
        """
        try:
            links = {}
            data = self.__scrape_page()
            website_link = data.find("a", rel="nofollow", itemprop="url", href=True)[
                "href"
            ]
            links["website"] = website_link
            gmail = data.find("a", itemprop="email", href=True)["href"]
            links["gmail"] = gmail
            other_link = data.find_all("a", rel="nofollow", href=True)
            for o in other_link:
                name = (
                    o["href"]
                    .split("//")[1]
                    .split("/")[0]
                    .replace("www.", "")
                    .split(".")[0]
                )
                if name != self.organization or name.find(self.organization) == -1:
                    if not name in links:
                        links[name] = o["href"]
            return links
        except:
            return "An exception occured, information cannot be printed"


# src/scrape_up/github/pull_request.py

import requests
from bs4 import BeautifulSoup


class PullRequest:
    def __init__(self, username: str, repository_name: str, pull_request_number: int):
        self.username = username
        self.repository = repository_name
        self.pr_number = pull_request_number

    def __scrape_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/pull/{self.pr_number}"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def labels(self):
        """
        Class - `PullRequest`
        Example:
        ```
        repository = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
        labels = repository.labels()
        ```
        Returns:
        {
        "data": labels_found,
        "message": f"Found labels for {self.repository}",
        } 
        """
        labels_found = []
        data = self.__scrape_page()
        label_raw = data.find_all(
            "a", class_="IssueLabel hx_IssueLabel width-fit mb-1 mr-1"
        )
        try:
            for d in label_raw:
                labels_found.append(d.get_text().strip())
            labels_found + 1
            # return labels_found
            return {
                "data": labels_found,
                "message": f"Found labels for {self.repository}",
            }
        except:
            message = f"No labels found for {self.repository}"
            return {
                "data": labels_found,
                "message": message,
            }

    def commits(self):
        """
        Class - `PullRequest`
        Example:
        ```
        repository = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
        commits = repository.commits()
        ```
        Returns:
        {
        "data": commits_count,
        "message": f"Found {commits_count} commits for {self.pr_number}",
        } 
        """
        data = self.__scrape_page()
        try:
            commits_count = data.find("span", id="commits_tab_counter").text.strip()
            return {
                "data": commits_count,
                "message": f"Found {commits_count} commits for {self.pr_number}",
            }
        except:
            message = f"No commits found for {self.pr_number}"
            return {
                "data": None,
                "message": message,
            }

    def title(self):
        """
    
        Class - `PullRequest`
        Example:
        ```
        repository = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
        title = repository.title()
        ```
        Returns:
        {
        "data": title,
        "message": f"Found title for {self.pr_number}",
        } 
        """
        data = self.__scrape_page()
        try:
            title_body = data.find("bdi", class_="js-issue-title markdown-title")
            title = title_body.text.strip()
            return {
                "data": title,
                "message": f"Found title for {self.pr_number}",
            }
        except:
            message = f"No title found for {self.pr_number}"
            return {
                "data": None,
                "message": message,
            }

    def __files_changed_body(self):
        """
        scrape the data of files changed in a pull request
        """
        link = f"https://github.com/{self.username}/{self.repository}/pull/{self.pr_number}/files"
        data = requests.get(link)
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def files_changed(self):
        """
        
        Class - `PullRequest`
        Example:
        ```
        repository = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
        files_changed = repository.files_changed()
        ```
        Returns:
        {
        "data": files_changed,
        "message": f"Found {files_changed} files changed for {self.pr_number}",
        } 
        """
        data = self.__files_changed_body()
        try:
            files_changed_body = data.find("span", id="files_tab_counter")
            files_changed = files_changed_body.text.strip()
            return {
                "data": files_changed,
                "message": f"Found {files_changed} files changed for {self.pr_number}",
            }
        except:
            message = f"No files changed found for {self.pr_number}"
            return {
                "data": None,
                "message": message,
            }

    def reviewers(self):
        """
        Class - `PullRequest`
        Example:
        ```
        repository = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
        reviewers = repository.reviewers()
        ```
        Returns:
        {
        "data": reviewerList,
        "message": f"Found {len(reviewerList)} reviewers for {self.pr_number}",
        } 
        """
        data = self.__scrape_page()
        try:
            reviewerList = []
            reviewers = data.find_all(
                "span", class_="css-truncate-target width-fit v-align-middle"
            )
            if len(reviewers) == 0:
                message = f"No reviewers found for {self.pr_number}"
                return {
                    "data": reviewerList,
                    "message": message,
                }
            else:
                for reviewer in reviewers:
                    reviewerList.append(reviewer.text)
                return {
                    "data": reviewerList,
                    "message": f"Found {len(reviewerList)} reviewers for {self.pr_number}",
                }
        except:
            message = f"No reviewers found for {self.pr_number}"
            return {
                "data": None,
                "message": message,
            }


# src/scrape_up/github/repository.py

import requests
from bs4 import BeautifulSoup

import requests_html
import os


class Repository:
    def __init__(self, username: str, repository_name: str):
        self.username = username
        self.repository = repository_name

    def __str__(self):
        return f"{self.repository} belongs to {self.username}"

    def __scrape_page(self):
        data = requests.get(f"https://github.com/{self.username}/{self.repository}")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_tags_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/tags"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_issues_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/issues"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_pull_requests_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/pulls"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_deployments_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/deployments/activity_log"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_watchers_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/watchers"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def languagesUsed(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        languagesUsed = repository.languagesUsed()
        ```
        Returns:
        {
        "data": allLanguages,
        "message": f"Languages used in {self.repository} repository",
        }
        """
        data = self.__scrape_page()

        try:
            languages = data.find_all(class_="color-fg-default text-bold mr-1")
            allLanguages = []
            for item in languages:
                item = str(item)
                item = item[46:]
                item = item[:-7]
                allLanguages.append(item)
            # return allLanguages  # return list of languages
            return {
                "data": allLanguages,
                "message": f"Languages used in {self.repository} repository",
            }
        except:
            message = f"No languages found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def about(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        about = repository.about()
        ```
        Returns:
        {
        "data": about,
        "message": f"About {self.repository} repository",
        }
        """
        data = self.__scrape_page()

        try:
            tag = data.find(class_="f4 mb-3")
            about = tag.get_text()
            return {
                "data": about,
                "message": f"About {self.repository} repository",
            }
        except:
            message = (
                f"No details found in the about section of {self.repository} repository"
            )
            return {
                "data": None,
                "message": message,
            }

    def fork_count(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        fork_count = repository.fork_count()
        ```
        Returns:
        {
        "data": fork_count,
        "message": f"Number of forks of {self.repository} repository",

        }
        """
        data = self.__scrape_page()
        try:
            stats_body = data.find(
                "ul", class_="pagehead-actions flex-shrink-0 d-none d-md-inline"
            )
            forks = stats_body.find("span", id="repo-network-counter")
            fork_count = forks.text.strip()
            return {
                "data": fork_count,
                "message": f"Number of forks of {self.repository} repository",
            }
        except:
            message = f"No forks found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def topics(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        topics = repository.topics()
        ```
        Returns:
        {
        "data": allTopics,
        "message": f"Topics of {self.repository} repository",
        }
        """
        data = self.__scrape_page()

        try:
            topics = data.find_all(class_="topic-tag topic-tag-link")
            allTopics = []
            print(allTopics)
            for item in topics:
                allTopics.append(item.text)
            return {
                "data": allTopics,
                "message": f"Topics of {self.repository} repository",
            }
        except:
            message = f"No topics found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def star_count(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        star_count = repository.star_count()
        ```
        Returns:
        {
        "data": starCount,
        "message": f"Star count of {self.repository} repository",
        }
        """
        try:
            data = self.__scrape_page()
            starCount = (
                data.find("a", href=f"/{self.username}/{self.repository}/stargazers")
                .find("span")
                .text.strip()
            )
            return {
                "data": starCount,
                "message": f"Star count of {self.repository} repository",
            }
        except:
            message = f"No stars found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def pull_requests(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        pull_requests = repository.pull_requests()
        ```
        Returns:
        {
        "data": pull_requests,
        "message": f"Pull requests of {self.repository} repository",
        }
        """
        data = self.__scrape_page()
        try:
            pull_requests = (
                data.find_all(class_="UnderlineNav-item mr-0 mr-md-1 mr-lg-3")[2]
                .find_all("span")[1]
                .text.strip()
            )
            return {
                "data": pull_requests,
                "message": f"Pull requests of {self.repository} repository",
            }
        except:
            message = f"No pull requests found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def tags(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        tags = repository.tags()
        ```
        Returns:
        {
        "data": allTags,
        "message": f"Tags of {self.repository} repository",
        }
        """
        data = self.__scrape_tags_page()
        try:
            tags = data.find_all(class_="Link--primary")
            allTags = []
            for item in tags:
                allTags.append(item.text)
            return {
                "data": allTags,
                "message": f"Tags of {self.repository} repository",
            }
        except:
            message = f"No tags found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def releases(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        releases = repository.releases()
        ```
        Returns:
        {
        "data": allReleases,
        "message": f"Releases of {self.repository} repository",
        }
        """
        data = self.__scrape_tags_page()
        try:
            releases = data.find_all(class_="Link--primary")
            allReleases = []
            for item in releases:
                allReleases.append(item.text)
            return {
                "data": allReleases,
                "message": f"Releases of {self.repository} repository",
            }
        except:
            message = f"No releases found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def issues_count(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        issues_count = repository.issues_count()
        ```
        Returns:
        {
        "data": issues,
        "message": f"Total issues in {self.repository} repository",
        }
        """
        data = self.__scrape_page()
        try:
            issues = data.find("span", {"id": "issues-repo-tab-count"}).text.strip()
            return {
                "data": issues,
                "message": f"Total issues in {self.repository} repository",
            }
        except:
            message = f"No issues found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def readme(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        readme = repository.readme()
        ```
        Returns:
        {
         message = f"No readme found in {self.repository} repository"
        }
        """
        session = requests_html.HTMLSession()
        r = session.get(
            f"https://github.com/{self.username}/{self.username}/blob/main/README.md"
        )
        markdown_content = r.text

        try:
            with open("out.md", "w", encoding="utf-8") as f:
                f.write(markdown_content)
        except:
            message = f"No readme found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def get_pull_requests_ids(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        get_pull_requests_ids = repository.get_pull_requests_ids()
        ```
        Returns:
        {
        "data": pull_requests_ids,
        "message": f"Pull requests of {self.repository} repository",
        }
        """
        data = self.__scrape_pull_requests_page()
        try:
            pr_body = data.find(
                "div", class_="js-navigation-container js-active-navigation-container"
            )
            pull_requests_ids = []
            for each_pr in pr_body.find_all(
                "a",
                class_="Link--primary v-align-middle no-underline h4 js-navigation-open markdown-title",
            ):
                pr_id = each_pr["href"].split("/")[-1]
                pull_requests_ids.append(pr_id)

            return {
                "data": pull_requests_ids,
                "message": f"Pull requests of {self.repository} repository",
            }
        except:
            message = f"No pull requests found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def commits(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        commits = repository.commits()
        ```
        Returns:
        {
        "data": commits,
        "message": f"Commits of {self.repository} repository",
        }
        """
        data = self.__scrape_page()
        try:
            commits = str(data.find_all(class_="d-none d-sm-inline"))
            s = commits.split("<strong>")
            s = s[1].split("</strong>")
            commits = int(s[0])
            return {
                "data": commits,
                "message": f"Commits of {self.repository} repository",
            }
        except:
            message = f"No commits found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def get_issues(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        get_issues = repository.get_issues()
        ```
        Returns:
        {
        "data": allIssues,
        "message": f"Issues of {self.repository} repository",
        }
        """
        data = self.__scrape_issues_page()
        try:
            issues = data.find_all(
                class_="Link--primary v-align-middle no-underline h4 js-navigation-open markdown-title"
            )
            allIssues = []

            for item in issues:
                allIssues.append(item.text)
            return {
                "data": allIssues,
                "message": f"Issues of {self.repository} repository",
            }
        except:
            message = f"No issues found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def get_contributors(self):
        data = self.__scrape_page()

        try:
            contributors = data.find_all(
                "a", href=f"/{self.username}/{self.repository}/graphs/contributors"
            )
            contributor = []
            for it in contributors:
                contributor.append(it.get_text())
            return {
                "data": contributor[0].strip(),
                "message": f"Contributors of {self.repository} repository",
            }
        except:
            message = f"No contributors found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def last_update_at(self):
        data = self.__scrape_page()
        try:
            update = data.find_all("relative-time", class_="no-wrap")
            return {
                "data": update[0].get_text(),
                "message": f" last Updated of {self.repository} repository",
            }
        except:
            message = f"No updation found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def get_readme(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        get_readme = repository.get_readme()
        ```
        Returns:
        {
        "data": None,
        "message": message,
        }
        """
        data = requests.get(
            f"https://raw.githubusercontent.com/{self.username}/{self.username}/master/README.md"
        )
        if data.status_code == 404:
            data = requests.get(
                f"https://raw.githubusercontent.com/{self.username}/{self.username}/main/README.md"
            )
            if data.status_code == 404:
                message = f"No special repository found with username {self.username}"
                return {
                    "data": None,
                    "message": message,
                }
        else:
            path = f"./{self.username}"
            try:
                os.mkdir(path)
            except OSError as error:
                return error
            data = data.text
            readmeFile = os.open(path + "/README.md", os.O_RDWR | os.O_CREAT)
            os.write(readmeFile, data.encode("utf-8"))
            message = "README.md found & saved"
            return message

    def get_environment(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        get_environment = repository.get_environment()
        ```
        Returns:
        {
        "data": link,
        "message": f"Latest enviornment link for {self.repository} is {link}",
        }
        """
        try:
            data = self.__scrape_deployments_page()
            link = data.find(
                "a", class_="btn btn-outline flex-self-start mt-2 mt-md-0"
            ).get("href")
            return {
                "data": link,
                "message": f"Latest enviornment link for {self.repository} is {link}",
            }
        except:
            message = f"No link found for {self.repository}"
            return {
                "data": None,
                "message": message,
            }

    def get_branch(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/branches"
        )
        data = BeautifulSoup(data.text, "html.parser")
        try:
            branch = data.find_all(
                class_="branch-name css-truncate-target v-align-baseline width-fit mr-2 Details-content--shown"
            )
            allBranches = []
            for branchNames in branch:
                allBranches.append(branchNames.text.strip())
            return {
                "data": allBranches,
                "message": f"The branches of {self.repository}/{self.username} is {allBranches}",
            }
        except:
            message = f"Failed to fetch branches of {self.username}/{self.repository}"
            return {
                "data": None,
                "message": message,
            }

    def watch_count(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        watch_count = repository.watch_count()
        ```
        Returns:
        {
        "data": watches,
        "message": f"Total watches in {self.repository} repository",
        }
        """
        data = self.__scrape_watchers_page()
        try:
            watches = len(data.find("ol", {"class": "gutter"}).find_all("li"))
            return {
                "data": watches,
                "message": f"Total watches in {self.repository} repository",
            }
        except:
            message = f"No watches found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def all_watchers(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        all_watchers = repository.all_watchers()
        ```
        Returns:
        {
          return watchers
        }
        """
        data = self.__scrape_watchers_page()
        try:
            all = data.find("ol", {"class": "gutter"}).find_all(
                "a", {"data-hovercard-type": "user"}
            )[1::2]
            watchers = []
            for watcher in all:
                watchers.append(watcher.text.strip())
            return watchers

        except:
            message = f"No watchers found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }


# src/scrape_up/github/users.py

import requests
from bs4 import BeautifulSoup
import repository
import asyncio
import aiohttp
from tqdm import tqdm

# from github.repository import Repository


class Users:
    def __init__(self, username: str):
        self.username = username

    def __str__(self):
        return f" The username is: {self.username}"

    def __scrape_page(self):
        username = self.username
        data = requests.get(f"https://github.com/{username}")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def followers(self) -> str:
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        followers = user.followers()
        ```
        Return\n
        ```python
        return
        {
            "data": followers.text,
            "message":f"Followers found for user {self.username}"
        }
        ```
        """
        page = self.__scrape_page()
        try:
            followers = page.find(class_="text-bold color-fg-default")
            return {
                "data": followers.text,
                "message": f"Followers found for user {self.username}",
            }
        except:
            message = f"{self.username} not found !"
            return {"data": None, "message": message}

    def following(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        following = user.following()
        ```
        Return\n
        ```python
        return
        {
            following = page.find_all(class_="text-bold color-fg-default")
            # print(page.find_all("span"))
            return following[1].text
        }
        ```
        """
        page = self.__scrape_page()
        try:
            following = page.find_all(class_="text-bold color-fg-default")
            # print(page.find_all("span"))
            return following[1].text
        except:
            message = f"{self.username} not found !"
            return message

    def get_avatar(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_avatar = user.get_avatar()
        ```
        Return\n
        ```python
        return
        {
            return avatar["src"]
        }
        ```
        """
        page = self.__scrape_page()
        try:
            avatar = page.find(
                class_="avatar avatar-user width-full border color-bg-default"
            )
            return avatar["src"]
        except:
            message = f"Avatar not found for username {self.username}"
            return message

    def get_bio(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_bio = user.get_bio()
        ```
        Return\n
        ```python
        return
        {
            return bio.text
        }
        ```
        """
        page = self.__scrape_page()
        try:
            bio = page.find(
                class_="p-note user-profile-bio mb-3 js-user-profile-bio f4"
            )
            return bio.text
        except:
            message = f"Bio not found for username {self.username}"
            return message

    def get_repo(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_repo = user.get_repo()
        ```
        Return\n
        ```python
        return
        {
            return titles
        }
        ```
        """
        page = self.__scrape_page()
        try:
            pinned_repos = page.find_all(
                class_="mb-3 d-flex flex-content-stretch col-12 col-md-6 col-lg-6"
            )
            titles = [repo.find("span", class_="repo").text for repo in pinned_repos]
            return titles
        except:
            message = f"pinned repositories not found for username {self.username}"
            return message

    def repo_count(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        repo_count = user.repo_count()
        ```
        Return\n
        ```python
        return
        {
           return count_repo_list[0]
        }
        ```
        """
        page = self.__scrape_page()
        try:
            count_repo = page.find_all(class_="Counter")
            count_repo_list = []
            for word in count_repo:
                find_all_example = word.get_text()
                count_repo_list.append(find_all_example)
            return count_repo_list[0]
        except:
            message = f"No. of Repos not found for username {self.username}"
            return message

    def star_count(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        star_count = user.star_count()
        ```
        Return\n
        ```python
        return
        {
            return count_star_list[3]
        }
        ```
        """
        page = self.__scrape_page()
        try:
            count_star = page.find_all(class_="Counter")
            count_star_list = []
            for words in count_star:
                find_all_example = words.get_text()
                count_star_list.append(find_all_example)
            return count_star_list[3]
        except:
            message = f"Starred repo not found for username {self.username}"
            return message

    def get_yearly_contributions(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_yearly_contributions = user.get_yearly_contributions()
        ```
        Return\n
        ```python
        return
        {
           return " ".join(contributions.text.split())
        }
        ```
        """
        page = self.__scrape_page()
        try:
            contributions = page.find("h2", class_="f4 text-normal mb-2")
            return " ".join(contributions.text.split())
        except:
            message = f"Yearly contributions not found for username {self.username}"
            return message

    def __get_repo_page(self):
        """
        Scrape the repositories page of a GitHub user.
        """
        username = self.username
        repo_data = requests.get(f"https://github.com/{username}?tab=repositories")
        repo_data = BeautifulSoup(repo_data.text, "html.parser")
        return repo_data

    def get_repositories(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_repositories = user.get_repositories()
        ```
        Return\n
        ```python
        return
        {
            return repositories
        }
        ```
        """
        page = self.__get_repo_page()
        try:
            repo_body = page.find("div", id="user-repositories-list")
            repositories = []
            if repo_body != None:
                for repo in repo_body.find_all(
                    "div", class_="col-10 col-lg-9 d-inline-block"
                ):
                    repositories.append("https://github.com" + repo.a["href"])
            return repositories
        except:
            message = f"Repositories not found for username {self.username}"
            return message

    def get_organizations(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_organizations = user.get_organizations()
        ```
        Return\n
        ```python
        return
        {
            return orgs
        }
        ```
        """
        page = self.__scrape_page()
        try:
            orgs = [org.login for org in page.get_orgs()]
            return orgs
        except:
            message = f"No organizations found for the username {self.username}"
            return message

    def get_achievements(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_achievements = user.get_achievements()
        ```
        Return\n
        ```python
        return
        {
            return achievement
        }
        ```
        """
        try:
            achievement = []
            data = self.__scrape_page()
            data = data.find_all("img", class_="achievement-badge-sidebar", alt=True)
            itr = 0
            while itr < len(data) / 2:
                achievement.append(data[itr]["alt"].split(":")[1].strip(" "))
                itr = itr + 1

            return achievement
        except:
            return "Achievements cannot be fetched"

    def __get_starred_page(self):
        """
        Scrape the starred page of a GitHub user.
        """
        username = self.username
        starred_data = requests.get(f"https://github.com/{username}?tab=stars")
        starred_data = BeautifulSoup(starred_data.text, "html.parser")
        return starred_data

    def get_starred_repos(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_starred_repos = user.get_starred_repos()
        ```
        Return\n
        ```python
        return
        {
             return starred_repos
        }
        ```
        """
        page = self.__get_starred_page()
        try:
            starred_body = page.find("turbo-frame", id="user-starred-repos")
            starred_repos = []
            if starred_body != None:
                for repo in starred_body.find_all(
                    "div",
                    class_="col-12 d-block width-full py-4 border-bottom color-border-muted",
                ):
                    starred_repos.append("https://github.com" + repo.a["href"])
            return starred_repos
        except:
            message = f"Starred repositories not found for username {self.username}"
            return message

    def __scrape_followers_page(self):
        """
        Scrape the followers page of a GitHub user.
        """
        username = self.username
        followers_data = requests.get(f"https://github.com/{username}?tab=followers")
        followers_data = BeautifulSoup(followers_data.text, "html.parser")
        return followers_data

    def get_followers(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_followers = user.get_followers()
        ```
        Return\n
        ```python
        return
        {
            return followers
        }
        ```
        """
        page = self.__scrape_followers_page()
        try:
            followers_body = page.find("turbo-frame", id="user-profile-frame")
            followers = []
            for user in followers_body.find_all("span", class_="Link--secondary"):
                followers.append(user.text.strip())

            return followers
        except:
            message = f"Followers not found for username {self.username}"
            return message

    def __scrape_following_page(self):
        """

        Scrape the following page of a GitHub user.
        """
        username = self.username
        following_data = requests.get(f"https://github.com/{username}?tab=following")
        following_data = BeautifulSoup(following_data.text, "html.parser")
        return following_data

    def get_following_users(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_following_users = user.get_following_users()
        ```
        Return\n
        ```python
        return
        {
            return following
        }
        ```
        """
        page = self.__scrape_following_page()
        try:
            following_body = page.find("turbo-frame", id="user-profile-frame")
            following = []
            for user in following_body.find_all("span", class_="Link--secondary"):
                following.append(user.text.strip())

            return following
        except:
            message = f"Following users not found for username {self.username}"
            return message

    def company(self):
        page = self.__scrape_following_page()
        try:
            cmp = page.find(class_="Link--primary")
            print(cmp.text)
            # print(page.find_all("a"))
        except:
            message = f"Following users not found for username {self.username}"
            message = f"Following users not found for username {self.username}"
            return message

    def get_status(self):
        """Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_status = user.get_status()
        ```
        Return\n
        ```python
        return
        {
            return t.text.strip().replace("\n", "")
        }

        """
        try:
            data = self.__scrape_page()
            t = data.find(
                "div", class_="user-status-container position-relative hide-sm hide-md"
            )
            return t.text.strip().replace("\n", "")
        except:
            message = f"Status not found for username {self.username}"
            return message

    def get_contribution_streak(self):
        """
         Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_contribution_streak = user.get_contribution_streak()
        ```
        Return\n
        ```python
        return
        {
             return int(result)
        }

        """
        try:
            data = self.__scrape_page()
            t = data.find_all("rect", class_="ContributionCalendar-day")
            array = []
            for a in t:
                contri = a.get_text()
                if contri:
                    if contri[0] == "N":
                        array.append(0)
                    else:
                        array.append(1)

            count = 0
            result = 0

            for i in range(0, len(array)):
                if array[i] == 0:
                    count = 0.0
                else:
                    count += 1
                    result = max(result, count)
            return int(result)
        except:
            return "contribution streak cannot be obtained"

    def __get_page_details(self, link):
        """
        scrape the data in the page
        """
        data = requests.get(link)
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def get_pages(self, curr_repo_link, pages_links):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_pages = user.get_pages()
        ```
        Return\n
        ```python
        return
        {
            return pages_links
        }
        ```
        """
        data = self.__get_page_details(curr_repo_link)
        pages_body = data.find("div", class_="paginate-container")
        if pages_body.find("a", class_="next_page") != None:
            pages_links.append(
                "https://github.com" + pages_body.find("a", class_="next_page")["href"]
            )
            self.get_pages(pages_links[-1], pages_links)

        return pages_links

    def get_repository_details(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_repository_details = user.get_repository_details()
        ```
        Return\n
        ```python
        return
        {
            return repositories
        }
        ```
        """
        username = self.username
        repository_link = f"https://github.com/{username}?tab=repositories"
        try:
            # get all the pages of the repositories
            pages_links = self.get_pages(repository_link, [repository_link])
            repositories = []
            for page in pages_links:
                page_data = self.__get_page_details(page)
                # get the repositories in the page
                repositories_body = page_data.find("div", id="user-repositories-list")
                for repo in repositories_body.find_all("li"):
                    repo_name = repo.find(
                        "a", attrs={"itemprop": "name codeRepository"}
                    ).text.strip()
                    repo_url = f"https://github.com/{repo.find('a', attrs = {'itemprop': 'name codeRepository'})['href']}"
                    repo_description_body = repo.find(
                        "p", attrs={"itemprop": "description"}
                    )
                    repo_description = (
                        repo_description_body.text.strip()
                        if repo_description_body != None
                        else "No description"
                    )
                    repo_language_body = repo.find(
                        "span", attrs={"itemprop": "programmingLanguage"}
                    )
                    repo_language = (
                        repo_language_body.text.strip()
                        if repo_language_body != None
                        else "No language"
                    )
                    # create a repository object
                    repository_name = repo_url.split("/")[-1]
                    repository = repository.Repository(username, repository_name)
                    repo_forks, repo_stars, repo_issues, repo_pull_requests = (
                        repository.fork_count(),
                        repository.star_count(),
                        repository.issues_count(),
                        repository.pull_requests(),
                    )
                    repositories.append(
                        {
                            "name": repo_name,
                            "url": repo_url,
                            "description": repo_description,
                            "language": repo_language,
                            "forks": repo_forks,
                            "stars": repo_stars,
                            "issues": repo_issues,
                            "pull_requests": repo_pull_requests,
                        }
                    )

            return repositories
        except:
            return "No repositories found"

    def get_all_repo_details(self):
        page = self.__get_repo_page()

        repo_elements = page.select("#user-repositories-list ul li")
        repositories = []

        for repo_element in tqdm(repo_elements):
            link = repo_element.find("a")["href"]
            repositories.append(f"https://github.com{link}")

            name = repo_element.find("a").text.split("/")[-1]
            repositories.append(name.replace("\n", ""))

            description = (
                repo_element.find("p").get_text(strip=True)
                if repo_element.find("p")
                else None
            )
            # description= description.replace('\n','')
            repositories.append(description)

            url1 = "https://github.com" + link
            response1 = requests.get(url1)
            soup = BeautifulSoup(response1.content, "html.parser")
            li_elements = soup.find_all("li", class_="d-inline")

            for li in li_elements:
                language = li.text.strip()
                repositories.append(language.replace("\n", ""))

                stars = soup.find("span", class_="text-bold")
                num_of_stars = stars.text if stars else "N/A"
                repositories.append(num_of_stars.replace("\n", ""))

            pullurl = url1 + "/pulls"
            issuesurl = url1 + "/issues"
            pullresponse = requests.get(
                pullurl
            )  # getting the content of pull requests page
            issueresponse = requests.get(
                issuesurl
            )  # getting the content of issues page

            p_soup = BeautifulSoup(
                pullresponse.content, "html.parser"
            )  # creating beautifulsoup object for pull requests page
            i_soup = BeautifulSoup(
                issueresponse.content, "html.parser"
            )  # creating beautifulsoup object for issues page

            # to find number of pull requests
            pullrequests = p_soup.find(
                "div", class_="table-list-header-toggle states flex-auto pl-0"
            )
            num_of_pull_requests = pullrequests.text.strip() if pullrequests else "N/A"
            repositories.append(num_of_pull_requests.replace("\n", ""))

            issues = i_soup.select_one("span#issues-repo-tab-count")
            num_of_issues = issues.text.strip() if issues else "N/A"
            repositories.append(num_of_issues.replace("\n", ""))

        return repositories


# src/scrape_up/gitlab/__init__.py

from gitlab.issue import Issue
from scrape_up.gitlab.pull_request import PullRequest
from scrape_up.gitlab.organization import Organization
from scrape_up.gitlab.users import User
from scrape_up.gitlab.repository import Repository

__all__ = ["Issue", "PullRequest", "Organization", "Gitlab", "Repository"]


# src/scrape_up/gitlab/issue.py

import requests
from bs4 import BeautifulSoup


class Issue:
    def __init__(self, username, repository, issue_number):
        self.username = username
        self.repository = repository
        self.issue_number = issue_number

    def __scrape_page(self):
        url = f"https://gitlab.com/{self.username}/{self.repository}/-/issues/{self.issue_number}"
        response = requests.get(url)
        response.raise_for_status()  # Raise an exception for any HTTP errors
        soup = BeautifulSoup(response.text, "html.parser")
        return soup

    def get_title(self):
        try:
            data = self.__scrape_page()
            title = data.find("h1", class_="title").text.strip()
            return {"data": title, "message": "Retrieved issue title"}
        except Exception as e:
            return {"data": None, "message": f"Error retrieving issue title: {str(e)}"}

    def get_description(self):
        try:
            data = self.__scrape_page()
            description = data.find("div", class_="description").text.strip()
            return {"data": description, "message": "Retrieved issue description"}
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving issue description: {str(e)}",
            }

    def get_author(self):
        try:
            data = self.__scrape_page()
            author = data.find("span", class_="author").text.strip()
            return {"data": author, "message": "Retrieved issue author"}
        except Exception as e:
            return {"data": None, "message": f"Error retrieving issue author: {str(e)}"}


# src/scrape_up/gitlab/organization.py

import requests
from bs4 import BeautifulSoup


class Organization:
    def init(self, organization_name):
        self.organization_name = organization_name

    def __scrape_page(self, url):
        response = requests.get(url)
        response.raise_for_status()  # Raise an exception for any HTTP errors
        soup = BeautifulSoup(response.text, "html.parser")
        return soup

    def get_members(self):
        """
        Fetch a list of usernames of the members in the organization.
        """
        url = f"https://gitlab.com/{self.organization_name}/-/group_members"
        try:
            data = self.__scrape_page(url)
            members = []
            member_elements = data.find_all("li", class_="group-member-row")
            for member_element in member_elements:
                username = member_element.find(
                    "a", class_="group-member-username"
                ).text.strip()
                members.append(username)
            return {
                "data": members,
                "message": f"Retrieved members for organization {self.organization_name}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving members for organization {self.organization_name}: {str(e)}",
            }

    def get_projects(self):
        """
        Fetch a list of project names associated with the organization.
        """
        url = f"https://gitlab.com/groups/{self.organization_name}/-/projects"
        try:
            data = self.__scrape_page(url)
            projects = []
            project_elements = data.find_all("div", class_="gl-project-card")
            for project_element in project_elements:
                name = project_element.find(
                    "a", class_="gl-project-card-title"
                ).text.strip()
                projects.append(name)
            return {
                "data": projects,
                "message": f"Retrieved projects for organization {self.organization_name}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving projects for organization {self.organization_name}: {str(e)}",
            }


# src/scrape_up/gitlab/pull_request.py

import requests
from bs4 import BeautifulSoup


class PullRequest:
    def __init__(self, username, repository, pull_request_number):
        self.username = username
        self.repository = repository
        self.pull_request_number = pull_request_number

    def __scrape_page(self):
        url = f"https://gitlab.com/{self.username}/{self.repository}/-/merge_requests/{self.pull_request_number}"
        response = requests.get(url)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, "html.parser")
        return soup

    def get_title(self):
        try:
            data = self.__scrape_page()
            title = data.find("h1", class_="title").text.strip()
            return {"data": title, "message": "Retrieved pull request title"}
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving pull request title: {str(e)}",
            }

    def get_description(self):
        try:
            data = self.__scrape_page()
            description = data.find("div", class_="description").text.strip()
            return {
                "data": description,
                "message": "Retrieved pull request description",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving pull request description: {str(e)}",
            }

    def get_author(self):
        try:
            data = self.__scrape_page()
            author = data.find("span", class_="author").text.strip()
            return {"data": author, "message": "Retrieved pull request author"}
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving pull request author: {str(e)}",
            }


# src/scrape_up/gitlab/repository.py

import requests
from bs4 import BeautifulSoup


class Repository:
    def __init__(self, username, repository_name):
        self.username = username
        self.repository_name = repository_name

    def __scrape_page(self):
        url = f"https://gitlab.com/{self.username}/{self.repository_name}"
        response = requests.get(url)
        response.raise_for_status()  # Raise an exception for any HTTP errors
        soup = BeautifulSoup(response.text, "html.parser")
        return soup

    def get_name(self):
        try:
            data = self.__scrape_page()
            name = data.find("h1", class_="project-title").text.strip()
            return {
                "data": name,
                "message": f"Retrieved name for repository {self.repository_name}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving name for repository {self.repository_name}: {str(e)}",
            }

    def get_description(self):
        try:
            data = self.__scrape_page()
            description = data.find("div", class_="project-description").text.strip()
            return {
                "data": description,
                "message": f"Retrieved description for repository {self.repository_name}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving description for repository {self.repository_name}: {str(e)}",
            }


# src/scrape_up/gitlab/users.py

import requests
from bs4 import BeautifulSoup


class Users:
    def __init__(self, username):
        self.username = username

    def __scrape_page(self, url):
        response = requests.get(url)
        response.raise_for_status()  # Raise an exception for any HTTP errors
        soup = BeautifulSoup(response.text, "html.parser")
        return soup

    def get_name(self):
        url = f"https://gitlab.com/{self.username}"
        try:
            data = self.__scrape_page(url)
            name = data.find("h1", class_="user-title").text.strip()
            return {"data": name, "message": f"Retrieved name for user {self.username}"}
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving name for user {self.username}: {str(e)}",
            }

    def get_bio(self):
        url = f"https://gitlab.com/{self.username}"
        try:
            data = self.__scrape_page(url)
            bio = data.find("div", class_="user-info").text.strip()
            return {"data": bio, "message": f"Retrieved bio for user {self.username}"}
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving bio for user {self.username}: {str(e)}",
            }

    def get_avatar_url(self):
        url = f"https://gitlab.com/{self.username}"
        try:
            data = self.__scrape_page(url)
            avatar_url = data.find("img", class_="avatar s40")["src"]
            return {
                "data": avatar_url,
                "message": f"Retrieved avatar URL for user {self.username}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving avatar URL for user {self.username}: {str(e)}",
            }

    def get_repositories(self):
        url = f"https://gitlab.com/{self.username}?tab=repositories"
        try:
            data = self.__scrape_page(url)
            repositories = []
            repo_elements = data.find_all("li", class_="project-row")
            for repo_element in repo_elements:
                name = repo_element.find("h3", class_="project-title").text.strip()
                description = repo_element.find(
                    "p", class_="project-description"
                ).text.strip()
                repositories.append({"name": name, "description": description})
            return {
                "data": repositories,
                "message": f"Retrieved repositories for user {self.username}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving repositories for user {self.username}: {str(e)}",
            }

    def get_project_details(self, project_id):
        url = f"https://gitlab.com/{self.username}/{project_id}"
        try:
            data = self.__scrape_page(url)
            name = data.find("h1", class_="project-title").text.strip()
            last_activity = data.find("span", class_="last_activity").text.strip()
            return {
                "data": {"name": name, "last_activity": last_activity},
                "message": f"Retrieved project details for project {project_id}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving project details for project {project_id}: {str(e)}",
            }


# src/scrape_up/hacker_news/__init__.py

from hacker_news.articles import HackerNews

__all__ = ["HackerNews"]


# src/scrape_up/hacker_news/articles.py

import requests
from bs4 import BeautifulSoup

class HackerNews:
    """
    Class - `HackerNews` \n
    Creates a scraper for https://news.ycombinator.com/
    Example - \n
    ```python
    hacker_news = HackerNews()
    """
    def __init__(self):
        self.help = "This scrapes articles"

    def __scrap_page(self):
        data = requests.get("https://news.ycombinator.com/")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def articles_list(self):
        """
        Class - `HackerNews` \n
        Example - \n
        ```python
        hacker_news = HackerNews()
        articles = hacker_news.articles_list()
        ```
        Return \n
        ```python
        return 
        {
            "data": [
                {
                    "Article": "Article 1's title/text",
                    "Link": "https://www.article1.com"
                },
                {
                    "Article": "Article 2's title/text",
                    "Link": "https://www.article2.com/"
                },
                ...
                {
                {
                    "Article": "Article 30's title/text",
                    "Link": "https://www.article30.com/"
                }
                }
            ],
            "message": "Successfully fetched data."
        }
        ```
        """
        page = self.__scrap_page()
        article_list = []
        try:
            articles = page.find_all("span", class_="titleline")
            for article in articles:
                link = article.find("a")
                article_list.append({"Article": article.text, "Link": link["href"]})
            return {"data": article_list, "message": "Successfully fetched data."}

        except:
            message = "An Error Occurred!"
            return {"data": article_list, "message": message}



# src/scrape_up/imdb1/__init__.py

from .imdb import Movie

__all__ = ['Movie']

"""
You can also use -
from scrape_up.imdb.imdb import Movie

__all__ = ['Movie']


"""  


# src/scrape_up/imdb1/imdb.py

from bs4 import BeautifulSoup
import requests
import openpyxl


class IMDB:
    def __init__(self):
        pass

    def __scrape_page(self):
        try:
            source = requests.get("https://www.imdb.com/chart/top/?ref_=nv_mv_250")
            source.raise_for_status()
            soup = BeautifulSoup(source.text, "html.parser")
            movies = soup.find("tbody", class_="lister-list").find_all("tr")
            return movies
        except:
            return None

    def top_rated(self):
        """
        Class - `IMDB`\n
        Example -\n
        ```python
        top_250 = IMDB()
        print(top_250.top_rated())
        ```
        Return\n
        ```python
        return
        {
            "data": movie_data,
            "message": f"Top rated movie listed on IMDB has been fetched",
        }
        ```
        """
        try:
            movies = self.__scrape_page()
            if movies is not None:
                movie_data = []
                for movie in movies:
                    movie_name = movie.find("td", class_="titleColumn").a.text
                    rank = (
                        movie.find("td", class_="titleColumn")
                        .get_text(strip=True)
                        .split(".")[0]
                    )
                    year = movie.find("td", class_="titleColumn").span.text.strip("()")
                    rating = movie.find(
                        "td", class_="ratingColumn imdbRating"
                    ).strong.text

                    movie_data.append([rank, movie_name, year, rating])

                return {
                    "data": movie_data,
                    "message": f"Top rated movie listed on IMDB has been fetched",
                }
            else:
                return {
                    "data": None,
                    "message": f"Unable to fetch top rate movie",
                }

        except requests.exceptions.RequestException as e:
            return {
                "data": None,
                "message": f"Unable to fetch top rate movie",
            }


# src/scrape_up/instagram/__init__.py

from scrape_up.instagram.users import Users

__all__=["Users"]

# src/scrape_up/instagram/users.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.firefox import GeckoDriverManager
from selenium.webdriver.firefox.service import Service

driver = webdriver.Firefox(service=Service(GeckoDriverManager().install()))


class User:
    def __init__(self, username: str):
        self.username = username

    def user_details(self):
        """
        user = User(username=" ")
        print(user.user_details())
        """
        try:
            driver.get(f"https://www.instagram.com/{self.username}/")
            wait = WebDriverWait(driver, 180)
            account_details = wait.until(
                EC.presence_of_all_elements_located(
                    (By.XPATH, '//span[@class="_ac2a"]')
                )
            )
            return {
                "Number of Posts:": account_details[0].text,
                "Number of Followers:": account_details[1].text,
                "Number of Following:": account_details[2].text,
            }

        except Exception as e:
            message = f"{self.username} not found!"
            return {"data": None, "message": message}
        finally:
            driver.quit()


# src/scrape_up/internshala/__init__.py

from scrape_up.internshala.internships import Internships

__all__ = ["Internships"]


# src/scrape_up/internshala/internships.py

import requests
from bs4 import BeautifulSoup


class Internshala:
    """
    Class to scrape internships from Internshala website.
    """

    def __init__(self, search_type):
        """
        Initializes the Internships class.
        """


class Internshala:
    def __init__(self, search_type):
        self.base_url = "https://internshala.com/internships/"
        self.search_type = search_type

    def scrape_page(self, url):
        """
        Fetches the HTML content of the specified URL.

        Args:
            url (str): The URL to fetch.

        Returns:
            str: The HTML content of the URL.

        Raises:
            Exception: If an error occurs while fetching the page.
        """
        try:
            html_text = requests.get(url)
            html_text.raise_for_status()
            return html_text.text
        except requests.exceptions.RequestException as e:
            raise Exception(f"An error occurred while fetching the page: {str(e)}")

    def parse_page(self, html):
        """
        Parses the HTML content using BeautifulSoup.

        Args:
            html (str): The HTML content to parse.

        Returns:
            BeautifulSoup: The parsed BeautifulSoup object.

        Raises:
            Exception: If an error occurs while parsing the page.
        """
        try:
            soup = BeautifulSoup(html, "lxml")
            return soup
        except Exception as e:
            raise Exception(f"An error occurred while parsing the page: {str(e)}")

    def internships(self):
        """
        Fetches the internships data.

        Class - `Internships`
        Example:
        ```
        scraper = Internshala(search_type="keyword")
        internships = scraper.internships()
        ```
        Returns:
        List of dictionaries, where each dictionary represents an internship and contains the following keys:
        - 'title': Title of the internship
        - 'company': Company offering the internship
        - 'location': Location of the internship
        - 'duration': Duration of the internship
        - 'stipend': Stipend offered for the internship
        """
        try:
            url = self.base_url + self.search_type
            html = self.scrape_page(url)
            page = self.parse_page(html)
            internships = []

            internships_container = page.find_all("div", class_="individual_internship")

            for internship in internships_container:
                title = internship.find("h3", class_="heading_4_5 profile").text.strip()
                company = internship.find(
                    "h4", class_="heading_6 company_name"
                ).text.strip()
                location = internship.find("a", class_="location_link").text.strip()
                other_details = internship.find_all(class_="item_body")
                duration = (
                    other_details[1].text.strip() if len(other_details) > 2 else "N/A"
                )
                stipend_element = internship.find("span", class_="stipend")
                stipend = stipend_element.text.strip() if stipend_element else "N/A"

                internship_data = {
                    "title": title,
                    "company": company,
                    "location": location,
                    "duration": duration,
                    "stipend": stipend,
                }

                internships.append(internship_data)

            return {
                "data": internships,
                "message": "Internships are now fetched",
            }
        except Exception as e:
            raise Exception(f"An error occurred while scraping internships: {str(e)}")


# src/scrape_up/kindle_bookstore/__init__.py

from kindle import AmazonKindle

__all__ = ["AmazonKindle"]


# src/scrape_up/kindle_bookstore/kindle.py

import requests
from bs4 import BeautifulSoup


# scraping kindle product page
class AmazonKindle:
    def __init__(self):
        pass

    def bestsellers(self):
        """
        Class - `Book`\n
        Example -\n
        ```python
        books = AmazonKindle()
        book.bestsellers()
        ```
        Return\n
        ```python
        return {all_books with details in json}
        ```
        """
        try:
            url = "https://www.amazon.in/gp/bestsellers/books/"
            response = requests.get(url)
            page_contents = response.text
            doc = BeautifulSoup(page_contents, "html.parser")

            book_title_tags = doc.find_all("div", {"class": "zg-grid-general-faceout"})
            author_name_tags = doc.find_all("div", {"class": "zg-grid-general-faceout"})
            rating = "a-icon a-icon-star-small a-star-small-4-5 aok-align-top"
            rating_tags = doc.find_all("i", {"class": rating})
            book_price_tags = doc.find_all("div", {"class": "zg-grid-general-faceout"})
            book_url_tag = doc.find_all("div", {"class": "zg-grid-general-faceout"})

            all_books = []

            for i in range(len(book_title_tags)):
                book_title = book_title_tags[i].find("span").text
                author_name = (
                    author_name_tags[i]
                    .find("div", {"class": "a-row a-size-small"})
                    .text
                )

                # Check if the index is within the range of rating_tags
                if i < len(rating_tags):
                    star_rating = rating_tags[i].find("span").text
                else:
                    star_rating = (
                        "N/A"  # Assign a default value if rating is not available
                    )

                book_price = (
                    book_price_tags[i].find("span", {"class": "p13n-sc-price"}).text
                )
                book_url = (
                    "https://www.amazon.in"
                    + book_url_tag[i].find("a", {"class": "a-link-normal"})["href"]
                )

                book_details = {
                    "Book_Title": book_title,
                    "Author_Name": author_name,
                    "Stars": star_rating,
                    "Price": book_price,
                    "URL": book_url,
                }

                all_books.append(book_details)

            return all_books

        except Exception as e:
            return None



# src/scrape_up/kooapp/__init__.py

from scrape_up.kooapp.users import KooUser

__all__=["KooUser"]

# src/scrape_up/kooapp/users.py

import requests
import json
from bs4 import BeautifulSoup
from datetime import datetime


class KooUser:
    """
    class - `KooUser`

    Example:
    ```py
    user = KooUser('krvishal')
    avatar = user.get_avatar_url()
    ```
    Returns:
    ```
    >>> 'https://images.kooapp.com/transcode_input/8074390/profile16851706949505ew9y8.jpg'
    """

    def __init__(self, username: str) -> None:
        self.username = username

    def __str__(self) -> str:
        return f"The username is: {self.username}"

    @property
    def profile_url(self) -> str:
        return f"https://www.kooapp.com/profile/{self.username}"

    def __scrape_page(self) -> requests.Response:
        try:
            res = requests.get(self.profile_url)
            if res.status_code == 404:
                raise Exception(f"User not found with username: {self.username}")
            return res
        except Exception as e:
            raise Exception(f"An error occurred while fetching the page: {str(e)}")

    def __parse_page(self) -> dict:
        res = self.__scrape_page()
        try:
            page = BeautifulSoup(res.text, "html.parser")
            data = json.loads(page.find("script", attrs={"id": "__NEXT_DATA__"}).text)
            return data
        except Exception as e:
            raise Exception(f"An error occurred while parsing the page: {str(e)}")

    def __parse_profile_data(self) -> dict:
        data = self.__parse_page()
        try:
            userdata = (
                data.get("props")
                .get("pageProps")
                .get("initialState")
                .get("profileReducers")
                .get("profileItems")
            )
            return userdata
        except Exception as e:
            raise Exception(f"An error occured while parsing the data: {e}")

    def get_name(self) -> str:
        """
        Fetch the name of the user.
        """
        userdata = self.__parse_profile_data()
        name = userdata.get("name")
        return {
            "data": name or None,
            "message": f"Name of the user {self.username} is {name}"
            if name
            else f"No name found for the user {self.username}",
        }

    def get_bio(self) -> str:
        """
        Fetch the bio dsecription of the user.
        """
        userdata = self.__parse_profile_data()
        bio = userdata.get("description")
        return {
            "data": bio or None,
            "message": f"Bio found for the user {self.username}"
            if bio
            else f"No bio found for the user {self.username}",
        }

    def get_avatar_url(self) -> str:
        """
        Fetch the avatar url of the user.
        """
        userdata = self.__parse_profile_data()
        avatar = userdata.get("profileImage")
        return {
            "data": avatar or None,
            "message": f"Avatar found for the user {self.username}"
            if avatar
            else f"Avatar not found for the user {self.username}",
        }

    def followers(self) -> int:
        """
        Fetch the number of followers of the Koo user.
        """
        userdata = self.__parse_profile_data()
        followers = userdata.get("followerCount")
        return {
            "data": followers,
            "message": f"There are {followers} follower(s) of the user {self.username}",
        }

    def following(self) -> int:
        """
        Fetch the number of following of the Koo user.
        """
        userdata = self.__parse_profile_data()
        following = userdata.get("followingCount")
        return {
            "data": following,
            "message": f"There are {following} user(s) followed by {self.username}",
        }

    def get_social_profiles(self) -> dict[str, str]:
        """
        Fetch all the listed social media profiles of user.
        """
        userdata = self.__parse_profile_data()
        profiles = {
            s_media: handle
            for s_media, handle in userdata.get("socialProfile").items()
            if handle
        }
        return {
            "data": profiles or None,
            "message": f"Found {len(profiles)} social profiles for the user {self.username}"
            if profiles
            else f"No social profiles found for the user {self.username}",
        }

    def get_profession(self) -> str:
        """
        Fetch the profession of the user.
        """
        userdata = self.__parse_profile_data()
        profession = userdata.get("title")
        return {
            "data": profession or None,
            "message": f"Profession found for the user {self.username}"
            if profession
            else f"No profession found for the user {self.username}",
        }


# src/scrape_up/leetcode/__init__.py

from scrape_up.leetcode.leetcode_scraper import LeetCodeScraper

__all__= ['LeetCodeScraper']

# src/scrape_up/leetcode/leetcode_scraper.py

import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
from bs4 import BeautifulSoup


class LeetCode:
    """
    ```python
    LeetCode = LeetCode(username="nikhil25803")\n
    ```
    **username** might not require in few methods\n
    | Methods                    | Details                                          |
    | -------------------------- | ------------------------------------------------ |
    | `.scrape_rank()`   | Used to scrape the rank of a user on LeetCode.   |
    | `.scrape_rating()` | Used to scrape the rating of a user on LeetCode. |
    | `.get_problems_solved()`   | Used to scrape total problems solved by a user on LeetCode.   |
    | `.get_solved_by_difficulty()` | Used to scrape difficulty wise problems solved by a user on LeetCode. |
    | `.get_github_link()`   | Used to scrape github link of a user on LeetCode.   |
    | `.get_linkedin_link()` | Used to scrape linkedin link of a user on LeetCode. |
    | `.get_community_stats()` | Used to scrape community stats of a user on LeetCode. |
    """

    def __init__(self, username: str = ""):
        self.username = username
        self.user_profile = self.__scrape_user_profile()

        self.chrome_options = Options()
        self.chrome_options.add_argument("--headless")
        self.chrome_options.add_argument("--window-size=1920,1080")
        self.chrome_options.add_argument("--disable-gpu")
        self.chrome_options.add_argument("--no-sandbox")
        self.chrome_options.add_argument("--disable-dev-shm-usage")
        self.chrome_options.add_argument("--disable-extensions")
        self.chrome_options.add_argument("--disable-logging")
        self.chrome_options.add_argument("--log-level=3")
        self.chrome_options.add_argument("--silent")
        self.chrome_options.add_argument("--blink-settings=imagesEnabled=false")
        self.chrome_options.add_experimental_option(
            "excludeSwitches", ["enable-logging"]
        )

        self.service = Service(ChromeDriverManager().install())

        self.driver = webdriver.Chrome(
            service=self.service, options=self.chrome_options
        )
        self.wait = WebDriverWait(self.driver, 100)

    def __scrape_user_profile(self):
        url = f"https://leetcode.com/{self.username}"
        data = requests.get(url)
        data.raise_for_status()
        soup = BeautifulSoup(data.text, "html.parser")
        return soup

    def scrape_rank(self):
        """
        Method to scrape the rank of the user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.scrape_rank()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return {"data": None, "message": message}

        soup = self.user_profile
        try:
            rank_element = soup.find(
                "span", {"class": "ttext-label-1 dark:text-dark-label-1 font-medium"}
            )
            rank = rank_element.text.strip() if rank_element else "Not Ranked"

            return {
                "data": rank,
                "message": f"Successfully scraped rank for user '{self.username}'",
            }
        except:
            return {
                "data": None,
                "message": f"Failed to scrape rank for user '{self.username}'",
            }

    def scrape_rating(self):
        """
        Method to scrape the contest rating of the user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.scrape_rating()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return {"data": None, "message": message}

        soup = self.user_profile
        try:
            rating_element = soup.find(
                "div",
                {
                    "class": "text-label-1 dark:text-dark-label-1 flex items-center text-2xl"
                },
            )
            rating = rating_element.text.strip() if rating_element else "No Rating"

            return {
                "data": rating,
                "message": f"Successfully scraped rating for user '{self.username}'",
            }
        except:
            return {
                "data": None,
                "message": f"Failed to scrape rating for user '{self.username}'",
            }

    def get_problems_solved(self):
        """
        Method to scrape the number of problem solved by the user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.get_problems_solved()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return {"data": None, "message": message}

        soup = self.user_profile
        try:
            total_problems = soup.find(
                "div",
                {
                    "class": "text-[24px] font-medium text-label-1 dark:text-dark-label-1"
                },
            )
            return {
                "data": total_problems.text,
                "message": f"Found total problems solved for user '{self.username}'",
            }
        except:
            message = f"Failed to scrape total problems for user '{self.username}'"
            return {"data": None, "message": message}

    def get_solved_by_difficulty(self):
        """
        Method to scrape the number of problem solved by the user of each Easy, Medium and Hard\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.get_solved_by_difficulty()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return {"data": None, "message": message}

        soup = self.user_profile
        try:
            difficulty_wise = soup.find_all(
                "span",
                {
                    "class": "mr-[5px] text-base font-medium leading-[20px] text-label-1 dark:text-dark-label-1"
                },
            )

            solved = {
                "easy": difficulty_wise[0].text,
                "medium": difficulty_wise[1].text,
                "hard": difficulty_wise[2].text,
            }

            return {
                "data": solved,
                "message": f"Found difficulty wise problems solved for user '{self.username}'",
            }
        except:
            message = (
                f"Failed to scrape difficulty wise problems for user '{self.username}'"
            )
            return {"data": None, "message": message}

    def get_github_link(self):
        """
        Method to scrape the GitHub link of the Useleetoce user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.get_github_link()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return message

        soup = self.user_profile
        try:
            links = soup.find_all(
                "a",
                {
                    "class": "hover:text-label-1 dark:hover:text-dark-label-1 cursor-pointer"
                },
            )

            for link in links:
                if "github" in link["href"]:
                    return {
                        "data": link["href"],
                        "message": f"Found github link for user '{self.username}'",
                    }

            message = f"No github link found for user '{self.username}'"
            return {"data": None, "message": message}
        except:
            message = f"Failed to scrape github link for user '{self.username}'"
            return {"data": None, "message": message}

    def get_linkedin_link(self):
        """
        Method to scrape the LinkedIn link of the Useleetoce user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.get_linkedin_link()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return message

        soup = self.user_profile
        try:
            links = soup.find_all(
                "a",
                {
                    "class": "hover:text-label-1 dark:hover:text-dark-label-1 cursor-pointer"
                },
            )

            for link in links:
                if "linkedin" in link["href"]:
                    return {
                        "data": link["href"],
                        "message": f"Found linkedin link for user '{self.username}'",
                    }

            message = f"No linkedin link found for user '{self.username}'"
            return {"data": None, "message": message}
        except:
            message = f"Failed to scrape linkedin link for user '{self.username}'"
            return {"data": None, "message": message}

    def get_community_stats(self):
        """
        Method to scrape the community stat of the user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.get_community_stats()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return message

        soup = self.user_profile
        try:
            stats = soup.find_all(
                "div",
                {"class": "flex items-center space-x-2 text-[14px]"},
            )

            stats = {
                "views": stats[0].find_all("div")[-1].text,
                "solution": stats[1].find_all("div")[-1].text,
                "discuss": stats[2].find_all("div")[-1].text,
                "reputation": stats[3].find_all("div")[-1].text,
            }
            return {"data": stats, "message": f"Found stats for user '{self.username}'"}
        except:
            message = f"Failed to scrape community stats for user '{self.username}'"
            return {"data": None, "message": message}

    def get_problems(self, difficulty="", tags=[], search_key=""):
        """
        Method to scrape problems with keyword `difficulty` and `tags`\n
        Example\n
        ```python
        LeetCode = LeetCode()
        LeetCode.get_problems(difficulty="medium", tags=["dynamic-programming"],
        search_key="palindrome")
        ```
        \n
        """
        url = "https://leetcode.com/problemset/all/?"

        if difficulty != "":
            url += "difficulty=" + difficulty.upper()

        if len(tags) > 0:
            url += "&topicSlugs=" + tags[0]

            for i in range(1, len(tags)):
                url += "%2C" + tags[i]

        if search_key != "":
            url += "&search=" + search_key

        self.driver.get(url)
        self.wait.until(EC.visibility_of_element_located((By.CLASS_NAME, "space-x-1")))
        soup = BeautifulSoup(self.driver.page_source, "html.parser")

        try:
            problems = soup.find_all(
                "a",
                {"class": "h-5 hover:text-blue-s dark:hover:text-dark-blue-s"},
            )
            problems_premium = soup.find_all(
                "a",
                {
                    "class": "h-5 hover:text-blue-s dark:hover:text-dark-blue-s opacity-60"
                },
            )
            problems.extend(problems_premium)

            problems_list = []

            for problem in problems:
                problem_title = problem.text
                problem_url = "https://leetcode.com" + problem["href"]
                problems_list.append({"title": problem_title, "link": problem_url})

            return {"data": problems_list, "message": f"Found problems list"}
        except:
            message = f"Failed to scrape problems for given filters"
            return {"data": None, "message": message}

    def get_contests(self):
        """
        Method to scrape the list of active contests on Leetcode\n
        Example\n
        ```python
        LeetCode = LeetCode()
        LeetCode.get_contests()
        ```
        \n
        """
        url = "https://leetcode.com/contest/"
        data = requests.get(url)
        data.raise_for_status()
        soup = BeautifulSoup(data.text, "html.parser")

        try:
            contests = soup.find(
                "div",
                {"class": "swiper-wrapper"},
            ).find_all("div", {"class": "h-[54px]"})

            contest_list = []
            for contest in contests:
                contest_list.append(
                    {
                        "title": contest.div.text,
                        "date": contest.text.replace(contest.div.text, ""),
                    }
                )

            return {"data": contest_list, "message": f"Found contest list"}
        except:
            message = f"Failed to scrape contest details"
            return {"data": None, "message": message}

    def get_daily_challenge(self):
        """
        Method to scrape the list of daily challanges on LeetCode\n
        Example\n
        ```python
        LeetCode = LeetCode()
        LeetCode.get_daily_challenge()
        ```
        \n
        """
        try:
            problemset_url = "https://leetcode.com/problemset/all/"
            self.driver.get(problemset_url)
            self.wait.until(EC.visibility_of_element_located((By.CLASS_NAME, "flex-0")))
            soup = BeautifulSoup(self.driver.page_source, "html.parser")

            daily = soup.find(
                "a",
                {"class": "h-5 hover:text-blue-s dark:hover:text-dark-blue-s"},
            )
            daily_link = "https://leetcode.com" + daily["href"]

            self.driver.get(daily_link)
            self.wait.until(EC.visibility_of_element_located((By.CLASS_NAME, "_1l1MA")))
            soup = BeautifulSoup(self.driver.page_source, "html.parser")

            daily_challenge = {}
            daily_challenge["title"] = daily.text

            difficulty = soup.find("div", class_="rounded-[21px]").text
            daily_challenge["difficulty"] = difficulty

            p_tags = soup.find("div", class_="_1l1MA").find_all("p")
            pre_tags = soup.find("div", class_="_1l1MA").find_all("pre")

            i = 0
            description = ""
            while "Example" not in p_tags[i].text:
                description += p_tags[i].text
                i += 1
            daily_challenge["description"] = description

            examples = []
            j = 0

            while "Example" in p_tags[i].text:
                examples.append(pre_tags[j].text)
                i += 1
                j += 1
            daily_challenge["examples"] = examples

            constraints = soup.find(
                "div",
                {"class": "_1l1MA"},
            ).find("ul")
            daily_challenge["constraints"] = constraints.text

            return {
                "data": daily_challenge,
                "message": f"Found daily challenge problem",
            }
        except:
            message = f"Failed to scrape daily challenge"
            return {"data": None, "message": message}


# src/scrape_up/linkedIn/__init__.py

#Profile list can contain as many as public profiles
Profile_List = ['reidhoffman', 'another_profile']

#Import and Install Python Scrapy 

# src/scrape_up/linkedIn/linkedInspider.py

import scrapy

class LinkedInPeopleProfileSpider(scrapy.Spider):
    name = "linkedin_people_profile"

    custom_settings = {
        'FEEDS': { 'data/%(name)s_%(time)s.jsonl': { 'format': 'jsonlines',}}
    }

    def start_requests(self):
        profile_list = ['reidhoffman']  # Any public profile username can be added
        for profile in profile_list:
            linkedin_people_url = f'https://www.linkedin.com/in/{profile}/' 
            yield scrapy.Request(url=linkedin_people_url, callback=self.parse_profile, meta={'profile': profile, 'linkedin_url': linkedin_people_url})

    def parse_profile(self, response):
        item = self.extract_profile_info(response)

        item['about'] = self.extract_about_section(response)
        item['experience'] = self.extract_experience_section(response)
        item['education'] = self.extract_education_section(response)

        yield item

    def extract_profile_info(self, response):
        item = {}
        item['profile'] = response.meta['profile']
        item['url'] = response.meta['linkedin_url']

        summary_box = response.css("section.top-card-layout")
        item['name'] = summary_box.css("h1::text").get().strip()
        item['description'] = summary_box.css("h2::text").get().strip()

        try:
            item['location'] = summary_box.css('div.top-card__subline-item::text').get()
        except:
            item['location'] = summary_box.css('span.top-card__subline-item::text').get().strip()
            if 'followers' in item['location'] or 'connections' in item['location']:
                item['location'] = ''

        item['followers'] = ''
        item['connections'] = ''

        for span_text in summary_box.css('span.top-card__subline-item::text').getall():
            if 'followers' in span_text:
                item['followers'] = span_text.replace(' followers', '').strip()
            if 'connections' in span_text:
                item['connections'] = span_text.replace(' connections', '').strip()

        return item

    def extract_about_section(self, response):
        about_section = response.css('section.summary div.core-section-container__content p::text')
        return about_section.get(default='')

    def extract_experience_section(self, response):
        experience_blocks = response.css('li.experience-item')
        experience_list = []

        for block in experience_blocks:
            experience = {}
            experience['organisation_profile'] = block.css('h4 a::attr(href)').get(default='').split('?')[0]
            experience['location'] = block.css('p.experience-item__location::text').get(default='').strip()

            try:
                experience['description'] = block.css('p.show-more-less-text__text--more::text').get().strip()
            except Exception as e:
                print('experience --> description', e)
                try:
                    experience['description'] = block.css('p.show-more-less-text__text--less::text').get().strip()
                except Exception as e:
                    print('experience --> description', e)
                    experience['description'] = ''

            try:
                date_ranges = block.css('span.date-range time::text').getall()
                if len(date_ranges) == 2:
                    experience['start_time'] = date_ranges[0]
                    experience['end_time'] = date_ranges[1]
                    experience['duration'] = block.css('span.date-range__duration::text').get()
               
                elif len(date_ranges) == 1:
                    experience['start_time'] = date_ranges[0]
                    experience['end_time'] = 'present'
                    experience['duration'] = block.css('span.date-range__duration::text').get()
                
            except Exception as e:
                print('experience --> time ranges', e)
                experience['start_time'] = ''
                experience['end_time'] = ''
                experience['duration'] = ''
                experience_list.append(experience)

        return experience_list

def extract_education_section(self, response):
    education_blocks = response.css('li.education__list-item')
    education_list = []

    for block in education_blocks:
        education = {}
        education['organisation'] = block.css('h3::text').get(default='').strip()
        education['organisation_profile'] = block.css('a::attr(href)').get(default='').split('?')[0]

        try:
            education['course_details'] = ''
            for text in block.css('h4 span::text').getall():
                education['course_details'] = education['course_details'] + text.strip() + ' '
            education['course_details'] = education['course_details'].strip()
        except Exception as e:
            print("education --> course_details", e)
            education['course_details'] = ''

        education['description'] = block.css('div.education__item--details p::text').get(default='').strip()

        try:
            date_ranges = block.css('span.date-range time::text').getall()
            if len(date_ranges) == 2:
                education['start_time'] = date_ranges[0]
                education['end_time'] = date_ranges[1]
            elif len(date_ranges) == 1:
                education['start_time'] = date_ranges[0]
                education['end_time'] = 'present'
        except Exception as e:
            print("education --> time_ranges", e)
            education['start_time'] = ''
            education['end_time'] = ''

        education_list.append(education)

    return education_list


# src/scrape_up/medium/__init__.py

from scrape_up.medium.trending import Trending
from scrape_up.medium.publication import Publication
from scrape_up.medium.user import User

__all__ = ["Trending", "Publication", "User"]

# src/scrape_up/medium/publication.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time


options = Options()
options.add_argument("--headless")
options.add_argument("--log-level=3")
options.add_experimental_option("excludeSwitches", ["enable-logging"])
driver = webdriver.Chrome(options=options)


class Publication:
    def __init__(self, link):
        self.link = link

    def get_articles(self):
        """
        Class - `Publication`
        Example
        ```python
        publication = medium.Publication("https://towardsdatascience.com")
        articles = publication.get_articles()
        for article in articles:
            print(article) #For better readability/clarity
        ```
        Returns the articles of the publication which are arranged in the form of a list
        """
        try:
            articles = []
            link = self.link
            driver.get(link)
            scroll_pause = 0.5
            # Get scroll height
            last_height = driver.execute_script(
                "return document.documentElement.scrollHeight"
            )
            run_time, max_run_time = 0, 1
            while True:
                iteration_start = time.time()
                # Scroll down to bottom
                driver.execute_script(
                    "window.scrollTo(0, 1000*document.documentElement.scrollHeight);"
                )

                # Wait to load page
                time.sleep(scroll_pause)

                # Calculate new scroll height and compare with last scroll height
                new_height = driver.execute_script(
                    "return document.documentElement.scrollHeight"
                )
                scrolled = new_height != last_height
                timed_out = run_time >= max_run_time
                if scrolled:
                    run_time = 0
                    last_height = new_height
                elif not scrolled and not timed_out:
                    run_time += time.time() - iteration_start
                elif not scrolled and timed_out:
                    break
            elements = driver.find_elements(By.XPATH, "//h2 | //h3")
            for x in elements:
                articles.append(x.text)
            return articles
        except:
            return "page/publication not found."


# publication = Publication("https://pub.towardsai.net")
# articles = publication.get_articles_list()
# for article in articles:
#     print(article)


# src/scrape_up/medium/trending.py

import requests
from bs4 import BeautifulSoup as bs

headers = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
}  # mimics a browser's request


class Trending:
    def get_trending():
        """
        Class - `Trending`
        Example
        ```python
        trending = Trending.get_trending()
        for trend in trending:
            print(trend) #For better readability/clarity
        ```
        Returns a list of trending titles

        """
        try:
            titles = []
            r = requests.get("https://medium.com/", headers=headers)
            soup = bs(r.text, "html.parser")
            elements = soup.select('h2[class^="by j"]')
            for x in elements:
                titles.append(x.text)
            return titles

        except:
            return {"data": None, "message": "Something went wrong! Try again!"}


# src/scrape_up/medium/user.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time

options = Options()
options.add_argument("--headless")
options.add_argument("--log-level=3")
options.add_experimental_option("excludeSwitches", ["enable-logging"])
driver = webdriver.Chrome(options=options)


class User:
    def __init__(self, username):
        self.username = username

    def get_articles(self):
        """
        Class `Users`
        Example:
        ```python
        user = medium.User(username='karthikbhandary2')
        article_titles = user.get_articles()
        for article in article_titles:
            print(article) # For better readability/clarity
        ```
        Returns a list of the titles.
        """
        try:
            titles = []
            username = self.username
            driver.get(f"https://{username}.medium.com")
            scroll_pause = 0.5
            # Get scroll height
            last_height = driver.execute_script(
                "return document.documentElement.scrollHeight"
            )
            run_time, max_run_time = 0, 1
            while True:
                iteration_start = time.time()
                # Scroll down to bottom
                driver.execute_script(
                    "window.scrollTo(0, 1000*document.documentElement.scrollHeight);"
                )

                # Wait to load page
                time.sleep(scroll_pause)

                # Calculate new scroll height and compare with last scroll height
                new_height = driver.execute_script(
                    "return document.documentElement.scrollHeight"
                )
                scrolled = new_height != last_height
                timed_out = run_time >= max_run_time
                if scrolled:
                    run_time = 0
                    last_height = new_height
                elif not scrolled and not timed_out:
                    run_time += time.time() - iteration_start
                elif not scrolled and timed_out:
                    break
            elements = driver.find_elements(By.CSS_SELECTOR, "h2")
            for x in elements:
                titles.append(x.text)
            return titles

        except:
            return f"{username} not found."


# src/scrape_up/stackoverflow/__init__.py

from scrape_up.stackoverflow.questions import StackOverflow
__all__ = ["StackOverflow"]

# src/scrape_up/stackoverflow/questions.py

from bs4 import BeautifulSoup
import requests
import json


class StackOverflow:
    def __init__(self, topic):
        self.topic = topic

    def questions(self):
        """
        Returns the questions, views, votes, answer counts, and descriptions in JSON format\n
        Class - `StackOverflow`
        Example:
        ```
        que = StackOverflow(topic="github")
        scrape = que.questions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://stackoverflow.com/questions/tagged/" + self.topic
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                    .replace("  ", "")
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson


ques = StackOverflow(topic="Python")
print(ques.result())


# src/scrape_up/techCrunch/__init__.py

from scrape_up.techCrunch.techCrunch import TechCrunch
__all__ = ["TechCrunch"]

# src/scrape_up/techCrunch/techCrunch.py

import requests
from bs4 import BeautifulSoup
import json


class TechCrunch:
    """
    Class - `TechCrunch`
    Example:
    ```
    articles = TechCrunch(category = "topic")
    ```\n
    Methods :\n
    1. ``.getArticles() | Response - Articles with title, descriptions, images, date and link.
    """

    def __init__(self, category):
        self.category = category

    def getArticles(self):
        """
        Class - `TechCrunch`
        Example:
        ```
        articles = TechCrunch("artificial-intelligence")
        articles.getArticles()
        ```
        Returns:
        {
            "title": Tile of the article
            "description": Description of the article
            "image": Image of the article
            "author": Author of the Article
            "date": Date the article was posted
            "link": Link to the article
        }
        """
        url = (
            "https://techcrunch.com/category/" + self.category.replace(" ", "-").lower()
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            articles_data = {"articles": []}

            articles = soup.find_all(
                "div", class_="post-block post-block--image post-block--unread"
            )
            for n in articles:
                name = (
                    n.select_one(".post-block__title__link")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                desc = (
                    n.select_one(".post-block__content")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                img = n.find_all("img", src=True)
                image = img[0]["src"]
                author = (
                    n.select_one(".river-byline__authors")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                time = n.find_all("div", class_="river-byline")
                date = (
                    time[0]
                    .select_one(".river-byline__time")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                links = n.find_all("a", class_="post-block__title__link", href=True)
                link = links[0]["href"]
                articles_data["articles"].append(
                    {
                        "title": name,
                        "description": desc,
                        "image": image,
                        "author": author,
                        "date": date,
                        "link": link,
                    }
                )
            res_json = json.dumps(articles_data)
            return res_json
        except:
            error_message = {
                "message": "Can't fetch any articles from the topic provided."
            }
            ejson = json.dumps(error_message)
            return ejson


# src/scrape_up/twitter/__init__.py

from scrape_up.twitter.numidconverter import TwitterScraper

__all__=['TwitterScraper']

# src/scrape_up/twitter/numidconverter.py

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options
from time import sleep
from bs4 import BeautifulSoup
import json


class TwitterScraper:
    def __init__(self):
        self.chrome_options = Options()
        self.chrome_options.add_argument("--headless")
        self.chrome_options.add_argument("--window-size=1920,1080")
        self.chrome_options.add_argument("--disable-gpu")
        self.chrome_options.add_argument("--no-sandbox")
        self.chrome_options.add_argument("--disable-dev-shm-usage")
        self.chrome_options.add_argument("--disable-extensions")
        self.chrome_options.add_argument("--disable-logging")
        self.chrome_options.add_argument("--log-level=3")
        self.chrome_options.add_argument("--silent")
        self.chrome_options.add_argument("--blink-settings=imagesEnabled=false")

    def unametoid(self, username):
        url = "https://twitter.com/{}".format(username)
        # print(url)
        driver = webdriver.Chrome(options=self.chrome_options)
        driver.get(url)

        html = driver.page_source
        soup = BeautifulSoup(html, "html.parser")
        try:
            user_id = soup.find("script", {"data-testid": "UserProfileSchema-test"})
            data = json.loads(user_id.string)
            # driver.quit()
            return {
                "data": data["author"]["identifier"],
                "message": f"Numerical id found for username {username}",
            }
        except:
            return {
                "data": None,
                "message": f"Numerical id not found for username {username}",
            }

    def idtouname(self, numid):
        url = "https://twitter.com/i/user/{}".format(numid)
        driver = webdriver.Chrome(options=self.chrome_options)
        driver.get(url)
        html = driver.page_source
        soup = BeautifulSoup(html, "html.parser")
        try:
            user_id = soup.find("script", {"data-testid": "UserProfileSchema-test"})
            data = json.loads(user_id.string)
            # driver.quit()
            return {
                "data": data["author"]["additionalName"],
                "message": f"Username found for numerical id {numid}",
            }
        except:
            return {
                "data": None,
                "message": f"Username not found for numerical id {numid}",
            }


# src/scrape_up/wikipedia/__init__.py

from scrape_up.wikipedia.wikipedia import wikipedia

__all__ = ["wikipedia"]


# src/scrape_up/wikipedia/wikipedia.py

from bs4 import BeautifulSoup
import requests
import json

class WikipediaScraper:
    def __init__(self, url):
        self.url = url

    def scrape(self):
        response = requests.get(self.url)
        soup = BeautifulSoup(response.text, 'html.parser')

        # Extract the title
        title = soup.find(id='firstHeading').text

        # Extract all the headings and their content
        sections = soup.find_all('h2')
        data = {}
        for section in sections:
            heading = section.find('span', class_='mw-headline')
            if heading:
                content = []
                next_node = section.find_next_sibling(['h2', 'h3', 'h4', 'h5', 'h6'])
                while next_node and next_node.name != 'h2':
                    if next_node.name in ['h3', 'h4', 'h5', 'h6']:
                        content.append({'heading': next_node.text.strip()})
                    elif next_node.name == 'p':
                        content.append({'text': next_node.text.strip()})
                    next_node = next_node.find_next_sibling(['h2', 'h3', 'h4', 'h5', 'h6', 'p'])
                data[heading.text] = content

        # Return the data as JSON
        result = {
            'title': title,
            'sections': data
        }
        return json.dumps(result, indent=4)


# src/scrape_up/devCommunity/articles.py
from bs4 import BeautifulSoup
from pprint import pprint
import json
import requests

class DevCommunity:
    """
    Class - `DevCommunity` \n

    Creates a scraper for latest articles from https://www.dev.to.
    Creates a scraper for latest articles from https://www.dev.to/t/{tag} with a given tag.
    Creates a scraper for  user details, pinned articles written by a user, and all articles 
    written by a user from https://www.dev.to/{username}.

    Example - \n
    ```python
    dev = DevCommunity('python','francescoxx')
    """
    def __init__(self, tag: str, username: str):
        self.help = "This scrapes articles from the DevCommunity website."
        self.tag = tag
        self.username = username

    
    def __scrape_page(self):
        try:
            data = requests.get('https://www.dev.to')
            data.raise_for_status()
            data = BeautifulSoup(data.text, "html.parser")
            return data
        except:
            message = "An Error Occurred!"
            return message
    
    def all_articles(self):
        """
		Class - `DevCommunity` \n
        Example - \n
        ```python
        articles = dev.all_articles()
        pprint(articles)
        ```
        Return \n
        ```python
        return 
        {
            "{'data': '{\n'
         '  "articles": [\n'
         '    {\n'
         '      "Title": "With GitHub apparently down right now...",\n'       
         '      "Author": "Ben Halpern",\n'
         '      "Date": "Jun 29",\n'
         '      "Tags": "#watercooler\\n#github",\n'
         '      "Reactions": "22\\u00a0reactions",\n'
         '      "Comments": "Comments\\n\\n              6\\u00a0comments",\n'
         '      "Read Time": "2 min read"\n'
         '    },\n'
         '    {\n'
         '      "Title": "Sloan\'s Inbox: What is the Secret to a Truly Great '
         'Portfolio?",\n'
         '      "Author": "Sloan the DEV Moderator",\n'
         '      "Date": "Jun 29",\n'
         '      "Tags": '
         '"#discuss\\n#beginners\\n#career\\n#careerdevelopment",\n'
         '      "Reactions": "10\\u00a0reactions",\n'
         '      "Comments": "Comments\\n\\n              1\\u00a0comment",\n'  
         '      "Read Time": "1 min read"\n'
         '    },\n'
         '    {\n'
         '      "Title": "Sticky sessions with Apache APISIX",\n'
         '      "Author": "Nicolas Frankel",\n'
         '      "Date": "Jun 29",\n'
         '      "Tags": '
         '"#stickysessions\\n#sessionaffinity\\n#apacheapisix\\n#devops",\n'
         '      "Reactions": "44\\u00a0reactions",\n'
         '      "Comments": "Comments\\nAdd\\u00a0Comment",\n'
         '      "Read Time": "4 min read"\n'
         '    }
         '  ]\n'
         '}',
 		'message': 'Successfully fetched all articles.'}
        
        ```
		"""
        page = self.__scrape_page()
        articles_list = {"articles": []}
        try:
            articles = page.find_all( class_="substories")
            for x in articles:
                title = x.find_all(class_="crayons-story__title")        
                author = x.find_all('a', class_="crayons-story__secondary fw-medium m:hidden")
                date = x.find_all('a', class_="crayons-story__tertiary fs-xs")       
                tags = x.find_all(class_="crayons-story__tags")
                reactions = x.find_all('span', class_="aggregate_reactions_counter")        
                comments = x.find_all('a',class_="crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon-left flex items-center")        
                readTime = x.find_all(class_="crayons-story__save")
    
            for ti,a,d,ta,r,c,re, in zip(title, author, date, tags, reactions, comments, readTime):
                articles_list["articles"].append({
                "Title": ti.text.strip(),
                "Author": a.text.strip(),
                "Date": d.text.strip(),
                "Tags": ta.text.strip(),
                "Reactions": r.text.strip(),
                "Comments": c.text.strip(),
                "Read Time": re.text.strip(),
            })
            json_data = json.dumps(articles_list, indent=2)
            return {"data": json_data, "message": "Successfully fetched all articles."}
        except:
            message = "An Error Occurred!"
            return message


    def __strTag__(self):
        """
		Class - `DevCommunity` \n
    	Example - \n
    	```python
        tag_name = dev.__strTag__() 
        print(tag_name)
        ```
        Return \n
        ```python
		The list of articles we want to scrape have the tag: python
		"""
        return f" The list of articles we want to scrape have the tag: {self.tag}"
    
    def __scrape_tag(self):
        try:
            tag = self.tag
            data = requests.get(f"https://www.dev.to/t/{tag}")
            data.raise_for_status()
            data = BeautifulSoup(data.text, "html.parser")
            return data
        except:
            message = "An Error Occurred!"
            return message
    
    
    def tag_articles(self):
        """
		Class - `DevCommunity` \n
    	Example - \n
    	```python
        tagged_articles = dev.tag_articles()
        pprint(tagged_articles)
        ```
        Return \n
        ```python
		return
		{
		{'data': '{\n'
		'  "articles": [\n'
		'    {\n'
		'      "Title": "Introducing The Spatial Cypher Cheat Sheet",\n'
		'      "nAuthor": "William Lyon",\n'
		'      "Date": "Jun 28",\n'
		'      "Tags": "#neo4j\\n#python\\n#datascience\\n#gis",\n'
		'      "Reactions": "5\\u00a0reactions",\n'
		'      "Comments": "Comments\\nAdd\\u00a0Comment",\n'
		'      "Read Time": "10 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "Redis",\n'
		'      "nAuthor": "Phan T\\u1ea5n Th\\u1eafng",\n'
		'      "Date": "Jun 28",\n'
		'      "Tags": "#devops\\n#python\\n#javascript\\n#tutorial",\n'
		'      "Reactions": "2\\u00a0reactions",\n'
		'      "Comments": "Comments\\nAdd\\u00a0Comment",\n'
		'      "Read Time": "1 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "Making a Cloud Resume",\n'
		'      "nAuthor": "Tristan Armbrister",\n'
		'      "Date": "Jun 27",\n'
		'      "Tags": "#cloud\\n#azure\\n#python\\n#webdev",\n'
		'      "Reactions": "5\\u00a0reactions",\n'
		'      "Comments": "Comments\\nAdd\\u00a0Comment",\n'
		'      "Read Time": "7 min read"\n'
		'    },\n'
		]\n'
		'}',
 		'message': 'Successfully fetched all articles with the given tag.'
		}
		"""
        page = self.__scrape_tag()
        articles_list = {"articles": []}
        try:
            articles = page.find_all( class_="substories")
            for x in articles:
                title = x.find_all(class_="crayons-story__title")        
                author = x.find_all('a', class_="crayons-story__secondary fw-medium m:hidden")
                date = x.find_all('a', class_="crayons-story__tertiary fs-xs")       
                tags_ = x.find_all(class_="crayons-story__tags")
                reactions = x.find_all('span', class_="aggregate_reactions_counter")        
                comments = x.find_all('a',class_="crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon-left flex items-center")        
                readTime = x.find_all(class_="crayons-story__save")
    
            for ti,a,d,ta,r,c,re, in zip(title, author, date, tags_, reactions, comments, readTime):
                articles_list["articles"].append({
                "Title": ti.text.strip(),
                "nAuthor": a.text.strip(),
                "Date": d.text.strip(),
                "Tags": ta.text.strip(),
                "Reactions": r.text.strip(),
                "Comments": c.text.strip(),
                "Read Time": re.text.strip(),
            })
            json_data = json.dumps(articles_list, indent=2)
            return {"data": json_data, "message": "Successfully fetched all articles with the given tag."}
        except:
            message = "An Error Occurred!"
            return message


    def __strUser__(self):
        """
		Class - `DevCommunity` \n
    	Example - \n
    	```python
        user = dev.__strUser__()
		print(user)
        ```
        Return \n
        ```python
		The username is: francescoxx
		"""
        return f" The username is: {self.username}"

    def __scrape_User(self):
        try:
            username = self.username 
            data = requests.get(f"https://www.dev.to/{username}")
            data.raise_for_status()
            data = BeautifulSoup(data.text, "html.parser")
            return data
        except:
            message = "An Error Occurred!"
            return message

    def user_details(self):
        """
		Class - `User` \n
    	Example - \n
    	```python        
        user_detail = dev.user_details()
        print(user_detail)
        ```
        Return \n
        ```python
		return
		{'data': [{'Name': 'Francesco Ciulla', 'Bio': 'Check YouTube: https://www.youtube.com/@francescociulla\n', 
		'Address': ['Location', 'Rome,', 'Italy']}], 'message': 'Successfully fetched user details.'}
		"""
        page = self.__scrape_User()
        user_details = []
        try:	
            user = page.find(class_="profile-header__details")
            name = user.find(class_="crayons-title fw-heavy mb-2").text
            bio = user.find(class_="fs-base m:fs-l color-base-90 mb-4 mx-auto max-w-100 m:max-w-75").text
            address = user.find(class_="profile-header__meta__item").text.split()
            user_details.append({"Name": name, "Bio": bio, "Address": address})
            return {"data": user_details, "message": "Successfully fetched user details."}		
        except:
            message = "An Error Occurred!"
            return {"data": user_details, "message": message}

    def pinned_articles(self):	
        """
		Class - `User` \n
    	Example - \n
    	```python
        pin_articles = dev.pinned_articles()
		pprint(pin_articles)
        ```
        Return \n
        ```python
		return
		{'data': '{\n'
		'  "articles": [\n'
		'    {\n'
		'      "Title": "Rust \\ud83e\\udd80 CRUD Rest API with Docker '
		'\\ud83d\\udc33",\n'
		'      "Date": "May 4",\n'
		'      "Tags": "#rust\\n#webdev\\n#beginners\\n#devops",\n'
		'      "Reactions": "129\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              37\\u00a0comments",\n'
		'      "Read Time": "15 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "TypeScript CRUD Rest API, using Nest.js, TypeORM, '
		'Postgres, Docker and Docker Compose",\n'
		'      "Date": "Mar 26",\n'
		'      "Tags": "#typescript\\n#webdev\\n#beginners\\n#programming",\n'
		'      "Reactions": "231\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              16\\u00a0comments",\n'
		'      "Read Time": "11 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "How to Learn Web3",\n'
		'      "Date": "Sep 1 \'22",\n'
		'      "Tags": "#web3\\n#blockchain\\n#beginners\\n#programming",\n'
		'      "Reactions": "494\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              49\\u00a0comments",\n'
		'      "Read Time": "10 min read"\n'
		'    }\n'
		'  ]\n'
		'}',
 		'message': 'Successfully fetched pinned articles.'}
		"""
        try:
            page = self.__scrape_User()
            articles_list = {"articles": []}
            articles = page.find_all(class_="crayons-card mb-2 border-2 border-solid border-accent-brand")
            
            for article in articles:
                title = article.find_all(class_="crayons-story__title")
                date = article.find_all('a', class_="crayons-story__tertiary fs-xs")
                tags = article.find_all(class_="crayons-story__tags")
                reactions = article.find_all('span', class_="aggregate_reactions_counter")
                comments = article.find_all('a',class_="crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon-left flex items-center")
                readTime = article.find_all(class_="crayons-story__save")
                
            for ti,d,ta,r,c,re, in zip(title, date, tags, reactions, comments, readTime):
                articles_list["articles"].append({
                "Title": ti.text.strip(),
                "Date": d.text.strip(),
                "Tags": ta.text.strip(),
                "Reactions": r.text.strip(),
                "Comments": c.text.strip(),
                "Read Time": re.text.strip(),
            })
            json_data = json.dumps(articles_list, indent=2)
            return {"data": json_data, "message": "Successfully fetched pinned articles."}

        except:
                message = "An Error Occurred!"
                return {"data": json_data, "message": message}

    def user_articles(self):
        """
		Class - `User` \n
    	Example - \n
    	```python
        user_article = dev.user_articles()
		pprint(user_article)
        ```
        Return \n
        ```python
		return
		{'data': '{\n'
		'  "articles": [\n'
		'    {\n'
		'      "Title": "Rust \\ud83e\\udd80 CRUD Rest API with Docker '
		'\\ud83d\\udc33",\n'
		'      "Date": "May 4",\n'
		'      "Tags": "#rust",\n'
		'      "Reactions": "129\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              37\\u00a0comments",\n'
		'      "Read Time": "15 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "Kubernetes quick tutorial",\n'
		'      "Date": "Apr 18",\n'
		'      "Tags": "#webdev",\n'
		'      "Reactions": "55\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              11\\u00a0comments",\n'
		'      "Read Time": "5 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "TypeScript CRUD Rest API, using Nest.js, TypeORM, '
		'Postgres, Docker and Docker Compose",\n'
		'      "Date": "Mar 26",\n'
		'      "Tags": "#beginners",\n'
		'      "Reactions": "231\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              16\\u00a0comments",\n'
		'      "Read Time": "11 min read"\n'
		'    },\n'
		'  ]\n'
		'}',
 		'message': 'Successfully fetched all articles.'}
		"""
        page = self.__scrape_User()
        articles_list = {"articles": []}
        try:
            articles = page.find_all(class_="substories")
            for article in articles:
                date = article.find_all(class_="crayons-story__tertiary fs-xs")
                title = article.find_all(class_="crayons-story__title")
                tags = article.find_all(class_="crayons-tag crayons-tag--monochrome")
                reactions = article.find_all(class_="aggregate_reactions_counter")
                comments = article.find_all('a',class_="crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon-left flex items-center")
                readTime = article.find_all(class_="crayons-story__tertiary fs-xs mr-2")

            for ti,d,ta,r,c,re, in zip(title, date, tags, reactions, comments, readTime):
                articles_list["articles"].append({
                "Title": ti.text.strip(),
                "Date": d.text.strip(),
                "Tags": ta.text.strip(),
                "Reactions": r.text.strip(),
                "Comments": c.text.strip(),
                "Read Time": re.text.strip(),
            })
            json_data = json.dumps(articles_list, indent=2)
            
            return {"data": json_data, "message": "Successfully fetched all articles."}

        except:
                message = "An Error Occurred!"
                return {"data": json_data, "message": message}
            


dev = DevCommunity('python','francescoxx')
articles = dev.all_articles()
pprint(articles)

tag_name = dev.__strTag__()
tagged_articles = dev.tag_articles()
print(tag_name)
pprint(tagged_articles)

user = dev.__strUser__()
user_detail = dev.user_details()
pin_articles = dev.pinned_articles()
user_article = dev.user_articles()
print(user)
print(user_detail)
pprint(pin_articles)
pprint(user_article)

    