# test/GithubTest.py

from scrape_up import github

class UserTest():

    def __init__(self, username):
        self.username = username
        #SetUp
        self.user = github.Users(username=self.username)

    def test_followers(self):
        followers = self.user.followers()
        return followers

    def test_following(self):
        following = self.user.following()
        return following

    def test_get_avatar(self):
        avatar = self.user.get_avatar()
        return avatar  

    def test_get_bio(self):
        bio = self.user.get_bio()
        return bio

    def test_get_repo(self):
        repos = self.user.get_repo()
        return repos

    def test_repo_count(self):
        repo_count = self.user.repo_count()
        return repo_count

    def test_star_count(self):
        star_count = self.user.star_count()
        return star_count

    def test_get_yearly_contributions(self):
        contributions = self.user.get_yearly_contributions()
        return contributions

    def test_get_repositories(self):
        repositories = self.user.get_repositories()
        return repositories

    def test_get_starred_repos(self):
        starred_repos = self.user.get_starred_repos()
        return starred_repos
        
    def test_pull_requests(self):
        pull_requests = self.user.pul_requests()
        return pull_requests

    def test_get_followers(self):
        followers = self.user.get_followers()
        return followers

    def test_get_following_users(self):
        following_users = self.user.get_following_users()
        return following_users

    def test_get_achievements(self):
        achievements = self.user.get_achievements()
        return achievements

    def test_get_status(self):
        status = self.user.get_status()
        return status
        
    def test_get_contribution_streak(self):
        contribution_streak = self.user.get_contribution_streak()
        return contribution_streak
        
    def test_get_repository_details(self):
        repository_details = self.user.get_repository_details()
        return repository_details
        
    def test_get_branch(self):
        branches = self.user.get_branch()
        return branches

    def test_get_merged_pull_requests(self):
        merged_pull_requests = self.user.get_merged_pull_requests()
        return merged_pull_requests

class RepositoryTest:

    def __init__(self, username, repo):
        self.username = username
        self.repo = repo
        #SetUp
        self.repository = github.Repository(self.username, repo)
    
    def test_fork_count(self):
        fork_count = self.repository.fork_count()
        return fork_count

    def test_get_contributors(self):
        contributors = self.repository.get_contributors()
        return contributors
    
    def test_topics(self):
        topics = self.repository.topics()
        return topics
    
    def test_pull_requests(self):
        pull_requests = self.repository.pull_requests()
        return pull_requests

    def test_last_updated_at(self):
        last_updated_at = self.repository.last_update_at()
        return last_updated_at

    def test_tags(self):
        tags = self.repository.tags()
        return tags

    def test_releases(self):
        releases = self.repository.releases()
        return releases

    def test_issues_count(self):
        issues_count = self.repository.issues_count()
        return issues_count        

    def test_readme(self):
        readme_path = self.repository.readme()
        return readme_path

    def test_get_pull_requests_ids(self):
        pull_requests_ids = self.repository.get_pull_requests_ids()
        return pull_requests_ids

    def test_get_issues(self):
        issues = self.repository.get_issues()
        return issues

    def test_commits(self):
        commits = self.repository.commits()
        return commits    

    def test_get_readme(self):
        readme = self.repository.get_readme()
        return readme

    def test_get_environment(self):
        environment = self.repository.get_environment()
        return environment    

    def test_watch_count(self):
        watch_count = self.repository.watch_count()
        return watch_count

    def test_all_watchers(self):
        watchers = self.repository.all_watchers()
        return watchers

class IssueTest:

    def __init__(self, username, repo, issue_no):
        self.username = username
        self.repo = repo
        self.issue_no = issue_no
        #SetUp
        self.issue = github.Issue(self.username, self.repo, self.issue_no)

    def test_assignees(self):
        assignees = self.issue.assignees()
        return assignees

    def test_labels(self):
        labels = self.issue.labels()
        return labels

    def test_opened_by(self):
        opened_by = self.issue.opened_by()
        return opened_by

    def test_title(self):
        title = self.issue.title()
        return title

    def test_is_milestone(self):
        milestone = self.issue.is_milestone()
        return milestone

    def test_opened_at(self):
        opened_at = self.issue.opened_at()
        return opened_at

class PullRequestTest:

    def __init__(self, username, repo, pr_no):
        self.username= username
        self.repo = repo
        self.pr_no = pr_no
        #SetUp
        self.pull_request = github.PullRequest(self.username, self.repo, self.pr_no)
    
    def test_commits(self):
        commits = self.pull_request.commits()
        return commits

    def test_title(self):
        title = self.pull_request.title()
        return title

    def test_labels(self):
        labels = self.pull_request.labels()
        return labels    

    def test_files_changed(self):
        files_changed = self.pull_request.files_changed()
        return files_changed

    def test_reviewers(self):
        reviewers = self.pull_request.reviewers()
        return reviewers

class OrganizationTest:

    def __init__(self, org_name):
        self.org_name = org_name
        #SetUp
        self.organization = github.Organization(self.org_name)
    
    def test_top_topics(self):
        top_topics = self.organization.top_topics()
        return top_topics

    def test_followers(self):
        followers = self.organization.followers()
        return followers

    def test_top_languages(self):
        top_languages = self.organization.top_languages()
        return top_languages

    def test_avatar(self):
        avatar = self.organization.avatar()
        return avatar

    def test_repositories(self):
        repositories = self.organization.repositories()
        return repositories

    def test_people(self):
        people = self.organization.people()
        return people

    def test_peoples(self):
        peoples = self.organization.peoples()
        return peoples

    def test_get_location(self):
        location = self.organization.get_location()
        return location

    def test_repository_details(self):
        repository_details = self.organization.repository_details()
        return repository_details

    def test_pinned_repository(self):
        pinned_repository = self.organization.pinned_repository()
        return pinned_repository

    def test_get_organization_links(self):
        organization_links = self.organization.get_organization_links()
        return organization_links

# src/test/leetcode_test.py

from scrape_up.leetcode.leetcode_scraper import LeetCodeScraper

leetcodeScraper = LeetCodeScraper()
print("problems list: ", leetcodeScraper.get_problems(difficulty="medium", tags=["dynamic-programming"], search_key="palindrome"))
print("contests: ", leetcodeScraper.get_contests())
print("daily challenge: ", leetcodeScraper.get_daily_challenge())

leetcodeScraper = LeetCodeScraper(username="test")
print("rank: ", leetcodeScraper.scrape_rank())
print("rating: ", leetcodeScraper.scrape_rating())
print("total problems: ", leetcodeScraper.get_problems_solved())
print("difficulty wise problems: " ,leetcodeScraper.get_solved_by_difficulty())
print("github link: ", leetcodeScraper.get_github_link())
print("linkedin link: ", leetcodeScraper.get_linkedin_link())
print("community stats: ", leetcodeScraper.get_community_stats())

# src/scrape_up/BBCNews/__init__.py

from bbc_news import BBCNewsScraper

__all__=["BBCNewsScraper"]

# src/scrape_up/BBCNews/bbc_news.py

import requests
from bs4 import BeautifulSoup


class BBCNews:
    """
    First create an object of class `User`\n
    ```python
    user = instagram.Users(username="nikhil25803")
    ```
    | Methods        | Details                                             |
    | -------------- | --------------------------------------------------- |
    | `.get_headlines()` | Returns the list of object containig the headlines          |
    """

    def __init__(self):
        self.base_url = "https://www.bbc.co.uk"
        self.headlines_url = self.base_url + "/news"

    def get_headlines(self):
        """
        Create an instance of the class - `BBCNews`\n
        ```python
        scraper = BBCNews()
        headlines = scraper.get_headlines()
        print(headlines)
        ```
        """
        response = requests.get(self.headlines_url)
        soup = BeautifulSoup(response.content, "html.parser")
        headlines = soup.find_all("h3", class_="gs-c-promo-heading__title")

        news_set = set()
        news_list = []
        index = 1

        for headline in headlines:
            news_text = headline.get_text(strip=True)
            if news_text not in news_set:
                news_set.add(news_text)
                news_list.append({"index": index, "headline": news_text})
                index += 1

        return news_list


# src/scrape_up/CovidInfo/__init__.py

from covid_info import CovidInfo

__all__ = ["CovidInfo"]


# src/scrape_up/CovidInfo/covid_info.py

import pandas as pd
import requests
from bs4 import BeautifulSoup


class CovidInfo:
    """
    Class - `CovidInfo`\n
    | Methods                     | Details                                                                                              |
    | --------------------------- | ---------------------------------------------------------------------------------------------------- |
    | `.scrape()`                 | Returns the scraped data from the target website of all the countries in the form of a list          |
    | `.findCountry()`            | Returns the data related to the specific country passed as a parameter into the function             |
    | `.sortbycases()`            | Returns the list of datas of all the country, sorted according to the number of cases                |
    | `.sortbydeaths()`           | Returns the list of datas of all the country, sorted according to the number of deaths               |
    | `.totalcases()`             | Returns the total number of covid cases as of yet in the form of a string of numbers                 |
    | `.totaldeaths()`            | Returns the total number of covid deaths as of yet in the form of a string of numbers                |
    """

    def __init__(self):
        pass

    def covid_data(self):
        """
        Class - `CovidInfo`\n
        ```python
        response = CovidInfo()
        response.covid_data()
        ```
        Returns\n
        ```js
        {'Country': 'United States', 'Number of Cases': 107365548, 'Deaths': 1168558, 'Continent': 'North America'}
        ```
        """

        url = "https://www.worldometers.info/coronavirus/countries-where-coronavirus-has-spread/"
        page = requests.get(url)
        soup = BeautifulSoup(page.text, "html.parser")
        keys_data = ["Country", "Number of Cases", "Deaths", "Continent"]
        response_data = []
        data_iterator = iter(soup.find_all("td"))
        while True:
            try:
                country = next(data_iterator).text
                confirmed = str(
                    next(data_iterator).text.replace(",", "").replace(" ", "")
                )
                deaths = str(next(data_iterator).text.replace(",", "").replace(" ", ""))
                continent = next(data_iterator).text

                values_data = [country, int(confirmed), int(deaths), continent]
                zipped_data = dict(zip(keys_data, values_data))
                response_data.append(zipped_data)

            except StopIteration:
                break
        if len(response_data) == 1:
            return None
        else:
            return response_data

    def total_cases(self):
        """
        Get the total number of COVID cases in the world\n
        Class - `CovidInfo`\n
        ```python
        response = CovidInfo()
        response.total_cases()
        ```
        """
        try:
            url = "https://www.worldometers.info/coronavirus/countries-where-coronavirus-has-spread/"
            page = requests.get(url)
            soup = BeautifulSoup(page.text, "html.parser")
            req = soup.find_all("span", {"class": "bold_number"})

            z = req[0].find_all("a")
            fin = ""
            for i in z:
                fin = fin + i.contents[0]
            sol = fin.split(" ")
            return sol[0]
        except:
            return None

    def total_deaths(self):
        """
        Get the total number of COVID-deaths in the world\n
        Class - `CovidInfo`\n
        ```python
        response = CovidInfo()
        response.total_cases()
        ```
        """
        try:
            url = "https://www.worldometers.info/coronavirus/countries-where-coronavirus-has-spread/"
            page = requests.get(url)
            soup = BeautifulSoup(page.text, "html.parser")
            req = soup.find_all("a", {"href": "/coronavirus/coronavirus-death-toll/"})
            k = req[0].find("strong").contents[0].split(" ")
            return k[0]
        except:
            return None


# src/scrape_up/Library_Genesis/__init__.py

from library import LibGen

__all__ = ["LibGen"]


# src/scrape_up/Library_Genesis/library.py

import requests
from bs4 import BeautifulSoup as bs


def link_to_get(link):
    """This function will get the url of the image & book download direct link using the given link for book download"""
    response = requests.get(link)
    th_html = bs(response.text, "html.parser")
    td_all = th_html.find_all("td", id="info")
    td_all = td_all[0]
    td_a = td_all.find_all("a")
    link_href = td_a[1].get("href")
    img_link_td = td_all.find("img", alt="cover")
    img_link_src = img_link_td.get("src")
    img_link = f"http://library.lol{img_link_src}"
    return [link_href, img_link]


class LibGen:
    """
    Create an object of the 'LibGen' class\n
    ```python
    scraper = LibGen()
    ```
    | Methods       | Details                       |
    | --------------| ----------------------------- |
    | `.getBooks(book_name=" ")` | Returns the books with name, author, size, format, book link, book cover link, language |
    """

    def getBooks(self, book_name):
        """
        Class - `LibGen`
        Example:
        ```
        courses = LibGen(book_name="Python")
        courses.getBooks()
        ```
        Returns:
        ```
        [
            {
                "name":"The Silent Patient 9781250301697",
                "author":"Alex Michaelides",
                "size":"411 Kb",
                "format":"epub",
                "link":"https://cloudflare-ipfs.com/ipfs/bafykbzaceanodrmdg7zm2i5yj7v3uv5ep4qnvfv53jj4eydfn53gxsjjkyhgi?filename=Alex%20Michaelides%20-%20The%20Silent%20Patient.epub",
                "language":"http://library.lol/covers/2443000/20ee5ec38fbb07bfe9502007c2a2396f-g.jpg"
            }
        ]
        ```
        """
        try:
            Books = []
            name = book_name
            results = 10
            mainres = 25
            if name == "":
                return "Error: enter name"
            name = name.replace(" ", "+")

            url = f"http://libgen.is/search.php?req={name}&lg_topic=libgen&open=0&view=simple&res={mainres}&phrase=1&column=def"

            response = requests.get(url)
            bs_html = bs(response.text, "html.parser")

            if "Search string must contain minimum 3 characters.." in bs_html.body:
                return "Error: Title Too Short"

            table = bs_html.find_all("table")
            table = table[2]
            table_rows = table.find_all("tr")
            a = len(table_rows)
            table_rows.pop(0)
            obj_keys = ["name", "author", "size", "format", "link", "language"]
            if a > 1:
                counter = 1
                for i in table_rows:
                    if counter <= results:
                        book_lst = []

                        table_datas = i.find_all("td")

                        book_name = table_datas[2].get_text()

                        author = table_datas[1].get_text()

                        link_row = table_datas[9]
                        a = link_row.find("a", href=True)
                        link = a.get("href")

                        link_all = link_to_get(link)

                        language_row = table_datas[6]
                        language = language_row.get_text()

                        size_row = table_datas[7]
                        size = size_row.get_text()

                        type_row = table_datas[8]
                        type_ofit = type_row.get_text()

                        if (
                            type_ofit != "pdf" and type_ofit != "epub"
                        ) or language != "English":
                            continue
                        book_lst.append(book_name)
                        book_lst.append(author)
                        book_lst.append(size)
                        book_lst.append(type_ofit)
                        book_lst.append(link_all[0])
                        book_lst.append(link_all[1])
                        book_lst.append(language)

                        Books.append(dict(zip(obj_keys, book_lst)))
                        counter += 1
                if len(Books) >= 1:
                    return Books
                else:
                    return "Error: no results found"
            else:
                return "Error: no results found"
        except:
            return "Error"


# src/scrape_up/MediEncyclopedia/__init__.py

from mediencyclopedia import MediEncyclopedia

__all__ = ["MediEncyclopedia"]


# src/scrape_up/MediEncyclopedia/mediencyclopedia.py

from bs4 import BeautifulSoup
import requests


class MediEncyclopedia:
    """
    Create an instance of `MediEncyclopedia` class.
    ```python
    ency = MediEncyclopedia()
    ```
    | Methods               | Details                                                                                      |
    | --------------------- | -------------------------------------------------------------------------------------------- |
    | `.scrapebyurl()`      | Returns the medical dictation of associated topic url                                        |
    | `.query()`            | It takes a user query parameter as an argument and returns all relevant terms related to it. |
    | `.byletter()`         | Returns the list of medical relics starting with a particular letter                         |
    """

    def __init__(self):
      pass

    def scrapebyurl(self, url):
        """
        Class - `MediEncyclopedia`
        Example:
        ```
        ency = MediEncyclopedia()
        ency.scrapebyurl('url goes here')
        ```
        Returns:
        ```js
        {
            "headline": The name of the encyclopedia entry
            "text": Link to the entry
        }
        ```
        """

        try:
            content = requests.get(url)
            soup = BeautifulSoup(content.content, "html.parser")
            headline = soup.find("h1", attrs={"class": "with-also"}).text
            article = soup.find("article")
            for script in article(["script", "style"]):
                script.extract()
            text = (
                article.get_text()
                .replace("Browse the Encyclopedia", "")
                .replace("Clinical Trials", "")
            )
            text = text.replace(
                "To use the sharing features on this page, please enable JavaScript.",
                "",
            )

            return [headline, text]
        except:
            return None

    def query(self, userquery):
        """
        Class - `MediEncyclopedia`
        Example:
        ```
        ency = MediEncyclopedia()
        ency.query(userquery='query goes here')
        ```
        Returns:
        ```js
        {
            "headline": The name of the encyclopedia entry
            "text": Link to the entry
        }
        ```
        """
        try:
            fl = userquery[0].upper()
            resq = []
            vals = MediEncyclopedia().byletter(fl)
            ls = userquery.split(" ")
            for i in ls:
                for j in vals:
                    if i in j[0]:
                        resq.append(j)
            resq = list(resq)
            return resq
        except:
            return None

    def byletter(self, character):
        """
        Class - `MediEncyclopedia`
        Example:
        ```
        ency = MediEncyclopedia()
        ency.byletter(character='single english character goes here')
        ```
        Returns:
        ```js
        {
            "headline": The name of the encyclopedia entry
            "text": Link to the entry
        }
        ```
        """
        try:
            character = character.upper()
            chk = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
            if character not in chk:
                return None
            url = "https://medlineplus.gov/ency/encyclopedia_{}.htm".format(character)
            content = requests.get(url)
            base = "https://medlineplus.gov/ency/"
            soup = BeautifulSoup(content.content, "html.parser")
            vals = []
            heads = soup.find_all("li")
            allheads = heads[42 : len(heads) - 16]
            for i in allheads:
                t = base + i.find("a")["href"]
                h = i.find("a").text
                vals.append([h, t])

            return vals
        except:
            return None


# src/scrape_up/NewsCNN/__init__.py

from newscnn import NewsCNN

__all__=["NewsCNN"]


# src/scrape_up/NewsCNN/newscnn.py

import bs4
from bs4 import BeautifulSoup
import requests


class NewsCNN:
    """
    Create an instance of `NewsCNN` class.\n
    ```python
    news = newsCNN()
    ```
    | Methods               | Details                                                            |
    | `.newsbylocation(country="india)`   | Returns the list of articles by a specific country.               |
    """

    def __init__(self):
        pass

    def news_by_location(self, country:str):
        """
        Returns the relevant news articles corresponding to that particular geo-continent or country\n
        Class - `NewsCNN`
        Parameters: \n
        - country: Name of the country\n
        ```python
        news = newsCNN()
        news.news_by_location()
        ```
        """
        
        try:
            sol = []
            obj_keys = ['news','link']
            location = country.lower()
            URL = f"https://edition.cnn.com/world/{location}"
            page = requests.get(URL)
            parse = BeautifulSoup(page.content, "html.parser")
            heads = parse.find_all("span", attrs={"data-editable": "headline"})
            links1 = parse.find_all(
                "a",
                attrs={
                    "class": "container__link container_lead-plus-headlines-with-images__link"
                },
            )
            links2 = parse.find_all(
                "a", attrs={"class": "container__link container_vertical-strip__link"}
            )
            links3 = parse.find_all(
                "a",
                attrs={"class": "container__link container_lead-plus-headlines__link"},
            )

            base = "https://edition.cnn.com/"
            allurls = []
            allheads = []

            for i in heads:
                tmp = i.text
                allheads.append(tmp)

            for i in links1 + links2 + links3:
                t = base + i["href"]
                allurls.append(t)
            allurls = list(set(allurls))

            for i in range(len(allurls)):
                obj_values = [allheads[i], allurls[i]]
                new_obj = dict(zip(obj_keys, obj_values))
                sol.append(new_obj)

            return sol
        except:
            return None


news = NewsCNN()
print(news.news_by_location(country="usa"))


# src/scrape_up/Quora/__init__.py

from quora import Quora

__all__ = ["Quora"]


# src/scrape_up/Quora/quora.py

import urllib
import requests
import pandas as pd
import json
from bs4 import BeautifulSoup as Soup


class Quora:

    """
    Class - `Quora`\n
    | Methods                     | Details                                                                                              |
    | --------------------------- | ---------------------------------------------------------------------------------------------------- |
    | `.fetch_answers()`          | Returns the list of answers pertaining to a particular url gien by the user as parameter.            |
    | `.get_by_query()`             | Returns the list of answers pertaining to a particular query given by the user.                      |
    | `.profile_details()`             | Returns the list of the name of a user along with their quora profile link.                          |

    """

    def __init__(self):
        pass

    def fetch_answers(self, url):
        """
        Class - `Quora`
        Example:
        ```
        quora = Quora()
        quora.fetch_answers('url goes here')
        ```
        Returns:
        ```js
        {
            "answers": The list of all the answers aviliable for a particular link of question.
        }
        ```
        """
        try:
            req = requests.get(url)

            page_soup = Soup(req.content, "html.parser")
            main_box = page_soup.findAll("script", {"type": "application/ld+json"})[
                0
            ].text

            data = json.loads(main_box)
            answers = []
            try:
                answers += [x["text"] for x in data["mainEntity"]["acceptedAnswer"]]
            except:
                pass
            answers += [x["text"] for x in data["mainEntity"]["suggestedAnswer"]]

            return answers
        except:
            return None

    def get_by_query(self, query):
        """
        Returns the list of all the answers aviliable for a particular query of question given. If there is no data aviliable, then empty list is returned.
        Class - `Quora`
        Example:
        ```python
        quora = Quora()
        quora.get_by_query(query="How-should-I-start-learning-Python-1")
        ```
        Returns:
        ```js
        [ ... ]
        ```
        """
        try:
            base = "https://www.quora.com/"
            query = query.replace(" ", "-")
            url = base + query
            k = Quora().fetch_answers(url)
            return k[0]
        except:
            return None

    def profile_details(self, username):
        """
        Class - `Quora`
        Example:
        ```python
        quora = Quora()
        quora.profile_details(username="Nikhil-Raj-1727")
        ```
        Returns:
        ```js
        {'name': 'Nikhil Raj', 'url': 'https://www.quora.com/profile/NIKHIL-RAJ-1727'}
        ```
        """
        try:
            username = username.upper()
            username = username.split()
            for i in range(1, len(username)):
                username[i] = "-" + username[i]

            res = "".join(username)
            base = "https://www.quora.com/profile/"
            url = base + res
            req = requests.get(url)
            soup = Soup(req.content, "html.parser")

            name = soup.find_all("meta")[3]["content"]
            obj_keys = ["name", "url"]
            obj_values = [name, url]

            return dict(zip(obj_keys, obj_values))
        except:
            return None



# src/scrape_up/Robu/__init__.py

from robu import Robu

__all__ = ["Robu"]


# src/scrape_up/Robu/robu.py

import json
import requests
from bs4 import BeautifulSoup


class Robu:
    """
    Create a new instance of the `Robu` class\n
    ```python
    robu = Robu()
    ```
    | Methods     | Details                                                                                                         |
    | ----------- | --------------------------------------------------------------------------------------------------------------- |
    | `.search()` | Returns the json data of all the details related to search with informing about the total amount of items found |
    """

    def __init__(self):
        self.base_url = "https://www.robo.in"
        self.outputs = []

    def __get_html_content(self, url, headers=None):
        if headers is None:
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
            }

        session = requests.Session()
        session.headers.update(headers)
        response = session.get(url)
        response.raise_for_status()
        return response.content

    def __get_soup(self, url):
        request_content = self.__get_html_content(url)
        soup = BeautifulSoup(request_content, "html.parser")
        return soup

    def __is_matching_title(self, title, search_item):
        search_words = search_item.lower().split()
        return all(word in title.lower() for word in search_words)

    def __scrape_results(self, soup, search_item):
        results_container = soup.find("ul", {"class": "products"})
        results = results_container.find_all("li")

        for result in results:
            try:
                title_element = result.find(
                    "h2", class_="woocommerce-loop-product__title"
                )
                title = title_element.text.strip() if title_element else None

                if title and self.__is_matching_title(title, search_item):
                    price_element = result.find(
                        "span", class_="woocommerce-Price-amount amount"
                    )
                    price = (
                        price_element.text.strip().split("\u20b9\u00a0")[-1]
                        if price_element
                        else None
                    )

                    link_element = result.find(
                        "a",
                        class_="woocommerce-LoopProduct-link woocommerce-loop-product__link",
                    )
                    link = link_element["href"] if link_element else None

                    img_element = result.find(
                        "img",
                        class_="attachment-woocommerce_thumbnail size-woocommerce_thumbnail",
                    )
                    img = img_element["src"] if img_element else None

                    rating_element = result.find("div", class_="product-rating")
                    rating = rating_element.text.strip() if rating_element else None

                    short_description_element = result.find(
                        "div", class_="product-short-description"
                    )
                    short_description = (
                        short_description_element.text.strip().split("\n")
                        if short_description_element
                        else None
                    )

                    product_id_element = result.find("div", class_="product-sku")
                    product_id = (
                        product_id_element.text.strip() if product_id_element else None
                    )

                    self.outputs.append(
                        {
                            "title": title,
                            "price": "Rs. " + price,
                            "link": link,
                            "img": img,
                            "rating": rating,
                            "short_description": short_description,
                            "product_id": product_id,
                        }
                    )
            except Exception as e:
                print(f"Exception occurred during Robu scraping: {e}")
                continue

    def search(self, search_item, number=-1, orderby="relevance"):
        """
        Class - `RobuSearch`\n
        Args:
        + `search_item` (str): The search item to match.
        + `number` (int): The number of the items to search [ Options : 20 | 40 | 80 | 160 | 320 | -1 ] Default -1 (all)
        + `orderby` (str): The order by which the items will be returned [ Options : "relevance" | "popularity" | "rating" | "date" | "price" | "price-desc" ] Default : "relevance"
        Example -
        ```python
        robu = Robu()
        robu.search("arduino")
        ```
        Return
        ```js
        [
            {
                "title": "Arduino Uno R3 with Cable",
                "price": "Rs. 631.68",
                "link": "https://robu.in/product/arduino-uno-r3/",
                "img": "https://robu.in/wp-content/uploads/2015/11/SKU-6337-314x252.png",
                "rating": "Rated 5.00 out of 5 (25)",
                "short_description": [
                    "Micro-controller : ATmega328.",
                    "Operating Voltage : 5V.",
                    "Input Voltage (recommended) : 7-12V.",
                    "Digital I/O Pins : 14 (of which 6 provide PWM output).",
                    "Analog Input Pins : 6."
                ],
                "product_id": "SKU: 6337"
            },
        ]
        ```
        """
        if number not in [20, 40, 80, 160, 320, -1]:
            raise ValueError("Number must be from 20 | 40 | 80 | 160 | 320 | -1")
        if orderby not in [
            "relevance",
            "popularity",
            "rating",
            "date",
            "price",
            "price-desc",
        ]:
            raise ValueError(
                "Order by must be one of 'relevance','popularity','rating','date','price','price-desc'"
            )
        try:
            url = f"https://robu.in/?s={search_item}&product_cat=0&post_type=product&ppp={number}&orderby={orderby}"
            soup = self.__get_soup(url)
            self.__scrape_results(soup, search_item)
        except requests.exceptions.RequestException as e:
            return None

        except AttributeError as e:
            return None

        except Exception as e:
            return None

        result = self.outputs

        return result


# src/scrape_up/TImesJobs/__init__.py

from timesjobs_scraper import TimesJobs


__all__ = ["TimesJobs"]


# src/scrape_up/TImesJobs/timesjobs_scraper.py

import requests
from bs4 import BeautifulSoup
import json


class TimesJobs:
    def __init__(self, role: str):
        self.role = role

    def list_jobs(self):
        """
        Class - `TimesJobs`\n
        Example -\n
        ```python

        jobs = TimesJobs(role="developer")
        jobs.list_jobs()
        ```
        Return\n
        ```python
        return
        {
            "Company": "Name of the comapny",
            "Location": "Location at which the company is located",
            "Experience": "Experience of the applicants required for that post",
            "Posted": "Number of days before which this job was posted on this webiste",
            "Apply here": "Link which directly takes you to the Web-page where you can apply for the job"
        }
        """
        try:
            spl = self.role.split()
            self.role = "%20".join(spl)
        except:
            return None
        try:
            url = f"https://m.timesjobs.com/mobile/jobs-search-result.html?txtKeywords={self.role}&txtLocation=India&cboWorkExp1=-1"
            response = requests.get(url)
            soup = BeautifulSoup(response.text, "html.parser")
            companies = soup.find_all("h4")
            experiences = soup.find_all("div", class_="srp-exp")
            locations = soup.find_all("div", class_="srp-loc")
            days_ago = soup.find_all("span", class_="posting-time")
            application_links = soup.find_all("h3")

            job_data = []

            for i in range(len(companies)):
                company = companies[i].text
                location = locations[i].text
                experience = experiences[i].text
                days = days_ago[i].text
                href_value = application_links[i].a["href"]

                job_info = {
                    "Company": company,
                    "Location": location,
                    "Experience": experience,
                    "Posted": days,
                    "Apply here": href_value,
                }
                job_data.append(job_info)

            return job_data

        except Exception as e:
            print("Not possible to webscrape")
            return None


# src/scrape_up/UCI/UCI.py

from bs4 import BeautifulSoup
import requests


class UCI:
    """
    Create an instance of UCI class
    ```python
    uci = UCI()
    ```
    | Methods       | Details                               |
    | ------------- | ------------------------------------- |
    | `.datasets()` | Fetches datasets information from UCI |
    """

    def __init__(self):
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36"
        }

    def datasets(self, number):
        """
        Get UCI datasets information.\n
        Args:
        `number (int)`: The number of datasets to fetch. The method fetches datasets in batches of 10.
        Example:
        ```python
        uci = UCI()
        datasets_info = uci.datasets(20)
        ```
        Returns:
        ```js
        [
            {
                "Name":"Iris",
                "Link":"https://archive.ics.uci.edu//dataset/53/iris",
                "Description":"A small classic dataset from Fisher, 1936. One of the earliest datasets used for evaluation of classification methodologies.\n",
                "Extra Info":" Classification  Multivariate  150 Instances  4 Attributes "
            }
        ]
        ```
        """
        try:
            number = number // 10
            dataset = []
            for i in range(0, number):
                url = "https://archive.ics.uci.edu/datasets?skip={}&take=10&sort=desc&orderBy=NumHits&search=s".format(
                    i * 10
                )
                html_text = requests.get(url, headers=self.headers).text
                soup = BeautifulSoup(html_text, "lxml")

                container = soup.find("div", {"class": "flex flex-col gap-1"})

                for items in container.find_all(
                    "div", {"class": "rounded-box bg-base-100"}
                ):
                    title = items.find("h2").text
                    link = (
                        "https://archive.ics.uci.edu/"
                        + items.find("a", href=True)["href"]
                    )
                    description = items.find("p").text
                    extra_info = ""
                    for item in items.find_all(
                        "div", {"class": "col-span-3 flex items-center gap-2"}
                    ):
                        extra_info = extra_info + item.text + " "
                    data = {
                        "Name": title,
                        "Link": link,
                        "Description": description,
                        "Extra Info": extra_info,
                    }
                    dataset.append(data)
            return dataset
        except:
            return None

# src/scrape_up/UCI/__init__.py

from UCI import UCI

__all__ = ["UCI"]


# src/scrape_up/Udemy/__init__.py

from courses import Courses

__all__ = ["Courses"]


# src/scrape_up/Udemy/courses.py

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
import undetected_chromedriver as uc
import re
import json


class Courses:
    """
    Class - courses\n
    First, create an object of class `Courses`
    ```python
    topic = Udemy.courses(keyword="data science")
    ```
    | Methods           | Details                                  |
    | ----------------- | ---------------------------------------- |
    | `.get_courses()`  | Returns the list of top courses.         |
    """

    def __init__(self, keyword):
        self.keyword = keyword

    def __scrape_page(self):
        chrome_options = uc.ChromeOptions()
        chrome_options.add_argument("--headless")
        driver = uc.Chrome(options=chrome_options)
        driver.get(
            "https://www.udemy.com/courses/search/?src=ukw&q="
            + self.keyword.replace(" ", "+")
        )
        wait = WebDriverWait(driver, 10)
        return wait, driver

    def __replace_after_newline(self, text):
        index = text.find("\n")  # Find the index of the first occurrence of '\n'
        if index != -1:
            return text[:index]  # Return the substring up to the '\n'
        else:
            return text

    def __remove_html_tags(self, text):
        clean_text = re.sub("<.*?>", "", text)  # Use regex to remove HTML tags
        return clean_text

    def __extract_numbers(self, string):
        if string == None:
            return None
        number = re.findall(
            r"\d+", string
        )  # Find all numeric substrings using regex  # Convert the substrings to integers
        return number[0]

    def __extract_amount(self, string):
        if string == None:
            return None
        rupee_index = string.find("₹")  # Find the index of the "₹" sign
        if rupee_index != -1:
            amount = string[
                rupee_index + 1 :
            ]  # Extract the substring after the "₹" sign
            amount = amount.replace(
                ",", ""
            )  # Remove any commas from the extracted amount
            return amount.strip()  # Remove leading/trailing whitespaces
        else:
            return None

    def get_courses(self):
        """
        Class - `Courses`
        Example:
        ```
        topic = Courses(keyword="data science")
        topic.get_courses()
        ```
        Returns:
        ```js
        [
            { 
            'title' : 'Title of course',
            'link' : 'Link to course',
            'description' : 'Description of course',
            'instructor' : 'Instructor of course',
            'rating' : 'Rating of course out of 5',
            'reviews' : 'Number of reviews for course',
            'discount_price' : 'Discounted price of course',
            'list_price' : 'List price of course',
            'duration' : 'Duration of course',
            'lectures' : 'Number of lectures in course',
            'level' : 'Level of course',
            }
        ]
        ```
        """
        wait, driver = self.__scrape_page()
        coursesArr = []
        courses = wait.until(
            lambda d: d.find_elements(By.CLASS_NAME, "course-card--container--1QM2W")
        )
        if not courses:
            return "error in scraping..."

        try:
            for course in courses:
                title = course.find_element(
                    By.CLASS_NAME, "course-card--course-title--vVEjC"
                )

                link = title.find_element(By.TAG_NAME, "a").get_attribute("href")

                description = course.find_element(
                    By.CLASS_NAME, "course-card--course-headline--2DAqq"
                )

                instructor = course.find_element(
                    By.CLASS_NAME, "course-card--instructor-list--nH1OC"
                ).text.replace("\\", "")

                rating = course.find_element(
                    By.CLASS_NAME, "star-rating-module--rating-number--2xeHu"
                )

                try:
                    discount_price = course.find_element(
                        By.CLASS_NAME, "course-card--discount-price--1bQ5Q"
                    ).text
                except Exception as inst:
                    discount_price = None

                try:
                    list_price = course.find_element(
                        By.CLASS_NAME, "course-card--list-price--3RTcj"
                    ).text
                except:
                    list_price = None

                reviews = course.find_element(
                    By.CLASS_NAME, "course-card--reviews-text--1yloi"
                )

                meta_info = course.find_element(
                    By.CLASS_NAME, "course-card--course-meta-info--2jTzN"
                )

                meta_info_arr = meta_info.find_elements(
                    By.CLASS_NAME, "course-card--row--29Y0w"
                )

                coursesArr.append(
                    {
                        "title": self.__replace_after_newline(title.text),
                        "link": link,
                        "description": self.__remove_html_tags(description.text),
                        "instructor": instructor,
                        "rating": rating.text,
                        "reviews": reviews.text[1 : (len(reviews.text) - 1)],
                        "discount_price": self.__extract_amount(
                            discount_price if discount_price else None
                        ),
                        "list_price": self.__extract_amount(
                            list_price if list_price else None
                        ),
                        "duration": self.__extract_numbers(meta_info_arr[0].text),
                        "lectures": self.__extract_numbers(meta_info_arr[1].text),
                        "level": meta_info_arr[2].text,
                    }
                )

            return coursesArr

        except Exception as inst:
            return json.dumps(
                {"data": None, "message": "No courses found for " + self.keyword + ""}
            )
        finally:
            driver.quit()


# src/scrape_up/amazon/__init__.py

from products import Product

__all__ = ["Product"]


# src/scrape_up/amazon/products.py

import requests
from bs4 import BeautifulSoup


# scraping amazon product page
class Product:
    def __init__(self, product_name: str):
        self.product_name = product_name

    def get_product(self):
        """
        Class - `Product`\n
        Example -\n
        ```python
        product = Product(product_name="watch")
        product.get_product()
        ```
        Return\n
        ```python
        return
        {
            "data": product_link,
            "message": f"Product data has been fetched",
        }
        ```
        """
        try:
            product_name = self.product_name
            product_name = product_name.replace(" ", "+")
            url = f"https://www.amazon.in/s?k={product_name}"
            headers = {
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \
                    (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
            }
            r = requests.get(url, headers=headers)
            soup = BeautifulSoup(r.content, "html.parser")
            product = soup.find("div", {"class": "s-product-image-container"})
            product_link = product.find("a", {"class": "a-link-normal"})["href"]
            product_link = "https://www.amazon.in" + product_link
            return {
                "data": product_link,
                "message": f"Product data has been fetched",
            }
        except:
            return {
                "data": None,
                "message": f"Unable to fetch product's data",
            }

    # Get product details
    def get_product_details(self):
        """
        Class - `Product`\n
        Example -\n
        ```python
        product = Product(product_name="watch")
        product.get_product_details()
        ```
        Return\n
        ```python
        return
        {
            "data": product_details,
            "message": f"Product detail has been fetched",
        }
        ```
        """
        try:
            product_link = self.get_product()["data"]
            headers = {
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \
                    (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
            }
            r = requests.get(product_link, headers=headers)
            soup = BeautifulSoup(r.content, "html.parser")
            product_name = soup.find("span", {"id": "productTitle"}).text.strip()
            product_price = soup.find("span", {"class": "a-price-whole"}).text.strip()
            product_rating = soup.find(
                "span", {"class": "a-size-base a-color-base"}
            ).text.strip()
            product_details = {
                "product_name": product_name,
                "product_price": product_price,
                "product_rating": product_rating,
                "product_link": product_link,
            }
            return {
                "data": product_details,
                "message": f"Product detail has been fetched",
            }
        except:
            return {
                "data": None,
                "message": f"Unable to fetch product detail",
            }

    # Get product image
    def get_product_image(self):
        """
        Class - `Product`\n
        Example -\n
        ```python
        product = Product(product_name="watch")
        product.get_product_image()
        ```
        Return\n
        ```python
        return
        {
            "data": product_image,
            "message": f"Product image has been fetched",
        }
        ```
        """
        try:
            product_link = self.get_product()["data"]
            headers = {
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \
                    (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
            }
            r = requests.get(product_link, headers=headers)
            soup = BeautifulSoup(r.content, "html.parser")
            product_image = soup.find(
                "img", {"class": "a-dynamic-image a-stretch-horizontal"}
            )["src"]

            return {
                "data": product_image,
                "message": f"Product image has been fetched",
            }
        except:
            return {
                "data": None,
                "message": f"Unable to fetch product image",
            }

    # Get customer reviews
    def customer_review(self):
        """
        Class - `Product`\n
        Example -\n
        ```python
        product = Product(product_name="watch")
        product.customer_review()
        ```
        Return\n
        ```python
        return
        {
            "data": review,
            "message": f"Product review has been fetched",
        }
        ```
        """
        try:
            product_link = self.get_product()["data"]
            headers = {
                "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \
                    (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
            }
            r = requests.get(product_link, headers=headers)
            soup = BeautifulSoup(r.content, "html.parser")

            review_elements = soup.find_all("div", {"data-hook": "review"})

            for review_element in review_elements:
                reviewer_name = review_element.find(
                    "span", {"class": "a-profile-name"}
                ).text
                rating = (
                    review_element.find("i", {"class": "a-icon-star"})
                    .find("span", {"class": "a-icon-alt"})
                    .text
                )
                review_title = review_element.find(
                    "a", {"data-hook": "review-title"}
                ).text.strip()
                review_date = review_element.find(
                    "span", {"data-hook": "review-date"}
                ).text
                review_text = review_element.find(
                    "span", {"data-hook": "review-body"}
                ).text.strip()
                review = [reviewer_name, rating, review_title, review_date, review_text]
            return {
                "data": review,
                "message": f"Product review has been fetched",
            }
        except:
            return {
                "data": None,
                "message": f"Unable to fetch product review",
            }


# src/scrape_up/askUbuntu/__init__.py

from scrape_up.askUbuntu.questions import AskUbuntu
__all__ = ["AskUbuntu"]

# src/scrape_up/askUbuntu/questions.py

from bs4 import BeautifulSoup
import requests
import json


class AskUbuntu:
    """
    Create an instance of `AskUbuntu` class.

    ```python
    questions = AskUbuntu("topic")
    ```

    | Methods                     | Details                                                                                              |
    | --------------------------- | ---------------------------------------------------------------------------------------------------- |
    | `.getNewQuestions()`        | Returns the new questions, views, votes, answer counts, and descriptions in JSON format              |
    | `.getActiveQuestions()`     | Returns the active questions, views, votes, answer counts, and descriptions in JSON format           |
    | `.getUnansweredQuestions()` | Returns the unanswered questions, views, votes, answer counts, and descriptions in JSON format       |
    | `.getBountiedQuestions()`   | Returns the bountied questions, views, votes, answer counts, and descriptions in JSON format         |
    | `.getFrequentQuestions()`   | Returns the frequently asked questions, views, votes, answer counts, and descriptions in JSON format |
    | `.getHighScoredQuestions()` | Returns the most voted questions, views, votes, answer counts, and descriptions in JSON format       |
    """

    def __init__(self, topic):
        self.topic = topic

    def getNewQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getNewQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Newest"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getActiveQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getActiveQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Active"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getUnansweredQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getUnansweredQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Unanswered"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getBountiedQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getBountiedQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Bountied"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getFrequentQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getFrequentQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Frequent"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getHighScoredQuestions(self):
        """
        Class - `AskUbuntu`
        Example:
        ```
        que = AskUbuntu("github")
        scrape = que.getHighScoredQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://askubuntu.com/questions/tagged/" + self.topic + "?tab=Votes"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

ask = AskUbuntu(topic="Python")
print(ask.getActiveQuestions())

# src/scrape_up/codeforces/__init__.py

from user import USER
__all__ = ['USER']

# src/scrape_up/codeforces/user.py

import requests
from bs4 import BeautifulSoup


class USER:
    """
    Fetches user data from CodeForces Competitive Platform
    """

    user_url = "https://codeforces.com/profile/{}"

    def __init__(self, username):
        self.username = username
        self.userdata = self.user_get_data()

    def user_get_data(self):
        """
        Gets all information of a username from codeforces.

        Class - `codeforces.user()`\n
        Example -\n
        ```python
        user = USER("benq")
        print(user.user_get_data())
        ```
        Return\n
        ```python
        return
        lastName → Qi
        country → United States
        lastOnlineTimeSeconds → 1686275190
        city → Princeton
        rating → 3783
        friendOfCount → 12513
        titlePhoto → https://userpic.codeforces.org/312472/title/7cf0a442d4071e87.jpg
        handle → Benq
        avatar → https://userpic.codeforces.org/312472/avatar/5716ac69aea8159a.jpg
        firstName → Benjamin
        contribution → 50
        organization → MIT
        rank → legendary grandmaster
        maxRating → 3813
        registrationTimeSeconds → 1435099979
        maxRank → legendary grandmaster
        lastName → Qi
        country → United States
        lastOnlineTimeSeconds → 1686275190
        city → Princeton
        rating → 3783
        friendOfCount → 12513
        titlePhoto → https://userpic.codeforces.org/312472/title/7cf0a442d4071e87.jpg
        handle → Benq
        avatar → https://userpic.codeforces.org/312472/avatar/5716ac69aea8159a.jpg
        firstName → Benjamin
        contribution → 50
        organization → MIT
        rank → legendary grandmaster
        maxRating → 3813
        registrationTimeSeconds → 1435099979
        maxRank → legendary grandmaster
        Data Fetching Successful
        ```
        """
        try:
            url = self.user_url.format(self.username)
            info = requests.get(url)
            soup = BeautifulSoup(info.content)
            user_info_set = info.json()
            user_info_data = user_info_set["result"][0]
            for data in user_info_data:
                print(f"{data} → {user_info_data[data]}")
            return (
                "Data Fetching Successful"
                if user_info_set["status"] == "OK"
                else "Failed"
            )
        except IndexError:
            raise Exception("Invalid User Name.")
        except TypeError:
            raise Exception("Invalid User Name")
        except requests.exceptions.ConnectionError:
            raise Exception("Connection Error, Please try again.")


if __name__ == "__main__":
    user = USER("benq")
    print(user.user_get_data())


# src/scrape_up/coursera/__init__.py

from courses import Coursera

__all__ = ["Coursera"]


# src/scrape_up/coursera/courses.py

import requests
from bs4 import BeautifulSoup
import json


class Coursera:
    """
    Create an object of the 'Courses' class:
    ```python
    scraper = Courses("topic")
    ```
    | Methods                               | Details                                                                                    |
    | ------------------------------------- | ------------------------------------------------------------------------------------------ |
    | `.get_courses()`                       | Returns the courses with title, teached by, skills, rating, review count, img url and link |
    | `.fetchModules(course='Course Name')` | Returns the modules associated with the Coursera.                                          |
    """

    def __init__(self, topic):
        self.topic = topic

    def get_courses(self):
        """
        Class - `Coursera`
        Example:
        ```
        courses = Coursera(topic="ml")
        courses.get_courses()
        ```
        Returns:
        ```js
        [
            {
                "title": Title of the course
                "teached_by": Organization which teaches the course
                "skills": Skills learnt from the course
                "rating": Rating of the course
                "review_count": Np. of review of the course
                "img_url": Image URL of the course
                "link": Link to the course
            }
            ...
        ]
        ```
        """
        url = "https://www.coursera.org/search?query=" + self.topic
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            courses_data = {"courses": []}

            courses = soup.find_all("div", class_="css-1cj5od")

            for c in courses:
                try:
                    title = c.find("h2", class_="cds-119 css-h1jogs cds-121").getText()
                    teached_by = c.find(
                        "span", class_="cds-119 css-1mru19s cds-121"
                    ).getText()
                    skills = c.find("p", class_="cds-119 css-12ksubz cds-121").getText()
                    rating = c.find("p", class_="cds-119 css-11uuo4b cds-121").getText()
                    review_count = (
                        c.find("p", class_="cds-119 css-dmxkm1 cds-121")
                        .getText()
                        .replace("(", "")
                        .replace(")", "")
                    )
                    img = c.find("div", class_="css-1doy6bd")
                    img_url = img.find("img")["src"]
                    link = "https://www.coursera.org" + c.find("a")["href"]
                except:
                    pass

                courses_data["courses"].append(
                    {
                        "title": title,
                        "teached_by": teached_by,
                        "skills": skills,
                        "rating": rating,
                        "review_count": review_count,
                        "img_url": img_url,
                        "link": link,
                    }
                )
            return courses_data["courses"]
        except:
            return None

    def fetch_modules(self, course):
        """
        Class - `Coursera`
        Example:
        ```
        courses = Coursera(topic="ml")
        courses.fetch_modules()
        ```
        Returns:
        ```js
        [ modules ]
        ```
        """
        courseList = self.get_courses()
        global ccourseURL
        for i in courseList:
            if i["title"] == course:
                courseURL = i["link"]
        try:
            res = requests.get(courseURL)
            if res.status_code == 200:
                soup = BeautifulSoup(res.text, "html.parser")
                script_tag = soup.find("script", {"id": "__NEXT_DATA__"})
                if script_tag is not None:
                    json_blob = json.loads(script_tag.get_text())
                    product_data = json_blob["props"]["pageProps"]["initialData"][
                        "data"
                    ]["product"]

                modules = soup.find_all("div", class_="SyllabusModule")
                modules_data = []
                for m in modules:
                    mod = m.find("h3", class_="headline-2-text bold m-b-2").getText()
                    modules_data.append(mod)

                if modules_data == []:
                    modules = soup.find_all("div", class_="css-13tws8d")
                    for m in modules:
                        mod = m.find(
                            "a", class_="cds-119 cds-113 cds-115 css-1uw69sh cds-142"
                        ).getText()
                        modules_data.append(mod)

                return modules_data
            else:
                return "Server Error. Retry"
        except:
            return "No modules for this course"


# src/scrape_up/cricbuzz/__init__.py

from scrape_up.cricbuzz.cricbuzz import Cricbuzz

__all__ = ["Cricbuzz"]


# src/scrape_up/cricbuzz/cricbuzz.py

import requests
from bs4 import BeautifulSoup


class Cricbuzz:
    """
    Create an instance of `Cricubzz` class.
    ```python
    cricbuzz = Cricubzz()
    ```
    Methods\n
    | Methods                      | Details                                                                |
    | ---------------------------- | ---------------------------------------------------------------------- |
    | `.get_live_matches()`        | Returns a list of live matches from Cricbuzz.                          |
    | `.get_recent_matches()`      | Returns a list of recent matches from Cricbuzz.                        |
    | `.get_upcoming_matches()`    | Returns a list of upcoming matches from Cricbuzz.                      |
    | `.get_series()`              | Returns a dictionary of series in month and year format from Cricbuzz. |
    | `.get_series_from_archive()` | Returns a list of series from archive from Cricbuzz.                   |
    | `.get_matches_by_day()`      | Returns a dictionary of matches by day from Cricbuzz.                  |
    | `.get_series_matches()`      | Returns a list of matches in a series from Cricbuzz.                   |
    | `.get_series_stats()`        | Returns a list of stats of players in a series from Cricbuzz.          |
    | `.get_teams_list()`          | Returns a list of teams from Cricbuzz.                                 |
    | `.get_team_schedule()`       | Returns a list of matches of a team from Cricbuzz.                     |
    | `.get_team_players()`        | Returns a list of players of a team from Cricbuzz.                     |
    | `.get_team_results()`        | Returns a list of past results of a team from Cricbuzz.                |
    | `.get_team_stats()`          | Returns a list of player stats of a team from Cricbuzz.                |
    """

    BASE_URL = "https://www.cricbuzz.com/"
    TYPES = ["all", "international", "domestic", "league", "women"]
    FORMATS = ["Test", "ODI", "T20I"]
    STATS = [
        "most-runs",
        "highest-score",
        "highest-avg",
        "highest-sr",
        "most-hundreds",
        "most-fifties",
        "most-fours",
        "most-sixes",
        "most-nineties",
        "most-wickets",
        "lowest-avg",
        "best-bowling-innnings",
        "most-five-wickets",
        "lowest-econ",
        "lowest-sr",
    ]

    def __init__(self):
        self.session = requests.Session()

    def __timestamp_to_date(self, timestamp):
        """
        Converts timestamp to date
        """
        from datetime import datetime

        dt_obj = datetime.utcfromtimestamp(timestamp / 1000)
        return dt_obj.strftime("%d-%m-%Y %H:%M:%S")

    def __scrape_match(self, url, type, isUpcoming=False):
        try:
            res = self.session.get(url)
            if res.status_code != 200:
                return [{"error": "Unable to fetch data from cricbuzz"}]
            soup = BeautifulSoup(res.text, "html.parser")
            search_paras = {
                "class": "cb-col cb-col-100 cb-plyr-tbody cb-rank-hdr cb-lv-main"
            }
            if type.lower() not in self.TYPES:
                return [{"error": "Invalid type"}]
            elif type.lower() != "all":
                search_paras["ng-show"] = f"active_match_type == '{type.lower()}-tab'"
            series = soup.find_all(
                "div",
                attrs=search_paras,
            )
            if len(series) == 0:
                return [{"error": "No live matches"}]
            matches = []
            for s in series:
                series_name = s.find("h2").text.strip()
                series_matches = s.find_all(
                    "div", attrs={"class": "cb-mtch-lst cb-col cb-col-100 cb-tms-itm"}
                )
                for m in series_matches:
                    match_data = {}
                    match_data["series_name"] = series_name
                    match_data["match_name"] = (
                        m.find(
                            "h3", class_="cb-lv-scr-mtch-hdr inline-block"
                        ).text.strip()
                        + " "
                        + m.find("span", class_="text-gray").text.strip()
                    )
                    match_data["start_date_time"] = (
                        self.__timestamp_to_date(
                            int(
                                m.find("div", class_="text-gray")
                                .find("span")["ng-if"]
                                .split("|")[0]
                                .strip()[1:]
                            )
                        )
                        + " GMT"
                    )
                    match_data["location"] = "at".join(
                        m.find("div", class_="text-gray")
                        .find_all("span")[-1]
                        .text.split("at")[1:]
                    ).strip()
                    match_data_id = m.find("a")["href"].split("/")
                    match_id = match_data_id[
                        match_data_id.index("live-cricket-scores") + 1
                    ]
                    match_data["match_id"] = match_id
                    if not isUpcoming:
                        match_data["status"] = m.find("div", class_="cb-text-live")
                        if match_data["status"] == None:
                            match_data["status"] = m.find(
                                "div", class_="cb-text-complete"
                            )
                        if match_data["status"] == None:
                            match_data["status"] = m.find(
                                "div", class_="cb-text-preview"
                            )
                        if match_data["status"] != None:
                            match_data["status"] = match_data["status"].text.strip()
                        else:
                            del match_data["status"]
                        match_score = []
                        score_div = m.find(
                            "div", class_="cb-scr-wll-chvrn cb-lv-scrs-col"
                        )
                        if score_div == None:
                            matches.append(match_data)
                            continue
                        bat_scores_div = score_div.find(
                            "div", class_="cb-hmscg-bat-txt"
                        )
                        bowl_scores_div = score_div.find(
                            "div", class_="cb-hmscg-bwl-txt"
                        )
                        if bat_scores_div == None and bowl_scores_div == None:
                            matches.append(match_data)
                            continue
                        elif bat_scores_div == None:
                            bat_scores_div = score_div.find_all(
                                "div", class_="cb-hmscg-bwl-txt"
                            )[1]
                        elif bowl_scores_div == None:
                            bowl_scores_div = score_div.find_all(
                                "div", class_="cb-hmscg-bat-txt"
                            )[1]
                        bat_team_names = bat_scores_div.find(
                            "div", class_="cb-hmscg-tm-nm"
                        ).text.strip()
                        bowl_team_names = bowl_scores_div.find(
                            "div", class_="cb-hmscg-tm-nm"
                        ).text.strip()
                        bat_scores = bat_scores_div.find_all(
                            "div", class_="cb-ovr-flo"
                        )[-1].text.strip()
                        bowl_scores = bowl_scores_div.find_all(
                            "div", class_="cb-ovr-flo"
                        )[-1].text.strip()
                        bat_team_scores = {
                            "team_name": bat_team_names,
                            "scores": [i.strip() for i in bat_scores.split("&")],
                        }
                        bowl_team_scores = {
                            "team_name": bowl_team_names,
                            "scores": [i.strip() for i in bowl_scores.split("&")],
                        }
                        match_score.append(bat_team_scores)
                        match_score.append(bowl_team_scores)
                        match_data["score"] = match_score
                    matches.append(match_data)
            return matches

        except Exception as e:
            return [{"error": f"Something Went Wrong: {e}"}]

    def get_live_matches(self, type="all"):
        """
        Class - Cricbuzz\n
        Params required
        - TYPES = ["all", "international", "domestic", "league", "women"]\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_live_matches(type="all")
        ```
        """
        try:
            URL = self.BASE_URL + "cricket-match/live-scores"
            response = self.__scrape_match(url=URL, type=type)
            return response
        except:
            return None

    def get_recent_matches(self, type="all"):
        """
        Class - Cricbuzz\n
        Params required
        - TYPES = ["all", "international", "domestic", "league", "women"]\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_recent_matches(type="all")
        ```
        """
        try:
            URL = self.BASE_URL + "cricket-match/live-scores/recent-matches"
            response = self.__scrape_match(url=URL, type=type)
            return response
        except:
            return None

    def get_upcoming_matches(self, type="all"):
        """
        Class - Cricbuzz\n
        Params required
        - TYPES = ["all", "international", "domestic", "league", "women"]\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_upcoming_matches(type="all")
        ```
        """
        try:
            URL = self.BASE_URL + "cricket-match/live-scores/upcoming-matches"
            response = self.__scrape_match(url=URL, type=type, isUpcoming=True)
            return response
        except:
            return None

    def __scrape_series(self, url, type="all"):
        try:
            res = self.session.get(url)
            if res.status_code != 200:
                return [{"error": "Unable to fetch data from cricbuzz"}]
            soup = BeautifulSoup(res.text, "html.parser")
            series = soup.find_all(
                "div",
                class_="cb-col-100 cb-col",
            )
            international_series = [
                i
                for i in series
                if i["ng-if"] == "((filtered_category == 0 || filtered_category == 9))"
            ]
            domestic_series = [
                i
                for i in series
                if i["ng-if"] == "((filtered_category == 1 || filtered_category == 9))"
            ]
            league_series = [
                i
                for i in series
                if i["ng-if"] == "((filtered_category == 2 || filtered_category == 9))"
            ]
            womens_series = [
                i
                for i in series
                if i["ng-if"] == "((filtered_category == 3 || filtered_category == 9))"
            ]
            diff_series = [
                international_series,
                domestic_series,
                league_series,
                womens_series,
            ]
            if type.lower() not in self.TYPES:
                return [{"error": "Invalid type"}]
            elif type.lower() != "all":
                series = diff_series[self.TYPES.index(type.lower()) - 1]
            series_data = {}
            for s in series:
                series_month = s.find(
                    "div", class_="cb-col-16 cb-col text-bold cb-mnth"
                )
                monthly_series_data = []
                series_div = s.find_all("div", class_="cb-sch-lst-itm")
                for i in series_div:
                    series_data_id = i.find("a")["href"].split("/")
                    series_id = series_data_id[
                        series_data_id.index("cricket-series") + 1
                    ]
                    monthly_series_data.append(
                        {
                            "series_name": i.find("a").text.strip(),
                            "series_id": series_id,
                            "series_dates": i.find(
                                "div", class_="text-gray cb-font-12"
                            ).text.strip(),
                        }
                    )
                if series_data.get(series_month.text.strip()) == None:
                    series_data[series_month.text.strip()] = monthly_series_data
                else:
                    series_data[series_month.text.strip()].extend(monthly_series_data)
            return series_data
        except Exception as e:
            return [{"error": f"Something Went Wrong: {e}"}]

    def get_series(self, type="all"):
        """
        Class - Cricbuzz\n
        Params required
        - TYPES = ["all", "international", "domestic", "league", "women"]\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_series(type="all")
        ```
        """
        try:
            URL = self.BASE_URL + "cricket-schedule/series"
            response = self.__scrape_series(url=URL, type=type)
            return response
        except:
            return None

    def __scrape_series_from_archive(self, url, type="all"):
        try:
            res = self.session.get(url)
            if res.status_code != 200:
                return [{"error": "Unable to fetch data from cricbuzz"}]
            soup = BeautifulSoup(res.text, "html.parser")
            data = soup.select_one(
                "#page-wrapper > div:nth-child(5) > div.cb-left.cb-col-67.cb-col.cb-schdl"
            )
            data_elements = [
                i.text.lower()
                for i in data.find_all(
                    "h2",
                    class_="cb-col-16 cb-col text-bold cb-srs-cat cb-lv-scr-mtch-hdr",
                )
            ]
            data_divs = data.find_all("div", class_="cb-col-84 cb-col")
            divs = data_divs
            if type.lower() != "all":
                if type.lower() == "league":
                    divs = [data_divs[data_elements.index("t20 league")]]
                else:
                    divs = [data_divs[data_elements.index(type.lower())]]
            series = []
            for d in divs:
                data = []
                series_data = d.find_all("div", class_="cb-srs-lst-itm")
                for s in series_data:
                    series_data_id = s.find("a")["href"].split("/")
                    series_id = series_data_id[
                        series_data_id.index("cricket-series") + 1
                    ]
                    data.append(
                        {
                            "series_name": s.find("a").text.strip(),
                            "series_id": series_id,
                            "series_dates": s.find(
                                "span", class_="text-gray cb-font-12"
                            ).text.strip(),
                        }
                    )
                series.extend(data)
            return series
        except Exception as e:
            return [{"error": f"Something Went Wrong: {e}"}]

    def get_series_from_archive(self, year, type="all"):
        """
        Class - Cricbuzz\n
        Params required
        - TYPES = ["all", "international", "domestic", "league", "women"]
        - Year\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_series_from_archive(year, type="all")
        ```
        """
        try:
            URL = self.BASE_URL + f"cricket-scorecard-archives/{year}"
            if type.lower() not in self.TYPES:
                return [{"error": "Invalid type"}]
            response = self.__scrape_series_from_archive(url=URL, type=type)
            return response
        except:
            return None

    def __scarpe_matches_by_day(self, url, type):
        try:
            res = self.session.get(url)
            if res.status_code != 200:
                return [{"error": "Unable to fetch data from cricbuzz"}]
            soup = BeautifulSoup(res.text, "html.parser")
            data = soup.find("div", id=f"{type.lower()}-list")
            days_data = data.select(f"#{type.lower()}-list > div:nth-child(n)")
            matches_data = {}
            for i in days_data:
                day = i.find("div", class_="cb-lv-grn-strip text-bold").text.strip()
                match_data = i.find_all("div", class_="cb-col-100 cb-col")
                matches = []
                for m in match_data:
                    match_data = {}
                    match_series = m.find("a")
                    match_series_data_id = match_series["href"].split("/")
                    match_data["series_name"] = match_series.text.strip()
                    match_data["series_id"] = match_series_data_id[
                        match_series_data_id.index("cricket-series") + 1
                    ]
                    match_elem = m.find("div", class_="cb-col-67 cb-col").find("a")
                    match_data_id = match_elem["href"].split("/")
                    match_data["match_name"] = match_elem.text.strip()
                    match_data["match_id"] = match_data_id[
                        match_data_id.index("live-cricket-scores") + 1
                    ]
                    match_data["location"] = m.find(
                        "div", attrs={"itemprop": "location"}
                    ).text.strip()
                    match_data["start_date_time"] = (
                        self.__timestamp_to_date(
                            int(m.find("span", class_="schedule-date")["timestamp"])
                        )
                        + " GMT"
                    )
                    matches.append(match_data)
                matches_data[day] = matches
            return matches_data

        except Exception as e:
            return [{"error": f"Something Went Wrong: {e}"}]

    def get_matches_by_day(self, type="all"):
        """
        Class - Cricbuzz\n
        Params required
        - TYPES = ["all", "international", "domestic", "league", "women"]\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_matches_by_day(type="all")
        ```
        """
        try:
            URL = self.BASE_URL + "cricket-schedule/upcoming-series"
            if type.lower() not in self.TYPES:
                return [{"error": "Invalid type"}]
            else:
                URL += f"/{type.lower()}"
            response = self.__scarpe_matches_by_day(url=URL, type=type)
            return response
        except:
            return None

    def __scrape_series_matches(self, url):
        try:
            res = self.session.get(url)
            if res.status_code != 200:
                return [{"error": "Unable to fetch data from cricbuzz"}]
            soup = BeautifulSoup(res.text, "html.parser")

            data = soup.find_all("div", class_="cb-series-matches")
            matches = []
            for m in data:
                match_data = {}
                match_data["match_start_data_time"] = (
                    self.__timestamp_to_date(
                        int(
                            m.find("div", class_="schedule-date")
                            .find("span")["ng-bind"]
                            .split("|")[0]
                            .strip()
                        )
                    )
                    + " GMT"
                )
                match_data["match_title"] = (
                    m.find("div", class_="cb-col-60 cb-col cb-srs-mtchs-tm")
                    .find("span")
                    .text.strip()
                )
                match_data["match_location"] = (
                    m.find("div", class_="cb-col-60 cb-col cb-srs-mtchs-tm")
                    .find("div", class_="text-gray")
                    .text.strip()
                )
                if (
                    m.find("div", class_="cb-col-60 cb-col cb-srs-mtchs-tm")
                    .find("div", class_="text-gray")
                    .find_next_sibling("a")
                    is not None
                ):
                    match_data["match_status"] = (
                        m.find("div", class_="cb-col-60 cb-col cb-srs-mtchs-tm")
                        .find("div", class_="text-gray")
                        .find_next_sibling("a")
                        .text.strip()
                    )
                match_id_data = m.find("a")["href"].split("/")
                try:
                    match_data["match_id"] = match_id_data[
                        match_id_data.index("cricket-scores") + 1
                    ]
                except:
                    match_data["match_id"] = match_id_data[
                        match_id_data.index("live-cricket-scores") + 1
                    ]

                matches.append(match_data)
            return matches
        except Exception as e:
            return [{"error": f"Something Went Wrong: {e}"}]

    def get_series_matches(self, series_id):
        """
        Class - Cricbuzz\n
        Params required
        - `series_id` = Cricket mathc series Id(can be found on Cricbuzz Website)\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_series_matches(series_id="abc")
        ```
        """
        try:
            URL = self.BASE_URL + f"cricket-series/{series_id}/series/matches"
            return self.__scrape_series_matches(url=URL)
        except:
            return None

    def __scrape_series_stats(self, series_id, match_format="Test", stat="most-runs"):
        try:
            format_index = self.FORMATS.index(match_format) + 1
            URL = f"https://www.cricbuzz.com/api/html/series/{series_id}/{stat}/{format_index}/0/0"
            res = self.session.get(URL)
            if res.status_code != 200:
                return [{"error": "Unable to fetch data from cricbuzz"}]
            soup = BeautifulSoup(res.text, "html.parser")
            if soup.find("tbody") == None:
                return {"error": "No data found"}
            keys = [i.text.strip() for i in soup.find("thead").find_all("th")]
            players_data = soup.find("tbody").find_all("tr")
            players = []
            for i in players_data:
                player_data_id = i.find("a")["href"].split("/")
                player_id = player_data_id[player_data_id.index("profiles") + 1]
                player_data = [i.text.strip() for i in i.find_all("td")]
                player_data_dict = dict(zip(keys, player_data))
                player_data_dict["PLAYER ID"] = player_id
                players.append(player_data_dict)
            return players
        except Exception as e:
            return [{"error": f"Something Went Wrong: {e}"}]

    def get_series_stats(self, series_id, match_format="Test", stat="most-runs"):
        """
        Class - Cricbuzz\n
        Params required
        - `series_id` = Cricket mathc series Id(can be found on Cricbuzz Website)\n
        - `match_format` = ["Test", "ODI", "T20I"]\n
        - `stat` = [
            "most-runs",
            "highest-score",
            "highest-avg",
            "highest-sr",
            "most-hundreds",
            "most-fifties",
            "most-fours",
            "most-sixes",
            "most-nineties",
            "most-wickets",
            "lowest-avg",
            "best-bowling-innnings",
            "most-five-wickets",
            "lowest-econ",
            "lowest-sr",
        ]\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_series_stats(series_id, match_format="Test", stat="most-runs")
        ```
        """
        try:
            if match_format not in self.FORMATS:
                return [
                    {"error": "Invalid match format", "valid_formats": self.FORMATS}
                ]
            if stat not in self.STATS:
                return [{"error": "Invalid stat", "valid_stats": self.STATS}]
            return self.__scrape_series_stats(
                series_id=series_id, match_format=match_format, stat=stat
            )
        except:
            return None

    def __scrape_team_data(self, url):
        try:
            res = self.session.get(url)
            if res.status_code != 200:
                return [{"error": "Unable to fetch data from cricbuzz"}]
            soup = BeautifulSoup(res.text, "html.parser")
            data = soup.select_one(
                "#page-wrapper > div:nth-child(6) > div.cb-col.cb-col-67.cb-nws-lft-col > div.cb-col.cb-col-100"
            )
            teams = []
            teams_data = data.find_all("div", class_="cb-team-item")
            for t in teams_data:
                team_name = t.find("a")["title"].strip().title()
                team_code = t.find("a")["href"].split("/")[-1].strip()
                teams.append({"team_name": team_name, "team_code": team_code})
            return teams
        except Exception as e:
            return [{"error": f"Something Went Wrong: {e}"}]

    def get_teams_list(self, type="all"):
        """
        Class - Cricbuzz\n
        Params required
        - `type` = ["all", "international", "domestic", "league", "women"]\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_teams_list(type="all")
        ```
        """
        URL = self.BASE_URL + "cricket-team"
        if type.lower() not in self.TYPES:
            return [{"error": "Invalid type"}]
        elif type.lower() != "all" and type.lower() != "international":
            URL += f"/{type.lower()}"
        if type.lower() == "all":
            return {
                "international": self.__scrape_team_data(url=URL),
                "domestic": self.__scrape_team_data(url=URL + "/domestic"),
                "league": self.__scrape_team_data(url=URL + "/league"),
                "women": self.__scrape_team_data(url=URL + "/women"),
            }
        return self.__scrape_team_data(url=URL)

    def __scrape_team_schedule(self, url):
        try:
            res = self.session.get(url)
            if res.status_code != 200:
                return [{"error": "Unable to fetch data from cricbuzz"}]
            soup = BeautifulSoup(res.text, "html.parser")

            data = soup.find_all("div", class_="cb-series-matches")
            matches = []
            for m in data:
                match_data = {}
                match_data["match_start_data_time"] = (
                    self.__timestamp_to_date(
                        int(
                            m.find("div", class_="schedule-date")
                            .find("span")["ng-bind"]
                            .split("|")[0]
                            .strip()
                        )
                    )
                    + " GMT"
                )
                match_data["match_title"] = (
                    m.find("div", class_="cb-srs-mtchs-tm").find("span").text.strip()
                )
                match_data["match_location"] = (
                    m.find("div", class_="cb-srs-mtchs-tm")
                    .find("div", class_="text-gray cb-ovr-flo")
                    .text.strip()
                )
                match_data["match_series_name"] = (
                    m.find("div", class_="cb-srs-mtchs-tm")
                    .find("div", class_="text-gray")
                    .text.strip()
                )
                if (
                    m.find("div", class_="cb-srs-mtchs-tm")
                    .find("div", class_="text-gray")
                    .find_next_sibling("a")
                    is not None
                ):
                    match_data["match_status"] = (
                        m.find("div", class_="cb-srs-mtchs-tm")
                        .find("div", class_="text-gray")
                        .find_next_sibling("a")
                        .text.strip()
                    )
                match_id_data = m.find("a")["href"].split("/")
                try:
                    match_data["match_id"] = match_id_data[
                        match_id_data.index("cricket-scores") + 1
                    ]
                except:
                    match_data["match_id"] = match_id_data[
                        match_id_data.index("live-cricket-scores") + 1
                    ]

                matches.append(match_data)
            return matches
        except Exception as e:
            return [{"error": f"Something Went Wrong: {e}"}]

    def get_team_schedule(self, team_id):
        """
        Class - Cricbuzz\n
        Params required
        - `team_id` = Can be extracted from Cricbuzz\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_team_schedule(team_id=" ")
        """
        URL = self.BASE_URL + f"cricket-team/team/{team_id}/schedule"
        return self.__scrape_team_schedule(url=URL)

    def __scrape_team_players(self, url):
        # try:
        res = self.session.get(url)
        if res.status_code != 200:
            return [{"error": "Unable to fetch data from cricbuzz"}]
        soup = BeautifulSoup(res.text, "html.parser")
        players_data = soup.find("div", class_="cb-top-zero").find_all(
            "a", class_="cb-col cb-col-50"
        )
        players = []
        for player in players_data:
            player_data_id = player["href"].split("/")
            player_id = player_data_id[player_data_id.index("profiles") + 1]
            players.append(
                {
                    "player_name": player.text.strip(),
                    "player_id": player_id,
                }
            )
        return players

    def get_team_players(self, team_id):
        """
        Class - Cricbuzz\n
        Params required
        - `team_id` = Can be extracted from Cricbuzz\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_team_players(team_id=" ")
        ```
        """
        try:
            URL = self.BASE_URL + f"cricket-team/team/{team_id}/players"
            return self.__scrape_team_players(url=URL)
        except:
            return None

    def __scrape_team_results(self, url):
        try:
            res = self.session.get(url)
            if res.status_code != 200:
                return [{"error": "Unable to fetch data from cricbuzz"}]
            soup = BeautifulSoup(res.text, "html.parser")

            data = soup.find("div", id="series-matches").find_all(
                "div", class_="cb-col-100 cb-col cb-brdr-thin-btm"
            )
            matches = []
            for m in data:
                match_data = {}
                match_data["match_start_data_time"] = (
                    self.__timestamp_to_date(
                        int(m.find("span")["ng-bind"].split("|")[0].strip())
                    )
                    + " GMT"
                )
                match_data["match_title"] = (
                    m.find("div", class_="cb-srs-mtchs-tm").find("a").text.strip()
                )
                match_data["match_series_name"] = (
                    m.find("div", class_="cb-srs-mtchs-tm")
                    .find("div", class_="text-gray")
                    .text.strip()
                )
                if (
                    len(
                        m.find("div", class_="cb-srs-mtchs-tm").find_all(
                            "div", class_="text-gray"
                        )
                    )
                    > 1
                ):
                    match_data["match_location"] = (
                        m.find("div", class_="cb-srs-mtchs-tm")
                        .find_all("div", class_="text-gray")[1]
                        .text.strip()
                    )
                if (
                    m.find("div", class_="cb-srs-mtchs-tm")
                    .find("div", class_="text-gray")
                    .find_next_sibling("a")
                    is not None
                ):
                    match_data["match_status"] = (
                        m.find("div", class_="cb-srs-mtchs-tm")
                        .find("div", class_="text-gray")
                        .find_next_sibling("a")
                        .text.strip()
                    )
                match_id_data = m.find("a")["href"].split("/")
                try:
                    match_data["match_id"] = match_id_data[
                        match_id_data.index("cricket-scores") + 1
                    ]
                except:
                    match_data["match_id"] = match_id_data[
                        match_id_data.index("live-cricket-scores") + 1
                    ]

                matches.append(match_data)
            return matches
        except Exception as e:
            return [{"error": f"Something Went Wrong: {e}"}]

    def get_team_results(self, team_id):
        """
        Class - Cricbuzz\n
        Params required
        - `team_id` = Can be extracted from Cricbuzz\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_team_results(team_id=" ")
        ```
        """
        try:
            URL = self.BASE_URL + f"cricket-team/team/{team_id}/results"
            return self.__scrape_team_results(url=URL)
        except:
            return None

    def __scrape_team_stats(self, team_id, year, match_format="Test", stat="most-runs"):
        try:
            format_index = self.FORMATS.index(match_format) + 1
            URL = f"https://www.cricbuzz.com/cricket-team/team/{team_id}/stats-table/{stat}/{format_index}/{year}/all"
            res = self.session.get(URL)
            if res.status_code != 200:
                return [{"error": "Unable to fetch data from cricbuzz"}]
            soup = BeautifulSoup(res.text, "html.parser")
            if soup.find("tbody") == None:
                return {"error": "No data found"}
            keys = [i.text.strip() for i in soup.find("thead").find_all("th")]
            players_data = soup.find("tbody").find_all("tr")
            players = []
            for i in players_data:
                player_data_id = i.find("a")["href"].split("/")
                player_id = player_data_id[player_data_id.index("profiles") + 1]
                player_data = [i.text.strip() for i in i.find_all("td")]
                player_data_dict = dict(zip(keys, player_data))
                player_data_dict["PLAYER ID"] = player_id
                players.append(player_data_dict)
            return players
        except Exception as e:
            return [{"error": f"Something Went Wrong: {e}"}]

    def get_team_stats(
        self, team_id, year="all", match_format="Test", stat="most-runs"
    ):
        """
        Class - Cricbuzz\n
        Params required
        - `team_id` = Team Id(can be found on Cricbuzz Website)\n
        - `match_format` = ["Test", "ODI", "T20I"]\n
        - `stat` = [
            "most-runs",
            "highest-score",
            "highest-avg",
            "highest-sr",
            "most-hundreds",
            "most-fifties",
            "most-fours",
            "most-sixes",
            "most-nineties",
            "most-wickets",
            "lowest-avg",
            "best-bowling-innnings",
            "most-five-wickets",
            "lowest-econ",
            "lowest-sr",
        ]\n
        ```python
        cricbuzz = Cricubzz()
        cricbuzz.get_team_stats(team_id, match_format="Test", stat="most-runs")
        ```
        """
        try:
            if match_format not in self.FORMATS:
                return [
                    {"error": "Invalid match format", "valid_formats": self.FORMATS}
                ]
            if stat not in self.STATS:
                return [{"error": "Invalid stat", "valid_stats": self.STATS}]
            return self.__scrape_team_stats(
                team_id=team_id, year=year, match_format=match_format, stat=stat
            )
        except:
            return None

cric = Cricbuzz()
print(cric.get_live_matches())

# src/scrape_up/devCommunity/__init__.py

from scrape_up.devCommunity.articles import DevCommunity

__all__= ["DevCommunity"]

# src/scrape_up/devCommunity/articles.py

from bs4 import BeautifulSoup
from pprint import pprint
import json
import requests

class DevCommunity:
    """
    Class - `DevCommunity` \n

    Creates a scraper for latest articles from https://www.dev.to.
    Creates a scraper for latest articles from https://www.dev.to/t/{tag} with a given tag.
    Creates a scraper for  user details, pinned articles written by a user, and all articles 
    written by a user from https://www.dev.to/{username}.

    Example - \n
    ```python
    dev = DevCommunity('francescoxx')
    """
    def __init__(self, username: str):
        self.help = "This scrapes articles from the DevCommunity website."
        self.username = username

    
    def __scrape_page(self):
        try:
            data = requests.get('https://www.dev.to')
            data.raise_for_status()
            data = BeautifulSoup(data.text, "html.parser")
            return data
        except:
            message = "An Error Occurred!"
            return message
    
    def all_articles(self):
        """
		Class - `DevCommunity` \n
        Example - \n
        ```python
        articles = dev.all_articles()
        pprint(articles)
        ```
        Return \n
        ```python
        return 
        {
            "{'data': '{\n'
         '  "articles": [\n'
         '    {\n'
         '      "Title": "With GitHub apparently down right now...",\n'       
         '      "Author": "Ben Halpern",\n'
         '      "Date": "Jun 29",\n'
         '      "Tags": "#watercooler\\n#github",\n'
         '      "Reactions": "22\\u00a0reactions",\n'
         '      "Comments": "Comments\\n\\n              6\\u00a0comments",\n'
         '      "Read Time": "2 min read"\n'
         '    },\n'
         '    {\n'
         '      "Title": "Sloan\'s Inbox: What is the Secret to a Truly Great '
         'Portfolio?",\n'
         '      "Author": "Sloan the DEV Moderator",\n'
         '      "Date": "Jun 29",\n'
         '      "Tags": '
         '"#discuss\\n#beginners\\n#career\\n#careerdevelopment",\n'
         '      "Reactions": "10\\u00a0reactions",\n'
         '      "Comments": "Comments\\n\\n              1\\u00a0comment",\n'  
         '      "Read Time": "1 min read"\n'
         '    },\n'
         '    {\n'
         '      "Title": "Sticky sessions with Apache APISIX",\n'
         '      "Author": "Nicolas Frankel",\n'
         '      "Date": "Jun 29",\n'
         '      "Tags": '
         '"#stickysessions\\n#sessionaffinity\\n#apacheapisix\\n#devops",\n'
         '      "Reactions": "44\\u00a0reactions",\n'
         '      "Comments": "Comments\\nAdd\\u00a0Comment",\n'
         '      "Read Time": "4 min read"\n'
         '    }
         '  ]\n'
         '}',
 		'message': 'Successfully fetched all articles.'}
        
        ```
		"""
        page = self.__scrape_page()
        articles_list = {"articles": []}
        try:
            articles = page.find_all( class_="substories")
            for x in articles:
                title = x.find_all(class_="crayons-story__title")        
                author = x.find_all('a', class_="crayons-story__secondary fw-medium m:hidden")
                date = x.find_all('a', class_="crayons-story__tertiary fs-xs")       
                tags = x.find_all(class_="crayons-story__tags")
                reactions = x.find_all('span', class_="aggregate_reactions_counter")        
                comments = x.find_all('a',class_="crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon-left flex items-center")        
                readTime = x.find_all(class_="crayons-story__save")
    
            for ti,a,d,ta,r,c,re, in zip(title, author, date, tags, reactions, comments, readTime):
                articles_list["articles"].append({
                "Title": ti.text.strip(),
                "Author": a.text.strip(),
                "Date": d.text.strip(),
                "Tags": ta.text.strip(),
                "Reactions": r.text.strip(),
                "Comments": c.text.strip(),
                "Read Time": re.text.strip(),
            })
            json_data = json.dumps(articles_list, indent=2)
            return {"data": json_data, "message": "Successfully fetched all articles."}
        except:
            message = "An Error Occurred!"
            return message


    def __strTag__(self, tag: str):
        """
		Class - `DevCommunity` \n
    	Example - \n
    	```python
        tag_name = dev.__strTag__(tag='python')
        print(tag_name)
        ```
        Return \n
        ```python
		The list of articles we want to scrape have the tag: python
		"""
        self.tag = tag
        return f" The list of articles we want to scrape have the tag: {self.tag}"
    
    def __scrape_tag(self, tag: str):
        try:
            self.tag = tag  
            data = requests.get(f"https://www.dev.to/t/{self.tag}")
            data.raise_for_status()
            data = BeautifulSoup(data.text, "html.parser")
            return data
        except:
            message = "An Error Occurred!"
            return message
    
    
    def tag_articles(self, tag: str):
        """
		Class - `DevCommunity` \n
    	Example - \n
    	```python
        tagged_articles = dev.tag_articles(tag='python')
        pprint(tagged_articles)
        ```
        Return \n
        ```python
		return
		{
		{'data': '{\n'
		'  "articles": [\n'
		'    {\n'
		'      "Title": "Introducing The Spatial Cypher Cheat Sheet",\n'
		'      "nAuthor": "William Lyon",\n'
		'      "Date": "Jun 28",\n'
		'      "Tags": "#neo4j\\n#python\\n#datascience\\n#gis",\n'
		'      "Reactions": "5\\u00a0reactions",\n'
		'      "Comments": "Comments\\nAdd\\u00a0Comment",\n'
		'      "Read Time": "10 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "Redis",\n'
		'      "nAuthor": "Phan T\\u1ea5n Th\\u1eafng",\n'
		'      "Date": "Jun 28",\n'
		'      "Tags": "#devops\\n#python\\n#javascript\\n#tutorial",\n'
		'      "Reactions": "2\\u00a0reactions",\n'
		'      "Comments": "Comments\\nAdd\\u00a0Comment",\n'
		'      "Read Time": "1 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "Making a Cloud Resume",\n'
		'      "nAuthor": "Tristan Armbrister",\n'
		'      "Date": "Jun 27",\n'
		'      "Tags": "#cloud\\n#azure\\n#python\\n#webdev",\n'
		'      "Reactions": "5\\u00a0reactions",\n'
		'      "Comments": "Comments\\nAdd\\u00a0Comment",\n'
		'      "Read Time": "7 min read"\n'
		'    },\n'
		]\n'
		'}',
 		'message': 'Successfully fetched all articles with the given tag.'
		}
		"""  
        self.tag = tag
        page = self.__scrape_tag(self.tag)
        articles_list = {"articles": []}
        try:
            articles = page.find_all( class_="substories")
            for x in articles:
                title = x.find_all(class_="crayons-story__title")        
                author = x.find_all('a', class_="crayons-story__secondary fw-medium m:hidden")
                date = x.find_all('a', class_="crayons-story__tertiary fs-xs")       
                tags_ = x.find_all(class_="crayons-story__tags")
                reactions = x.find_all('span', class_="aggregate_reactions_counter")        
                comments = x.find_all('a',class_="crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon-left flex items-center")        
                readTime = x.find_all(class_="crayons-story__save")
    
            for ti,a,d,ta,r,c,re, in zip(title, author, date, tags_, reactions, comments, readTime):
                articles_list["articles"].append({
                "Title": ti.text.strip(),
                "nAuthor": a.text.strip(),
                "Date": d.text.strip(),
                "Tags": ta.text.strip(),
                "Reactions": r.text.strip(),
                "Comments": c.text.strip(),
                "Read Time": re.text.strip(),
            })
            json_data = json.dumps(articles_list, indent=2)
            return {"data": json_data, "message": "Successfully fetched all articles with the given tag."}
        except:
            message = "An Error Occurred!"
            return message


    def __strUser__(self):
        """
		Class - `DevCommunity` \n
    	Example - \n
    	```python
        user = dev.__strUser__()
		print(user)
        ```
        Return \n
        ```python
		The username is: francescoxx
		"""
        return f" The username is: {self.username}"

    def __scrape_User(self):
        try:
            username = self.username 
            data = requests.get(f"https://www.dev.to/{username}")
            data.raise_for_status()
            data = BeautifulSoup(data.text, "html.parser")
            return data
        except:
            message = "An Error Occurred!"
            return message

    def user_details(self):
        """
		Class - `User` \n
    	Example - \n
    	```python        
        user_detail = dev.user_details()
        print(user_detail)
        ```
        Return \n
        ```python
		return
		{'data': [{'Name': 'Francesco Ciulla', 'Bio': 'Check YouTube: https://www.youtube.com/@francescociulla\n', 
		'Address': ['Location', 'Rome,', 'Italy']}], 'message': 'Successfully fetched user details.'}
		"""
        page = self.__scrape_User()
        user_details = []
        try:	
            user = page.find(class_="profile-header__details")
            name = user.find(class_="crayons-title fw-heavy mb-2").text
            bio = user.find(class_="fs-base m:fs-l color-base-90 mb-4 mx-auto max-w-100 m:max-w-75").text
            address = user.find(class_="profile-header__meta__item").text.split()
            user_details.append({"Name": name, "Bio": bio, "Address": address})
            return {"data": user_details, "message": "Successfully fetched user details."}		
        except:
            message = "An Error Occurred!"
            return {"data": user_details, "message": message}

    def pinned_articles(self):	
        """
		Class - `User` \n
    	Example - \n
    	```python
        pin_articles = dev.pinned_articles()
		pprint(pin_articles)
        ```
        Return \n
        ```python
		return
		{'data': '{\n'
		'  "articles": [\n'
		'    {\n'
		'      "Title": "Rust \\ud83e\\udd80 CRUD Rest API with Docker '
		'\\ud83d\\udc33",\n'
		'      "Date": "May 4",\n'
		'      "Tags": "#rust\\n#webdev\\n#beginners\\n#devops",\n'
		'      "Reactions": "129\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              37\\u00a0comments",\n'
		'      "Read Time": "15 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "TypeScript CRUD Rest API, using Nest.js, TypeORM, '
		'Postgres, Docker and Docker Compose",\n'
		'      "Date": "Mar 26",\n'
		'      "Tags": "#typescript\\n#webdev\\n#beginners\\n#programming",\n'
		'      "Reactions": "231\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              16\\u00a0comments",\n'
		'      "Read Time": "11 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "How to Learn Web3",\n'
		'      "Date": "Sep 1 \'22",\n'
		'      "Tags": "#web3\\n#blockchain\\n#beginners\\n#programming",\n'
		'      "Reactions": "494\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              49\\u00a0comments",\n'
		'      "Read Time": "10 min read"\n'
		'    }\n'
		'  ]\n'
		'}',
 		'message': 'Successfully fetched pinned articles.'}
		"""
        try:
            page = self.__scrape_User()
            articles_list = {"articles": []}
            articles = page.find_all(class_="crayons-card mb-2 border-2 border-solid border-accent-brand")
            
            for article in articles:
                title = article.find_all(class_="crayons-story__title")
                date = article.find_all('a', class_="crayons-story__tertiary fs-xs")
                tags = article.find_all(class_="crayons-story__tags")
                reactions = article.find_all('span', class_="aggregate_reactions_counter")
                comments = article.find_all('a',class_="crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon-left flex items-center")
                readTime = article.find_all(class_="crayons-story__save")
                
            for ti,d,ta,r,c,re, in zip(title, date, tags, reactions, comments, readTime):
                articles_list["articles"].append({
                "Title": ti.text.strip(),
                "Date": d.text.strip(),
                "Tags": ta.text.strip(),
                "Reactions": r.text.strip(),
                "Comments": c.text.strip(),
                "Read Time": re.text.strip(),
            })
            json_data = json.dumps(articles_list, indent=2)
            return {"data": json_data, "message": "Successfully fetched pinned articles."}

        except:
                message = "An Error Occurred!"
                return {"data": json_data, "message": message}

    def user_articles(self):
        """
		Class - `User` \n
    	Example - \n
    	```python
        user_article = dev.user_articles()
		pprint(user_article)
        ```
        Return \n
        ```python
		return
		{'data': '{\n'
		'  "articles": [\n'
		'    {\n'
		'      "Title": "Rust \\ud83e\\udd80 CRUD Rest API with Docker '
		'\\ud83d\\udc33",\n'
		'      "Date": "May 4",\n'
		'      "Tags": "#rust",\n'
		'      "Reactions": "129\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              37\\u00a0comments",\n'
		'      "Read Time": "15 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "Kubernetes quick tutorial",\n'
		'      "Date": "Apr 18",\n'
		'      "Tags": "#webdev",\n'
		'      "Reactions": "55\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              11\\u00a0comments",\n'
		'      "Read Time": "5 min read"\n'
		'    },\n'
		'    {\n'
		'      "Title": "TypeScript CRUD Rest API, using Nest.js, TypeORM, '
		'Postgres, Docker and Docker Compose",\n'
		'      "Date": "Mar 26",\n'
		'      "Tags": "#beginners",\n'
		'      "Reactions": "231\\u00a0reactions",\n'
		'      "Comments": "Comments\\n\\n              16\\u00a0comments",\n'
		'      "Read Time": "11 min read"\n'
		'    },\n'
		'  ]\n'
		'}',
 		'message': 'Successfully fetched all articles.'}
		"""
        page = self.__scrape_User()
        articles_list = {"articles": []}
        try:
            articles = page.find_all(class_="substories")
            for article in articles:
                date = article.find_all(class_="crayons-story__tertiary fs-xs")
                title = article.find_all(class_="crayons-story__title")
                tags = article.find_all(class_="crayons-tag crayons-tag--monochrome")
                reactions = article.find_all(class_="aggregate_reactions_counter")
                comments = article.find_all('a',class_="crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon-left flex items-center")
                readTime = article.find_all(class_="crayons-story__tertiary fs-xs mr-2")

            for ti,d,ta,r,c,re, in zip(title, date, tags, reactions, comments, readTime):
                articles_list["articles"].append({
                "Title": ti.text.strip(),
                "Date": d.text.strip(),
                "Tags": ta.text.strip(),
                "Reactions": r.text.strip(),
                "Comments": c.text.strip(),
                "Read Time": re.text.strip(),
            })
            json_data = json.dumps(articles_list, indent=2)
            
            return {"data": json_data, "message": "Successfully fetched all articles."}

        except:
                message = "An Error Occurred!"
                return {"data": json_data, "message": message}
            


dev = DevCommunity('francescoxx')
articles = dev.all_articles()
pprint(articles)

tag_name = dev.__strTag__(tag='python')
tagged_articles = dev.tag_articles(tag='python')
print(tag_name)
pprint(tagged_articles)

user = dev.__strUser__()
user_detail = dev.user_details()
pin_articles = dev.pinned_articles()
user_article = dev.user_articles()
print(user)
print(user_detail)
pprint(pin_articles)
pprint(user_article)

    

# src/scrape_up/eBay/__init__.py

from ebay import eBay

__all__ = ["eBay"]


# src/scrape_up/eBay/ebay.py

from bs4 import BeautifulSoup
import requests


class EBAY:
    """
    Create an instance of EBAY class
    ```python
    quora = EBAY()
    ```
    | Methods             | Details                             |
    | ------------------- | ----------------------------------- |
    | `.spotlights()`     | Returns spotlight deals on EBAY.    |
    | `.featured()`       | Returns the featured deals on EBAY. |
    | `.specific_deals()` | Returns the specific deals on EBAY. |
    """

    def __init__(self):
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36"
        }
        url = "https://www.ebay.com/globaldeals"
        html_text = requests.get(url, headers=headers).text
        soup = BeautifulSoup(html_text, "lxml")

        self.container = soup.find("div", {"class": "sections-container"})

    def spotlights(self):
        """
        Get the spotlight deal information.
        Example:
        ```python
        ebay = EBAY()
        spotlight_data = EBAY.spotlights()
        ```
        Returns:
        ```js
        {
            "Description":"Spotlight Deal",
            "Product":"Apple iPhone 12 Pro 128GB Unlocked Smartphone - Very Good",
            "Price":"US $524.95",
            "Link":"https://www.ebay.com/itm/363490711983?_trkparms=5373%3A0%7C5374%3AFeatured"
        }
        ```
        """
        try:
            spotlight_deal = self.container.find(
                "div",
                {
                    "class": "ebayui-dne-summary-card card ebayui-dne-item-featured-card--topDeals"
                },
            ).find("div", {"class": "dne-itemtile-detail"})
            title = spotlight_deal.find("h3").text
            price = spotlight_deal.find("span", {"class": "first"}).text
            url = spotlight_deal.find("a", href=True)
            spotlight = {
                "Description": "Spotlight Deal",
                "Product": title,
                "Price": price,
                "Link": url["href"],
            }
            return spotlight
        except:
            return None

    def featured(self):
        """
        Get the featured deals information.
        Example:
        ```python
        ebay = eBay()
        featured_data = ebay.featured()
        ```
        Returns:
        ```
        [
            {
                "Product":"Intel Core i5-13600KF Unlocked Desktop Processor - 14 Cores (6P+8E) & 20 Threads",
                "Price":"US $313.41",
                "Link":"https://www.ebay.com/itm/295247209215?_trkparms=5373%3A0%7C5374%3AFeatured"
            }
            ...
        ]
        ```
        """
        try:
            featured_deal = self.container.find(
                "div", {"class": "ebayui-dne-banner-text"}
            ).next_sibling
            Featured_deal = []
            for cols in featured_deal.find_all("div", {"class", "col"}):
                cols = cols.find("div", {"class": "dne-itemtile-detail"})
                title = cols.find("h3").text
                url = cols.find("a", href=True)["href"]
                price = cols.find("span", {"class": "first"}).text
                featured = {"Product": title, "Price": price, "Link": url}
                Featured_deal.append(featured)
            return Featured_deal
        except:
            return None

    def specific_deals(self):
        """
        Get the specific deals information.
        Example:
        ```python
        ebay = EBAY()
        specific_deals_data = EBAY.specific_deals()
        ```

        Returns:
        ```js
        [
            {
                "Product":"STRAY KIDS MAXIDENT Album GO LIMITED Ver. CD+P.Book+Poster+2 Card+Pre-Order+GIFT",
                "Price":"US $32.50",
                "Link":"https://www.ebay.com/itm/165661661490?_trkparms=5373%3A0%7C5374%3AFeatured%7C5079%3A6000012162"
            }
            ...
        ]
        ``
        """
        try:
            for items in self.container.find_all(
                "div",
                {
                    "class": "ebayui-dne-item-pattern-card ebayui-dne-item-pattern-card-no-padding ebayui-dne-item-pattern-card--desktop"
                },
            ):
                specific = []
                i = 0
                for cols in items.find("div", {"class": "row"}).find_all(
                    "div", {"class": "col"}
                ):
                    for item in cols.find_all("div", {"class": "item"}):
                        if i == 4:
                            break
                        i += 1
                        title = item.find("h3").text
                        price = item.find("span", {"class": "first"}).text
                        url = item.find("a", href=True)["href"]
                        deal = {"Product": title, "Price": price, "Link": url}
                        specific.append(deal)
            return specific
        except:
            return None


# src/scrape_up/eazydiner/__init__.py

from scrape_up.eazydiner.eazydiner import EazyDiner
__all__ = ["EazyDiner"]

# src/scrape_up/eazydiner/eazydiner.py

import requests
from bs4 import BeautifulSoup
import json


class EazyDiner:
    """
    Create an instance of `EazyDiner` class.\n
    ```python
    restaurants = EazyDiner(location="city-name")
    ```
    | Methods             | Details                                                                               |
    | ------------------- | ------------------------------------------------------------------------------------- |
    | `.get_restaurants()` | Returns the restaurants name, location, rating, cuisine and prices in JSON format.  |
    | `.get_breakfast()`   | Returns the restaurants name, location, rating, cuisine and prices in JSON format for Breakfast.|
    | `.get_lunch()`       | Returns the restaurants name, location, rating, cuisine and prices in JSON format for Lunch. |
    | `.get_dinner()`      | Returns the restaurants name, location, rating, cuisine and prices in JSON format for Dinner.|
    | `.dinner_with_discount()`      | Returns list of resturant from the entered location with 50% offer.|
    """

    def __init__(self, location):
        self.location = location

    def get_restaurants(self):
        """
        Class - `EazyDiner`
        Example:
        ```
        del = EazyDiner("Delhi NCR") or del = EazyDiner("delhi-ncr")
        del.getRestaurants()
        ```
        Returns:
        {
            "restaurant": restaurant name
            "location": location of restaurant
            "rating": rating
            "cuisine": cuisines provided
            "price": price for two people
        }
        """
        url = (
            "https://www.eazydiner.com/restaurants?location="
            + self.location.replace(" ", "-").replace(",", "").lower()
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            restaurant_data = {"restaurants": []}

            restaurants = soup.select(".restaurant")
            for r in restaurants:
                name = r.find("h3", class_="res_name").getText().strip()
                location = r.find("h3", class_="res_loc").getText().strip()
                rating = r.find("span", class_="critic").getText().strip()
                cuisine = (
                    r.find("div", class_="res_cuisine").getText().replace(",", ", ")
                )
                price = (
                    r.find("span", class_="cost_for_two")
                    .getText()
                    .encode("ascii", "ignore")
                    .decode()
                    .strip()
                )
                restaurant_data["restaurants"].append(
                    {
                        "restaurant": name,
                        "location": location,
                        "rating": rating,
                        "cuisine": cuisine,
                        "price": "Rs. " + price + " for two",
                    }
                )
            res_json = json.dumps(restaurant_data)
            return res_json
        except:
            error_message = {
                "message": "There are no restaurants in the given location."
            }
            ejson = json.dumps(error_message)
            return ejson

    def get_breakfast(self):
        """
        Class - `EazyDiner`
        Example:
        ```
        del = EazyDiner("Delhi NCR") or del = EazyDiner("delhi-ncr")
        del.getBreakfast()
        ```
        Returns:
        {
            "restaurant": restaurant name
            "location": location of restaurant
            "rating": rating
            "cuisine": cuisines provided
            "price": price for two people
        }
        """
        url = (
            "https://www.eazydiner.com/restaurants?location="
            + self.location.replace(" ", "-").replace(",", "").lower()
            + "&meal_period=breakfast"
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            restaurant_data = {"restaurants": []}

            restaurants = soup.select(".restaurant")
            for r in restaurants:
                name = r.find("h3", class_="res_name").getText().strip()
                location = r.find("h3", class_="res_loc").getText().strip()
                rating = r.find("span", class_="critic").getText().strip()
                cuisine = (
                    r.find("div", class_="res_cuisine").getText().replace(",", ", ")
                )
                price = (
                    r.find("span", class_="cost_for_two")
                    .getText()
                    .encode("ascii", "ignore")
                    .decode()
                    .strip()
                )
                restaurant_data["restaurants"].append(
                    {
                        "restaurant": name,
                        "location": location,
                        "rating": rating,
                        "cuisine": cuisine,
                        "price": "Rs. " + price + " for two",
                    }
                )
            res_json = json.dumps(restaurant_data)
            return res_json
        except:
            error_message = {
                "message": "There are no restaurants in the given location."
            }
            ejson = json.dumps(error_message)
            return ejson

    def get_lunch(self):
        """
        Class - `EazyDiner`
        Example:
        ```
        del = EazyDiner("Delhi NCR") or del = EazyDiner("delhi-ncr")
        del.getLunch()
        ```
        Returns:
        {
            "restaurant": restaurant name
            "location": location of restaurant
            "rating": rating
            "cuisine": cuisines provided
            "price": price for two people
        }
        """
        url = (
            "https://www.eazydiner.com/restaurants?location="
            + self.location.replace(" ", "-").replace(",", "").lower()
            + "&meal_period=lunch"
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            restaurant_data = {"restaurants": []}

            restaurants = soup.select(".restaurant")
            for r in restaurants:
                name = r.find("h3", class_="res_name").getText().strip()
                location = r.find("h3", class_="res_loc").getText().strip()
                rating = r.find("span", class_="critic").getText().strip()
                cuisine = (
                    r.find("div", class_="res_cuisine").getText().replace(",", ", ")
                )
                price = (
                    r.find("span", class_="cost_for_two")
                    .getText()
                    .encode("ascii", "ignore")
                    .decode()
                    .strip()
                )
                restaurant_data["restaurants"].append(
                    {
                        "restaurant": name,
                        "location": location,
                        "rating": rating,
                        "cuisine": cuisine,
                        "price": "Rs. " + price + " for two",
                    }
                )
            res_json = json.dumps(restaurant_data)
            return res_json
        except:
            error_message = {
                "message": "There are no restaurants in the given location."
            }
            ejson = json.dumps(error_message)
            return ejson

    def get_dinner(self):
        """
        Class - `EazyDiner`
        Example:
        ```
        del = EazyDiner("Delhi NCR") or del = EazyDiner("delhi-ncr")
        del.getDinner()
        ```
        Returns:
        {
            "restaurant": restaurant name
            "location": location of restaurant
            "rating": rating
            "cuisine": cuisines provided
            "price": price for two people
        }
        """
        url = (
            "https://www.eazydiner.com/restaurants?location="
            + self.location.replace(" ", "-").replace(",", "").lower()
            + "&meal_period=dinner"
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            restaurant_data = {"restaurants": []}

            restaurants = soup.select(".restaurant")
            for r in restaurants:
                name = r.find("h3", class_="res_name").getText().strip()
                location = r.find("h3", class_="res_loc").getText().strip()
                rating = r.find("span", class_="critic").getText().strip()
                cuisine = (
                    r.find("div", class_="res_cuisine").getText().replace(",", ", ")
                )
                price = (
                    r.find("span", class_="cost_for_two")
                    .getText()
                    .encode("ascii", "ignore")
                    .decode()
                    .strip()
                )
                restaurant_data["restaurants"].append(
                    {
                        "restaurant": name,
                        "location": location,
                        "rating": rating,
                        "cuisine": cuisine,
                        "price": "Rs. " + price + " for two",
                    }
                )
            res_json = json.dumps(restaurant_data)
            return res_json
        except:
            error_message = {
                "message": "There are no restaurants in the given location."
            }
            ejson = json.dumps(error_message)
            return ejson

    def dinner_with_discount(self):
        """
        Returns list of resturant from the entered location with 50% offer.\n
        Class - `EazyDiner`
        Example:
        ```
        deldiner = EazyDiner("Delhi NCR")
        deldiner.dinner_with_discount()
        ```
        Returns:
        {
            "restaurant": restaurant name
            "location": location of restaurant
            "rating": rating
            "cuisine": cuisines provided
            "price": price for two people
        }
        """
        url = (
            "https://www.eazydiner.com/restaurants?location="
            + self.location.replace(" ", "-").replace(",", "").lower()
            + "&meal_period=dinner&buckets%5B%5D=fifty-percent-discounts"
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            restaurant_data = {"restaurants": []}

            restaurants = soup.select(".restaurant")
            for r in restaurants:
                name = r.find("h3", class_="res_name").getText().strip()
                location = r.find("h3", class_="res_loc").getText().strip()
                rating = r.find("span", class_="critic").getText().strip()
                cuisine = (
                    r.find("div", class_="res_cuisine").getText().replace(",", ", ")
                )
                price = (
                    r.find("span", class_="cost_for_two")
                    .getText()
                    .encode("ascii", "ignore")
                    .decode()
                    .strip()
                )
                restaurant_data["restaurants"].append(
                    {
                        "restaurant": name,
                        "location": location,
                        "rating": rating,
                        "cuisine": cuisine,
                        "price": "Rs. " + price + " for two",
                    }
                )
            return restaurant_data["restaurants"]
        except:
            return None


# src/scrape_up/finance/__init__.py

from scrape_up.finance.stock_price import StockPrice

__all__ = ["StockPrice"]


# src/scrape_up/finance/bse.py

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import json


# All the prices returned from this class are in INR
class BSE:
    """
    Handles all data request for bse stock.
    """

    autocomplete_url = "https://api.bseindia.com/Msource/90D/getQouteSearch.aspx?Type=EQ&text={}&flag=gq"
    latest_price_url = "https://api.bseindia.com/BseIndiaAPI/api/getScripHeaderData/w?Debtflag=&scripcode={code}&seriesid="
    historical_data_url = "https://api.bseindia.com/BseIndiaAPI/api/StockReachGraph/w?scripcode={code}&flag=1&fromdate={from_date}&todate={to_date}&seriesid="  # date format yyyymmdd
    chart_data_url = "https://api.bseindia.com/BseIndiaAPI/api/StockReachGraph/w?scripcode={code}&flag=0&fromdate=&todate=&seriesid="
    currency_type = "INR"

    headers = {
        "Connection": "keep-alive",
        "Cache-Control": "max-age=0",
        "DNT": "1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36",
        "Sec-Fetch-User": "?1",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-Mode": "navigate",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9,hi;q=0.8",
    }

    # Creating request.Session object so cookies persist among requests
    fetcher = requests.Session()
    fetcher.headers.update(headers)

    def __init__(self, stock_name):
        self.stock_name = stock_name
        self.stock_code = self.get_data()["code"]
        self.latest_price_url = self.latest_price_url.format(code=self.stock_code)
        self.chart_data_url = self.chart_data_url.format(code=self.stock_code)

    # gets closest matching name and symbol of stock based given stock_name
    def get_data(self):
        try:
            data_html = self.fetcher.get(self.autocomplete_url.format(self.stock_name))
            soup = BeautifulSoup(data_html.text, "html.parser")
            li_element = soup.find("li", class_="quotemenu")
            stock_code = li_element.find("span").text.split()[-1]
            stock_name = li_element.find("strong").text
            if li_element.find("strong").next_sibling.strip:
                stock_name += str(li_element.find("strong").next_sibling.strip())

        except IndexError:
            raise Exception("Invalid Stock Name.")
        except TypeError:
            raise Exception("Invalid Stock Name")
        except requests.exceptions.ConnectionError:
            raise Exception("Connection Error, Please try again.")
        else:
            self.stock_name = stock_name
            return {"code": stock_code, "symbol_info": stock_name}

    def get_latest_price(self):
        """
        Gets Latest stock price info of given bse stock.
        """
        price_info = self.fetcher.get(self.latest_price_url).json()["CurrRate"]
        return {
            "latestPrice": float(price_info["LTP"]),
            "rateChange": float(price_info["Chg"]),
            "pChange": float(price_info["PcChg"]),
        }

    # Please give dates in DD-MM-YYYY format
    def get_historical_data(self, from_date, to_date):
        """
        Gets historical stock price (vwap) in range from_date to to_date
        """

        from_date = datetime.strptime(from_date, "%d-%m-%Y").strftime("%Y%m%d")
        to_date = datetime.strptime(to_date, "%d-%m-%Y").strftime("%Y%m%d")

        historical_price_data_raw = self.fetcher.get(
            self.historical_data_url.format(
                code=self.stock_code, from_date=from_date, to_date=to_date
            )
        ).json()["Data"]

        historical_price_data_raw = json.loads(historical_price_data_raw)
        historical_price_data = {}

        for item in historical_price_data_raw:
            dttm = datetime.strptime(item["dttm"], "%a %b %d %Y %H:%M:%S").strftime(
                "%d-%m-%Y"
            )
            vale1 = float(item["vale1"])
            historical_price_data[dttm] = vale1

        return historical_price_data


if __name__ == "__main__":
    infosys = BSE(stock_name="infosys ltd")
    print(infosys.get_latest_price())
    print(infosys.get_historical_data("05-06-2023", "10-06-2023"))


# src/scrape_up/finance/nasdaq.py

import requests
from datetime import datetime


# All prices returned form this class are in USD
class NASDAQ:
    """
    Handles all requests for nasdaq stock
    """

    autocomplete_url = "https://api.nasdaq.com/api/search/result?query={}&order=relevence&offset=0&lang=en"
    latest_price_url = (
        "https://api.nasdaq.com/api/quote/{symbol}/info?assetclass=stocks"
    )
    chart_data_url = "https://api.nasdaq.com/api/quote/{symbol}/chart?assetclass=stocks"
    historical_data_url = "https://api.nasdaq.com/api/quote/{symbol}/historical?assetclass=stocks&fromdate={from_date}&limit=0&todate={to_date}"  # date in yyyy-mm-dd format
    currency_type = "USD"

    headers = {
        "Connection": "keep-alive",
        "Cache-Control": "max-age=0",
        "DNT": "1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36",
        "Sec-Fetch-User": "?1",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-Mode": "navigate",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9,hi;q=0.8",
    }

    # Creating request.Session object so cookies persist among requests
    fetcher = requests.Session()
    fetcher.headers.update(headers)

    def __init__(self, stock_name):
        self.stock_name = stock_name
        self.stock_symbol = self.get_data()["symbol"]
        self.latest_price_url = self.latest_price_url.format(symbol=self.stock_symbol)
        self.chart_data_url = self.chart_data_url.format(symbol=self.stock_symbol)

    def get_data(self):
        try:
            data = self.fetcher.get(
                self.autocomplete_url.format(self.stock_name)
            ).json()["data"]["results"][0]
        except IndexError:
            raise Exception("Invalid Stock Name.")
        except TypeError:
            raise Exception("Invalid Stock Name")
        except requests.exceptions.ConnectionError:
            raise Exception("Connection Error, Please try again.")
        else:
            self.stock_name = data["name"][0]
            return {"symbol": data["symbol"][0], "symbol_info": data["name"][0]}

    def get_latest_price(self):
        """
        Gets Latest stock price info of given nse stock.
        Class - `finance.NASDAQ()`\n
        Example -\n
        ```python
        google = NASDAQ("alphabet")
        print(google.get_latest_price())
        ```
        Return\n
        ```python
        return
        {
            "data": data,
            "message": f"Latest price of the stock has been fetched",
        }
        ```
        """
        try:
            price_info = self.fetcher.get(self.latest_price_url).json()["data"][
                "primaryData"
            ]
            data = {
                "latestPrice": float(price_info["lastSalePrice"][1:]),
                "rateChange": float(price_info["netChange"]),
                "pChange": float(price_info["percentageChange"][:-1]),
            }
            return {
                "data": data,
                "message": f"Latest price of the stock has been fetched",
            }
        except:
            return {
                "data": None,
                "message": f"Not able to fetch latest price of the stock",
            }

    # Please give dates in DD-MM-YYYY format
    def get_historical_data(self, from_date, to_date):
        """
        Gets Latest stock price info of given nse stock.
        Class - `finance.NASDAQ()`\n
        Example -\n
        ```python
        google = NASDAQ("alphabet")
        print(google.get_historical_data("05-06-2023", "09-06-2023"))
        ```
        Return\n
        ```python
        return
        {
            "data": historical_price_data,
            "message": f"Sucessfylly scrapped the historic data of the stock.",
        }
        ```
        """
        try:
            from_date = datetime.strptime(from_date, "%d-%m-%Y").strftime("%Y-%m-%d")
            to_date = datetime.strptime(to_date, "%d-%m-%Y").strftime("%Y-%m-%d")
            historical_price_data_raw = self.fetcher.get(
                self.historical_data_url.format(
                    symbol=self.stock_symbol, from_date=from_date, to_date=to_date
                )
            ).json()["data"]["tradesTable"]["rows"]
            historical_price_data = {}
            for i in historical_price_data_raw:
                date_formatted = datetime.strptime(i["date"], "%m/%d/%Y").strftime(
                    "%d-%m-%Y"
                )
                historical_price_data[date_formatted] = float(i["open"][1:])
            return {
                "data": historical_price_data,
                "message": f"Sucessfylly scrapped the historic data of the stock.",
            }
        except:
            return {
                "data": None,
                "message": f"Not able to fetch historical data of the stock.",
            }


if __name__ == "__main__":
    google = NASDAQ("alphabet")
    print(google.get_latest_price())
    print(google.get_historical_data("05-06-2023", "09-06-2023"))


# src/scrape_up/finance/nse.py

import requests


# All the prices returned from this class are in INR
class NSE:
    """
    Handles all data request for nse stock.
    """

    autocomplete_url = "https://www.nseindia.com/api/search/autocomplete?q={}"
    latest_price_url = "https://www.nseindia.com/api/quote-equity?symbol={symbol}"
    chart_data_url = "https://www.nseindia.com/api/chart-databyindex?index={symbol}"
    historical_data_url = 'https://www.nseindia.com/api/historical/cm/equity?symbol={symbol}&series=["EQ"]&from={from_date}&to={to_date}'
    currency_type = 'INR'
    
    headers = {
        "Connection": "keep-alive",
        "Cache-Control": "max-age=0",
        "DNT": "1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36",
        "Sec-Fetch-User": "?1",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-Mode": "navigate",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "en-US,en;q=0.9,hi;q=0.8",
    }

    # Creating request.Session object so cookies persist among requests
    fetcher = requests.Session()
    fetcher.headers.update(headers)

    def __init__(self, stock_name):
        self.stock_name = stock_name
        self.stock_symbol = self.get_data()["symbol"]
        self.latest_price_url = self.latest_price_url.format(symbol=self.stock_symbol)
        self.chart_data_url = self.chart_data_url.format(symbol=self.stock_symbol)

    # gets closest matching name and symbol of stock based given stock_name
    def get_data(self):
        try:
            """
            The line below is necessary as nseindia.com does no give access to its
            api unless you have cookies of visiting main page.
            Hence visiting mainpage would imitate behaviour of a human user and let us
            access all the nseindia APIs.
            """
            main_page = self.fetcher.get("https://www.nseindia.com")
            data = self.fetcher.get(
                self.autocomplete_url.format(self.stock_name)
            ).json()["symbols"][0]
        except IndexError:
            raise Exception("Invalid Stock Name.")
        except TypeError:
            raise Exception("Invalid Stock Name")
        except requests.exceptions.ConnectionError:
            raise Exception("Connection Error, Please try again.")
        else:
            self.stock_name = data["symbol_info"]
            return {"symbol": data["symbol"], "symbol_info": data["symbol_info"]}

    # please use this return format for other classes (like bse, nasdaq) too if they are created in future
    def get_latest_price(self):
        """
        Gets Latest stock price info of given nse stock.
        """
        price_info = self.fetcher.get(self.latest_price_url).json()["priceInfo"]
        return {
            "latestPrice": price_info["lastPrice"],
            "rateChange": price_info["change"],
            "pChange": price_info["pChange"],
            "open": price_info["open"],
            "close": price_info["close"],
            "vwap": price_info["vwap"],
        }

    # Please give dates in DD-MM-YYYY format
    def get_historical_data(self, from_date, to_date):
        """
        Gets historical stock price (vwap) in range from_date to to_date
        """
        historical_price_data_raw = self.fetcher.get(
            self.historical_data_url.format(
                symbol=self.stock_symbol, from_date=from_date, to_date=to_date
            )
        ).json()["data"]
        historical_price_data = {}
        for i in historical_price_data_raw:
            historical_price_data[i["mTIMESTAMP"]] = i["VWAP"]
        return historical_price_data




# src/scrape_up/finance/stock_price.py

try:
    from nse import NSE
    from nasdaq import NASDAQ
    from bse import BSE
except ModuleNotFoundError:
    from scrape_up.finance.nse import NSE
    from scrape_up.finance.nasdaq import NASDAQ
    from scrape_up.finance.bse import BSE


class StockPrice:
    """
    This is entry point for users to scrape data.
    Usage:
        Create an instance of Stock price class for Infosys
        >>> infosys_stock = StockPrice('infosys','nse')
        >>> infosys_stock = StockPrice('infosys')

        # the stock_index option is optional as it is set to 'nse' by default for now

        To get latest price info of Infosys's Stock
        >>> latest_price_info = infosys_stock.get_latest_price()

        To get stock price of Infosys from 02-05-2023 to 31-05-2023
        >>> historical_price_info = infosys_stock.get_historical_data('02-05-2023', '31-05-2023')

    """

    def __init__(self, stock_name, stock_index):
        self.stock_name = stock_name.strip()
        self.stock_index = stock_index.strip().lower()
        stock_class = None
        if self.stock_index == "nse":
            stock_class = NSE
        elif self.stock_index == "nasdaq":
            stock_class = NASDAQ
        elif self.stock_index == "bse":
            stock_class = BSE
        if stock_class:
            print(f"Searching for {self.stock_name} {self.stock_index} stock...")
            try:
                self.stock = stock_class(stock_name)
                self.stock_name = self.stock.stock_name
                print(
                    f"Found stock based on provided name: {self.stock_name}\nStock instance created"
                )
            except:
                self.stock = None
                print(f"No stock found based on provided name: {self.stock_name}")
        else:
            print(
                f"{self.stock_index} stock scraping code under development.\nPlease wait for Update."
            )

    # these two functions could potentially be modified to
    # also return data as pandas format
    def get_latest_price(self):
        """
        Gets Latest stock price info of given stock
        Gets Latest stock price info of given nse stock.
        Class - `finance.StockPrice()`\n
        Example -\n
        ```python
        infosys_stock = StockPrice("infosys", "nse")
        print(infosys_stock.get_latest_price())
        ```
        Return\n
        ```python
        return
        {
            "data": data,
            "message": f"No latest stock price found for {self.stock_name}"
        }
        ```
        """
        try:
            data = self.stock.get_latest_price()
            message = f"Found latest stock price(in {self.stock.currency_type}) for {self.stock_name}"
        except:
            data = None
            message = f"No latest stock price found for {self.stock_name}"
        finally:
            return {
                "data": data,
                "message": message,
            }

    def get_historical_data(self, from_date, to_date):
        """
        Gets historical stock price (vwap) in range from_date to to_date
        Class - `finance.StockPrice()`\n
        Example -\n
        ```python
        infosys_stock = StockPrice("infosys", "nse")
        print(infosys_stock.get_historical_data('02-05-2023', '31-05-2023'))
        ```
        Return\n
        ```python
        return
        {
            "data": data,
            "message": f"No historial stock price found for {self.stock_name}"
        }
        ```
        """
        try:
            data = self.stock.get_historical_data(from_date, to_date)
            message = f"Historical stock price(in {self.stock.currency_type}) found for {self.stock_name} in range {from_date} to {to_date}"
        except:
            data = None
            message = f"No historial stock price found for {self.stock_name}"
        finally:
            return {
                "data": data,
                "message": message,
            }


# src/scrape_up/flipkart/__init__.py

from flipkart_file import Flipkart
from flipkart_clothing import FlipkartTees


__all__ = ["Flipkart", "FlipkartTees"]


# src/scrape_up/flipkart/flipkart_clothing.py

import bs4
from bs4 import BeautifulSoup
import requests


class FlipkartClothing:
    """
    Create an instance of `FlipkartTees` class.
    ```python
    cloth = FlipkartTees()
    ```
    | Methods                    | Details                                                        |
    | -------------------------- | -------------------------------------------------------------- |
    | `.scrape()`                | Returns the list of t-shirts with other relevant info          |
    | `.range()`                 | Returns the list of t-shirts between a particular price range. |
    | `.minrating()`             | Returns the list of t-shirts havinga minimum given rating.     |
    | `.gendermale()`            | Returns the list of t-shirts which are for males.              |
    | `.genderfemale()`          | Returns the list of t-shirts which are there for females.      |
    | `.size()`                  | Returns the list of tshirts havning a particular size.         |
    | `formal_shirts_for_male()` | It returns those t-shirts which are of a particular size       |
    """

    def __init__(self):
        pass

    def tshirts(
        self,
        url="https://www.flipkart.com/search?q=tshirt&as=on&as-show=on&otracker=AS_Query_TrendingAutoSuggest_1_0_na_na_na&otracker1=AS_Query_TrendingAutoSuggest_1_0_na_na_na&as-pos=1&as-type=TRENDING&suggestionId=mobiles&requestId=fbbac901-d12f-4dc5-9a8c-b034cde11af9",
    ):
        """
        It returns the t-shirt data of products from flipkart\n
        Class - `FlipkartClothing()`
        ```python
        clothes = FlipkartClothing()
        clothes.tshirts()
        ```
        """
        obj_keys = ["name", "price", "description", "image", "link"]
        try:
            content = requests.get(url)

            soup = BeautifulSoup(content.content, "html.parser")

            imgs = soup.find_all("img", attrs={"class": "_2r_T1I"})
            names = soup.find_all("div", attrs={"class": "_2WkVRV"})
            prices = soup.find_all("div", attrs={"class": "_30jeq3"})
            desc = soup.find_all("a", attrs={"class": "IRpwTa"})

            res = []
            startlink = "https://www.flipkart.com"

            for i in range(len(imgs)):
                nm = desc[i].attrs["href"]
                name = names[i].text
                img = imgs[i].attrs["src"]
                des = desc[i].text
                price = prices[i].text

                obj_values = [name, price, des, img, startlink + nm]

                res.append(dict(zip(obj_keys, obj_values)))
            return res
        except:
            return None

    def tshirts_by_price_range(self, min, max):
        """
        It returns the t-shirts in a particular price bracket(min-max). The information regarding parameters is as follows:\n
        - min: Bottom range of price\n
        - max: Max range of price\n
        Class - `FlipkartClothing()`
        ```python
        clothes = FlipkartClothing()
        clothes.tshirts_by_price_range(min=500,max=2000)
        ```
        """
        try:
            max = int(max)
            min = int(min)
            if min == max:
                return None
            url = "https://www.flipkart.com/search?q=tshirt&as=on&as-show=on&otracker=AS_Query_TrendingAutoSuggest_1_0_na_na_na&otracker1=AS_Query_TrendingAutoSuggest_1_0_na_na_na&as-pos=1&as-type=TRENDING&suggestionId=mobiles&requestId=fbbac901-d12f-4dc5-9a8c-b034cde11af9&p%5B%5D=facets.price_range.from%3D{}&p%5B%5D=facets.price_range.to%3D{}".format(
                min, max
            )
            k = FlipkartClothing().tshirts(url)
            return k
        except:
            return None

    def tshirts_by_rating(self, rating):
        """
        It returns the t-shirts having the user rating greater than or equal to the minimum passed rating(r)\n
        - range: minimum rating\n
        Class - `FlipkartClothing()`
        ```python
        tees = FlipkartClothing()
        tees.tshirts_by_rating(rating=4.1)
        ```
        """
        r = rating
        try:
            if int(r) < 1 and int(r) >= 5:
                return None
            url = "https://www.flipkart.com/search?q=tshirt&as=on&as-show=on&otracker=AS_Query_TrendingAutoSuggest_1_0_na_na_na&otracker1=AS_Query_TrendingAutoSuggest_1_0_na_na_na&as-pos=1&as-type=TRENDING&suggestionId=mobiles&requestId=fbbac901-d12f-4dc5-9a8c-b034cde11af9&p%5B%5D=facets.price_range.from%3DMin&p%5B%5D=facets.price_range.to%3DMax&p%5B%5D=facets.rating%255B%255D%3D{}%25E2%2598%2585%2B%2526%2Babove".format(
                int(r)
            )
            k = FlipkartClothing().tshirts(url)
            return k
        except:
            return None

    def tshirts_for_male(self):
        """
        It returns all the t-shirts pertaining to males\n
        Class - `FlipkartClothing()`\n
        ```python
        tees = FlipkartClothing()
        tees.tshirts_for_male()
        ```
        """
        try:
            url = "https://www.flipkart.com/clothing-and-accessories/topwear/tshirts/mens-tshirts/pr?sid=clo,ash,ank,edy&q=tshirt&otracker=categorytree"
            k = FlipkartClothing().tshirts(url)
            return k
        except:
            return None

    def tshirts_for_female(self):
        """
        It returns all the t-shirts pertaining to females\n
        Class - `FlipkartClothing()`\n
        ```python
        tees = FlipkartClothing()
        tees.tshirts_for_female()
        ```
        """
        try:
            url = "https://www.flipkart.com/clothing-and-accessories/topwear/tshirts/womens-tshirts/pr?sid=clo,ash,ank,loi&q=tshirt&otracker=categorytree"
            k = FlipkartClothing().tshirts(url)
            return k
        except:
            return None

    def tshirt_by_size(self, size="M"):
        """
        It returns those t-shirts which are of a particular size\n
        Required parameter = `size` - `["XS", "S", "M", "L", "XL", "2XL", "3XL"]`\n
        Class - `FlipkartClothing()`
        ```python
        tees = FlipkartClothing()
        tees.tshirt_by_size()
        ```
        """
        try:
            m = size.upper()
            curavl = ["XS", "S", "M", "L", "XL", "2XL", "3XL"]
            if m not in curavl:
                return None
            url = "https://www.flipkart.com/search?q=tshirt&as=on&as-show=on&otracker=AS_Query_TrendingAutoSuggest_1_0_na_na_na&otracker1=AS_Query_TrendingAutoSuggest_1_0_na_na_na&as-pos=1&as-type=TRENDING&suggestionId=mobiles&requestId=fbbac901-d12f-4dc5-9a8c-b034cde11af9&p%5B%5D=facets.size%255B%255D%3D{}".format(
                m
            )
            k = FlipkartClothing().tshirts(url)
            return k
        except:
            return None

    def formal_shirts_for_male():
        """
        It returns all the t-shirts pertaining to females\n
        Class - `FlipkartClothing()`\n
        ```python
        tees = FlipkartClothing()
        tees.formal_shirts_for_male()
        ```
        Return\n
        ```js
        [
            {
                "name":"FIBERMILL",
                "price":"₹389",
                "description":"Men Regular Fit Solid Spread Collar Formal Shirt",
                "image":"https://rukminim2.flixcart.com/image/612/612/xif0q/shirt/k/g/1/l-fbrml-r-fibermill-original-imagjfk6ytwytyg3.jpeg?q=70",
                "link":"..."
            }
            ...
        ]
        ```
        """
        try:
            url = "https://www.flipkart.com/search?q=formal+shirts+for+men&sid=clo%2Cash%2Caxc%2Cmmk%2Cbk1&as=on&as-show=on&otracker=AS_QueryStore_OrganicAutoSuggest_1_11_na_na_ps&otracker1=AS_QueryStore_OrganicAutoSuggest_1_11_na_na_ps&as-pos=1&as-type=RECENT&suggestionId=formal+shirts+for+men%7CFormal+Shirts&requestId=2d7ee0f8-6dd3-4857-9a84-6e8b378e7767&as-backfill=on&page=1"
            k = FlipkartClothing().tshirts(url)
            return k
        except:
            return None


# src/scrape_up/flipkart/flipkart_file.py

import bs4
from bs4 import BeautifulSoup as bs
import requests


class Flipkart:
    """
    Create an instance of `Flipkart` class.
    ```python
    item = Flipkart()
    ```
    | Methods               | Details                                                            |
    | --------------------- | ------------------------------------------------------------------ |
    | `.TVs()`              | Returns the list of TV sets on flipkart                            |
    | `.bestseller_books()` | Returns the list of bestselling books data listed on Flipkart.     |
    | `.mobiles()`          | Returns the list of mobile phones under 50K along with their data. |
    | `.sport_shoes()`      | Returns the list of trendong sport shoes data.                     |
    | `.laptops()`          | Returns the list of laptop from flipkart.                          |
    | `.camera()`           | Returns the list of camera from flipkart.                           |
    | `.computer()`         | Returns the list of computer from flipkart.                       |
    | `.tablets()`          | Returns the list of tablets from flipkart.                         |
    """

    def __init__(self):
        pass

    def tvs(self):
        """
        Class - `Flipkart`\n
        Example -\n
        ```python
        item = Flipkart()
        item.tvs()
        """
        try:
            link = "https://www.flipkart.com/search?q=tv&as=on&as-show=on&otracker=AS_Query_TrendingAutoSuggest_8_0_na_na_na&otracker1=AS_Query_TrendingAutoSuggest_8_0_na_na_na&as-pos=8&as-type=TRENDING&suggestionId=tv&requestId=9c9fa553-b7e5-454b-a65b-bbb7a9c74a29"
            page = requests.get(link)
            soup = bs(page.content, "html.parser")

            all_items = []

            for data in soup.findAll("div", class_="_3pLy-c row"):
                names = data.find("div", attrs={"class": "_4rR01T"})
                price = data.find("div", attrs={"class": "_30jeq3 _1_WHN1"})
                rating = data.find("div", attrs={"class": "_3LWZlK"})
                specification = data.find("div", attrs={"class": "fMghEO"})

                specs = []
                for spec in specification.find_all("li", attrs={"class": "rgWa7D"}):
                    specs.append(spec.text)

                item_details = {
                    "Item_Name": names.text,
                    "Price": price.text,
                    "Rating": rating.text,
                    "Specifications": ", ".join(specs),
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None

    def bestseller_books(self):
        try:
            """
            Class - `Flipkart`\n
            Example -\n
            ```python
            item = Flipkart()
            item.bestseller_books()
            """

            link = "https://www.flipkart.com/search?q=bestsellers&otracker=search&otracker1=search&marketplace=FLIPKART&as-show=on&as=off"
            page = requests.get(link)
            soup = bs(page.content, "html.parser")

            all_items = []

            for data in soup.findAll("div", class_="_4ddWXP"):
                names = data.find("a", attrs={"class": "s1Q9rs"})
                price = data.find("div", attrs={"class": "_30jeq3"})
                rating = data.find("div", attrs={"class": "_3LWZlK"})
                specification = data.find("div", attrs={"class": "_3Djpdu"})

                item_details = {
                    "Item_Name": names.text,
                    "Price": price.text,
                    "Rating": rating.text if rating else None,
                    "Specifications": specification.text if specification else None,
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None

    def mobiles(self):
        """
        Get the list of mobiles under 50K\n
        Class - `Flipkart`\n
        Example -\n
        ```python
        item = Flipkart()
        item.mobiles()
        """
        try:
            link = "https://www.flipkart.com/search?q=MOBILE+PHONE+UNDER+50000&otracker=search&otracker1=search&marketplace=FLIPKART&as-show=on&as=off&page=3"
            page = requests.get(link)
            soup = bs(page.content, "html.parser")

            all_items = []

            for data in soup.findAll("div", class_="_1AtVbE"):
                names = data.find("a", class_="_1fQZEK")
                price = data.find("div", class_="_30jeq3 _1_WHN1")
                description = data.find("ul", class_="_1xgFaf")
                review = data.find("div", class_="_3LWZlK")

                item_details = {
                    "Item_Name": names.text if names else None,
                    "Price": price.text if price else None,
                    "Description": description.text if description else None,
                    "Review": review.text if review else None,
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None

    def sport_shoes(self):
        """
        Class - `Flipkart`\n
        Example -\n
        ```python
        item = Flipkart()
        item.BestsellersBooks()
        item.sport_shoes()
        """
        try:
            all_items = []
            url = "https://www.flipkart.com/mens-footwear/sports-shoes/pr?sid=osp,cil,1cu&otracker=nmenu_sub_Men_0_Sports%20Shoes"
            response = requests.get(url)
            html_content = response.content
            soup = bs(html_content, "html.parser")

            all_items = []

            div_elements = soup.find_all("div", class_="_1xHGtK _373qXS")

            for div in div_elements:
                name_element = div.find("a", class_="IRpwTa")
                name = name_element.text.strip() if name_element else ""

                image_element = div.find("img", class_="_2r_T1I")
                image_url = image_element["src"] if image_element else ""

                details_element = div.find("a", class_="_3bPFwb")
                details = details_element.text.strip() if details_element else ""

                item_details = {
                    "Name": name,
                    "Image URL": image_url,
                    "Details": details,
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            print("Error:", str(e))
            return None

    def laptops(self):
        """
        Get the list of mobiles under 50K\n
        Class - `Flipkart`\n
        Example -\n
        ```python
        item = Flipkart()
        item.laptops()
        """
        try:
            link = "https://www.flipkart.com/search?q=laptops&otracker=search&otracker1=search&marketplace=FLIPKART&as-show=on&as=1"
            page = requests.get(link)
            soup = bs(page.content, "html.parser")

            all_items = []

            for data in soup.findAll("div", class_="_1AtVbE col-12-12"):
                names = data.find("div", class_="_4rR01T")
                price = data.find("div", class_="_30jeq3 _1_WHN1")
                description = data.find("ul", class_="_1xgFaf")
                review = data.find("div", class_="_3LWZlK")

                item_details = {
                    "Item_Name": names.text if names else None,
                    "Price": price.text if price else None,
                    "Description": description.text if description else None,
                    "Review": review.text if review else None,
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None

    def headphones(self):
        """
        Class - `Flipkart`\n
        ```python
        item = Flipkart()
        item.headphones()
        ```
        """
        try:
            link = "https://www.flipkart.com/search?q=headphones+bluetooth&sid=0pm%2Cfcn%2Cgc3%2Cka8&as=on&as-show=on&otracker=AS_QueryStore_OrganicAutoSuggest_1_17_na_na_ps&otracker1=AS_QueryStore_OrganicAutoSuggest_1_17_na_na_ps&as-pos=1&as-type=RECENT&suggestionId=headphones+bluetooth%7CWireless+Headphones&requestId=beefc9a9-55a8-4b64-b510-b9bc44ae6680&as-backfill=on&page=1"
            page = requests.get(link)
            soup = bs(page.content, "html.parser")

            all_items = []

            for data in soup.findAll("div", class_="_1AtVbE col-12-12"):
                names = data.find("a", class_="s1Q9rs")
                price = data.find("div", class_="_30jeq3")
                description = data.find("div", class_="_3xFhiH")
                review = data.find("div", class_="_3LWZlK")

                item_details = {
                    "Item_Name": names.text if names else None,
                    "Price": price.text if price else None,
                    "Description": description.text if description else None,
                    "Review": review.text if review else None,
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None

    def camera(self):
        """
        Class - `Flipkart`\n
        ```python
        item = Flipkart()
        item.camera()
        ```
        """
        try:
            link = "https://www.flipkart.com/search?q=camera&otracker=search&otracker1=search&marketplace=FLIPKART&as-show=on&as=off&page=1"
            page = requests.get(link)
            soup = bs(page.content, "html.parser")

            all_items = []

            for data in soup.findAll("div", class_="_1AtVbE col-12-12"):
                names = data.find("div", class_="_4rR01T")
                price = data.find("div", class_="_30jeq3")
                description = data.find("ul", class_="_1xgFaf")
                review = data.find("div", class_="_3LWZlK")

                item_details = {
                    "Item_Name": names.text if names else None,
                    "Price": price.text if price else None,
                    "Description": description.text if description else None,
                    "Review": review.text if review else None,
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None

    def computer(self):
        """
        Class - `Flipkart`\n
        ```python
        item = Flipkart()
        item.computer()
        ```
        """
        try:
            link = "https://www.flipkart.com/search?q=computer&sid=6bo%2Cnl4%2Cigk&as=on&as-show=on&otracker=AS_QueryStore_OrganicAutoSuggest_2_8_na_na_na&otracker1=AS_QueryStore_OrganicAutoSuggest_2_8_na_na_na&as-pos=2&as-type=RECENT&suggestionId=computer%7CAll+In+One+PCs&requestId=a640e175-ccac-4757-81de-8580730de6ef&as-backfill=on&page=1"
            page = requests.get(link)
            soup = bs(page.content, "html.parser")

            all_items = []

            for data in soup.findAll("div", class_="_1AtVbE col-12-12"):
                names = data.find("a", class_="s1Q9rs")
                price = data.find("div", class_="_30jeq3")
                delivery = data.find("div", class_="_2Tpdn3")
                review = data.find("div", class_="_3LWZlK")

                item_details = {
                    "Item_Name": names.text if names else None,
                    "Price": price.text if price else None,
                    "delivery": delivery.text if delivery else None,
                    "Review": review.text if review else None,
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None

    def tablets():
        """
        Get the list of mobiles under 50K\n
        Class - `Flipkart`\n
        Example -\n
        ```python
        item = Flipkart()
        item.tablets()
        """
        try:
            link = "https://www.flipkart.com/search?q=tablet&sid=tyy%2Chry&as=on&as-show=on&otracker=AS_QueryStore_OrganicAutoSuggest_1_5_na_na_na&otracker1=AS_QueryStore_OrganicAutoSuggest_1_5_na_na_na&as-pos=1&as-type=RECENT&suggestionId=tablet%7CTablets&requestId=07b434d5-21a5-48e0-9170-8d9f4a90928f&as-searchtext=tablet&page=1"
            page = requests.get(link)
            soup = bs(page.content, "html.parser")

            all_items = []

            for data in soup.findAll("div", class_="_1AtVbE col-12-12"):
                names = data.find("div", class_="_4rR01T")
                price = data.find("div", class_="_30jeq3 _1_WHN1")
                description = data.find("ul", class_="_1xgFaf")
                review = data.find("div", class_="gUuXy-")
                offer_price = data.find("div", class_="_2ZdXDB")

                item_details = {
                    "Item_Name": names.text if names else None,
                    "Price": price.text if price else None,
                    "Description": description.text if description else None,
                    "Review": review.text if review else None,
                    "Offer_Price": offer_price.text if review else None,
                }

                all_items.append(item_details)

            return all_items

        except Exception as e:
            return None


# src/scrape_up/flyrobu/__init__.py

from flyrobu import Flyrobu

__all__ = ["Flyrobu"]


# src/scrape_up/flyrobu/flyrobu.py

import requests
from bs4 import BeautifulSoup
import re


class Flyrobu:
    """
    Create an instance of `Flyrobu` class.

    ```python
    flyrobu = Flyrobu()
    print(flyrobu.search("arduino"))
    ```

    | Methods             | Details                                                                                                         |
    | ------------------- | --------------------------------------------------------------------------------------------------------------- |
    | `.search(keyword)`  | Returns the json data of all the details related to search with informing about the total amount of items found |
    | `.get_product_details(product_name)` | Returns the json data of the product details based on the given `product_name` |
    """

    def __init__(self):
        self.base_url = "https://www.flyrobo.in"
        self.outputs = []

    def __get_html_content(self, url, headers=None):
        if headers is None:
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
            }

        session = requests.Session()
        session.headers.update(headers)
        response = session.get(url)
        response.raise_for_status()
        return response.content

    def __get_soup(self, url):
        request_content = self.__get_html_content(url)
        soup = BeautifulSoup(request_content, "html.parser")
        return soup

    def __is_matching_title(self, title, search_item):
        search_words = search_item.lower().split()
        return all(word in title.lower() for word in search_words)

    def __scrape_results(self, soup, search_item):
        results_container = soup.find("div", class_="product-grid")
        results = results_container.find_all("div", class_="product-layout")

        for result in results:
            try:
                title_element = result.select_one(".name a")
                if title_element:
                    title = title_element.text.strip()
                    if not self.__is_matching_title(title, search_item):
                        continue

                    price_new_element = result.select_one(".price-new")
                    price_element = price_new_element or result.select_one(
                        ".price-normal"
                    )
                    price = (
                        price_element.text.strip().replace("₹", "Rs. ")
                        if price_element
                        else None
                    )

                    brand_name = (
                        result.select_one(".stat-1 a").text.strip()
                        if result.select_one(".stat-1 a")
                        else None
                    )
                    image = (
                        result.select_one(".image img")["src"]
                        if result.select_one(".image img")
                        else None
                    )
                    description = (
                        result.select_one(".description").text.strip()
                        if result.select_one(".description")
                        else None
                    )
                    rating_stars = result.select(".rating-stars .fa-star")
                    rating = len(rating_stars)
                    model_id = (
                        result.select_one(".stat-2 span:nth-of-type(2)").text.strip()
                        if result.select_one(".stat-2 span:nth-of-type(2)")
                        else None
                    )
                    link = title_element["href"] if title_element else None

                    self.outputs.append(
                        {
                            "title": title,
                            "price": price,
                            "brand_name": brand_name,
                            "image": image,
                            "description": description,
                            "rating": rating,
                            "model_id": model_id,
                            "link": link,
                        }
                    )

            except Exception as e:
                print(f"Error scraping result: {str(e)}")

    def search(self, keyword):
        """
        Class - `Flyrobu`\n
        Example -\n
        ```python
        from flyrobu import Flyrobu

        flyrobu = Flyrobu()
        results = flyrobu.search("arduino")
        print(results)
        ```
        Return\n
        ```js
        [
            {
                "title": "Arduino Uno R3 Compatible board + Cable for Arduino Uno",
                "price": "Rs. 777",
                "brand_name": "Arduino",
                "image": "https://www.flyrobo.in/image/cache/catalog/product/arduino-uno-r3-+-cable-for-arduino-uno-250x250.jpg",
                "description": "The Arduino Uno R3 Compatible board is an electronic hardware device used to build and program electronic circuits and projects. The board is based on the ATmega3:
                "rating": 4,
                "model_id": "FRC-01-401",
                "link": "https://www.flyrobo.io/productsn/arduino-uno-r3-compatible-board-plus-cable-for-arduino-uno-1?search-arduino&description=true"
                },
            }
        ]
        ```
        """
        url = f"{self.base_url}/index.php?route=product/search&search={keyword}&description=true&limit=100"

        while True:
            try:
                soup = self.__get_soup(url)
                self.__scrape_results(soup, keyword)

                next_page = soup.find("ul", class_="pagination").find(
                    "a", class_="next"
                )
                if next_page:
                    url = next_page["href"]
                else:
                    break

            except requests.exceptions.RequestException as e:
                return None
            except AttributeError as e:
                return None
            except Exception as e:
                return None

        result = self.outputs
        return result

    def get_product_details(self, product_name):
        """
        Retrieve Detailed Product Description and Specification based on the given `product_name`.\n
        Parameters required: \n
        `product_name: str`: The name of the product to fetch details for.
        Example Usage:
        ```python
        flyrobu = Flyrobu()
        product_name = input("Enter the product name: ")
        product_details = flyrobu.get_product_details(product_name)
        ```
        Output:
        ```json
        {
            "Description": "E-Bike Power Lock Ignition Key Switch is brand new and unused and comes with 2 keys. E-Bike Power Lock Ignition Key Switch has 2 Female Pin, Plastic Make.\nIt is Suitable for MY1016 controllers.\n\n \nFeatures:\n     \n\nEasy installation\nProtect the security\nDesigned with a reliable ignition switch locking mechanism\nDurable, convenient to use",
            "Features": [
                "Easy installation",
                "Protect the security",
                "Designed with a reliable ignition switch locking mechanism",
                "Durable, convenient to use"
            ],
            "Technical Details": {
                "Cable Length": "27 cm",
                "Connector": "2 Pin Female"
            },
            "Current Price": 349,
            "Original Price": 399,
            "Image": "https://www.flyrobo.in/image/cache/catalog/e-bile-power-lock-ignition-key-switch/e-bile-power-lock-ignition-key-switch-2-550x550.jpeg",
            "Package Includes": "1 x E-Bike Power Lock Ignition Key Switch"
        }
        ```
        """
        try:
            product_url = f"{self.base_url}/{product_name}"

            soup = self.__get_soup(product_url)
            product_details = {}

            # Extract the product description and features
            description = (
                soup.find("div", class_="product_extra-242").find("div").text.strip()
            )
            features_list = (
                soup.find("div", class_="product_extra-242").find("ul").find_all("li")
            )
            features = [feature.text.strip() for feature in features_list]

            product_details["Description"] = description
            product_details["Features"] = features

            # Extract the technical specifications
            technical_details_table = soup.find("table", class_="table-bordered")
            technical_details = {}
            if technical_details_table:
                rows = technical_details_table.find_all("tr")
                for row in rows[1:]:
                    attribute_name = row.find(
                        "td", class_="attribute_name"
                    ).b.text.strip()
                    attribute_value = row.find(
                        "td", class_="attribute_value"
                    ).p.text.strip()
                    technical_details[attribute_name] = attribute_value

            product_details["Technical Details"] = technical_details

            # Extract the price details
            price_section = soup.find("div", class_="price-group")
            current_price_elem = price_section.find("div", class_="product-price-new")
            original_price_elem = price_section.find("div", class_="product-price-old")

            # Check if both current_price_elem and original_price_elem are None, then try to find the "product-price" element
            if current_price_elem is None and original_price_elem is None:
                fallback_price_elem = price_section.find("div", class_="product-price")
                if fallback_price_elem:
                    current_price = fallback_price_elem.text.strip()
                    original_price = None
                else:
                    current_price = None
                    original_price = None
            else:
                current_price = (
                    current_price_elem.text.strip() if current_price_elem else None
                )
                original_price = (
                    original_price_elem.text.strip() if original_price_elem else None
                )

            # Remove the Rupee symbol and convert the prices to integers
            if current_price:
                current_price = int(current_price.replace("₹", "").replace(",", ""))
                product_details["Current Price"] = current_price

            if original_price:
                original_price = int(original_price.replace("₹", "").replace(",", ""))
                product_details["Original Price"] = original_price

            # Extract the image URL
            image_div = soup.find("div", class_="swiper-wrapper")
            image_url = image_div.find("img")["src"]
            product_details["Image"] = image_url

            # Extract the "Package Includes" details from the description using regular expression
            package_includes_pattern = r"Package Includes:\s*(.*)"
            match = re.search(package_includes_pattern, description)
            if match:
                package_includes_text = match.group(1).strip()
                product_details["Package Includes"] = package_includes_text

                # Remove the "Package Includes" part from the description
                description = re.sub(package_includes_pattern, "", description).strip()
                product_details["Description"] = description

            return product_details

        except requests.exceptions.RequestException as e:
            return None
        except AttributeError as e:
            return None
        except Exception as e:
            return None


# src/scrape_up/github/__init__.py

from users import Users
from repository import Repository
from issue import Issue
from organization import Organization
from pull_request import PullRequest


__all__ = ["Users", "Repository", "Issue", "Organization", "PullRequest"]


# src/scrape_up/github/issue.py

import requests
from bs4 import BeautifulSoup


class Issue:
    """
    Create an instance of the class `Issue`
    ```python
    repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
    ```

    | Methods           | Details                                                                            |
    | ----------------- | ---------------------------------------------------------------------------------- |
    | `.assignees()`    | Returns the assignees of an issue.                                                 |
    | `.labels()`       | Returns the labels of an issue.                                                    |
    | `.opened_by()`    | Returns the name of the user, who opened the issue.                                |
    | `.title()`        | Returns the title of an issue.                                                     |
    | `.is_milestone()` | Returns the milestone, if the issue is part of one or 'No milestone', if it's not. |
    | `.opened_at()`    | Returns a string containing the time when the issue was opened in ISO format.      |
    """

    def __init__(self, username: str, repository_name: str, issue_number: int):
        self.username = username
        self.repository = repository_name
        self.issue_number = issue_number

    def __scrape_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/issues/{self.issue_number}"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def assignees(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        assignees = repository.assignees()
        ```
        Returns: Assignees | None
        ```
        """
        data = self.__scrape_page()
        try:
            assignees_body = data.find("span", class_="css-truncate js-issue-assignees")
            assignees = []
            for assignee in assignees_body.find_all(
                "a", class_="assignee Link--primary css-truncate-target width-fit"
            ):
                assignees.append(assignee.text.replace("\n", "").strip())
            assignees
        except:
            None

    def labels(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        labels = repository.labels()
        ```
        Returns: Labels | None
        """
        data = self.__scrape_page()
        try:
            labelsDiv = data.find(class_="js-issue-labels d-flex flex-wrap")
            allLabelsHtml = labelsDiv.find_all(
                class_="css-truncate css-truncate-target width-fit"
            )
            allLabels = []
            for label in allLabelsHtml:
                allLabels.append(label.text)
            allLabels
        except:
            return None

    def opened_by(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        opened_by = repository.opened_by()
        ```
        Returns: Author Name | None
        """
        data = self.__scrape_page()
        try:
            author_name = data.find("a", class_="author text-bold Link--secondary").text
            author_name
        except:
            return None

    def title(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        title = repository.title()
        ```
        Returns: Title | None
        """
        data = self.__scrape_page()
        try:
            title_body = data.find("bdi", class_="js-issue-title markdown-title")
            title = title_body.text.strip()
            return title
        except:
            return None

    def opened_at(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        opened_at = repository.opened_at()
        ```
        Returns: Opened at | None
        """
        try:
            data = self.__scrape_page()
            return data.find("relative-time").text
        except:
            return None

    def is_milestone(self):
        """
        Class - `Issues`
        Example:
        ```
        repository = github.Issue(username="nikhil25803", repository_name="scrape-up", issue_number=59)
        is_milestone = repository.is_milestone()
        ```js
        Returns: Milestones | None
        """
        data = self.__scrape_page()
        try:
            milestone = data.find(
                "a", class_="Link--secondary mt-1 d-block text-bold css-truncate"
            ).text.strip()
            return milestone
        except:
            return None


# src/scrape_up/github/organization.py

import requests
from bs4 import BeautifulSoup
import json


class Organization:
    """
    Create an instance of class `Organization`
    ```python
    organization = github.Organization(organization_name="Clueless-Community")
    ```
    | Methods                     | Details                                                         |
    | --------------------------- | --------------------------------------------------------------- |
    | `.top_topics()`             | Returns a list of the most used topics in an organization.      |
    | `.followers()`              | Returns the number of followers of an organization.             |
    | `.top_languages()`          | Returns the top languages used in an organization.              |
    | `.followers()`              | Returns the number of followers of an organization.             |
    | `.avatar()`                 | Returns the avatar URL of an organization.                      |
    | `.repositories()`           | Returns the list of repositories of an organization.            |
    | `.people()`                 | Returns the list of people in an organization.                  |
    | `.peoples() `               | Returns the number of people in an organization.                |
    | `.get_location() `          | Returns the location of an organization.                        |
    | `.repository_details()`     | Returns the list of repositories with their details.            |
    | `.pinned_repository()`      | Returns the list of pinned repositories with their details.     |
    | `.get_organization_links()` | Returns a dictionary of important website links of a community. |
    """

    def __init__(self, organization_name: str):
        self.organization = organization_name

    def __scrape_page(self):
        data = requests.get(f"https://github.com/{self.organization}")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def top_languages(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        top_languages = repository.top_languages()
        ```
        """
        try:
            languages = []
            data = self.__scrape_page()
            lang_raw = data.find_all(
                "a", class_="no-wrap color-fg-muted d-inline-block Link--muted mt-2"
            )
            for lang in lang_raw:
                languages.append(lang.get_text().strip())
            return languages
        except:
            return None

    def top_topics(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        top_topics = repository.top_topics()
        ```
        """
        page = self.__scrape_page()
        try:
            all_topics = page.find_all(class_="topic-tag topic-tag-link")
            topics = []
            for topic in all_topics:
                topics.append(topic.text.strip())
            return topics
        except:
            return None

    def followers(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        followers = repository.followers()
        ```
        """
        page = self.__scrape_page()
        try:
            followers_body = page.find(
                "a", class_="Link--secondary no-underline no-wrap"
            )
            followers = followers_body.span.text.strip()
            return followers
        except:
            return None

    def avatar(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        avatar = repository.avatar()
        ```
        """
        page = self.__scrape_page()
        try:
            avatar = page.find("a", attrs={"itemprop": "url"})
            avatar_url = avatar.text.strip()
            return avatar_url
        except:
            return None

    def __scrape_repositories_page(self):
        """
        scrapes the head page of repositories of an organization
        """
        organization = self.organization
        data = requests.get(f"https://github.com/orgs/{organization}/repositories")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_repositories(self, page):
        """
        scrapes the repositories page of an organization
        """
        data = requests.get(page)
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def repositories(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        repositories = repository.repositories()
        ```
        """
        organization = self.organization
        data = self.__scrape_repositories_page()
        try:
            pages_body = data.find("div", class_="paginate-container")
            current_page = pages_body.find("em", class_="current")
            total_pages = 1
            if current_page != None:
                total_pages = (int)(current_page["data-total-pages"])

            pages = []
            if total_pages == 1:
                pages.append(f"https://github.com/orgs/{organization}/repositories")
            else:
                for i in range(1, total_pages + 1):
                    pages.append(
                        f"https://github.com/orgs/{organization}/repositories?page={i}"
                    )

            repositories = []
            for page in pages:
                page_data = self.__scrape_repositories(page)
                repositories_body = page_data.find("div", id="org-repositories")
                for repo in repositories_body.find_all(
                    "a", attrs={"itemprop": "name codeRepository"}
                ):
                    repositories.append(repo.text.strip())

            return repositories
        except:
            return None

    def __scrape_people_page(self):
        """
        scrapes the head page of people of an organization

        """
        organization = self.organization
        data = requests.get(f"https://github.com/orgs/{organization}/people")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_people(self, page):
        """
        scrapes the people page of an organization
        """
        organization = self.organization
        data = requests.get(page)
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def people(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        people = repository.people()
        ```
        Returns:
        {
           "data": people,
           "message": f"Found {len(people)} people for {organization}",
        }
        """
        organization = self.organization
        data = self.__scrape_people_page()
        try:
            pages_body = data.find("div", class_="paginate-container")
            current_page = pages_body.find("em", class_="current")
            total_pages = 1
            if current_page != None:
                total_pages = (int)(current_page["data-total-pages"])

            pages = []
            if total_pages == 1:
                pages.append(f"https://github.com/orgs/{organization}/people")
            else:
                for i in range(1, total_pages + 1):
                    pages.append(
                        f"https://github.com/orgs/{organization}/people?page={i}"
                    )

            peoples = []
            for page in pages:
                page_data = self.__scrape_people(page)
                people_body = page_data.find("div", id="org-members-table")
                for person in people_body.find_all("li"):
                    person_username = person.find("a", class_="d-inline-block")
                    peoples.append(person_username["href"][1:])

            return peoples
        except:
            return None

    def peoples(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        peoples = repository.peoples()
        ```
        """
        data = self.__scrape_people_page()
        try:
            body = data.find("div", class_="paginate-container")
            current_page = body.find("em", class_="current")
            page_count = 1
            if current_page != None:
                page_count = int((current_page["data-total-pages"]))

            pages = []

            if page_count == 1:
                pages.append(f"https://github.com/orgs/{self.organization}/people")
            else:
                for i in range(1, page_count + 1):
                    pages.append(
                        f"https://github.com/orgs/{self.organization}/people?page={i}"
                    )

            people_count = 0
            for page in pages:
                page_data = self.__scrape_people(page)
                people_body = page_data.find("div", id="org-members-table")
                people_count = len(people_body.find_all("li"))

            return people_count
        except:
            return None

    def get_location(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        get_location = repository.get_location()
        ```
        """
        page = self.__scrape_page()
        try:
            lc = page.find("span", itemprop="location")
            return lc.text.strip()
        except:
            return None

    def repository_stats(self, repo_url):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        repository_stats = repository.repository_stats()
        ```
        """
        data = self.__scrape_repositories(repo_url)
        try:
            # forks
            forksCount = data.find("span", id="repo-network-counter").text.strip()
            # stars
            starCount = data.find("span", id="repo-stars-counter-star").text.strip()
            # issues
            issuesCount = data.find("span", id="issues-repo-tab-count").text.strip()
            # pull requests
            pullRequests = data.find(
                "span", id="pull-requests-repo-tab-count"
            ).text.strip()
            data = {
                "forks": forksCount,
                "stars": starCount,
                "issues": issuesCount,
                "pullRequests": pullRequests,
            }
            return data
        except:
            return None

    def repository_details(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        repository_details = repository.repository_details()
        ```
        """
        organization = self.organization
        data = self.__scrape_repositories_page()
        try:
            pages_body = data.find("div", class_="paginate-container")
            current_page = pages_body.find("em", class_="current")
            total_pages = 1
            if current_page != None:
                total_pages = (int)(current_page["data-total-pages"])

            pages = []
            if total_pages == 1:
                pages.append(f"https://github.com/orgs/{organization}/repositories")
            else:
                for i in range(1, total_pages + 1):
                    pages.append(
                        f"https://github.com/orgs/{organization}/repositories?page={i}"
                    )

            repositories = []
            for page in pages:
                page_data = self.__scrape_repositories(page)
                repositories_body = page_data.find("div", id="org-repositories")
                for repo in repositories_body.find_all("li"):
                    repo_name = repo.find(
                        "a", attrs={"itemprop": "name codeRepository"}
                    ).text.strip()
                    repo_url = f"https://github.com{repo.find('a', attrs = {'itemprop': 'name codeRepository'})['href']}"
                    repo_description_body = repo.find(
                        "p", attrs={"itemprop": "description"}
                    )
                    repo_description = (
                        repo_description_body.text.strip()
                        if repo_description_body != None
                        else "No description"
                    )
                    repo_language_body = repo.find(
                        "span", attrs={"itemprop": "programmingLanguage"}
                    )
                    repo_language = (
                        repo_language_body.text.strip()
                        if repo_language_body != None
                        else "No language"
                    )
                    (
                        repo_forks,
                        repo_stars,
                        repo_issues,
                        repo_pull_requests,
                    ) = self.repository_stats(repo_url)
                    repositories.append(
                        {
                            "name": repo_name,
                            "url": repo_url,
                            "description": repo_description,
                            "language": repo_language,
                            "forks": repo_forks,
                            "stars": repo_stars,
                            "issues": repo_issues,
                            "pull_requests": repo_pull_requests,
                        }
                    )

            return repositories
        except:
            return None

    def pinned_repository(self):
        """
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        pinned_repository = repository.pinned_repository()
        ```
        """
        organization = self.organization

        data = self.__scrape_page()

        try:
            pinned_repos = data.find(
                "ol",
                class_="d-flex flex-wrap list-style-none gutter-condensed mb-2 js-pinned-items-reorder-list",
            )

            repo_info_list = []

            for repo in pinned_repos.find_all("li"):
                name = repo.find("span", class_="repo").text.strip()

                desc = repo.find(
                    "p", class_="pinned-item-desc color-fg-muted text-small mt-2 mb-0"
                ).text.strip()

                top_tech = repo.find(
                    "span", itemprop="programmingLanguage"
                ).text.strip()

                url = "https://github.com" + repo.find("a", href=True)["href"]
                response = requests.get(url)

                url_parts = url.split("/")

                organization = url_parts[3]

                repository = url_parts[4]

                soup = BeautifulSoup(response.content, "html.parser")

                star_count_elem = soup.find(
                    "a", href=f"/{organization}/{repository}/stargazers"
                ).find("span")
                star_count = int(star_count_elem.text.strip())

                stats_body = soup.find(
                    "ul", class_="pagehead-actions flex-shrink-0 d-none d-md-inline"
                )
                forks = stats_body.find("span", id="repo-network-counter")
                fork_count = forks.text.strip()

                repo_info = {
                    "name": name,
                    "link": url,
                    "detail": desc,
                    "top_lang": top_tech,
                    "stars": star_count,
                    "forks": fork_count,
                }
                repo_info_list.append(repo_info)
            return repo_info_list
        except:
            return None

    def get_organization_links(self):
        """ "
        Class - `Organisation`
        Example:
        ```
        repository = github.Organization(organization_name="Clueless-Community")
        get_organization_links = repository.get_organization_links()
        ```
        """
        try:
            links = {}
            data = self.__scrape_page()
            website_link = data.find("a", rel="nofollow", itemprop="url", href=True)[
                "href"
            ]
            links["website"] = website_link
            gmail = data.find("a", itemprop="email", href=True)["href"]
            links["gmail"] = gmail
            other_link = data.find_all("a", rel="nofollow", href=True)
            for o in other_link:
                name = (
                    o["href"]
                    .split("//")[1]
                    .split("/")[0]
                    .replace("www.", "")
                    .split(".")[0]
                )
                if name != self.organization or name.find(self.organization) == -1:
                    if not name in links:
                        links[name] = o["href"]
            return links
        except:
            return None


# src/scrape_up/github/pull_request.py

import requests
from bs4 import BeautifulSoup


class PullRequest:
    """
    Create an instance of the class `PullRequest`
    ```python
    pull_request = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
    ```

    | Methods            | Details                                                                    |
    | ------------------ | -------------------------------------------------------------------------- |
    | `.commits()`       | Returns the number of commits made in a pull request.                      |
    | `.title()`         | Returns the title of a pull request.                                       |
    | `.labels()`        | Returns all the labels of a pull request, empty list in case of no labels. |
    | `.files_changed()` | Returns the number of files changed in a pull request.                     |
    | `.reviewers()`     | Return the list of reviewers assigned in a pull request.                   |
    """

    def __init__(self, username: str, repository_name: str, pull_request_number: int):
        self.username = username
        self.repository = repository_name
        self.pr_number = pull_request_number

    def __scrape_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/pull/{self.pr_number}"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def labels(self):
        """
        Class - `PullRequest`
        Example:
        ```
        repository = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
        labels = repository.labels()
        ```
        """
        labels_found = []
        data = self.__scrape_page()
        label_raw = data.find_all(
            "a", class_="IssueLabel hx_IssueLabel width-fit mb-1 mr-1"
        )
        try:
            for d in label_raw:
                labels_found.append(d.get_text().strip())
            labels_found + 1
            # return labels_found
            return labels_found
        except:
            return None

    def commits(self):
        """
        Class - `PullRequest`
        Example:
        ```python
        repository = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
        commits = repository.commits()
        ```
        """
        data = self.__scrape_page()
        try:
            commits_count = data.find("span", id="commits_tab_counter").text.strip()
            return commits_count
        except:
            return None

    def title(self):
        """
        Class - `PullRequest`
        Example:
        ```python
        repository = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
        title = repository.title()
        ```
        """
        data = self.__scrape_page()
        try:
            title_body = data.find("bdi", class_="js-issue-title markdown-title")
            title = title_body.text.strip()
            return title
        except:
            return None

    def __files_changed_body(self):
        """
        scrape the data of files changed in a pull request
        """
        link = f"https://github.com/{self.username}/{self.repository}/pull/{self.pr_number}/files"
        data = requests.get(link)
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def files_changed(self):
        """
        Class - `PullRequest`
        Example:
        ```python
        repository = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
        files_changed = repository.files_changed()
        ```
        """
        data = self.__files_changed_body()
        try:
            files_changed_body = data.find("span", id="files_tab_counter")
            files_changed = files_changed_body.text.strip()
            return files_changed
        except:
            return None

    def reviewers(self):
        """
        Class - `PullRequest`
        Example:
        ```python
        repository = github.PullRequest(username="nikhil25803", repository_name="scrape-up", pull_request_number=30)
        reviewers = repository.reviewers()
        ```
        """
        data = self.__scrape_page()
        try:
            reviewerList = []
            reviewers = data.find_all(
                "span", class_="css-truncate-target width-fit v-align-middle"
            )
            if len(reviewers) == 0:
                message = f"No reviewers found for {self.pr_number}"
                return reviewerList
            else:
                for reviewer in reviewers:
                    reviewerList.append(reviewer.text)
                return reviewerList
        except:
            return None


# src/scrape_up/github/repository.py

import requests
from bs4 import BeautifulSoup
import requests_html
import os


class Repository:
    """
    Create an instance of the class `Repository`
    ```python
    repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
    ```

    | Methods                    | Details                                                                                                                                                        |
    | -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- |
    | `.fork_count()`            | Returns the number of forks of a repository.                                                                                                                   |
    | `.get_contributors()`      | Returns the number of contributors of a repository.                                                                                                            |
    | `.topics()`                | Returns the topics of a repository.                                                                                                                            |
    | `.pull_requests()`         | Returns the number of pull requests opened in a repository.                                                                                                    |
    | `.last_updated_at()`       | Returns the last updated date of a repository.                                                                                                                 |
    | `.tags()`                  | Returns the last ten tags of a repository.                                                                                                                     |
    | `.releases()`              | Returns the last ten releases of a repository.                                                                                                                 |
    | `.issues_count()`          | Returns number of issues in a repository                                                                                                                       |
    | `.readme`                  | Saves the readme.md file of the given user to the current working directory. To view the readme.md with a live server, change ".md" to ".html" in "readme.md". |
    | `.get_pull_requests_ids()` | Returns all ids of opened pull requests in a repository.                                                                                                       |
    | `.get_issues()`            | Returns the list of all open issues in a repository.                                                                                                           |
    | `.commits()`               | Returns the number of commits in a repository.                                                                                                                 |
    | `.get_readme()`            | Returns & saves README.md file of the special repository (if exists)                                                                                           |
    | `.get_environment()`       | Returns the latest deployed link of a repository (if exists).                                                                                                  |
    | `.watch_count()`           | Returns the number of watchers of a repository                                                                                                                 |
    | `.all_watchers()`          | Returns the username of all watches of a repository                                                                                                            |
    """

    def __init__(self, username: str, repository_name: str):
        self.username = username
        self.repository = repository_name

    def __str__(self):
        return f"{self.repository} belongs to {self.username}"

    def __scrape_page(self):
        data = requests.get(f"https://github.com/{self.username}/{self.repository}")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_tags_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/tags"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_issues_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/issues"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_pull_requests_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/pulls"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_deployments_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/deployments/activity_log"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def __scrape_watchers_page(self):
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/watchers"
        )
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def languagesUsed(self):
        """
        Class - `Repository`
        Example:
        ```pyhon
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        languagesUsed = repository.languagesUsed()
        ```
        """
        data = self.__scrape_page()

        try:
            languages = data.find_all(class_="color-fg-default text-bold mr-1")
            allLanguages = []
            for item in languages:
                item = str(item)
                item = item[46:]
                item = item[:-7]
                allLanguages.append(item)
            # return allLanguages  # return list of languages
            return allLanguages
        except:
            return None

    def about(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        about = repository.about()
        ```
        """
        data = self.__scrape_page()

        try:
            tag = data.find(class_="f4 mb-3")
            about = tag.get_text()
            return about
        except:
            return None

    def fork_count(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        fork_count = repository.fork_count()
        ```
        """
        data = self.__scrape_page()
        try:
            stats_body = data.find(
                "ul", class_="pagehead-actions flex-shrink-0 d-none d-md-inline"
            )
            forks = stats_body.find("span", id="repo-network-counter")
            fork_count = forks.text.strip()
            return fork_count
        except:
            return None

    def topics(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        topics = repository.topics()
        ```
        """
        data = self.__scrape_page()

        try:
            topics = data.find_all(class_="topic-tag topic-tag-link")
            allTopics = []
            print(allTopics)
            for item in topics:
                allTopics.append(item.text)
            return allTopics
        except:
            return None

    def star_count(self):
        """
        Class - `Repository`
        Example:
        ```pyhton
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        star_count = repository.star_count()
        ```
        """
        try:
            data = self.__scrape_page()
            starCount = (
                data.find("a", href=f"/{self.username}/{self.repository}/stargazers")
                .find("span")
                .text.strip()
            )
            return starCount
        except:
            return None

    def pull_requests(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        pull_requests = repository.pull_requests()
        ```
        """
        data = self.__scrape_page()
        try:
            pull_requests = (
                data.find_all(class_="UnderlineNav-item mr-0 mr-md-1 mr-lg-3")[2]
                .find_all("span")[1]
                .text.strip()
            )
            return pull_requests
        except:
            return None

    def tags(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        tags = repository.tags()
        ```
        """
        data = self.__scrape_tags_page()
        try:
            tags = data.find_all(class_="Link--primary")
            allTags = []
            for item in tags:
                allTags.append(item.text)
            return allTags
        except:
            return None

    def releases(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        releases = repository.releases()
        ```
        """
        data = self.__scrape_tags_page()
        try:
            releases = data.find_all(class_="Link--primary")
            allReleases = []
            for item in releases:
                allReleases.append(item.text)
            return allReleases
        except:
            return None

    def issues_count(self):
        """
        Class - `Repository`
        Example:
        ```
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        issues_count = repository.issues_count()
        ```
        """
        data = self.__scrape_page()
        try:
            issues = data.find("span", {"id": "issues-repo-tab-count"}).text.strip()
            return issues
        except:
            return None

    def readme(self):
        """
        Class - `Repository`\n
        *This downloads the README.md in the root directory*\n
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        readme = repository.readme()
        ```
        """
        session = requests_html.HTMLSession()
        r = session.get(
            f"https://github.com/{self.username}/{self.username}/blob/main/README.md"
        )
        markdown_content = r.text

        try:
            with open("out.md", "w", encoding="utf-8") as f:
                f.write(markdown_content)
        except:
            return None

    def get_pull_requests_ids(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        get_pull_requests_ids = repository.get_pull_requests_ids()
        ```
        """
        data = self.__scrape_pull_requests_page()
        try:
            pr_body = data.find(
                "div", class_="js-navigation-container js-active-navigation-container"
            )
            pull_requests_ids = []
            for each_pr in pr_body.find_all(
                "a",
                class_="Link--primary v-align-middle no-underline h4 js-navigation-open markdown-title",
            ):
                pr_id = each_pr["href"].split("/")[-1]
                pull_requests_ids.append(pr_id)

            return pull_requests_ids
        except:
            return None

    def commits(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        commits = repository.commits()
        ```
        """
        data = self.__scrape_page()
        try:
            commits = str(data.find_all(class_="d-none d-sm-inline"))
            s = commits.split("<strong>")
            s = s[1].split("</strong>")
            commits = int(s[0])
            return commits
        except:
            return None

    def get_issues(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        get_issues = repository.get_issues()
        ```
        """
        data = self.__scrape_issues_page()
        try:
            issues = data.find_all(
                class_="Link--primary v-align-middle no-underline h4 js-navigation-open markdown-title"
            )
            allIssues = []

            for item in issues:
                allIssues.append(item.text)
            return allIssues
        except:
            return None

    def get_contributors(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        get_contributors = repository.get_contributors()
        ```
        """
        data = self.__scrape_page()

        try:
            contributors = data.find_all(
                "a", href=f"/{self.username}/{self.repository}/graphs/contributors"
            )
            contributor = []
            for it in contributors:
                contributor.append(it.get_text())
            return contributor[0].strip()
        except:
            return None

    def last_update_at(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        last_update_at = repository.last_update_at()
        ```
        """
        data = self.__scrape_page()
        try:
            update = data.find_all("relative-time", class_="no-wrap")
            return update[0].get_text()
        except:
            return None

    def get_readme(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        get_readme = repository.get_readme()
        ```
        """
        data = requests.get(
            f"https://raw.githubusercontent.com/{self.username}/{self.username}/master/README.md"
        )
        if data.status_code == 404:
            data = requests.get(
                f"https://raw.githubusercontent.com/{self.username}/{self.username}/main/README.md"
            )
            if data.status_code == 404:
                message = f"No special repository found with username {self.username}"
                return {
                    "data": None,
                    "message": message,
                }
        else:
            path = f"./{self.username}"
            try:
                os.mkdir(path)
            except OSError as error:
                return None
            data = data.text
            readmeFile = os.open(path + "/README.md", os.O_RDWR | os.O_CREAT)
            os.write(readmeFile, data.encode("utf-8"))
            message = "README.md found & saved"
            return message

    def get_environment(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        get_environment = repository.get_environment()
        ```
        """
        try:
            data = self.__scrape_deployments_page()
            link = data.find(
                "a", class_="btn btn-outline flex-self-start mt-2 mt-md-0"
            ).get("href")
            return link
        except:
            return None

    def get_branch(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        get_branch = repository.get_branch()
        ```
        """
        data = requests.get(
            f"https://github.com/{self.username}/{self.repository}/branches"
        )
        data = BeautifulSoup(data.text, "html.parser")
        try:
            branch = data.find_all(
                class_="branch-name css-truncate-target v-align-baseline width-fit mr-2 Details-content--shown"
            )
            allBranches = []
            for branchNames in branch:
                allBranches.append(branchNames.text.strip())
            return allBranches
        except:
            return None

    def watch_count(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        watch_count = repository.watch_count()
        ```
        """
        data = self.__scrape_watchers_page()
        try:
            watches = len(data.find("ol", {"class": "gutter"}).find_all("li"))
            return {
                "data": watches,
                "message": f"Total watches in {self.repository} repository",
            }
        except:
            message = f"No watches found in {self.repository} repository"
            return {
                "data": None,
                "message": message,
            }

    def all_watchers(self):
        """
        Class - `Repository`
        Example:
        ```python
        repository = github.Repository(username="nikhil25803", repository_name="scrape-up")
        all_watchers = repository.all_watchers()
        ```
        """
        data = self.__scrape_watchers_page()
        try:
            all = data.find("ol", {"class": "gutter"}).find_all(
                "a", {"data-hovercard-type": "user"}
            )[1::2]
            watchers = []
            for watcher in all:
                watchers.append(watcher.text.strip())
            return watchers

        except:
            return None


# src/scrape_up/github/users.py

import requests
from bs4 import BeautifulSoup


class Users:
    """
    Create an instance of the class `Users`
    ```python
    user = github.Users(username="nikhil25803")
    ```
    | Methods                       | Details                                                                                            |
    | ----------------------------- | -------------------------------------------------------------------------------------------------- |
    | `.followers()`                | Returns the number of followers of a user.                                                         |
    | `.following()`                | Returns the number of following of a user.                                                         |
    | `.get_avatar()`               | Returns the avatar URL of a user.                                                                  |
    | `.get_bio()`                  | Returns the bio of a user.                                                                         |
    | `.get_repo()`                 | Returns the list of pinned repositories for a user.                                                |
    | `.repo_count()`               | Returns the number of Repositories of a user.                                                      |
    | `.star_count()`               | Returns the number of stars of a user.                                                             |
    | `.get_yearly_contributions()` | Returns the number of contributions made in 365 days frame.                                        |
    | `.get_repositories()`         | Returns the list of repositories of a user.                                                        |
    | `.get_starred_repos()`        | Return the list of starred repositories of a user.                                                 |
    | `.pul_requests()`             | Return the number of pull requests opened in a repository.                                         |
    | `.get_followers()`            | Returns the list of followers of a user.                                                           |
    | `.get_following_users()`      | Returns the list of users followed by a user.                                                      |
    | `.get_achievements()`         | Returns the list of achievements of a user.                                                        |
    | `.get_status()`               | Returns the status of a user.                                                                      |
    | `.get_contribution_streak()`  | Returns the maximum contribution streak of a user in the past year starting from the current date. |
    | `.get_repository_details()`   | Returns the list of repositories with their details.                                               |
    | `.get_branch()`               | Returns the list of branches in a repository.                                                      |
    | `.get_merged_pull_requests()` | Returns the list of merged pull requests                                                           |
    | `.get_open_issues()`          | Returns the list of open issues                                                                    |
    """

    def __init__(self, username: str):
        self.username = username

    def __str__(self):
        return f" The username is: {self.username}"

    def __scrape_page(self):
        username = self.username
        data = requests.get(f"https://github.com/{username}")
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def followers(self) -> str:
        """
        Class - `Users`
        ```python
        user = github.User(username="nikhil25803")
        followers = user.followers()
        ```
        """
        page = self.__scrape_page()
        try:
            followers = page.find(class_="text-bold color-fg-default")
            return followers.text
        except:
            return None

    def following(self):
        """
        Class - `Users`\n
        ```python
        user = github.User(username="nikhil25803")
        following = user.following()
        ```
        """
        page = self.__scrape_page()
        try:
            following = page.find_all(class_="text-bold color-fg-default")

            return following[1].text
        except:
            return None

    def get_avatar(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_avatar = user.get_avatar()
        ```
        """
        page = self.__scrape_page()
        try:
            avatar = page.find(
                class_="avatar avatar-user width-full border color-bg-default"
            )
            return avatar["src"]
        except:
            return None

    def get_bio(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_bio = user.get_bio()
        ```
        """
        page = self.__scrape_page()
        try:
            bio = page.find(
                class_="p-note user-profile-bio mb-3 js-user-profile-bio f4"
            )
            return bio.text
        except:
            return None

    def get_repo(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_repo = user.get_repo()
        ```
        """
        page = self.__scrape_page()
        try:
            pinned_repos = page.find_all(
                class_="mb-3 d-flex flex-content-stretch col-12 col-md-6 col-lg-6"
            )
            titles = [repo.find("span", class_="repo").text for repo in pinned_repos]
            return titles
        except:
            return None

    def repo_count(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        repo_count = user.repo_count()
        ```
        """
        page = self.__scrape_page()
        try:
            count_repo = page.find_all(class_="Counter")
            count_repo_list = []
            for word in count_repo:
                find_all_example = word.get_text()
                count_repo_list.append(find_all_example)
            return count_repo_list[0]
        except:
            return None

    def star_count(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        star_count = user.star_count()
        ```
        """
        page = self.__scrape_page()
        try:
            count_star = page.find_all(class_="Counter")
            count_star_list = []
            for words in count_star:
                find_all_example = words.get_text()
                count_star_list.append(find_all_example)
            return count_star_list[3]
        except:
            return None

    def get_yearly_contributions(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_yearly_contributions = user.get_yearly_contributions()
        ```
        """
        page = self.__scrape_page()
        try:
            contributions = page.find("h2", class_="f4 text-normal mb-2")
            return " ".join(contributions.text.split())
        except:
            return None

    def __get_repo_page(self):
        """
        Scrape the repositories page of a GitHub user.
        """
        username = self.username
        repo_data = requests.get(f"https://github.com/{username}?tab=repositories")
        repo_data = BeautifulSoup(repo_data.text, "html.parser")
        return repo_data

    def get_repositories(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_repositories = user.get_repositories()
        ```
        """
        page = self.__get_repo_page()
        try:
            repo_body = page.find("div", id="user-repositories-list")
            repositories = []
            if repo_body != None:
                for repo in repo_body.find_all(
                    "div", class_="col-10 col-lg-9 d-inline-block"
                ):
                    repositories.append("https://github.com" + repo.a["href"])
            return repositories
        except:
            return None

    def get_organizations(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_organizations = user.get_organizations()
        ```
        """
        page = self.__scrape_page()
        try:
            orgs = [org.login for org in page.get_orgs()]
            return orgs
        except:
            return None

    def get_achievements(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_achievements = user.get_achievements()
        ```
        """
        try:
            achievement = []
            data = self.__scrape_page()
            data = data.find_all("img", class_="achievement-badge-sidebar", alt=True)
            itr = 0
            while itr < len(data) / 2:
                achievement.append(data[itr]["alt"].split(":")[1].strip(" "))
                itr = itr + 1

            return achievement
        except:
            return None

    def __get_starred_page(self):
        """
        Scrape the starred page of a GitHub user.
        """
        username = self.username
        starred_data = requests.get(f"https://github.com/{username}?tab=stars")
        starred_data = BeautifulSoup(starred_data.text, "html.parser")
        return starred_data

    def get_starred_repos(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_starred_repos = user.get_starred_repos()
        ```
        """
        page = self.__get_starred_page()
        try:
            starred_body = page.find("turbo-frame", id="user-starred-repos")
            starred_repos = []
            if starred_body != None:
                for repo in starred_body.find_all(
                    "div",
                    class_="col-12 d-block width-full py-4 border-bottom color-border-muted",
                ):
                    starred_repos.append("https://github.com" + repo.a["href"])
            return starred_repos
        except:
            return None

    def __scrape_followers_page(self):
        """
        Scrape the followers page of a GitHub user.
        """
        username = self.username
        followers_data = requests.get(f"https://github.com/{username}?tab=followers")
        followers_data = BeautifulSoup(followers_data.text, "html.parser")
        return followers_data

    def get_followers(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_followers = user.get_followers()
        ```
        """
        page = self.__scrape_followers_page()
        try:
            followers_body = page.find("turbo-frame", id="user-profile-frame")
            followers = []
            for user in followers_body.find_all("span", class_="Link--secondary"):
                followers.append(user.text.strip())

            return followers
        except:
            return None

    def __scrape_following_page(self):
        """

        Scrape the following page of a GitHub user.
        """
        username = self.username
        following_data = requests.get(f"https://github.com/{username}?tab=following")
        following_data = BeautifulSoup(following_data.text, "html.parser")
        return following_data

    def get_following_users(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_following_users = user.get_following_users()
        ```
        """
        page = self.__scrape_following_page()
        try:
            following_body = page.find("turbo-frame", id="user-profile-frame")
            following = []
            for user in following_body.find_all("span", class_="Link--secondary"):
                following.append(user.text.strip())

            return following
        except:
            return None

    def company(self):
        page = self.__scrape_following_page()
        try:
            cmp = page.find(class_="Link--primary")
            return cmp.text
        except:
            return None

    def get_status(self):
        """Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_status = user.get_status()
        ```
        """
        try:
            data = self.__scrape_page()
            t = data.find(
                "div", class_="user-status-container position-relative hide-sm hide-md"
            )
            return t.text.strip().replace("\n", "")
        except:
            return None

    def get_contribution_streak(self):
        """
         Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_contribution_streak = user.get_contribution_streak()
        ```
        """
        try:
            data = self.__scrape_page()
            t = data.find_all("rect", class_="ContributionCalendar-day")
            array = []
            for a in t:
                contri = a.get_text()
                if contri:
                    if contri[0] == "N":
                        array.append(0)
                    else:
                        array.append(1)

            count = 0
            result = 0

            for i in range(0, len(array)):
                if array[i] == 0:
                    count = 0.0
                else:
                    count += 1
                    result = max(result, count)
            return int(result)
        except:
            return None

    def __get_page_details(self, link):
        """
        scrape the data in the page
        """
        data = requests.get(link)
        data = BeautifulSoup(data.text, "html.parser")
        return data

    def get_pages(self, curr_repo_link, pages_links):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_pages = user.get_pages()
        ```
        """
        try:
            data = self.__get_page_details(curr_repo_link)
            pages_body = data.find("div", class_="paginate-container")
            if pages_body.find("a", class_="next_page") != None:
                pages_links.append(
                    "https://github.com"
                    + pages_body.find("a", class_="next_page")["href"]
                )
                self.get_pages(pages_links[-1], pages_links)

            return pages_links
        except:
            return None

    def get_repository_details(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_repository_details = user.get_repository_details()
        ```
        """
        username = self.username
        repository_link = f"https://github.com/{username}?tab=repositories"
        try:
            # get all the pages of the repositories
            pages_links = self.get_pages(repository_link, [repository_link])
            repositories = []
            for page in pages_links:
                page_data = self.__get_page_details(page)
                # get the repositories in the page
                repositories_body = page_data.find("div", id="user-repositories-list")
                for repo in repositories_body.find_all("li"):
                    repo_name = repo.find(
                        "a", attrs={"itemprop": "name codeRepository"}
                    ).text.strip()
                    repo_url = f"https://github.com/{repo.find('a', attrs = {'itemprop': 'name codeRepository'})['href']}"
                    repo_description_body = repo.find(
                        "p", attrs={"itemprop": "description"}
                    )
                    repo_description = (
                        repo_description_body.text.strip()
                        if repo_description_body != None
                        else "No description"
                    )
                    repo_language_body = repo.find(
                        "span", attrs={"itemprop": "programmingLanguage"}
                    )
                    repo_language = (
                        repo_language_body.text.strip()
                        if repo_language_body != None
                        else "No language"
                    )
                    # create a repository object
                    repository_name = repo_url.split("/")[-1]
                    repository = repository.Repository(username, repository_name)
                    repo_forks, repo_stars, repo_issues, repo_pull_requests = (
                        repository.fork_count(),
                        repository.star_count(),
                        repository.issues_count(),
                        repository.pull_requests(),
                    )
                    repositories.append(
                        {
                            "name": repo_name,
                            "url": repo_url,
                            "description": repo_description,
                            "language": repo_language,
                            "forks": repo_forks,
                            "stars": repo_stars,
                            "issues": repo_issues,
                            "pull_requests": repo_pull_requests,
                        }
                    )

            return repositories
        except:
            return None

    def get_all_repo_details(self):
        """
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_all_repo_details = user.get_all_repo_details()
        ```
        """
        page = self.__get_repo_page()

        repo_elements = page.select("#user-repositories-list ul li")
        repositories = []

        try:
            for repo_element in repo_elements:
                link = repo_element.find("a")["href"]
                repositories.append(f"https://github.com{link}")

                name = repo_element.find("a").text.split("/")[-1]
                repositories.append(name.replace("\n", ""))

                description = (
                    repo_element.find("p").get_text(strip=True)
                    if repo_element.find("p")
                    else None
                )
                # description= description.replace('\n','')
                repositories.append(description)

                url1 = "https://github.com" + link
                response1 = requests.get(url1)
                soup = BeautifulSoup(response1.content, "html.parser")
                li_elements = soup.find_all("li", class_="d-inline")

                for li in li_elements:
                    language = li.text.strip()
                    repositories.append(language.replace("\n", ""))

                    stars = soup.find("span", class_="text-bold")
                    num_of_stars = stars.text if stars else "N/A"
                    repositories.append(num_of_stars.replace("\n", ""))

                pullurl = url1 + "/pulls"
                issuesurl = url1 + "/issues"
                pullresponse = requests.get(
                    pullurl
                )  # getting the content of pull requests page
                issueresponse = requests.get(
                    issuesurl
                )  # getting the content of issues page

                p_soup = BeautifulSoup(
                    pullresponse.content, "html.parser"
                )  # creating beautifulsoup object for pull requests page
                i_soup = BeautifulSoup(
                    issueresponse.content, "html.parser"
                )  # creating beautifulsoup object for issues page

                # to find number of pull requests
                pullrequests = p_soup.find(
                    "div", class_="table-list-header-toggle states flex-auto pl-0"
                )
                num_of_pull_requests = (
                    pullrequests.text.strip() if pullrequests else "N/A"
                )
                repositories.append(num_of_pull_requests.replace("\n", ""))

                issues = i_soup.select_one("span#issues-repo-tab-count")
                num_of_issues = issues.text.strip() if issues else "N/A"
                repositories.append(num_of_issues.replace("\n", ""))

            return repositories
        except:
            return None

    def get_merged_pull_requests(self):
        """
        Returns the list of all the merged PRs\n
        Warning -This methods take longer time to run.\n
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_merged_pull_requests = user.get_merged_pull_requests()
        ```
        """

        page = self.__get_repo_page()

        repo_elements = page.select("#user-repositories-list ul li")
        merged_pull_requests = []

        try:
            forked_repos = []
            for repo_element in repo_elements:
                forked_repo = repo_element.select_one("h3+span a")

                if forked_repo:
                    forked_repo = "https://github.com" + forked_repo.get("href")
                    forked_repos.append(forked_repo)

            for repo in forked_repos:
                closed_pr_url = (
                    repo + f"/pulls?q=is:pr+author:{self.username}+is:merged"
                )

                response = self.__get_page_details(closed_pr_url)
                pr_links = response.select('a[data-hovercard-type="pull_request"]')
                links = ["https://github.com" + link["href"] for link in pr_links]

                merged_pull_requests.extend(links)
        except:
            return None

        return merged_pull_requests

    def get_open_issues(self):
        """
        Return the list of open issues.\n
        Warning -This methods take longer time to run.\n
        Class - `Users`\n
        Example -\n
        ```python
        user = github.User(username="nikhil25803")
        get_open_issues = user.get_open_issues()
        ```
        """

        page = self.__get_repo_page()

        repo_elements = page.select("#user-repositories-list ul li")
        open_issues = []

        try:
            forked_repos = []
            for repo_element in repo_elements:
                forked_repo = repo_element.select_one("h3+span a")

                if forked_repo:
                    forked_repo = "https://github.com" + forked_repo.get("href")
                    forked_repos.append(forked_repo)

            for repo in forked_repos:
                open_issues_url = repo + f"/issues/created_by/{self.username}"

                response = self.__get_page_details(open_issues_url)
                issues_list = response.select(
                    'div[aria-label="Issues"] div[data-pjax="#repo-content-pjax-container"]'
                )
                issue_links = [
                    "https://github.com" + issue.select_one("a").get("href")
                    for issue in issues_list
                ]

                open_issues.extend(issue_links)
        except:
            return None

        return open_issues


user = Users(username="nikhil25803")
print(user.get_open_issues())


# src/scrape_up/gitlab/__init__.py

from gitlab.issue import Issue
from scrape_up.gitlab.pull_request import PullRequest
from scrape_up.gitlab.organization import Organization
from scrape_up.gitlab.users import User
from scrape_up.gitlab.repository import Repository

__all__ = ["Issue", "PullRequest", "Organization", "Gitlab", "Repository"]


# src/scrape_up/gitlab/issue.py

import requests
from bs4 import BeautifulSoup


class Issue:
    def __init__(self, username, repository, issue_number):
        self.username = username
        self.repository = repository
        self.issue_number = issue_number

    def __scrape_page(self):
        url = f"https://gitlab.com/{self.username}/{self.repository}/-/issues/{self.issue_number}"
        response = requests.get(url)
        response.raise_for_status()  # Raise an exception for any HTTP errors
        soup = BeautifulSoup(response.text, "html.parser")
        return soup

    def get_title(self):
        try:
            data = self.__scrape_page()
            title = data.find("h1", class_="title").text.strip()
            return {"data": title, "message": "Retrieved issue title"}
        except Exception as e:
            return {"data": None, "message": f"Error retrieving issue title: {str(e)}"}

    def get_description(self):
        try:
            data = self.__scrape_page()
            description = data.find("div", class_="description").text.strip()
            return {"data": description, "message": "Retrieved issue description"}
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving issue description: {str(e)}",
            }

    def get_author(self):
        try:
            data = self.__scrape_page()
            author = data.find("span", class_="author").text.strip()
            return {"data": author, "message": "Retrieved issue author"}
        except Exception as e:
            return {"data": None, "message": f"Error retrieving issue author: {str(e)}"}


# src/scrape_up/gitlab/organization.py

import requests
from bs4 import BeautifulSoup


class Organization:
    def init(self, organization_name):
        self.organization_name = organization_name

    def __scrape_page(self, url):
        response = requests.get(url)
        response.raise_for_status()  # Raise an exception for any HTTP errors
        soup = BeautifulSoup(response.text, "html.parser")
        return soup

    def get_members(self):
        """
        Fetch a list of usernames of the members in the organization.
        """
        url = f"https://gitlab.com/{self.organization_name}/-/group_members"
        try:
            data = self.__scrape_page(url)
            members = []
            member_elements = data.find_all("li", class_="group-member-row")
            for member_element in member_elements:
                username = member_element.find(
                    "a", class_="group-member-username"
                ).text.strip()
                members.append(username)
            return {
                "data": members,
                "message": f"Retrieved members for organization {self.organization_name}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving members for organization {self.organization_name}: {str(e)}",
            }

    def get_projects(self):
        """
        Fetch a list of project names associated with the organization.
        """
        url = f"https://gitlab.com/groups/{self.organization_name}/-/projects"
        try:
            data = self.__scrape_page(url)
            projects = []
            project_elements = data.find_all("div", class_="gl-project-card")
            for project_element in project_elements:
                name = project_element.find(
                    "a", class_="gl-project-card-title"
                ).text.strip()
                projects.append(name)
            return {
                "data": projects,
                "message": f"Retrieved projects for organization {self.organization_name}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving projects for organization {self.organization_name}: {str(e)}",
            }


# src/scrape_up/gitlab/pull_request.py

import requests
from bs4 import BeautifulSoup


class PullRequest:
    def __init__(self, username, repository, pull_request_number):
        self.username = username
        self.repository = repository
        self.pull_request_number = pull_request_number

    def __scrape_page(self):
        url = f"https://gitlab.com/{self.username}/{self.repository}/-/merge_requests/{self.pull_request_number}"
        response = requests.get(url)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, "html.parser")
        return soup

    def get_title(self):
        try:
            data = self.__scrape_page()
            title = data.find("h1", class_="title").text.strip()
            return {"data": title, "message": "Retrieved pull request title"}
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving pull request title: {str(e)}",
            }

    def get_description(self):
        try:
            data = self.__scrape_page()
            description = data.find("div", class_="description").text.strip()
            return {
                "data": description,
                "message": "Retrieved pull request description",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving pull request description: {str(e)}",
            }

    def get_author(self):
        try:
            data = self.__scrape_page()
            author = data.find("span", class_="author").text.strip()
            return {"data": author, "message": "Retrieved pull request author"}
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving pull request author: {str(e)}",
            }


# src/scrape_up/gitlab/repository.py

import requests
from bs4 import BeautifulSoup


class Repository:
    def __init__(self, username, repository_name):
        self.username = username
        self.repository_name = repository_name

    def __scrape_page(self):
        url = f"https://gitlab.com/{self.username}/{self.repository_name}"
        response = requests.get(url)
        response.raise_for_status()  # Raise an exception for any HTTP errors
        soup = BeautifulSoup(response.text, "html.parser")
        return soup

    def get_name(self):
        try:
            data = self.__scrape_page()
            name = data.find("h1", class_="project-title").text.strip()
            return {
                "data": name,
                "message": f"Retrieved name for repository {self.repository_name}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving name for repository {self.repository_name}: {str(e)}",
            }

    def get_description(self):
        try:
            data = self.__scrape_page()
            description = data.find("div", class_="project-description").text.strip()
            return {
                "data": description,
                "message": f"Retrieved description for repository {self.repository_name}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving description for repository {self.repository_name}: {str(e)}",
            }


# src/scrape_up/gitlab/users.py

import requests
from bs4 import BeautifulSoup


class Users:
    def __init__(self, username):
        self.username = username

    def __scrape_page(self, url):
        response = requests.get(url)
        response.raise_for_status()  # Raise an exception for any HTTP errors
        soup = BeautifulSoup(response.text, "html.parser")
        return soup

    def get_name(self):
        url = f"https://gitlab.com/{self.username}"
        try:
            data = self.__scrape_page(url)
            name = data.find("h1", class_="user-title").text.strip()
            return {"data": name, "message": f"Retrieved name for user {self.username}"}
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving name for user {self.username}: {str(e)}",
            }

    def get_bio(self):
        url = f"https://gitlab.com/{self.username}"
        try:
            data = self.__scrape_page(url)
            bio = data.find("div", class_="user-info").text.strip()
            return {"data": bio, "message": f"Retrieved bio for user {self.username}"}
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving bio for user {self.username}: {str(e)}",
            }

    def get_avatar_url(self):
        url = f"https://gitlab.com/{self.username}"
        try:
            data = self.__scrape_page(url)
            avatar_url = data.find("img", class_="avatar s40")["src"]
            return {
                "data": avatar_url,
                "message": f"Retrieved avatar URL for user {self.username}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving avatar URL for user {self.username}: {str(e)}",
            }

    def get_repositories(self):
        url = f"https://gitlab.com/{self.username}?tab=repositories"
        try:
            data = self.__scrape_page(url)
            repositories = []
            repo_elements = data.find_all("li", class_="project-row")
            for repo_element in repo_elements:
                name = repo_element.find("h3", class_="project-title").text.strip()
                description = repo_element.find(
                    "p", class_="project-description"
                ).text.strip()
                repositories.append({"name": name, "description": description})
            return {
                "data": repositories,
                "message": f"Retrieved repositories for user {self.username}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving repositories for user {self.username}: {str(e)}",
            }

    def get_project_details(self, project_id):
        url = f"https://gitlab.com/{self.username}/{project_id}"
        try:
            data = self.__scrape_page(url)
            name = data.find("h1", class_="project-title").text.strip()
            last_activity = data.find("span", class_="last_activity").text.strip()
            return {
                "data": {"name": name, "last_activity": last_activity},
                "message": f"Retrieved project details for project {project_id}",
            }
        except Exception as e:
            return {
                "data": None,
                "message": f"Error retrieving project details for project {project_id}: {str(e)}",
            }


# src/scrape_up/googleNews/__init__.py

from scrape_up.googleNews.googleNews import GoogleNews
__all__ = ["GoogleNews"]

# src/scrape_up/googleNews/googleNews.py

import requests
from bs4 import BeautifulSoup


class GoogleNews:
    """
    Create an instance of `GoogleNews` class.
    ```python
    articles = GoogleNews()
    ```
    | Methods                | Details                                                                                          |
    | ---------------------- | ------------------------------------------------------------------------------------------------ |
    | `.getArticles(topic="github")`       | Returns the list of articles with title, descriptions, news source, date and link in JSON format |
    | `.top_stories()`       | Returns the list of top stories listed regarding the mentioned topic                             |
    | `.timed_aticles(time)` | Returns the list of top stories listed regarding the mentioned topic and within that time frame  |
    """

    def __init__(self):
        pass

    def get_articles(self, topic: str):
        """
        Class - `GoogleNews`
        Example:
        ```python
        articles = GoogleNews()
        articles.get_articles(topic="github")
        ```
        Parameters required  -  `topic`\n
        Returns:
        ```js
        {
            "link": "Link to the article",
            "title": "Tile of the article",
            "date": "Date the article was posted",
        }
        ```
        """
        url = "https://news.google.com/rss/search?q=" + topic
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, features="xml")
            # find all li tags
            lis = soup.find_all("item")
            sub_articles = []
            for li in lis:
                sub_articles.append(
                    {
                        "link": li.link.text,
                        "title": li.title.text,
                        "Date & Time": li.pubDate.text,
                    }
                )
            return sub_articles[:8]
        except:
            return None

    def top_stories(self):
        """
        Class - `GoogleNews`
        Example:
        ```python
        articles = GoogleNews()
        articles.top_stories()
        ```
        Returns:
        ```js
        [
            {
                "link": "Link of the news",
                "title": "Title of the top story",
                "date": "Date of the top story"
            },
            ...
        ]
        ```
        """
        url = "https://news.google.com/news/rss"
        try:
            page = requests.get(url)
            soup = BeautifulSoup(page.content, features="xml")
            lis = soup.find_all("item")
            sub_articles = []
            for li in lis:
                sub_articles.append(
                    {
                        "link": li.link.text,
                        "title": li.title.text,
                        "Date & Time": li.pubDate.text,
                    }
                )
            return sub_articles
        except:
            return None

    def timed_articles(self, topic: str, time="1h"):
        """
        Class - `GoogleNews`
        Example:
        ```python
        articles = GoogleNews(topic="github")
        articles.timed_articles(time)
        ```
        Parameter required - `topic` and `time`\n
        Time format: \n
        + 1h - within 1 hour (default)
        + 1d - within 24 hours
        + 7d - within a week
        + 1y - within a year
        Returns:
        ```js
        [
            {
                "link": "Link of the news",
                "title": "Title of the top story",
                "date": "Date of the top story"
            },
            ...
        ]
        ```
        """
        if time != "":
            time = " when:" + time
        url = "https://news.google.com/news/rss/search?q=" + topic + time
        try:
            page = requests.get(url)
            soup = BeautifulSoup(page.content, features="xml")
            lis = soup.find_all("item")
            sub_articles = []
            for li in lis:
                sub_articles.append(
                    {
                        "link": li.link.text,
                        "title": li.title.text,
                        "Date & Time": li.pubDate.text,
                    }
                )
            return sub_articles
        except:
            return None


gn = GoogleNews(topic="github")
print(gn.top_stories())


# src/scrape_up/hacker_news/__init__.py

from articles import HackerNews

__all__ = ["HackerNews"]


# src/scrape_up/hacker_news/articles.py

import requests
from bs4 import BeautifulSoup


class HackerNews:
    """
    Create an instance of `HackerNews` class.
    ```py
    articles = HackerNews()
    ```
    | Methods            | Details                                                                                                              |
    | ------------------ | -------------------------------------------------------------------------------------------------------------------- |
    | `.articles_list()` | Returns the latest articles along with their score, author, author url, time, comment count and link in JSON format. |
    """

    def articles_list(self):
        """
        Class - `HackerNews`
        Example -
        ```python
        hacker_news = HackerNews()
        articles = hacker_news.articles_list()
        ```
        Return
        ```js
        [
            {
                "title":"I have written a JVM in Rust",
                "score":"507 points",
                "author":"lukastyrychtr",
                "author_url":"user?id=lukastyrychtr",
                "time":"9 hours ago",
                "comment_count":"79",
                "link":"https://andreabergia.com/blog/2023/07/i-have-written-a-jvm-in-rust/"
            }
            ...
        ]
        ```
        """
        url = "https://news.ycombinator.com/"

        articles_data = {"articles": []}

        try:
            res = requests.get(url)

            soup = BeautifulSoup(res.text, "html.parser")

            titles = soup.find_all("span", class_="titleline")
            subs = soup.find_all("span", class_="subline")

            for i, j in zip(titles, subs):
                title = i.find("a").getText()
                score = j.find("span", class_="score").getText()
                author = j.find("a", class_="hnuser").getText()
                author_url = j.find("a", class_="hnuser")["href"]
                time = j.find("span", class_="age").find("a").getText()
                comments_link = j.find_all("a")[-1]
                comment_count = (
                    "0" if not comments_link else comments_link.text.split()[0]
                )
                link = i.find("a")["href"]

                articles_data["articles"].append(
                    {
                        "title": title,
                        "score": score,
                        "author": author,
                        "author_url": author_url,
                        "time": time,
                        "comment_count": comment_count,
                        "link": link,
                    }
                )
            return articles_data["articles"]

        except:
            return None

# src/scrape_up/hashnode/__init__.py

from hashnode import Hashnode

__all__ = ["Hashnode"]


# src/scrape_up/hashnode/hashnode.py

import requests
from bs4 import BeautifulSoup


class Hashnode:
    """
    Create an instance of `Hashnode` class.
    ```python
    blogs = Hashnode()
    ```
    | Methods          | Details                                                                                              |
    | ---------------- | ---------------------------------------------------------------------------------------------------- |
    | `.get_feed()`     | Returns the blogs with title, descriptions, author, read time, like and comment count, date and link |
    | `.get_featured()` | Returns the featured blogs with title, descriptions, author, like and comment count, date and link   |
    | `.get_recent()`   | Returns the recent blogs with title, descriptions, author, like and comment count, date and link     |
    """

    def get_feed(self):
        """
        Class - `Hashnode`
        Example:
        ```pyton
        blogs = Hashnode()
        blogs.get_feed()
        ```
        Returns:
        ```js
        {
            "title": Title of the blog
            "description": Description of the blog
            "author": Author of the blog
            "read_time": Time required to read the blog
            "like_count": No. of likes of the blog
            "comment_count": No. of comments of the blog
            "date": Date the blog was posted
            "link": Link to the blog
        }
        ```
        """
        url = "https://hashnode.com/community"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            blogs_data = {"blogs": []}

            blogs = soup.find_all("div", class_="css-1s8wn94")

            for b in blogs:
                title = b.find("a", class_="css-4zleql").getText()
                desc = (
                    b.find("p", class_="css-1m4ptby")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                author = b.find("a", class_="css-9ssaz8").getText()
                read_time = b.find("span", class_="css-1r5gb7q").getText()
                try:
                    like_section = b.find("button", class_="css-a0cwys")
                    like_count = like_section.find("span").getText()
                except:
                    like_count = "0"
                try:
                    comment_section = b.find("a", class_="css-q5e5vl")
                    comment_count = comment_section.find("span").getText()
                except:
                    comment_count = "0"
                date = b.find("a", class_="css-15gyiyx").getText()
                link = b.find("a", class_="css-4zleql", href=True)["href"]

                blogs_data["blogs"].append(
                    {
                        "title": title,
                        "description": desc,
                        "author": author,
                        "read_time": read_time,
                        "like_count": like_count,
                        "comment_count": comment_count,
                        "date": date,
                        "link": link,
                    }
                )
            return blogs_data
        except:
            return None

    def get_featured(self):
        """
        Class - `Hashnode`
        Example:
        ```python
        blogs = Hashnode()
        blogs.get_featured()
        ```
        Returns:
        ```js
        [
            {
                "title": Title of the blog
                "description": Description of the blog
                "author": Author of the blog
                "like_count": No. of likes of the blog
                "comment_count": No. of comments of the blog
                "date": Date the blog was posted
                "link": Link to the blog
            }
            ...
        ]
        ```js
        """
        url = "https://hashnode.com/featured"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            blogs_data = {"blogs": []}

            blogs = soup.find_all(
                "div",
                class_="w-full first-of-type:border-t-0 border-t lg:!border border-slate-200 dark:border-slate-800 rounded-none lg:rounded-2xl pt-5 md:pt-8 lg:p-6 lg:pb-5 bg-white dark:bg-slate-950 flex flex-col gap-2 sm:gap-4",
            )

            for b in blogs:
                title = (
                    b.find("div", class_="flex flex-col gap-1")
                    .find("div")
                    .find("a")
                    .getText()
                )
                author = b.find(
                    "span",
                    class_="font-semibold text-slate-700 dark:text-slate-200 cursor-pointer",
                ).getText()
                date = b.find(
                    "p", class_="text-sm text-slate-500 dark:text-slate-400 font-normal"
                ).getText()

                try:
                    try:
                        desc = b.find(
                            "span",
                            class_="text-base hidden font-normal text-slate-500 dark:text-slate-400 hn-break-words cursor-pointer md:line-clamp-2",
                        ).getText()
                    except:
                        desc = b.find(
                            "span",
                            class_="text-base hidden font-normal text-slate-500 dark:text-slate-400 hn-break-words cursor-pointer md:line-clamp-3",
                        ).getText()
                except:
                    desc = ""

                link = (
                    b.find("div", class_="flex flex-col gap-1")
                    .find("div")
                    .find("a")["href"]
                )

                try:
                    like_count = b.find(
                        "button", attrs={"aria-label": "Like reaction"}
                    ).getText()
                except:
                    like_count = 0

                try:
                    comment_count = b.find(
                        "button", attrs={"aria-label": "Comment"}
                    ).getText()
                except:
                    comment_count = 0

                blogs_data["blogs"].append(
                    {
                        "title": title,
                        "description": desc,
                        "author": author,
                        "like_count": like_count,
                        "comment_count": comment_count,
                        "date": date,
                        "link": link,
                    }
                )
            return blogs_data["blogs"]
        except:
            return None

    def get_recent(self):
        """
        Class - `Hashnode`
        Example:
        ```python
        blogs = Hashnode()
        blogs.get_recent()
        ```
        Returns:
        ```js
        [
            {
                "title": Title of the blog
                "description": Description of the blog
                "author": Author of the blog
                "like_count": No. of likes of the blog
                "comment_count": No. of comments of the blog
                "date": Date the blog was posted
                "link": Link to the blog
            }
            ...
        ]
        """
        url = "https://hashnode.com/recent"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            blogs_data = {"blogs": []}

            blogs = soup.find_all(
                "div",
                class_="w-full first-of-type:border-t-0 border-t lg:!border border-slate-200 dark:border-slate-800 rounded-none lg:rounded-2xl pt-5 md:pt-8 lg:p-6 lg:pb-5 bg-white dark:bg-slate-950 flex flex-col gap-2 sm:gap-4",
            )

            for b in blogs:
                title = (
                    b.find("div", class_="flex flex-col gap-1")
                    .find("div")
                    .find("a")
                    .getText()
                )
                author = b.find(
                    "span",
                    class_="font-semibold text-slate-700 dark:text-slate-200 cursor-pointer",
                ).getText()
                date = b.find(
                    "p", class_="text-sm text-slate-500 dark:text-slate-400 font-normal"
                ).getText()

                try:
                    try:
                        desc = b.find(
                            "span",
                            class_="text-base hidden font-normal text-slate-500 dark:text-slate-400 hn-break-words cursor-pointer md:line-clamp-2",
                        ).getText()
                    except:
                        desc = b.find(
                            "span",
                            class_="text-base hidden font-normal text-slate-500 dark:text-slate-400 hn-break-words cursor-pointer md:line-clamp-3",
                        ).getText()
                except:
                    desc = ""

                link = (
                    b.find("div", class_="flex flex-col gap-1")
                    .find("div")
                    .find("a")["href"]
                )

                try:
                    like_count = b.find(
                        "button", attrs={"aria-label": "Like reaction"}
                    ).getText()
                except:
                    like_count = 0

                try:
                    comment_count = b.find(
                        "button", attrs={"aria-label": "Comment"}
                    ).getText()
                except:
                    comment_count = 0

                blogs_data["blogs"].append(
                    {
                        "title": title,
                        "description": desc,
                        "author": author,
                        "like_count": like_count,
                        "comment_count": comment_count,
                        "date": date,
                        "link": link,
                    }
                )
            return blogs_data["blogs"]
        except:
            return None


# src/scrape_up/icc/__init__.py

from icc_rankings import ICC

__all__ = ["ICC"]


# src/scrape_up/icc/icc_rankings.py

from bs4 import BeautifulSoup
import requests


class ICC:
    """
    Create an instance of `ICC` class.
    ```python
    scraper = ICC()
    ```
    | Method                       | Details                                                             |
    | ---------------------------- | ------------------------------------------------------------------- |
    | `.team_rankings(format)`     | Returns the list of rankings of teams of desired format             |
    |`.player_ranking(type,format)`| Returns the list of player ranking of desired type and format       |
    """

    def __init__(self):
        self.url = "https://www.icc-cricket.com/rankings/mens/"

    def team_rankings(self, format):
        """
        Create an instance of `ICC` class.\n
        Required Params - `format` - "ODI","T20" or "TEST"
        ```python
        icc = ICC()
        icc.team_rankings(format="odi")
        ```
        ```js
        [
            {
                "rank":1,
                "team":"Australia"
            }
        ]
        ```
        """
        try:
            format = format.lower()
            obj_keys = ["rank", "team"]
            resposne_list = []
            url = self.url + "team-rankings/" + format
            response = requests.get(url)
            soup = BeautifulSoup(response.content, "html.parser")
            teams = soup.find_all("span", class_="u-hide-phablet")
            for rank, team in enumerate(teams, 1):
                obj_values = [rank, team.get_text()]
                resposne_list.append(dict(zip(obj_keys, obj_values)))

            return resposne_list
        except:
            return None

    def player_ranking(self, type, format):
        """
        Create an instance of `ICC` class.\n
        Required Params
        - `format` - "ODI","T20" or "TEST"
        - `type` - "batting","bowling" or "all-rounder"\n
        ```python
        icc = ICC()
        icc.team_player(format="test",type="batting")
        ```
        Returns \n
        ```js
        [
            {
                "rank":1,
                "team":"Kane Williamson"
            }
            ...
        ]
        ```
        """
        try:
            format = format.lower()
            type = type.lower()
            response_list = []
            obj_keys = ["rank", "name"]
            url = self.url + f"/player-rankings/{format}/{type}"
            response = requests.get(url)
            soup = BeautifulSoup(response.content, "html.parser")
            top_player = soup.find(
                "div", class_="rankings-block__banner--name-large"
            ).get_text()
            rest_players = soup.find_all(
                "td", class_="table-body__cell rankings-table__name name"
            )
            response_list.append(dict(zip(obj_keys,[1,top_player])))
            for rank, player in enumerate(rest_players, 2):
                obj_values = [rank, player.get_text().replace("\n", "")]
                response_list.append(dict(zip(obj_keys, obj_values)))

            return response_list
        except:
            return None


# src/scrape_up/imdb1/__init__.py

from .imdb import IMDB

__all__ = ["IMDB"]


# src/scrape_up/imdb1/imdb.py

from bs4 import BeautifulSoup
import requests


class IMDB:
    """
    Create an instance of the `Movie` class.
    ```python
    scraper = IMDB()
    ```
    | Methods        | Details                                      |
    | -------------- | -------------------------------------------- |
    | `.top_rated()` | Returns the top-rated movies listed on IMDB. |
    | `.scrape_genre_movies(genre)` | Returns the list of movies related to the genre you mentioned. |
    | `.top_rated_shows()` | Returns the top-rated shows listed on IMDB. |
    """

    def __init__(self):
        self.headers = headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36"
        }

    def top_rated(self):
        """
        Class: `IMDB`\n
        Retrieves the top-rated movies listed on IMDB.
        Example:
        ```python
        top_movies = IMDB()
        result = top_movies.top_rated()
        ```
        Returns:
        ```js
        [
            {
                "title":"1. The Shawshank Redemption",
                "year":"1994",
                "duration":"2h 22m",
                "rating":"9.3"
            }
            ...
        ]
        ```
        """
        try:
            url = "https://www.imdb.com/chart/top/?ref_=nv_mv_250"
            html_text = requests.get(url, headers=self.headers).text
            soup = BeautifulSoup(html_text, "lxml")
            movies_container = soup.find(
                "ul",
                {
                    "class": "ipc-metadata-list ipc-metadata-list--dividers-between sc-3a353071-0 wTPeg compact-list-view ipc-metadata-list--base"
                },
            )
            movies = []

            for items in movies_container.find_all("li"):
                title = items.find("h3").text
                years = items.find(
                    "span", {"class": "sc-14dd939d-6 kHVqMR cli-title-metadata-item"}
                )
                duration = years.next_sibling.text
                rating = items.find(
                    "span",
                    {
                        "class": "ipc-rating-star ipc-rating-star--base ipc-rating-star--imdb ratingGroup--imdb-rating"
                    },
                ).text

                data = {
                    "title": title,
                    "year": years.text,
                    "duration": duration,
                    "rating": rating,
                }

                movies.append(data)

            return movies
        except requests.exceptions.RequestException as e:
            return None

    def scrape_genre_movies(self, genre):
        """
        Class - `IMDB`\n
        Example -\n
        ```python
        scraper = IMDB()
        genre = "Adventure"
        genre_data = scraper.scrape_genre_movies(genre)

        json_data = json.dumps(genre_data, indent=4)
        print(json_data)
        ```
        Return\n
        ```python
        return
        {
            "data": movie_data,
        }
        ```
        """
        try:
            url = "https://www.imdb.com/search/title/?genres={}&sort=user_rating,desc&title_type=feature&num_votes=25000,&pf_rd_m=A2FGELUUNOQJNL&pf_rd_p=5aab685f-35eb-40f3-95f7-c53f09d542c3&pf_rd_r=N97GEQS6R7J9EV7V770D&pf_rd_s=right-6&pf_rd_t=15506&pf_rd_i=top&ref_=chttp_gnr_16"
            formatted_url = url.format(genre)
            print(formatted_url)

            resp = requests.get(formatted_url, headers=self.headers)
            content = BeautifulSoup(resp.content, "lxml")
            genres = [
                "Adventure",
                "Animation",
                "Biography",
                "Comedy",
                "Crime",
                "Drama",
                "Family",
                "Fantasy",
                "Film-Noir",
                "History",
                "Horror",
                "Music",
                "Musical",
                "Mystery",
                "Romance",
                "Sci-Fi",
                "Sport",
                "Thriller",
                "War",
                "Western",
            ]

            movie_list = []

            for movie in content.select(".lister-item-content"):
                try:
                    title_info = movie.select(".lister-item-header")[0]
                    movie_name = title_info.a.text
                    year = title_info.find(
                        "span", class_="lister-item-year"
                    ).text.strip("()")
                    certificate = movie.select(".certificate")[0].text.strip()
                    time = movie.select(".runtime")[0].text.strip()
                    genre = movie.select(".genre")[0].text.strip()
                    rating = movie.select(".ratings-imdb-rating")[0].strong.text.strip()
                    simple_desc = movie.select(".text-muted")[2].text.strip()
                    votes = (
                        movie.select(".sort-num_votes-visible")[0]
                        .text.strip()
                        .split("|")[0]
                        .replace("Votes:", "")
                        .strip()
                    )

                    data = {
                        "title": movie_name,
                        "year": year,
                        "certificate": certificate,
                        "time": time,
                        "genre": genre,
                        "rating": rating,
                        "simple_desc": simple_desc,
                        "votes": votes,
                    }
                    movie_list.append(data)
                except IndexError:
                    continue

            return movie_list

        except requests.exceptions.RequestException as e:
            return None

    def top_rated_shows(self):
        """
        Class: `IMDB`\n
        Retrieves the top-rated TV shows listed on IMDb.
        Example:
        ```python
        top_shows = IMDB()
        result = top_shows.top_rated_shows()
        ```
        Returns:
        ```js
        [
            {
                "title":"1. Breaking Bad",
                "year":"2008-2013",
                "episode":"62",
                "rating":"9.5"
            }
        ]
        ```
        """
        try:
            url = "https://www.imdb.com/chart/toptv/?ref_=nv_tvv_250"
            html_text = requests.get(url, headers=self.headers).text
            soup = BeautifulSoup(html_text, "lxml")
            shows_container = soup.find(
                "ul",
                {
                    "class": "ipc-metadata-list ipc-metadata-list--dividers-between sc-3a353071-0 wTPeg compact-list-view ipc-metadata-list--base"
                },
            )
            shows = []

            for items in shows_container.find_all("li"):
                title = items.find("h3").text
                years = items.find(
                    "span", {"class": "sc-14dd939d-6 kHVqMR cli-title-metadata-item"}
                )
                eps = years.next_sibling.text.split()[0]
                rating = items.find(
                    "span",
                    {
                        "class": "ipc-rating-star ipc-rating-star--base ipc-rating-star--imdb ratingGroup--imdb-rating"
                    },
                ).text

                data = {
                    "title": title,
                    "year": years.text,
                    "episode": eps,
                    "rating": rating,
                }

                shows.append(data)

            return shows
        except requests.exceptions.RequestException as e:
            return None


# src/scrape_up/instagram/__init__.py

from scrape_up.instagram.users import Users

__all__=["Users"]

# src/scrape_up/instagram/users.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.firefox import GeckoDriverManager
from selenium.webdriver.firefox.service import Service

driver = webdriver.Firefox(service=Service(GeckoDriverManager().install()))


class User:
    def __init__(self, username: str):
        self.username = username

    def user_details(self):
        """
        user = User(username=" ")
        print(user.user_details())
        """
        try:
            driver.get(f"https://www.instagram.com/{self.username}/")
            wait = WebDriverWait(driver, 180)
            account_details = wait.until(
                EC.presence_of_all_elements_located(
                    (By.XPATH, '//span[@class="_ac2a"]')
                )
            )
            return {
                "Number of Posts:": account_details[0].text,
                "Number of Followers:": account_details[1].text,
                "Number of Following:": account_details[2].text,
            }

        except Exception as e:
            message = f"{self.username} not found!"
            return {"data": None, "message": message}
        finally:
            driver.quit()


# src/scrape_up/internshala/__init__.py

from scrape_up.internshala.internships import Internships

__all__ = ["Internships"]


# src/scrape_up/internshala/internships.py

import requests
from bs4 import BeautifulSoup


class Internshala:
    """
    Create an object for the 'Internships' class :\n
    ```python
    scraper = Internships()
    ```
    | Methods          | Details                                                              |
    | ---------------- | -------------------------------------------------------------------- |
    | `.internships()` | Scrapes and returns a list of dictionaries representing internships. |
    | `.jobs()`        | Scrapes and returns a list of dictionaries representing jobs.        |
    """

    def __init__(self, search_type):
        self.base_url = "https://internshala.com/"
        self.search_type = search_type

    def __scrape_page(self, url):
        try:
            html_text = requests.get(url)
            html_text.raise_for_status()
            return html_text.text
        except requests.exceptions.RequestException as e:
            raise Exception(f"An error occurred while fetching the page: {str(e)}")

    def __parse_page(self, html):
        try:
            soup = BeautifulSoup(html, "lxml")
            return soup
        except Exception as e:
            raise Exception(f"An error occurred while parsing the page: {str(e)}")

    def internships(self):
        """
        Fetches the jobs data.\n
        Class - `Jobs`
        Example:
        ```
        search = Internshala(search_type="machine learning")
        search.jobs()
        ```
        Returns:
        ```js
        [
            {
            "title":"Machine Learning Instructor",
            "company":"Future Skills",
            "location":"Hyderabad",
            "CTC":"₹  3 - 5 LPA",
            "experience(in years)":"0-5"
            }
        ]
        ```
        """
        try:
            self.search_type = self.search_type.replace(" ", "%20")
            url = self.base_url + "internships/" + "keywords-" + self.search_type
            html = self.__scrape_page(url)
            page = self.__parse_page(html)
            internships = []

            internships_container = page.find_all("div", class_="individual_internship")

            if not internships_container:
                return {"message": "No internships found"}

            else:
                for internship in internships_container:
                    title = internship.find(
                        "h3", class_="heading_4_5 profile"
                    ).text.strip()
                    company = internship.find(
                        "h4", class_="heading_6 company_name"
                    ).text.strip()
                    location = internship.find("a", class_="location_link").text.strip()
                    other_details = internship.find_all(class_="item_body")
                    duration = (
                        other_details[1].text.strip()
                        if len(other_details) > 2
                        else "N/A"
                    )
                    stipend_element = internship.find("span", class_="stipend")
                    stipend = stipend_element.text.strip() if stipend_element else "N/A"

                    internship_data = {
                        "title": title,
                        "company": company,
                        "location": location,
                        "duration": duration,
                        "stipend": stipend,
                    }

                    internships.append(internship_data)

                return {
                    "data": internships,
                    "message": "Internships are now fetched",
                }
        except Exception as e:
            raise Exception(f"An error occurred while scraping internships: {str(e)}")

    def jobs(self):
        """
        Fetches the jobs data.\n
        Class - `Jobs`
        Example:
        ```
        search = Internshala(search_type="machine learning")
        search.internships()
        ```
        Returns:
        ```js
        [
            {
                "title":"Data Science",
                "company":"Vedasis Analytics",
                "location":"Work From Home",
                "duration":"3 Months",
                "stipend":"₹ 5,000 /month"
            }
        ]
        ```
        """
        try:
            self.search_type = self.search_type.replace(" ", "%20")
            url = self.base_url + "jobs/" + "keywords-" + self.search_type
            html = self.__scrape_page(url)
            page = self.__parse_page(html)
            jobs = []

            jobs_container = page.find("div", {"id": "internship_list_container_1"})

            if not jobs_container.text:
                return {"message": "No jobs found"}
            else:
                for item in jobs_container.find_all(
                    "div",
                    {
                        "class": "container-fluid individual_internship visibilityTrackerItem"
                    },
                ):
                    title = item.find(
                        "h3", {"class": "heading_4_5 profile"}
                    ).text.strip()
                    company = item.find(
                        "h4", {"class": "heading_6 company_name"}
                    ).text.strip()
                    location = item.find("p", {"id": "location_names"}).text.strip()
                    ctc = item.find("div", {"class": "item_body salary"}).text.strip()
                    experience = (
                        item.find("div", {"class": "item_body desktop-text"})
                        .text.strip()
                        .split()[0]
                    )

                    job_data = {
                        "title": title,
                        "company": company,
                        "location": location,
                        "CTC": ctc,
                        "experience(in years)": experience,
                    }

                    jobs.append(job_data)

                return {
                    "data": jobs,
                    "message": "Jobs are now fetched",
                }
        except Exception as e:
            raise Exception(f"An error occurred while scraping jobs: {str(e)}")


internship = Internshala(search_type="machine learning")
print(internship.internships())


# src/scrape_up/kindle_bookstore/__init__.py

from kindle import AmazonKindle

__all__ = ["AmazonKindle"]


# src/scrape_up/kindle_bookstore/kindle.py

import requests
from bs4 import BeautifulSoup


class AmazonKindle:
    """
    Create an instance of `Book` class.
    ```python
    books = AmazonKindle()
    ```
    | Methods          | Details                                                |
    | ---------------- | ------------------------------------------------------ |
    | `.bestsellers()` | Returns the list of best-selling books on AmazonKindle |
    | `.topbooks()`    | Returns the list of top books on AmazonKindle          |
    """
    def __init__(self):
        pass

    def bestsellers(self):
        """
        Class - `Book`\n
        Example -\n
        ```python
        books = AmazonKindle()
        book.bestsellers()
        ```
        Return\n
        ```js
        [
            {
                "Book_Title":"101 Brain Booster Activity Book: Fun Activity Book For Children",
                "Author_Name":"Wonder House Books",
                "Stars":"N/A",
                "Price":"₹149.00",
                "URL":"https://www.amazon.in/101-Brain-Booster-ActivityBook/dp/9388369793/ref=zg_bs_g_books_sccl_20/000-0000000-0000000?psc=1"
            }
        ]
        ```
        """
        try:
            url = "https://www.amazon.in/gp/bestsellers/books/"
            response = requests.get(url)
            page_contents = response.text
            doc = BeautifulSoup(page_contents, "html.parser")

            book_title_tags = doc.find_all("div", {"class": "zg-grid-general-faceout"})
            author_name_tags = doc.find_all("div", {"class": "zg-grid-general-faceout"})
            rating = "a-icon a-icon-star-small a-star-small-4-5 aok-align-top"
            rating_tags = doc.find_all("i", {"class": rating})
            book_price_tags = doc.find_all("div", {"class": "zg-grid-general-faceout"})
            book_url_tag = doc.find_all("div", {"class": "zg-grid-general-faceout"})

            all_books = []

            for i in range(len(book_title_tags)):
                book_title = book_title_tags[i].find("span").text
                author_name = (
                    author_name_tags[i]
                    .find("div", {"class": "a-row a-size-small"})
                    .text
                )

                # Check if the index is within the range of rating_tags
                if i < len(rating_tags):
                    star_rating = rating_tags[i].find("span").text
                else:
                    star_rating = (
                        "N/A"  # Assign a default value if rating is not available
                    )

                book_price = (
                    book_price_tags[i].find("span", {"class": "p13n-sc-price"}).text
                )
                book_url = (
                    "https://www.amazon.in"
                    + book_url_tag[i].find("a", {"class": "a-link-normal"})["href"]
                )

                book_details = {
                    "Book_Title": book_title,
                    "Author_Name": author_name,
                    "Stars": star_rating,
                    "Price": book_price,
                    "URL": book_url,
                }

                all_books.append(book_details)

            return all_books

        except Exception as e:
            return None

    def topbooks(self):
        """
        Class - `Book`\n
        Example -\n
        ```python
        books = AmazonKindle()
        book.topbooks()
        ```
        Return\n
        ```js
        [
            {
                "Title":"Master Your Emotions: A Practical Guide to Overcome Negativity and Better Manage Your Feelings (Mastery Series Book 1)",
                "Rating(out of 5)":"4.5",
                "Price(in rupees)":"₹199.00",
                "Link":"https://www.amazon.in//Master-Your-Emotions-Practical-Negativity-ebook/dp/B07CX8H6YH/ref=zg_bs_g_digital-text_sccl_1/260-0078955-4830376?psc=1"
            }
            ...
        ]
        ```
        """
        try:
            all_books = []
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36"
            }
            for page_link in [
                "https://www.amazon.in/gp/bestsellers/digital-text/ref=zg_bs_nav_0",
                "https://www.amazon.in/gp/bestsellers/digital-text/ref=zg_bs_pg_2_digital-text?ie=UTF8&pg=2",
            ]:
                webpage = requests.get(page_link, headers=headers).text
                soup = BeautifulSoup(webpage, "lxml")
                books_container = soup.find(
                    "div", {"class": "p13n-gridRow _cDEzb_grid-row_3Cywl"}
                )
                for items in books_container.find_all("div", {"id": "gridItemRoot"}):
                    title = items.find(
                        "div", {"class": "_cDEzb_p13n-sc-css-line-clamp-1_1Fn1y"}
                    ).text
                    stars = items.find("span", {"class": "a-icon-alt"})
                    if stars is None:
                        stars = "Rating not available"

                    else:
                        stars = stars.text.split()[0]
                    price = items.find("span", {"class": "a-size-base"}).text
                    link = items.find("a", {"class": "a-link-normal"}, href=True)
                    url = "https://www.amazon.in/" + link["href"]
                    books = {
                        "Title": title,
                        "Rating(out of 5)": stars,
                        "Price(in rupees)": price,
                        "Link": url,
                    }
                    all_books.append(books)
            return all_books

        except Exception as e:
            return None

# src/scrape_up/kooapp/__init__.py

from scrape_up.kooapp.users import KooUser

__all__=["KooUser"]

# src/scrape_up/kooapp/users.py

import requests
import json
from bs4 import BeautifulSoup
from datetime import datetime


class KooUser:
    """
    class - `KooUser`

    Example:
    ```py
    user = KooUser('krvishal')
    avatar = user.get_avatar_url()
    ```
    Returns:
    ```
    >>> 'https://images.kooapp.com/transcode_input/8074390/profile16851706949505ew9y8.jpg'
    """

    def __init__(self, username: str) -> None:
        self.username = username

    def __str__(self) -> str:
        return f"The username is: {self.username}"

    @property
    def profile_url(self) -> str:
        return f"https://www.kooapp.com/profile/{self.username}"

    def __scrape_page(self) -> requests.Response:
        try:
            res = requests.get(self.profile_url)
            if res.status_code == 404:
                raise Exception(f"User not found with username: {self.username}")
            return res
        except Exception as e:
            raise Exception(f"An error occurred while fetching the page: {str(e)}")

    def __parse_page(self) -> dict:
        res = self.__scrape_page()
        try:
            page = BeautifulSoup(res.text, "html.parser")
            data = json.loads(page.find("script", attrs={"id": "__NEXT_DATA__"}).text)
            return data
        except Exception as e:
            raise Exception(f"An error occurred while parsing the page: {str(e)}")

    def __parse_profile_data(self) -> dict:
        data = self.__parse_page()
        try:
            userdata = (
                data.get("props")
                .get("pageProps")
                .get("initialState")
                .get("profileReducers")
                .get("profileItems")
            )
            return userdata
        except Exception as e:
            raise Exception(f"An error occured while parsing the data: {e}")

    def get_name(self) -> str:
        """
        Fetch the name of the user.
        """
        userdata = self.__parse_profile_data()
        name = userdata.get("name")
        return {
            "data": name or None,
            "message": f"Name of the user {self.username} is {name}"
            if name
            else f"No name found for the user {self.username}",
        }

    def get_bio(self) -> str:
        """
        Fetch the bio dsecription of the user.
        """
        userdata = self.__parse_profile_data()
        bio = userdata.get("description")
        return {
            "data": bio or None,
            "message": f"Bio found for the user {self.username}"
            if bio
            else f"No bio found for the user {self.username}",
        }

    def get_avatar_url(self) -> str:
        """
        Fetch the avatar url of the user.
        """
        userdata = self.__parse_profile_data()
        avatar = userdata.get("profileImage")
        return {
            "data": avatar or None,
            "message": f"Avatar found for the user {self.username}"
            if avatar
            else f"Avatar not found for the user {self.username}",
        }

    def followers(self) -> int:
        """
        Fetch the number of followers of the Koo user.
        """
        userdata = self.__parse_profile_data()
        followers = userdata.get("followerCount")
        return {
            "data": followers,
            "message": f"There are {followers} follower(s) of the user {self.username}",
        }

    def following(self) -> int:
        """
        Fetch the number of following of the Koo user.
        """
        userdata = self.__parse_profile_data()
        following = userdata.get("followingCount")
        return {
            "data": following,
            "message": f"There are {following} user(s) followed by {self.username}",
        }

    def get_social_profiles(self) -> dict[str, str]:
        """
        Fetch all the listed social media profiles of user.
        """
        userdata = self.__parse_profile_data()
        profiles = {
            s_media: handle
            for s_media, handle in userdata.get("socialProfile").items()
            if handle
        }
        return {
            "data": profiles or None,
            "message": f"Found {len(profiles)} social profiles for the user {self.username}"
            if profiles
            else f"No social profiles found for the user {self.username}",
        }

    def get_profession(self) -> str:
        """
        Fetch the profession of the user.
        """
        userdata = self.__parse_profile_data()
        profession = userdata.get("title")
        return {
            "data": profession or None,
            "message": f"Profession found for the user {self.username}"
            if profession
            else f"No profession found for the user {self.username}",
        }


# src/scrape_up/leetcode/__init__.py

from scrape_up.leetcode.leetcode_scraper import LeetCodeScraper

__all__= ['LeetCodeScraper']

# src/scrape_up/leetcode/leetcode_scraper.py

import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
from bs4 import BeautifulSoup


class LeetCode:
    """
    ```python
    LeetCode = LeetCode(username="nikhil25803")\n
    ```
    **username** might not require in few methods\n
    | Methods                    | Details                                          |
    | -------------------------- | ------------------------------------------------ |
    | `.scrape_rank()`   | Used to scrape the rank of a user on LeetCode.   |
    | `.scrape_rating()` | Used to scrape the rating of a user on LeetCode. |
    | `.get_problems_solved()`   | Used to scrape total problems solved by a user on LeetCode.   |
    | `.get_solved_by_difficulty()` | Used to scrape difficulty wise problems solved by a user on LeetCode. |
    | `.get_github_link()`   | Used to scrape github link of a user on LeetCode.   |
    | `.get_linkedin_link()` | Used to scrape linkedin link of a user on LeetCode. |
    | `.get_community_stats()` | Used to scrape community stats of a user on LeetCode. |
    """

    def __init__(self, username: str = ""):
        self.username = username
        self.user_profile = self.__scrape_user_profile()

        self.chrome_options = Options()
        self.chrome_options.add_argument("--headless")
        self.chrome_options.add_argument("--window-size=1920,1080")
        self.chrome_options.add_argument("--disable-gpu")
        self.chrome_options.add_argument("--no-sandbox")
        self.chrome_options.add_argument("--disable-dev-shm-usage")
        self.chrome_options.add_argument("--disable-extensions")
        self.chrome_options.add_argument("--disable-logging")
        self.chrome_options.add_argument("--log-level=3")
        self.chrome_options.add_argument("--silent")
        self.chrome_options.add_argument("--blink-settings=imagesEnabled=false")
        self.chrome_options.add_experimental_option(
            "excludeSwitches", ["enable-logging"]
        )

        self.service = Service(ChromeDriverManager().install())

        self.driver = webdriver.Chrome(
            service=self.service, options=self.chrome_options
        )
        self.wait = WebDriverWait(self.driver, 100)

    def __scrape_user_profile(self):
        url = f"https://leetcode.com/{self.username}"
        data = requests.get(url)
        data.raise_for_status()
        soup = BeautifulSoup(data.text, "html.parser")
        return soup

    def scrape_rank(self):
        """
        Method to scrape the rank of the user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.scrape_rank()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return {"data": None, "message": message}

        soup = self.user_profile
        try:
            rank_element = soup.find(
                "span", {"class": "ttext-label-1 dark:text-dark-label-1 font-medium"}
            )
            rank = rank_element.text.strip() if rank_element else "Not Ranked"

            return {
                "data": rank,
                "message": f"Successfully scraped rank for user '{self.username}'",
            }
        except:
            return {
                "data": None,
                "message": f"Failed to scrape rank for user '{self.username}'",
            }

    def scrape_rating(self):
        """
        Method to scrape the contest rating of the user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.scrape_rating()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return {"data": None, "message": message}

        soup = self.user_profile
        try:
            rating_element = soup.find(
                "div",
                {
                    "class": "text-label-1 dark:text-dark-label-1 flex items-center text-2xl"
                },
            )
            rating = rating_element.text.strip() if rating_element else "No Rating"

            return {
                "data": rating,
                "message": f"Successfully scraped rating for user '{self.username}'",
            }
        except:
            return {
                "data": None,
                "message": f"Failed to scrape rating for user '{self.username}'",
            }

    def get_problems_solved(self):
        """
        Method to scrape the number of problem solved by the user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.get_problems_solved()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return {"data": None, "message": message}

        soup = self.user_profile
        try:
            total_problems = soup.find(
                "div",
                {
                    "class": "text-[24px] font-medium text-label-1 dark:text-dark-label-1"
                },
            )
            return {
                "data": total_problems.text,
                "message": f"Found total problems solved for user '{self.username}'",
            }
        except:
            message = f"Failed to scrape total problems for user '{self.username}'"
            return {"data": None, "message": message}

    def get_solved_by_difficulty(self):
        """
        Method to scrape the number of problem solved by the user of each Easy, Medium and Hard\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.get_solved_by_difficulty()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return {"data": None, "message": message}

        soup = self.user_profile
        try:
            difficulty_wise = soup.find_all(
                "span",
                {
                    "class": "mr-[5px] text-base font-medium leading-[20px] text-label-1 dark:text-dark-label-1"
                },
            )

            solved = {
                "easy": difficulty_wise[0].text,
                "medium": difficulty_wise[1].text,
                "hard": difficulty_wise[2].text,
            }

            return {
                "data": solved,
                "message": f"Found difficulty wise problems solved for user '{self.username}'",
            }
        except:
            message = (
                f"Failed to scrape difficulty wise problems for user '{self.username}'"
            )
            return {"data": None, "message": message}

    def get_github_link(self):
        """
        Method to scrape the GitHub link of the Useleetoce user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.get_github_link()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return message

        soup = self.user_profile
        try:
            links = soup.find_all(
                "a",
                {
                    "class": "hover:text-label-1 dark:hover:text-dark-label-1 cursor-pointer"
                },
            )

            for link in links:
                if "github" in link["href"]:
                    return {
                        "data": link["href"],
                        "message": f"Found github link for user '{self.username}'",
                    }

            message = f"No github link found for user '{self.username}'"
            return {"data": None, "message": message}
        except:
            message = f"Failed to scrape github link for user '{self.username}'"
            return {"data": None, "message": message}

    def get_linkedin_link(self):
        """
        Method to scrape the LinkedIn link of the Useleetoce user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.get_linkedin_link()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return message

        soup = self.user_profile
        try:
            links = soup.find_all(
                "a",
                {
                    "class": "hover:text-label-1 dark:hover:text-dark-label-1 cursor-pointer"
                },
            )

            for link in links:
                if "linkedin" in link["href"]:
                    return {
                        "data": link["href"],
                        "message": f"Found linkedin link for user '{self.username}'",
                    }

            message = f"No linkedin link found for user '{self.username}'"
            return {"data": None, "message": message}
        except:
            message = f"Failed to scrape linkedin link for user '{self.username}'"
            return {"data": None, "message": message}

    def get_community_stats(self):
        """
        Method to scrape the community stat of the user\n
        Required argument - **username**
        Example\n
        ```python
        user = LeetCode(username="nikhil25803")
        user.get_community_stats()
        ```
        \n
        """
        if self.username == "":
            message = f"username is not given"
            return message

        soup = self.user_profile
        try:
            stats = soup.find_all(
                "div",
                {"class": "flex items-center space-x-2 text-[14px]"},
            )

            stats = {
                "views": stats[0].find_all("div")[-1].text,
                "solution": stats[1].find_all("div")[-1].text,
                "discuss": stats[2].find_all("div")[-1].text,
                "reputation": stats[3].find_all("div")[-1].text,
            }
            return {"data": stats, "message": f"Found stats for user '{self.username}'"}
        except:
            message = f"Failed to scrape community stats for user '{self.username}'"
            return {"data": None, "message": message}

    def get_problems(self, difficulty="", tags=[], search_key=""):
        """
        Method to scrape problems with keyword `difficulty` and `tags`\n
        Example\n
        ```python
        LeetCode = LeetCode()
        LeetCode.get_problems(difficulty="medium", tags=["dynamic-programming"],
        search_key="palindrome")
        ```
        \n
        """
        url = "https://leetcode.com/problemset/all/?"

        if difficulty != "":
            url += "difficulty=" + difficulty.upper()

        if len(tags) > 0:
            url += "&topicSlugs=" + tags[0]

            for i in range(1, len(tags)):
                url += "%2C" + tags[i]

        if search_key != "":
            url += "&search=" + search_key

        self.driver.get(url)
        self.wait.until(EC.visibility_of_element_located((By.CLASS_NAME, "space-x-1")))
        soup = BeautifulSoup(self.driver.page_source, "html.parser")

        try:
            problems = soup.find_all(
                "a",
                {"class": "h-5 hover:text-blue-s dark:hover:text-dark-blue-s"},
            )
            problems_premium = soup.find_all(
                "a",
                {
                    "class": "h-5 hover:text-blue-s dark:hover:text-dark-blue-s opacity-60"
                },
            )
            problems.extend(problems_premium)

            problems_list = []

            for problem in problems:
                problem_title = problem.text
                problem_url = "https://leetcode.com" + problem["href"]
                problems_list.append({"title": problem_title, "link": problem_url})

            return {"data": problems_list, "message": f"Found problems list"}
        except:
            message = f"Failed to scrape problems for given filters"
            return {"data": None, "message": message}

    def get_contests(self):
        """
        Method to scrape the list of active contests on Leetcode\n
        Example\n
        ```python
        LeetCode = LeetCode()
        LeetCode.get_contests()
        ```
        \n
        """
        url = "https://leetcode.com/contest/"
        data = requests.get(url)
        data.raise_for_status()
        soup = BeautifulSoup(data.text, "html.parser")

        try:
            contests = soup.find(
                "div",
                {"class": "swiper-wrapper"},
            ).find_all("div", {"class": "h-[54px]"})

            contest_list = []
            for contest in contests:
                contest_list.append(
                    {
                        "title": contest.div.text,
                        "date": contest.text.replace(contest.div.text, ""),
                    }
                )

            return {"data": contest_list, "message": f"Found contest list"}
        except:
            message = f"Failed to scrape contest details"
            return {"data": None, "message": message}

    def get_daily_challenge(self):
        """
        Method to scrape the list of daily challanges on LeetCode\n
        Example\n
        ```python
        LeetCode = LeetCode()
        LeetCode.get_daily_challenge()
        ```
        \n
        """
        try:
            problemset_url = "https://leetcode.com/problemset/all/"
            self.driver.get(problemset_url)
            self.wait.until(EC.visibility_of_element_located((By.CLASS_NAME, "flex-0")))
            soup = BeautifulSoup(self.driver.page_source, "html.parser")

            daily = soup.find(
                "a",
                {"class": "h-5 hover:text-blue-s dark:hover:text-dark-blue-s"},
            )
            daily_link = "https://leetcode.com" + daily["href"]

            self.driver.get(daily_link)
            self.wait.until(EC.visibility_of_element_located((By.CLASS_NAME, "_1l1MA")))
            soup = BeautifulSoup(self.driver.page_source, "html.parser")

            daily_challenge = {}
            daily_challenge["title"] = daily.text

            difficulty = soup.find("div", class_="rounded-[21px]").text
            daily_challenge["difficulty"] = difficulty

            p_tags = soup.find("div", class_="_1l1MA").find_all("p")
            pre_tags = soup.find("div", class_="_1l1MA").find_all("pre")

            i = 0
            description = ""
            while "Example" not in p_tags[i].text:
                description += p_tags[i].text
                i += 1
            daily_challenge["description"] = description

            examples = []
            j = 0

            while "Example" in p_tags[i].text:
                examples.append(pre_tags[j].text)
                i += 1
                j += 1
            daily_challenge["examples"] = examples

            constraints = soup.find(
                "div",
                {"class": "_1l1MA"},
            ).find("ul")
            daily_challenge["constraints"] = constraints.text

            return {
                "data": daily_challenge,
                "message": f"Found daily challenge problem",
            }
        except:
            return None

    def __scrape_solved_count(self):
        username = self.username
        url = f"https://leetcode.com/{username}"
        driver = webdriver.Chrome(options=self.chrome_options)
        driver.get(url)

        html = driver.page_source
        soup = BeautifulSoup(html, "html.parser")
        try:
            solved_element = soup.find(
                "div", {"data-cy": "header__side__progress__solved-question-count"}
            )
            solved_count = solved_element.text.strip() if solved_element else "Unknown"

            return {
                "solved_count": solved_count,
                "message": f"Successfully scraped solved count for user '{username}'",
            }
        except:
            return {
                "solved_count": None,
                "message": f"Failed to scrape solved count for user '{username}'",
            }

    def __submissions_count(self):
        username = self.username
        url = f"https://leetcode.com/{username}"
        driver = webdriver.Chrome(options=self.chrome_options)
        driver.get(url)

        html = driver.page_source
        soup = BeautifulSoup(html, "html.parser")
        try:
            submissions_element = soup.find(
                "div",
                {"data-cy": "header__side__progress__total-submission-count"},
            )
            submissions_count = (
                submissions_element.text.strip() if submissions_element else "Unknown"
            )

            return {
                "submissions_count": submissions_count,
                "message": f"Successfully scraped submissions count for user '{username}'",
            }
        except:
            return {
                "submissions_count": None,
                "message": f"Failed to scrape submissions count for user '{username}'",
            }


# src/scrape_up/linkedIn/__init__.py

#Profile list can contain as many as public profiles
Profile_List = ['reidhoffman', 'another_profile']

#Import and Install Python Scrapy 

# src/scrape_up/linkedIn/linkedInspider.py

import scrapy

class LinkedInPeopleProfileSpider(scrapy.Spider):
    name = "linkedin_people_profile"

    custom_settings = {
        'FEEDS': { 'data/%(name)s_%(time)s.jsonl': { 'format': 'jsonlines',}}
    }

    def start_requests(self):
        profile_list = ['reidhoffman']  # Any public profile username can be added
        for profile in profile_list:
            linkedin_people_url = f'https://www.linkedin.com/in/{profile}/' 
            yield scrapy.Request(url=linkedin_people_url, callback=self.parse_profile, meta={'profile': profile, 'linkedin_url': linkedin_people_url})

    def parse_profile(self, response):
        item = self.extract_profile_info(response)

        item['about'] = self.extract_about_section(response)
        item['experience'] = self.extract_experience_section(response)
        item['education'] = self.extract_education_section(response)

        yield item

    def extract_profile_info(self, response):
        item = {}
        item['profile'] = response.meta['profile']
        item['url'] = response.meta['linkedin_url']

        summary_box = response.css("section.top-card-layout")
        item['name'] = summary_box.css("h1::text").get().strip()
        item['description'] = summary_box.css("h2::text").get().strip()

        try:
            item['location'] = summary_box.css('div.top-card__subline-item::text').get()
        except:
            item['location'] = summary_box.css('span.top-card__subline-item::text').get().strip()
            if 'followers' in item['location'] or 'connections' in item['location']:
                item['location'] = ''

        item['followers'] = ''
        item['connections'] = ''

        for span_text in summary_box.css('span.top-card__subline-item::text').getall():
            if 'followers' in span_text:
                item['followers'] = span_text.replace(' followers', '').strip()
            if 'connections' in span_text:
                item['connections'] = span_text.replace(' connections', '').strip()

        return item

    def extract_about_section(self, response):
        about_section = response.css('section.summary div.core-section-container__content p::text')
        return about_section.get(default='')

    def extract_experience_section(self, response):
        experience_blocks = response.css('li.experience-item')
        experience_list = []

        for block in experience_blocks:
            experience = {}
            experience['organisation_profile'] = block.css('h4 a::attr(href)').get(default='').split('?')[0]
            experience['location'] = block.css('p.experience-item__location::text').get(default='').strip()

            try:
                experience['description'] = block.css('p.show-more-less-text__text--more::text').get().strip()
            except Exception as e:
                print('experience --> description', e)
                try:
                    experience['description'] = block.css('p.show-more-less-text__text--less::text').get().strip()
                except Exception as e:
                    print('experience --> description', e)
                    experience['description'] = ''

            try:
                date_ranges = block.css('span.date-range time::text').getall()
                if len(date_ranges) == 2:
                    experience['start_time'] = date_ranges[0]
                    experience['end_time'] = date_ranges[1]
                    experience['duration'] = block.css('span.date-range__duration::text').get()
               
                elif len(date_ranges) == 1:
                    experience['start_time'] = date_ranges[0]
                    experience['end_time'] = 'present'
                    experience['duration'] = block.css('span.date-range__duration::text').get()
                
            except Exception as e:
                print('experience --> time ranges', e)
                experience['start_time'] = ''
                experience['end_time'] = ''
                experience['duration'] = ''
                experience_list.append(experience)

        return experience_list

def extract_education_section(self, response):
    education_blocks = response.css('li.education__list-item')
    education_list = []

    for block in education_blocks:
        education = {}
        education['organisation'] = block.css('h3::text').get(default='').strip()
        education['organisation_profile'] = block.css('a::attr(href)').get(default='').split('?')[0]

        try:
            education['course_details'] = ''
            for text in block.css('h4 span::text').getall():
                education['course_details'] = education['course_details'] + text.strip() + ' '
            education['course_details'] = education['course_details'].strip()
        except Exception as e:
            print("education --> course_details", e)
            education['course_details'] = ''

        education['description'] = block.css('div.education__item--details p::text').get(default='').strip()

        try:
            date_ranges = block.css('span.date-range time::text').getall()
            if len(date_ranges) == 2:
                education['start_time'] = date_ranges[0]
                education['end_time'] = date_ranges[1]
            elif len(date_ranges) == 1:
                education['start_time'] = date_ranges[0]
                education['end_time'] = 'present'
        except Exception as e:
            print("education --> time_ranges", e)
            education['start_time'] = ''
            education['end_time'] = ''

        education_list.append(education)

    return education_list


# src/scrape_up/medium/__init__.py

from scrape_up.medium.trending import Trending
from scrape_up.medium.publication import Publication
from scrape_up.medium.user import User

__all__ = ["Trending", "Publication", "User"]

# src/scrape_up/medium/publication.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time


options = Options()
options.add_argument("--headless")
options.add_argument("--log-level=3")
options.add_experimental_option("excludeSwitches", ["enable-logging"])
driver = webdriver.Chrome(options=options)


class Publication:
    def __init__(self, link):
        self.link = link

    def get_articles(self):
        """
        Class - `Publication`
        Example
        ```python
        publication = medium.Publication("https://towardsdatascience.com")
        articles = publication.get_articles()
        for article in articles:
            print(article) #For better readability/clarity
        ```
        Returns the articles of the publication which are arranged in the form of a list
        """
        try:
            articles = []
            link = self.link
            driver.get(link)
            scroll_pause = 0.5
            # Get scroll height
            last_height = driver.execute_script(
                "return document.documentElement.scrollHeight"
            )
            run_time, max_run_time = 0, 1
            while True:
                iteration_start = time.time()
                # Scroll down to bottom
                driver.execute_script(
                    "window.scrollTo(0, 1000*document.documentElement.scrollHeight);"
                )

                # Wait to load page
                time.sleep(scroll_pause)

                # Calculate new scroll height and compare with last scroll height
                new_height = driver.execute_script(
                    "return document.documentElement.scrollHeight"
                )
                scrolled = new_height != last_height
                timed_out = run_time >= max_run_time
                if scrolled:
                    run_time = 0
                    last_height = new_height
                elif not scrolled and not timed_out:
                    run_time += time.time() - iteration_start
                elif not scrolled and timed_out:
                    break
            elements = driver.find_elements(By.XPATH, "//h2 | //h3")
            for x in elements:
                articles.append(x.text)
            return articles
        except:
            return "page/publication not found."


# publication = Publication("https://pub.towardsai.net")
# articles = publication.get_articles_list()
# for article in articles:
#     print(article)


# src/scrape_up/medium/trending.py

import requests
from bs4 import BeautifulSoup as bs

headers = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
}  # mimics a browser's request


class Trending:
    def get_trending():
        """
        Class - `Trending`
        Example
        ```python
        trending = Trending.get_trending()
        for trend in trending:
            print(trend) #For better readability/clarity
        ```
        Returns a list of trending titles

        """
        try:
            titles = []
            r = requests.get("https://medium.com/", headers=headers)
            soup = bs(r.text, "html.parser")
            elements = soup.select('h2[class^="by j"]')
            for x in elements:
                titles.append(x.text)
            return titles

        except:
            return {"data": None, "message": "Something went wrong! Try again!"}


# src/scrape_up/medium/user.py

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time

options = Options()
options.add_argument("--headless")
options.add_argument("--log-level=3")
options.add_experimental_option("excludeSwitches", ["enable-logging"])
driver = webdriver.Chrome(options=options)


class User:
    def __init__(self, username):
        self.username = username

    def get_articles(self):
        """
        Class `Users`
        Example:
        ```python
        user = medium.User(username='karthikbhandary2')
        article_titles = user.get_articles()
        for article in article_titles:
            print(article) # For better readability/clarity
        ```
        Returns a list of the titles.
        """
        try:
            titles = []
            username = self.username
            driver.get(f"https://{username}.medium.com")
            scroll_pause = 0.5
            # Get scroll height
            last_height = driver.execute_script(
                "return document.documentElement.scrollHeight"
            )
            run_time, max_run_time = 0, 1
            while True:
                iteration_start = time.time()
                # Scroll down to bottom
                driver.execute_script(
                    "window.scrollTo(0, 1000*document.documentElement.scrollHeight);"
                )

                # Wait to load page
                time.sleep(scroll_pause)

                # Calculate new scroll height and compare with last scroll height
                new_height = driver.execute_script(
                    "return document.documentElement.scrollHeight"
                )
                scrolled = new_height != last_height
                timed_out = run_time >= max_run_time
                if scrolled:
                    run_time = 0
                    last_height = new_height
                elif not scrolled and not timed_out:
                    run_time += time.time() - iteration_start
                elif not scrolled and timed_out:
                    break
            elements = driver.find_elements(By.CSS_SELECTOR, "h2")
            for x in elements:
                titles.append(x.text)
            return titles

        except:
            return f"{username} not found."


# src/scrape_up/reddit/__init__.py

from scrape_up.reddit.reddit import Reddit
__all__ = ["Reddit"]

# src/scrape_up/reddit/reddit.py

import requests
from bs4 import BeautifulSoup


class Reddit:
    """
    Create an instance of `Reddit` class.
    ```python
    posts = Reddit()
    ```

    | Methods      | Details                                                                                                                         |
    | ------------ | ------------------------------------------------------------------------------------------------------------------------------- |
    | `.getFeed()` | Returns the posts with title, descriptions, subreddit, subreddit avatar, time, vote and comment count, image, category and link |
    """

    def getFeed(self):
        """
        Class - `Reddit`
        Example:
        ```python
        posts = Reddit()
        posts.getFeed()
        ```
        Returns:
        ```js
        {
            "title": Title of the post
            "description": Description of the post
            "subreddit": subreddit name,
            "subreddit_avatar": subreddit avatar,
            "time": Time the post was posted,
            "vote_count": No. of votes of the post,
            "comment_count": No. of comments of the post,
            "img_url": URL of any image provided in the post,
            "category": Category of the post,
        }
        ```
        """
        url = "https://www.reddit.com/"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            posts_data = {"posts": []}

            posts = soup.find_all("div", attrs={"data-testid": "post-container"})

            for p in posts:
                base = p.find("a", attrs={"data-click-id": "subreddit"})
                subreddit = base["href"]
                subreddit_avatar = base.find("img")["src"]
                title = p.find("h3", class_="_eYtD2XCVieq6emjKBH3m").getText()
                try:
                    desc = p.find("p").getText()
                except:
                    desc = ""
                votes = p.find(
                    "div", class_="_1rZYMD_4xY3gRcSS3p8ODO _3a2ZHWaih05DgAOtvu6cIo"
                ).getText()
                time = p.find("span", class_="_2VF2J19pUIMSLJFky-7PEI").getText()
                comment_count = p.find("span", class_="FHCV02u6Cp2zYL0fhQPsO").getText()
                try:
                    img = p.find("img", alt="Post image")["src"]
                except:
                    img = ""
                try:
                    category = p.find(
                        "span",
                        class_="_1jNPl3YUk6zbpLWdjaJT1r _2VqfzH0dZ9dIl3XWNxs42y aJrgrewN9C8x1Fusdx4hh _1Dl-kvSxyJMWO9nuoTof8N",
                    ).getText()
                except:
                    category = ""
                link = p.find(
                    "a", class_="SQnoC3ObvgnGjWt90zD9Z _2INHSNB8V5eaWp4P0rY_mE"
                )["href"]

                posts_data["posts"].append(
                    {
                        "title": title,
                        "description": desc,
                        "subreddit": subreddit,
                        "subreddit_avatar": subreddit_avatar,
                        "time": time,
                        "vote_count": votes,
                        "comment_count": comment_count,
                        "img_url": img,
                        "category": category,
                        "link": link,
                    }
                )
            return posts_data
        except:
            return None


# src/scrape_up/rottenTomatoes/__init__.py

from rot_tom import RottenTomatoes

__all__ = ["RottenTomatoes"]


# src/scrape_up/rottenTomatoes/rot_tom.py

from bs4 import BeautifulSoup
import requests


class RottenTomatoes:
    """
    Create an instance of `RottenTomatoes` class.
    ```python
    scraper = RottenTomatoes()
    ```
    | Method                       | Details                                                             |
    | ---------------------------- | ------------------------------------------------------------------- |
    | `.top_rated()`               | Returns the top-rated movies listed on the Rotten Tomatoes website. |
    | `.movie_details(movie_name)` | Fetches and returns detailed information about a specific movie.    |
    """

    def __init__(self):
        self.url = "https://www.rottentomatoes.com/"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36"
        }

    def top_rated(self):
        """
        Create an instance of `RottenTomatoes` class.
        ```python
        scraper = RottenTomatoes()
        scrpaer.top_rated()
        ```
        ```js
        [
            {
                "name":"The Cow Who Sang a Song Into the Future",
                "streaming_date":"Streaming Jul 18, 2023"
            }
            ...
        ]
        ```
        """
        try:
            url = self.url + "top/bestofrt/"
            response = requests.get(url)
            soup = BeautifulSoup(response.content, "html.parser")

            movie_titles = []
            title_elements = soup.find_all(
                "span",
                class_="p--small",
                attrs={"data-qa": "discovery-media-list-item-title"},
            )
            for title_element in title_elements:
                movie_titles.append(title_element.text.strip())

            # Extract streaming dates
            streaming_dates = []
            streaming_elements = soup.find_all(
                "span",
                class_="smaller",
                attrs={"data-qa": "discovery-media-list-item-start-date"},
            )
            for streaming_element in streaming_elements:
                streaming_dates.append(streaming_element.text.strip())

            response_list = []
            if len(movie_titles) == len(streaming_dates):
                for i in range(len(movie_titles)):
                    response_list.append(
                        dict(
                            zip(
                                ["name", "streaming_date"],
                                [movie_titles[i], streaming_dates[i]],
                            )
                        )
                    )

            return response_list
        except:
            return None

    def movie_details(self, movie_name):
        """
        Create an instance of `RottenTomatoes` class.
        ```python
        scraper = RottenTomatoes()
        scrpaer.movie_details(movie_name="iron man")
        ```
        Returns\n
        ```js
        {
            "rating":"PG-13(SuggestiveReferences|BriefLanguage)",
            "genre":"Comedy",
            "director":"English",
            "producers":[
                "David Heyman",
                "Margot Robbie",
                "Tom Ackerley",
                "Robbie Brenner"
            ],
            "writers":[
                "Greta Gerwig",
                "Noah Baumbach"
            ],
            "release_date":"Jul 21, 2023\n\\xa0wide",
            "runtime":"1h 54m",
            "distributor":"WarnerBros.Pictures",
            "production_co":"NB/GGPictures,HeydayFilms,LuckyChapEntertainment,Mattel"
        }
        ```
        """
        try:
            url = self.url + "m/" + movie_name
            response = requests.get(url)
            soup = BeautifulSoup(response.content, "html.parser")
            movie_details = {}
            # Extract the movie details from the <ul> element with id="info"
            ul_element = soup.find("ul", id="info")
            producers = []
            writers = []
            movie_details = ul_element.find_all(
                "span",
                class_="info-item-value",
                attrs={"data-qa": "movie-info-item-value"},
            )
            rating = movie_details[0].text.strip().replace("\n", "").replace(" ", "")
            genre = movie_details[1].text.strip().replace("\n", "").replace(" ", "")
            director = movie_details[2].text.strip()
            producers_name = movie_details[4].find_all("a")
            writers_name = movie_details[5].find_all("a")
            release_date = movie_details[6].text.strip()
            runtime = movie_details[7].text.strip()
            distributor = (
                movie_details[8].text.strip().replace("\n", "").replace(" ", "")
            )
            production_co = (
                movie_details[9].text.strip().replace("\n", "").replace(" ", "")
            )

            for writer in writers_name:
                writer = writer.text.strip()
                writers.append(writer)
            for producer in producers_name:
                producer = producer.text.strip()
                producers.append(producer)

            movie_details = {
                "rating": rating,
                "genre": genre,
                "director": director,
                "producers": producers,
                "writers": writers,
                "release_date": release_date,
                "runtime": runtime,
                "distributor": distributor,
                "production_co": production_co,
            }

            return movie_details
        except:
            return "Movie not found"


# src/scrape_up/spotify/__init__.py

from spotify_file import Spotify

__all__ = ["Spotify"]


# src/scrape_up/spotify/spotify_file.py

import selenium
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By


class Spotify:
    def __init__(self):
        self.driver = selenium.webdriver.Chrome()

    def __scrape_songs_by_keyword(self, keyword):
        """
        Class - `Spotify`\n
        Example -\n
        ```python
        scraper = Spotify()
        songs = scrape_songs_by_keyword("love")
        for song in songs:
            print("Name:", song["name"])
            print("Artist:", song["artist"])

        scraper.close()
        ```
        Return\n
        ```python
        return {all_books with details in json}
        ```
        """
        try:
            self.driver.get("https://open.spotify.com/search/")
            self.driver.implicitly_wait(100)

            search_box = self.driver.find_element(
                By.XPATH, "//input[@data-testid='search-input']"
            )
            search_box.send_keys(keyword)
            search_box.send_keys(Keys.RETURN)

            songs = []
            for song in self.driver.find_elements(
                By.XPATH,
                "//div[@class='search-results']/section//div[@data-testid='tracklist-row']",
            ):
                songs.append(
                    {
                        "name": song.find_element(
                            By.XPATH,
                            ".//span[@data-testid='tracklist-row__track-name']",
                        ).text,
                        "artist": song.find_element(
                            By.XPATH,
                            ".//span[@data-testid='tracklist-row__artist-name-link']",
                        ).text,
                    }
                )

            return songs

        except Exception as e:
            return None

    def __scrape_homepage(self):
        """
        Class - `Spotify`\n
        Example -\n
        ```python
        scraper = Spotify()
        songs = scrape_homepage()
        for song in songs:
            print("Name:", song["name"])
            print("Artist:", song["artist"])

        scraper.close()
        ```
        Return\n
        ```python
        return {playlists on homepage}
        ```
        """
        try:
            self.driver.get("https://open.spotify.com/")
            self.driver.implicitly_wait(100)

            playlists = []
            for song in self.driver.find_elements(
                By.XPATH, "//div[@data-testid='tracklist-row']"
            ):
                playlists.append(
                    {
                        "name": song.find_element(
                            By.XPATH,
                            ".//span[@data-testid='tracklist-row__track-name']",
                        ).text,
                        "artist": song.find_element(
                            By.XPATH,
                            ".//span[@data-testid='tracklist-row__artist-name-link']",
                        ).text,
                    }
                )

            return playlists

        except Exception as e:
            return None

    def close(self):
        self.driver.quit()



# src/scrape_up/stackoverflow/__init__.py

from scrape_up.stackoverflow.questions import StackOverflow
__all__ = ["StackOverflow"]

# src/scrape_up/stackoverflow/questions.py

from bs4 import BeautifulSoup
import requests
import json


class StackOverflow:
    """
        Class - `StackOverflow`\n
        Example
        ```python
        que = StackOverflow(topic:str)
        ```
        \n
    | Methods                     | Details                                                                                              |
    | --------------------------- | ---------------------------------------------------------------------------------------------------- |
    | `.getNewQuestions()`        | Returns the new questions, views, votes, answer counts, and descriptions in JSON format              |
    | `.getActiveQuestions()`     | Returns the active questions, views, votes, answer counts, and descriptions in JSON format           |
    | `.getUnansweredQuestions()` | Returns the unanswered questions, views, votes, answer counts, and descriptions in JSON format       |
    | `.getBountiedQuestions()`   | Returns the bountied questions, views, votes, answer counts, and descriptions in JSON format         |
    | `.getFrequentQuestions()`   | Returns the frequently asked questions, views, votes, answer counts, and descriptions in JSON format |
    | `.getHighScoredQuestions()` | Returns the most voted questions, views, votes, answer counts, and descriptions in JSON format       |
    """

    def __init__(self, topic):
        self.topic = topic

    def getNewQuestions(self):
        """
        Returns the list of newest questions, views, votes, answer counts, and descriptions.\n
        Class - `StackOverflow`
        Example:
        ```
        que = StackOverflow(topic="github")
        scrape = que.getNewQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://stackoverflow.com/questions/tagged/" + self.topic + "?tab=Newest"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                    .replace("  ", "")
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            json_data = json.dumps(questions_data)
            return json_data
        except:
            error_message = {"message": "No questions related to the topic found"}

            ejson = json.dumps(error_message)
            return ejson

    def getActiveQuestions(self):
        """
        Returns the list of active questions, views, votes, answer counts, and descriptions.\n
        Class - `StackOverflow`
        Example:
        ```
        que = StackOverflow(topic="github")
        scrape = que.getActiveQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://stackoverflow.com/questions/tagged/" + self.topic + "?tab=Active"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                    .replace("  ", "")
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            questions_list = questions_data["questions"]
            return questions_list
        except:
            return None

    def getBountiedQuestions(self):
        """
        Returns the list of bountied questions, views, votes, answer counts, and descriptions.\n
        Class - `StackOverflow`
        Example:
        ```
        que = StackOverflow(topic="github")
        scrape = que.getBountiedQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = (
            "https://stackoverflow.com/questions/tagged/" + self.topic + "?tab=Bounties"
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                    .replace("  ", "")
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            questions_list = questions_data["questions"]
            return questions_list
        except:
            return None

    def getUnansweredQuestions(self):
        """
        Returns the list of unanswered questions, views, votes, answer counts, and descriptions.\n
        Class - `StackOverflow`
        Example:
        ```
        que = StackOverflow(topic="github")
        scrape = que.getUnansweredQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = (
            "https://stackoverflow.com/questions/tagged/"
            + self.topic
            + "?tab=Unanswered"
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                    .replace("  ", "")
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            questions_list = questions_data["questions"]
            return questions_list
        except:
            return None

    def getFrequentQuestions(self):
        """
        Returns the list of frequent questions, views, votes, answer counts, and descriptions.\n
        Class - `StackOverflow`
        Example:
        ```
        que = StackOverflow(topic="github")
        scrape = que.getFrequentQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = (
            "https://stackoverflow.com/questions/tagged/" + self.topic + "?tab=Frequent"
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                    .replace("  ", "")
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            questions_list = questions_data["questions"]
            return questions_list
        except:
            return None

    def getHighScoredQuestions(self):
        """
        Returns the list of high scored questions, views, votes, answer counts, and descriptions.\n
        Class - `StackOverflow`
        Example:
        ```
        que = StackOverflow(topic="github")
        scrape = que.getHighScoredQuestions()
        ```
        Returns:
        {
            "question": question title
            "views": view count of question
            "vote_count": vote count of question
            "answer_count": no. of answers to the question
            "description": description of the question
        }
        """
        url = "https://stackoverflow.com/questions/tagged/" + self.topic + "?tab=Votes"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            questions_data = {"questions": []}

            questions = soup.select(".s-post-summary")
            for que in questions:
                title = que.select_one(".s-link").getText()
                stats = que.select(".s-post-summary--stats-item-number")
                vote = stats[0].getText()
                ans = stats[1].getText()
                views = stats[2].getText()
                desc = (
                    que.select_one(".s-post-summary--content-excerpt")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                    .replace("  ", "")
                )
                questions_data["questions"].append(
                    {
                        "question": title,
                        "views": views,
                        "vote_count": vote,
                        "answer_count": ans,
                        "description": desc,
                    }
                )
            questions_list = questions_data["questions"]
            return questions_list
        except:
            return None


stack = StackOverflow(topic="python")
print(stack.getBountiedQuestions())


# src/scrape_up/swiggy/__init__.py

from swiggy import Swiggy

__all__ = ["Swiggy"]


# src/scrape_up/swiggy/swiggy.py

import requests
from bs4 import BeautifulSoup as bs


class Swiggy:
    """
    First, create an object of class `Swiggy`
    ```python
    store1 = Swiggy()
    ```
    | Methods         | Details                                                                  |
    | --------------- | ------------------------------------------------------------------------ |
    | `get_restraunt_details(restraunt_url = " ")` | Returns the restraunt data with name, cuisine, area, rating, offers, etc |
    """

    def __init__(self):
        pass

    def get_restraunt_details(self,restraunt_url):
        """
        Create an object of the 'Swiggy' class\n
        ```python
        scraper = Swiggy()
        scraper.(restraunt_url="https://www.swiggy.com/restaurants/pizza-hut-western-extension-area-karol-bagh-delhi-435678")
        ```
        Response
        ```js
        {
            "name":"Pizza Hut",
            "cuisine":"Pizzas",
            "area":"Karol Bagh",
            "rating":"3.7",
            "rating_count":"1K+ ratings",
            "cost_per_person":"₹350 for two",
            "offers":[
                {
                    "15% OFF UPTO ₹300":"USE CITIFOODIE | ABOVE ₹1200"
                }
                ...
            ]
        }
        ```
        """
        try:
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36"
            }
            response = requests.get(restraunt_url, headers=headers).text
            soup = bs(response, "lxml")
            restaurant_data = []
            name = soup.find(
                "p", {"class": "RestaurantNameAddress_name__2IaTv"}
            ).text.strip()
            cuisine = soup.find(
                "p", {"class": "RestaurantNameAddress_cuisines__mBHr2"}
            ).text.strip()
            area = soup.find(
                "p", {"class": "RestaurantNameAddress_area__2P9ib"}
            ).text.strip()
            rating = soup.find(
                "span", {"class": "RestaurantRatings_avgRating__1TOWY"}
            ).text.strip()
            rating_count = soup.find(
                "span", {"class": "RestaurantRatings_totalRatings__3d6Zc"}
            ).text.strip()
            cost_per = soup.find_all("li", {"class": "RestaurantTimeCost_item__2HCUz"})[
                -1
            ].text.strip()
            offers = []
            offer_box = soup.find_all(
                "div", {"class": "RestaurantOffer_infoWrapper__2trmg"}
            )
            for offer in offer_box:
                offer_ = {}
                offer_header = offer.find(
                    "p", {"class": "RestaurantOffer_header__3FBtQ"}
                ).text.strip()
                offer_content_1 = offer.find("span").text.strip()
                offer_content_2 = offer.find(
                    "span", {"class": "RestaurantOffer_description__1SRJf"}
                ).text.strip()
                offer_content = offer_content_1 + " | " + offer_content_2
                offer_[offer_header] = offer_content
                offers.append(offer_)
            restaurant_data = {
                "name": name,
                "cuisine": cuisine,
                "area": area,
                "rating": rating,
                "rating_count": rating_count,
                "cost_per_person": cost_per,
                "offers": offers,
            }
            return restaurant_data
        except:
            return None


# src/scrape_up/techCrunch/__init__.py

from scrape_up.techCrunch.techCrunch import TechCrunch
__all__ = ["TechCrunch"]

# src/scrape_up/techCrunch/techCrunch.py

import requests
from bs4 import BeautifulSoup
import json


class TechCrunch:
    """
    Class - `TechCrunch`
    Example:
    ```
    articles = TechCrunch()
    ```\n
    Methods :\n
    1. ``.getArticles() | Response - Articles with title, descriptions, images, date and link.
    """

    def getArticles(self, category):
        self.category = category
        """
        Class - `TechCrunch`
        Example:
        ```
        articles = TechCrunch()
        articles.getArticles("artificial-intelligence")
        ```
        Returns:
        {
            "title": Tile of the article
            "description": Description of the article
            "image": Image of the article
            "author": Author of the Article
            "date": Date the article was posted
            "link": Link to the article
        }
        """
        url = (
            "https://techcrunch.com/category/" + self.category.replace(" ", "-").lower()
        )
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            articles_data = {"articles": []}

            articles = soup.find_all(
                "div", class_="post-block post-block--image post-block--unread"
            )
            for n in articles:
                name = (
                    n.select_one(".post-block__title__link")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                desc = (
                    n.select_one(".post-block__content")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                img = n.find_all("img", src=True)
                image = img[0]["src"]
                author = (
                    n.select_one(".river-byline__authors")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                time = n.find_all("div", class_="river-byline")
                date = (
                    time[0]
                    .select_one(".river-byline__time")
                    .getText()
                    .strip()
                    .encode("ascii", "ignore")
                    .decode()
                )
                links = n.find_all("a", class_="post-block__title__link", href=True)
                link = links[0]["href"]
                articles_data["articles"].append(
                    {
                        "title": name,
                        "description": desc,
                        "image": image,
                        "author": author,
                        "date": date,
                        "link": link,
                    }
                )
            res_json = json.dumps(articles_data)
            return res_json
        except:
            error_message = {
                "message": "Can't fetch any articles from the topic provided."
            }
            ejson = json.dumps(error_message)
            return ejson

    def search(self, topic):
        self.topic = topic
        """
        Class - `TechCrunch`
        Example:
        ```
        articles = TechCrunch()
        articles.search("github")
        ```
        Returns:
        {
            "title": Tile of the article
            "description": Description of the article
            "image": Image of the article
            "author": Author of the Article
            "date": Date the article was posted
            "link": Link to the article
        }
        """
        url = "https://search.techcrunch.com/search?p=" + self.topic + "&fr=techcrunch"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")

            articles_data = {"articles": []}

            articles = soup.find_all("li", class_="ov-a mt-0 pt-26 pb-26 bt-dbdbdb")
            for i in articles:
                name = i.find("a", class_="fz-20 lh-22 fw-b").getText()
                desc = i.find("p", class_="fz-14 lh-20 c-777").getText()
                img = i.find("img", class_="s-img mr-10 s-img-errchk", src=True)
                image = img["src"]
                author = i.find("span", class_="mr-15").getText()
                date = i.find("span", class_="pl-15 bl-1-666").getText()
                links = i.find("a", class_="fz-20 lh-22 fw-b", href=True)
                link = links["href"]
                articles_data["articles"].append(
                    {
                        "title": name,
                        "description": desc,
                        "image": image,
                        "author": author,
                        "date": date,
                        "link": link,
                    }
                )
            return articles_data
        except:
            return None


# src/scrape_up/twitter/__init__.py

from scrape_up.twitter.numidconverter import TwitterScraper

__all__=['TwitterScraper']

# src/scrape_up/twitter/numidconverter.py

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options
from time import sleep
from bs4 import BeautifulSoup
import json


class TwitterScraper:
    def __init__(self):
        self.chrome_options = Options()
        self.chrome_options.add_argument("--headless")
        self.chrome_options.add_argument("--window-size=1920,1080")
        self.chrome_options.add_argument("--disable-gpu")
        self.chrome_options.add_argument("--no-sandbox")
        self.chrome_options.add_argument("--disable-dev-shm-usage")
        self.chrome_options.add_argument("--disable-extensions")
        self.chrome_options.add_argument("--disable-logging")
        self.chrome_options.add_argument("--log-level=3")
        self.chrome_options.add_argument("--silent")
        self.chrome_options.add_argument("--blink-settings=imagesEnabled=false")

    def unametoid(self, username):
        url = "https://twitter.com/{}".format(username)
        # print(url)
        driver = webdriver.Chrome(options=self.chrome_options)
        driver.get(url)

        html = driver.page_source
        soup = BeautifulSoup(html, "html.parser")
        try:
            user_id = soup.find("script", {"data-testid": "UserProfileSchema-test"})
            data = json.loads(user_id.string)
            # driver.quit()
            return {
                "data": data["author"]["identifier"],
                "message": f"Numerical id found for username {username}",
            }
        except:
            return {
                "data": None,
                "message": f"Numerical id not found for username {username}",
            }

    def idtouname(self, numid):
        url = "https://twitter.com/i/user/{}".format(numid)
        driver = webdriver.Chrome(options=self.chrome_options)
        driver.get(url)
        html = driver.page_source
        soup = BeautifulSoup(html, "html.parser")
        try:
            user_id = soup.find("script", {"data-testid": "UserProfileSchema-test"})
            data = json.loads(user_id.string)
            # driver.quit()
            return {
                "data": data["author"]["additionalName"],
                "message": f"Username found for numerical id {numid}",
            }
        except:
            return {
                "data": None,
                "message": f"Username not found for numerical id {numid}",
            }


# src/scrape_up/wikipedia/__init__.py

from scrape_up.wikipedia.wikipedia import wikipedia

__all__ = ["wikipedia"]


# src/scrape_up/wikipedia/wikipedia.py

from bs4 import BeautifulSoup
import requests
import json

class WikipediaScraper:
    def __init__(self, url):
        self.url = url

    def scrape(self):
        response = requests.get(self.url)
        soup = BeautifulSoup(response.text, 'html.parser')

        # Extract the title
        title = soup.find(id='firstHeading').text

        # Extract all the headings and their content
        sections = soup.find_all('h2')
        data = {}
        for section in sections:
            heading = section.find('span', class_='mw-headline')
            if heading:
                content = []
                next_node = section.find_next_sibling(['h2', 'h3', 'h4', 'h5', 'h6'])
                while next_node and next_node.name != 'h2':
                    if next_node.name in ['h3', 'h4', 'h5', 'h6']:
                        content.append({'heading': next_node.text.strip()})
                    elif next_node.name == 'p':
                        content.append({'text': next_node.text.strip()})
                    next_node = next_node.find_next_sibling(['h2', 'h3', 'h4', 'h5', 'h6', 'p'])
                data[heading.text] = content

        # Return the data as JSON
        result = {
            'title': title,
            'sections': data
        }
        return json.dumps(result, indent=4)


# src/scrape_up/youtube/__init__.py

from video import Video
from channel import Channel

__all__ = ["Video", "Channel"]


# src/scrape_up/youtube/channel.py

import requests
from bs4 import BeautifulSoup
import json


class Channel:
    """
    Create an instance of `Channel` class.
    ```python
    channel_data = Channel(channel_username="BeABetterDev")
    ```

    | Methods       | Details                                                                |
    | ------------- | ---------------------------------------------------------------------- |
    | `.getAbout()` | Returns the channel details mentioned in the about page of the channel |
    """

    def __init__(self, channel_username):
        self.channel_username = channel_username

    def getAbout(self):
        """
        Class - `Channel`
        ```py
        channel_data = Channel(channel_username="BeABetterDev")
        channel_data.getAbout()
        ```
        Returns:
        ```js
        {
            "name": Name of the channel
            "description": Description of the channel
            "channel_url": Link to the channel
            "channel_avatar": Channel avatar
            "channel_banner": Channel banner
            "subscriber_count": No. of subscribers of the channel
            "toal_videos": Total videos uploaded in the channel
            "total_views": Total views till date of the channel
            "join_date": Date the channel joined YouTube
            "country": Country of origin of the channel
            "links": Additional links provided from the channel
        }
        ```
        """
        url = f"https://www.youtube.com/@{self.channel_username}/about"
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")
            channel_data = {"channel_data": []}
            link_data = {"link_data": []}
            scripts = soup.find_all("script")
            req_script = scripts[35].text.strip()
            script = req_script[20:-1]
            data = json.loads(script)

            metadata = data["metadata"]["channelMetadataRenderer"]
            title = metadata["title"]
            desc = metadata["description"]
            channel_url = metadata["vanityChannelUrl"]
            channel_avatar = metadata["avatar"]["thumbnails"][0]["url"]
            header = data["header"]["c4TabbedHeaderRenderer"]
            channel_banner = header["banner"]["thumbnails"][5]["url"]
            subs = header["subscriberCountText"]["simpleText"]
            total_videos = header["videosCountText"]["runs"][0]["text"]

            baser = data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
            for b in baser:
                try:
                    base = b["tabRenderer"]["content"]["sectionListRenderer"][
                        "contents"
                    ][0]["itemSectionRenderer"]["contents"][0][
                        "channelAboutFullMetadataRenderer"
                    ]

                    total_views = base["viewCountText"]["simpleText"]
                    join_date = base["joinedDateText"]["runs"][1]["text"]
                    country = base["country"]["simpleText"]

                    links = base["primaryLinks"]
                    for i in links:
                        link_data["link_data"].append(
                            {
                                "link_url": i["navigationEndpoint"]["urlEndpoint"][
                                    "url"
                                ],
                                "link_name": i["title"]["simpleText"],
                                "link_icon": i["icon"]["thumbnails"][0]["url"],
                            }
                        )
                except:
                    pass

            channel_data["channel_data"].append(
                {
                    "name": title,
                    "description": desc,
                    "channel_url": channel_url,
                    "channel_avatar": channel_avatar,
                    "channel_banner": channel_banner,
                    "subscriber_count": subs,
                    "toal_videos": total_videos,
                    "total_views": total_views,
                    "join_date": join_date,
                    "country": country,
                    "links": link_data,
                }
            )
            return channel_data["channel_data"][0]
        except:
            return None


# src/scrape_up/youtube/video.py

import requests
from bs4 import BeautifulSoup
import json


class Video:
    """
    Create an instance of `Video` class.
    ```python
    video = Video(video_url="video_url")
    ```

    | Methods         | Details                  |
    | --------------- | ------------------------ |
    | `.getDetails()` | Return the video details |
    """

    def __init__(self, video_url):
        self.video_url = video_url

    def getDetails(self):
        """
        Class - `Video`
        Example:
        ```
        video_data = Video("https://www.youtube.com/watch?v=pBy1zgt0XPc")
        video_data.getDetails()
        ```
        Returns:
        {
            "title": Tile of the video
            "description": Description of the video
            "views_count": No. of views of the video,
            "upload_date": Upload date of the video,
            "comment_count": No. of comments,
            "channel_name": Name of the channel,
            "channel_avatar": Channel avatar,
            "subscriber_count": No. of channel subscribers,
            "channel_url": Link to the channel
        }
        """
        url = self.video_url
        try:
            res = requests.get(url)
            soup = BeautifulSoup(res.text, "html.parser")
            video_data = {"video_data": []}

            scripts = soup.find_all("script")
            req_script = scripts[44].text.strip()
            script = req_script[20:-1]
            data = json.loads(script)

            base = data["contents"]["twoColumnWatchNextResults"]["results"]["results"][
                "contents"
            ]

            first = base[0]["videoPrimaryInfoRenderer"]
            title = (
                first["title"]["runs"][0]["text"]
                .strip()
                .encode("ascii", "ignore")
                .decode()
            )
            views = first["viewCount"]["videoViewCountRenderer"]["viewCount"][
                "simpleText"
            ]
            date = first["dateText"]["simpleText"]

            channel_data = base[1]["videoSecondaryInfoRenderer"]["owner"][
                "videoOwnerRenderer"
            ]
            avatar = channel_data["thumbnail"]["thumbnails"][2]["url"]
            name = channel_data["title"]["runs"][0]["text"]
            channel_url = channel_data["title"]["runs"][0]["navigationEndpoint"][
                "commandMetadata"
            ]["webCommandMetadata"]["url"]
            subs = channel_data["subscriberCountText"]["accessibility"][
                "accessibilityData"
            ]["label"]

            desc = (
                base[1]["videoSecondaryInfoRenderer"]["attributedDescription"][
                    "content"
                ]
                .strip()
                .encode("ascii", "ignore")
                .decode()
            )
            comment_count = base[2]["itemSectionRenderer"]["contents"][0][
                "commentsEntryPointHeaderRenderer"
            ]["commentCount"]["simpleText"]

            video_data["video_data"].append(
                {
                    "title": title,
                    "description": desc[:200] + "...",
                    "views_count": views,
                    "upload_date": date,
                    "comment_count": comment_count,
                    "channel_name": name,
                    "channel_avatar": avatar,
                    "subscriber_count": subs,
                    "channel_url": "https://youtube.com" + channel_url,
                }
            )
            return video_data
        except:
            return None


# src/scrape_up/zomato/__init__.py

from zomato import Zomato

__all__ = ["Zomato"]


# src/scrape_up/zomato/zomato.py

import requests
from bs4 import BeautifulSoup as bs


class Zomato:
    """
    Create an object of the 'Zomato' class\n
    ```python
    scraper = Zomato()
    ```
    | Methods                    | Details                                                                   |
    | -------------------------- | ------------------------------------------------------------------------- |
    | `.get_restaurants_details()` | Returns the restraunt data with name, cuisine, area, rating, offers, etc  |
    """

    def __init__(self):
        pass

    def get_restaurants_details(self, page_url):
        """
        Create an object of the 'Zomato' class\n
        ```python
        scraper = Zomato()
        scraper.get_restaurants_details(page_url="https://www.zomato.com/ncr/music-mountains-hillside-cafe-greater-kailash-gk-1-new-delhi/order")
        ```
        Response
        ```js
        {
            'name': 'Music & Mountains - Hillside Cafe',
            'cuisine':
                [
                    'Italian',
                    'Continental',
                    'Fast Food',
                    'Salad',
                    'Desserts',
                    'Beverages',
                    'Shake'
                ],
            'area': 'M Block Market, Greater Kailash 1 (GK1), New Delhi',
            'dining_rating': '4.6',
            'dining_review_count': '6,165',
            'delivery_rating': '4.5',
            'delivery_review_count': '1,097',
            'offers':
                [
                    {'50% OFF up to ₹100': 'use code HUNGRY'},
                    {'Flat ₹125 OFF': 'use code GET125'},
                    {'Flat ₹125 OFF': 'use code CANARAPARTY'},
                    {'₹25 - ₹250 cashback': 'use code PAYTMWEE...'}
                ]
        }
        ```
        """
        try:
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36"
            }
            response = requests.get(page_url, headers=headers).text
            soup = bs(response, "lxml")
            restaurant_data = []
            name = soup.find(
                "h1", {"class": "sc-7kepeu-0 sc-iSDuPN fwzNdh"}
            ).text.strip()
            cuisine_box = soup.find("div", {"class": "sc-fgfRvd gBMRZZ"})
            cuisines_all = cuisine_box.find_all("span")
            cuisines = []
            for cuisine in cuisines_all:
                cuisine_add = cuisine.text.strip().replace(",", "")
                if cuisine_add != "," and cuisine_add != "":
                    cuisines.append(cuisine_add)
            area = soup.find("a", {"class": "sc-clNaTc vNCcy"}).text.strip()
            rating_boxes = soup.find("section", {"class": "sc-dxZgTM QvFZM"})
            ratings = rating_boxes.find_all("div", {"class": "sc-1q7bklc-1 cILgox"})
            dining_rating = ratings[0].text.strip()
            delivery_rating = ratings[1].text.strip()
            review_count_boxes = rating_boxes.find_all(
                "div", {"class": "sc-1q7bklc-8 kEgyiI"}
            )
            dining_review_count = review_count_boxes[0].text.strip()
            delivery_review_count = review_count_boxes[1].text.strip()
            offers = []
            offer_boxes = soup.find_all("div", {"class": "sc-1a03l6b-2 gerWzu"})
            for offer in offer_boxes:
                offer_ = {}
                offer_text = offer.find_all("div")
                offer_[offer_text[0].text.strip()] = offer_text[1].text.strip()
                offers.append(offer_)
            restaurant_data = {
                "name": name,
                "cuisine": cuisines,
                "area": area,
                "dining_rating": dining_rating,
                "dining_review_count": dining_review_count,
                "delivery_rating": delivery_rating,
                "delivery_review_count": delivery_review_count,
                "offers": offers,
            }
            return restaurant_data
        except:
            return None


# src/scrape_up/hackerrank/hackerrank.py
import requests
from bs4 import BeautifulSoup as bs


class HackerRank:
    """
    First, create an object of class `HackerRank`
    ```python
    user1 = HackerRank(id="helloguys289")
    ```
    | Methods         | Details                                                                                   |
    | --------------- | ----------------------------------------------------------------------------------------- |
    | `get_profile()` | Returns name, username, country, user_type, details, badges, verified_skills, social etc. |
    """

    def __init__(self, id):
        self.id = id

    def get_profile(self):
        """
        Create an object of the 'HackerRank' class\n
        ```python
        user1 = HackerRank(id="helloguys289")
        user1.get_profile()
        ```
        Response
        ```js
       {
        'name': 'Karri Srinivasa Rao', 
        'username': 'helloguys289', 
        'country': 'India', 
        'user_type': 'Student', 
        'details': 
        [
            {
                'Education': 'IIITM, Gwalior (ABV Indian Institute of Information Technology and Management)'
            }
        ], 
        'badges': 
        [
            {
                'badge': 'Problem Solving', 
                'stars': 4
            }, 
            {
                'badge': 'CPP', 
                'stars': 3
            }, 
            {
                'badge': 'Days of Code', 
                'stars': 1
            }, 
            {
                'badge': 'Days of JS', 
                'stars': 1
            }, 
            {
                'badge': 'C language', 
                'stars': 2
            }
        ], 
        'verified_skills': 
        [
            {
                'certificate': 'Problem Solving (Basic)', 
                'certificate_link': 'https://www.hackerrank.com/certificates/d8c5c5dcd307'
            }, 
            {
                'certificate': 'Problem Solving (Intermediate)', 
                'certificate_link': 'https://www.hackerrank.com/certificates/4b16deda4e6a'
            }, 
            {
                'certificate': 'Python (Basic)', 
                'certificate_link': 'https://www.hackerrank.com/certificates/026a64ec0c87'
            }, 
            {
                'certificate': 'Rest API (Intermediate)', 
                'certificate_link': 'https://www.hackerrank.com/certificates/227540142706'
            }
        ], 
        'social': 
        [
            {
                'Github': 'https://github.com/srinivaskarri1999'
            }, 
            {
                'Linkedin': 'https://www.linkedin.com/in/srinivas1999'
            }
        ]
        }
        ```
        """
        try:
            url = f"https://www.hackerrank.com/{self.id}"
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win 64 ; x64) Apple WeKit /537.36(KHTML , like Gecko) Chrome/80.0.3987.162 Safari/537.36"
            }
            response = requests.get(url, headers=headers).text
            soup = bs(response, "lxml")
            
            user_details_box = soup.find("div", {"class": "profile-summary-card"})
            extra_details_box = soup.find("div", {"class": "profile-details-extra"})
            # Name
            try:
                profile_name = user_details_box.find("h1", {"class": "profile-heading"}).text.strip()
            except:
                profile_name = ""
            # Username
            try:
                profile_username = user_details_box.find("p",{"class": "profile-username-heading"}).text.strip()[1:]
            except:
                profile_username = ""
            # Country
            try:
                country_name = user_details_box.find("p",{"class": "profile-country"}).text.strip()
            except:
                country_name = ""
            # User Type
            try:
                user_type = user_details_box.find("p", {"class": "profile-subheading"}).text.strip()
            except:
                user_type = ""

            # Social Links
            try:
                social_box = user_details_box.find("ul", {"class": "profile-social-links-list"})
                social_box_li = social_box.find_all("li")
                socials = []
                for li in social_box_li:
                    social_a = li.find("a")
                    social_heading = social_a["aria-label"]
                    social_link = social_a["href"]
                    social_details = {
                        social_heading : social_link
                    }
                    socials.append(social_details)
            except:
                socials = []

            # Extra Details
            try:
                extra_detail_div = extra_details_box.find_all("div")
                extra_details = []
                for div in extra_detail_div:
                    div_heading = div.find("p", {"class": "profile-details-label"}).text.strip()
                    div_content = div.find("p", {"class": "profile-details-value"}).text.strip()
                    details = {
                        div_heading : div_content
                    }
                    extra_details.append(details)
            except:
                extra_details = []
            
            # Right Pane
            user_details_box_right = soup.find("div", {"class": "profile-right-pane"})
            user_details_sections = user_details_box_right.find_all("section")

            # Badges
            try:
                badges_section_heading = user_details_sections[0].find("header").text.strip()
                badges_list = user_details_sections[0].find_all("div", {"class": "hacker-badge"})
                badges = []
                for badge in badges_list:
                    badge_heading = badge.find("text", {"class": "badge-title"}).text.strip()
                    star_count = len(badge.find_all("svg", {"class": "badge-star"}))
                    badge_details = {
                        "badge" : badge_heading,
                        "stars" : star_count
                    }
                    badges.append(badge_details)
            except:
                badges = []

            # Verified Skills
            try:
                verified_skill_section_heading = user_details_sections[1].find("header").text.strip()
                verified_certificates = user_details_sections[1].find_all("a")
                verified_skills = []
                for cert in verified_certificates:
                    cert_heading = cert.text.replace("Verified","").replace("Certificate: ", "").strip()
                    cert_link =  "https://www.hackerrank.com" + cert["href"]
                    cert_details = {
                        "certificate" : cert_heading,
                        "certificate_link" : cert_link
                    }
                    verified_skills.append(cert_details)
            except:
                verified_skills = []

            profile_data = {
                'name': profile_name,
                'username': profile_username,
                'country': country_name,
                'user_type': user_type,
                'details': extra_details,
                'badges': badges, 
                'verified_skills': verified_skills,
                'social': socials
            }
            return profile_data
        except:
            return None
