def get_unique_links ( self ) : page_url = self . get_current_url ( ) soup = self . get_beautiful_soup ( self . get_page_source ( ) ) links = page_utils . _get_unique_links ( page_url , soup ) return links
all links to html pages with no duplicates vode in python
Get all unique links in the html of the page source . Page links include those obtained from : a - > href img - > src link - > href and script - > src .
def get_unique_links(self):
        """ Get all unique links in the html of the page source.
            Page links include those obtained from:
            "a"->"href", "img"->"src", "link"->"href", and "script"->"src". """
        page_url = self.get_current_url()
        soup = self.get_beautiful_soup(self.get_page_source())
        links = page_utils._get_unique_links(page_url, soup)
        return links