{"action":"created","comment":{"url":"https://api.github.com/repos/junhaodong/jana/pulls/comments/46748141","id":46748141,"diff_hunk":"@@ -0,0 +1,107 @@\n+import dryscrape, re, sys\n+from bs4 import BeautifulSoup\n+\n+class EmailCrawler:\n+    \"\"\"\n+    Takes a domain name and prints out a list of email addresses found on that website.\n+    Stores emails, paths, and visited_paths as sets to avoid duplicates.\n+    Uses Dryscrape to dynamically scrape text of JavaScript generated and static websites.\n+    Uses BeautifulSoup to search for valid href's to continue crawling on.\n+    \"\"\"\n+    emailRE = re.compile(\"[\\w.+-]+@[\\w-]+\\.[\\w.-]+\")\n+\n+    def __init__(self, domain):\n+        if 'http' in domain:\n+            self.url = domain\n+        else:\n+            self.url = 'http://' + domain\n+        self.session = dryscrape.Session(base_url=self.url)\n+        self.emails = set()\n+        self.paths = set()\n+        self.visited_paths = set()\n+        self.num_pages_limit = 50\n+\n+        self.session.set_attribute('auto_load_images', False)\n+\n+    def is_valid_tag(self, tag):\n+        \"\"\"Checks if a tag contains a valid href that hasn't been visited yet.\"\"\"\n+\n+        if tag.has_attr('href') and len(tag['href']) > 0:\n+            href = tag['href']\n+\n+            is_visited = self.session.complete_url(href) in self.visited_paths\n+            is_style_sheet = tag.name == \"link\"\n+            is_jumpTo = \"#\" in href\n+            is_mailTo = \"mailto\" in href\n+            is_js = \"javascript:\" in href\n+            is_ad = \"/ads/\" in href","path":"find_email_addresses.py","position":37,"original_position":37,"commit_id":"3d1f0df463dbc903f74904e35540070903c4d4b6","original_commit_id":"3d1f0df463dbc903f74904e35540070903c4d4b6","user":{"login":"mmcmahon-jana","id":5712683,"avatar_url":"https://avatars.githubusercontent.com/u/5712683?v=3","gravatar_id":"","url":"https://api.github.com/users/mmcmahon-jana","html_url":"https://github.com/mmcmahon-jana","followers_url":"https://api.github.com/users/mmcmahon-jana/followers","following_url":"https://api.github.com/users/mmcmahon-jana/following{/other_user}","gists_url":"https://api.github.com/users/mmcmahon-jana/gists{/gist_id}","starred_url":"https://api.github.com/users/mmcmahon-jana/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mmcmahon-jana/subscriptions","organizations_url":"https://api.github.com/users/mmcmahon-jana/orgs","repos_url":"https://api.github.com/users/mmcmahon-jana/repos","events_url":"https://api.github.com/users/mmcmahon-jana/events{/privacy}","received_events_url":"https://api.github.com/users/mmcmahon-jana/received_events","type":"User","site_admin":false},"body":"What would you use as a length cutoff?  A better approach would be to verity that any non-relative links are being served from the domain passed in as the original argument.  So for example, if you ran `python find_email_addresses.py mcent.com`, a link to \"http://srv.carbonads.net/ads/click/x/GTND...redirect=...\" would not be followed since the domain is srv.carbionads.net.\r\nTrying to limit based on length would always run into edge cases and false positives.","created_at":"2015-12-05T00:17:58Z","updated_at":"2015-12-05T00:17:58Z","html_url":"https://github.com/junhaodong/jana/pull/1#discussion_r46748141","pull_request_url":"https://api.github.com/repos/junhaodong/jana/pulls/1","_links":{"self":{"href":"https://api.github.com/repos/junhaodong/jana/pulls/comments/46748141"},"html":{"href":"https://github.com/junhaodong/jana/pull/1#discussion_r46748141"},"pull_request":{"href":"https://api.github.com/repos/junhaodong/jana/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/junhaodong/jana/pulls/1","id":52639375,"html_url":"https://github.com/junhaodong/jana/pull/1","diff_url":"https://github.com/junhaodong/jana/pull/1.diff","patch_url":"https://github.com/junhaodong/jana/pull/1.patch","issue_url":"https://api.github.com/repos/junhaodong/jana/issues/1","number":1,"state":"open","locked":false,"title":"Solution","user":{"login":"junhaodong","id":5431678,"avatar_url":"https://avatars.githubusercontent.com/u/5431678?v=3","gravatar_id":"","url":"https://api.github.com/users/junhaodong","html_url":"https://github.com/junhaodong","followers_url":"https://api.github.com/users/junhaodong/followers","following_url":"https://api.github.com/users/junhaodong/following{/other_user}","gists_url":"https://api.github.com/users/junhaodong/gists{/gist_id}","starred_url":"https://api.github.com/users/junhaodong/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/junhaodong/subscriptions","organizations_url":"https://api.github.com/users/junhaodong/orgs","repos_url":"https://api.github.com/users/junhaodong/repos","events_url":"https://api.github.com/users/junhaodong/events{/privacy}","received_events_url":"https://api.github.com/users/junhaodong/received_events","type":"User","site_admin":false},"body":"In addition to the issue I mentioned in the README, my program has one small redundancy. If the user enters `jana.com` as the domain for example, since \"/\" shows up as a valid href, the program searches on both domains, `jana.com` and `jana.com/`. I figured this one extra crawl would be insignificant especially as the number of visited pages gets larger.","created_at":"2015-12-04T07:52:55Z","updated_at":"2015-12-05T00:17:58Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d67bd7ed5226cceb273d5e0c7a4529722a481b63","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/junhaodong/jana/pulls/1/commits","review_comments_url":"https://api.github.com/repos/junhaodong/jana/pulls/1/comments","review_comment_url":"https://api.github.com/repos/junhaodong/jana/pulls/comments{/number}","comments_url":"https://api.github.com/repos/junhaodong/jana/issues/1/comments","statuses_url":"https://api.github.com/repos/junhaodong/jana/statuses/3d1f0df463dbc903f74904e35540070903c4d4b6","head":{"label":"junhaodong:workspace","ref":"workspace","sha":"3d1f0df463dbc903f74904e35540070903c4d4b6","user":{"login":"junhaodong","id":5431678,"avatar_url":"https://avatars.githubusercontent.com/u/5431678?v=3","gravatar_id":"","url":"https://api.github.com/users/junhaodong","html_url":"https://github.com/junhaodong","followers_url":"https://api.github.com/users/junhaodong/followers","following_url":"https://api.github.com/users/junhaodong/following{/other_user}","gists_url":"https://api.github.com/users/junhaodong/gists{/gist_id}","starred_url":"https://api.github.com/users/junhaodong/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/junhaodong/subscriptions","organizations_url":"https://api.github.com/users/junhaodong/orgs","repos_url":"https://api.github.com/users/junhaodong/repos","events_url":"https://api.github.com/users/junhaodong/events{/privacy}","received_events_url":"https://api.github.com/users/junhaodong/received_events","type":"User","site_admin":false},"repo":{"id":47234996,"name":"jana","full_name":"junhaodong/jana","owner":{"login":"junhaodong","id":5431678,"avatar_url":"https://avatars.githubusercontent.com/u/5431678?v=3","gravatar_id":"","url":"https://api.github.com/users/junhaodong","html_url":"https://github.com/junhaodong","followers_url":"https://api.github.com/users/junhaodong/followers","following_url":"https://api.github.com/users/junhaodong/following{/other_user}","gists_url":"https://api.github.com/users/junhaodong/gists{/gist_id}","starred_url":"https://api.github.com/users/junhaodong/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/junhaodong/subscriptions","organizations_url":"https://api.github.com/users/junhaodong/orgs","repos_url":"https://api.github.com/users/junhaodong/repos","events_url":"https://api.github.com/users/junhaodong/events{/privacy}","received_events_url":"https://api.github.com/users/junhaodong/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/junhaodong/jana","description":"Coding Challenge","fork":false,"url":"https://api.github.com/repos/junhaodong/jana","forks_url":"https://api.github.com/repos/junhaodong/jana/forks","keys_url":"https://api.github.com/repos/junhaodong/jana/keys{/key_id}","collaborators_url":"https://api.github.com/repos/junhaodong/jana/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/junhaodong/jana/teams","hooks_url":"https://api.github.com/repos/junhaodong/jana/hooks","issue_events_url":"https://api.github.com/repos/junhaodong/jana/issues/events{/number}","events_url":"https://api.github.com/repos/junhaodong/jana/events","assignees_url":"https://api.github.com/repos/junhaodong/jana/assignees{/user}","branches_url":"https://api.github.com/repos/junhaodong/jana/branches{/branch}","tags_url":"https://api.github.com/repos/junhaodong/jana/tags","blobs_url":"https://api.github.com/repos/junhaodong/jana/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/junhaodong/jana/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/junhaodong/jana/git/refs{/sha}","trees_url":"https://api.github.com/repos/junhaodong/jana/git/trees{/sha}","statuses_url":"https://api.github.com/repos/junhaodong/jana/statuses/{sha}","languages_url":"https://api.github.com/repos/junhaodong/jana/languages","stargazers_url":"https://api.github.com/repos/junhaodong/jana/stargazers","contributors_url":"https://api.github.com/repos/junhaodong/jana/contributors","subscribers_url":"https://api.github.com/repos/junhaodong/jana/subscribers","subscription_url":"https://api.github.com/repos/junhaodong/jana/subscription","commits_url":"https://api.github.com/repos/junhaodong/jana/commits{/sha}","git_commits_url":"https://api.github.com/repos/junhaodong/jana/git/commits{/sha}","comments_url":"https://api.github.com/repos/junhaodong/jana/comments{/number}","issue_comment_url":"https://api.github.com/repos/junhaodong/jana/issues/comments{/number}","contents_url":"https://api.github.com/repos/junhaodong/jana/contents/{+path}","compare_url":"https://api.github.com/repos/junhaodong/jana/compare/{base}...{head}","merges_url":"https://api.github.com/repos/junhaodong/jana/merges","archive_url":"https://api.github.com/repos/junhaodong/jana/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/junhaodong/jana/downloads","issues_url":"https://api.github.com/repos/junhaodong/jana/issues{/number}","pulls_url":"https://api.github.com/repos/junhaodong/jana/pulls{/number}","milestones_url":"https://api.github.com/repos/junhaodong/jana/milestones{/number}","notifications_url":"https://api.github.com/repos/junhaodong/jana/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/junhaodong/jana/labels{/name}","releases_url":"https://api.github.com/repos/junhaodong/jana/releases{/id}","created_at":"2015-12-02T03:35:41Z","updated_at":"2015-12-02T03:35:41Z","pushed_at":"2015-12-04T07:52:55Z","git_url":"git://github.com/junhaodong/jana.git","ssh_url":"git@github.com:junhaodong/jana.git","clone_url":"https://github.com/junhaodong/jana.git","svn_url":"https://github.com/junhaodong/jana","homepage":null,"size":30,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"junhaodong:master","ref":"master","sha":"a67e925035b8fcf3253653c32da8bf61b50f7da1","user":{"login":"junhaodong","id":5431678,"avatar_url":"https://avatars.githubusercontent.com/u/5431678?v=3","gravatar_id":"","url":"https://api.github.com/users/junhaodong","html_url":"https://github.com/junhaodong","followers_url":"https://api.github.com/users/junhaodong/followers","following_url":"https://api.github.com/users/junhaodong/following{/other_user}","gists_url":"https://api.github.com/users/junhaodong/gists{/gist_id}","starred_url":"https://api.github.com/users/junhaodong/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/junhaodong/subscriptions","organizations_url":"https://api.github.com/users/junhaodong/orgs","repos_url":"https://api.github.com/users/junhaodong/repos","events_url":"https://api.github.com/users/junhaodong/events{/privacy}","received_events_url":"https://api.github.com/users/junhaodong/received_events","type":"User","site_admin":false},"repo":{"id":47234996,"name":"jana","full_name":"junhaodong/jana","owner":{"login":"junhaodong","id":5431678,"avatar_url":"https://avatars.githubusercontent.com/u/5431678?v=3","gravatar_id":"","url":"https://api.github.com/users/junhaodong","html_url":"https://github.com/junhaodong","followers_url":"https://api.github.com/users/junhaodong/followers","following_url":"https://api.github.com/users/junhaodong/following{/other_user}","gists_url":"https://api.github.com/users/junhaodong/gists{/gist_id}","starred_url":"https://api.github.com/users/junhaodong/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/junhaodong/subscriptions","organizations_url":"https://api.github.com/users/junhaodong/orgs","repos_url":"https://api.github.com/users/junhaodong/repos","events_url":"https://api.github.com/users/junhaodong/events{/privacy}","received_events_url":"https://api.github.com/users/junhaodong/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/junhaodong/jana","description":"Coding Challenge","fork":false,"url":"https://api.github.com/repos/junhaodong/jana","forks_url":"https://api.github.com/repos/junhaodong/jana/forks","keys_url":"https://api.github.com/repos/junhaodong/jana/keys{/key_id}","collaborators_url":"https://api.github.com/repos/junhaodong/jana/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/junhaodong/jana/teams","hooks_url":"https://api.github.com/repos/junhaodong/jana/hooks","issue_events_url":"https://api.github.com/repos/junhaodong/jana/issues/events{/number}","events_url":"https://api.github.com/repos/junhaodong/jana/events","assignees_url":"https://api.github.com/repos/junhaodong/jana/assignees{/user}","branches_url":"https://api.github.com/repos/junhaodong/jana/branches{/branch}","tags_url":"https://api.github.com/repos/junhaodong/jana/tags","blobs_url":"https://api.github.com/repos/junhaodong/jana/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/junhaodong/jana/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/junhaodong/jana/git/refs{/sha}","trees_url":"https://api.github.com/repos/junhaodong/jana/git/trees{/sha}","statuses_url":"https://api.github.com/repos/junhaodong/jana/statuses/{sha}","languages_url":"https://api.github.com/repos/junhaodong/jana/languages","stargazers_url":"https://api.github.com/repos/junhaodong/jana/stargazers","contributors_url":"https://api.github.com/repos/junhaodong/jana/contributors","subscribers_url":"https://api.github.com/repos/junhaodong/jana/subscribers","subscription_url":"https://api.github.com/repos/junhaodong/jana/subscription","commits_url":"https://api.github.com/repos/junhaodong/jana/commits{/sha}","git_commits_url":"https://api.github.com/repos/junhaodong/jana/git/commits{/sha}","comments_url":"https://api.github.com/repos/junhaodong/jana/comments{/number}","issue_comment_url":"https://api.github.com/repos/junhaodong/jana/issues/comments{/number}","contents_url":"https://api.github.com/repos/junhaodong/jana/contents/{+path}","compare_url":"https://api.github.com/repos/junhaodong/jana/compare/{base}...{head}","merges_url":"https://api.github.com/repos/junhaodong/jana/merges","archive_url":"https://api.github.com/repos/junhaodong/jana/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/junhaodong/jana/downloads","issues_url":"https://api.github.com/repos/junhaodong/jana/issues{/number}","pulls_url":"https://api.github.com/repos/junhaodong/jana/pulls{/number}","milestones_url":"https://api.github.com/repos/junhaodong/jana/milestones{/number}","notifications_url":"https://api.github.com/repos/junhaodong/jana/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/junhaodong/jana/labels{/name}","releases_url":"https://api.github.com/repos/junhaodong/jana/releases{/id}","created_at":"2015-12-02T03:35:41Z","updated_at":"2015-12-02T03:35:41Z","pushed_at":"2015-12-04T07:52:55Z","git_url":"git://github.com/junhaodong/jana.git","ssh_url":"git@github.com:junhaodong/jana.git","clone_url":"https://github.com/junhaodong/jana.git","svn_url":"https://github.com/junhaodong/jana","homepage":null,"size":30,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/junhaodong/jana/pulls/1"},"html":{"href":"https://github.com/junhaodong/jana/pull/1"},"issue":{"href":"https://api.github.com/repos/junhaodong/jana/issues/1"},"comments":{"href":"https://api.github.com/repos/junhaodong/jana/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/junhaodong/jana/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/junhaodong/jana/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/junhaodong/jana/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/junhaodong/jana/statuses/3d1f0df463dbc903f74904e35540070903c4d4b6"}}}}