{"action":"created","comment":{"url":"https://api.github.com/repos/sunlightlabs/openstates/pulls/comments/40130316","id":40130316,"diff_hunk":"@@ -194,21 +194,31 @@ def scrape_bill(self, bill):\n         if srow:\n             bill['subjects'] = [s.strip() for s in srow if s.strip()]\n \n+        # Attempt to find link to bill text/documents.\n         ver_link = page.xpath(\"//a[contains(@href, 'display_ps.asp')]\")[0]\n         ver_url = ver_link.get('href')\n+\n         try:\n             ver_html = self.get(ver_url, retry_on_404=True).text\n-        except socket.timeout:\n+        except (socket.timeout, requests.exceptions.HTTPError):\n             pass\n         else:\n             if ver_html:\n                 vdoc = lxml.html.fromstring(ver_html)\n-                vdoc.make_links_absolute(ver_url)\n-                # various versions: billtexts, billdocs, billpdfs\n-                vurl = vdoc.xpath('//a[contains(@href, \"billtexts/\")]/@href')\n-                if vurl:\n-                    bill.add_version('Initial Version', vurl[0],\n-                                     mimetype='text/html')\n+\n+                # Check whether the bill text is missing.\n+                bill_text_missing = vdoc.xpath('string(//div[@id = \"sec0\"])').strip()\n+                match = re.search('Cannot find requested paper', bill_text_missing)\n+\n+                if not match:\n+                    vdoc.make_links_absolute(ver_url)\n+\n+                    # various versions: billtexts, billdocs, billpdfs\n+                    vurl = vdoc.xpath('//a[contains(@href, \"billtexts/\")]/@href')\n+\n+                    if vurl:\n+                        bill.add_version('Initial Version', vurl[0],","path":"openstates/me/bills.py","position":41,"original_position":34,"commit_id":"6b872a6ab67b24378ae780b383a73ef3fd55ca4b","original_commit_id":"ec7906d6f35c03daf9821115b22ebc125a952b4a","user":{"login":"loandy","id":865525,"avatar_url":"https://avatars.githubusercontent.com/u/865525?v=3","gravatar_id":"","url":"https://api.github.com/users/loandy","html_url":"https://github.com/loandy","followers_url":"https://api.github.com/users/loandy/followers","following_url":"https://api.github.com/users/loandy/following{/other_user}","gists_url":"https://api.github.com/users/loandy/gists{/gist_id}","starred_url":"https://api.github.com/users/loandy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/loandy/subscriptions","organizations_url":"https://api.github.com/users/loandy/orgs","repos_url":"https://api.github.com/users/loandy/repos","events_url":"https://api.github.com/users/loandy/events{/privacy}","received_events_url":"https://api.github.com/users/loandy/received_events","type":"User","site_admin":false},"body":"I made this change on a separate branch, which I'll make a pull request for in just a moment.","created_at":"2015-09-22T19:10:54Z","updated_at":"2015-09-22T19:10:54Z","html_url":"https://github.com/sunlightlabs/openstates/pull/841#discussion_r40130316","pull_request_url":"https://api.github.com/repos/sunlightlabs/openstates/pulls/841","_links":{"self":{"href":"https://api.github.com/repos/sunlightlabs/openstates/pulls/comments/40130316"},"html":{"href":"https://github.com/sunlightlabs/openstates/pull/841#discussion_r40130316"},"pull_request":{"href":"https://api.github.com/repos/sunlightlabs/openstates/pulls/841"}}},"pull_request":{"url":"https://api.github.com/repos/sunlightlabs/openstates/pulls/841","id":45667821,"html_url":"https://github.com/sunlightlabs/openstates/pull/841","diff_url":"https://github.com/sunlightlabs/openstates/pull/841.diff","patch_url":"https://github.com/sunlightlabs/openstates/pull/841.patch","issue_url":"https://api.github.com/repos/sunlightlabs/openstates/issues/841","number":841,"state":"open","locked":false,"title":"ME: Added cases for skipping retrieval of inaccessible bill document links.","user":{"login":"loandy","id":865525,"avatar_url":"https://avatars.githubusercontent.com/u/865525?v=3","gravatar_id":"","url":"https://api.github.com/users/loandy","html_url":"https://github.com/loandy","followers_url":"https://api.github.com/users/loandy/followers","following_url":"https://api.github.com/users/loandy/following{/other_user}","gists_url":"https://api.github.com/users/loandy/gists{/gist_id}","starred_url":"https://api.github.com/users/loandy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/loandy/subscriptions","organizations_url":"https://api.github.com/users/loandy/orgs","repos_url":"https://api.github.com/users/loandy/repos","events_url":"https://api.github.com/users/loandy/events{/privacy}","received_events_url":"https://api.github.com/users/loandy/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-09-22T14:10:39Z","updated_at":"2015-09-22T19:10:54Z","closed_at":null,"merged_at":null,"merge_commit_sha":"06e3f5d2cd098b174a42a85930614141de2e5f49","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/sunlightlabs/openstates/pulls/841/commits","review_comments_url":"https://api.github.com/repos/sunlightlabs/openstates/pulls/841/comments","review_comment_url":"https://api.github.com/repos/sunlightlabs/openstates/pulls/comments{/number}","comments_url":"https://api.github.com/repos/sunlightlabs/openstates/issues/841/comments","statuses_url":"https://api.github.com/repos/sunlightlabs/openstates/statuses/6b872a6ab67b24378ae780b383a73ef3fd55ca4b","head":{"label":"loandy:hotfix-me-bill-text-retrieval","ref":"hotfix-me-bill-text-retrieval","sha":"6b872a6ab67b24378ae780b383a73ef3fd55ca4b","user":{"login":"loandy","id":865525,"avatar_url":"https://avatars.githubusercontent.com/u/865525?v=3","gravatar_id":"","url":"https://api.github.com/users/loandy","html_url":"https://github.com/loandy","followers_url":"https://api.github.com/users/loandy/followers","following_url":"https://api.github.com/users/loandy/following{/other_user}","gists_url":"https://api.github.com/users/loandy/gists{/gist_id}","starred_url":"https://api.github.com/users/loandy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/loandy/subscriptions","organizations_url":"https://api.github.com/users/loandy/orgs","repos_url":"https://api.github.com/users/loandy/repos","events_url":"https://api.github.com/users/loandy/events{/privacy}","received_events_url":"https://api.github.com/users/loandy/received_events","type":"User","site_admin":false},"repo":{"id":42575153,"name":"openstates","full_name":"loandy/openstates","owner":{"login":"loandy","id":865525,"avatar_url":"https://avatars.githubusercontent.com/u/865525?v=3","gravatar_id":"","url":"https://api.github.com/users/loandy","html_url":"https://github.com/loandy","followers_url":"https://api.github.com/users/loandy/followers","following_url":"https://api.github.com/users/loandy/following{/other_user}","gists_url":"https://api.github.com/users/loandy/gists{/gist_id}","starred_url":"https://api.github.com/users/loandy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/loandy/subscriptions","organizations_url":"https://api.github.com/users/loandy/orgs","repos_url":"https://api.github.com/users/loandy/repos","events_url":"https://api.github.com/users/loandy/events{/privacy}","received_events_url":"https://api.github.com/users/loandy/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/loandy/openstates","description":"source for Open States' scrapers","fork":true,"url":"https://api.github.com/repos/loandy/openstates","forks_url":"https://api.github.com/repos/loandy/openstates/forks","keys_url":"https://api.github.com/repos/loandy/openstates/keys{/key_id}","collaborators_url":"https://api.github.com/repos/loandy/openstates/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/loandy/openstates/teams","hooks_url":"https://api.github.com/repos/loandy/openstates/hooks","issue_events_url":"https://api.github.com/repos/loandy/openstates/issues/events{/number}","events_url":"https://api.github.com/repos/loandy/openstates/events","assignees_url":"https://api.github.com/repos/loandy/openstates/assignees{/user}","branches_url":"https://api.github.com/repos/loandy/openstates/branches{/branch}","tags_url":"https://api.github.com/repos/loandy/openstates/tags","blobs_url":"https://api.github.com/repos/loandy/openstates/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/loandy/openstates/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/loandy/openstates/git/refs{/sha}","trees_url":"https://api.github.com/repos/loandy/openstates/git/trees{/sha}","statuses_url":"https://api.github.com/repos/loandy/openstates/statuses/{sha}","languages_url":"https://api.github.com/repos/loandy/openstates/languages","stargazers_url":"https://api.github.com/repos/loandy/openstates/stargazers","contributors_url":"https://api.github.com/repos/loandy/openstates/contributors","subscribers_url":"https://api.github.com/repos/loandy/openstates/subscribers","subscription_url":"https://api.github.com/repos/loandy/openstates/subscription","commits_url":"https://api.github.com/repos/loandy/openstates/commits{/sha}","git_commits_url":"https://api.github.com/repos/loandy/openstates/git/commits{/sha}","comments_url":"https://api.github.com/repos/loandy/openstates/comments{/number}","issue_comment_url":"https://api.github.com/repos/loandy/openstates/issues/comments{/number}","contents_url":"https://api.github.com/repos/loandy/openstates/contents/{+path}","compare_url":"https://api.github.com/repos/loandy/openstates/compare/{base}...{head}","merges_url":"https://api.github.com/repos/loandy/openstates/merges","archive_url":"https://api.github.com/repos/loandy/openstates/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/loandy/openstates/downloads","issues_url":"https://api.github.com/repos/loandy/openstates/issues{/number}","pulls_url":"https://api.github.com/repos/loandy/openstates/pulls{/number}","milestones_url":"https://api.github.com/repos/loandy/openstates/milestones{/number}","notifications_url":"https://api.github.com/repos/loandy/openstates/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/loandy/openstates/labels{/name}","releases_url":"https://api.github.com/repos/loandy/openstates/releases{/id}","created_at":"2015-09-16T08:37:02Z","updated_at":"2015-09-16T08:37:04Z","pushed_at":"2015-09-22T19:09:40Z","git_url":"git://github.com/loandy/openstates.git","ssh_url":"git@github.com:loandy/openstates.git","clone_url":"https://github.com/loandy/openstates.git","svn_url":"https://github.com/loandy/openstates","homepage":"http://openstates.org","size":20652,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"sunlightlabs:master","ref":"master","sha":"e6e1d905177bd221efab6c0088747241e17a53cc","user":{"login":"sunlightlabs","id":40420,"avatar_url":"https://avatars.githubusercontent.com/u/40420?v=3","gravatar_id":"","url":"https://api.github.com/users/sunlightlabs","html_url":"https://github.com/sunlightlabs","followers_url":"https://api.github.com/users/sunlightlabs/followers","following_url":"https://api.github.com/users/sunlightlabs/following{/other_user}","gists_url":"https://api.github.com/users/sunlightlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/sunlightlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sunlightlabs/subscriptions","organizations_url":"https://api.github.com/users/sunlightlabs/orgs","repos_url":"https://api.github.com/users/sunlightlabs/repos","events_url":"https://api.github.com/users/sunlightlabs/events{/privacy}","received_events_url":"https://api.github.com/users/sunlightlabs/received_events","type":"Organization","site_admin":false},"repo":{"id":138635,"name":"openstates","full_name":"sunlightlabs/openstates","owner":{"login":"sunlightlabs","id":40420,"avatar_url":"https://avatars.githubusercontent.com/u/40420?v=3","gravatar_id":"","url":"https://api.github.com/users/sunlightlabs","html_url":"https://github.com/sunlightlabs","followers_url":"https://api.github.com/users/sunlightlabs/followers","following_url":"https://api.github.com/users/sunlightlabs/following{/other_user}","gists_url":"https://api.github.com/users/sunlightlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/sunlightlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sunlightlabs/subscriptions","organizations_url":"https://api.github.com/users/sunlightlabs/orgs","repos_url":"https://api.github.com/users/sunlightlabs/repos","events_url":"https://api.github.com/users/sunlightlabs/events{/privacy}","received_events_url":"https://api.github.com/users/sunlightlabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/sunlightlabs/openstates","description":"source for Open States' scrapers","fork":false,"url":"https://api.github.com/repos/sunlightlabs/openstates","forks_url":"https://api.github.com/repos/sunlightlabs/openstates/forks","keys_url":"https://api.github.com/repos/sunlightlabs/openstates/keys{/key_id}","collaborators_url":"https://api.github.com/repos/sunlightlabs/openstates/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/sunlightlabs/openstates/teams","hooks_url":"https://api.github.com/repos/sunlightlabs/openstates/hooks","issue_events_url":"https://api.github.com/repos/sunlightlabs/openstates/issues/events{/number}","events_url":"https://api.github.com/repos/sunlightlabs/openstates/events","assignees_url":"https://api.github.com/repos/sunlightlabs/openstates/assignees{/user}","branches_url":"https://api.github.com/repos/sunlightlabs/openstates/branches{/branch}","tags_url":"https://api.github.com/repos/sunlightlabs/openstates/tags","blobs_url":"https://api.github.com/repos/sunlightlabs/openstates/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/sunlightlabs/openstates/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/sunlightlabs/openstates/git/refs{/sha}","trees_url":"https://api.github.com/repos/sunlightlabs/openstates/git/trees{/sha}","statuses_url":"https://api.github.com/repos/sunlightlabs/openstates/statuses/{sha}","languages_url":"https://api.github.com/repos/sunlightlabs/openstates/languages","stargazers_url":"https://api.github.com/repos/sunlightlabs/openstates/stargazers","contributors_url":"https://api.github.com/repos/sunlightlabs/openstates/contributors","subscribers_url":"https://api.github.com/repos/sunlightlabs/openstates/subscribers","subscription_url":"https://api.github.com/repos/sunlightlabs/openstates/subscription","commits_url":"https://api.github.com/repos/sunlightlabs/openstates/commits{/sha}","git_commits_url":"https://api.github.com/repos/sunlightlabs/openstates/git/commits{/sha}","comments_url":"https://api.github.com/repos/sunlightlabs/openstates/comments{/number}","issue_comment_url":"https://api.github.com/repos/sunlightlabs/openstates/issues/comments{/number}","contents_url":"https://api.github.com/repos/sunlightlabs/openstates/contents/{+path}","compare_url":"https://api.github.com/repos/sunlightlabs/openstates/compare/{base}...{head}","merges_url":"https://api.github.com/repos/sunlightlabs/openstates/merges","archive_url":"https://api.github.com/repos/sunlightlabs/openstates/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/sunlightlabs/openstates/downloads","issues_url":"https://api.github.com/repos/sunlightlabs/openstates/issues{/number}","pulls_url":"https://api.github.com/repos/sunlightlabs/openstates/pulls{/number}","milestones_url":"https://api.github.com/repos/sunlightlabs/openstates/milestones{/number}","notifications_url":"https://api.github.com/repos/sunlightlabs/openstates/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/sunlightlabs/openstates/labels{/name}","releases_url":"https://api.github.com/repos/sunlightlabs/openstates/releases{/id}","created_at":"2009-02-26T22:49:19Z","updated_at":"2015-09-12T17:44:39Z","pushed_at":"2015-09-22T16:39:01Z","git_url":"git://github.com/sunlightlabs/openstates.git","ssh_url":"git@github.com:sunlightlabs/openstates.git","clone_url":"https://github.com/sunlightlabs/openstates.git","svn_url":"https://github.com/sunlightlabs/openstates","homepage":"http://openstates.org","size":27515,"stargazers_count":352,"watchers_count":352,"language":"Python","has_issues":false,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":214,"mirror_url":null,"open_issues_count":42,"forks":214,"open_issues":42,"watchers":352,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/sunlightlabs/openstates/pulls/841"},"html":{"href":"https://github.com/sunlightlabs/openstates/pull/841"},"issue":{"href":"https://api.github.com/repos/sunlightlabs/openstates/issues/841"},"comments":{"href":"https://api.github.com/repos/sunlightlabs/openstates/issues/841/comments"},"review_comments":{"href":"https://api.github.com/repos/sunlightlabs/openstates/pulls/841/comments"},"review_comment":{"href":"https://api.github.com/repos/sunlightlabs/openstates/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/sunlightlabs/openstates/pulls/841/commits"},"statuses":{"href":"https://api.github.com/repos/sunlightlabs/openstates/statuses/6b872a6ab67b24378ae780b383a73ef3fd55ca4b"}}}}