{"action":"created","comment":{"url":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/comments/29118464","id":29118464,"diff_hunk":"@@ -75,7 +75,7 @@ def url_for(base_url, page = 1):\n def report_from(result, base_url):\n   report = {\n     'inspector': 'gsa',\n-    'inspector_url': 'http://gsaig.gov/',\n+    'inspector_url': 'http://www.gsaig.gov/',","path":"inspectors/gsa.py","position":5,"original_position":5,"commit_id":"437ce33c515e5d4b502e31189a2066180a47c0ce","original_commit_id":"437ce33c515e5d4b502e31189a2066180a47c0ce","user":{"login":"konklone","id":4592,"avatar_url":"https://avatars.githubusercontent.com/u/4592?v=3","gravatar_id":"","url":"https://api.github.com/users/konklone","html_url":"https://github.com/konklone","followers_url":"https://api.github.com/users/konklone/followers","following_url":"https://api.github.com/users/konklone/following{/other_user}","gists_url":"https://api.github.com/users/konklone/gists{/gist_id}","starred_url":"https://api.github.com/users/konklone/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/konklone/subscriptions","organizations_url":"https://api.github.com/users/konklone/orgs","repos_url":"https://api.github.com/users/konklone/repos","events_url":"https://api.github.com/users/konklone/events{/privacy}","received_events_url":"https://api.github.com/users/konklone/received_events","type":"User","site_admin":false},"body":"Looks like the GSA IG has moved to HTTPS: https://www.gsaig.gov","created_at":"2015-04-27T02:48:24Z","updated_at":"2015-04-27T02:48:24Z","html_url":"https://github.com/unitedstates/inspectors-general/pull/227#discussion_r29118464","pull_request_url":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/227","_links":{"self":{"href":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/comments/29118464"},"html":{"href":"https://github.com/unitedstates/inspectors-general/pull/227#discussion_r29118464"},"pull_request":{"href":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/227"}}},"pull_request":{"url":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/227","id":34133584,"html_url":"https://github.com/unitedstates/inspectors-general/pull/227","diff_url":"https://github.com/unitedstates/inspectors-general/pull/227.diff","patch_url":"https://github.com/unitedstates/inspectors-general/pull/227.patch","issue_url":"https://api.github.com/repos/unitedstates/inspectors-general/issues/227","number":227,"state":"open","locked":false,"title":"(WIP) Detect \"Soft 404s\"","user":{"login":"divergentdave","id":181772,"avatar_url":"https://avatars.githubusercontent.com/u/181772?v=3","gravatar_id":"","url":"https://api.github.com/users/divergentdave","html_url":"https://github.com/divergentdave","followers_url":"https://api.github.com/users/divergentdave/followers","following_url":"https://api.github.com/users/divergentdave/following{/other_user}","gists_url":"https://api.github.com/users/divergentdave/gists{/gist_id}","starred_url":"https://api.github.com/users/divergentdave/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/divergentdave/subscriptions","organizations_url":"https://api.github.com/users/divergentdave/orgs","repos_url":"https://api.github.com/users/divergentdave/repos","events_url":"https://api.github.com/users/divergentdave/events{/privacy}","received_events_url":"https://api.github.com/users/divergentdave/received_events","type":"User","site_admin":false},"body":"Some servers do not properly return an error status code when they can't find the page for a URL. This pull request will register a requests TransportAdapter for these servers to detect such cases and throw an exception.\r\n\r\nIn most cases, the server first gives a redirect to a \"page not found\" page, which is in turn served with a 200 status code. In those cases, the TransportAdapter checks the redirect URL against a blacklist, and raises an exception based on that. For other sites where no redirect is involved, and the server always responds with a status code of 200, the TransportAdapter will check a telltale header, read the data, and check it for specific strings. If the TransportAdapter had to read the body of a response, it will mock up a new response to pass along the data.\r\n\r\nSo far, I found a lot of broken links in the DOD and Smithsonian scrapers; I'll investigate those and round out this PR with appropriate fixes.\r\n\r\n- [x] cftc (No broken links found)\r\n- [ ] dod (Fixed some broken links, more to check)\r\n- [x] exim (No broken links found)\r\n- [x] fec (No broken links found)\r\n- [x] gpo (No broken links found)\r\n- [x] cpb (No broken links found)\r\n- [x] ncua (No broken links found)\r\n- [ ] smithsonian (Many broken links to fix)","created_at":"2015-04-27T00:56:52Z","updated_at":"2015-04-27T02:48:24Z","closed_at":null,"merged_at":null,"merge_commit_sha":"afc9c00b39b81acf328fae22bf9de7f757b8e193","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/227/commits","review_comments_url":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/227/comments","review_comment_url":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/comments{/number}","comments_url":"https://api.github.com/repos/unitedstates/inspectors-general/issues/227/comments","statuses_url":"https://api.github.com/repos/unitedstates/inspectors-general/statuses/437ce33c515e5d4b502e31189a2066180a47c0ce","head":{"label":"unitedstates:soft-404-handler","ref":"soft-404-handler","sha":"437ce33c515e5d4b502e31189a2066180a47c0ce","user":{"login":"unitedstates","id":2200203,"avatar_url":"https://avatars.githubusercontent.com/u/2200203?v=3","gravatar_id":"","url":"https://api.github.com/users/unitedstates","html_url":"https://github.com/unitedstates","followers_url":"https://api.github.com/users/unitedstates/followers","following_url":"https://api.github.com/users/unitedstates/following{/other_user}","gists_url":"https://api.github.com/users/unitedstates/gists{/gist_id}","starred_url":"https://api.github.com/users/unitedstates/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/unitedstates/subscriptions","organizations_url":"https://api.github.com/users/unitedstates/orgs","repos_url":"https://api.github.com/users/unitedstates/repos","events_url":"https://api.github.com/users/unitedstates/events{/privacy}","received_events_url":"https://api.github.com/users/unitedstates/received_events","type":"Organization","site_admin":false},"repo":{"id":13230665,"name":"inspectors-general","full_name":"unitedstates/inspectors-general","owner":{"login":"unitedstates","id":2200203,"avatar_url":"https://avatars.githubusercontent.com/u/2200203?v=3","gravatar_id":"","url":"https://api.github.com/users/unitedstates","html_url":"https://github.com/unitedstates","followers_url":"https://api.github.com/users/unitedstates/followers","following_url":"https://api.github.com/users/unitedstates/following{/other_user}","gists_url":"https://api.github.com/users/unitedstates/gists{/gist_id}","starred_url":"https://api.github.com/users/unitedstates/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/unitedstates/subscriptions","organizations_url":"https://api.github.com/users/unitedstates/orgs","repos_url":"https://api.github.com/users/unitedstates/repos","events_url":"https://api.github.com/users/unitedstates/events{/privacy}","received_events_url":"https://api.github.com/users/unitedstates/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/unitedstates/inspectors-general","description":"Collecting reports from Inspectors General across the US federal government.","fork":false,"url":"https://api.github.com/repos/unitedstates/inspectors-general","forks_url":"https://api.github.com/repos/unitedstates/inspectors-general/forks","keys_url":"https://api.github.com/repos/unitedstates/inspectors-general/keys{/key_id}","collaborators_url":"https://api.github.com/repos/unitedstates/inspectors-general/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/unitedstates/inspectors-general/teams","hooks_url":"https://api.github.com/repos/unitedstates/inspectors-general/hooks","issue_events_url":"https://api.github.com/repos/unitedstates/inspectors-general/issues/events{/number}","events_url":"https://api.github.com/repos/unitedstates/inspectors-general/events","assignees_url":"https://api.github.com/repos/unitedstates/inspectors-general/assignees{/user}","branches_url":"https://api.github.com/repos/unitedstates/inspectors-general/branches{/branch}","tags_url":"https://api.github.com/repos/unitedstates/inspectors-general/tags","blobs_url":"https://api.github.com/repos/unitedstates/inspectors-general/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/unitedstates/inspectors-general/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/unitedstates/inspectors-general/git/refs{/sha}","trees_url":"https://api.github.com/repos/unitedstates/inspectors-general/git/trees{/sha}","statuses_url":"https://api.github.com/repos/unitedstates/inspectors-general/statuses/{sha}","languages_url":"https://api.github.com/repos/unitedstates/inspectors-general/languages","stargazers_url":"https://api.github.com/repos/unitedstates/inspectors-general/stargazers","contributors_url":"https://api.github.com/repos/unitedstates/inspectors-general/contributors","subscribers_url":"https://api.github.com/repos/unitedstates/inspectors-general/subscribers","subscription_url":"https://api.github.com/repos/unitedstates/inspectors-general/subscription","commits_url":"https://api.github.com/repos/unitedstates/inspectors-general/commits{/sha}","git_commits_url":"https://api.github.com/repos/unitedstates/inspectors-general/git/commits{/sha}","comments_url":"https://api.github.com/repos/unitedstates/inspectors-general/comments{/number}","issue_comment_url":"https://api.github.com/repos/unitedstates/inspectors-general/issues/comments{/number}","contents_url":"https://api.github.com/repos/unitedstates/inspectors-general/contents/{+path}","compare_url":"https://api.github.com/repos/unitedstates/inspectors-general/compare/{base}...{head}","merges_url":"https://api.github.com/repos/unitedstates/inspectors-general/merges","archive_url":"https://api.github.com/repos/unitedstates/inspectors-general/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/unitedstates/inspectors-general/downloads","issues_url":"https://api.github.com/repos/unitedstates/inspectors-general/issues{/number}","pulls_url":"https://api.github.com/repos/unitedstates/inspectors-general/pulls{/number}","milestones_url":"https://api.github.com/repos/unitedstates/inspectors-general/milestones{/number}","notifications_url":"https://api.github.com/repos/unitedstates/inspectors-general/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/unitedstates/inspectors-general/labels{/name}","releases_url":"https://api.github.com/repos/unitedstates/inspectors-general/releases{/id}","created_at":"2013-09-30T23:55:47Z","updated_at":"2015-04-21T22:39:21Z","pushed_at":"2015-04-27T00:56:52Z","git_url":"git://github.com/unitedstates/inspectors-general.git","ssh_url":"git@github.com:unitedstates/inspectors-general.git","clone_url":"https://github.com/unitedstates/inspectors-general.git","svn_url":"https://github.com/unitedstates/inspectors-general","homepage":"https://sunlightfoundation.com/blog/2014/11/07/opengov-voices-opening-up-government-reports-through-teamwork-and-open-data/","size":4263,"stargazers_count":44,"watchers_count":44,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":15,"mirror_url":null,"open_issues_count":14,"forks":15,"open_issues":14,"watchers":44,"default_branch":"master"}},"base":{"label":"unitedstates:master","ref":"master","sha":"3d3fe092c11a340ab8c49b37382ae6a94b0ac744","user":{"login":"unitedstates","id":2200203,"avatar_url":"https://avatars.githubusercontent.com/u/2200203?v=3","gravatar_id":"","url":"https://api.github.com/users/unitedstates","html_url":"https://github.com/unitedstates","followers_url":"https://api.github.com/users/unitedstates/followers","following_url":"https://api.github.com/users/unitedstates/following{/other_user}","gists_url":"https://api.github.com/users/unitedstates/gists{/gist_id}","starred_url":"https://api.github.com/users/unitedstates/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/unitedstates/subscriptions","organizations_url":"https://api.github.com/users/unitedstates/orgs","repos_url":"https://api.github.com/users/unitedstates/repos","events_url":"https://api.github.com/users/unitedstates/events{/privacy}","received_events_url":"https://api.github.com/users/unitedstates/received_events","type":"Organization","site_admin":false},"repo":{"id":13230665,"name":"inspectors-general","full_name":"unitedstates/inspectors-general","owner":{"login":"unitedstates","id":2200203,"avatar_url":"https://avatars.githubusercontent.com/u/2200203?v=3","gravatar_id":"","url":"https://api.github.com/users/unitedstates","html_url":"https://github.com/unitedstates","followers_url":"https://api.github.com/users/unitedstates/followers","following_url":"https://api.github.com/users/unitedstates/following{/other_user}","gists_url":"https://api.github.com/users/unitedstates/gists{/gist_id}","starred_url":"https://api.github.com/users/unitedstates/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/unitedstates/subscriptions","organizations_url":"https://api.github.com/users/unitedstates/orgs","repos_url":"https://api.github.com/users/unitedstates/repos","events_url":"https://api.github.com/users/unitedstates/events{/privacy}","received_events_url":"https://api.github.com/users/unitedstates/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/unitedstates/inspectors-general","description":"Collecting reports from Inspectors General across the US federal government.","fork":false,"url":"https://api.github.com/repos/unitedstates/inspectors-general","forks_url":"https://api.github.com/repos/unitedstates/inspectors-general/forks","keys_url":"https://api.github.com/repos/unitedstates/inspectors-general/keys{/key_id}","collaborators_url":"https://api.github.com/repos/unitedstates/inspectors-general/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/unitedstates/inspectors-general/teams","hooks_url":"https://api.github.com/repos/unitedstates/inspectors-general/hooks","issue_events_url":"https://api.github.com/repos/unitedstates/inspectors-general/issues/events{/number}","events_url":"https://api.github.com/repos/unitedstates/inspectors-general/events","assignees_url":"https://api.github.com/repos/unitedstates/inspectors-general/assignees{/user}","branches_url":"https://api.github.com/repos/unitedstates/inspectors-general/branches{/branch}","tags_url":"https://api.github.com/repos/unitedstates/inspectors-general/tags","blobs_url":"https://api.github.com/repos/unitedstates/inspectors-general/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/unitedstates/inspectors-general/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/unitedstates/inspectors-general/git/refs{/sha}","trees_url":"https://api.github.com/repos/unitedstates/inspectors-general/git/trees{/sha}","statuses_url":"https://api.github.com/repos/unitedstates/inspectors-general/statuses/{sha}","languages_url":"https://api.github.com/repos/unitedstates/inspectors-general/languages","stargazers_url":"https://api.github.com/repos/unitedstates/inspectors-general/stargazers","contributors_url":"https://api.github.com/repos/unitedstates/inspectors-general/contributors","subscribers_url":"https://api.github.com/repos/unitedstates/inspectors-general/subscribers","subscription_url":"https://api.github.com/repos/unitedstates/inspectors-general/subscription","commits_url":"https://api.github.com/repos/unitedstates/inspectors-general/commits{/sha}","git_commits_url":"https://api.github.com/repos/unitedstates/inspectors-general/git/commits{/sha}","comments_url":"https://api.github.com/repos/unitedstates/inspectors-general/comments{/number}","issue_comment_url":"https://api.github.com/repos/unitedstates/inspectors-general/issues/comments{/number}","contents_url":"https://api.github.com/repos/unitedstates/inspectors-general/contents/{+path}","compare_url":"https://api.github.com/repos/unitedstates/inspectors-general/compare/{base}...{head}","merges_url":"https://api.github.com/repos/unitedstates/inspectors-general/merges","archive_url":"https://api.github.com/repos/unitedstates/inspectors-general/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/unitedstates/inspectors-general/downloads","issues_url":"https://api.github.com/repos/unitedstates/inspectors-general/issues{/number}","pulls_url":"https://api.github.com/repos/unitedstates/inspectors-general/pulls{/number}","milestones_url":"https://api.github.com/repos/unitedstates/inspectors-general/milestones{/number}","notifications_url":"https://api.github.com/repos/unitedstates/inspectors-general/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/unitedstates/inspectors-general/labels{/name}","releases_url":"https://api.github.com/repos/unitedstates/inspectors-general/releases{/id}","created_at":"2013-09-30T23:55:47Z","updated_at":"2015-04-21T22:39:21Z","pushed_at":"2015-04-27T00:56:52Z","git_url":"git://github.com/unitedstates/inspectors-general.git","ssh_url":"git@github.com:unitedstates/inspectors-general.git","clone_url":"https://github.com/unitedstates/inspectors-general.git","svn_url":"https://github.com/unitedstates/inspectors-general","homepage":"https://sunlightfoundation.com/blog/2014/11/07/opengov-voices-opening-up-government-reports-through-teamwork-and-open-data/","size":4263,"stargazers_count":44,"watchers_count":44,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":15,"mirror_url":null,"open_issues_count":14,"forks":15,"open_issues":14,"watchers":44,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/227"},"html":{"href":"https://github.com/unitedstates/inspectors-general/pull/227"},"issue":{"href":"https://api.github.com/repos/unitedstates/inspectors-general/issues/227"},"comments":{"href":"https://api.github.com/repos/unitedstates/inspectors-general/issues/227/comments"},"review_comments":{"href":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/227/comments"},"review_comment":{"href":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/unitedstates/inspectors-general/pulls/227/commits"},"statuses":{"href":"https://api.github.com/repos/unitedstates/inspectors-general/statuses/437ce33c515e5d4b502e31189a2066180a47c0ce"}}}}