{"action":"created","comment":{"url":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/comments/23225188","id":23225188,"diff_hunk":"@@ -127,8 +127,21 @@ public String createPrefixForInternalLinks(String sourceUrl) {\n     return prefixForInternalLinks;\n   }\n \n+    private String deletePHP(String url) {\n+        if(url.contains(\"?\")) {\n+            return url.substring(0, url.indexOf('?'));\n+        }\n+        return url;\n+    }\n+\n     public String normalize(String urlString)\n             throws MalformedURLException {\n+\n+        urlString = urlString.trim();\n+\n+        urlString = deletePHP(urlString);\n+\n+\n         if (\"\".equals(urlString))                     // permit empty","path":"extraction/src/main/java/io/ssc/trackthetrackers/extraction/resources/URLNormalizer.java","position":19,"original_position":19,"commit_id":"85aa2ba45413369caab319c88b4b48d7a956403e","original_commit_id":"85aa2ba45413369caab319c88b4b48d7a956403e","user":{"login":"sscdotopen","id":409707,"avatar_url":"https://avatars.githubusercontent.com/u/409707?v=3","gravatar_id":"","url":"https://api.github.com/users/sscdotopen","html_url":"https://github.com/sscdotopen","followers_url":"https://api.github.com/users/sscdotopen/followers","following_url":"https://api.github.com/users/sscdotopen/following{/other_user}","gists_url":"https://api.github.com/users/sscdotopen/gists{/gist_id}","starred_url":"https://api.github.com/users/sscdotopen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sscdotopen/subscriptions","organizations_url":"https://api.github.com/users/sscdotopen/orgs","repos_url":"https://api.github.com/users/sscdotopen/repos","events_url":"https://api.github.com/users/sscdotopen/events{/privacy}","received_events_url":"https://api.github.com/users/sscdotopen/received_events","type":"User","site_admin":false},"body":"please always use braces with if","created_at":"2015-01-20T14:33:22Z","updated_at":"2015-01-20T14:33:22Z","html_url":"https://github.com/sscdotopen/trackthetrackers/pull/5#discussion_r23225188","pull_request_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/5","_links":{"self":{"href":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/comments/23225188"},"html":{"href":"https://github.com/sscdotopen/trackthetrackers/pull/5#discussion_r23225188"},"pull_request":{"href":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/5"}}},"pull_request":{"url":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/5","id":27619834,"html_url":"https://github.com/sscdotopen/trackthetrackers/pull/5","diff_url":"https://github.com/sscdotopen/trackthetrackers/pull/5.diff","patch_url":"https://github.com/sscdotopen/trackthetrackers/pull/5.patch","issue_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/issues/5","number":5,"state":"open","locked":false,"title":"Google parser","user":{"login":"FelixNeutatz","id":5217389,"avatar_url":"https://avatars.githubusercontent.com/u/5217389?v=3","gravatar_id":"","url":"https://api.github.com/users/FelixNeutatz","html_url":"https://github.com/FelixNeutatz","followers_url":"https://api.github.com/users/FelixNeutatz/followers","following_url":"https://api.github.com/users/FelixNeutatz/following{/other_user}","gists_url":"https://api.github.com/users/FelixNeutatz/gists{/gist_id}","starred_url":"https://api.github.com/users/FelixNeutatz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/FelixNeutatz/subscriptions","organizations_url":"https://api.github.com/users/FelixNeutatz/orgs","repos_url":"https://api.github.com/users/FelixNeutatz/repos","events_url":"https://api.github.com/users/FelixNeutatz/events{/privacy}","received_events_url":"https://api.github.com/users/FelixNeutatz/received_events","type":"User","site_admin":false},"body":"This pull request contains:\r\n\r\n- Integration of Google Javascript Parser (now we parse all Javascript Strings and search for URLs)\r\n- Minor fixes for URLNormalizer\r\n- Fixing commons dependencies\r\n\r\nMetrics:\r\nExecution time for CommonCrawlTest (250MB): 154 s\r\nOutput size of Parquet files: 1.2 MB","created_at":"2015-01-19T16:44:54Z","updated_at":"2015-01-20T14:33:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"765e24919eb59e830105384226c27df9f1a650f1","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/5/commits","review_comments_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/5/comments","review_comment_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/comments/{number}","comments_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/issues/5/comments","statuses_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/statuses/85aa2ba45413369caab319c88b4b48d7a956403e","head":{"label":"FelixNeutatz:GoogleParser","ref":"GoogleParser","sha":"85aa2ba45413369caab319c88b4b48d7a956403e","user":{"login":"FelixNeutatz","id":5217389,"avatar_url":"https://avatars.githubusercontent.com/u/5217389?v=3","gravatar_id":"","url":"https://api.github.com/users/FelixNeutatz","html_url":"https://github.com/FelixNeutatz","followers_url":"https://api.github.com/users/FelixNeutatz/followers","following_url":"https://api.github.com/users/FelixNeutatz/following{/other_user}","gists_url":"https://api.github.com/users/FelixNeutatz/gists{/gist_id}","starred_url":"https://api.github.com/users/FelixNeutatz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/FelixNeutatz/subscriptions","organizations_url":"https://api.github.com/users/FelixNeutatz/orgs","repos_url":"https://api.github.com/users/FelixNeutatz/repos","events_url":"https://api.github.com/users/FelixNeutatz/events{/privacy}","received_events_url":"https://api.github.com/users/FelixNeutatz/received_events","type":"User","site_admin":false},"repo":{"id":26955619,"name":"trackthetrackers","full_name":"FelixNeutatz/trackthetrackers","owner":{"login":"FelixNeutatz","id":5217389,"avatar_url":"https://avatars.githubusercontent.com/u/5217389?v=3","gravatar_id":"","url":"https://api.github.com/users/FelixNeutatz","html_url":"https://github.com/FelixNeutatz","followers_url":"https://api.github.com/users/FelixNeutatz/followers","following_url":"https://api.github.com/users/FelixNeutatz/following{/other_user}","gists_url":"https://api.github.com/users/FelixNeutatz/gists{/gist_id}","starred_url":"https://api.github.com/users/FelixNeutatz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/FelixNeutatz/subscriptions","organizations_url":"https://api.github.com/users/FelixNeutatz/orgs","repos_url":"https://api.github.com/users/FelixNeutatz/repos","events_url":"https://api.github.com/users/FelixNeutatz/events{/privacy}","received_events_url":"https://api.github.com/users/FelixNeutatz/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/FelixNeutatz/trackthetrackers","description":"","fork":true,"url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers","forks_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/forks","keys_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/keys{/key_id}","collaborators_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/teams","hooks_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/hooks","issue_events_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/issues/events{/number}","events_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/events","assignees_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/assignees{/user}","branches_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/branches{/branch}","tags_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/tags","blobs_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/git/refs{/sha}","trees_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/git/trees{/sha}","statuses_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/statuses/{sha}","languages_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/languages","stargazers_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/stargazers","contributors_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/contributors","subscribers_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/subscribers","subscription_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/subscription","commits_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/commits{/sha}","git_commits_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/git/commits{/sha}","comments_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/comments{/number}","issue_comment_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/issues/comments/{number}","contents_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/contents/{+path}","compare_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/compare/{base}...{head}","merges_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/merges","archive_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/downloads","issues_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/issues{/number}","pulls_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/pulls{/number}","milestones_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/milestones{/number}","notifications_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/labels{/name}","releases_url":"https://api.github.com/repos/FelixNeutatz/trackthetrackers/releases{/id}","created_at":"2014-11-21T10:21:15Z","updated_at":"2014-12-11T15:00:02Z","pushed_at":"2015-01-19T16:39:43Z","git_url":"git://github.com/FelixNeutatz/trackthetrackers.git","ssh_url":"git@github.com:FelixNeutatz/trackthetrackers.git","clone_url":"https://github.com/FelixNeutatz/trackthetrackers.git","svn_url":"https://github.com/FelixNeutatz/trackthetrackers","homepage":null,"size":244484,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"sscdotopen:master","ref":"master","sha":"c269fea4747338c362c9c01c36f90d25c43fb72a","user":{"login":"sscdotopen","id":409707,"avatar_url":"https://avatars.githubusercontent.com/u/409707?v=3","gravatar_id":"","url":"https://api.github.com/users/sscdotopen","html_url":"https://github.com/sscdotopen","followers_url":"https://api.github.com/users/sscdotopen/followers","following_url":"https://api.github.com/users/sscdotopen/following{/other_user}","gists_url":"https://api.github.com/users/sscdotopen/gists{/gist_id}","starred_url":"https://api.github.com/users/sscdotopen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sscdotopen/subscriptions","organizations_url":"https://api.github.com/users/sscdotopen/orgs","repos_url":"https://api.github.com/users/sscdotopen/repos","events_url":"https://api.github.com/users/sscdotopen/events{/privacy}","received_events_url":"https://api.github.com/users/sscdotopen/received_events","type":"User","site_admin":false},"repo":{"id":26857656,"name":"trackthetrackers","full_name":"sscdotopen/trackthetrackers","owner":{"login":"sscdotopen","id":409707,"avatar_url":"https://avatars.githubusercontent.com/u/409707?v=3","gravatar_id":"","url":"https://api.github.com/users/sscdotopen","html_url":"https://github.com/sscdotopen","followers_url":"https://api.github.com/users/sscdotopen/followers","following_url":"https://api.github.com/users/sscdotopen/following{/other_user}","gists_url":"https://api.github.com/users/sscdotopen/gists{/gist_id}","starred_url":"https://api.github.com/users/sscdotopen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sscdotopen/subscriptions","organizations_url":"https://api.github.com/users/sscdotopen/orgs","repos_url":"https://api.github.com/users/sscdotopen/repos","events_url":"https://api.github.com/users/sscdotopen/events{/privacy}","received_events_url":"https://api.github.com/users/sscdotopen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/sscdotopen/trackthetrackers","description":"","fork":false,"url":"https://api.github.com/repos/sscdotopen/trackthetrackers","forks_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/forks","keys_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/keys{/key_id}","collaborators_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/teams","hooks_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/hooks","issue_events_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/issues/events{/number}","events_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/events","assignees_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/assignees{/user}","branches_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/branches{/branch}","tags_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/tags","blobs_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/git/refs{/sha}","trees_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/git/trees{/sha}","statuses_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/statuses/{sha}","languages_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/languages","stargazers_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/stargazers","contributors_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/contributors","subscribers_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/subscribers","subscription_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/subscription","commits_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/commits{/sha}","git_commits_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/git/commits{/sha}","comments_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/comments{/number}","issue_comment_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/issues/comments/{number}","contents_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/contents/{+path}","compare_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/compare/{base}...{head}","merges_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/merges","archive_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/downloads","issues_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/issues{/number}","pulls_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls{/number}","milestones_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/milestones{/number}","notifications_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/labels{/name}","releases_url":"https://api.github.com/repos/sscdotopen/trackthetrackers/releases{/id}","created_at":"2014-11-19T11:09:15Z","updated_at":"2014-12-11T06:49:28Z","pushed_at":"2014-12-11T06:49:28Z","git_url":"git://github.com/sscdotopen/trackthetrackers.git","ssh_url":"git@github.com:sscdotopen/trackthetrackers.git","clone_url":"https://github.com/sscdotopen/trackthetrackers.git","svn_url":"https://github.com/sscdotopen/trackthetrackers","homepage":null,"size":244416,"stargazers_count":1,"watchers_count":1,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":1,"forks":1,"open_issues":1,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/5"},"html":{"href":"https://github.com/sscdotopen/trackthetrackers/pull/5"},"issue":{"href":"https://api.github.com/repos/sscdotopen/trackthetrackers/issues/5"},"comments":{"href":"https://api.github.com/repos/sscdotopen/trackthetrackers/issues/5/comments"},"review_comments":{"href":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/5/comments"},"review_comment":{"href":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/sscdotopen/trackthetrackers/pulls/5/commits"},"statuses":{"href":"https://api.github.com/repos/sscdotopen/trackthetrackers/statuses/85aa2ba45413369caab319c88b4b48d7a956403e"}}}}