{"action":"created","comment":{"url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/comments/25569918","id":25569918,"diff_hunk":"@@ -0,0 +1,107 @@\n+package com.digitalpebble.storm.crawler.filtering.basic;\n+\n+import java.io.BufferedReader;\n+import java.io.InputStream;\n+import java.io.InputStreamReader;\n+import java.net.MalformedURLException;\n+import java.net.URI;\n+import java.net.URISyntaxException;\n+import java.net.URL;\n+import java.nio.charset.StandardCharsets;\n+import java.util.ArrayList;\n+import java.util.Collections;\n+import java.util.List;\n+import java.util.Map;\n+\n+import com.digitalpebble.storm.crawler.Metadata;\n+import com.digitalpebble.storm.crawler.filtering.URLFilter;\n+import com.fasterxml.jackson.databind.JsonNode;\n+\n+import org.apache.http.NameValuePair;\n+import org.apache.http.client.utils.URIBuilder;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+\n+/**\n+ * Basic filter to remove query parameters from urls so parameters that don't change the content of\n+ * the page can be removed. An example would be a google analytics query parameter like\n+ * \"utm_campaign\" which might have several different values for a url that points to the same\n+ * content.\n+ */\n+public class QueryParameterNormalizer implements URLFilter {\n+\n+    private List<String> parametersToRemove;\n+    private static final Logger LOG = LoggerFactory.getLogger(QueryParameterNormalizer.class);\n+\n+    @Override\n+    public void configure(Map stormConf, JsonNode filterParams) {\n+        JsonNode filenameNode = filterParams.get(\"parameterNormalizerFile\");\n+        String rulesFileName = filenameNode.textValue();\n+        parametersToRemove = readRules(rulesFileName);\n+    }\n+\n+    @Override\n+    public String filter(URL sourceUrl, Metadata sourceMetadata, String urlToFilter) {\n+        URIBuilder uriBuilder;\n+        try {\n+            // Handle illegal characters by making a url first\n+            // this will clean illegal characters like |\n+            URL url = new URL(urlToFilter);\n+            URI uri =\n+                    new URI(url.getProtocol(), url.getAuthority(), url.getPath(), url.getQuery(),\n+                            url.getRef());\n+            uriBuilder = new URIBuilder(uri);\n+        } catch (URISyntaxException e) {\n+            LOG.warn(\"Invalid urlToFilter {}. {}\", urlToFilter, e);\n+            return urlToFilter;\n+        } catch (MalformedURLException e) {\n+            LOG.warn(\"Innvalid urlToFilter {}. {}\", urlToFilter, e);\n+            return urlToFilter;","path":"core/src/main/java/com/digitalpebble/storm/crawler/filtering/basic/QueryParameterNormalizer.java","position":60,"original_position":60,"commit_id":"45c6372b40f29394b163f077676a3fd5578a8beb","original_commit_id":"45c6372b40f29394b163f077676a3fd5578a8beb","user":{"login":"GuiForget","id":2585258,"avatar_url":"https://avatars.githubusercontent.com/u/2585258?v=3","gravatar_id":"","url":"https://api.github.com/users/GuiForget","html_url":"https://github.com/GuiForget","followers_url":"https://api.github.com/users/GuiForget/followers","following_url":"https://api.github.com/users/GuiForget/following{/other_user}","gists_url":"https://api.github.com/users/GuiForget/gists{/gist_id}","starred_url":"https://api.github.com/users/GuiForget/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GuiForget/subscriptions","organizations_url":"https://api.github.com/users/GuiForget/orgs","repos_url":"https://api.github.com/users/GuiForget/repos","events_url":"https://api.github.com/users/GuiForget/events{/privacy}","received_events_url":"https://api.github.com/users/GuiForget/received_events","type":"User","site_admin":false},"body":"Wouldn't you want to filter out the URLs that are not valid?","created_at":"2015-03-01T18:52:09Z","updated_at":"2015-03-01T18:52:09Z","html_url":"https://github.com/DigitalPebble/storm-crawler/pull/98#discussion_r25569918","pull_request_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/98","_links":{"self":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/comments/25569918"},"html":{"href":"https://github.com/DigitalPebble/storm-crawler/pull/98#discussion_r25569918"},"pull_request":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/98"}}},"pull_request":{"url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/98","id":30237867,"html_url":"https://github.com/DigitalPebble/storm-crawler/pull/98","diff_url":"https://github.com/DigitalPebble/storm-crawler/pull/98.diff","patch_url":"https://github.com/DigitalPebble/storm-crawler/pull/98.patch","issue_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/98","number":98,"state":"open","locked":false,"title":"Add a normalizer to remove specified query parameters.","user":{"login":"acinader","id":700572,"avatar_url":"https://avatars.githubusercontent.com/u/700572?v=3","gravatar_id":"","url":"https://api.github.com/users/acinader","html_url":"https://github.com/acinader","followers_url":"https://api.github.com/users/acinader/followers","following_url":"https://api.github.com/users/acinader/following{/other_user}","gists_url":"https://api.github.com/users/acinader/gists{/gist_id}","starred_url":"https://api.github.com/users/acinader/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acinader/subscriptions","organizations_url":"https://api.github.com/users/acinader/orgs","repos_url":"https://api.github.com/users/acinader/repos","events_url":"https://api.github.com/users/acinader/events{/privacy}","received_events_url":"https://api.github.com/users/acinader/received_events","type":"User","site_admin":false},"body":"Easier to reason about that writing regular expressions to normalize a particular query\r\nparameter.  Handles things like remove the ? if there are no more query parameters.\r\n\r\n@jnioche not sure if you'd think this generally useful? I can easily pull into our own project, but would like your feedback anyway :) so started here..\r\n\r\n@GuiForget any pointers on java idioms, etc always welcome.","created_at":"2015-03-01T06:35:42Z","updated_at":"2015-03-01T18:52:09Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3338006b2e77fad5b30bdf2a5e91da31e84aefaa","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/98/commits","review_comments_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/98/comments","review_comment_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/comments{/number}","comments_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/98/comments","statuses_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/statuses/45c6372b40f29394b163f077676a3fd5578a8beb","head":{"label":"acinader:add-query-parameter-normalizer","ref":"add-query-parameter-normalizer","sha":"45c6372b40f29394b163f077676a3fd5578a8beb","user":{"login":"acinader","id":700572,"avatar_url":"https://avatars.githubusercontent.com/u/700572?v=3","gravatar_id":"","url":"https://api.github.com/users/acinader","html_url":"https://github.com/acinader","followers_url":"https://api.github.com/users/acinader/followers","following_url":"https://api.github.com/users/acinader/following{/other_user}","gists_url":"https://api.github.com/users/acinader/gists{/gist_id}","starred_url":"https://api.github.com/users/acinader/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acinader/subscriptions","organizations_url":"https://api.github.com/users/acinader/orgs","repos_url":"https://api.github.com/users/acinader/repos","events_url":"https://api.github.com/users/acinader/events{/privacy}","received_events_url":"https://api.github.com/users/acinader/received_events","type":"User","site_admin":false},"repo":{"id":26672505,"name":"storm-crawler","full_name":"acinader/storm-crawler","owner":{"login":"acinader","id":700572,"avatar_url":"https://avatars.githubusercontent.com/u/700572?v=3","gravatar_id":"","url":"https://api.github.com/users/acinader","html_url":"https://github.com/acinader","followers_url":"https://api.github.com/users/acinader/followers","following_url":"https://api.github.com/users/acinader/following{/other_user}","gists_url":"https://api.github.com/users/acinader/gists{/gist_id}","starred_url":"https://api.github.com/users/acinader/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/acinader/subscriptions","organizations_url":"https://api.github.com/users/acinader/orgs","repos_url":"https://api.github.com/users/acinader/repos","events_url":"https://api.github.com/users/acinader/events{/privacy}","received_events_url":"https://api.github.com/users/acinader/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/acinader/storm-crawler","description":"Storm-based web crawler ","fork":true,"url":"https://api.github.com/repos/acinader/storm-crawler","forks_url":"https://api.github.com/repos/acinader/storm-crawler/forks","keys_url":"https://api.github.com/repos/acinader/storm-crawler/keys{/key_id}","collaborators_url":"https://api.github.com/repos/acinader/storm-crawler/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/acinader/storm-crawler/teams","hooks_url":"https://api.github.com/repos/acinader/storm-crawler/hooks","issue_events_url":"https://api.github.com/repos/acinader/storm-crawler/issues/events{/number}","events_url":"https://api.github.com/repos/acinader/storm-crawler/events","assignees_url":"https://api.github.com/repos/acinader/storm-crawler/assignees{/user}","branches_url":"https://api.github.com/repos/acinader/storm-crawler/branches{/branch}","tags_url":"https://api.github.com/repos/acinader/storm-crawler/tags","blobs_url":"https://api.github.com/repos/acinader/storm-crawler/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/acinader/storm-crawler/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/acinader/storm-crawler/git/refs{/sha}","trees_url":"https://api.github.com/repos/acinader/storm-crawler/git/trees{/sha}","statuses_url":"https://api.github.com/repos/acinader/storm-crawler/statuses/{sha}","languages_url":"https://api.github.com/repos/acinader/storm-crawler/languages","stargazers_url":"https://api.github.com/repos/acinader/storm-crawler/stargazers","contributors_url":"https://api.github.com/repos/acinader/storm-crawler/contributors","subscribers_url":"https://api.github.com/repos/acinader/storm-crawler/subscribers","subscription_url":"https://api.github.com/repos/acinader/storm-crawler/subscription","commits_url":"https://api.github.com/repos/acinader/storm-crawler/commits{/sha}","git_commits_url":"https://api.github.com/repos/acinader/storm-crawler/git/commits{/sha}","comments_url":"https://api.github.com/repos/acinader/storm-crawler/comments{/number}","issue_comment_url":"https://api.github.com/repos/acinader/storm-crawler/issues/comments{/number}","contents_url":"https://api.github.com/repos/acinader/storm-crawler/contents/{+path}","compare_url":"https://api.github.com/repos/acinader/storm-crawler/compare/{base}...{head}","merges_url":"https://api.github.com/repos/acinader/storm-crawler/merges","archive_url":"https://api.github.com/repos/acinader/storm-crawler/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/acinader/storm-crawler/downloads","issues_url":"https://api.github.com/repos/acinader/storm-crawler/issues{/number}","pulls_url":"https://api.github.com/repos/acinader/storm-crawler/pulls{/number}","milestones_url":"https://api.github.com/repos/acinader/storm-crawler/milestones{/number}","notifications_url":"https://api.github.com/repos/acinader/storm-crawler/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/acinader/storm-crawler/labels{/name}","releases_url":"https://api.github.com/repos/acinader/storm-crawler/releases{/id}","created_at":"2014-11-15T07:13:54Z","updated_at":"2014-11-15T07:13:56Z","pushed_at":"2015-03-01T06:34:15Z","git_url":"git://github.com/acinader/storm-crawler.git","ssh_url":"git@github.com:acinader/storm-crawler.git","clone_url":"https://github.com/acinader/storm-crawler.git","svn_url":"https://github.com/acinader/storm-crawler","homepage":"","size":568,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"DigitalPebble:master","ref":"master","sha":"4617c7fe9946e33916302fdc6e88d06fc95c19cd","user":{"login":"DigitalPebble","id":1726647,"avatar_url":"https://avatars.githubusercontent.com/u/1726647?v=3","gravatar_id":"","url":"https://api.github.com/users/DigitalPebble","html_url":"https://github.com/DigitalPebble","followers_url":"https://api.github.com/users/DigitalPebble/followers","following_url":"https://api.github.com/users/DigitalPebble/following{/other_user}","gists_url":"https://api.github.com/users/DigitalPebble/gists{/gist_id}","starred_url":"https://api.github.com/users/DigitalPebble/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DigitalPebble/subscriptions","organizations_url":"https://api.github.com/users/DigitalPebble/orgs","repos_url":"https://api.github.com/users/DigitalPebble/repos","events_url":"https://api.github.com/users/DigitalPebble/events{/privacy}","received_events_url":"https://api.github.com/users/DigitalPebble/received_events","type":"Organization","site_admin":false},"repo":{"id":9395761,"name":"storm-crawler","full_name":"DigitalPebble/storm-crawler","owner":{"login":"DigitalPebble","id":1726647,"avatar_url":"https://avatars.githubusercontent.com/u/1726647?v=3","gravatar_id":"","url":"https://api.github.com/users/DigitalPebble","html_url":"https://github.com/DigitalPebble","followers_url":"https://api.github.com/users/DigitalPebble/followers","following_url":"https://api.github.com/users/DigitalPebble/following{/other_user}","gists_url":"https://api.github.com/users/DigitalPebble/gists{/gist_id}","starred_url":"https://api.github.com/users/DigitalPebble/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DigitalPebble/subscriptions","organizations_url":"https://api.github.com/users/DigitalPebble/orgs","repos_url":"https://api.github.com/users/DigitalPebble/repos","events_url":"https://api.github.com/users/DigitalPebble/events{/privacy}","received_events_url":"https://api.github.com/users/DigitalPebble/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DigitalPebble/storm-crawler","description":"Storm-based web crawler ","fork":false,"url":"https://api.github.com/repos/DigitalPebble/storm-crawler","forks_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/forks","keys_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/teams","hooks_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/hooks","issue_events_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/events{/number}","events_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/events","assignees_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/assignees{/user}","branches_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/branches{/branch}","tags_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/tags","blobs_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/refs{/sha}","trees_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/statuses/{sha}","languages_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/languages","stargazers_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/stargazers","contributors_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/contributors","subscribers_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/subscribers","subscription_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/subscription","commits_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/commits{/sha}","git_commits_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/commits{/sha}","comments_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/comments{/number}","issue_comment_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/comments{/number}","contents_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/contents/{+path}","compare_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/merges","archive_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/downloads","issues_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues{/number}","pulls_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls{/number}","milestones_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/milestones{/number}","notifications_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/labels{/name}","releases_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/releases{/id}","created_at":"2013-04-12T14:13:30Z","updated_at":"2015-02-28T22:27:58Z","pushed_at":"2015-02-27T17:06:08Z","git_url":"git://github.com/DigitalPebble/storm-crawler.git","ssh_url":"git@github.com:DigitalPebble/storm-crawler.git","clone_url":"https://github.com/DigitalPebble/storm-crawler.git","svn_url":"https://github.com/DigitalPebble/storm-crawler","homepage":"","size":2532,"stargazers_count":55,"watchers_count":55,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":16,"mirror_url":null,"open_issues_count":16,"forks":16,"open_issues":16,"watchers":55,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/98"},"html":{"href":"https://github.com/DigitalPebble/storm-crawler/pull/98"},"issue":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/98"},"comments":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/98/comments"},"review_comments":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/98/comments"},"review_comment":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/98/commits"},"statuses":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/statuses/45c6372b40f29394b163f077676a3fd5578a8beb"}}}}