{"action":"created","comment":{"url":"https://api.github.com/repos/medallia/krakenj/pulls/comments/45259065","id":45259065,"diff_hunk":"@@ -0,0 +1,61 @@\n+package totemcaf.kraken.processor;\n+\n+\n+import com.google.common.base.Charsets;\n+import com.google.common.io.Resources;\n+import totemcaf.kraken.Request;\n+\n+import java.io.IOException;\n+import java.util.HashMap;\n+import java.util.List;\n+import java.util.Random;\n+\n+/**\n+ * Adds a UserAgent to the {@link Request}'s headers with a random strategy from a list.\n+ */\n+public class RandomUserAgentAssignerProcessor implements UserAgentAssignerProcessor {\n+\tprivate final List<String> userAgents;\n+\tprivate boolean overrideIfPresent;\n+\n+\tprivate RandomUserAgentAssignerProcessor(List<String> userAgents, boolean overrideIfPresent) {\n+\t\tthis.userAgents = userAgents;\n+\t\tthis.overrideIfPresent = overrideIfPresent;\n+\t}\n+\n+\t/**\n+\t * Builds a RandomUserAgentAssignerProcessor with the default list of UserAgents\n+\t * @param overrideIfPresent if true, set a UserAgent header in the request even if it already has one.\n+\t * @return a new RandomUserAgentAssignerProcessor with the default UserAgents list\n+\t */\n+\tpublic static RandomUserAgentAssignerProcessor withDefaultList(boolean overrideIfPresent) {\n+\t\treturn fromFile(\"userAgents.txt\", overrideIfPresent);\n+\t}\n+\n+\t/**\n+\t * Builds a RandomUserAgentAssignerProcessor with a custom list of UserAgents\n+\t * @param userAgentsFile name of the file with a custom list of UserAgents\n+\t * @param overrideIfPresent if true, set a UserAgent header in the request even if it already has one.\n+\t * @return a new RandomUserAgentAssignerProcessor with the desired file as its proxies list\n+\t */\n+\tprivate static RandomUserAgentAssignerProcessor fromFile(String userAgentsFile, boolean overrideIfPresent) {\n+\t\ttry {\n+\t\t\treturn new RandomUserAgentAssignerProcessor(Resources.readLines(Resources.getResource(RandomUserAgentAssignerProcessor.class, userAgentsFile), Charsets.UTF_8), overrideIfPresent);\n+\t\t} catch (IOException e) {\n+\t\t\tthrow new RuntimeException(e);\n+\t\t}\n+\t}\n+\n+\t@Override\n+\tpublic Request apply(Request request) {\n+\t\tif (request.getHeaders() != null && request.getHeaders().containsKey(USER_AGENT_KEY) && !overrideIfPresent)\n+\t\t\treturn request;\n+\n+\t\tHEADERS_KEY.computeIfAbsent(request.getJob(), HashMap::new).computeIfAbsent(USER_AGENT_KEY, k -> this.getRandomUserAgent());","path":"src/main/java/totemcaf/kraken/processor/RandomUserAgentAssignerProcessor.java","position":null,"original_position":53,"commit_id":"ed99c35054d5a6301c632da1dd4135bbdef34881","original_commit_id":"009e28a76025d38d53dbced1dea4c3262097e82f","user":{"login":"aandres-medallia","id":10635858,"avatar_url":"https://avatars.githubusercontent.com/u/10635858?v=3","gravatar_id":"","url":"https://api.github.com/users/aandres-medallia","html_url":"https://github.com/aandres-medallia","followers_url":"https://api.github.com/users/aandres-medallia/followers","following_url":"https://api.github.com/users/aandres-medallia/following{/other_user}","gists_url":"https://api.github.com/users/aandres-medallia/gists{/gist_id}","starred_url":"https://api.github.com/users/aandres-medallia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aandres-medallia/subscriptions","organizations_url":"https://api.github.com/users/aandres-medallia/orgs","repos_url":"https://api.github.com/users/aandres-medallia/repos","events_url":"https://api.github.com/users/aandres-medallia/events{/privacy}","received_events_url":"https://api.github.com/users/aandres-medallia/received_events","type":"User","site_admin":false},"body":"ConcurrentHashMap::new","created_at":"2015-11-18T20:58:53Z","updated_at":"2015-11-18T20:58:53Z","html_url":"https://github.com/medallia/krakenj/pull/4#discussion_r45259065","pull_request_url":"https://api.github.com/repos/medallia/krakenj/pulls/4","_links":{"self":{"href":"https://api.github.com/repos/medallia/krakenj/pulls/comments/45259065"},"html":{"href":"https://github.com/medallia/krakenj/pull/4#discussion_r45259065"},"pull_request":{"href":"https://api.github.com/repos/medallia/krakenj/pulls/4"}}},"pull_request":{"url":"https://api.github.com/repos/medallia/krakenj/pulls/4","id":51138555,"html_url":"https://github.com/medallia/krakenj/pull/4","diff_url":"https://github.com/medallia/krakenj/pull/4.diff","patch_url":"https://github.com/medallia/krakenj/pull/4.patch","issue_url":"https://api.github.com/repos/medallia/krakenj/issues/4","number":4,"state":"open","locked":false,"title":"RandomUserAgentAssigner","user":{"login":"docd","id":1208832,"avatar_url":"https://avatars.githubusercontent.com/u/1208832?v=3","gravatar_id":"","url":"https://api.github.com/users/docd","html_url":"https://github.com/docd","followers_url":"https://api.github.com/users/docd/followers","following_url":"https://api.github.com/users/docd/following{/other_user}","gists_url":"https://api.github.com/users/docd/gists{/gist_id}","starred_url":"https://api.github.com/users/docd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/docd/subscriptions","organizations_url":"https://api.github.com/users/docd/orgs","repos_url":"https://api.github.com/users/docd/repos","events_url":"https://api.github.com/users/docd/events{/privacy}","received_events_url":"https://api.github.com/users/docd/received_events","type":"User","site_admin":false},"body":"Processor for assigning a random UserAgent to the request's headers.","created_at":"2015-11-18T20:37:22Z","updated_at":"2015-11-18T20:58:53Z","closed_at":null,"merged_at":null,"merge_commit_sha":"00a8b25bb755eb24f4910a019cc61296e0abd049","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/medallia/krakenj/pulls/4/commits","review_comments_url":"https://api.github.com/repos/medallia/krakenj/pulls/4/comments","review_comment_url":"https://api.github.com/repos/medallia/krakenj/pulls/comments{/number}","comments_url":"https://api.github.com/repos/medallia/krakenj/issues/4/comments","statuses_url":"https://api.github.com/repos/medallia/krakenj/statuses/ed99c35054d5a6301c632da1dd4135bbdef34881","head":{"label":"medallia:useragentassigner","ref":"useragentassigner","sha":"ed99c35054d5a6301c632da1dd4135bbdef34881","user":{"login":"medallia","id":2198222,"avatar_url":"https://avatars.githubusercontent.com/u/2198222?v=3","gravatar_id":"","url":"https://api.github.com/users/medallia","html_url":"https://github.com/medallia","followers_url":"https://api.github.com/users/medallia/followers","following_url":"https://api.github.com/users/medallia/following{/other_user}","gists_url":"https://api.github.com/users/medallia/gists{/gist_id}","starred_url":"https://api.github.com/users/medallia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medallia/subscriptions","organizations_url":"https://api.github.com/users/medallia/orgs","repos_url":"https://api.github.com/users/medallia/repos","events_url":"https://api.github.com/users/medallia/events{/privacy}","received_events_url":"https://api.github.com/users/medallia/received_events","type":"Organization","site_admin":false},"repo":{"id":45934632,"name":"krakenj","full_name":"medallia/krakenj","owner":{"login":"medallia","id":2198222,"avatar_url":"https://avatars.githubusercontent.com/u/2198222?v=3","gravatar_id":"","url":"https://api.github.com/users/medallia","html_url":"https://github.com/medallia","followers_url":"https://api.github.com/users/medallia/followers","following_url":"https://api.github.com/users/medallia/following{/other_user}","gists_url":"https://api.github.com/users/medallia/gists{/gist_id}","starred_url":"https://api.github.com/users/medallia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medallia/subscriptions","organizations_url":"https://api.github.com/users/medallia/orgs","repos_url":"https://api.github.com/users/medallia/repos","events_url":"https://api.github.com/users/medallia/events{/privacy}","received_events_url":"https://api.github.com/users/medallia/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/medallia/krakenj","description":"Modular Web crawler that allows continued crawling.","fork":true,"url":"https://api.github.com/repos/medallia/krakenj","forks_url":"https://api.github.com/repos/medallia/krakenj/forks","keys_url":"https://api.github.com/repos/medallia/krakenj/keys{/key_id}","collaborators_url":"https://api.github.com/repos/medallia/krakenj/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/medallia/krakenj/teams","hooks_url":"https://api.github.com/repos/medallia/krakenj/hooks","issue_events_url":"https://api.github.com/repos/medallia/krakenj/issues/events{/number}","events_url":"https://api.github.com/repos/medallia/krakenj/events","assignees_url":"https://api.github.com/repos/medallia/krakenj/assignees{/user}","branches_url":"https://api.github.com/repos/medallia/krakenj/branches{/branch}","tags_url":"https://api.github.com/repos/medallia/krakenj/tags","blobs_url":"https://api.github.com/repos/medallia/krakenj/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/medallia/krakenj/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/medallia/krakenj/git/refs{/sha}","trees_url":"https://api.github.com/repos/medallia/krakenj/git/trees{/sha}","statuses_url":"https://api.github.com/repos/medallia/krakenj/statuses/{sha}","languages_url":"https://api.github.com/repos/medallia/krakenj/languages","stargazers_url":"https://api.github.com/repos/medallia/krakenj/stargazers","contributors_url":"https://api.github.com/repos/medallia/krakenj/contributors","subscribers_url":"https://api.github.com/repos/medallia/krakenj/subscribers","subscription_url":"https://api.github.com/repos/medallia/krakenj/subscription","commits_url":"https://api.github.com/repos/medallia/krakenj/commits{/sha}","git_commits_url":"https://api.github.com/repos/medallia/krakenj/git/commits{/sha}","comments_url":"https://api.github.com/repos/medallia/krakenj/comments{/number}","issue_comment_url":"https://api.github.com/repos/medallia/krakenj/issues/comments{/number}","contents_url":"https://api.github.com/repos/medallia/krakenj/contents/{+path}","compare_url":"https://api.github.com/repos/medallia/krakenj/compare/{base}...{head}","merges_url":"https://api.github.com/repos/medallia/krakenj/merges","archive_url":"https://api.github.com/repos/medallia/krakenj/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/medallia/krakenj/downloads","issues_url":"https://api.github.com/repos/medallia/krakenj/issues{/number}","pulls_url":"https://api.github.com/repos/medallia/krakenj/pulls{/number}","milestones_url":"https://api.github.com/repos/medallia/krakenj/milestones{/number}","notifications_url":"https://api.github.com/repos/medallia/krakenj/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/medallia/krakenj/labels{/name}","releases_url":"https://api.github.com/repos/medallia/krakenj/releases{/id}","created_at":"2015-11-10T19:19:00Z","updated_at":"2015-11-10T19:19:01Z","pushed_at":"2015-11-18T20:55:27Z","git_url":"git://github.com/medallia/krakenj.git","ssh_url":"git@github.com:medallia/krakenj.git","clone_url":"https://github.com/medallia/krakenj.git","svn_url":"https://github.com/medallia/krakenj","homepage":null,"size":783,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"base":{"label":"medallia:master","ref":"master","sha":"d08c78ce7f3eabf8f00c2dcdb1484b194980e9cf","user":{"login":"medallia","id":2198222,"avatar_url":"https://avatars.githubusercontent.com/u/2198222?v=3","gravatar_id":"","url":"https://api.github.com/users/medallia","html_url":"https://github.com/medallia","followers_url":"https://api.github.com/users/medallia/followers","following_url":"https://api.github.com/users/medallia/following{/other_user}","gists_url":"https://api.github.com/users/medallia/gists{/gist_id}","starred_url":"https://api.github.com/users/medallia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medallia/subscriptions","organizations_url":"https://api.github.com/users/medallia/orgs","repos_url":"https://api.github.com/users/medallia/repos","events_url":"https://api.github.com/users/medallia/events{/privacy}","received_events_url":"https://api.github.com/users/medallia/received_events","type":"Organization","site_admin":false},"repo":{"id":45934632,"name":"krakenj","full_name":"medallia/krakenj","owner":{"login":"medallia","id":2198222,"avatar_url":"https://avatars.githubusercontent.com/u/2198222?v=3","gravatar_id":"","url":"https://api.github.com/users/medallia","html_url":"https://github.com/medallia","followers_url":"https://api.github.com/users/medallia/followers","following_url":"https://api.github.com/users/medallia/following{/other_user}","gists_url":"https://api.github.com/users/medallia/gists{/gist_id}","starred_url":"https://api.github.com/users/medallia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medallia/subscriptions","organizations_url":"https://api.github.com/users/medallia/orgs","repos_url":"https://api.github.com/users/medallia/repos","events_url":"https://api.github.com/users/medallia/events{/privacy}","received_events_url":"https://api.github.com/users/medallia/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/medallia/krakenj","description":"Modular Web crawler that allows continued crawling.","fork":true,"url":"https://api.github.com/repos/medallia/krakenj","forks_url":"https://api.github.com/repos/medallia/krakenj/forks","keys_url":"https://api.github.com/repos/medallia/krakenj/keys{/key_id}","collaborators_url":"https://api.github.com/repos/medallia/krakenj/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/medallia/krakenj/teams","hooks_url":"https://api.github.com/repos/medallia/krakenj/hooks","issue_events_url":"https://api.github.com/repos/medallia/krakenj/issues/events{/number}","events_url":"https://api.github.com/repos/medallia/krakenj/events","assignees_url":"https://api.github.com/repos/medallia/krakenj/assignees{/user}","branches_url":"https://api.github.com/repos/medallia/krakenj/branches{/branch}","tags_url":"https://api.github.com/repos/medallia/krakenj/tags","blobs_url":"https://api.github.com/repos/medallia/krakenj/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/medallia/krakenj/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/medallia/krakenj/git/refs{/sha}","trees_url":"https://api.github.com/repos/medallia/krakenj/git/trees{/sha}","statuses_url":"https://api.github.com/repos/medallia/krakenj/statuses/{sha}","languages_url":"https://api.github.com/repos/medallia/krakenj/languages","stargazers_url":"https://api.github.com/repos/medallia/krakenj/stargazers","contributors_url":"https://api.github.com/repos/medallia/krakenj/contributors","subscribers_url":"https://api.github.com/repos/medallia/krakenj/subscribers","subscription_url":"https://api.github.com/repos/medallia/krakenj/subscription","commits_url":"https://api.github.com/repos/medallia/krakenj/commits{/sha}","git_commits_url":"https://api.github.com/repos/medallia/krakenj/git/commits{/sha}","comments_url":"https://api.github.com/repos/medallia/krakenj/comments{/number}","issue_comment_url":"https://api.github.com/repos/medallia/krakenj/issues/comments{/number}","contents_url":"https://api.github.com/repos/medallia/krakenj/contents/{+path}","compare_url":"https://api.github.com/repos/medallia/krakenj/compare/{base}...{head}","merges_url":"https://api.github.com/repos/medallia/krakenj/merges","archive_url":"https://api.github.com/repos/medallia/krakenj/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/medallia/krakenj/downloads","issues_url":"https://api.github.com/repos/medallia/krakenj/issues{/number}","pulls_url":"https://api.github.com/repos/medallia/krakenj/pulls{/number}","milestones_url":"https://api.github.com/repos/medallia/krakenj/milestones{/number}","notifications_url":"https://api.github.com/repos/medallia/krakenj/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/medallia/krakenj/labels{/name}","releases_url":"https://api.github.com/repos/medallia/krakenj/releases{/id}","created_at":"2015-11-10T19:19:00Z","updated_at":"2015-11-10T19:19:01Z","pushed_at":"2015-11-18T20:55:27Z","git_url":"git://github.com/medallia/krakenj.git","ssh_url":"git@github.com:medallia/krakenj.git","clone_url":"https://github.com/medallia/krakenj.git","svn_url":"https://github.com/medallia/krakenj","homepage":null,"size":783,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/medallia/krakenj/pulls/4"},"html":{"href":"https://github.com/medallia/krakenj/pull/4"},"issue":{"href":"https://api.github.com/repos/medallia/krakenj/issues/4"},"comments":{"href":"https://api.github.com/repos/medallia/krakenj/issues/4/comments"},"review_comments":{"href":"https://api.github.com/repos/medallia/krakenj/pulls/4/comments"},"review_comment":{"href":"https://api.github.com/repos/medallia/krakenj/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/medallia/krakenj/pulls/4/commits"},"statuses":{"href":"https://api.github.com/repos/medallia/krakenj/statuses/ed99c35054d5a6301c632da1dd4135bbdef34881"}}}}