{"action":"created","comment":{"url":"https://api.github.com/repos/Wikia/pandora/pulls/comments/46273813","id":46273813,"diff_hunk":"@@ -24,12 +29,28 @@\n    *\n    * @return Set\n    */\n-  public Set matchTemplateTypes(String html) {\n+  public Set matchTemplateTypes(String html, List<String> categories) {\n     //remove comments as they are not parsed and bring no value\n     Document document = Jsoup.parse(html.replaceAll(\"(?s)<!--.*?-->\", \"\"));\n \n+    //clean up template contents before parsing\n+    Elements whitelist = this.getElementsBetween(document, \"OnlyInclude\");\n+    whitelist.addAll(this.getElementsBetween(document, \"IncludeOnly\"));\n+    Elements blacklist = this.getElementsBetween(document, \"NoInclude\");\n+\n+    // if include only is set, remove others","path":"workers/template-classification-worker/src/main/java/com/wikia/workers/templateclassificationworker/domain/TemplateParser.java","position":27,"original_position":27,"commit_id":"e96fa3d2f3f7115f90afe89c85cfa838c9a1d6ef","original_commit_id":"e96fa3d2f3f7115f90afe89c85cfa838c9a1d6ef","user":{"login":"SebastianMarzjan","id":2051084,"avatar_url":"https://avatars.githubusercontent.com/u/2051084?v=3","gravatar_id":"","url":"https://api.github.com/users/SebastianMarzjan","html_url":"https://github.com/SebastianMarzjan","followers_url":"https://api.github.com/users/SebastianMarzjan/followers","following_url":"https://api.github.com/users/SebastianMarzjan/following{/other_user}","gists_url":"https://api.github.com/users/SebastianMarzjan/gists{/gist_id}","starred_url":"https://api.github.com/users/SebastianMarzjan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SebastianMarzjan/subscriptions","organizations_url":"https://api.github.com/users/SebastianMarzjan/orgs","repos_url":"https://api.github.com/users/SebastianMarzjan/repos","events_url":"https://api.github.com/users/SebastianMarzjan/events{/privacy}","received_events_url":"https://api.github.com/users/SebastianMarzjan/received_events","type":"User","site_admin":false},"body":"Per offline conversation, please add a comment why the behavior differs :)","created_at":"2015-12-01T12:55:26Z","updated_at":"2015-12-01T12:55:26Z","html_url":"https://github.com/Wikia/pandora/pull/675#discussion_r46273813","pull_request_url":"https://api.github.com/repos/Wikia/pandora/pulls/675","_links":{"self":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/comments/46273813"},"html":{"href":"https://github.com/Wikia/pandora/pull/675#discussion_r46273813"},"pull_request":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/675"}}},"pull_request":{"url":"https://api.github.com/repos/Wikia/pandora/pulls/675","id":52008148,"html_url":"https://github.com/Wikia/pandora/pull/675","diff_url":"https://github.com/Wikia/pandora/pull/675.diff","patch_url":"https://github.com/Wikia/pandora/pull/675.patch","issue_url":"https://api.github.com/repos/Wikia/pandora/issues/675","number":675,"state":"open","locked":false,"title":"DAT-3532 handle includes tags in template output","user":{"login":"idradm","id":3169661,"avatar_url":"https://avatars.githubusercontent.com/u/3169661?v=3","gravatar_id":"","url":"https://api.github.com/users/idradm","html_url":"https://github.com/idradm","followers_url":"https://api.github.com/users/idradm/followers","following_url":"https://api.github.com/users/idradm/following{/other_user}","gists_url":"https://api.github.com/users/idradm/gists{/gist_id}","starred_url":"https://api.github.com/users/idradm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/idradm/subscriptions","organizations_url":"https://api.github.com/users/idradm/orgs","repos_url":"https://api.github.com/users/idradm/repos","events_url":"https://api.github.com/users/idradm/events{/privacy}","received_events_url":"https://api.github.com/users/idradm/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-11-27T15:59:08Z","updated_at":"2015-12-01T12:55:26Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a642d6286b6739c3af8e26e1c889cb70cb61d983","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/Wikia/pandora/pulls/675/commits","review_comments_url":"https://api.github.com/repos/Wikia/pandora/pulls/675/comments","review_comment_url":"https://api.github.com/repos/Wikia/pandora/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Wikia/pandora/issues/675/comments","statuses_url":"https://api.github.com/repos/Wikia/pandora/statuses/e96fa3d2f3f7115f90afe89c85cfa838c9a1d6ef","head":{"label":"Wikia:DAT-3532","ref":"DAT-3532","sha":"e96fa3d2f3f7115f90afe89c85cfa838c9a1d6ef","user":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"repo":{"id":26663749,"name":"pandora","full_name":"Wikia/pandora","owner":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Wikia/pandora","description":"API Service","fork":false,"url":"https://api.github.com/repos/Wikia/pandora","forks_url":"https://api.github.com/repos/Wikia/pandora/forks","keys_url":"https://api.github.com/repos/Wikia/pandora/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Wikia/pandora/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Wikia/pandora/teams","hooks_url":"https://api.github.com/repos/Wikia/pandora/hooks","issue_events_url":"https://api.github.com/repos/Wikia/pandora/issues/events{/number}","events_url":"https://api.github.com/repos/Wikia/pandora/events","assignees_url":"https://api.github.com/repos/Wikia/pandora/assignees{/user}","branches_url":"https://api.github.com/repos/Wikia/pandora/branches{/branch}","tags_url":"https://api.github.com/repos/Wikia/pandora/tags","blobs_url":"https://api.github.com/repos/Wikia/pandora/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Wikia/pandora/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Wikia/pandora/git/refs{/sha}","trees_url":"https://api.github.com/repos/Wikia/pandora/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Wikia/pandora/statuses/{sha}","languages_url":"https://api.github.com/repos/Wikia/pandora/languages","stargazers_url":"https://api.github.com/repos/Wikia/pandora/stargazers","contributors_url":"https://api.github.com/repos/Wikia/pandora/contributors","subscribers_url":"https://api.github.com/repos/Wikia/pandora/subscribers","subscription_url":"https://api.github.com/repos/Wikia/pandora/subscription","commits_url":"https://api.github.com/repos/Wikia/pandora/commits{/sha}","git_commits_url":"https://api.github.com/repos/Wikia/pandora/git/commits{/sha}","comments_url":"https://api.github.com/repos/Wikia/pandora/comments{/number}","issue_comment_url":"https://api.github.com/repos/Wikia/pandora/issues/comments{/number}","contents_url":"https://api.github.com/repos/Wikia/pandora/contents/{+path}","compare_url":"https://api.github.com/repos/Wikia/pandora/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Wikia/pandora/merges","archive_url":"https://api.github.com/repos/Wikia/pandora/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Wikia/pandora/downloads","issues_url":"https://api.github.com/repos/Wikia/pandora/issues{/number}","pulls_url":"https://api.github.com/repos/Wikia/pandora/pulls{/number}","milestones_url":"https://api.github.com/repos/Wikia/pandora/milestones{/number}","notifications_url":"https://api.github.com/repos/Wikia/pandora/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Wikia/pandora/labels{/name}","releases_url":"https://api.github.com/repos/Wikia/pandora/releases{/id}","created_at":"2014-11-15T00:29:06Z","updated_at":"2015-11-04T16:44:11Z","pushed_at":"2015-12-01T12:47:31Z","git_url":"git://github.com/Wikia/pandora.git","ssh_url":"git@github.com:Wikia/pandora.git","clone_url":"https://github.com/Wikia/pandora.git","svn_url":"https://github.com/Wikia/pandora","homepage":null,"size":17129,"stargazers_count":3,"watchers_count":3,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":9,"forks":1,"open_issues":9,"watchers":3,"default_branch":"master"}},"base":{"label":"Wikia:master","ref":"master","sha":"00f7309ba2053e7a531f423b22f7d43a9952cdcd","user":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"repo":{"id":26663749,"name":"pandora","full_name":"Wikia/pandora","owner":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Wikia/pandora","description":"API Service","fork":false,"url":"https://api.github.com/repos/Wikia/pandora","forks_url":"https://api.github.com/repos/Wikia/pandora/forks","keys_url":"https://api.github.com/repos/Wikia/pandora/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Wikia/pandora/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Wikia/pandora/teams","hooks_url":"https://api.github.com/repos/Wikia/pandora/hooks","issue_events_url":"https://api.github.com/repos/Wikia/pandora/issues/events{/number}","events_url":"https://api.github.com/repos/Wikia/pandora/events","assignees_url":"https://api.github.com/repos/Wikia/pandora/assignees{/user}","branches_url":"https://api.github.com/repos/Wikia/pandora/branches{/branch}","tags_url":"https://api.github.com/repos/Wikia/pandora/tags","blobs_url":"https://api.github.com/repos/Wikia/pandora/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Wikia/pandora/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Wikia/pandora/git/refs{/sha}","trees_url":"https://api.github.com/repos/Wikia/pandora/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Wikia/pandora/statuses/{sha}","languages_url":"https://api.github.com/repos/Wikia/pandora/languages","stargazers_url":"https://api.github.com/repos/Wikia/pandora/stargazers","contributors_url":"https://api.github.com/repos/Wikia/pandora/contributors","subscribers_url":"https://api.github.com/repos/Wikia/pandora/subscribers","subscription_url":"https://api.github.com/repos/Wikia/pandora/subscription","commits_url":"https://api.github.com/repos/Wikia/pandora/commits{/sha}","git_commits_url":"https://api.github.com/repos/Wikia/pandora/git/commits{/sha}","comments_url":"https://api.github.com/repos/Wikia/pandora/comments{/number}","issue_comment_url":"https://api.github.com/repos/Wikia/pandora/issues/comments{/number}","contents_url":"https://api.github.com/repos/Wikia/pandora/contents/{+path}","compare_url":"https://api.github.com/repos/Wikia/pandora/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Wikia/pandora/merges","archive_url":"https://api.github.com/repos/Wikia/pandora/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Wikia/pandora/downloads","issues_url":"https://api.github.com/repos/Wikia/pandora/issues{/number}","pulls_url":"https://api.github.com/repos/Wikia/pandora/pulls{/number}","milestones_url":"https://api.github.com/repos/Wikia/pandora/milestones{/number}","notifications_url":"https://api.github.com/repos/Wikia/pandora/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Wikia/pandora/labels{/name}","releases_url":"https://api.github.com/repos/Wikia/pandora/releases{/id}","created_at":"2014-11-15T00:29:06Z","updated_at":"2015-11-04T16:44:11Z","pushed_at":"2015-12-01T12:47:31Z","git_url":"git://github.com/Wikia/pandora.git","ssh_url":"git@github.com:Wikia/pandora.git","clone_url":"https://github.com/Wikia/pandora.git","svn_url":"https://github.com/Wikia/pandora","homepage":null,"size":17129,"stargazers_count":3,"watchers_count":3,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":9,"forks":1,"open_issues":9,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/675"},"html":{"href":"https://github.com/Wikia/pandora/pull/675"},"issue":{"href":"https://api.github.com/repos/Wikia/pandora/issues/675"},"comments":{"href":"https://api.github.com/repos/Wikia/pandora/issues/675/comments"},"review_comments":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/675/comments"},"review_comment":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/675/commits"},"statuses":{"href":"https://api.github.com/repos/Wikia/pandora/statuses/e96fa3d2f3f7115f90afe89c85cfa838c9a1d6ef"}}}}