{"action":"created","comment":{"url":"https://api.github.com/repos/Wikia/pandora/pulls/comments/39033208","id":39033208,"diff_hunk":"@@ -0,0 +1,186 @@\n+package com.wikia.workers.articlesnippetworker;\n+\n+import org.jsoup.Jsoup;\n+import org.jsoup.nodes.Document;\n+import org.jsoup.nodes.Element;\n+import org.jsoup.nodes.Node;\n+import org.jsoup.nodes.TextNode;\n+import org.jsoup.select.Elements;\n+\n+import java.util.ArrayList;\n+import java.util.Arrays;\n+import java.util.List;\n+import java.util.regex.Matcher;\n+import java.util.regex.Pattern;\n+\n+public class SnippetParser {\n+\n+  public static final int PARAGRAPHS_COUNT = 2;\n+\n+  public enum NotSupportedTag {\n+    div, figure, nav, table, script, sup, dl;\n+\n+    public static String getList(String seperator, NotSupportedTag... exclude) {\n+      final List<NotSupportedTag> excludedList = Arrays.asList(exclude);\n+      List<String> tagList = new ArrayList<String>() {{\n+        for (NotSupportedTag aTag : NotSupportedTag.values()) {\n+          if (!excludedList.contains(aTag)) {\n+            add(aTag.name());\n+          }\n+        }\n+      }};\n+      return String.join(seperator, tagList);\n+    }\n+  }\n+\n+  //remove space before any punctuation except ()\"-<>[]\n+  private Pattern punct = Pattern.compile(\" ((?![\\\\(\\\\)\\\\[\\\\]\\\"\\\\-<>])\\\\p{Punct})\");\n+  //symbol pattern\n+  private Pattern symbols = Pattern.compile(\"\\\\p{S}\");\n+\n+  public String getSnippet(String html) {\n+    List<String> result;\n+    Document document = Jsoup.parse(html);\n+    //remove buttons and contents table, and hidden data\n+    document.select(\"span.editsection, nav.toc, .wikia-button\").remove();\n+    document.select(\"*[style*=display:none]\").remove();\n+\n+    //clone so it wont influence future parsing\n+    result = this.parseBody(document.body().clone(), NotSupportedTag.getList(\", \"));\n+    if (result.size() > 0) {\n+      return this.clear(result);\n+    }\n+\n+    //go through uncertain parent nodes\n+    Element aClone = document.body().clone();\n+    Elements toCheck = aClone.select(\"div > :not(div)\");\n+    while (!toCheck.isEmpty()) {\n+      Element aDiv = toCheck.first().parent();\n+      result\n+          .addAll(this.parseBody(aDiv.clone(), NotSupportedTag.getList(\", \", NotSupportedTag.div)));\n+      if (result.size() >= PARAGRAPHS_COUNT) {\n+        return this.clear(result);\n+      }\n+      aDiv.remove();\n+      //refresh list after each removal\n+      toCheck = aClone.select(\"div > :not(div)\");\n+    }\n+    if (result.size() > 0) {\n+      return this.clear(result);\n+    }\n+\n+    //parse not cleaned article in case of no results\n+    result.addAll(this.parseBody(document.body().clone(), \"\"));\n+    return this.clear(result);\n+  }\n+\n+  private List<String> parseBody(Element body, String toRemove) {\n+    List<String> result = new ArrayList<>();\n+    List<String> listResult = new ArrayList<>();\n+    String p = \"\";\n+    String temp;\n+    Element item;\n+    Elements filters = body.select(\"h1, h2, h3, h4\");\n+    //remove not parsed elements\n+    if (!toRemove.isEmpty()) {\n+      Elements elementsToRemove = body.select(toRemove);\n+      if (elementsToRemove.contains(body)) {\n+        return result;\n+      }\n+      elementsToRemove.remove();\n+    }\n+\n+    for (Node aNode : body.childNodes()) {\n+      if (aNode instanceof TextNode) {\n+        temp = ((TextNode) aNode).text();\n+        if (!this.isEmpty(temp)) {\n+          p += temp;\n+        }\n+      } else if (aNode instanceof Element) {\n+        item = (Element) aNode;\n+        if (\"ul ol\".contains(item.tagName())) {\n+          temp = this.parseElement(item);\n+          if (!this.isEmpty(temp)) {\n+            listResult.add(temp);\n+          }\n+        } else if (!filters.contains(item)) {\n+          if (item.tagName().equals(\"p\") && !p.equals(\"\")) {\n+            result.add(p);\n+            p = \"\";\n+          }\n+          p += this.parseElement(item);\n+        }\n+      }\n+    }\n+    if (!p.equals(\"\")) {\n+      result.add(p);\n+    }\n+    result.addAll(listResult);\n+    return result;\n+  }\n+\n+  private String parseElement(Element item) {\n+    Element sibling = item.nextElementSibling();\n+    if (\"ul ol\".contains(item.tagName())) {\n+      String parsed = this.parseList(item, item.previousElementSibling());\n+      if (!this.isEmpty(parsed)) {\n+        return parsed;\n+      }\n+    } else if (!this.isEmpty(item.text()) && (sibling == null || !\"ul ol\"\n+        .contains(sibling.tagName()))) {\n+      return item.text() + \" \";\n+    }\n+    return \"\";\n+  }\n+\n+  private String parseList(Element ul, Element prevSibling) {\n+    List<String> predecessors = new ArrayList<String>() {{\n+      add(\"p\");\n+      add(\"h1\");\n+      add(\"h2\");\n+      add(\"h3\");\n+      add(\"h4\");\n+    }};\n+    Elements li = ul.select(\"li\");\n+    String result = \"\";\n+    for (Element aLi : li) {\n+      if (!this.isEmpty(li.text())) {\n+        if (!result.equals(\"\")) {\n+          result += \", \";\n+        }\n+        result += aLi.text();\n+      }\n+    }\n+    // append previous if exists\n+    if (prevSibling != null && predecessors.contains(prevSibling.tagName())) {\n+      result = prevSibling.text() + \" \" + result;\n+    }\n+    return result;\n+  }\n+\n+  private boolean isEmpty(String text) {\n+    return this.symbols.matcher(text)\n+        .replaceAll(\"\")\n+        .replace(\"\\u00a0\", \"\")\n+        .trim()\n+        .equals(\"\");\n+  }\n+\n+  private String clear(List<String> result) {\n+    if (result.size() > PARAGRAPHS_COUNT) {\n+      // take only first two elements\n+      result = result.subList(0, PARAGRAPHS_COUNT);","path":"workers/article-snippet-worker/src/main/java/com/wikia/workers/articlesnippetworker/SnippetParser.java","position":172,"original_position":172,"commit_id":"65e54e4d5fbf3fb0aaff94017d493a9fcad24486","original_commit_id":"65e54e4d5fbf3fb0aaff94017d493a9fcad24486","user":{"login":"WikiaTech","id":13279985,"avatar_url":"https://avatars.githubusercontent.com/u/13279985?v=3","gravatar_id":"","url":"https://api.github.com/users/WikiaTech","html_url":"https://github.com/WikiaTech","followers_url":"https://api.github.com/users/WikiaTech/followers","following_url":"https://api.github.com/users/WikiaTech/following{/other_user}","gists_url":"https://api.github.com/users/WikiaTech/gists{/gist_id}","starred_url":"https://api.github.com/users/WikiaTech/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/WikiaTech/subscriptions","organizations_url":"https://api.github.com/users/WikiaTech/orgs","repos_url":"https://api.github.com/users/WikiaTech/repos","events_url":"https://api.github.com/users/WikiaTech/events{/privacy}","received_events_url":"https://api.github.com/users/WikiaTech/received_events","type":"User","site_admin":false},"body":"![MAJOR](https://raw.githubusercontent.com/SonarCommunity/sonar-github/master/images/severity-major.png) Introduce a new variable instead of reusing the parameter \"result\". [![rule](https://raw.githubusercontent.com/SonarCommunity/sonar-github/master/images/rule.png)](http://nemo.sonarqube.org/coding_rules#rule_key=squid%3AS1226)\n","created_at":"2015-09-09T11:57:16Z","updated_at":"2015-09-09T11:57:16Z","html_url":"https://github.com/Wikia/pandora/pull/363#discussion_r39033208","pull_request_url":"https://api.github.com/repos/Wikia/pandora/pulls/363","_links":{"self":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/comments/39033208"},"html":{"href":"https://github.com/Wikia/pandora/pull/363#discussion_r39033208"},"pull_request":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/363"}}},"pull_request":{"url":"https://api.github.com/repos/Wikia/pandora/pulls/363","id":41094049,"html_url":"https://github.com/Wikia/pandora/pull/363","diff_url":"https://github.com/Wikia/pandora/pull/363.diff","patch_url":"https://github.com/Wikia/pandora/pull/363.patch","issue_url":"https://api.github.com/repos/Wikia/pandora/issues/363","number":363,"state":"open","locked":false,"title":"Dat 3017","user":{"login":"rybmat","id":1534495,"avatar_url":"https://avatars.githubusercontent.com/u/1534495?v=3","gravatar_id":"","url":"https://api.github.com/users/rybmat","html_url":"https://github.com/rybmat","followers_url":"https://api.github.com/users/rybmat/followers","following_url":"https://api.github.com/users/rybmat/following{/other_user}","gists_url":"https://api.github.com/users/rybmat/gists{/gist_id}","starred_url":"https://api.github.com/users/rybmat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rybmat/subscriptions","organizations_url":"https://api.github.com/users/rybmat/orgs","repos_url":"https://api.github.com/users/rybmat/repos","events_url":"https://api.github.com/users/rybmat/events{/privacy}","received_events_url":"https://api.github.com/users/rybmat/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-07-29T07:24:51Z","updated_at":"2015-09-09T11:57:16Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3708789dc147721ba4bbe26276e858ec877676c6","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/Wikia/pandora/pulls/363/commits","review_comments_url":"https://api.github.com/repos/Wikia/pandora/pulls/363/comments","review_comment_url":"https://api.github.com/repos/Wikia/pandora/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Wikia/pandora/issues/363/comments","statuses_url":"https://api.github.com/repos/Wikia/pandora/statuses/65e54e4d5fbf3fb0aaff94017d493a9fcad24486","head":{"label":"Wikia:DAT-3017","ref":"DAT-3017","sha":"65e54e4d5fbf3fb0aaff94017d493a9fcad24486","user":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"repo":{"id":26663749,"name":"pandora","full_name":"Wikia/pandora","owner":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Wikia/pandora","description":"API Service","fork":false,"url":"https://api.github.com/repos/Wikia/pandora","forks_url":"https://api.github.com/repos/Wikia/pandora/forks","keys_url":"https://api.github.com/repos/Wikia/pandora/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Wikia/pandora/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Wikia/pandora/teams","hooks_url":"https://api.github.com/repos/Wikia/pandora/hooks","issue_events_url":"https://api.github.com/repos/Wikia/pandora/issues/events{/number}","events_url":"https://api.github.com/repos/Wikia/pandora/events","assignees_url":"https://api.github.com/repos/Wikia/pandora/assignees{/user}","branches_url":"https://api.github.com/repos/Wikia/pandora/branches{/branch}","tags_url":"https://api.github.com/repos/Wikia/pandora/tags","blobs_url":"https://api.github.com/repos/Wikia/pandora/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Wikia/pandora/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Wikia/pandora/git/refs{/sha}","trees_url":"https://api.github.com/repos/Wikia/pandora/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Wikia/pandora/statuses/{sha}","languages_url":"https://api.github.com/repos/Wikia/pandora/languages","stargazers_url":"https://api.github.com/repos/Wikia/pandora/stargazers","contributors_url":"https://api.github.com/repos/Wikia/pandora/contributors","subscribers_url":"https://api.github.com/repos/Wikia/pandora/subscribers","subscription_url":"https://api.github.com/repos/Wikia/pandora/subscription","commits_url":"https://api.github.com/repos/Wikia/pandora/commits{/sha}","git_commits_url":"https://api.github.com/repos/Wikia/pandora/git/commits{/sha}","comments_url":"https://api.github.com/repos/Wikia/pandora/comments{/number}","issue_comment_url":"https://api.github.com/repos/Wikia/pandora/issues/comments{/number}","contents_url":"https://api.github.com/repos/Wikia/pandora/contents/{+path}","compare_url":"https://api.github.com/repos/Wikia/pandora/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Wikia/pandora/merges","archive_url":"https://api.github.com/repos/Wikia/pandora/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Wikia/pandora/downloads","issues_url":"https://api.github.com/repos/Wikia/pandora/issues{/number}","pulls_url":"https://api.github.com/repos/Wikia/pandora/pulls{/number}","milestones_url":"https://api.github.com/repos/Wikia/pandora/milestones{/number}","notifications_url":"https://api.github.com/repos/Wikia/pandora/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Wikia/pandora/labels{/name}","releases_url":"https://api.github.com/repos/Wikia/pandora/releases{/id}","created_at":"2014-11-15T00:29:06Z","updated_at":"2015-08-18T21:15:19Z","pushed_at":"2015-09-09T11:56:43Z","git_url":"git://github.com/Wikia/pandora.git","ssh_url":"git@github.com:Wikia/pandora.git","clone_url":"https://github.com/Wikia/pandora.git","svn_url":"https://github.com/Wikia/pandora","homepage":null,"size":22837,"stargazers_count":3,"watchers_count":3,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":6,"forks":1,"open_issues":6,"watchers":3,"default_branch":"master"}},"base":{"label":"Wikia:master","ref":"master","sha":"350ce78a8203aeaae16951c6d02c15ef289637a8","user":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"repo":{"id":26663749,"name":"pandora","full_name":"Wikia/pandora","owner":{"login":"Wikia","id":1171011,"avatar_url":"https://avatars.githubusercontent.com/u/1171011?v=3","gravatar_id":"","url":"https://api.github.com/users/Wikia","html_url":"https://github.com/Wikia","followers_url":"https://api.github.com/users/Wikia/followers","following_url":"https://api.github.com/users/Wikia/following{/other_user}","gists_url":"https://api.github.com/users/Wikia/gists{/gist_id}","starred_url":"https://api.github.com/users/Wikia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Wikia/subscriptions","organizations_url":"https://api.github.com/users/Wikia/orgs","repos_url":"https://api.github.com/users/Wikia/repos","events_url":"https://api.github.com/users/Wikia/events{/privacy}","received_events_url":"https://api.github.com/users/Wikia/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Wikia/pandora","description":"API Service","fork":false,"url":"https://api.github.com/repos/Wikia/pandora","forks_url":"https://api.github.com/repos/Wikia/pandora/forks","keys_url":"https://api.github.com/repos/Wikia/pandora/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Wikia/pandora/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Wikia/pandora/teams","hooks_url":"https://api.github.com/repos/Wikia/pandora/hooks","issue_events_url":"https://api.github.com/repos/Wikia/pandora/issues/events{/number}","events_url":"https://api.github.com/repos/Wikia/pandora/events","assignees_url":"https://api.github.com/repos/Wikia/pandora/assignees{/user}","branches_url":"https://api.github.com/repos/Wikia/pandora/branches{/branch}","tags_url":"https://api.github.com/repos/Wikia/pandora/tags","blobs_url":"https://api.github.com/repos/Wikia/pandora/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Wikia/pandora/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Wikia/pandora/git/refs{/sha}","trees_url":"https://api.github.com/repos/Wikia/pandora/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Wikia/pandora/statuses/{sha}","languages_url":"https://api.github.com/repos/Wikia/pandora/languages","stargazers_url":"https://api.github.com/repos/Wikia/pandora/stargazers","contributors_url":"https://api.github.com/repos/Wikia/pandora/contributors","subscribers_url":"https://api.github.com/repos/Wikia/pandora/subscribers","subscription_url":"https://api.github.com/repos/Wikia/pandora/subscription","commits_url":"https://api.github.com/repos/Wikia/pandora/commits{/sha}","git_commits_url":"https://api.github.com/repos/Wikia/pandora/git/commits{/sha}","comments_url":"https://api.github.com/repos/Wikia/pandora/comments{/number}","issue_comment_url":"https://api.github.com/repos/Wikia/pandora/issues/comments{/number}","contents_url":"https://api.github.com/repos/Wikia/pandora/contents/{+path}","compare_url":"https://api.github.com/repos/Wikia/pandora/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Wikia/pandora/merges","archive_url":"https://api.github.com/repos/Wikia/pandora/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Wikia/pandora/downloads","issues_url":"https://api.github.com/repos/Wikia/pandora/issues{/number}","pulls_url":"https://api.github.com/repos/Wikia/pandora/pulls{/number}","milestones_url":"https://api.github.com/repos/Wikia/pandora/milestones{/number}","notifications_url":"https://api.github.com/repos/Wikia/pandora/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Wikia/pandora/labels{/name}","releases_url":"https://api.github.com/repos/Wikia/pandora/releases{/id}","created_at":"2014-11-15T00:29:06Z","updated_at":"2015-08-18T21:15:19Z","pushed_at":"2015-09-09T11:56:43Z","git_url":"git://github.com/Wikia/pandora.git","ssh_url":"git@github.com:Wikia/pandora.git","clone_url":"https://github.com/Wikia/pandora.git","svn_url":"https://github.com/Wikia/pandora","homepage":null,"size":22837,"stargazers_count":3,"watchers_count":3,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":6,"forks":1,"open_issues":6,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/363"},"html":{"href":"https://github.com/Wikia/pandora/pull/363"},"issue":{"href":"https://api.github.com/repos/Wikia/pandora/issues/363"},"comments":{"href":"https://api.github.com/repos/Wikia/pandora/issues/363/comments"},"review_comments":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/363/comments"},"review_comment":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Wikia/pandora/pulls/363/commits"},"statuses":{"href":"https://api.github.com/repos/Wikia/pandora/statuses/65e54e4d5fbf3fb0aaff94017d493a9fcad24486"}}}}