{"action":"created","comment":{"url":"https://api.github.com/repos/caskdata/cdap/pulls/comments/35668932","id":35668932,"diff_hunk":"@@ -0,0 +1,192 @@\n+/*\n+ * Copyright © 2015 Cask Data, Inc.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n+ * use this file except in compliance with the License. You may obtain a copy of\n+ * the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n+ * License for the specific language governing permissions and limitations under\n+ * the License.\n+ */\n+\n+package co.cask.cdap.examples.wikipedia;\n+\n+import co.cask.cdap.api.Resources;\n+import co.cask.cdap.api.common.Bytes;\n+import co.cask.cdap.api.mapreduce.AbstractMapReduce;\n+import co.cask.cdap.api.mapreduce.MapReduceContext;\n+import co.cask.cdap.api.workflow.Value;\n+import co.cask.cdap.api.workflow.WorkflowToken;\n+import com.google.gson.Gson;\n+import com.google.gson.JsonSyntaxException;\n+import com.google.gson.annotations.SerializedName;\n+import org.apache.hadoop.mapreduce.Job;\n+import org.apache.hadoop.mapreduce.Mapper;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+import org.sweble.wikitext.engine.EngineException;\n+import org.sweble.wikitext.engine.PageId;\n+import org.sweble.wikitext.engine.PageTitle;\n+import org.sweble.wikitext.engine.WtEngineImpl;\n+import org.sweble.wikitext.engine.config.WikiConfig;\n+import org.sweble.wikitext.engine.nodes.EngProcessedPage;\n+import org.sweble.wikitext.engine.utils.DefaultConfigEnWp;\n+import org.sweble.wikitext.example.TextConverter;\n+import org.sweble.wikitext.parser.parser.LinkTargetException;\n+\n+import java.io.IOException;\n+import java.util.List;\n+import java.util.Map;\n+\n+/**\n+ * MapReduce program to validate and extract content from raw wikidata blobs.\n+ * Normalizes the data by converting the Wikitext to plain text for easier analysis in later stages.\n+ */\n+public class WikiContentValidatorAndNormalizer extends AbstractMapReduce {\n+\n+  public static final String NAME = WikiContentValidatorAndNormalizer.class.getSimpleName();\n+\n+  @Override\n+  protected void configure() {\n+    setName(NAME);\n+    setDescription(\"A MapReduce program that dumps page titles to a dataset.\");\n+    setMapperResources(new Resources(512));\n+    setInputDataset(WikipediaPipelineApp.RAW_WIKIPEDIA_DATASET);\n+    setOutputDataset(WikipediaPipelineApp.NORMALIZED_WIKIPEDIA_DATASET);\n+  }\n+\n+  @Override\n+  public void beforeSubmit(MapReduceContext context) throws Exception {\n+    Job job = context.getHadoopJob();\n+    job.setMapperClass(FilterNormalizerMapper.class);\n+    job.setNumReduceTasks(0);\n+  }\n+\n+  @Override\n+  public void onFinish(boolean succeeded, MapReduceContext context) throws Exception {\n+    WorkflowToken workflowToken = context.getWorkflowToken();\n+    if (workflowToken != null) {\n+      workflowToken.put(\"result\", Value.of(succeeded));\n+    }\n+  }\n+\n+  /**\n+   * Mapper that:\n+   * - Filters records that are null or empty or cannot be parsed as JSON.\n+   * - Removes meta fields from the raw Wikipedia JSON blobs.\n+   * - Normalizes data formatted as Wikitext to plain text.\n+   */\n+  public static class FilterNormalizerMapper extends Mapper<byte [], byte [], byte [], byte []> {\n+    private static final Logger LOG = LoggerFactory.getLogger(FilterNormalizerMapper.class);\n+    private static final Gson GSON = new Gson();\n+\n+    @Override\n+    protected void map(byte[] key, byte[] value, Context context) throws IOException, InterruptedException {\n+      if (key == null) {\n+        LOG.debug(\"Found null key. Skipping record.\");\n+        return;\n+      }\n+      if (key.length == 0) {\n+        LOG.debug(\"Found empty key. Skipping record.\");\n+        return;\n+      }\n+      if (value == null) {\n+        LOG.debug(\"Found null value. Skipping record.\");\n+        return;\n+      }\n+      if (value.length == 0) {\n+        LOG.debug(\"Found null value. Skipping record.\");\n+        return;\n+      }\n+      WikiTitleAndText titleAndText;\n+      try {\n+        titleAndText = parse(value);\n+      } catch (JsonSyntaxException e) {\n+        LOG.debug(\"Malformed JSON found as value. Wikipedia may not have an entry for the page '{}'\",\n+                  Bytes.toString(key));\n+        return;\n+      }\n+      String plainText;\n+      try {\n+        plainText = toPlainText(titleAndText);\n+      } catch (EngineException | LinkTargetException e) {\n+        LOG.debug(\"Error while parsing wikitext for '{}'. Skipping record.\", Bytes.toString(key));\n+        return;\n+      }\n+      context.write(key, Bytes.toBytes(plainText));\n+      context.getCounter(\"custom\", \"num.records\").increment(1);\n+    }\n+\n+    private WikiTitleAndText parse(byte[] rawWikiData) {\n+      WikiContents json = GSON.fromJson(Bytes.toString(rawWikiData), WikiContents.class);\n+      Map<String, WikiContents.Query.Page> pages = json.query.pages;\n+      // there is only one entry in this map, with the key as the page id\n+      String key = pages.keySet().iterator().next();\n+      WikiContents.Query.Page page = pages.get(key);","path":"cdap-examples/WikipediaPipeline/src/main/java/co/cask/cdap/examples/wikipedia/WikiContentValidatorAndNormalizer.java","position":130,"original_position":130,"commit_id":"2302d167d267ad8cbd4f1072a33efa0c4e6b5879","original_commit_id":"2302d167d267ad8cbd4f1072a33efa0c4e6b5879","user":{"login":"gsps1","id":1328150,"avatar_url":"https://avatars.githubusercontent.com/u/1328150?v=3","gravatar_id":"","url":"https://api.github.com/users/gsps1","html_url":"https://github.com/gsps1","followers_url":"https://api.github.com/users/gsps1/followers","following_url":"https://api.github.com/users/gsps1/following{/other_user}","gists_url":"https://api.github.com/users/gsps1/gists{/gist_id}","starred_url":"https://api.github.com/users/gsps1/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gsps1/subscriptions","organizations_url":"https://api.github.com/users/gsps1/orgs","repos_url":"https://api.github.com/users/gsps1/repos","events_url":"https://api.github.com/users/gsps1/events{/privacy}","received_events_url":"https://api.github.com/users/gsps1/received_events","type":"User","site_admin":false},"body":"you could rather do ``pages.entrySet().iterator().next().getValue()`` and remove the previous line to get key.","created_at":"2015-07-28T16:34:11Z","updated_at":"2015-07-28T16:34:11Z","html_url":"https://github.com/caskdata/cdap/pull/3307#discussion_r35668932","pull_request_url":"https://api.github.com/repos/caskdata/cdap/pulls/3307","_links":{"self":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/comments/35668932"},"html":{"href":"https://github.com/caskdata/cdap/pull/3307#discussion_r35668932"},"pull_request":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/3307"}}},"pull_request":{"url":"https://api.github.com/repos/caskdata/cdap/pulls/3307","id":40864535,"html_url":"https://github.com/caskdata/cdap/pull/3307","diff_url":"https://github.com/caskdata/cdap/pull/3307.diff","patch_url":"https://github.com/caskdata/cdap/pull/3307.patch","issue_url":"https://api.github.com/repos/caskdata/cdap/issues/3307","number":3307,"state":"open","locked":false,"title":"[CDAP-1983] Added an application for processing and analyzing Wikipedia data using a CDAP Workflow.","user":{"login":"bdmogal","id":3895099,"avatar_url":"https://avatars.githubusercontent.com/u/3895099?v=3","gravatar_id":"","url":"https://api.github.com/users/bdmogal","html_url":"https://github.com/bdmogal","followers_url":"https://api.github.com/users/bdmogal/followers","following_url":"https://api.github.com/users/bdmogal/following{/other_user}","gists_url":"https://api.github.com/users/bdmogal/gists{/gist_id}","starred_url":"https://api.github.com/users/bdmogal/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bdmogal/subscriptions","organizations_url":"https://api.github.com/users/bdmogal/orgs","repos_url":"https://api.github.com/users/bdmogal/repos","events_url":"https://api.github.com/users/bdmogal/events{/privacy}","received_events_url":"https://api.github.com/users/bdmogal/received_events","type":"User","site_admin":false},"body":"The app contains a CDAP Workflow that runs in either online or offline mode. In the offline mode, it expects Wikipedia data to be available in a Stream. In the online mode, it attempts to download wikipedia data for a provided set of page titles (the output of the Facebook Likes API). Once wikipedia data is available it runs a map-only job to filter/normalize data formatted as text/wiki-text into text/plain. It then runs two analyses on it in a fork -\r\n1. A Spark program that does topic modeling using Latent Dirichlet Allocation (LDA).\r\n2. A MapReduce program that does a simple 'Top N' word generation.\r\n\r\nThis app also demonstrates Workflow Token usage in:\r\n1. Condition Predicates\r\n2. Determining Job configuration (setting it based on values in the token)\r\n3. map/reduce functions\r\n4. Spark Programs\r\n\r\nIt also shows usage of getDataset + put() in Spark programs.\r\n\r\nBuild: http://builds.cask.co/browse/CDAP-RBT355\r\n\r\n\r\nPending/WIP:\r\n- [ ] The app currently does not have a good way to explore output or a service to retrieve results. The LDA output dataset does not have a schema set (it currently has a column per term, so the schema is variable). However, since the aim of the app is to demonstrate workflows, this can be fixed in future.\r\n- [ ] An example of how to use workflow tokens in a Java Spark program.\r\n\r\n![image](https://cloud.githubusercontent.com/assets/3895099/8898257/45699f34-33d6-11e5-8218-374489956fb3.png)\r\n","created_at":"2015-07-27T01:46:02Z","updated_at":"2015-07-28T16:34:12Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e3b5cff7d0af5810cef3626f53adf5ea77f855c6","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/caskdata/cdap/pulls/3307/commits","review_comments_url":"https://api.github.com/repos/caskdata/cdap/pulls/3307/comments","review_comment_url":"https://api.github.com/repos/caskdata/cdap/pulls/comments{/number}","comments_url":"https://api.github.com/repos/caskdata/cdap/issues/3307/comments","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/2302d167d267ad8cbd4f1072a33efa0c4e6b5879","head":{"label":"caskdata:feature/workflow-example","ref":"feature/workflow-example","sha":"2302d167d267ad8cbd4f1072a33efa0c4e6b5879","user":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"repo":{"id":22542759,"name":"cdap","full_name":"caskdata/cdap","owner":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/caskdata/cdap","description":"Cask Data Application Platform (CDAP)","fork":false,"url":"https://api.github.com/repos/caskdata/cdap","forks_url":"https://api.github.com/repos/caskdata/cdap/forks","keys_url":"https://api.github.com/repos/caskdata/cdap/keys{/key_id}","collaborators_url":"https://api.github.com/repos/caskdata/cdap/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/caskdata/cdap/teams","hooks_url":"https://api.github.com/repos/caskdata/cdap/hooks","issue_events_url":"https://api.github.com/repos/caskdata/cdap/issues/events{/number}","events_url":"https://api.github.com/repos/caskdata/cdap/events","assignees_url":"https://api.github.com/repos/caskdata/cdap/assignees{/user}","branches_url":"https://api.github.com/repos/caskdata/cdap/branches{/branch}","tags_url":"https://api.github.com/repos/caskdata/cdap/tags","blobs_url":"https://api.github.com/repos/caskdata/cdap/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/caskdata/cdap/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/caskdata/cdap/git/refs{/sha}","trees_url":"https://api.github.com/repos/caskdata/cdap/git/trees{/sha}","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/{sha}","languages_url":"https://api.github.com/repos/caskdata/cdap/languages","stargazers_url":"https://api.github.com/repos/caskdata/cdap/stargazers","contributors_url":"https://api.github.com/repos/caskdata/cdap/contributors","subscribers_url":"https://api.github.com/repos/caskdata/cdap/subscribers","subscription_url":"https://api.github.com/repos/caskdata/cdap/subscription","commits_url":"https://api.github.com/repos/caskdata/cdap/commits{/sha}","git_commits_url":"https://api.github.com/repos/caskdata/cdap/git/commits{/sha}","comments_url":"https://api.github.com/repos/caskdata/cdap/comments{/number}","issue_comment_url":"https://api.github.com/repos/caskdata/cdap/issues/comments{/number}","contents_url":"https://api.github.com/repos/caskdata/cdap/contents/{+path}","compare_url":"https://api.github.com/repos/caskdata/cdap/compare/{base}...{head}","merges_url":"https://api.github.com/repos/caskdata/cdap/merges","archive_url":"https://api.github.com/repos/caskdata/cdap/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/caskdata/cdap/downloads","issues_url":"https://api.github.com/repos/caskdata/cdap/issues{/number}","pulls_url":"https://api.github.com/repos/caskdata/cdap/pulls{/number}","milestones_url":"https://api.github.com/repos/caskdata/cdap/milestones{/number}","notifications_url":"https://api.github.com/repos/caskdata/cdap/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/caskdata/cdap/labels{/name}","releases_url":"https://api.github.com/repos/caskdata/cdap/releases{/id}","created_at":"2014-08-02T09:00:10Z","updated_at":"2015-06-30T16:06:18Z","pushed_at":"2015-07-28T16:05:40Z","git_url":"git://github.com/caskdata/cdap.git","ssh_url":"git@github.com:caskdata/cdap.git","clone_url":"https://github.com/caskdata/cdap.git","svn_url":"https://github.com/caskdata/cdap","homepage":"http://cdap.io","size":839003,"stargazers_count":56,"watchers_count":56,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":19,"mirror_url":null,"open_issues_count":22,"forks":19,"open_issues":22,"watchers":56,"default_branch":"develop"}},"base":{"label":"caskdata:release/3.1","ref":"release/3.1","sha":"d8bb4a6fd9aa3fab73c57051837b367f9944fd33","user":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"repo":{"id":22542759,"name":"cdap","full_name":"caskdata/cdap","owner":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/caskdata/cdap","description":"Cask Data Application Platform (CDAP)","fork":false,"url":"https://api.github.com/repos/caskdata/cdap","forks_url":"https://api.github.com/repos/caskdata/cdap/forks","keys_url":"https://api.github.com/repos/caskdata/cdap/keys{/key_id}","collaborators_url":"https://api.github.com/repos/caskdata/cdap/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/caskdata/cdap/teams","hooks_url":"https://api.github.com/repos/caskdata/cdap/hooks","issue_events_url":"https://api.github.com/repos/caskdata/cdap/issues/events{/number}","events_url":"https://api.github.com/repos/caskdata/cdap/events","assignees_url":"https://api.github.com/repos/caskdata/cdap/assignees{/user}","branches_url":"https://api.github.com/repos/caskdata/cdap/branches{/branch}","tags_url":"https://api.github.com/repos/caskdata/cdap/tags","blobs_url":"https://api.github.com/repos/caskdata/cdap/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/caskdata/cdap/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/caskdata/cdap/git/refs{/sha}","trees_url":"https://api.github.com/repos/caskdata/cdap/git/trees{/sha}","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/{sha}","languages_url":"https://api.github.com/repos/caskdata/cdap/languages","stargazers_url":"https://api.github.com/repos/caskdata/cdap/stargazers","contributors_url":"https://api.github.com/repos/caskdata/cdap/contributors","subscribers_url":"https://api.github.com/repos/caskdata/cdap/subscribers","subscription_url":"https://api.github.com/repos/caskdata/cdap/subscription","commits_url":"https://api.github.com/repos/caskdata/cdap/commits{/sha}","git_commits_url":"https://api.github.com/repos/caskdata/cdap/git/commits{/sha}","comments_url":"https://api.github.com/repos/caskdata/cdap/comments{/number}","issue_comment_url":"https://api.github.com/repos/caskdata/cdap/issues/comments{/number}","contents_url":"https://api.github.com/repos/caskdata/cdap/contents/{+path}","compare_url":"https://api.github.com/repos/caskdata/cdap/compare/{base}...{head}","merges_url":"https://api.github.com/repos/caskdata/cdap/merges","archive_url":"https://api.github.com/repos/caskdata/cdap/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/caskdata/cdap/downloads","issues_url":"https://api.github.com/repos/caskdata/cdap/issues{/number}","pulls_url":"https://api.github.com/repos/caskdata/cdap/pulls{/number}","milestones_url":"https://api.github.com/repos/caskdata/cdap/milestones{/number}","notifications_url":"https://api.github.com/repos/caskdata/cdap/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/caskdata/cdap/labels{/name}","releases_url":"https://api.github.com/repos/caskdata/cdap/releases{/id}","created_at":"2014-08-02T09:00:10Z","updated_at":"2015-06-30T16:06:18Z","pushed_at":"2015-07-28T16:05:40Z","git_url":"git://github.com/caskdata/cdap.git","ssh_url":"git@github.com:caskdata/cdap.git","clone_url":"https://github.com/caskdata/cdap.git","svn_url":"https://github.com/caskdata/cdap","homepage":"http://cdap.io","size":839003,"stargazers_count":56,"watchers_count":56,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":19,"mirror_url":null,"open_issues_count":22,"forks":19,"open_issues":22,"watchers":56,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/3307"},"html":{"href":"https://github.com/caskdata/cdap/pull/3307"},"issue":{"href":"https://api.github.com/repos/caskdata/cdap/issues/3307"},"comments":{"href":"https://api.github.com/repos/caskdata/cdap/issues/3307/comments"},"review_comments":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/3307/comments"},"review_comment":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/3307/commits"},"statuses":{"href":"https://api.github.com/repos/caskdata/cdap/statuses/2302d167d267ad8cbd4f1072a33efa0c4e6b5879"}}}}