{"action":"created","comment":{"url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/comments/34132003","id":34132003,"diff_hunk":"@@ -59,5 +72,83 @@ class JsonPediaParser(lang:String)(implicit val sc: SparkContext,implicit val sq\n   }\n \n \n+  /*\n+     Method to Get the list of Surface forms from the wiki\n+   */\n+  def getSfs() : RDD[String] = {\n+\n+    dfWikiRDD.select(\"wid\",\"links.description\")\n+             .rdd\n+             .map(artRow => (artRow.getList[String](1)))\n+             .flatMap(sf => sf)\n+  }\n+\n+  /*\n+  Method to get the wid and article text from the wiki dump\n+   */\n+  //TODO Need to improve the filter condition of blank articles. we can include in the where clause of dataframe\n+  def getArticleText(): RDD[(Long,String)] = {\n+\n+    dfWikiRDD.select(\"wid\",\"wikiText\")\n+             .rdd\n+             .map(artRow => {\n+                 (artRow.getLong(0),artRow.getString(1))\n+             })\n+             .filter(artRow => artRow._2.length > 0)\n+  }\n+\n+  /*\n+   Logic to Get Surface Forms and URIs from the wikiDump\n+   */\n+  def getSfURI(): RDD[(Long,String,String)]= {\n+\n+    val sfUriRDD = dfWikiRDD.select(\"wid\",\"links.description\",\"links.id\")\n+             .rdd\n+             .map(artRow => (artRow.getLong(0),artRow.getList[String](1),artRow.getList[String](2)))\n+             .map{case (wid,sfArray,uriArray) => (wid,sfArray.zip(uriArray))}\n+             .flatMap{case (wid,uriSf) => for(x <- uriSf) yield (wid,x._1,x._2)}\n+    sfUriRDD\n+\n+  }\n+  /*\n+\n+  Logic to Create Memory Token Store\n+   */\n+  def createTokenTypeStore(tokenTypes:List[TokenType]): MemoryTokenTypeStore =  {\n+\n+    val tokenTypeStore = new MemoryTokenTypeStore()\n+    val tokens = new Array[String](tokenTypes.size + 1)\n+    val counts = new Array[Int](tokenTypes.size + 1)\n+\n+    tokenTypes.map(token => {\n+      tokens(token.id) = token.tokenType\n+      counts(token.id) = token.count\n+\n+    })\n+\n+    tokenTypeStore.tokenForId  = tokens.array\n+    tokenTypeStore.counts = counts.array\n+    tokenTypeStore.loaded()\n+\n+    return tokenTypeStore\n+  }\n+\n+  /*\n+  Logic for building the memory type tokens\n+   */\n+  def getTokens(): List[TokenType] ={\n+\n+    val stemmer = new Stemmer()\n+    val locale = new Locale(lang)\n+    val lst = new LanguageIndependentStringTokenizer(locale, stemmer)\n+    //Below Logic is for creating Token Store from the Surface forms\n+    //TODO Getting Searlization error Hence Using Collect and ToList. May need to change in Future","path":"src/main/scala/org/dbpedia/spotlight/wikistats/JsonPediaParser.scala","position":117,"original_position":117,"commit_id":"fa79698ce4366f72b8f53c2375b571e5a744033c","original_commit_id":"fa79698ce4366f72b8f53c2375b571e5a744033c","user":{"login":"dav009","id":1659415,"avatar_url":"https://avatars.githubusercontent.com/u/1659415?v=3","gravatar_id":"","url":"https://api.github.com/users/dav009","html_url":"https://github.com/dav009","followers_url":"https://api.github.com/users/dav009/followers","following_url":"https://api.github.com/users/dav009/following{/other_user}","gists_url":"https://api.github.com/users/dav009/gists{/gist_id}","starred_url":"https://api.github.com/users/dav009/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dav009/subscriptions","organizations_url":"https://api.github.com/users/dav009/orgs","repos_url":"https://api.github.com/users/dav009/repos","events_url":"https://api.github.com/users/dav009/events{/privacy}","received_events_url":"https://api.github.com/users/dav009/received_events","type":"User","site_admin":false},"body":"Could you add this as an Issue to the repo.\r\nIn my experience all of these `ToDo` get forgotten.","created_at":"2015-07-08T09:54:43Z","updated_at":"2015-07-08T09:54:43Z","html_url":"https://github.com/dbpedia-spotlight/wikipedia-stats-extractor/pull/2#discussion_r34132003","pull_request_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/2","_links":{"self":{"href":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/comments/34132003"},"html":{"href":"https://github.com/dbpedia-spotlight/wikipedia-stats-extractor/pull/2#discussion_r34132003"},"pull_request":{"href":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/2"}}},"pull_request":{"url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/2","id":39429500,"html_url":"https://github.com/dbpedia-spotlight/wikipedia-stats-extractor/pull/2","diff_url":"https://github.com/dbpedia-spotlight/wikipedia-stats-extractor/pull/2.diff","patch_url":"https://github.com/dbpedia-spotlight/wikipedia-stats-extractor/pull/2.patch","issue_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/issues/2","number":2,"state":"open","locked":false,"title":"Uri, Pair, Sf Counts PR","user":{"login":"naveenmadhire","id":8851548,"avatar_url":"https://avatars.githubusercontent.com/u/8851548?v=3","gravatar_id":"","url":"https://api.github.com/users/naveenmadhire","html_url":"https://github.com/naveenmadhire","followers_url":"https://api.github.com/users/naveenmadhire/followers","following_url":"https://api.github.com/users/naveenmadhire/following{/other_user}","gists_url":"https://api.github.com/users/naveenmadhire/gists{/gist_id}","starred_url":"https://api.github.com/users/naveenmadhire/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/naveenmadhire/subscriptions","organizations_url":"https://api.github.com/users/naveenmadhire/orgs","repos_url":"https://api.github.com/users/naveenmadhire/repos","events_url":"https://api.github.com/users/naveenmadhire/events{/privacy}","received_events_url":"https://api.github.com/users/naveenmadhire/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-07-07T23:56:25Z","updated_at":"2015-07-08T09:54:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"920f8327a2956251209c28c852a5926759c59fcd","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/2/commits","review_comments_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/2/comments","review_comment_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/issues/2/comments","statuses_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/statuses/fa79698ce4366f72b8f53c2375b571e5a744033c","head":{"label":"naveenmadhire:master","ref":"master","sha":"fa79698ce4366f72b8f53c2375b571e5a744033c","user":{"login":"naveenmadhire","id":8851548,"avatar_url":"https://avatars.githubusercontent.com/u/8851548?v=3","gravatar_id":"","url":"https://api.github.com/users/naveenmadhire","html_url":"https://github.com/naveenmadhire","followers_url":"https://api.github.com/users/naveenmadhire/followers","following_url":"https://api.github.com/users/naveenmadhire/following{/other_user}","gists_url":"https://api.github.com/users/naveenmadhire/gists{/gist_id}","starred_url":"https://api.github.com/users/naveenmadhire/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/naveenmadhire/subscriptions","organizations_url":"https://api.github.com/users/naveenmadhire/orgs","repos_url":"https://api.github.com/users/naveenmadhire/repos","events_url":"https://api.github.com/users/naveenmadhire/events{/privacy}","received_events_url":"https://api.github.com/users/naveenmadhire/received_events","type":"User","site_admin":false},"repo":{"id":35684009,"name":"wikipedia-stats-extractor","full_name":"naveenmadhire/wikipedia-stats-extractor","owner":{"login":"naveenmadhire","id":8851548,"avatar_url":"https://avatars.githubusercontent.com/u/8851548?v=3","gravatar_id":"","url":"https://api.github.com/users/naveenmadhire","html_url":"https://github.com/naveenmadhire","followers_url":"https://api.github.com/users/naveenmadhire/followers","following_url":"https://api.github.com/users/naveenmadhire/following{/other_user}","gists_url":"https://api.github.com/users/naveenmadhire/gists{/gist_id}","starred_url":"https://api.github.com/users/naveenmadhire/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/naveenmadhire/subscriptions","organizations_url":"https://api.github.com/users/naveenmadhire/orgs","repos_url":"https://api.github.com/users/naveenmadhire/repos","events_url":"https://api.github.com/users/naveenmadhire/events{/privacy}","received_events_url":"https://api.github.com/users/naveenmadhire/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/naveenmadhire/wikipedia-stats-extractor","description":"Raw Wikipedia counts for entity linking","fork":true,"url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor","forks_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/forks","keys_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/keys{/key_id}","collaborators_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/teams","hooks_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/hooks","issue_events_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/issues/events{/number}","events_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/events","assignees_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/assignees{/user}","branches_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/branches{/branch}","tags_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/tags","blobs_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/git/refs{/sha}","trees_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/git/trees{/sha}","statuses_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/statuses/{sha}","languages_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/languages","stargazers_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/stargazers","contributors_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/contributors","subscribers_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/subscribers","subscription_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/subscription","commits_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/commits{/sha}","git_commits_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/git/commits{/sha}","comments_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/comments{/number}","issue_comment_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/issues/comments{/number}","contents_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/contents/{+path}","compare_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/compare/{base}...{head}","merges_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/merges","archive_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/downloads","issues_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/issues{/number}","pulls_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/pulls{/number}","milestones_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/milestones{/number}","notifications_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/labels{/name}","releases_url":"https://api.github.com/repos/naveenmadhire/wikipedia-stats-extractor/releases{/id}","created_at":"2015-05-15T16:17:08Z","updated_at":"2015-07-06T03:47:36Z","pushed_at":"2015-07-07T23:50:05Z","git_url":"git://github.com/naveenmadhire/wikipedia-stats-extractor.git","ssh_url":"git@github.com:naveenmadhire/wikipedia-stats-extractor.git","clone_url":"https://github.com/naveenmadhire/wikipedia-stats-extractor.git","svn_url":"https://github.com/naveenmadhire/wikipedia-stats-extractor","homepage":null,"size":207,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"dbpedia-spotlight:master","ref":"master","sha":"4847ffd0335d91c460ce437f484af53f55886b08","user":{"login":"dbpedia-spotlight","id":1667837,"avatar_url":"https://avatars.githubusercontent.com/u/1667837?v=3","gravatar_id":"","url":"https://api.github.com/users/dbpedia-spotlight","html_url":"https://github.com/dbpedia-spotlight","followers_url":"https://api.github.com/users/dbpedia-spotlight/followers","following_url":"https://api.github.com/users/dbpedia-spotlight/following{/other_user}","gists_url":"https://api.github.com/users/dbpedia-spotlight/gists{/gist_id}","starred_url":"https://api.github.com/users/dbpedia-spotlight/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dbpedia-spotlight/subscriptions","organizations_url":"https://api.github.com/users/dbpedia-spotlight/orgs","repos_url":"https://api.github.com/users/dbpedia-spotlight/repos","events_url":"https://api.github.com/users/dbpedia-spotlight/events{/privacy}","received_events_url":"https://api.github.com/users/dbpedia-spotlight/received_events","type":"Organization","site_admin":false},"repo":{"id":35672462,"name":"wikipedia-stats-extractor","full_name":"dbpedia-spotlight/wikipedia-stats-extractor","owner":{"login":"dbpedia-spotlight","id":1667837,"avatar_url":"https://avatars.githubusercontent.com/u/1667837?v=3","gravatar_id":"","url":"https://api.github.com/users/dbpedia-spotlight","html_url":"https://github.com/dbpedia-spotlight","followers_url":"https://api.github.com/users/dbpedia-spotlight/followers","following_url":"https://api.github.com/users/dbpedia-spotlight/following{/other_user}","gists_url":"https://api.github.com/users/dbpedia-spotlight/gists{/gist_id}","starred_url":"https://api.github.com/users/dbpedia-spotlight/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dbpedia-spotlight/subscriptions","organizations_url":"https://api.github.com/users/dbpedia-spotlight/orgs","repos_url":"https://api.github.com/users/dbpedia-spotlight/repos","events_url":"https://api.github.com/users/dbpedia-spotlight/events{/privacy}","received_events_url":"https://api.github.com/users/dbpedia-spotlight/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dbpedia-spotlight/wikipedia-stats-extractor","description":"Raw Wikipedia counts for entity linking","fork":false,"url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor","forks_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/forks","keys_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/teams","hooks_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/hooks","issue_events_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/issues/events{/number}","events_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/events","assignees_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/assignees{/user}","branches_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/branches{/branch}","tags_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/tags","blobs_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/git/refs{/sha}","trees_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/statuses/{sha}","languages_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/languages","stargazers_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/stargazers","contributors_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/contributors","subscribers_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/subscribers","subscription_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/subscription","commits_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/commits{/sha}","git_commits_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/git/commits{/sha}","comments_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/comments{/number}","issue_comment_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/issues/comments{/number}","contents_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/contents/{+path}","compare_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/merges","archive_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/downloads","issues_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/issues{/number}","pulls_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls{/number}","milestones_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/milestones{/number}","notifications_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/labels{/name}","releases_url":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/releases{/id}","created_at":"2015-05-15T12:21:09Z","updated_at":"2015-07-05T15:48:50Z","pushed_at":"2015-07-07T23:56:25Z","git_url":"git://github.com/dbpedia-spotlight/wikipedia-stats-extractor.git","ssh_url":"git@github.com:dbpedia-spotlight/wikipedia-stats-extractor.git","clone_url":"https://github.com/dbpedia-spotlight/wikipedia-stats-extractor.git","svn_url":"https://github.com/dbpedia-spotlight/wikipedia-stats-extractor","homepage":null,"size":688,"stargazers_count":3,"watchers_count":3,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":1,"forks":1,"open_issues":1,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/2"},"html":{"href":"https://github.com/dbpedia-spotlight/wikipedia-stats-extractor/pull/2"},"issue":{"href":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/issues/2"},"comments":{"href":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/issues/2/comments"},"review_comments":{"href":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/2/comments"},"review_comment":{"href":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/pulls/2/commits"},"statuses":{"href":"https://api.github.com/repos/dbpedia-spotlight/wikipedia-stats-extractor/statuses/fa79698ce4366f72b8f53c2375b571e5a744033c"}}}}