{"action":"created","comment":{"url":"https://api.github.com/repos/locationtech/geomesa/pulls/comments/45253486","id":45253486,"diff_hunk":"@@ -0,0 +1,119 @@\n+/***********************************************************************\n+* Copyright (c) 2013-2015 Commonwealth Computer Research, Inc.\n+* All rights reserved. This program and the accompanying materials\n+* are made available under the terms of the Apache License, Version 2.0 which\n+* accompanies this distribution and is available at\n+* http://www.opensource.org/licenses/apache2.0.php.\n+*************************************************************************/\n+package org.locationtech.geomesa.tools.ingest\n+\n+import java.net.URLDecoder\n+import java.nio.charset.StandardCharsets\n+\n+import com.twitter.scalding.{Args, Hdfs, Job, Local, Mode}\n+import com.typesafe.config.ConfigFactory\n+import com.typesafe.scalalogging.slf4j.Logging\n+import org.geotools.data.{DataStoreFinder, Transaction}\n+import org.geotools.filter.identity.FeatureIdImpl\n+import org.locationtech.geomesa.accumulo.data.AccumuloDataStore\n+import org.locationtech.geomesa.accumulo.data.AccumuloDataStoreFactory.{params => dsp}\n+import org.locationtech.geomesa.accumulo.index.Constants\n+import org.locationtech.geomesa.convert.SimpleFeatureConverters\n+import org.locationtech.geomesa.convert.Transformers.DefaultCounter\n+import org.locationtech.geomesa.jobs.scalding.MultipleUsefulTextLineFiles\n+import org.locationtech.geomesa.tools.Utils.IngestParams\n+import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes\n+\n+class ScaldingConverterIngestJob(args: Args) extends Job(args) with Logging {\n+  import scala.collection.JavaConversions._\n+\n+  val counter = new DefaultCounter\n+\n+  lazy val pathList         = DelimitedIngest.decodeFileList(args(IngestParams.FILE_PATH))\n+  lazy val sftSpec          = URLDecoder.decode(args(IngestParams.SFT_SPEC), StandardCharsets.UTF_8.displayName)\n+  lazy val featureName      = args(IngestParams.FEATURE_NAME)\n+  lazy val converterConfig  = URLDecoder.decode(args(IngestParams.CONVERTER_CONFIG), StandardCharsets.UTF_8.displayName)\n+  lazy val isTestRun        = args(IngestParams.IS_TEST_INGEST).toBoolean\n+\n+  //Data Store parameters\n+  lazy val dsConfig =\n+    Map(\n+      dsp.zookeepersParam.getName -> args(IngestParams.ZOOKEEPERS),\n+      dsp.instanceIdParam.getName -> args(IngestParams.ACCUMULO_INSTANCE),\n+      dsp.tableNameParam.getName  -> args(IngestParams.CATALOG_TABLE),\n+      dsp.userParam.getName       -> args(IngestParams.ACCUMULO_USER),\n+      dsp.passwordParam.getName   -> args(IngestParams.ACCUMULO_PASSWORD),\n+      dsp.authsParam.getName      -> args.optional(IngestParams.AUTHORIZATIONS),\n+      dsp.visibilityParam.getName -> args.optional(IngestParams.VISIBILITIES),\n+      dsp.mockParam.getName       -> args.optional(IngestParams.ACCUMULO_MOCK)\n+    ).collect{ case (key, Some(value)) => (key, value); case (key, value: String) => (key, value) }\n+\n+  lazy val sft = {\n+    val ret = SimpleFeatureTypes.createType(featureName, sftSpec)\n+    args.optional(IngestParams.INDEX_SCHEMA_FMT).foreach { indexSchema =>\n+      ret.getUserData.put(Constants.SFT_INDEX_SCHEMA, indexSchema)\n+    }\n+    ret\n+  }\n+\n+  // non-serializable resources.\n+  class Resources {\n+    val ds = DataStoreFinder.getDataStore(dsConfig).asInstanceOf[AccumuloDataStore]\n+    lazy val fw = ds.getFeatureWriterAppend(featureName, Transaction.AUTO_COMMIT)\n+    val converter = SimpleFeatureConverters.build[String](sft, ConfigFactory.parseString(converterConfig))\n+    val callback = converter.processWithCallback(counter = counter)\n+    def release(): Unit = {\n+      logger.trace(\"Releasing ingest resources\")\n+      converter.close()\n+      fw.close()\n+    }\n+  }\n+\n+  def printStatInfo() {\n+    Mode.getMode(args) match {\n+      case Some(Local(_)) =>\n+        logger.info(getStatInfo(counter.getSuccess, counter.getFailure, \"Local ingest completed, total lines:\"))\n+      case Some(Hdfs(_, _)) =>\n+        logger.info(\"Ingest completed in HDFS mode\")\n+      case _ =>\n+        logger.warn(\"Could not determine job mode\")\n+    }\n+  }\n+\n+  def getStatInfo(successes: Int, failures: Int, pref: String): String = {\n+    val successPvsS = if (successes == 1) \"feature\" else \"features\"\n+    val failurePvsS = if (failures == 1) \"feature\" else \"features\"\n+    val failureString = if (failures == 0) \"with no failures\" else s\"and failed to ingest: $failures $failurePvsS\"\n+    s\"$pref ${counter.getLineCount}, ingested: $successes $successPvsS, $failureString.\"\n+  }\n+\n+  // Check to see if this an actual ingest job or just a test.\n+  if (!isTestRun) {\n+    new MultipleUsefulTextLineFiles(pathList: _*).using(new Resources)\n+      .foreach('line) { (cres: Resources, s: String) =>\n+        processLine(cres, s)\n+      }\n+  }\n+\n+  private def processLine(resources: Resources, s: String) =\n+    resources.callback(s)\n+      .foreach { sf =>\n+        val toWrite = resources.fw.next()\n+        toWrite.setAttributes(sf.getAttributes)\n+        toWrite.getIdentifier.asInstanceOf[FeatureIdImpl].setID(sf.getID)\n+        toWrite.getUserData.putAll(sf.getUserData)","path":"geomesa-tools/src/main/scala/org/locationtech/geomesa/tools/ingest/ScaldingConverterIngestJob.scala","position":104,"original_position":104,"commit_id":"33effe8e8f022be6e1cfab28a623b3773c79db77","original_commit_id":"33effe8e8f022be6e1cfab28a623b3773c79db77","user":{"login":"elahrvivaz","id":7715322,"avatar_url":"https://avatars.githubusercontent.com/u/7715322?v=3","gravatar_id":"","url":"https://api.github.com/users/elahrvivaz","html_url":"https://github.com/elahrvivaz","followers_url":"https://api.github.com/users/elahrvivaz/followers","following_url":"https://api.github.com/users/elahrvivaz/following{/other_user}","gists_url":"https://api.github.com/users/elahrvivaz/gists{/gist_id}","starred_url":"https://api.github.com/users/elahrvivaz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elahrvivaz/subscriptions","organizations_url":"https://api.github.com/users/elahrvivaz/orgs","repos_url":"https://api.github.com/users/elahrvivaz/repos","events_url":"https://api.github.com/users/elahrvivaz/events{/privacy}","received_events_url":"https://api.github.com/users/elahrvivaz/received_events","type":"User","site_admin":false},"body":"should set Hints.USE_PROVIDED_FID (or have an option for it at least)","created_at":"2015-11-18T20:11:28Z","updated_at":"2015-11-18T20:11:28Z","html_url":"https://github.com/locationtech/geomesa/pull/719#discussion_r45253486","pull_request_url":"https://api.github.com/repos/locationtech/geomesa/pulls/719","_links":{"self":{"href":"https://api.github.com/repos/locationtech/geomesa/pulls/comments/45253486"},"html":{"href":"https://github.com/locationtech/geomesa/pull/719#discussion_r45253486"},"pull_request":{"href":"https://api.github.com/repos/locationtech/geomesa/pulls/719"}}},"pull_request":{"url":"https://api.github.com/repos/locationtech/geomesa/pulls/719","id":51013430,"html_url":"https://github.com/locationtech/geomesa/pull/719","diff_url":"https://github.com/locationtech/geomesa/pull/719.diff","patch_url":"https://github.com/locationtech/geomesa/pull/719.patch","issue_url":"https://api.github.com/repos/locationtech/geomesa/issues/719","number":719,"state":"open","locked":false,"title":"For Comment: GEOMESA-937 Use Geomesa-Convert for Ingest Tools","user":{"login":"jahhulbert-ccri","id":7433591,"avatar_url":"https://avatars.githubusercontent.com/u/7433591?v=3","gravatar_id":"","url":"https://api.github.com/users/jahhulbert-ccri","html_url":"https://github.com/jahhulbert-ccri","followers_url":"https://api.github.com/users/jahhulbert-ccri/followers","following_url":"https://api.github.com/users/jahhulbert-ccri/following{/other_user}","gists_url":"https://api.github.com/users/jahhulbert-ccri/gists{/gist_id}","starred_url":"https://api.github.com/users/jahhulbert-ccri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jahhulbert-ccri/subscriptions","organizations_url":"https://api.github.com/users/jahhulbert-ccri/orgs","repos_url":"https://api.github.com/users/jahhulbert-ccri/repos","events_url":"https://api.github.com/users/jahhulbert-ccri/events{/privacy}","received_events_url":"https://api.github.com/users/jahhulbert-ccri/received_events","type":"User","site_admin":false},"body":"* Use the geomesa-convert framework for ingesting data via the command line tools\r\n* Replace old ingest command line args with args for converter configuration\r\n* Add transforms to converter framework for lists and maps\r\n* Allow SFT to be specified in TypeSafe conf for ingest (as a string or a filename)\r\n* Allow conf files with SFT and converter in the same file\r\n* Configurable paths to SFT and Convert Conf objects in TypeSafe config files\r\n\r\nSigned-off-by: Andrew Hulbert <andrew.hulbert@ccri.com>","created_at":"2015-11-17T22:18:38Z","updated_at":"2015-11-18T20:11:28Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e4cc1d9f6617a940b11605f0c58eeaf20ec62dce","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/locationtech/geomesa/pulls/719/commits","review_comments_url":"https://api.github.com/repos/locationtech/geomesa/pulls/719/comments","review_comment_url":"https://api.github.com/repos/locationtech/geomesa/pulls/comments{/number}","comments_url":"https://api.github.com/repos/locationtech/geomesa/issues/719/comments","statuses_url":"https://api.github.com/repos/locationtech/geomesa/statuses/33effe8e8f022be6e1cfab28a623b3773c79db77","head":{"label":"jahhulbert-ccri:f_GEOMESA-937","ref":"f_GEOMESA-937","sha":"33effe8e8f022be6e1cfab28a623b3773c79db77","user":{"login":"jahhulbert-ccri","id":7433591,"avatar_url":"https://avatars.githubusercontent.com/u/7433591?v=3","gravatar_id":"","url":"https://api.github.com/users/jahhulbert-ccri","html_url":"https://github.com/jahhulbert-ccri","followers_url":"https://api.github.com/users/jahhulbert-ccri/followers","following_url":"https://api.github.com/users/jahhulbert-ccri/following{/other_user}","gists_url":"https://api.github.com/users/jahhulbert-ccri/gists{/gist_id}","starred_url":"https://api.github.com/users/jahhulbert-ccri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jahhulbert-ccri/subscriptions","organizations_url":"https://api.github.com/users/jahhulbert-ccri/orgs","repos_url":"https://api.github.com/users/jahhulbert-ccri/repos","events_url":"https://api.github.com/users/jahhulbert-ccri/events{/privacy}","received_events_url":"https://api.github.com/users/jahhulbert-ccri/received_events","type":"User","site_admin":false},"repo":{"id":20939620,"name":"geomesa","full_name":"jahhulbert-ccri/geomesa","owner":{"login":"jahhulbert-ccri","id":7433591,"avatar_url":"https://avatars.githubusercontent.com/u/7433591?v=3","gravatar_id":"","url":"https://api.github.com/users/jahhulbert-ccri","html_url":"https://github.com/jahhulbert-ccri","followers_url":"https://api.github.com/users/jahhulbert-ccri/followers","following_url":"https://api.github.com/users/jahhulbert-ccri/following{/other_user}","gists_url":"https://api.github.com/users/jahhulbert-ccri/gists{/gist_id}","starred_url":"https://api.github.com/users/jahhulbert-ccri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jahhulbert-ccri/subscriptions","organizations_url":"https://api.github.com/users/jahhulbert-ccri/orgs","repos_url":"https://api.github.com/users/jahhulbert-ccri/repos","events_url":"https://api.github.com/users/jahhulbert-ccri/events{/privacy}","received_events_url":"https://api.github.com/users/jahhulbert-ccri/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jahhulbert-ccri/geomesa","description":"","fork":true,"url":"https://api.github.com/repos/jahhulbert-ccri/geomesa","forks_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/forks","keys_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/teams","hooks_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/hooks","issue_events_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/issues/events{/number}","events_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/events","assignees_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/assignees{/user}","branches_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/branches{/branch}","tags_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/tags","blobs_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/git/refs{/sha}","trees_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/statuses/{sha}","languages_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/languages","stargazers_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/stargazers","contributors_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/contributors","subscribers_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/subscribers","subscription_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/subscription","commits_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/commits{/sha}","git_commits_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/git/commits{/sha}","comments_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/comments{/number}","issue_comment_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/issues/comments{/number}","contents_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/contents/{+path}","compare_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/merges","archive_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/downloads","issues_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/issues{/number}","pulls_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/pulls{/number}","milestones_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/milestones{/number}","notifications_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/labels{/name}","releases_url":"https://api.github.com/repos/jahhulbert-ccri/geomesa/releases{/id}","created_at":"2014-06-17T21:07:18Z","updated_at":"2015-03-27T17:58:43Z","pushed_at":"2015-11-17T22:18:27Z","git_url":"git://github.com/jahhulbert-ccri/geomesa.git","ssh_url":"git@github.com:jahhulbert-ccri/geomesa.git","clone_url":"https://github.com/jahhulbert-ccri/geomesa.git","svn_url":"https://github.com/jahhulbert-ccri/geomesa","homepage":null,"size":7096,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"accumulo1.5.x/1.x"}},"base":{"label":"locationtech:master","ref":"master","sha":"b655edee477d5fbc5ba4e16e2809dcd814d0525d","user":{"login":"locationtech","id":1991411,"avatar_url":"https://avatars.githubusercontent.com/u/1991411?v=3","gravatar_id":"","url":"https://api.github.com/users/locationtech","html_url":"https://github.com/locationtech","followers_url":"https://api.github.com/users/locationtech/followers","following_url":"https://api.github.com/users/locationtech/following{/other_user}","gists_url":"https://api.github.com/users/locationtech/gists{/gist_id}","starred_url":"https://api.github.com/users/locationtech/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/locationtech/subscriptions","organizations_url":"https://api.github.com/users/locationtech/orgs","repos_url":"https://api.github.com/users/locationtech/repos","events_url":"https://api.github.com/users/locationtech/events{/privacy}","received_events_url":"https://api.github.com/users/locationtech/received_events","type":"Organization","site_admin":false},"repo":{"id":13226069,"name":"geomesa","full_name":"locationtech/geomesa","owner":{"login":"locationtech","id":1991411,"avatar_url":"https://avatars.githubusercontent.com/u/1991411?v=3","gravatar_id":"","url":"https://api.github.com/users/locationtech","html_url":"https://github.com/locationtech","followers_url":"https://api.github.com/users/locationtech/followers","following_url":"https://api.github.com/users/locationtech/following{/other_user}","gists_url":"https://api.github.com/users/locationtech/gists{/gist_id}","starred_url":"https://api.github.com/users/locationtech/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/locationtech/subscriptions","organizations_url":"https://api.github.com/users/locationtech/orgs","repos_url":"https://api.github.com/users/locationtech/repos","events_url":"https://api.github.com/users/locationtech/events{/privacy}","received_events_url":"https://api.github.com/users/locationtech/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/locationtech/geomesa","description":"","fork":false,"url":"https://api.github.com/repos/locationtech/geomesa","forks_url":"https://api.github.com/repos/locationtech/geomesa/forks","keys_url":"https://api.github.com/repos/locationtech/geomesa/keys{/key_id}","collaborators_url":"https://api.github.com/repos/locationtech/geomesa/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/locationtech/geomesa/teams","hooks_url":"https://api.github.com/repos/locationtech/geomesa/hooks","issue_events_url":"https://api.github.com/repos/locationtech/geomesa/issues/events{/number}","events_url":"https://api.github.com/repos/locationtech/geomesa/events","assignees_url":"https://api.github.com/repos/locationtech/geomesa/assignees{/user}","branches_url":"https://api.github.com/repos/locationtech/geomesa/branches{/branch}","tags_url":"https://api.github.com/repos/locationtech/geomesa/tags","blobs_url":"https://api.github.com/repos/locationtech/geomesa/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/locationtech/geomesa/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/locationtech/geomesa/git/refs{/sha}","trees_url":"https://api.github.com/repos/locationtech/geomesa/git/trees{/sha}","statuses_url":"https://api.github.com/repos/locationtech/geomesa/statuses/{sha}","languages_url":"https://api.github.com/repos/locationtech/geomesa/languages","stargazers_url":"https://api.github.com/repos/locationtech/geomesa/stargazers","contributors_url":"https://api.github.com/repos/locationtech/geomesa/contributors","subscribers_url":"https://api.github.com/repos/locationtech/geomesa/subscribers","subscription_url":"https://api.github.com/repos/locationtech/geomesa/subscription","commits_url":"https://api.github.com/repos/locationtech/geomesa/commits{/sha}","git_commits_url":"https://api.github.com/repos/locationtech/geomesa/git/commits{/sha}","comments_url":"https://api.github.com/repos/locationtech/geomesa/comments{/number}","issue_comment_url":"https://api.github.com/repos/locationtech/geomesa/issues/comments{/number}","contents_url":"https://api.github.com/repos/locationtech/geomesa/contents/{+path}","compare_url":"https://api.github.com/repos/locationtech/geomesa/compare/{base}...{head}","merges_url":"https://api.github.com/repos/locationtech/geomesa/merges","archive_url":"https://api.github.com/repos/locationtech/geomesa/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/locationtech/geomesa/downloads","issues_url":"https://api.github.com/repos/locationtech/geomesa/issues{/number}","pulls_url":"https://api.github.com/repos/locationtech/geomesa/pulls{/number}","milestones_url":"https://api.github.com/repos/locationtech/geomesa/milestones{/number}","notifications_url":"https://api.github.com/repos/locationtech/geomesa/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/locationtech/geomesa/labels{/name}","releases_url":"https://api.github.com/repos/locationtech/geomesa/releases{/id}","created_at":"2013-09-30T20:13:46Z","updated_at":"2015-11-15T14:39:01Z","pushed_at":"2015-11-18T19:27:13Z","git_url":"git://github.com/locationtech/geomesa.git","ssh_url":"git@github.com:locationtech/geomesa.git","clone_url":"https://github.com/locationtech/geomesa.git","svn_url":"https://github.com/locationtech/geomesa","homepage":null,"size":13734,"stargazers_count":108,"watchers_count":108,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":67,"mirror_url":null,"open_issues_count":9,"forks":67,"open_issues":9,"watchers":108,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/locationtech/geomesa/pulls/719"},"html":{"href":"https://github.com/locationtech/geomesa/pull/719"},"issue":{"href":"https://api.github.com/repos/locationtech/geomesa/issues/719"},"comments":{"href":"https://api.github.com/repos/locationtech/geomesa/issues/719/comments"},"review_comments":{"href":"https://api.github.com/repos/locationtech/geomesa/pulls/719/comments"},"review_comment":{"href":"https://api.github.com/repos/locationtech/geomesa/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/locationtech/geomesa/pulls/719/commits"},"statuses":{"href":"https://api.github.com/repos/locationtech/geomesa/statuses/33effe8e8f022be6e1cfab28a623b3773c79db77"}}}}