{"action":"created","comment":{"url":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/comments/37010223","id":37010223,"diff_hunk":"@@ -0,0 +1,195 @@\n+package com.alpine.plugin.samples.ver1_0\n+\n+import com.alpine.plugin.core._\n+import com.alpine.plugin.core.datasource.OperatorDataSourceManager\n+import com.alpine.plugin.core.dialog.{ColumnFilter, OperatorDialog}\n+import com.alpine.plugin.core.io._\n+import com.alpine.plugin.core.spark.utils.SparkUtils\n+import com.alpine.plugin.core.spark.{SparkIOTypedPluginJob, SparkRuntimeWithIOTypedJob}\n+import org.apache.hadoop.fs.{FileSystem, Path}\n+import org.apache.spark.SparkContext\n+\n+import scala.collection.mutable\n+\n+class ColumnFilterPlugin {\n+\n+}\n+\n+class ColumnFilterSignature extends OperatorSignature[\n+  ColumnFilterGUINode,\n+  ColumnFilterRuntime] {\n+  def getMetadata(): OperatorMetadata = {\n+    new OperatorMetadata(\n+      name = \"ColumnFilter (Spark)\",\n+      category = \"Transformation\",\n+      author = \"Egor Pakhomov\",\n+      version = 1,\n+      helpURL = \"\",\n+      iconNamePrefix = \"\"\n+    )\n+  }\n+}\n+\n+\n+class ColumnFilterGUINode extends\n+OperatorGUINode[HdfsTabularDataset, HdfsTabularDataset] {\n+  override def onPlacement(\n+                            operatorDialog: OperatorDialog,\n+                            operatorDataSourceManager: OperatorDataSourceManager,\n+                            operatorSchemaManager: OperatorSchemaManager): Unit = {\n+\n+    operatorDialog.addTabularDatasetColumnCheckboxes(\n+      \"columnsToDelete\",\n+      \"Columns to delete\",\n+      ColumnFilter.All,\n+      \"main\"\n+    )\n+\n+    operatorDialog.addDropdownBox(\n+      \"storageFormat\",\n+      \"Storage format\",\n+      Array(\"Parquet\", \"Avro\", \"Tsv\").toSeq,\n+      \"Parquet\"\n+    )\n+\n+    operatorDialog.addHdfsFileSelector(\n+      \"outputPath\",\n+      \"Output Path\",\n+      \"/tmp\",\n+      operatorDataSourceManager\n+    )\n+\n+  }\n+\n+\n+  override def onInputOrParameterChange(\n+                                         inputSchemas: mutable.Map[String, TabularSchemaOutline],\n+                                         params: OperatorParameters,\n+                                         operatorSchemaManager: OperatorSchemaManager): Unit = {\n+    val (_, columnsToDelete) =\n+      params.getTabularDatasetSelectedColumns(\"columnsToDelete\")\n+\n+\n+    if (inputSchemas.size > 0) {\n+      val inputSchema = inputSchemas.values.iterator.next()\n+      if (inputSchema.getFixedColumns().length > 0) {\n+        val inputSchema = inputSchemas.values.head\n+\n+        val numCols: Int = inputSchema.getMaxNumColumns - columnsToDelete.size\n+\n+        val outputSchema = operatorSchemaManager.createTabularSchemaOutline(\n+          minNumCols = numCols,\n+          maxNumCols = numCols\n+        )\n+\n+        inputSchema.getFixedColumns.filter(name => !columnsToDelete.contains(name.columnName)).foreach(cd => outputSchema.addColumnDef(cd))\n+\n+\n+        operatorSchemaManager.setOutputSchemaOutline(outputSchema)\n+\n+        val storageFormat = params.getStringValue(\"storageFormat\")\n+        if (storageFormat.equals(\"Parquet\")) {\n+          outputSchema.setExpectedOutputFormat(\n+            TabularFormatAttributes.createParquetFormat()\n+          )\n+        } else if (storageFormat.equals(\"Avro\")) {\n+          outputSchema.setExpectedOutputFormat(\n+            TabularFormatAttributes.createAvroFormat()\n+          )\n+        } else {\n+          // Storage format is TSV.\n+          outputSchema.setExpectedOutputFormat(\n+            TabularFormatAttributes.createDelimitedFormat(\n+              \"\\t\",\n+              \"\\\\\",\n+              \"\\\"\"\n+            )\n+          )\n+        }\n+      }\n+    }\n+  }\n+\n+//  override def onOutputVisualization(\n+//                                      input: HdfsTabularDataset,\n+//                                      params: OperatorParameters,\n+//                                      output: HdfsTabularDataset,\n+//                                      visualFactory: VisualModelFactory): VisualModel = {\n+//    visualFactory.createTextVisualization(\n+//      \"Some visualisation\"\n+//    )\n+//  }\n+}\n+\n+class ColumnFilterRuntime extends SparkRuntimeWithIOTypedJob[\n+  ColumnFilterJob,\n+  HdfsTabularDataset,\n+  HdfsTabularDataset] {}\n+\n+class ColumnFilterJob extends\n+SparkIOTypedPluginJob[HdfsTabularDataset, HdfsTabularDataset] {\n+  override def onExecution(\n+                            sparkContext: SparkContext,\n+                            appConf: mutable.Map[String, String],\n+                            input: HdfsTabularDataset,\n+                            operatorParameters: OperatorParameters,\n+                            listener: OperatorListener,\n+                            ioFactory: IOFactory): HdfsTabularDataset = {\n+    val sparkUtils = new SparkUtils(\n+      sparkContext,\n+      ioFactory\n+    )\n+    val schemaOutline = input.getSchemaOutline()\n+    println(\"Input schema : \")\n+    schemaOutline.getFixedColumns().map(\n+      colDef =>\n+        println(colDef.columnName + \" : \" + colDef.columnType.name)\n+    )\n+    val dataFrame = sparkUtils.getDataFrame(","path":"PluginExampleProject/src/main/scala/com/alpine/plugin/samples/ver1_0/ColumnFilterPlugin.scala","position":148,"original_position":148,"commit_id":"b5797be8da8cd8416077c7e546164b10cb6c18e4","original_commit_id":"b5797be8da8cd8416077c7e546164b10cb6c18e4","user":{"login":"jthompson6","id":4349284,"avatar_url":"https://avatars.githubusercontent.com/u/4349284?v=3","gravatar_id":"","url":"https://api.github.com/users/jthompson6","html_url":"https://github.com/jthompson6","followers_url":"https://api.github.com/users/jthompson6/followers","following_url":"https://api.github.com/users/jthompson6/following{/other_user}","gists_url":"https://api.github.com/users/jthompson6/gists{/gist_id}","starred_url":"https://api.github.com/users/jthompson6/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jthompson6/subscriptions","organizations_url":"https://api.github.com/users/jthompson6/orgs","repos_url":"https://api.github.com/users/jthompson6/repos","events_url":"https://api.github.com/users/jthompson6/events{/privacy}","received_events_url":"https://api.github.com/users/jthompson6/received_events","type":"User","site_admin":false},"body":"We should probably be using the plugin template for this example.","created_at":"2015-08-13T18:43:22Z","updated_at":"2015-08-13T18:43:22Z","html_url":"https://github.com/AlpineNow/CustomPlugins/pull/2#discussion_r37010223","pull_request_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/2","_links":{"self":{"href":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/comments/37010223"},"html":{"href":"https://github.com/AlpineNow/CustomPlugins/pull/2#discussion_r37010223"},"pull_request":{"href":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/2"}}},"pull_request":{"url":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/2","id":42320810,"html_url":"https://github.com/AlpineNow/CustomPlugins/pull/2","diff_url":"https://github.com/AlpineNow/CustomPlugins/pull/2.diff","patch_url":"https://github.com/AlpineNow/CustomPlugins/pull/2.patch","issue_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/issues/2","number":2,"state":"open","locked":false,"title":"Adding plugin-test","user":{"login":"epahomov","id":2845390,"avatar_url":"https://avatars.githubusercontent.com/u/2845390?v=3","gravatar_id":"","url":"https://api.github.com/users/epahomov","html_url":"https://github.com/epahomov","followers_url":"https://api.github.com/users/epahomov/followers","following_url":"https://api.github.com/users/epahomov/following{/other_user}","gists_url":"https://api.github.com/users/epahomov/gists{/gist_id}","starred_url":"https://api.github.com/users/epahomov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/epahomov/subscriptions","organizations_url":"https://api.github.com/users/epahomov/orgs","repos_url":"https://api.github.com/users/epahomov/repos","events_url":"https://api.github.com/users/epahomov/events{/privacy}","received_events_url":"https://api.github.com/users/epahomov/received_events","type":"User","site_admin":false},"body":"@codedeft @jthompson6 @chesterxgchen ","created_at":"2015-08-13T07:09:22Z","updated_at":"2015-08-13T18:43:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"30e8bab639b8f0aa21f3b7288ac3c16bd270ece1","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/2/commits","review_comments_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/2/comments","review_comment_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/comments{/number}","comments_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/issues/2/comments","statuses_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/statuses/b5797be8da8cd8416077c7e546164b10cb6c18e4","head":{"label":"AlpineNow:ADD_TESTS","ref":"ADD_TESTS","sha":"b5797be8da8cd8416077c7e546164b10cb6c18e4","user":{"login":"AlpineNow","id":6020016,"avatar_url":"https://avatars.githubusercontent.com/u/6020016?v=3","gravatar_id":"","url":"https://api.github.com/users/AlpineNow","html_url":"https://github.com/AlpineNow","followers_url":"https://api.github.com/users/AlpineNow/followers","following_url":"https://api.github.com/users/AlpineNow/following{/other_user}","gists_url":"https://api.github.com/users/AlpineNow/gists{/gist_id}","starred_url":"https://api.github.com/users/AlpineNow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AlpineNow/subscriptions","organizations_url":"https://api.github.com/users/AlpineNow/orgs","repos_url":"https://api.github.com/users/AlpineNow/repos","events_url":"https://api.github.com/users/AlpineNow/events{/privacy}","received_events_url":"https://api.github.com/users/AlpineNow/received_events","type":"Organization","site_admin":false},"repo":{"id":38650675,"name":"CustomPlugins","full_name":"AlpineNow/CustomPlugins","owner":{"login":"AlpineNow","id":6020016,"avatar_url":"https://avatars.githubusercontent.com/u/6020016?v=3","gravatar_id":"","url":"https://api.github.com/users/AlpineNow","html_url":"https://github.com/AlpineNow","followers_url":"https://api.github.com/users/AlpineNow/followers","following_url":"https://api.github.com/users/AlpineNow/following{/other_user}","gists_url":"https://api.github.com/users/AlpineNow/gists{/gist_id}","starred_url":"https://api.github.com/users/AlpineNow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AlpineNow/subscriptions","organizations_url":"https://api.github.com/users/AlpineNow/orgs","repos_url":"https://api.github.com/users/AlpineNow/repos","events_url":"https://api.github.com/users/AlpineNow/events{/privacy}","received_events_url":"https://api.github.com/users/AlpineNow/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/AlpineNow/CustomPlugins","description":"","fork":false,"url":"https://api.github.com/repos/AlpineNow/CustomPlugins","forks_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/forks","keys_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/keys{/key_id}","collaborators_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/teams","hooks_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/hooks","issue_events_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/issues/events{/number}","events_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/events","assignees_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/assignees{/user}","branches_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/branches{/branch}","tags_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/tags","blobs_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/git/refs{/sha}","trees_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/git/trees{/sha}","statuses_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/statuses/{sha}","languages_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/languages","stargazers_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/stargazers","contributors_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/contributors","subscribers_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/subscribers","subscription_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/subscription","commits_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/commits{/sha}","git_commits_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/git/commits{/sha}","comments_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/comments{/number}","issue_comment_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/issues/comments{/number}","contents_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/contents/{+path}","compare_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/compare/{base}...{head}","merges_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/merges","archive_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/downloads","issues_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/issues{/number}","pulls_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls{/number}","milestones_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/milestones{/number}","notifications_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/labels{/name}","releases_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/releases{/id}","created_at":"2015-07-06T22:51:09Z","updated_at":"2015-07-08T21:12:02Z","pushed_at":"2015-08-13T07:09:23Z","git_url":"git://github.com/AlpineNow/CustomPlugins.git","ssh_url":"git@github.com:AlpineNow/CustomPlugins.git","clone_url":"https://github.com/AlpineNow/CustomPlugins.git","svn_url":"https://github.com/AlpineNow/CustomPlugins","homepage":null,"size":244,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"AlpineNow:master","ref":"master","sha":"39de2fa07e2970bb11331bcfcae6744889efb9dc","user":{"login":"AlpineNow","id":6020016,"avatar_url":"https://avatars.githubusercontent.com/u/6020016?v=3","gravatar_id":"","url":"https://api.github.com/users/AlpineNow","html_url":"https://github.com/AlpineNow","followers_url":"https://api.github.com/users/AlpineNow/followers","following_url":"https://api.github.com/users/AlpineNow/following{/other_user}","gists_url":"https://api.github.com/users/AlpineNow/gists{/gist_id}","starred_url":"https://api.github.com/users/AlpineNow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AlpineNow/subscriptions","organizations_url":"https://api.github.com/users/AlpineNow/orgs","repos_url":"https://api.github.com/users/AlpineNow/repos","events_url":"https://api.github.com/users/AlpineNow/events{/privacy}","received_events_url":"https://api.github.com/users/AlpineNow/received_events","type":"Organization","site_admin":false},"repo":{"id":38650675,"name":"CustomPlugins","full_name":"AlpineNow/CustomPlugins","owner":{"login":"AlpineNow","id":6020016,"avatar_url":"https://avatars.githubusercontent.com/u/6020016?v=3","gravatar_id":"","url":"https://api.github.com/users/AlpineNow","html_url":"https://github.com/AlpineNow","followers_url":"https://api.github.com/users/AlpineNow/followers","following_url":"https://api.github.com/users/AlpineNow/following{/other_user}","gists_url":"https://api.github.com/users/AlpineNow/gists{/gist_id}","starred_url":"https://api.github.com/users/AlpineNow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AlpineNow/subscriptions","organizations_url":"https://api.github.com/users/AlpineNow/orgs","repos_url":"https://api.github.com/users/AlpineNow/repos","events_url":"https://api.github.com/users/AlpineNow/events{/privacy}","received_events_url":"https://api.github.com/users/AlpineNow/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/AlpineNow/CustomPlugins","description":"","fork":false,"url":"https://api.github.com/repos/AlpineNow/CustomPlugins","forks_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/forks","keys_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/keys{/key_id}","collaborators_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/teams","hooks_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/hooks","issue_events_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/issues/events{/number}","events_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/events","assignees_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/assignees{/user}","branches_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/branches{/branch}","tags_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/tags","blobs_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/git/refs{/sha}","trees_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/git/trees{/sha}","statuses_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/statuses/{sha}","languages_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/languages","stargazers_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/stargazers","contributors_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/contributors","subscribers_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/subscribers","subscription_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/subscription","commits_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/commits{/sha}","git_commits_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/git/commits{/sha}","comments_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/comments{/number}","issue_comment_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/issues/comments{/number}","contents_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/contents/{+path}","compare_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/compare/{base}...{head}","merges_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/merges","archive_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/downloads","issues_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/issues{/number}","pulls_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls{/number}","milestones_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/milestones{/number}","notifications_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/labels{/name}","releases_url":"https://api.github.com/repos/AlpineNow/CustomPlugins/releases{/id}","created_at":"2015-07-06T22:51:09Z","updated_at":"2015-07-08T21:12:02Z","pushed_at":"2015-08-13T07:09:23Z","git_url":"git://github.com/AlpineNow/CustomPlugins.git","ssh_url":"git@github.com:AlpineNow/CustomPlugins.git","clone_url":"https://github.com/AlpineNow/CustomPlugins.git","svn_url":"https://github.com/AlpineNow/CustomPlugins","homepage":null,"size":244,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/2"},"html":{"href":"https://github.com/AlpineNow/CustomPlugins/pull/2"},"issue":{"href":"https://api.github.com/repos/AlpineNow/CustomPlugins/issues/2"},"comments":{"href":"https://api.github.com/repos/AlpineNow/CustomPlugins/issues/2/comments"},"review_comments":{"href":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/2/comments"},"review_comment":{"href":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/AlpineNow/CustomPlugins/pulls/2/commits"},"statuses":{"href":"https://api.github.com/repos/AlpineNow/CustomPlugins/statuses/b5797be8da8cd8416077c7e546164b10cb6c18e4"}}}}