{"action":"created","comment":{"url":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/comments/28746470","id":28746470,"diff_hunk":"@@ -0,0 +1,201 @@\n+package org.apache.spark.sql.cassandra\n+\n+import java.io.IOException\n+\n+import com.datastax.spark.connector._\n+import com.datastax.spark.connector.cql.{CassandraConnector, ColumnDef}\n+import com.datastax.spark.connector.rdd.ValidRDDType\n+import com.datastax.spark.connector.writer.SqlRowWriter\n+import org.apache.spark.Logging\n+\n+import org.apache.spark.rdd.RDD\n+import org.apache.spark.sql.cassandra.CassandraSQLRow.CassandraSQLRowReader\n+import org.apache.spark.sql.catalyst.expressions.{Literal, AttributeReference, Expression, Attribute}\n+import org.apache.spark.sql.sources._\n+import org.apache.spark.sql.types._\n+import org.apache.spark.sql.{DataFrame, Row, SQLContext}\n+\n+import scala.reflect.ClassTag\n+\n+/** Create Cassandra data source relation based on [[ScanType]]*/\n+case class CassandraSourceRelation(table: String, keyspace: String, scanType: ScanType, cluster: Option[String],\n+                                   userSpecifiedSchema: Option[StructType], sqlContext: SQLContext) {\n+  def apply(table: String, keyspace: String, scanType: ScanType, cluster: Option[String],\n+            userSpecifiedSchema: Option[StructType], sqlContext: SQLContext) = scanType.makeRelation(\n+    table, keyspace, cluster, userSpecifiedSchema, sqlContext)\n+}\n+\n+/** Base table scan relation*/\n+case object BaseScanType extends ScanType  {\n+  override def makeRelation(table: String, keyspace: String, cluster: Option[String],\n+                            userSpecifiedSchema: Option[StructType],sqlContext: SQLContext) =\n+    new TableScanRelationImpl(table, keyspace, cluster, userSpecifiedSchema, sqlContext)\n+}\n+\n+/** Pruned columns table scan relation*/\n+case object PrunedScanType extends ScanType {\n+  override def makeRelation(table: String, keyspace: String, cluster: Option[String],\n+                            userSpecifiedSchema: Option[StructType],sqlContext: SQLContext) =\n+    new PrunedScanRelationImpl(table, keyspace, cluster, userSpecifiedSchema, sqlContext)\n+}\n+\n+/** Pruned columns and filtered table scan relation*/\n+case object PrunedFilteredScanType extends ScanType {\n+  override def makeRelation(table: String, keyspace: String, cluster: Option[String],\n+                            userSpecifiedSchema: Option[StructType],sqlContext: SQLContext) =\n+    new PrunedFilteredScanRelationImpl(table, keyspace, cluster, userSpecifiedSchema, sqlContext)\n+}\n+\n+/** Pruned columns and Catalyst filtered table scan relation*/\n+case object CatalystScanType extends ScanType {\n+  override def makeRelation(table: String, keyspace: String, cluster: Option[String],\n+                            userSpecifiedSchema: Option[StructType],sqlContext: SQLContext) =\n+    new CatalystScanRelationImpl(table, keyspace, cluster, userSpecifiedSchema, sqlContext)\n+}\n+\n+/** Table scan type */\n+sealed trait ScanType {\n+  def makeRelation(table: String, keyspace: String, cluster: Option[String],\n+                   userSpecifiedSchema: Option[StructType], sqlContext: SQLContext) : BaseRelationImpl\n+}\n+\n+/** Base relation implements [[BaseRelation]] and [[InsertableRelation]] */\n+private[cassandra] class BaseRelationImpl(table: String, keyspace: String, cluster: Option[String],\n+                                          userSpecifiedSchema: Option[StructType], sqlcontext: SQLContext)\n+  extends BaseRelation with InsertableRelation with Serializable with Logging {\n+\n+  override def sqlContext: SQLContext = sqlcontext\n+\n+  protected[this] val tableDef = sqlContext\n+    .schemas.get(cluster.getOrElse(\"default\"))\n+    .keyspaceByName.getOrElse(keyspace, throw new IOException(s\"Keyspace not found: $keyspace\"))\n+    .tableByName.getOrElse(table, throw new IOException(s\"Table not found: $keyspace.$table\"))\n+\n+  protected[this] val columnNameByLowercase = tableDef.allColumns\n+    .map(c => (c.columnName.toLowerCase, c.columnName)).toMap\n+\n+  override def schema: StructType = {\n+    def columnToStructField(column: ColumnDef): StructField = {\n+      StructField(column.columnName, ColumnDataType.catalystDataType(column.columnType, nullable = true))","path":"spark-cassandra-connector/src/main/scala/org/apache/spark/sql/cassandra/CassandraSourceRelation.scala","position":84,"original_position":79,"commit_id":"2f41a60556a1a16143109e414ab2c0d96a56ea18","original_commit_id":"b876436e9848854d8fc4431500cc63de3e420e71","user":{"login":"alexliu68","id":1668804,"avatar_url":"https://avatars.githubusercontent.com/u/1668804?v=3","gravatar_id":"","url":"https://api.github.com/users/alexliu68","html_url":"https://github.com/alexliu68","followers_url":"https://api.github.com/users/alexliu68/followers","following_url":"https://api.github.com/users/alexliu68/following{/other_user}","gists_url":"https://api.github.com/users/alexliu68/gists{/gist_id}","starred_url":"https://api.github.com/users/alexliu68/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alexliu68/subscriptions","organizations_url":"https://api.github.com/users/alexliu68/orgs","repos_url":"https://api.github.com/users/alexliu68/repos","events_url":"https://api.github.com/users/alexliu68/events{/privacy}","received_events_url":"https://api.github.com/users/alexliu68/received_events","type":"User","site_admin":false},"body":"Will add a test for writing null.","created_at":"2015-04-21T02:47:43Z","updated_at":"2015-04-21T02:47:43Z","html_url":"https://github.com/datastax/spark-cassandra-connector/pull/629#discussion_r28746470","pull_request_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/629","_links":{"self":{"href":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/comments/28746470"},"html":{"href":"https://github.com/datastax/spark-cassandra-connector/pull/629#discussion_r28746470"},"pull_request":{"href":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/629"}}},"pull_request":{"url":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/629","id":33292962,"html_url":"https://github.com/datastax/spark-cassandra-connector/pull/629","diff_url":"https://github.com/datastax/spark-cassandra-connector/pull/629.diff","patch_url":"https://github.com/datastax/spark-cassandra-connector/pull/629.patch","issue_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/issues/629","number":629,"state":"open","locked":false,"title":"SPARKC-112 Cassandra Spark DataSource API Integration","user":{"login":"alexliu68","id":1668804,"avatar_url":"https://avatars.githubusercontent.com/u/1668804?v=3","gravatar_id":"","url":"https://api.github.com/users/alexliu68","html_url":"https://github.com/alexliu68","followers_url":"https://api.github.com/users/alexliu68/followers","following_url":"https://api.github.com/users/alexliu68/following{/other_user}","gists_url":"https://api.github.com/users/alexliu68/gists{/gist_id}","starred_url":"https://api.github.com/users/alexliu68/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alexliu68/subscriptions","organizations_url":"https://api.github.com/users/alexliu68/orgs","repos_url":"https://api.github.com/users/alexliu68/repos","events_url":"https://api.github.com/users/alexliu68/events{/privacy}","received_events_url":"https://api.github.com/users/alexliu68/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-04-14T19:48:37Z","updated_at":"2015-04-21T02:47:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0783594bc66ad7389282a6cbd806f64019b6b54b","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/629/commits","review_comments_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/629/comments","review_comment_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/comments{/number}","comments_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/issues/629/comments","statuses_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/statuses/2f41a60556a1a16143109e414ab2c0d96a56ea18","head":{"label":"datastax:SPARKC-112","ref":"SPARKC-112","sha":"2f41a60556a1a16143109e414ab2c0d96a56ea18","user":{"login":"datastax","id":573369,"avatar_url":"https://avatars.githubusercontent.com/u/573369?v=3","gravatar_id":"","url":"https://api.github.com/users/datastax","html_url":"https://github.com/datastax","followers_url":"https://api.github.com/users/datastax/followers","following_url":"https://api.github.com/users/datastax/following{/other_user}","gists_url":"https://api.github.com/users/datastax/gists{/gist_id}","starred_url":"https://api.github.com/users/datastax/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/datastax/subscriptions","organizations_url":"https://api.github.com/users/datastax/orgs","repos_url":"https://api.github.com/users/datastax/repos","events_url":"https://api.github.com/users/datastax/events{/privacy}","received_events_url":"https://api.github.com/users/datastax/received_events","type":"Organization","site_admin":false},"repo":{"id":21280887,"name":"spark-cassandra-connector","full_name":"datastax/spark-cassandra-connector","owner":{"login":"datastax","id":573369,"avatar_url":"https://avatars.githubusercontent.com/u/573369?v=3","gravatar_id":"","url":"https://api.github.com/users/datastax","html_url":"https://github.com/datastax","followers_url":"https://api.github.com/users/datastax/followers","following_url":"https://api.github.com/users/datastax/following{/other_user}","gists_url":"https://api.github.com/users/datastax/gists{/gist_id}","starred_url":"https://api.github.com/users/datastax/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/datastax/subscriptions","organizations_url":"https://api.github.com/users/datastax/orgs","repos_url":"https://api.github.com/users/datastax/repos","events_url":"https://api.github.com/users/datastax/events{/privacy}","received_events_url":"https://api.github.com/users/datastax/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/datastax/spark-cassandra-connector","description":"If you write a Spark application that needs access to Cassandra, this library is for you","fork":false,"url":"https://api.github.com/repos/datastax/spark-cassandra-connector","forks_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/forks","keys_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/keys{/key_id}","collaborators_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/teams","hooks_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/hooks","issue_events_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/issues/events{/number}","events_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/events","assignees_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/assignees{/user}","branches_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/branches{/branch}","tags_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/tags","blobs_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/git/refs{/sha}","trees_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/git/trees{/sha}","statuses_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/statuses/{sha}","languages_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/languages","stargazers_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/stargazers","contributors_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/contributors","subscribers_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/subscribers","subscription_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/subscription","commits_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/commits{/sha}","git_commits_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/git/commits{/sha}","comments_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/comments{/number}","issue_comment_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/issues/comments{/number}","contents_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/contents/{+path}","compare_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/compare/{base}...{head}","merges_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/merges","archive_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/downloads","issues_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/issues{/number}","pulls_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls{/number}","milestones_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/milestones{/number}","notifications_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/labels{/name}","releases_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/releases{/id}","created_at":"2014-06-27T15:45:54Z","updated_at":"2015-04-20T20:18:05Z","pushed_at":"2015-04-21T02:33:30Z","git_url":"git://github.com/datastax/spark-cassandra-connector.git","ssh_url":"git@github.com:datastax/spark-cassandra-connector.git","clone_url":"https://github.com/datastax/spark-cassandra-connector.git","svn_url":"https://github.com/datastax/spark-cassandra-connector","homepage":"","size":11147,"stargazers_count":478,"watchers_count":478,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":143,"mirror_url":null,"open_issues_count":18,"forks":143,"open_issues":18,"watchers":478,"default_branch":"master"}},"base":{"label":"datastax:master","ref":"master","sha":"a6fe12d4f4dc42754bd8acc9f295112ee78aacfe","user":{"login":"datastax","id":573369,"avatar_url":"https://avatars.githubusercontent.com/u/573369?v=3","gravatar_id":"","url":"https://api.github.com/users/datastax","html_url":"https://github.com/datastax","followers_url":"https://api.github.com/users/datastax/followers","following_url":"https://api.github.com/users/datastax/following{/other_user}","gists_url":"https://api.github.com/users/datastax/gists{/gist_id}","starred_url":"https://api.github.com/users/datastax/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/datastax/subscriptions","organizations_url":"https://api.github.com/users/datastax/orgs","repos_url":"https://api.github.com/users/datastax/repos","events_url":"https://api.github.com/users/datastax/events{/privacy}","received_events_url":"https://api.github.com/users/datastax/received_events","type":"Organization","site_admin":false},"repo":{"id":21280887,"name":"spark-cassandra-connector","full_name":"datastax/spark-cassandra-connector","owner":{"login":"datastax","id":573369,"avatar_url":"https://avatars.githubusercontent.com/u/573369?v=3","gravatar_id":"","url":"https://api.github.com/users/datastax","html_url":"https://github.com/datastax","followers_url":"https://api.github.com/users/datastax/followers","following_url":"https://api.github.com/users/datastax/following{/other_user}","gists_url":"https://api.github.com/users/datastax/gists{/gist_id}","starred_url":"https://api.github.com/users/datastax/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/datastax/subscriptions","organizations_url":"https://api.github.com/users/datastax/orgs","repos_url":"https://api.github.com/users/datastax/repos","events_url":"https://api.github.com/users/datastax/events{/privacy}","received_events_url":"https://api.github.com/users/datastax/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/datastax/spark-cassandra-connector","description":"If you write a Spark application that needs access to Cassandra, this library is for you","fork":false,"url":"https://api.github.com/repos/datastax/spark-cassandra-connector","forks_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/forks","keys_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/keys{/key_id}","collaborators_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/teams","hooks_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/hooks","issue_events_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/issues/events{/number}","events_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/events","assignees_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/assignees{/user}","branches_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/branches{/branch}","tags_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/tags","blobs_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/git/refs{/sha}","trees_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/git/trees{/sha}","statuses_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/statuses/{sha}","languages_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/languages","stargazers_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/stargazers","contributors_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/contributors","subscribers_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/subscribers","subscription_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/subscription","commits_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/commits{/sha}","git_commits_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/git/commits{/sha}","comments_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/comments{/number}","issue_comment_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/issues/comments{/number}","contents_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/contents/{+path}","compare_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/compare/{base}...{head}","merges_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/merges","archive_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/downloads","issues_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/issues{/number}","pulls_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls{/number}","milestones_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/milestones{/number}","notifications_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/labels{/name}","releases_url":"https://api.github.com/repos/datastax/spark-cassandra-connector/releases{/id}","created_at":"2014-06-27T15:45:54Z","updated_at":"2015-04-20T20:18:05Z","pushed_at":"2015-04-21T02:33:30Z","git_url":"git://github.com/datastax/spark-cassandra-connector.git","ssh_url":"git@github.com:datastax/spark-cassandra-connector.git","clone_url":"https://github.com/datastax/spark-cassandra-connector.git","svn_url":"https://github.com/datastax/spark-cassandra-connector","homepage":"","size":11147,"stargazers_count":478,"watchers_count":478,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":143,"mirror_url":null,"open_issues_count":18,"forks":143,"open_issues":18,"watchers":478,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/629"},"html":{"href":"https://github.com/datastax/spark-cassandra-connector/pull/629"},"issue":{"href":"https://api.github.com/repos/datastax/spark-cassandra-connector/issues/629"},"comments":{"href":"https://api.github.com/repos/datastax/spark-cassandra-connector/issues/629/comments"},"review_comments":{"href":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/629/comments"},"review_comment":{"href":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/datastax/spark-cassandra-connector/pulls/629/commits"},"statuses":{"href":"https://api.github.com/repos/datastax/spark-cassandra-connector/statuses/2f41a60556a1a16143109e414ab2c0d96a56ea18"}}}}