{"action":"created","comment":{"url":"https://api.github.com/repos/blaze/odo/pulls/comments/48045628","id":48045628,"diff_hunk":"@@ -0,0 +1,252 @@\n+from __future__ import absolute_import, division, print_function\n+\n+import errno\n+import os\n+import uuid\n+\n+from avro import schema, datafile, io\n+from avro.schema import AvroException\n+import pandas as pd\n+from datashape import discover, var, Record, Map, Var, \\\n+    Option, null, string, int32, int64, float64, float32, boolean\n+from collections import Iterator\n+from ..append import append\n+from ..convert import convert\n+from ..resource import resource\n+from ..temp import Temp\n+\n+AVRO_TYPE_MAP = {\n+    'string': string,\n+    'int': int32,\n+    'long': int64,\n+    'null': null,\n+    'double': float64,\n+    'float': float32,\n+    'bool': boolean,\n+    'map': Map,\n+    'record': Record,\n+    'array': Var,\n+}\n+\n+class AVRO(object):\n+    \"\"\"Wrapper object for reading and writing an Avro container file\n+\n+    Parameters\n+    ----------\n+\n+    uri : str\n+        uri of avro data\n+\n+    schema : avro.schema.Schema\n+        User specified Avro schema object.  Used to decode file or serialize new records to file.\n+         schema is required to create a new Avro file.\n+         If reading or appending to an existing Avro file, the writers_schema embedded in that file\n+         will be used.\n+\n+    codec : str\n+        compression codec.  Valid values: 'null', 'deflate', 'snappy'\n+\n+    \"\"\"\n+    def __init__(self, uri, schema=None, codec='null', **kwargs):\n+        self._uri = uri\n+        self._schema = schema\n+        self._codec = codec\n+        self._kwargs = kwargs #CURRENTLY UNUSED\n+\n+        if not schema:\n+            sch = self._get_writers_schema()\n+            if sch is None:\n+                raise AvroException(\"Couldn't extract writers schema from '{0}'.  User must provide a valid schema\".format(uri))\n+            self._schema = sch\n+\n+    def __iter__(self):\n+        return self.reader\n+\n+    def next(self):\n+        return self.reader.next()\n+\n+    def __enter__(self):\n+        return self\n+\n+    def __exit__(self, type, value, traceback):\n+        # Perform a close if there's no exception\n+        if type is None:\n+            self.reader.close()\n+            self.writer.close()\n+\n+    def _get_writers_schema(self):\n+        \"\"\"\n+        Extract writers schema embedded in an existing Avro file.\n+        \"\"\"\n+        reader = self.reader\n+        return schema.parse(self.reader.meta['avro.schema']) if reader else None\n+\n+    uri = property(lambda self: self._uri)\n+    codec = property(lambda self: self._codec)\n+    schema = property(lambda self: self._schema)\n+\n+    @property\n+    def reader(self):\n+        if hasattr(self, '_reader'):\n+            if hasattr(self, '_writer'):\n+                self.flush()\n+            return self._reader\n+        else:\n+            try:\n+                rec_reader = io.DatumReader(readers_schema=self.schema)\n+\n+                df_reader = datafile.DataFileReader(\n+                    open(self.uri, 'rb'),","path":"odo/backends/avro.py","position":99,"original_position":99,"commit_id":"491c4d19fcc4f107cbf87d72730b81fc4d3cd35c","original_commit_id":"491c4d19fcc4f107cbf87d72730b81fc4d3cd35c","user":{"login":"cpcloud","id":417981,"avatar_url":"https://avatars.githubusercontent.com/u/417981?v=3","gravatar_id":"","url":"https://api.github.com/users/cpcloud","html_url":"https://github.com/cpcloud","followers_url":"https://api.github.com/users/cpcloud/followers","following_url":"https://api.github.com/users/cpcloud/following{/other_user}","gists_url":"https://api.github.com/users/cpcloud/gists{/gist_id}","starred_url":"https://api.github.com/users/cpcloud/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cpcloud/subscriptions","organizations_url":"https://api.github.com/users/cpcloud/orgs","repos_url":"https://api.github.com/users/cpcloud/repos","events_url":"https://api.github.com/users/cpcloud/events{/privacy}","received_events_url":"https://api.github.com/users/cpcloud/received_events","type":"User","site_admin":false},"body":"this could potentially open a bunch of file handles if `self.reader` is called a bunch of times ... is that a valid concern?","created_at":"2015-12-18T16:57:49Z","updated_at":"2015-12-18T16:57:49Z","html_url":"https://github.com/blaze/odo/pull/386#discussion_r48045628","pull_request_url":"https://api.github.com/repos/blaze/odo/pulls/386","_links":{"self":{"href":"https://api.github.com/repos/blaze/odo/pulls/comments/48045628"},"html":{"href":"https://github.com/blaze/odo/pull/386#discussion_r48045628"},"pull_request":{"href":"https://api.github.com/repos/blaze/odo/pulls/386"}}},"pull_request":{"url":"https://api.github.com/repos/blaze/odo/pulls/386","id":54111983,"html_url":"https://github.com/blaze/odo/pull/386","diff_url":"https://github.com/blaze/odo/pull/386.diff","patch_url":"https://github.com/blaze/odo/pull/386.patch","issue_url":"https://api.github.com/repos/blaze/odo/issues/386","number":386,"state":"open","locked":false,"title":"Adding an Avro backend","user":{"login":"ahasha","id":2471955,"avatar_url":"https://avatars.githubusercontent.com/u/2471955?v=3","gravatar_id":"","url":"https://api.github.com/users/ahasha","html_url":"https://github.com/ahasha","followers_url":"https://api.github.com/users/ahasha/followers","following_url":"https://api.github.com/users/ahasha/following{/other_user}","gists_url":"https://api.github.com/users/ahasha/gists{/gist_id}","starred_url":"https://api.github.com/users/ahasha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ahasha/subscriptions","organizations_url":"https://api.github.com/users/ahasha/orgs","repos_url":"https://api.github.com/users/ahasha/repos","events_url":"https://api.github.com/users/ahasha/events{/privacy}","received_events_url":"https://api.github.com/users/ahasha/received_events","type":"User","site_admin":false},"body":"This PR will fix #370 \r\n\r\nI'm pretty sure there are edge cases that I haven't thought through, and there are a number of avro types that are not supported (i.e. ['null', 'error_union', 'bytes', 'enum', 'request', 'error', 'fixed']).  \r\n\r\nHowever, I figured it is now complete enough to share and get feedback.","created_at":"2015-12-18T15:05:21Z","updated_at":"2015-12-18T16:57:49Z","closed_at":null,"merged_at":null,"merge_commit_sha":"23a818482e16ae95c47923e2b64a8435f9ef8d6f","assignee":null,"milestone":{"url":"https://api.github.com/repos/blaze/odo/milestones/9","html_url":"https://github.com/blaze/odo/milestones/0.4.1","labels_url":"https://api.github.com/repos/blaze/odo/milestones/9/labels","id":1072665,"number":9,"title":"0.4.1","description":"","creator":{"login":"cpcloud","id":417981,"avatar_url":"https://avatars.githubusercontent.com/u/417981?v=3","gravatar_id":"","url":"https://api.github.com/users/cpcloud","html_url":"https://github.com/cpcloud","followers_url":"https://api.github.com/users/cpcloud/followers","following_url":"https://api.github.com/users/cpcloud/following{/other_user}","gists_url":"https://api.github.com/users/cpcloud/gists{/gist_id}","starred_url":"https://api.github.com/users/cpcloud/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cpcloud/subscriptions","organizations_url":"https://api.github.com/users/cpcloud/orgs","repos_url":"https://api.github.com/users/cpcloud/repos","events_url":"https://api.github.com/users/cpcloud/events{/privacy}","received_events_url":"https://api.github.com/users/cpcloud/received_events","type":"User","site_admin":false},"open_issues":55,"closed_issues":6,"state":"open","created_at":"2015-04-17T18:57:52Z","updated_at":"2015-12-18T16:28:17Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/blaze/odo/pulls/386/commits","review_comments_url":"https://api.github.com/repos/blaze/odo/pulls/386/comments","review_comment_url":"https://api.github.com/repos/blaze/odo/pulls/comments{/number}","comments_url":"https://api.github.com/repos/blaze/odo/issues/386/comments","statuses_url":"https://api.github.com/repos/blaze/odo/statuses/491c4d19fcc4f107cbf87d72730b81fc4d3cd35c","head":{"label":"ahasha:avro","ref":"avro","sha":"491c4d19fcc4f107cbf87d72730b81fc4d3cd35c","user":{"login":"ahasha","id":2471955,"avatar_url":"https://avatars.githubusercontent.com/u/2471955?v=3","gravatar_id":"","url":"https://api.github.com/users/ahasha","html_url":"https://github.com/ahasha","followers_url":"https://api.github.com/users/ahasha/followers","following_url":"https://api.github.com/users/ahasha/following{/other_user}","gists_url":"https://api.github.com/users/ahasha/gists{/gist_id}","starred_url":"https://api.github.com/users/ahasha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ahasha/subscriptions","organizations_url":"https://api.github.com/users/ahasha/orgs","repos_url":"https://api.github.com/users/ahasha/repos","events_url":"https://api.github.com/users/ahasha/events{/privacy}","received_events_url":"https://api.github.com/users/ahasha/received_events","type":"User","site_admin":false},"repo":{"id":45632560,"name":"odo","full_name":"ahasha/odo","owner":{"login":"ahasha","id":2471955,"avatar_url":"https://avatars.githubusercontent.com/u/2471955?v=3","gravatar_id":"","url":"https://api.github.com/users/ahasha","html_url":"https://github.com/ahasha","followers_url":"https://api.github.com/users/ahasha/followers","following_url":"https://api.github.com/users/ahasha/following{/other_user}","gists_url":"https://api.github.com/users/ahasha/gists{/gist_id}","starred_url":"https://api.github.com/users/ahasha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ahasha/subscriptions","organizations_url":"https://api.github.com/users/ahasha/orgs","repos_url":"https://api.github.com/users/ahasha/repos","events_url":"https://api.github.com/users/ahasha/events{/privacy}","received_events_url":"https://api.github.com/users/ahasha/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ahasha/odo","description":"Data Migration for the Blaze Project","fork":true,"url":"https://api.github.com/repos/ahasha/odo","forks_url":"https://api.github.com/repos/ahasha/odo/forks","keys_url":"https://api.github.com/repos/ahasha/odo/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ahasha/odo/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ahasha/odo/teams","hooks_url":"https://api.github.com/repos/ahasha/odo/hooks","issue_events_url":"https://api.github.com/repos/ahasha/odo/issues/events{/number}","events_url":"https://api.github.com/repos/ahasha/odo/events","assignees_url":"https://api.github.com/repos/ahasha/odo/assignees{/user}","branches_url":"https://api.github.com/repos/ahasha/odo/branches{/branch}","tags_url":"https://api.github.com/repos/ahasha/odo/tags","blobs_url":"https://api.github.com/repos/ahasha/odo/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ahasha/odo/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ahasha/odo/git/refs{/sha}","trees_url":"https://api.github.com/repos/ahasha/odo/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ahasha/odo/statuses/{sha}","languages_url":"https://api.github.com/repos/ahasha/odo/languages","stargazers_url":"https://api.github.com/repos/ahasha/odo/stargazers","contributors_url":"https://api.github.com/repos/ahasha/odo/contributors","subscribers_url":"https://api.github.com/repos/ahasha/odo/subscribers","subscription_url":"https://api.github.com/repos/ahasha/odo/subscription","commits_url":"https://api.github.com/repos/ahasha/odo/commits{/sha}","git_commits_url":"https://api.github.com/repos/ahasha/odo/git/commits{/sha}","comments_url":"https://api.github.com/repos/ahasha/odo/comments{/number}","issue_comment_url":"https://api.github.com/repos/ahasha/odo/issues/comments{/number}","contents_url":"https://api.github.com/repos/ahasha/odo/contents/{+path}","compare_url":"https://api.github.com/repos/ahasha/odo/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ahasha/odo/merges","archive_url":"https://api.github.com/repos/ahasha/odo/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ahasha/odo/downloads","issues_url":"https://api.github.com/repos/ahasha/odo/issues{/number}","pulls_url":"https://api.github.com/repos/ahasha/odo/pulls{/number}","milestones_url":"https://api.github.com/repos/ahasha/odo/milestones{/number}","notifications_url":"https://api.github.com/repos/ahasha/odo/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ahasha/odo/labels{/name}","releases_url":"https://api.github.com/repos/ahasha/odo/releases{/id}","created_at":"2015-11-05T18:58:54Z","updated_at":"2015-11-05T18:58:56Z","pushed_at":"2015-12-18T16:18:31Z","git_url":"git://github.com/ahasha/odo.git","ssh_url":"git@github.com:ahasha/odo.git","clone_url":"https://github.com/ahasha/odo.git","svn_url":"https://github.com/ahasha/odo","homepage":"http://odo.readthedocs.org/","size":11957,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"blaze:master","ref":"master","sha":"de9a081c55948b92c38087ed9c3f9ba9373d4837","user":{"login":"blaze","id":12833921,"avatar_url":"https://avatars.githubusercontent.com/u/12833921?v=3","gravatar_id":"","url":"https://api.github.com/users/blaze","html_url":"https://github.com/blaze","followers_url":"https://api.github.com/users/blaze/followers","following_url":"https://api.github.com/users/blaze/following{/other_user}","gists_url":"https://api.github.com/users/blaze/gists{/gist_id}","starred_url":"https://api.github.com/users/blaze/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/blaze/subscriptions","organizations_url":"https://api.github.com/users/blaze/orgs","repos_url":"https://api.github.com/users/blaze/repos","events_url":"https://api.github.com/users/blaze/events{/privacy}","received_events_url":"https://api.github.com/users/blaze/received_events","type":"Organization","site_admin":false},"repo":{"id":27790033,"name":"odo","full_name":"blaze/odo","owner":{"login":"blaze","id":12833921,"avatar_url":"https://avatars.githubusercontent.com/u/12833921?v=3","gravatar_id":"","url":"https://api.github.com/users/blaze","html_url":"https://github.com/blaze","followers_url":"https://api.github.com/users/blaze/followers","following_url":"https://api.github.com/users/blaze/following{/other_user}","gists_url":"https://api.github.com/users/blaze/gists{/gist_id}","starred_url":"https://api.github.com/users/blaze/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/blaze/subscriptions","organizations_url":"https://api.github.com/users/blaze/orgs","repos_url":"https://api.github.com/users/blaze/repos","events_url":"https://api.github.com/users/blaze/events{/privacy}","received_events_url":"https://api.github.com/users/blaze/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/blaze/odo","description":"Data Migration for the Blaze Project","fork":false,"url":"https://api.github.com/repos/blaze/odo","forks_url":"https://api.github.com/repos/blaze/odo/forks","keys_url":"https://api.github.com/repos/blaze/odo/keys{/key_id}","collaborators_url":"https://api.github.com/repos/blaze/odo/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/blaze/odo/teams","hooks_url":"https://api.github.com/repos/blaze/odo/hooks","issue_events_url":"https://api.github.com/repos/blaze/odo/issues/events{/number}","events_url":"https://api.github.com/repos/blaze/odo/events","assignees_url":"https://api.github.com/repos/blaze/odo/assignees{/user}","branches_url":"https://api.github.com/repos/blaze/odo/branches{/branch}","tags_url":"https://api.github.com/repos/blaze/odo/tags","blobs_url":"https://api.github.com/repos/blaze/odo/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/blaze/odo/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/blaze/odo/git/refs{/sha}","trees_url":"https://api.github.com/repos/blaze/odo/git/trees{/sha}","statuses_url":"https://api.github.com/repos/blaze/odo/statuses/{sha}","languages_url":"https://api.github.com/repos/blaze/odo/languages","stargazers_url":"https://api.github.com/repos/blaze/odo/stargazers","contributors_url":"https://api.github.com/repos/blaze/odo/contributors","subscribers_url":"https://api.github.com/repos/blaze/odo/subscribers","subscription_url":"https://api.github.com/repos/blaze/odo/subscription","commits_url":"https://api.github.com/repos/blaze/odo/commits{/sha}","git_commits_url":"https://api.github.com/repos/blaze/odo/git/commits{/sha}","comments_url":"https://api.github.com/repos/blaze/odo/comments{/number}","issue_comment_url":"https://api.github.com/repos/blaze/odo/issues/comments{/number}","contents_url":"https://api.github.com/repos/blaze/odo/contents/{+path}","compare_url":"https://api.github.com/repos/blaze/odo/compare/{base}...{head}","merges_url":"https://api.github.com/repos/blaze/odo/merges","archive_url":"https://api.github.com/repos/blaze/odo/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/blaze/odo/downloads","issues_url":"https://api.github.com/repos/blaze/odo/issues{/number}","pulls_url":"https://api.github.com/repos/blaze/odo/pulls{/number}","milestones_url":"https://api.github.com/repos/blaze/odo/milestones{/number}","notifications_url":"https://api.github.com/repos/blaze/odo/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/blaze/odo/labels{/name}","releases_url":"https://api.github.com/repos/blaze/odo/releases{/id}","created_at":"2014-12-09T22:17:14Z","updated_at":"2015-12-06T07:03:15Z","pushed_at":"2015-12-18T16:41:21Z","git_url":"git://github.com/blaze/odo.git","ssh_url":"git@github.com:blaze/odo.git","clone_url":"https://github.com/blaze/odo.git","svn_url":"https://github.com/blaze/odo","homepage":"http://odo.readthedocs.org/","size":12326,"stargazers_count":244,"watchers_count":244,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":52,"mirror_url":null,"open_issues_count":102,"forks":52,"open_issues":102,"watchers":244,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/blaze/odo/pulls/386"},"html":{"href":"https://github.com/blaze/odo/pull/386"},"issue":{"href":"https://api.github.com/repos/blaze/odo/issues/386"},"comments":{"href":"https://api.github.com/repos/blaze/odo/issues/386/comments"},"review_comments":{"href":"https://api.github.com/repos/blaze/odo/pulls/386/comments"},"review_comment":{"href":"https://api.github.com/repos/blaze/odo/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/blaze/odo/pulls/386/commits"},"statuses":{"href":"https://api.github.com/repos/blaze/odo/statuses/491c4d19fcc4f107cbf87d72730b81fc4d3cd35c"}}}}