{"action":"created","comment":{"url":"https://api.github.com/repos/ohmu/pghoard/pulls/comments/37850849","id":37850849,"diff_hunk":"@@ -40,19 +46,26 @@ def find_site_for_file(self, filepath):\n             return filepath.split(\"/\")[-4]\n         return filepath.split(\"/\")[-3]\n \n-    def compress_filepath(self, filepath, compression_dir):\n-        compressed_filepath, algorithm = self.compress_lzma_filepath(filepath, compression_dir)\n-        compressed_file_size = os.stat(compressed_filepath).st_size\n-        return compressed_filepath, algorithm, compressed_file_size\n+    def compress_filepath(self, filepath, targetfilepath):\n+        algorithm = self.compress_lzma_filepath(filepath, targetfilepath)\n+        compressed_file_size = os.stat(targetfilepath).st_size\n+        return algorithm, compressed_file_size\n \n-    def compress_lzma_filepath(self, filepath, compression_dir):\n-        lzma_filepath = os.path.join(compression_dir, os.path.basename(filepath)) + \".xz\"\n-        # NOTE: pyliblzma, which may be in use, is buggy and requires an explicit closing wrapper\n-        with contextlib.closing(lzma_open(lzma_filepath, mode=\"wb\", preset=0)) as lzma_file:\n-            # TODO: fsync, LZMAFile has no .fileno() on 2.7 etc\n-            with open(filepath, \"rb\") as input_file:\n-                shutil.copyfileobj(input_file, lzma_file, length=IO_BLOCK_SIZE)\n-        return lzma_filepath, \"lzma\"\n+    def compress_lzma_filepath(self, filepath, targetfilepath):\n+        compressor = lzma_compressor(preset=0)\n+        with open(filepath, \"rb\") as input_file:\n+            with open(targetfilepath, \"wb\") as output_file:\n+                while True:\n+                    input_data = input_file.read(IO_BLOCK_SIZE)\n+                    if not input_data:\n+                        break\n+                    compressed_data = compressor.compress(input_data)\n+                    if compressed_data:\n+                        output_file.write(compressed_data)\n+                compressed_data = compressor.flush()","path":"pghoard/compressor.py","position":71,"original_position":71,"commit_id":"eb88fbcebde8605246e0b40b483f6b2c1beffb79","original_commit_id":"eb88fbcebde8605246e0b40b483f6b2c1beffb79","user":{"login":"melor","id":396587,"avatar_url":"https://avatars.githubusercontent.com/u/396587?v=3","gravatar_id":"","url":"https://api.github.com/users/melor","html_url":"https://github.com/melor","followers_url":"https://api.github.com/users/melor/followers","following_url":"https://api.github.com/users/melor/following{/other_user}","gists_url":"https://api.github.com/users/melor/gists{/gist_id}","starred_url":"https://api.github.com/users/melor/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/melor/subscriptions","organizations_url":"https://api.github.com/users/melor/orgs","repos_url":"https://api.github.com/users/melor/repos","events_url":"https://api.github.com/users/melor/events{/privacy}","received_events_url":"https://api.github.com/users/melor/received_events","type":"User","site_admin":false},"body":"flushing can only be done once per compressor (at the end)","created_at":"2015-08-25T10:11:06Z","updated_at":"2015-08-25T10:11:06Z","html_url":"https://github.com/ohmu/pghoard/pull/13#discussion_r37850849","pull_request_url":"https://api.github.com/repos/ohmu/pghoard/pulls/13","_links":{"self":{"href":"https://api.github.com/repos/ohmu/pghoard/pulls/comments/37850849"},"html":{"href":"https://github.com/ohmu/pghoard/pull/13#discussion_r37850849"},"pull_request":{"href":"https://api.github.com/repos/ohmu/pghoard/pulls/13"}}},"pull_request":{"url":"https://api.github.com/repos/ohmu/pghoard/pulls/13","id":43258929,"html_url":"https://github.com/ohmu/pghoard/pull/13","diff_url":"https://github.com/ohmu/pghoard/pull/13.diff","patch_url":"https://github.com/ohmu/pghoard/pull/13.patch","issue_url":"https://api.github.com/repos/ohmu/pghoard/issues/13","number":13,"state":"open","locked":false,"title":"Local basebackup via compress flow","user":{"login":"hnousiainen","id":2621453,"avatar_url":"https://avatars.githubusercontent.com/u/2621453?v=3","gravatar_id":"","url":"https://api.github.com/users/hnousiainen","html_url":"https://github.com/hnousiainen","followers_url":"https://api.github.com/users/hnousiainen/followers","following_url":"https://api.github.com/users/hnousiainen/following{/other_user}","gists_url":"https://api.github.com/users/hnousiainen/gists{/gist_id}","starred_url":"https://api.github.com/users/hnousiainen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hnousiainen/subscriptions","organizations_url":"https://api.github.com/users/hnousiainen/orgs","repos_url":"https://api.github.com/users/hnousiainen/repos","events_url":"https://api.github.com/users/hnousiainen/events{/privacy}","received_events_url":"https://api.github.com/users/hnousiainen/received_events","type":"User","site_admin":false},"body":"Ensure we always have up-to-date local metadata including the compression information. We'll need this later for encrypted data. A round of random fixes preparing for the eventual stacked transformations in addition to just compression.","created_at":"2015-08-25T07:52:50Z","updated_at":"2015-08-25T10:11:07Z","closed_at":null,"merged_at":null,"merge_commit_sha":"30a7c72c35e6d8e4e6cdf8b2350590558cf7807e","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/ohmu/pghoard/pulls/13/commits","review_comments_url":"https://api.github.com/repos/ohmu/pghoard/pulls/13/comments","review_comment_url":"https://api.github.com/repos/ohmu/pghoard/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ohmu/pghoard/issues/13/comments","statuses_url":"https://api.github.com/repos/ohmu/pghoard/statuses/eb88fbcebde8605246e0b40b483f6b2c1beffb79","head":{"label":"hnousiainen:local-basebackup-via-compress-flow","ref":"local-basebackup-via-compress-flow","sha":"eb88fbcebde8605246e0b40b483f6b2c1beffb79","user":{"login":"hnousiainen","id":2621453,"avatar_url":"https://avatars.githubusercontent.com/u/2621453?v=3","gravatar_id":"","url":"https://api.github.com/users/hnousiainen","html_url":"https://github.com/hnousiainen","followers_url":"https://api.github.com/users/hnousiainen/followers","following_url":"https://api.github.com/users/hnousiainen/following{/other_user}","gists_url":"https://api.github.com/users/hnousiainen/gists{/gist_id}","starred_url":"https://api.github.com/users/hnousiainen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hnousiainen/subscriptions","organizations_url":"https://api.github.com/users/hnousiainen/orgs","repos_url":"https://api.github.com/users/hnousiainen/repos","events_url":"https://api.github.com/users/hnousiainen/events{/privacy}","received_events_url":"https://api.github.com/users/hnousiainen/received_events","type":"User","site_admin":false},"repo":{"id":40564371,"name":"pghoard","full_name":"hnousiainen/pghoard","owner":{"login":"hnousiainen","id":2621453,"avatar_url":"https://avatars.githubusercontent.com/u/2621453?v=3","gravatar_id":"","url":"https://api.github.com/users/hnousiainen","html_url":"https://github.com/hnousiainen","followers_url":"https://api.github.com/users/hnousiainen/followers","following_url":"https://api.github.com/users/hnousiainen/following{/other_user}","gists_url":"https://api.github.com/users/hnousiainen/gists{/gist_id}","starred_url":"https://api.github.com/users/hnousiainen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hnousiainen/subscriptions","organizations_url":"https://api.github.com/users/hnousiainen/orgs","repos_url":"https://api.github.com/users/hnousiainen/repos","events_url":"https://api.github.com/users/hnousiainen/events{/privacy}","received_events_url":"https://api.github.com/users/hnousiainen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/hnousiainen/pghoard","description":"PostgreSQL backup and restore service","fork":true,"url":"https://api.github.com/repos/hnousiainen/pghoard","forks_url":"https://api.github.com/repos/hnousiainen/pghoard/forks","keys_url":"https://api.github.com/repos/hnousiainen/pghoard/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hnousiainen/pghoard/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hnousiainen/pghoard/teams","hooks_url":"https://api.github.com/repos/hnousiainen/pghoard/hooks","issue_events_url":"https://api.github.com/repos/hnousiainen/pghoard/issues/events{/number}","events_url":"https://api.github.com/repos/hnousiainen/pghoard/events","assignees_url":"https://api.github.com/repos/hnousiainen/pghoard/assignees{/user}","branches_url":"https://api.github.com/repos/hnousiainen/pghoard/branches{/branch}","tags_url":"https://api.github.com/repos/hnousiainen/pghoard/tags","blobs_url":"https://api.github.com/repos/hnousiainen/pghoard/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hnousiainen/pghoard/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hnousiainen/pghoard/git/refs{/sha}","trees_url":"https://api.github.com/repos/hnousiainen/pghoard/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hnousiainen/pghoard/statuses/{sha}","languages_url":"https://api.github.com/repos/hnousiainen/pghoard/languages","stargazers_url":"https://api.github.com/repos/hnousiainen/pghoard/stargazers","contributors_url":"https://api.github.com/repos/hnousiainen/pghoard/contributors","subscribers_url":"https://api.github.com/repos/hnousiainen/pghoard/subscribers","subscription_url":"https://api.github.com/repos/hnousiainen/pghoard/subscription","commits_url":"https://api.github.com/repos/hnousiainen/pghoard/commits{/sha}","git_commits_url":"https://api.github.com/repos/hnousiainen/pghoard/git/commits{/sha}","comments_url":"https://api.github.com/repos/hnousiainen/pghoard/comments{/number}","issue_comment_url":"https://api.github.com/repos/hnousiainen/pghoard/issues/comments{/number}","contents_url":"https://api.github.com/repos/hnousiainen/pghoard/contents/{+path}","compare_url":"https://api.github.com/repos/hnousiainen/pghoard/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hnousiainen/pghoard/merges","archive_url":"https://api.github.com/repos/hnousiainen/pghoard/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hnousiainen/pghoard/downloads","issues_url":"https://api.github.com/repos/hnousiainen/pghoard/issues{/number}","pulls_url":"https://api.github.com/repos/hnousiainen/pghoard/pulls{/number}","milestones_url":"https://api.github.com/repos/hnousiainen/pghoard/milestones{/number}","notifications_url":"https://api.github.com/repos/hnousiainen/pghoard/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hnousiainen/pghoard/labels{/name}","releases_url":"https://api.github.com/repos/hnousiainen/pghoard/releases{/id}","created_at":"2015-08-11T20:45:45Z","updated_at":"2015-08-11T20:45:46Z","pushed_at":"2015-08-25T07:49:44Z","git_url":"git://github.com/hnousiainen/pghoard.git","ssh_url":"git@github.com:hnousiainen/pghoard.git","clone_url":"https://github.com/hnousiainen/pghoard.git","svn_url":"https://github.com/hnousiainen/pghoard","homepage":null,"size":269,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"ohmu:master","ref":"master","sha":"b96dd700dabe253ebd816e897fa4d3a902ffea34","user":{"login":"ohmu","id":986303,"avatar_url":"https://avatars.githubusercontent.com/u/986303?v=3","gravatar_id":"","url":"https://api.github.com/users/ohmu","html_url":"https://github.com/ohmu","followers_url":"https://api.github.com/users/ohmu/followers","following_url":"https://api.github.com/users/ohmu/following{/other_user}","gists_url":"https://api.github.com/users/ohmu/gists{/gist_id}","starred_url":"https://api.github.com/users/ohmu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ohmu/subscriptions","organizations_url":"https://api.github.com/users/ohmu/orgs","repos_url":"https://api.github.com/users/ohmu/repos","events_url":"https://api.github.com/users/ohmu/events{/privacy}","received_events_url":"https://api.github.com/users/ohmu/received_events","type":"Organization","site_admin":false},"repo":{"id":37595806,"name":"pghoard","full_name":"ohmu/pghoard","owner":{"login":"ohmu","id":986303,"avatar_url":"https://avatars.githubusercontent.com/u/986303?v=3","gravatar_id":"","url":"https://api.github.com/users/ohmu","html_url":"https://github.com/ohmu","followers_url":"https://api.github.com/users/ohmu/followers","following_url":"https://api.github.com/users/ohmu/following{/other_user}","gists_url":"https://api.github.com/users/ohmu/gists{/gist_id}","starred_url":"https://api.github.com/users/ohmu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ohmu/subscriptions","organizations_url":"https://api.github.com/users/ohmu/orgs","repos_url":"https://api.github.com/users/ohmu/repos","events_url":"https://api.github.com/users/ohmu/events{/privacy}","received_events_url":"https://api.github.com/users/ohmu/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ohmu/pghoard","description":"PostgreSQL backup and restore service","fork":false,"url":"https://api.github.com/repos/ohmu/pghoard","forks_url":"https://api.github.com/repos/ohmu/pghoard/forks","keys_url":"https://api.github.com/repos/ohmu/pghoard/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ohmu/pghoard/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ohmu/pghoard/teams","hooks_url":"https://api.github.com/repos/ohmu/pghoard/hooks","issue_events_url":"https://api.github.com/repos/ohmu/pghoard/issues/events{/number}","events_url":"https://api.github.com/repos/ohmu/pghoard/events","assignees_url":"https://api.github.com/repos/ohmu/pghoard/assignees{/user}","branches_url":"https://api.github.com/repos/ohmu/pghoard/branches{/branch}","tags_url":"https://api.github.com/repos/ohmu/pghoard/tags","blobs_url":"https://api.github.com/repos/ohmu/pghoard/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ohmu/pghoard/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ohmu/pghoard/git/refs{/sha}","trees_url":"https://api.github.com/repos/ohmu/pghoard/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ohmu/pghoard/statuses/{sha}","languages_url":"https://api.github.com/repos/ohmu/pghoard/languages","stargazers_url":"https://api.github.com/repos/ohmu/pghoard/stargazers","contributors_url":"https://api.github.com/repos/ohmu/pghoard/contributors","subscribers_url":"https://api.github.com/repos/ohmu/pghoard/subscribers","subscription_url":"https://api.github.com/repos/ohmu/pghoard/subscription","commits_url":"https://api.github.com/repos/ohmu/pghoard/commits{/sha}","git_commits_url":"https://api.github.com/repos/ohmu/pghoard/git/commits{/sha}","comments_url":"https://api.github.com/repos/ohmu/pghoard/comments{/number}","issue_comment_url":"https://api.github.com/repos/ohmu/pghoard/issues/comments{/number}","contents_url":"https://api.github.com/repos/ohmu/pghoard/contents/{+path}","compare_url":"https://api.github.com/repos/ohmu/pghoard/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ohmu/pghoard/merges","archive_url":"https://api.github.com/repos/ohmu/pghoard/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ohmu/pghoard/downloads","issues_url":"https://api.github.com/repos/ohmu/pghoard/issues{/number}","pulls_url":"https://api.github.com/repos/ohmu/pghoard/pulls{/number}","milestones_url":"https://api.github.com/repos/ohmu/pghoard/milestones{/number}","notifications_url":"https://api.github.com/repos/ohmu/pghoard/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ohmu/pghoard/labels{/name}","releases_url":"https://api.github.com/repos/ohmu/pghoard/releases{/id}","created_at":"2015-06-17T13:13:40Z","updated_at":"2015-06-17T14:06:13Z","pushed_at":"2015-08-25T07:52:50Z","git_url":"git://github.com/ohmu/pghoard.git","ssh_url":"git@github.com:ohmu/pghoard.git","clone_url":"https://github.com/ohmu/pghoard.git","svn_url":"https://github.com/ohmu/pghoard","homepage":null,"size":482,"stargazers_count":4,"watchers_count":4,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4,"mirror_url":null,"open_issues_count":1,"forks":4,"open_issues":1,"watchers":4,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ohmu/pghoard/pulls/13"},"html":{"href":"https://github.com/ohmu/pghoard/pull/13"},"issue":{"href":"https://api.github.com/repos/ohmu/pghoard/issues/13"},"comments":{"href":"https://api.github.com/repos/ohmu/pghoard/issues/13/comments"},"review_comments":{"href":"https://api.github.com/repos/ohmu/pghoard/pulls/13/comments"},"review_comment":{"href":"https://api.github.com/repos/ohmu/pghoard/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ohmu/pghoard/pulls/13/commits"},"statuses":{"href":"https://api.github.com/repos/ohmu/pghoard/statuses/eb88fbcebde8605246e0b40b483f6b2c1beffb79"}}}}