{"action":"created","comment":{"url":"https://api.github.com/repos/influxdb/influxdb/pulls/comments/48573528","id":48573528,"diff_hunk":"@@ -95,51 +98,152 @@ func (s *Service) serve() {\n \n // handleConn processes conn. This is run in a separate goroutine.\n func (s *Service) handleConn(conn net.Conn) error {\n-\t// Read manifest from connection.\n-\tm, err := s.readManifest(conn)\n+\tr, err := s.readRequest(conn)\n \tif err != nil {\n-\t\treturn fmt.Errorf(\"read manifest: %s\", err)\n+\t\treturn fmt.Errorf(\"read request: %s\", err)\n \t}\n \n-\t// Write snapshot to connection.\n-\tif err := s.writeSnapshot(conn, m); err != nil {\n-\t\treturn fmt.Errorf(\"write snapshot: %s\", err)\n+\tswitch r.Type {\n+\tcase RequestShardBackup:\n+\t\tif err := s.TSDBStore.BackupShard(r.ShardID, r.Since, conn); err != nil {\n+\t\t\treturn err\n+\t\t}\n+\tcase RequestMetastoreBackup:\n+\t\t// Retrieve and serialize the current meta data.\n+\t\tbuf, err := s.MetaStore.MarshalBinary()\n+\t\tif err != nil {\n+\t\t\treturn fmt.Errorf(\"marshal meta: %s\", err)\n+\t\t}\n+\t\tif _, err := conn.Write(buf); err != nil {\n+\t\t\treturn err\n+\t\t}\n+\tcase RequestDatabaseInfo:\n+\t\treturn s.writeDatabaseInfo(conn, r.Database)\n+\tcase RequestRetentionPolicyInfo:\n+\t\treturn s.writeRetentionPolicyInfo(conn, r.Database, r.RetentionPolicy)\n+\tdefault:\n+\t\treturn fmt.Errorf(\"request type unknown: %v\", r.Type)\n \t}\n \n \treturn nil\n }\n \n-// readManifest reads the manifest size and contents from conn.\n-// Unmarshals the bytes and returns a manifest object.\n-func (s *Service) readManifest(conn net.Conn) (snapshot.Manifest, error) {\n-\tvar m snapshot.Manifest\n-\tif err := json.NewDecoder(conn).Decode(&m); err != nil {\n-\t\treturn m, err\n+// writeDatabaseInfo will write the relative paths of all shards in the database on\n+// this server into the connection\n+func (s *Service) writeDatabaseInfo(conn net.Conn, database string) error {\n+\tres := Response{}\n+\tdb, err := s.MetaStore.Database(database)\n+\tif err != nil {\n+\t\treturn err\n+\t}\n+\tif db == nil {\n+\t\treturn influxdb.ErrDatabaseNotFound(database)\n \t}\n-\treturn m, nil\n+\n+\tfor _, rp := range db.RetentionPolicies {\n+\t\tfor _, sg := range rp.ShardGroups {\n+\t\t\tfor _, sh := range sg.Shards {\n+\t\t\t\t// ignore if the shard isn't on the server","path":"services/snapshotter/service.go","position":82,"original_position":82,"commit_id":"76996088699b1f7eec284c8605a7fa35e0936f0a","original_commit_id":"76996088699b1f7eec284c8605a7fa35e0936f0a","user":{"login":"otoolep","id":536312,"avatar_url":"https://avatars.githubusercontent.com/u/536312?v=3","gravatar_id":"","url":"https://api.github.com/users/otoolep","html_url":"https://github.com/otoolep","followers_url":"https://api.github.com/users/otoolep/followers","following_url":"https://api.github.com/users/otoolep/following{/other_user}","gists_url":"https://api.github.com/users/otoolep/gists{/gist_id}","starred_url":"https://api.github.com/users/otoolep/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/otoolep/subscriptions","organizations_url":"https://api.github.com/users/otoolep/orgs","repos_url":"https://api.github.com/users/otoolep/repos","events_url":"https://api.github.com/users/otoolep/events{/privacy}","received_events_url":"https://api.github.com/users/otoolep/received_events","type":"User","site_admin":false},"body":"OK, I see -- it's just skipped.","created_at":"2015-12-29T22:05:17Z","updated_at":"2015-12-29T22:05:17Z","html_url":"https://github.com/influxdb/influxdb/pull/5224#discussion_r48573528","pull_request_url":"https://api.github.com/repos/influxdb/influxdb/pulls/5224","_links":{"self":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/comments/48573528"},"html":{"href":"https://github.com/influxdb/influxdb/pull/5224#discussion_r48573528"},"pull_request":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/5224"}}},"pull_request":{"url":"https://api.github.com/repos/influxdb/influxdb/pulls/5224","id":54587258,"html_url":"https://github.com/influxdb/influxdb/pull/5224","diff_url":"https://github.com/influxdb/influxdb/pull/5224.diff","patch_url":"https://github.com/influxdb/influxdb/pull/5224.patch","issue_url":"https://api.github.com/repos/influxdb/influxdb/issues/5224","number":5224,"state":"open","locked":false,"title":"Implement backup/restore for TSM.","user":{"login":"pauldix","id":4331,"avatar_url":"https://avatars.githubusercontent.com/u/4331?v=3","gravatar_id":"","url":"https://api.github.com/users/pauldix","html_url":"https://github.com/pauldix","followers_url":"https://api.github.com/users/pauldix/followers","following_url":"https://api.github.com/users/pauldix/following{/other_user}","gists_url":"https://api.github.com/users/pauldix/gists{/gist_id}","starred_url":"https://api.github.com/users/pauldix/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pauldix/subscriptions","organizations_url":"https://api.github.com/users/pauldix/orgs","repos_url":"https://api.github.com/users/pauldix/repos","events_url":"https://api.github.com/users/pauldix/events{/privacy}","received_events_url":"https://api.github.com/users/pauldix/received_events","type":"User","site_admin":false},"body":"This present is just in time to get under the tree for Christmas!\r\n\r\nThis changes backup and restore to work for TSM. It breaks it for b1 and bz1, but since those are getting removed it's ok.\r\n\r\nThe backup runs against any host that is specified and can backup either the metasstore, a database, specific retention policy, or a specific shard. It can also take incremental backups with the `since` flag, which will only backup TSM files that have been created since that timestamp.\r\n\r\nThe backup is safe to run online. However, for shards that are still hot for writes, they won't be able to create new TSM files while the backup for that single shard runs. If the backup isn't too large and the write throughput isn't too high this shouldn't be a problem since the writes will just go into the WAL cache.\r\n\r\nFor details on usage use `influxd backup`","created_at":"2015-12-25T13:25:12Z","updated_at":"2015-12-29T22:05:17Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b95c9f75c18a3c47363a1126e437047a19e8e719","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/influxdb/influxdb/pulls/5224/commits","review_comments_url":"https://api.github.com/repos/influxdb/influxdb/pulls/5224/comments","review_comment_url":"https://api.github.com/repos/influxdb/influxdb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/influxdb/influxdb/issues/5224/comments","statuses_url":"https://api.github.com/repos/influxdb/influxdb/statuses/76996088699b1f7eec284c8605a7fa35e0936f0a","head":{"label":"influxdb:pd-backup-restore","ref":"pd-backup-restore","sha":"76996088699b1f7eec284c8605a7fa35e0936f0a","user":{"login":"influxdb","id":5713248,"avatar_url":"https://avatars.githubusercontent.com/u/5713248?v=3","gravatar_id":"","url":"https://api.github.com/users/influxdb","html_url":"https://github.com/influxdb","followers_url":"https://api.github.com/users/influxdb/followers","following_url":"https://api.github.com/users/influxdb/following{/other_user}","gists_url":"https://api.github.com/users/influxdb/gists{/gist_id}","starred_url":"https://api.github.com/users/influxdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/influxdb/subscriptions","organizations_url":"https://api.github.com/users/influxdb/orgs","repos_url":"https://api.github.com/users/influxdb/repos","events_url":"https://api.github.com/users/influxdb/events{/privacy}","received_events_url":"https://api.github.com/users/influxdb/received_events","type":"Organization","site_admin":false},"repo":{"id":13124802,"name":"influxdb","full_name":"influxdb/influxdb","owner":{"login":"influxdb","id":5713248,"avatar_url":"https://avatars.githubusercontent.com/u/5713248?v=3","gravatar_id":"","url":"https://api.github.com/users/influxdb","html_url":"https://github.com/influxdb","followers_url":"https://api.github.com/users/influxdb/followers","following_url":"https://api.github.com/users/influxdb/following{/other_user}","gists_url":"https://api.github.com/users/influxdb/gists{/gist_id}","starred_url":"https://api.github.com/users/influxdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/influxdb/subscriptions","organizations_url":"https://api.github.com/users/influxdb/orgs","repos_url":"https://api.github.com/users/influxdb/repos","events_url":"https://api.github.com/users/influxdb/events{/privacy}","received_events_url":"https://api.github.com/users/influxdb/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/influxdb/influxdb","description":"Scalable datastore for metrics, events, and real-time analytics","fork":false,"url":"https://api.github.com/repos/influxdb/influxdb","forks_url":"https://api.github.com/repos/influxdb/influxdb/forks","keys_url":"https://api.github.com/repos/influxdb/influxdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/influxdb/influxdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/influxdb/influxdb/teams","hooks_url":"https://api.github.com/repos/influxdb/influxdb/hooks","issue_events_url":"https://api.github.com/repos/influxdb/influxdb/issues/events{/number}","events_url":"https://api.github.com/repos/influxdb/influxdb/events","assignees_url":"https://api.github.com/repos/influxdb/influxdb/assignees{/user}","branches_url":"https://api.github.com/repos/influxdb/influxdb/branches{/branch}","tags_url":"https://api.github.com/repos/influxdb/influxdb/tags","blobs_url":"https://api.github.com/repos/influxdb/influxdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/influxdb/influxdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/influxdb/influxdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/influxdb/influxdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/influxdb/influxdb/statuses/{sha}","languages_url":"https://api.github.com/repos/influxdb/influxdb/languages","stargazers_url":"https://api.github.com/repos/influxdb/influxdb/stargazers","contributors_url":"https://api.github.com/repos/influxdb/influxdb/contributors","subscribers_url":"https://api.github.com/repos/influxdb/influxdb/subscribers","subscription_url":"https://api.github.com/repos/influxdb/influxdb/subscription","commits_url":"https://api.github.com/repos/influxdb/influxdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/influxdb/influxdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/influxdb/influxdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/influxdb/influxdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/influxdb/influxdb/contents/{+path}","compare_url":"https://api.github.com/repos/influxdb/influxdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/influxdb/influxdb/merges","archive_url":"https://api.github.com/repos/influxdb/influxdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/influxdb/influxdb/downloads","issues_url":"https://api.github.com/repos/influxdb/influxdb/issues{/number}","pulls_url":"https://api.github.com/repos/influxdb/influxdb/pulls{/number}","milestones_url":"https://api.github.com/repos/influxdb/influxdb/milestones{/number}","notifications_url":"https://api.github.com/repos/influxdb/influxdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/influxdb/influxdb/labels{/name}","releases_url":"https://api.github.com/repos/influxdb/influxdb/releases{/id}","created_at":"2013-09-26T14:31:10Z","updated_at":"2015-12-29T18:49:47Z","pushed_at":"2015-12-29T21:12:18Z","git_url":"git://github.com/influxdb/influxdb.git","ssh_url":"git@github.com:influxdb/influxdb.git","clone_url":"https://github.com/influxdb/influxdb.git","svn_url":"https://github.com/influxdb/influxdb","homepage":"http://www.influxdata.com","size":39694,"stargazers_count":7051,"watchers_count":7051,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":875,"mirror_url":null,"open_issues_count":541,"forks":875,"open_issues":541,"watchers":7051,"default_branch":"master"}},"base":{"label":"influxdb:master","ref":"master","sha":"cbbb01ce8e7ce70b8432b91e4bf32b6e0f6df883","user":{"login":"influxdb","id":5713248,"avatar_url":"https://avatars.githubusercontent.com/u/5713248?v=3","gravatar_id":"","url":"https://api.github.com/users/influxdb","html_url":"https://github.com/influxdb","followers_url":"https://api.github.com/users/influxdb/followers","following_url":"https://api.github.com/users/influxdb/following{/other_user}","gists_url":"https://api.github.com/users/influxdb/gists{/gist_id}","starred_url":"https://api.github.com/users/influxdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/influxdb/subscriptions","organizations_url":"https://api.github.com/users/influxdb/orgs","repos_url":"https://api.github.com/users/influxdb/repos","events_url":"https://api.github.com/users/influxdb/events{/privacy}","received_events_url":"https://api.github.com/users/influxdb/received_events","type":"Organization","site_admin":false},"repo":{"id":13124802,"name":"influxdb","full_name":"influxdb/influxdb","owner":{"login":"influxdb","id":5713248,"avatar_url":"https://avatars.githubusercontent.com/u/5713248?v=3","gravatar_id":"","url":"https://api.github.com/users/influxdb","html_url":"https://github.com/influxdb","followers_url":"https://api.github.com/users/influxdb/followers","following_url":"https://api.github.com/users/influxdb/following{/other_user}","gists_url":"https://api.github.com/users/influxdb/gists{/gist_id}","starred_url":"https://api.github.com/users/influxdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/influxdb/subscriptions","organizations_url":"https://api.github.com/users/influxdb/orgs","repos_url":"https://api.github.com/users/influxdb/repos","events_url":"https://api.github.com/users/influxdb/events{/privacy}","received_events_url":"https://api.github.com/users/influxdb/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/influxdb/influxdb","description":"Scalable datastore for metrics, events, and real-time analytics","fork":false,"url":"https://api.github.com/repos/influxdb/influxdb","forks_url":"https://api.github.com/repos/influxdb/influxdb/forks","keys_url":"https://api.github.com/repos/influxdb/influxdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/influxdb/influxdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/influxdb/influxdb/teams","hooks_url":"https://api.github.com/repos/influxdb/influxdb/hooks","issue_events_url":"https://api.github.com/repos/influxdb/influxdb/issues/events{/number}","events_url":"https://api.github.com/repos/influxdb/influxdb/events","assignees_url":"https://api.github.com/repos/influxdb/influxdb/assignees{/user}","branches_url":"https://api.github.com/repos/influxdb/influxdb/branches{/branch}","tags_url":"https://api.github.com/repos/influxdb/influxdb/tags","blobs_url":"https://api.github.com/repos/influxdb/influxdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/influxdb/influxdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/influxdb/influxdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/influxdb/influxdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/influxdb/influxdb/statuses/{sha}","languages_url":"https://api.github.com/repos/influxdb/influxdb/languages","stargazers_url":"https://api.github.com/repos/influxdb/influxdb/stargazers","contributors_url":"https://api.github.com/repos/influxdb/influxdb/contributors","subscribers_url":"https://api.github.com/repos/influxdb/influxdb/subscribers","subscription_url":"https://api.github.com/repos/influxdb/influxdb/subscription","commits_url":"https://api.github.com/repos/influxdb/influxdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/influxdb/influxdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/influxdb/influxdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/influxdb/influxdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/influxdb/influxdb/contents/{+path}","compare_url":"https://api.github.com/repos/influxdb/influxdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/influxdb/influxdb/merges","archive_url":"https://api.github.com/repos/influxdb/influxdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/influxdb/influxdb/downloads","issues_url":"https://api.github.com/repos/influxdb/influxdb/issues{/number}","pulls_url":"https://api.github.com/repos/influxdb/influxdb/pulls{/number}","milestones_url":"https://api.github.com/repos/influxdb/influxdb/milestones{/number}","notifications_url":"https://api.github.com/repos/influxdb/influxdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/influxdb/influxdb/labels{/name}","releases_url":"https://api.github.com/repos/influxdb/influxdb/releases{/id}","created_at":"2013-09-26T14:31:10Z","updated_at":"2015-12-29T18:49:47Z","pushed_at":"2015-12-29T21:12:18Z","git_url":"git://github.com/influxdb/influxdb.git","ssh_url":"git@github.com:influxdb/influxdb.git","clone_url":"https://github.com/influxdb/influxdb.git","svn_url":"https://github.com/influxdb/influxdb","homepage":"http://www.influxdata.com","size":39694,"stargazers_count":7051,"watchers_count":7051,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":875,"mirror_url":null,"open_issues_count":541,"forks":875,"open_issues":541,"watchers":7051,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/5224"},"html":{"href":"https://github.com/influxdb/influxdb/pull/5224"},"issue":{"href":"https://api.github.com/repos/influxdb/influxdb/issues/5224"},"comments":{"href":"https://api.github.com/repos/influxdb/influxdb/issues/5224/comments"},"review_comments":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/5224/comments"},"review_comment":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/5224/commits"},"statuses":{"href":"https://api.github.com/repos/influxdb/influxdb/statuses/76996088699b1f7eec284c8605a7fa35e0936f0a"}}}}