{"action":"created","comment":{"url":"https://api.github.com/repos/restic/restic/pulls/comments/29386447","id":29386447,"diff_hunk":"@@ -0,0 +1,258 @@\n+package server\n+\n+import (\n+\t\"encoding/json\"\n+\t\"errors\"\n+\t\"fmt\"\n+\t\"io\"\n+\t\"sync\"\n+\n+\t\"github.com/restic/restic/backend\"\n+\t\"github.com/restic/restic/debug\"\n+\t\"github.com/restic/restic/pack\"\n+)\n+\n+// Index holds a lookup table for id -> pack.\n+type Index struct {\n+\tm    sync.Mutex\n+\tpack map[string]indexEntry\n+}\n+\n+type indexEntry struct {\n+\ttpe    pack.BlobType\n+\tpackID backend.ID\n+\toffset uint\n+\tlength uint\n+\told    bool\n+}\n+\n+// NewIndex returns a new index.\n+func NewIndex() *Index {\n+\treturn &Index{\n+\t\tpack: make(map[string]indexEntry),\n+\t}\n+}\n+\n+func (idx *Index) store(t pack.BlobType, id, pack backend.ID, offset, length uint, old bool) {\n+\tidx.pack[id.String()] = indexEntry{\n+\t\ttpe:    t,\n+\t\tpackID: pack,\n+\t\toffset: offset,\n+\t\tlength: length,\n+\t\told:    old,\n+\t}\n+}\n+\n+// Store remembers the id and pack in the index.\n+func (idx *Index) Store(t pack.BlobType, id, pack backend.ID, offset, length uint) {\n+\tidx.m.Lock()\n+\tdefer idx.m.Unlock()\n+\n+\tdebug.Log(\"Index.Store\", \"pack %v contains id %v (%v), offset %v, length %v\",\n+\t\tpack.Str(), id.Str(), t, offset, length)\n+\n+\tidx.store(t, id, pack, offset, length, false)\n+}\n+\n+// Remove removes the pack ID from the index.\n+func (idx *Index) Remove(packID backend.ID) {\n+\tidx.m.Lock()\n+\tdefer idx.m.Unlock()\n+\n+\tdebug.Log(\"Index.Remove\", \"id %v removed\", packID.Str())\n+\n+\ts := packID.String()\n+\tif _, ok := idx.pack[s]; ok {\n+\t\tdelete(idx.pack, s)\n+\t}\n+}\n+\n+// Lookup returns the pack for the id.\n+func (idx *Index) Lookup(id backend.ID) (packID backend.ID, tpe pack.BlobType, offset, length uint, err error) {\n+\tidx.m.Lock()\n+\tdefer idx.m.Unlock()\n+\n+\tif p, ok := idx.pack[id.String()]; ok {\n+\t\tdebug.Log(\"Index.Lookup\", \"id %v found in pack %v at %d, length %d\",\n+\t\t\tid.Str(), p.packID.Str(), p.offset, p.length)\n+\t\treturn p.packID, p.tpe, p.offset, p.length, nil\n+\t}\n+\n+\tdebug.Log(\"Index.Lookup\", \"id %v not found\", id.Str())\n+\treturn nil, pack.Data, 0, 0, errors.New(\"id not found\")\n+}\n+\n+// Has returns true iff the id is listed in the index.\n+func (idx *Index) Has(id backend.ID) bool {\n+\t_, _, _, _, err := idx.Lookup(id)\n+\tif err == nil {\n+\t\treturn true\n+\t}\n+\n+\treturn false\n+}\n+\n+// Merge loads all items from other into idx.\n+func (idx *Index) Merge(other *Index) {\n+\tdebug.Log(\"Index.Merge\", \"Merge index with %p\", other)\n+\tidx.m.Lock()\n+\tdefer idx.m.Unlock()\n+\n+\tfor k, v := range other.pack {\n+\t\tif _, ok := idx.pack[k]; ok {\n+\t\t\tdebug.Log(\"Index.Merge\", \"index already has key %v, updating\", k[:8])\n+\t\t}\n+\n+\t\tidx.pack[k] = v\n+\t}\n+\tdebug.Log(\"Index.Merge\", \"done merging index\")\n+}\n+\n+// Each returns a channel that yields all blobs known to the index. If done is\n+// closed, the background goroutine terminates. This blocks any modification of\n+// the index.\n+func (idx *Index) Each(done chan struct{}) <-chan pack.Blob {\n+\tidx.m.Lock()\n+\n+\tch := make(chan pack.Blob)\n+\n+\tgo func() {\n+\t\tdefer idx.m.Unlock()\n+\t\tdefer func() {\n+\t\t\tclose(ch)\n+\t\t}()\n+\n+\t\tfor ids, blob := range idx.pack {\n+\t\t\tid, err := backend.ParseID(ids)\n+\t\t\tif err != nil {\n+\t\t\t\t// ignore invalid IDs\n+\t\t\t\tcontinue\n+\t\t\t}\n+\n+\t\t\tselect {\n+\t\t\tcase <-done:\n+\t\t\t\treturn\n+\t\t\tcase ch <- pack.Blob{\n+\t\t\t\tID:     id,\n+\t\t\t\tOffset: blob.offset,\n+\t\t\t\tType:   blob.tpe,\n+\t\t\t\tLength: uint32(blob.length),\n+\t\t\t}:\n+\t\t\t}\n+\t\t}\n+\t}()\n+\n+\treturn ch\n+}\n+\n+// Count returns the number of blobs of type t in the index.\n+func (idx *Index) Count(t pack.BlobType) (n uint) {\n+\tdebug.Log(\"Index.Count\", \"counting blobs of type %v\", t)\n+\tidx.m.Lock()\n+\tdefer idx.m.Unlock()\n+\n+\tfor id, blob := range idx.pack {\n+\t\tif blob.tpe == t {","path":"server/index.go","position":155,"original_position":155,"commit_id":"ec108fb708285944f5c1043f5a55a1191858cd34","original_commit_id":"ec108fb708285944f5c1043f5a55a1191858cd34","user":{"login":"fd0","id":123276,"avatar_url":"https://avatars.githubusercontent.com/u/123276?v=3","gravatar_id":"","url":"https://api.github.com/users/fd0","html_url":"https://github.com/fd0","followers_url":"https://api.github.com/users/fd0/followers","following_url":"https://api.github.com/users/fd0/following{/other_user}","gists_url":"https://api.github.com/users/fd0/gists{/gist_id}","starred_url":"https://api.github.com/users/fd0/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fd0/subscriptions","organizations_url":"https://api.github.com/users/fd0/orgs","repos_url":"https://api.github.com/users/fd0/repos","events_url":"https://api.github.com/users/fd0/events{/privacy}","received_events_url":"https://api.github.com/users/fd0/received_events","type":"User","site_admin":false},"body":"Good idea, but unnecessary: `Count()` is only used in the tests.","created_at":"2015-04-29T22:07:53Z","updated_at":"2015-04-29T22:07:53Z","html_url":"https://github.com/restic/restic/pull/151#discussion_r29386447","pull_request_url":"https://api.github.com/repos/restic/restic/pulls/151","_links":{"self":{"href":"https://api.github.com/repos/restic/restic/pulls/comments/29386447"},"html":{"href":"https://github.com/restic/restic/pull/151#discussion_r29386447"},"pull_request":{"href":"https://api.github.com/repos/restic/restic/pulls/151"}}},"pull_request":{"url":"https://api.github.com/repos/restic/restic/pulls/151","id":34403645,"html_url":"https://github.com/restic/restic/pull/151","diff_url":"https://github.com/restic/restic/pull/151.diff","patch_url":"https://github.com/restic/restic/pull/151.patch","issue_url":"https://api.github.com/repos/restic/restic/issues/151","number":151,"state":"open","locked":false,"title":"Pack blobs","user":{"login":"fd0","id":123276,"avatar_url":"https://avatars.githubusercontent.com/u/123276?v=3","gravatar_id":"","url":"https://api.github.com/users/fd0","html_url":"https://github.com/fd0","followers_url":"https://api.github.com/users/fd0/followers","following_url":"https://api.github.com/users/fd0/following{/other_user}","gists_url":"https://api.github.com/users/fd0/gists{/gist_id}","starred_url":"https://api.github.com/users/fd0/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fd0/subscriptions","organizations_url":"https://api.github.com/users/fd0/orgs","repos_url":"https://api.github.com/users/fd0/repos","events_url":"https://api.github.com/users/fd0/events{/privacy}","received_events_url":"https://api.github.com/users/fd0/received_events","type":"User","site_admin":false},"body":"This implements bundling of smaller chunks into bigger files (#56). Unfortunately, this is a big change that touches a lot of code in different places, and breaks the repository format and fsck. Let me know what you think!","created_at":"2015-04-29T20:40:24Z","updated_at":"2015-04-29T22:07:53Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7e92e49721a17aa4f6fcd7d1120b777bfb8cb1cd","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/restic/restic/pulls/151/commits","review_comments_url":"https://api.github.com/repos/restic/restic/pulls/151/comments","review_comment_url":"https://api.github.com/repos/restic/restic/pulls/comments{/number}","comments_url":"https://api.github.com/repos/restic/restic/issues/151/comments","statuses_url":"https://api.github.com/repos/restic/restic/statuses/ec108fb708285944f5c1043f5a55a1191858cd34","head":{"label":"restic:pack-blobs","ref":"pack-blobs","sha":"ec108fb708285944f5c1043f5a55a1191858cd34","user":{"login":"restic","id":10073512,"avatar_url":"https://avatars.githubusercontent.com/u/10073512?v=3","gravatar_id":"","url":"https://api.github.com/users/restic","html_url":"https://github.com/restic","followers_url":"https://api.github.com/users/restic/followers","following_url":"https://api.github.com/users/restic/following{/other_user}","gists_url":"https://api.github.com/users/restic/gists{/gist_id}","starred_url":"https://api.github.com/users/restic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/restic/subscriptions","organizations_url":"https://api.github.com/users/restic/orgs","repos_url":"https://api.github.com/users/restic/repos","events_url":"https://api.github.com/users/restic/events{/privacy}","received_events_url":"https://api.github.com/users/restic/received_events","type":"Organization","site_admin":false},"repo":{"id":19205896,"name":"restic","full_name":"restic/restic","owner":{"login":"restic","id":10073512,"avatar_url":"https://avatars.githubusercontent.com/u/10073512?v=3","gravatar_id":"","url":"https://api.github.com/users/restic","html_url":"https://github.com/restic","followers_url":"https://api.github.com/users/restic/followers","following_url":"https://api.github.com/users/restic/following{/other_user}","gists_url":"https://api.github.com/users/restic/gists{/gist_id}","starred_url":"https://api.github.com/users/restic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/restic/subscriptions","organizations_url":"https://api.github.com/users/restic/orgs","repos_url":"https://api.github.com/users/restic/repos","events_url":"https://api.github.com/users/restic/events{/privacy}","received_events_url":"https://api.github.com/users/restic/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/restic/restic","description":"restic backup program","fork":false,"url":"https://api.github.com/repos/restic/restic","forks_url":"https://api.github.com/repos/restic/restic/forks","keys_url":"https://api.github.com/repos/restic/restic/keys{/key_id}","collaborators_url":"https://api.github.com/repos/restic/restic/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/restic/restic/teams","hooks_url":"https://api.github.com/repos/restic/restic/hooks","issue_events_url":"https://api.github.com/repos/restic/restic/issues/events{/number}","events_url":"https://api.github.com/repos/restic/restic/events","assignees_url":"https://api.github.com/repos/restic/restic/assignees{/user}","branches_url":"https://api.github.com/repos/restic/restic/branches{/branch}","tags_url":"https://api.github.com/repos/restic/restic/tags","blobs_url":"https://api.github.com/repos/restic/restic/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/restic/restic/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/restic/restic/git/refs{/sha}","trees_url":"https://api.github.com/repos/restic/restic/git/trees{/sha}","statuses_url":"https://api.github.com/repos/restic/restic/statuses/{sha}","languages_url":"https://api.github.com/repos/restic/restic/languages","stargazers_url":"https://api.github.com/repos/restic/restic/stargazers","contributors_url":"https://api.github.com/repos/restic/restic/contributors","subscribers_url":"https://api.github.com/repos/restic/restic/subscribers","subscription_url":"https://api.github.com/repos/restic/restic/subscription","commits_url":"https://api.github.com/repos/restic/restic/commits{/sha}","git_commits_url":"https://api.github.com/repos/restic/restic/git/commits{/sha}","comments_url":"https://api.github.com/repos/restic/restic/comments{/number}","issue_comment_url":"https://api.github.com/repos/restic/restic/issues/comments{/number}","contents_url":"https://api.github.com/repos/restic/restic/contents/{+path}","compare_url":"https://api.github.com/repos/restic/restic/compare/{base}...{head}","merges_url":"https://api.github.com/repos/restic/restic/merges","archive_url":"https://api.github.com/repos/restic/restic/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/restic/restic/downloads","issues_url":"https://api.github.com/repos/restic/restic/issues{/number}","pulls_url":"https://api.github.com/repos/restic/restic/pulls{/number}","milestones_url":"https://api.github.com/repos/restic/restic/milestones{/number}","notifications_url":"https://api.github.com/repos/restic/restic/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/restic/restic/labels{/name}","releases_url":"https://api.github.com/repos/restic/restic/releases{/id}","created_at":"2014-04-27T14:07:58Z","updated_at":"2015-04-29T21:11:10Z","pushed_at":"2015-04-29T21:36:21Z","git_url":"git://github.com/restic/restic.git","ssh_url":"git@github.com:restic/restic.git","clone_url":"https://github.com/restic/restic.git","svn_url":"https://github.com/restic/restic","homepage":"","size":4032,"stargazers_count":111,"watchers_count":111,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":12,"mirror_url":null,"open_issues_count":70,"forks":12,"open_issues":70,"watchers":111,"default_branch":"master"}},"base":{"label":"restic:master","ref":"master","sha":"66b81a2a8bb8735ddcc64c1b0ff98bb29c832ca8","user":{"login":"restic","id":10073512,"avatar_url":"https://avatars.githubusercontent.com/u/10073512?v=3","gravatar_id":"","url":"https://api.github.com/users/restic","html_url":"https://github.com/restic","followers_url":"https://api.github.com/users/restic/followers","following_url":"https://api.github.com/users/restic/following{/other_user}","gists_url":"https://api.github.com/users/restic/gists{/gist_id}","starred_url":"https://api.github.com/users/restic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/restic/subscriptions","organizations_url":"https://api.github.com/users/restic/orgs","repos_url":"https://api.github.com/users/restic/repos","events_url":"https://api.github.com/users/restic/events{/privacy}","received_events_url":"https://api.github.com/users/restic/received_events","type":"Organization","site_admin":false},"repo":{"id":19205896,"name":"restic","full_name":"restic/restic","owner":{"login":"restic","id":10073512,"avatar_url":"https://avatars.githubusercontent.com/u/10073512?v=3","gravatar_id":"","url":"https://api.github.com/users/restic","html_url":"https://github.com/restic","followers_url":"https://api.github.com/users/restic/followers","following_url":"https://api.github.com/users/restic/following{/other_user}","gists_url":"https://api.github.com/users/restic/gists{/gist_id}","starred_url":"https://api.github.com/users/restic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/restic/subscriptions","organizations_url":"https://api.github.com/users/restic/orgs","repos_url":"https://api.github.com/users/restic/repos","events_url":"https://api.github.com/users/restic/events{/privacy}","received_events_url":"https://api.github.com/users/restic/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/restic/restic","description":"restic backup program","fork":false,"url":"https://api.github.com/repos/restic/restic","forks_url":"https://api.github.com/repos/restic/restic/forks","keys_url":"https://api.github.com/repos/restic/restic/keys{/key_id}","collaborators_url":"https://api.github.com/repos/restic/restic/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/restic/restic/teams","hooks_url":"https://api.github.com/repos/restic/restic/hooks","issue_events_url":"https://api.github.com/repos/restic/restic/issues/events{/number}","events_url":"https://api.github.com/repos/restic/restic/events","assignees_url":"https://api.github.com/repos/restic/restic/assignees{/user}","branches_url":"https://api.github.com/repos/restic/restic/branches{/branch}","tags_url":"https://api.github.com/repos/restic/restic/tags","blobs_url":"https://api.github.com/repos/restic/restic/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/restic/restic/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/restic/restic/git/refs{/sha}","trees_url":"https://api.github.com/repos/restic/restic/git/trees{/sha}","statuses_url":"https://api.github.com/repos/restic/restic/statuses/{sha}","languages_url":"https://api.github.com/repos/restic/restic/languages","stargazers_url":"https://api.github.com/repos/restic/restic/stargazers","contributors_url":"https://api.github.com/repos/restic/restic/contributors","subscribers_url":"https://api.github.com/repos/restic/restic/subscribers","subscription_url":"https://api.github.com/repos/restic/restic/subscription","commits_url":"https://api.github.com/repos/restic/restic/commits{/sha}","git_commits_url":"https://api.github.com/repos/restic/restic/git/commits{/sha}","comments_url":"https://api.github.com/repos/restic/restic/comments{/number}","issue_comment_url":"https://api.github.com/repos/restic/restic/issues/comments{/number}","contents_url":"https://api.github.com/repos/restic/restic/contents/{+path}","compare_url":"https://api.github.com/repos/restic/restic/compare/{base}...{head}","merges_url":"https://api.github.com/repos/restic/restic/merges","archive_url":"https://api.github.com/repos/restic/restic/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/restic/restic/downloads","issues_url":"https://api.github.com/repos/restic/restic/issues{/number}","pulls_url":"https://api.github.com/repos/restic/restic/pulls{/number}","milestones_url":"https://api.github.com/repos/restic/restic/milestones{/number}","notifications_url":"https://api.github.com/repos/restic/restic/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/restic/restic/labels{/name}","releases_url":"https://api.github.com/repos/restic/restic/releases{/id}","created_at":"2014-04-27T14:07:58Z","updated_at":"2015-04-29T21:11:10Z","pushed_at":"2015-04-29T21:36:21Z","git_url":"git://github.com/restic/restic.git","ssh_url":"git@github.com:restic/restic.git","clone_url":"https://github.com/restic/restic.git","svn_url":"https://github.com/restic/restic","homepage":"","size":4032,"stargazers_count":111,"watchers_count":111,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":12,"mirror_url":null,"open_issues_count":70,"forks":12,"open_issues":70,"watchers":111,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/restic/restic/pulls/151"},"html":{"href":"https://github.com/restic/restic/pull/151"},"issue":{"href":"https://api.github.com/repos/restic/restic/issues/151"},"comments":{"href":"https://api.github.com/repos/restic/restic/issues/151/comments"},"review_comments":{"href":"https://api.github.com/repos/restic/restic/pulls/151/comments"},"review_comment":{"href":"https://api.github.com/repos/restic/restic/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/restic/restic/pulls/151/commits"},"statuses":{"href":"https://api.github.com/repos/restic/restic/statuses/ec108fb708285944f5c1043f5a55a1191858cd34"}}}}