{"action":"created","comment":{"url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments/28785356","id":28785356,"diff_hunk":"@@ -532,3 +532,73 @@ func TestProgressWithDownNode(t *testing.T) {\n \tmtc.restartStore(1)\n \tverify([]int64{16, 16, 16})\n }\n+\n+func TestReplicateAddAndRemove(t *testing.T) {\n+\tdefer leaktest.AfterTest(t)\n+\n+\t// Run the test twice, once adding the replacement before removing\n+\t// the downed node, and once removing the downed node first.\n+\tfor _, addFirst := range []bool{true, false} {\n+\t\tmtc := startMultiTestContext(t, 4)\n+\t\tdefer mtc.Stop()\n+\n+\t\t// Replicate the initial range to three of the four nodes.\n+\t\traftID := int64(1)\n+\t\tmtc.replicateRange(raftID, 0, 3, 1)\n+\n+\t\tincArgs, incResp := incrementArgs([]byte(\"a\"), 5, raftID, mtc.stores[0].StoreID())\n+\t\tif err := mtc.stores[0].ExecuteCmd(proto.Increment, incArgs, incResp); err != nil {\n+\t\t\tt.Fatal(err)\n+\t\t}\n+\n+\t\tverify := func(expected []int64) {\n+\t\t\tutil.SucceedsWithin(t, time.Second, func() error {\n+\t\t\t\tvalues := []int64{}\n+\t\t\t\tfor _, eng := range mtc.engines {\n+\t\t\t\t\tval, err := engine.MVCCGet(eng, proto.Key(\"a\"), mtc.clock.Now(), true, nil)\n+\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\treturn err\n+\t\t\t\t\t}\n+\t\t\t\t\tvalues = append(values, val.GetInteger())\n+\t\t\t\t}\n+\t\t\t\tif !reflect.DeepEqual(expected, values) {\n+\t\t\t\t\treturn util.Errorf(\"expected %v, got %v\", expected, values)\n+\t\t\t\t}\n+\t\t\t\treturn nil\n+\t\t\t})\n+\t\t}\n+\n+\t\t// The first increment is visible on all three replicas.\n+\t\tverify([]int64{5, 5, 0, 5})\n+\n+\t\t// Stop a store and replace it.\n+\t\tmtc.stopStore(1)\n+\t\tif addFirst {\n+\t\t\tmtc.replicateRange(raftID, 0, 2)\n+\t\t\tmtc.unreplicateRange(raftID, 0, 1)\n+\t\t} else {\n+\t\t\tmtc.unreplicateRange(raftID, 0, 1)\n+\t\t\tmtc.replicateRange(raftID, 0, 2)\n+\t\t}\n+\t\tverify([]int64{5, 5, 5, 5})\n+\n+\t\t// Ensure that the rest of the group can make progress.\n+\t\tincArgs, incResp = incrementArgs([]byte(\"a\"), 11, raftID, mtc.stores[0].StoreID())\n+\t\tif err := mtc.stores[0].ExecuteCmd(proto.Increment, incArgs, incResp); err != nil {\n+\t\t\tt.Fatal(err)\n+\t\t}\n+\t\tverify([]int64{16, 5, 16, 16})\n+\n+\t\t// Bring the downed store back up (required for a clean shutdown).\n+\t\tmtc.restartStore(1)\n+\n+\t\t// Node 1 never (for small values of \"never\") sees the increment\n+\t\t// that was added while it was down.\n+\t\ttime.Sleep(10 * time.Millisecond)","path":"storage/client_raft_test.go","position":66,"original_position":66,"commit_id":"ed173ef6b2716f584ecea3b5bd81ed0ddbaa0e04","original_commit_id":"ed173ef6b2716f584ecea3b5bd81ed0ddbaa0e04","user":{"login":"tschottdorf","id":5076964,"avatar_url":"https://avatars.githubusercontent.com/u/5076964?v=3","gravatar_id":"","url":"https://api.github.com/users/tschottdorf","html_url":"https://github.com/tschottdorf","followers_url":"https://api.github.com/users/tschottdorf/followers","following_url":"https://api.github.com/users/tschottdorf/following{/other_user}","gists_url":"https://api.github.com/users/tschottdorf/gists{/gist_id}","starred_url":"https://api.github.com/users/tschottdorf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tschottdorf/subscriptions","organizations_url":"https://api.github.com/users/tschottdorf/orgs","repos_url":"https://api.github.com/users/tschottdorf/repos","events_url":"https://api.github.com/users/tschottdorf/events{/privacy}","received_events_url":"https://api.github.com/users/tschottdorf/received_events","type":"User","site_admin":false},"body":"why not do another increment instead and make sure that you end up with `{17, 5, 17, 17}` instead of the `time.Sleep` here?","created_at":"2015-04-21T14:52:04Z","updated_at":"2015-04-21T14:52:04Z","html_url":"https://github.com/cockroachdb/cockroach/pull/717#discussion_r28785356","pull_request_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/717","_links":{"self":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments/28785356"},"html":{"href":"https://github.com/cockroachdb/cockroach/pull/717#discussion_r28785356"},"pull_request":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/717"}}},"pull_request":{"url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/717","id":33716653,"html_url":"https://github.com/cockroachdb/cockroach/pull/717","diff_url":"https://github.com/cockroachdb/cockroach/pull/717.diff","patch_url":"https://github.com/cockroachdb/cockroach/pull/717.patch","issue_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/717","number":717,"state":"open","locked":false,"title":"Add a test for adding and removing replicas.","user":{"login":"bdarnell","id":160562,"avatar_url":"https://avatars.githubusercontent.com/u/160562?v=3","gravatar_id":"","url":"https://api.github.com/users/bdarnell","html_url":"https://github.com/bdarnell","followers_url":"https://api.github.com/users/bdarnell/followers","following_url":"https://api.github.com/users/bdarnell/following{/other_user}","gists_url":"https://api.github.com/users/bdarnell/gists{/gist_id}","starred_url":"https://api.github.com/users/bdarnell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bdarnell/subscriptions","organizations_url":"https://api.github.com/users/bdarnell/orgs","repos_url":"https://api.github.com/users/bdarnell/repos","events_url":"https://api.github.com/users/bdarnell/events{/privacy}","received_events_url":"https://api.github.com/users/bdarnell/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-04-20T23:35:21Z","updated_at":"2015-04-21T14:52:04Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1a9d4fad82226c956b2088d5adcc0627e188f381","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/717/commits","review_comments_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/717/comments","review_comment_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments{/number}","comments_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/717/comments","statuses_url":"https://api.github.com/repos/cockroachdb/cockroach/statuses/ed173ef6b2716f584ecea3b5bd81ed0ddbaa0e04","head":{"label":"bdarnell:add-remove-test","ref":"add-remove-test","sha":"ed173ef6b2716f584ecea3b5bd81ed0ddbaa0e04","user":{"login":"bdarnell","id":160562,"avatar_url":"https://avatars.githubusercontent.com/u/160562?v=3","gravatar_id":"","url":"https://api.github.com/users/bdarnell","html_url":"https://github.com/bdarnell","followers_url":"https://api.github.com/users/bdarnell/followers","following_url":"https://api.github.com/users/bdarnell/following{/other_user}","gists_url":"https://api.github.com/users/bdarnell/gists{/gist_id}","starred_url":"https://api.github.com/users/bdarnell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bdarnell/subscriptions","organizations_url":"https://api.github.com/users/bdarnell/orgs","repos_url":"https://api.github.com/users/bdarnell/repos","events_url":"https://api.github.com/users/bdarnell/events{/privacy}","received_events_url":"https://api.github.com/users/bdarnell/received_events","type":"User","site_admin":false},"repo":{"id":23885381,"name":"cockroach","full_name":"bdarnell/cockroach","owner":{"login":"bdarnell","id":160562,"avatar_url":"https://avatars.githubusercontent.com/u/160562?v=3","gravatar_id":"","url":"https://api.github.com/users/bdarnell","html_url":"https://github.com/bdarnell","followers_url":"https://api.github.com/users/bdarnell/followers","following_url":"https://api.github.com/users/bdarnell/following{/other_user}","gists_url":"https://api.github.com/users/bdarnell/gists{/gist_id}","starred_url":"https://api.github.com/users/bdarnell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bdarnell/subscriptions","organizations_url":"https://api.github.com/users/bdarnell/orgs","repos_url":"https://api.github.com/users/bdarnell/repos","events_url":"https://api.github.com/users/bdarnell/events{/privacy}","received_events_url":"https://api.github.com/users/bdarnell/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/bdarnell/cockroach","description":"A Scalable, Geo-Replicated, Transactional Datastore","fork":true,"url":"https://api.github.com/repos/bdarnell/cockroach","forks_url":"https://api.github.com/repos/bdarnell/cockroach/forks","keys_url":"https://api.github.com/repos/bdarnell/cockroach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/bdarnell/cockroach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/bdarnell/cockroach/teams","hooks_url":"https://api.github.com/repos/bdarnell/cockroach/hooks","issue_events_url":"https://api.github.com/repos/bdarnell/cockroach/issues/events{/number}","events_url":"https://api.github.com/repos/bdarnell/cockroach/events","assignees_url":"https://api.github.com/repos/bdarnell/cockroach/assignees{/user}","branches_url":"https://api.github.com/repos/bdarnell/cockroach/branches{/branch}","tags_url":"https://api.github.com/repos/bdarnell/cockroach/tags","blobs_url":"https://api.github.com/repos/bdarnell/cockroach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/bdarnell/cockroach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/bdarnell/cockroach/git/refs{/sha}","trees_url":"https://api.github.com/repos/bdarnell/cockroach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/bdarnell/cockroach/statuses/{sha}","languages_url":"https://api.github.com/repos/bdarnell/cockroach/languages","stargazers_url":"https://api.github.com/repos/bdarnell/cockroach/stargazers","contributors_url":"https://api.github.com/repos/bdarnell/cockroach/contributors","subscribers_url":"https://api.github.com/repos/bdarnell/cockroach/subscribers","subscription_url":"https://api.github.com/repos/bdarnell/cockroach/subscription","commits_url":"https://api.github.com/repos/bdarnell/cockroach/commits{/sha}","git_commits_url":"https://api.github.com/repos/bdarnell/cockroach/git/commits{/sha}","comments_url":"https://api.github.com/repos/bdarnell/cockroach/comments{/number}","issue_comment_url":"https://api.github.com/repos/bdarnell/cockroach/issues/comments{/number}","contents_url":"https://api.github.com/repos/bdarnell/cockroach/contents/{+path}","compare_url":"https://api.github.com/repos/bdarnell/cockroach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/bdarnell/cockroach/merges","archive_url":"https://api.github.com/repos/bdarnell/cockroach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/bdarnell/cockroach/downloads","issues_url":"https://api.github.com/repos/bdarnell/cockroach/issues{/number}","pulls_url":"https://api.github.com/repos/bdarnell/cockroach/pulls{/number}","milestones_url":"https://api.github.com/repos/bdarnell/cockroach/milestones{/number}","notifications_url":"https://api.github.com/repos/bdarnell/cockroach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/bdarnell/cockroach/labels{/name}","releases_url":"https://api.github.com/repos/bdarnell/cockroach/releases{/id}","created_at":"2014-09-10T17:55:56Z","updated_at":"2015-03-31T18:00:59Z","pushed_at":"2015-04-20T23:43:26Z","git_url":"git://github.com/bdarnell/cockroach.git","ssh_url":"git@github.com:bdarnell/cockroach.git","clone_url":"https://github.com/bdarnell/cockroach.git","svn_url":"https://github.com/bdarnell/cockroach","homepage":null,"size":13572,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"cockroachdb:master","ref":"master","sha":"07d7a4fa0024b221e17a4a755409aea9f7e09c44","user":{"login":"cockroachdb","id":6748139,"avatar_url":"https://avatars.githubusercontent.com/u/6748139?v=3","gravatar_id":"","url":"https://api.github.com/users/cockroachdb","html_url":"https://github.com/cockroachdb","followers_url":"https://api.github.com/users/cockroachdb/followers","following_url":"https://api.github.com/users/cockroachdb/following{/other_user}","gists_url":"https://api.github.com/users/cockroachdb/gists{/gist_id}","starred_url":"https://api.github.com/users/cockroachdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cockroachdb/subscriptions","organizations_url":"https://api.github.com/users/cockroachdb/orgs","repos_url":"https://api.github.com/users/cockroachdb/repos","events_url":"https://api.github.com/users/cockroachdb/events{/privacy}","received_events_url":"https://api.github.com/users/cockroachdb/received_events","type":"Organization","site_admin":false},"repo":{"id":16563587,"name":"cockroach","full_name":"cockroachdb/cockroach","owner":{"login":"cockroachdb","id":6748139,"avatar_url":"https://avatars.githubusercontent.com/u/6748139?v=3","gravatar_id":"","url":"https://api.github.com/users/cockroachdb","html_url":"https://github.com/cockroachdb","followers_url":"https://api.github.com/users/cockroachdb/followers","following_url":"https://api.github.com/users/cockroachdb/following{/other_user}","gists_url":"https://api.github.com/users/cockroachdb/gists{/gist_id}","starred_url":"https://api.github.com/users/cockroachdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cockroachdb/subscriptions","organizations_url":"https://api.github.com/users/cockroachdb/orgs","repos_url":"https://api.github.com/users/cockroachdb/repos","events_url":"https://api.github.com/users/cockroachdb/events{/privacy}","received_events_url":"https://api.github.com/users/cockroachdb/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cockroachdb/cockroach","description":"A Scalable, Geo-Replicated, Transactional Datastore","fork":false,"url":"https://api.github.com/repos/cockroachdb/cockroach","forks_url":"https://api.github.com/repos/cockroachdb/cockroach/forks","keys_url":"https://api.github.com/repos/cockroachdb/cockroach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cockroachdb/cockroach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cockroachdb/cockroach/teams","hooks_url":"https://api.github.com/repos/cockroachdb/cockroach/hooks","issue_events_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/events{/number}","events_url":"https://api.github.com/repos/cockroachdb/cockroach/events","assignees_url":"https://api.github.com/repos/cockroachdb/cockroach/assignees{/user}","branches_url":"https://api.github.com/repos/cockroachdb/cockroach/branches{/branch}","tags_url":"https://api.github.com/repos/cockroachdb/cockroach/tags","blobs_url":"https://api.github.com/repos/cockroachdb/cockroach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cockroachdb/cockroach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cockroachdb/cockroach/git/refs{/sha}","trees_url":"https://api.github.com/repos/cockroachdb/cockroach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cockroachdb/cockroach/statuses/{sha}","languages_url":"https://api.github.com/repos/cockroachdb/cockroach/languages","stargazers_url":"https://api.github.com/repos/cockroachdb/cockroach/stargazers","contributors_url":"https://api.github.com/repos/cockroachdb/cockroach/contributors","subscribers_url":"https://api.github.com/repos/cockroachdb/cockroach/subscribers","subscription_url":"https://api.github.com/repos/cockroachdb/cockroach/subscription","commits_url":"https://api.github.com/repos/cockroachdb/cockroach/commits{/sha}","git_commits_url":"https://api.github.com/repos/cockroachdb/cockroach/git/commits{/sha}","comments_url":"https://api.github.com/repos/cockroachdb/cockroach/comments{/number}","issue_comment_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/comments{/number}","contents_url":"https://api.github.com/repos/cockroachdb/cockroach/contents/{+path}","compare_url":"https://api.github.com/repos/cockroachdb/cockroach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cockroachdb/cockroach/merges","archive_url":"https://api.github.com/repos/cockroachdb/cockroach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cockroachdb/cockroach/downloads","issues_url":"https://api.github.com/repos/cockroachdb/cockroach/issues{/number}","pulls_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls{/number}","milestones_url":"https://api.github.com/repos/cockroachdb/cockroach/milestones{/number}","notifications_url":"https://api.github.com/repos/cockroachdb/cockroach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cockroachdb/cockroach/labels{/name}","releases_url":"https://api.github.com/repos/cockroachdb/cockroach/releases{/id}","created_at":"2014-02-06T00:18:47Z","updated_at":"2015-04-21T13:31:49Z","pushed_at":"2015-04-21T14:48:48Z","git_url":"git://github.com/cockroachdb/cockroach.git","ssh_url":"git@github.com:cockroachdb/cockroach.git","clone_url":"https://github.com/cockroachdb/cockroach.git","svn_url":"https://github.com/cockroachdb/cockroach","homepage":null,"size":24833,"stargazers_count":3323,"watchers_count":3323,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":252,"mirror_url":null,"open_issues_count":64,"forks":252,"open_issues":64,"watchers":3323,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/717"},"html":{"href":"https://github.com/cockroachdb/cockroach/pull/717"},"issue":{"href":"https://api.github.com/repos/cockroachdb/cockroach/issues/717"},"comments":{"href":"https://api.github.com/repos/cockroachdb/cockroach/issues/717/comments"},"review_comments":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/717/comments"},"review_comment":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/717/commits"},"statuses":{"href":"https://api.github.com/repos/cockroachdb/cockroach/statuses/ed173ef6b2716f584ecea3b5bd81ed0ddbaa0e04"}}}}