{"action":"created","comment":{"url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments/38587374","id":38587374,"diff_hunk":"@@ -392,6 +392,70 @@ func TestMembershipChange(t *testing.T) {\n \t\t}*/\n }\n \n+func TestRemoveLeader(t *testing.T) {\n+\tdefer leaktest.AfterTest(t)\n+\tstopper := stop.NewStopper()\n+\tcluster := newTestCluster(nil, 6, stopper, t)\n+\tdefer stopper.Stop()\n+\n+\t// Consume and apply the membership change events.\n+\tfor i := 0; i < 4; i++ {\n+\t\tgo func(i int) {\n+\t\t\tfor {\n+\t\t\t\tif e, ok := <-cluster.events[i].MembershipChangeCommitted; ok {\n+\t\t\t\t\te.Callback(nil)\n+\t\t\t\t} else {\n+\t\t\t\t\treturn\n+\t\t\t\t}\n+\t\t\t}\n+\t\t}(i)\n+\t}\n+\n+\t// Tick all the clocks in the background to ensure that all the\n+\t// necessary elections are triggered.\n+\t// TODO(bdarnell): newTestCluster should have an option to use a\n+\t// real clock instead of a manual one.\n+\tstopper.RunWorker(func() {\n+\t\tticker := time.NewTicker(10 * time.Millisecond)\n+\t\tdefer ticker.Stop()\n+\t\tfor {\n+\t\t\tselect {\n+\t\t\tcase <-stopper.ShouldStop():\n+\t\t\t\treturn\n+\t\t\tcase <-ticker.C:\n+\t\t\t\tfor _, t := range cluster.tickers {\n+\t\t\t\t\tt.Tick()\n+\t\t\t\t}\n+\t\t\t}\n+\t\t}\n+\t})\n+\n+\t// Create a group with three members.\n+\tgroupID := proto.RangeID(1)\n+\tcluster.createGroup(groupID, 0, 3)\n+\n+\t// Move the group one node at a time from the first three nodes to\n+\t// the last three. In the process, we necessarily remove the leader\n+\t// and trigger at least one new election among the new nodes.\n+\tfor i := 0; i < 3; i++ {\n+\t\tlog.Infof(\"adding node %d\", i+3)","path":"multiraft/multiraft_test.go","position":50,"original_position":50,"commit_id":"ba7abee304f800c5630803e03f0247c20ff6ab8b","original_commit_id":"ba7abee304f800c5630803e03f0247c20ff6ab8b","user":{"login":"bdarnell","id":160562,"avatar_url":"https://avatars.githubusercontent.com/u/160562?v=3","gravatar_id":"","url":"https://api.github.com/users/bdarnell","html_url":"https://github.com/bdarnell","followers_url":"https://api.github.com/users/bdarnell/followers","following_url":"https://api.github.com/users/bdarnell/following{/other_user}","gists_url":"https://api.github.com/users/bdarnell/gists{/gist_id}","starred_url":"https://api.github.com/users/bdarnell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bdarnell/subscriptions","organizations_url":"https://api.github.com/users/bdarnell/orgs","repos_url":"https://api.github.com/users/bdarnell/repos","events_url":"https://api.github.com/users/bdarnell/events{/privacy}","received_events_url":"https://api.github.com/users/bdarnell/received_events","type":"User","site_admin":false},"body":"`t.Logf` messages are buffered by the testing framework and printed at the end of the test, which means that they can't be correlated with any of our other log spam. I don't think it ever makes sense for us to use `t.Logf` from a package that already uses our `log` package.","created_at":"2015-09-02T21:32:40Z","updated_at":"2015-09-02T21:32:40Z","html_url":"https://github.com/cockroachdb/cockroach/pull/2334#discussion_r38587374","pull_request_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/2334","_links":{"self":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments/38587374"},"html":{"href":"https://github.com/cockroachdb/cockroach/pull/2334#discussion_r38587374"},"pull_request":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/2334"}}},"pull_request":{"url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/2334","id":44038886,"html_url":"https://github.com/cockroachdb/cockroach/pull/2334","diff_url":"https://github.com/cockroachdb/cockroach/pull/2334.diff","patch_url":"https://github.com/cockroachdb/cockroach/pull/2334.patch","issue_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/2334","number":2334,"state":"open","locked":false,"title":"Allow a node to be removed from raft while it is the leader.","user":{"login":"bdarnell","id":160562,"avatar_url":"https://avatars.githubusercontent.com/u/160562?v=3","gravatar_id":"","url":"https://api.github.com/users/bdarnell","html_url":"https://github.com/bdarnell","followers_url":"https://api.github.com/users/bdarnell/followers","following_url":"https://api.github.com/users/bdarnell/following{/other_user}","gists_url":"https://api.github.com/users/bdarnell/gists{/gist_id}","starred_url":"https://api.github.com/users/bdarnell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bdarnell/subscriptions","organizations_url":"https://api.github.com/users/bdarnell/orgs","repos_url":"https://api.github.com/users/bdarnell/repos","events_url":"https://api.github.com/users/bdarnell/events{/privacy}","received_events_url":"https://api.github.com/users/bdarnell/received_events","type":"User","site_admin":false},"body":"Previously, removed nodes would still be counted as members of the\r\ngroup for purposes of coalesced heartbeats. If the node was leader\r\nwhen it was removed, this would prevent any further elections in the\r\ngroup (as long as the removed node was alive).\r\n\r\nThis should unblock the repair/rebalancing work, although it doesn't fix #2291. I have some more tests for this change (to be added to `storage/client_raft_test.go`) but they are blocked on @tschottdorf's batch mega-PR.","created_at":"2015-09-02T20:33:40Z","updated_at":"2015-09-02T21:32:40Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c82ebdff6f0465ba0434e7fe4d905930dd3ca7e5","assignee":{"login":"tschottdorf","id":5076964,"avatar_url":"https://avatars.githubusercontent.com/u/5076964?v=3","gravatar_id":"","url":"https://api.github.com/users/tschottdorf","html_url":"https://github.com/tschottdorf","followers_url":"https://api.github.com/users/tschottdorf/followers","following_url":"https://api.github.com/users/tschottdorf/following{/other_user}","gists_url":"https://api.github.com/users/tschottdorf/gists{/gist_id}","starred_url":"https://api.github.com/users/tschottdorf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tschottdorf/subscriptions","organizations_url":"https://api.github.com/users/tschottdorf/orgs","repos_url":"https://api.github.com/users/tschottdorf/repos","events_url":"https://api.github.com/users/tschottdorf/events{/privacy}","received_events_url":"https://api.github.com/users/tschottdorf/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/2334/commits","review_comments_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/2334/comments","review_comment_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments{/number}","comments_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/2334/comments","statuses_url":"https://api.github.com/repos/cockroachdb/cockroach/statuses/ba7abee304f800c5630803e03f0247c20ff6ab8b","head":{"label":"bdarnell:remove-raft-leader2","ref":"remove-raft-leader2","sha":"ba7abee304f800c5630803e03f0247c20ff6ab8b","user":{"login":"bdarnell","id":160562,"avatar_url":"https://avatars.githubusercontent.com/u/160562?v=3","gravatar_id":"","url":"https://api.github.com/users/bdarnell","html_url":"https://github.com/bdarnell","followers_url":"https://api.github.com/users/bdarnell/followers","following_url":"https://api.github.com/users/bdarnell/following{/other_user}","gists_url":"https://api.github.com/users/bdarnell/gists{/gist_id}","starred_url":"https://api.github.com/users/bdarnell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bdarnell/subscriptions","organizations_url":"https://api.github.com/users/bdarnell/orgs","repos_url":"https://api.github.com/users/bdarnell/repos","events_url":"https://api.github.com/users/bdarnell/events{/privacy}","received_events_url":"https://api.github.com/users/bdarnell/received_events","type":"User","site_admin":false},"repo":{"id":23885381,"name":"cockroach","full_name":"bdarnell/cockroach","owner":{"login":"bdarnell","id":160562,"avatar_url":"https://avatars.githubusercontent.com/u/160562?v=3","gravatar_id":"","url":"https://api.github.com/users/bdarnell","html_url":"https://github.com/bdarnell","followers_url":"https://api.github.com/users/bdarnell/followers","following_url":"https://api.github.com/users/bdarnell/following{/other_user}","gists_url":"https://api.github.com/users/bdarnell/gists{/gist_id}","starred_url":"https://api.github.com/users/bdarnell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bdarnell/subscriptions","organizations_url":"https://api.github.com/users/bdarnell/orgs","repos_url":"https://api.github.com/users/bdarnell/repos","events_url":"https://api.github.com/users/bdarnell/events{/privacy}","received_events_url":"https://api.github.com/users/bdarnell/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/bdarnell/cockroach","description":"A Scalable, Geo-Replicated, Transactional Datastore","fork":true,"url":"https://api.github.com/repos/bdarnell/cockroach","forks_url":"https://api.github.com/repos/bdarnell/cockroach/forks","keys_url":"https://api.github.com/repos/bdarnell/cockroach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/bdarnell/cockroach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/bdarnell/cockroach/teams","hooks_url":"https://api.github.com/repos/bdarnell/cockroach/hooks","issue_events_url":"https://api.github.com/repos/bdarnell/cockroach/issues/events{/number}","events_url":"https://api.github.com/repos/bdarnell/cockroach/events","assignees_url":"https://api.github.com/repos/bdarnell/cockroach/assignees{/user}","branches_url":"https://api.github.com/repos/bdarnell/cockroach/branches{/branch}","tags_url":"https://api.github.com/repos/bdarnell/cockroach/tags","blobs_url":"https://api.github.com/repos/bdarnell/cockroach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/bdarnell/cockroach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/bdarnell/cockroach/git/refs{/sha}","trees_url":"https://api.github.com/repos/bdarnell/cockroach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/bdarnell/cockroach/statuses/{sha}","languages_url":"https://api.github.com/repos/bdarnell/cockroach/languages","stargazers_url":"https://api.github.com/repos/bdarnell/cockroach/stargazers","contributors_url":"https://api.github.com/repos/bdarnell/cockroach/contributors","subscribers_url":"https://api.github.com/repos/bdarnell/cockroach/subscribers","subscription_url":"https://api.github.com/repos/bdarnell/cockroach/subscription","commits_url":"https://api.github.com/repos/bdarnell/cockroach/commits{/sha}","git_commits_url":"https://api.github.com/repos/bdarnell/cockroach/git/commits{/sha}","comments_url":"https://api.github.com/repos/bdarnell/cockroach/comments{/number}","issue_comment_url":"https://api.github.com/repos/bdarnell/cockroach/issues/comments{/number}","contents_url":"https://api.github.com/repos/bdarnell/cockroach/contents/{+path}","compare_url":"https://api.github.com/repos/bdarnell/cockroach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/bdarnell/cockroach/merges","archive_url":"https://api.github.com/repos/bdarnell/cockroach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/bdarnell/cockroach/downloads","issues_url":"https://api.github.com/repos/bdarnell/cockroach/issues{/number}","pulls_url":"https://api.github.com/repos/bdarnell/cockroach/pulls{/number}","milestones_url":"https://api.github.com/repos/bdarnell/cockroach/milestones{/number}","notifications_url":"https://api.github.com/repos/bdarnell/cockroach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/bdarnell/cockroach/labels{/name}","releases_url":"https://api.github.com/repos/bdarnell/cockroach/releases{/id}","created_at":"2014-09-10T17:55:56Z","updated_at":"2015-03-31T18:00:59Z","pushed_at":"2015-09-02T20:29:39Z","git_url":"git://github.com/bdarnell/cockroach.git","ssh_url":"git@github.com:bdarnell/cockroach.git","clone_url":"https://github.com/bdarnell/cockroach.git","svn_url":"https://github.com/bdarnell/cockroach","homepage":null,"size":29923,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"cockroachdb:master","ref":"master","sha":"7c9fa4ab6395adce089956be5c6cb4aee0f31040","user":{"login":"cockroachdb","id":6748139,"avatar_url":"https://avatars.githubusercontent.com/u/6748139?v=3","gravatar_id":"","url":"https://api.github.com/users/cockroachdb","html_url":"https://github.com/cockroachdb","followers_url":"https://api.github.com/users/cockroachdb/followers","following_url":"https://api.github.com/users/cockroachdb/following{/other_user}","gists_url":"https://api.github.com/users/cockroachdb/gists{/gist_id}","starred_url":"https://api.github.com/users/cockroachdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cockroachdb/subscriptions","organizations_url":"https://api.github.com/users/cockroachdb/orgs","repos_url":"https://api.github.com/users/cockroachdb/repos","events_url":"https://api.github.com/users/cockroachdb/events{/privacy}","received_events_url":"https://api.github.com/users/cockroachdb/received_events","type":"Organization","site_admin":false},"repo":{"id":16563587,"name":"cockroach","full_name":"cockroachdb/cockroach","owner":{"login":"cockroachdb","id":6748139,"avatar_url":"https://avatars.githubusercontent.com/u/6748139?v=3","gravatar_id":"","url":"https://api.github.com/users/cockroachdb","html_url":"https://github.com/cockroachdb","followers_url":"https://api.github.com/users/cockroachdb/followers","following_url":"https://api.github.com/users/cockroachdb/following{/other_user}","gists_url":"https://api.github.com/users/cockroachdb/gists{/gist_id}","starred_url":"https://api.github.com/users/cockroachdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cockroachdb/subscriptions","organizations_url":"https://api.github.com/users/cockroachdb/orgs","repos_url":"https://api.github.com/users/cockroachdb/repos","events_url":"https://api.github.com/users/cockroachdb/events{/privacy}","received_events_url":"https://api.github.com/users/cockroachdb/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cockroachdb/cockroach","description":"A Scalable, Geo-Replicated, Transactional Datastore","fork":false,"url":"https://api.github.com/repos/cockroachdb/cockroach","forks_url":"https://api.github.com/repos/cockroachdb/cockroach/forks","keys_url":"https://api.github.com/repos/cockroachdb/cockroach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cockroachdb/cockroach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cockroachdb/cockroach/teams","hooks_url":"https://api.github.com/repos/cockroachdb/cockroach/hooks","issue_events_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/events{/number}","events_url":"https://api.github.com/repos/cockroachdb/cockroach/events","assignees_url":"https://api.github.com/repos/cockroachdb/cockroach/assignees{/user}","branches_url":"https://api.github.com/repos/cockroachdb/cockroach/branches{/branch}","tags_url":"https://api.github.com/repos/cockroachdb/cockroach/tags","blobs_url":"https://api.github.com/repos/cockroachdb/cockroach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cockroachdb/cockroach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cockroachdb/cockroach/git/refs{/sha}","trees_url":"https://api.github.com/repos/cockroachdb/cockroach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cockroachdb/cockroach/statuses/{sha}","languages_url":"https://api.github.com/repos/cockroachdb/cockroach/languages","stargazers_url":"https://api.github.com/repos/cockroachdb/cockroach/stargazers","contributors_url":"https://api.github.com/repos/cockroachdb/cockroach/contributors","subscribers_url":"https://api.github.com/repos/cockroachdb/cockroach/subscribers","subscription_url":"https://api.github.com/repos/cockroachdb/cockroach/subscription","commits_url":"https://api.github.com/repos/cockroachdb/cockroach/commits{/sha}","git_commits_url":"https://api.github.com/repos/cockroachdb/cockroach/git/commits{/sha}","comments_url":"https://api.github.com/repos/cockroachdb/cockroach/comments{/number}","issue_comment_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/comments{/number}","contents_url":"https://api.github.com/repos/cockroachdb/cockroach/contents/{+path}","compare_url":"https://api.github.com/repos/cockroachdb/cockroach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cockroachdb/cockroach/merges","archive_url":"https://api.github.com/repos/cockroachdb/cockroach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cockroachdb/cockroach/downloads","issues_url":"https://api.github.com/repos/cockroachdb/cockroach/issues{/number}","pulls_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls{/number}","milestones_url":"https://api.github.com/repos/cockroachdb/cockroach/milestones{/number}","notifications_url":"https://api.github.com/repos/cockroachdb/cockroach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cockroachdb/cockroach/labels{/name}","releases_url":"https://api.github.com/repos/cockroachdb/cockroach/releases{/id}","created_at":"2014-02-06T00:18:47Z","updated_at":"2015-09-02T21:25:42Z","pushed_at":"2015-09-02T21:19:06Z","git_url":"git://github.com/cockroachdb/cockroach.git","ssh_url":"git@github.com:cockroachdb/cockroach.git","clone_url":"https://github.com/cockroachdb/cockroach.git","svn_url":"https://github.com/cockroachdb/cockroach","homepage":null,"size":53720,"stargazers_count":5088,"watchers_count":5088,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":406,"mirror_url":null,"open_issues_count":159,"forks":406,"open_issues":159,"watchers":5088,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/2334"},"html":{"href":"https://github.com/cockroachdb/cockroach/pull/2334"},"issue":{"href":"https://api.github.com/repos/cockroachdb/cockroach/issues/2334"},"comments":{"href":"https://api.github.com/repos/cockroachdb/cockroach/issues/2334/comments"},"review_comments":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/2334/comments"},"review_comment":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/2334/commits"},"statuses":{"href":"https://api.github.com/repos/cockroachdb/cockroach/statuses/ba7abee304f800c5630803e03f0247c20ff6ab8b"}}}}