{"action":"created","comment":{"url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments/30826465","id":30826465,"diff_hunk":"@@ -101,45 +112,106 @@ func (t *rpcTransport) Stop(id proto.RaftNodeID) {\n \tdelete(t.servers, id)\n }\n \n-// Send a message to the recipient specified in the request.\n-func (t *rpcTransport) Send(req *multiraft.RaftMessageRequest) error {\n-\t// Convert internal to proto formats.\n-\tprotoReq := &proto.RaftMessageRequest{GroupID: req.GroupID}\n-\tvar err error\n-\tif protoReq.Msg, err = req.Message.Marshal(); err != nil {\n-\t\treturn err\n+// processQueue creates a client and sends messages from its designated queue\n+// via that client, exiting when the client fails or when it idles out. All\n+// messages remaining in the queue at that point are lost and a new instance of\n+// processQueue should be started by the next message to be sent.\n+// TODO(tschottdorf) should let MultiRaft know if the node is down;\n+// need a feedback mechanism for that. Potentially easiest is to arrange for\n+// the next call to Send() to fail appropriately.\n+func (t *rpcTransport) processQueue(raftNodeID proto.RaftNodeID) {\n+\tt.mu.Lock()\n+\tch, ok := t.queues[raftNodeID]\n+\tt.mu.Unlock()\n+\tif !ok {\n+\t\treturn\n \t}\n-\n-\tnodeID, _ := proto.DecodeRaftNodeID(proto.RaftNodeID(req.Message.To))\n+\t// Clean-up when the loop below shuts down.\n+\tdefer func() {\n+\t\tt.mu.Lock()\n+\t\tdelete(t.queues, raftNodeID)\n+\t\tt.mu.Unlock()\n+\t}()\n+\n+\tnodeID, _ := proto.DecodeRaftNodeID(raftNodeID)\n \taddr, err := t.gossip.GetNodeIDAddress(nodeID)\n \tif err != nil {\n-\t\treturn err\n+\t\tlog.Errorf(\"could not get address for node %d: %s\", nodeID, err)\n \t}\n-\n \tclient := rpc.NewClient(addr, nil, t.rpcContext)\n \tselect {\n-\tcase <-client.Ready:\n+\tcase <-t.rpcContext.Stopper.ShouldStop():\n+\t\treturn\n \tcase <-client.Closed:\n-\t\treturn util.Errorf(\"raft client failed to connect\")\n+\t\tlog.Warningf(\"raft client failed to connect\")\n+\t\treturn\n+\tcase <-time.After(raftIdleTimeout):\n+\t\t// Should never happen.\n+\t\tlog.Errorf(\"raft client stuck connecting\")\n+\t\treturn\n+\tcase <-client.Ready:\n \t}\n \n-\tcall := client.Go(raftMessageName, protoReq, &proto.RaftMessageResponse{}, nil)\n-\tselect {\n-\tcase <-call.Done:\n-\t\t// If the call failed synchronously, report an error.\n-\t\treturn call.Error\n-\tdefault:\n-\t\t// Otherwise, fire-and-forget.\n-\t\tgo func() {\n-\t\t\t<-call.Done\n+\tdone := make(chan *gorpc.Call, cap(ch))\n+\tvar req *multiraft.RaftMessageRequest\n+\tprotoReq := &proto.RaftMessageRequest{}\n+\tprotoResp := &proto.RaftMessageResponse{}\n+\tfor {\n+\t\tselect {\n+\t\tcase <-t.rpcContext.Stopper.ShouldStop():\n+\t\t\treturn\n+\t\tcase <-time.After(raftIdleTimeout):\n+\t\t\treturn\n+\t\tcase <-client.Closed:\n+\t\t\tlog.Warningf(\"raft client closed\")\n+\t\t\treturn\n+\t\tcase call := <-done:\n \t\t\tif call.Error != nil {\n \t\t\t\tlog.Errorf(\"raft message failed: %s\", call.Error)\n \t\t\t}\n-\t\t}()\n-\t\treturn nil\n+\t\t\tcontinue\n+\t\tcase req = <-ch:\n+\t\t}\n+\t\tif req == nil {\n+\t\t\treturn\n+\t\t}\n+\n+\t\t// Convert to proto format.\n+\t\tprotoReq.Reset()\n+\t\tprotoReq.GroupID = req.GroupID\n+\t\tvar err error\n+\t\tif protoReq.Msg, err = req.Message.Marshal(); err != nil {\n+\t\t\tlog.Errorf(\"could not marshal message: %s\", err)\n+\t\t}\n+\n+\t\tif !client.IsHealthy() {\n+\t\t\tlog.Warningf(\"client is unhealthy\")\n+\t\t\treturn\n+\t\t}\n+\t\tclient.Go(raftMessageName, protoReq, protoResp, done)\n \t}\n }\n \n+// Send a message to the recipient specified in the request.\n+func (t *rpcTransport) Send(req *multiraft.RaftMessageRequest) error {\n+\traftNodeID := proto.RaftNodeID(req.Message.To)\n+\tt.mu.Lock()\n+\tch, ok := t.queues[raftNodeID]\n+\tif !ok {\n+\t\tch = make(chan *multiraft.RaftMessageRequest, raftSendBufferSize)\n+\t\tt.queues[raftNodeID] = ch","path":"server/raft_transport.go","position":159,"original_position":159,"commit_id":"2e36ff818ec323ca0b7a4aee026d1212c0bfbefc","original_commit_id":"2e36ff818ec323ca0b7a4aee026d1212c0bfbefc","user":{"login":"bdarnell","id":160562,"avatar_url":"https://avatars.githubusercontent.com/u/160562?v=3","gravatar_id":"","url":"https://api.github.com/users/bdarnell","html_url":"https://github.com/bdarnell","followers_url":"https://api.github.com/users/bdarnell/followers","following_url":"https://api.github.com/users/bdarnell/following{/other_user}","gists_url":"https://api.github.com/users/bdarnell/gists{/gist_id}","starred_url":"https://api.github.com/users/bdarnell/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bdarnell/subscriptions","organizations_url":"https://api.github.com/users/bdarnell/orgs","repos_url":"https://api.github.com/users/bdarnell/repos","events_url":"https://api.github.com/users/bdarnell/events{/privacy}","received_events_url":"https://api.github.com/users/bdarnell/received_events","type":"User","site_admin":false},"body":"I don't think that completely solves the race. I agree that dropping messages while a node is down is not a problem. Dropping messages when a node is on the edge of the idle timeout is not ideal but probably not worth a complicated solution. The one thing that would concern me is if there were something that would try to send messages at regular intervals that happened to coincide with the idle timeout (and might therefore hit the race condition repeatedly), but the idle timeout is long enough that it's probably not an issue.","created_at":"2015-05-21T17:39:55Z","updated_at":"2015-05-21T17:39:55Z","html_url":"https://github.com/cockroachdb/cockroach/pull/1076#discussion_r30826465","pull_request_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1076","_links":{"self":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments/30826465"},"html":{"href":"https://github.com/cockroachdb/cockroach/pull/1076#discussion_r30826465"},"pull_request":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1076"}}},"pull_request":{"url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1076","id":35840431,"html_url":"https://github.com/cockroachdb/cockroach/pull/1076","diff_url":"https://github.com/cockroachdb/cockroach/pull/1076.diff","patch_url":"https://github.com/cockroachdb/cockroach/pull/1076.patch","issue_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/1076","number":1076,"state":"open","locked":false,"title":"dispatch Raft messages in goroutine","user":{"login":"tschottdorf","id":5076964,"avatar_url":"https://avatars.githubusercontent.com/u/5076964?v=3","gravatar_id":"","url":"https://api.github.com/users/tschottdorf","html_url":"https://github.com/tschottdorf","followers_url":"https://api.github.com/users/tschottdorf/followers","following_url":"https://api.github.com/users/tschottdorf/following{/other_user}","gists_url":"https://api.github.com/users/tschottdorf/gists{/gist_id}","starred_url":"https://api.github.com/users/tschottdorf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tschottdorf/subscriptions","organizations_url":"https://api.github.com/users/tschottdorf/orgs","repos_url":"https://api.github.com/users/tschottdorf/repos","events_url":"https://api.github.com/users/tschottdorf/events{/privacy}","received_events_url":"https://api.github.com/users/tschottdorf/received_events","type":"User","site_admin":false},"body":"Prior to this change, Transport.Send() would wait for the RPC client to be\r\nready, which can block for arbitrary amounts of time, on top of there not\r\nbeing a timeout after which the call was discarded (potentially forever).\r\n\r\nfixes #1073","created_at":"2015-05-20T14:55:21Z","updated_at":"2015-05-21T17:39:55Z","closed_at":null,"merged_at":null,"merge_commit_sha":"940968ed6300edf49ea48fc025034454d669d7a6","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1076/commits","review_comments_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1076/comments","review_comment_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments{/number}","comments_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/1076/comments","statuses_url":"https://api.github.com/repos/cockroachdb/cockroach/statuses/2e36ff818ec323ca0b7a4aee026d1212c0bfbefc","head":{"label":"tschottdorf:raft_messages","ref":"raft_messages","sha":"2e36ff818ec323ca0b7a4aee026d1212c0bfbefc","user":{"login":"tschottdorf","id":5076964,"avatar_url":"https://avatars.githubusercontent.com/u/5076964?v=3","gravatar_id":"","url":"https://api.github.com/users/tschottdorf","html_url":"https://github.com/tschottdorf","followers_url":"https://api.github.com/users/tschottdorf/followers","following_url":"https://api.github.com/users/tschottdorf/following{/other_user}","gists_url":"https://api.github.com/users/tschottdorf/gists{/gist_id}","starred_url":"https://api.github.com/users/tschottdorf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tschottdorf/subscriptions","organizations_url":"https://api.github.com/users/tschottdorf/orgs","repos_url":"https://api.github.com/users/tschottdorf/repos","events_url":"https://api.github.com/users/tschottdorf/events{/privacy}","received_events_url":"https://api.github.com/users/tschottdorf/received_events","type":"User","site_admin":false},"repo":{"id":23832429,"name":"cockroach","full_name":"tschottdorf/cockroach","owner":{"login":"tschottdorf","id":5076964,"avatar_url":"https://avatars.githubusercontent.com/u/5076964?v=3","gravatar_id":"","url":"https://api.github.com/users/tschottdorf","html_url":"https://github.com/tschottdorf","followers_url":"https://api.github.com/users/tschottdorf/followers","following_url":"https://api.github.com/users/tschottdorf/following{/other_user}","gists_url":"https://api.github.com/users/tschottdorf/gists{/gist_id}","starred_url":"https://api.github.com/users/tschottdorf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tschottdorf/subscriptions","organizations_url":"https://api.github.com/users/tschottdorf/orgs","repos_url":"https://api.github.com/users/tschottdorf/repos","events_url":"https://api.github.com/users/tschottdorf/events{/privacy}","received_events_url":"https://api.github.com/users/tschottdorf/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tschottdorf/cockroach","description":"A Scalable, Geo-Replicated, Transactional Datastore","fork":true,"url":"https://api.github.com/repos/tschottdorf/cockroach","forks_url":"https://api.github.com/repos/tschottdorf/cockroach/forks","keys_url":"https://api.github.com/repos/tschottdorf/cockroach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tschottdorf/cockroach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tschottdorf/cockroach/teams","hooks_url":"https://api.github.com/repos/tschottdorf/cockroach/hooks","issue_events_url":"https://api.github.com/repos/tschottdorf/cockroach/issues/events{/number}","events_url":"https://api.github.com/repos/tschottdorf/cockroach/events","assignees_url":"https://api.github.com/repos/tschottdorf/cockroach/assignees{/user}","branches_url":"https://api.github.com/repos/tschottdorf/cockroach/branches{/branch}","tags_url":"https://api.github.com/repos/tschottdorf/cockroach/tags","blobs_url":"https://api.github.com/repos/tschottdorf/cockroach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tschottdorf/cockroach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tschottdorf/cockroach/git/refs{/sha}","trees_url":"https://api.github.com/repos/tschottdorf/cockroach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tschottdorf/cockroach/statuses/{sha}","languages_url":"https://api.github.com/repos/tschottdorf/cockroach/languages","stargazers_url":"https://api.github.com/repos/tschottdorf/cockroach/stargazers","contributors_url":"https://api.github.com/repos/tschottdorf/cockroach/contributors","subscribers_url":"https://api.github.com/repos/tschottdorf/cockroach/subscribers","subscription_url":"https://api.github.com/repos/tschottdorf/cockroach/subscription","commits_url":"https://api.github.com/repos/tschottdorf/cockroach/commits{/sha}","git_commits_url":"https://api.github.com/repos/tschottdorf/cockroach/git/commits{/sha}","comments_url":"https://api.github.com/repos/tschottdorf/cockroach/comments{/number}","issue_comment_url":"https://api.github.com/repos/tschottdorf/cockroach/issues/comments{/number}","contents_url":"https://api.github.com/repos/tschottdorf/cockroach/contents/{+path}","compare_url":"https://api.github.com/repos/tschottdorf/cockroach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tschottdorf/cockroach/merges","archive_url":"https://api.github.com/repos/tschottdorf/cockroach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tschottdorf/cockroach/downloads","issues_url":"https://api.github.com/repos/tschottdorf/cockroach/issues{/number}","pulls_url":"https://api.github.com/repos/tschottdorf/cockroach/pulls{/number}","milestones_url":"https://api.github.com/repos/tschottdorf/cockroach/milestones{/number}","notifications_url":"https://api.github.com/repos/tschottdorf/cockroach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tschottdorf/cockroach/labels{/name}","releases_url":"https://api.github.com/repos/tschottdorf/cockroach/releases{/id}","created_at":"2014-09-09T12:26:29Z","updated_at":"2015-04-30T01:27:18Z","pushed_at":"2015-05-21T15:54:30Z","git_url":"git://github.com/tschottdorf/cockroach.git","ssh_url":"git@github.com:tschottdorf/cockroach.git","clone_url":"https://github.com/tschottdorf/cockroach.git","svn_url":"https://github.com/tschottdorf/cockroach","homepage":null,"size":19191,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":16,"forks":0,"open_issues":16,"watchers":0,"default_branch":"master"}},"base":{"label":"cockroachdb:master","ref":"master","sha":"5c685efdf1457c75ad2f0b84455eef0d998970e7","user":{"login":"cockroachdb","id":6748139,"avatar_url":"https://avatars.githubusercontent.com/u/6748139?v=3","gravatar_id":"","url":"https://api.github.com/users/cockroachdb","html_url":"https://github.com/cockroachdb","followers_url":"https://api.github.com/users/cockroachdb/followers","following_url":"https://api.github.com/users/cockroachdb/following{/other_user}","gists_url":"https://api.github.com/users/cockroachdb/gists{/gist_id}","starred_url":"https://api.github.com/users/cockroachdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cockroachdb/subscriptions","organizations_url":"https://api.github.com/users/cockroachdb/orgs","repos_url":"https://api.github.com/users/cockroachdb/repos","events_url":"https://api.github.com/users/cockroachdb/events{/privacy}","received_events_url":"https://api.github.com/users/cockroachdb/received_events","type":"Organization","site_admin":false},"repo":{"id":16563587,"name":"cockroach","full_name":"cockroachdb/cockroach","owner":{"login":"cockroachdb","id":6748139,"avatar_url":"https://avatars.githubusercontent.com/u/6748139?v=3","gravatar_id":"","url":"https://api.github.com/users/cockroachdb","html_url":"https://github.com/cockroachdb","followers_url":"https://api.github.com/users/cockroachdb/followers","following_url":"https://api.github.com/users/cockroachdb/following{/other_user}","gists_url":"https://api.github.com/users/cockroachdb/gists{/gist_id}","starred_url":"https://api.github.com/users/cockroachdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cockroachdb/subscriptions","organizations_url":"https://api.github.com/users/cockroachdb/orgs","repos_url":"https://api.github.com/users/cockroachdb/repos","events_url":"https://api.github.com/users/cockroachdb/events{/privacy}","received_events_url":"https://api.github.com/users/cockroachdb/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cockroachdb/cockroach","description":"A Scalable, Geo-Replicated, Transactional Datastore","fork":false,"url":"https://api.github.com/repos/cockroachdb/cockroach","forks_url":"https://api.github.com/repos/cockroachdb/cockroach/forks","keys_url":"https://api.github.com/repos/cockroachdb/cockroach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cockroachdb/cockroach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cockroachdb/cockroach/teams","hooks_url":"https://api.github.com/repos/cockroachdb/cockroach/hooks","issue_events_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/events{/number}","events_url":"https://api.github.com/repos/cockroachdb/cockroach/events","assignees_url":"https://api.github.com/repos/cockroachdb/cockroach/assignees{/user}","branches_url":"https://api.github.com/repos/cockroachdb/cockroach/branches{/branch}","tags_url":"https://api.github.com/repos/cockroachdb/cockroach/tags","blobs_url":"https://api.github.com/repos/cockroachdb/cockroach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cockroachdb/cockroach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cockroachdb/cockroach/git/refs{/sha}","trees_url":"https://api.github.com/repos/cockroachdb/cockroach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cockroachdb/cockroach/statuses/{sha}","languages_url":"https://api.github.com/repos/cockroachdb/cockroach/languages","stargazers_url":"https://api.github.com/repos/cockroachdb/cockroach/stargazers","contributors_url":"https://api.github.com/repos/cockroachdb/cockroach/contributors","subscribers_url":"https://api.github.com/repos/cockroachdb/cockroach/subscribers","subscription_url":"https://api.github.com/repos/cockroachdb/cockroach/subscription","commits_url":"https://api.github.com/repos/cockroachdb/cockroach/commits{/sha}","git_commits_url":"https://api.github.com/repos/cockroachdb/cockroach/git/commits{/sha}","comments_url":"https://api.github.com/repos/cockroachdb/cockroach/comments{/number}","issue_comment_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/comments{/number}","contents_url":"https://api.github.com/repos/cockroachdb/cockroach/contents/{+path}","compare_url":"https://api.github.com/repos/cockroachdb/cockroach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cockroachdb/cockroach/merges","archive_url":"https://api.github.com/repos/cockroachdb/cockroach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cockroachdb/cockroach/downloads","issues_url":"https://api.github.com/repos/cockroachdb/cockroach/issues{/number}","pulls_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls{/number}","milestones_url":"https://api.github.com/repos/cockroachdb/cockroach/milestones{/number}","notifications_url":"https://api.github.com/repos/cockroachdb/cockroach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cockroachdb/cockroach/labels{/name}","releases_url":"https://api.github.com/repos/cockroachdb/cockroach/releases{/id}","created_at":"2014-02-06T00:18:47Z","updated_at":"2015-05-21T11:41:19Z","pushed_at":"2015-05-21T17:33:19Z","git_url":"git://github.com/cockroachdb/cockroach.git","ssh_url":"git@github.com:cockroachdb/cockroach.git","clone_url":"https://github.com/cockroachdb/cockroach.git","svn_url":"https://github.com/cockroachdb/cockroach","homepage":null,"size":36989,"stargazers_count":3449,"watchers_count":3449,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":277,"mirror_url":null,"open_issues_count":77,"forks":277,"open_issues":77,"watchers":3449,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1076"},"html":{"href":"https://github.com/cockroachdb/cockroach/pull/1076"},"issue":{"href":"https://api.github.com/repos/cockroachdb/cockroach/issues/1076"},"comments":{"href":"https://api.github.com/repos/cockroachdb/cockroach/issues/1076/comments"},"review_comments":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1076/comments"},"review_comment":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1076/commits"},"statuses":{"href":"https://api.github.com/repos/cockroachdb/cockroach/statuses/2e36ff818ec323ca0b7a4aee026d1212c0bfbefc"}}}}