{"action":"created","comment":{"url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments/31668606","id":31668606,"diff_hunk":"@@ -480,6 +472,131 @@ func (ds *DistSender) sendRPC(raftID int64, replicas replicaSlice,\n \treturn err\n }\n \n+// sendAttempt is invoked by Send and handles retry logic and cache eviction\n+// for a call sent to a single range.\n+func (ds *DistSender) sendAttempt(desc *proto.RangeDescriptor, call client.Call) (retry.Status, error) {\n+\tleader := ds.leaderCache.Lookup(proto.RaftID(desc.RaftID))\n+\n+\t// Try to send the call.\n+\treplicas := newReplicaSlice(ds.gossip, desc)\n+\n+\t// Rearrange the replicas so that those replicas with long common\n+\t// prefix of attributes end up first. If there's no prefix, this is a\n+\t// no-op.\n+\torder := ds.optimizeReplicaOrder(replicas)\n+\n+\targs := call.Args\n+\treply := call.Reply\n+\n+\t// If this request needs to go to a leader and we know who that is, move\n+\t// it to the front.\n+\tif !(proto.IsRead(args) && args.Header().ReadConsistency == proto.INCONSISTENT) &&\n+\t\tleader.StoreID > 0 {\n+\t\tif i := replicas.FindReplica(leader.StoreID); i >= 0 {\n+\t\t\treplicas.MoveToFront(i)\n+\t\t\torder = rpc.OrderStable\n+\t\t}\n+\t}\n+\n+\trpcErr := ds.sendRPC(desc.RaftID, replicas, order, args, reply)\n+\n+\t// For an RPC error to occur, we must've been unable to contact any\n+\t// replicas. In this case, likely all nodes are down (or not getting back\n+\t// to us within a reasonable amount of time).\n+\t// We may simply not be trying to talk to the up-to-date replicas, so\n+\t// clearing the descriptor here should be a good idea.\n+\t// TODO(tschottdorf): If a replica group goes dead, this will cause clients\n+\t// to put high read pressure on the first range, so there should be some\n+\t// rate limiting here.\n+\tif rpcErr != nil {\n+\t\tds.rangeCache.EvictCachedRangeDescriptor(args.Header().Key, desc)\n+\t}\n+\n+\tif err := util.FirstError(rpcErr, reply.Header().GoError()); err != nil {","path":"kv/dist_sender.go","position":145,"original_position":145,"commit_id":"36763e303525b4f8c43c39d73e456eb9592d9482","original_commit_id":"36763e303525b4f8c43c39d73e456eb9592d9482","user":{"login":"tschottdorf","id":5076964,"avatar_url":"https://avatars.githubusercontent.com/u/5076964?v=3","gravatar_id":"","url":"https://api.github.com/users/tschottdorf","html_url":"https://github.com/tschottdorf","followers_url":"https://api.github.com/users/tschottdorf/followers","following_url":"https://api.github.com/users/tschottdorf/following{/other_user}","gists_url":"https://api.github.com/users/tschottdorf/gists{/gist_id}","starred_url":"https://api.github.com/users/tschottdorf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tschottdorf/subscriptions","organizations_url":"https://api.github.com/users/tschottdorf/orgs","repos_url":"https://api.github.com/users/tschottdorf/repos","events_url":"https://api.github.com/users/tschottdorf/events{/privacy}","received_events_url":"https://api.github.com/users/tschottdorf/received_events","type":"User","site_admin":false},"body":"Ok, I see what you mean. We just want to treat an RPC error like any other for simplicity (see the `default` case in the select below).","created_at":"2015-06-03T20:53:13Z","updated_at":"2015-06-03T20:53:13Z","html_url":"https://github.com/cockroachdb/cockroach/pull/1248#discussion_r31668606","pull_request_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1248","_links":{"self":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments/31668606"},"html":{"href":"https://github.com/cockroachdb/cockroach/pull/1248#discussion_r31668606"},"pull_request":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1248"}}},"pull_request":{"url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1248","id":36858749,"html_url":"https://github.com/cockroachdb/cockroach/pull/1248","diff_url":"https://github.com/cockroachdb/cockroach/pull/1248.diff","patch_url":"https://github.com/cockroachdb/cockroach/pull/1248.patch","issue_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/1248","number":1248,"state":"open","locked":false,"title":"break down DistSender.Send","user":{"login":"tschottdorf","id":5076964,"avatar_url":"https://avatars.githubusercontent.com/u/5076964?v=3","gravatar_id":"","url":"https://api.github.com/users/tschottdorf","html_url":"https://github.com/tschottdorf","followers_url":"https://api.github.com/users/tschottdorf/followers","following_url":"https://api.github.com/users/tschottdorf/following{/other_user}","gists_url":"https://api.github.com/users/tschottdorf/gists{/gist_id}","starred_url":"https://api.github.com/users/tschottdorf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tschottdorf/subscriptions","organizations_url":"https://api.github.com/users/tschottdorf/orgs","repos_url":"https://api.github.com/users/tschottdorf/repos","events_url":"https://api.github.com/users/tschottdorf/events{/privacy}","received_events_url":"https://api.github.com/users/tschottdorf/received_events","type":"User","site_admin":false},"body":"- remove the mechanism for removing a cached leader upon\r\n  caching the same leader twice in a row: silly idea, obviously\r\n  concurrent request going to the wrong leader will return\r\n  almost simultaneously and expire the freshly updated cache.\r\n- the leaderCache now only deals with proto.Replica{}, not pointers.\r\n- pretty complete refactoring of DistSender.Send to get more manageable\r\n  pieces and less error handling spagetthi.\r\n- atomic updates in ds.getNodeDescriptors","created_at":"2015-06-03T17:48:38Z","updated_at":"2015-06-03T20:53:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c8a42bf40e4b2829cd3e89e20036e8406a565b39","assignee":{"login":"spencerkimball","id":6147149,"avatar_url":"https://avatars.githubusercontent.com/u/6147149?v=3","gravatar_id":"","url":"https://api.github.com/users/spencerkimball","html_url":"https://github.com/spencerkimball","followers_url":"https://api.github.com/users/spencerkimball/followers","following_url":"https://api.github.com/users/spencerkimball/following{/other_user}","gists_url":"https://api.github.com/users/spencerkimball/gists{/gist_id}","starred_url":"https://api.github.com/users/spencerkimball/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spencerkimball/subscriptions","organizations_url":"https://api.github.com/users/spencerkimball/orgs","repos_url":"https://api.github.com/users/spencerkimball/repos","events_url":"https://api.github.com/users/spencerkimball/events{/privacy}","received_events_url":"https://api.github.com/users/spencerkimball/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1248/commits","review_comments_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1248/comments","review_comment_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments{/number}","comments_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/1248/comments","statuses_url":"https://api.github.com/repos/cockroachdb/cockroach/statuses/36763e303525b4f8c43c39d73e456eb9592d9482","head":{"label":"tschottdorf:distsender_refactor","ref":"distsender_refactor","sha":"36763e303525b4f8c43c39d73e456eb9592d9482","user":{"login":"tschottdorf","id":5076964,"avatar_url":"https://avatars.githubusercontent.com/u/5076964?v=3","gravatar_id":"","url":"https://api.github.com/users/tschottdorf","html_url":"https://github.com/tschottdorf","followers_url":"https://api.github.com/users/tschottdorf/followers","following_url":"https://api.github.com/users/tschottdorf/following{/other_user}","gists_url":"https://api.github.com/users/tschottdorf/gists{/gist_id}","starred_url":"https://api.github.com/users/tschottdorf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tschottdorf/subscriptions","organizations_url":"https://api.github.com/users/tschottdorf/orgs","repos_url":"https://api.github.com/users/tschottdorf/repos","events_url":"https://api.github.com/users/tschottdorf/events{/privacy}","received_events_url":"https://api.github.com/users/tschottdorf/received_events","type":"User","site_admin":false},"repo":{"id":23832429,"name":"cockroach","full_name":"tschottdorf/cockroach","owner":{"login":"tschottdorf","id":5076964,"avatar_url":"https://avatars.githubusercontent.com/u/5076964?v=3","gravatar_id":"","url":"https://api.github.com/users/tschottdorf","html_url":"https://github.com/tschottdorf","followers_url":"https://api.github.com/users/tschottdorf/followers","following_url":"https://api.github.com/users/tschottdorf/following{/other_user}","gists_url":"https://api.github.com/users/tschottdorf/gists{/gist_id}","starred_url":"https://api.github.com/users/tschottdorf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tschottdorf/subscriptions","organizations_url":"https://api.github.com/users/tschottdorf/orgs","repos_url":"https://api.github.com/users/tschottdorf/repos","events_url":"https://api.github.com/users/tschottdorf/events{/privacy}","received_events_url":"https://api.github.com/users/tschottdorf/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tschottdorf/cockroach","description":"A Scalable, Geo-Replicated, Transactional Datastore","fork":true,"url":"https://api.github.com/repos/tschottdorf/cockroach","forks_url":"https://api.github.com/repos/tschottdorf/cockroach/forks","keys_url":"https://api.github.com/repos/tschottdorf/cockroach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tschottdorf/cockroach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tschottdorf/cockroach/teams","hooks_url":"https://api.github.com/repos/tschottdorf/cockroach/hooks","issue_events_url":"https://api.github.com/repos/tschottdorf/cockroach/issues/events{/number}","events_url":"https://api.github.com/repos/tschottdorf/cockroach/events","assignees_url":"https://api.github.com/repos/tschottdorf/cockroach/assignees{/user}","branches_url":"https://api.github.com/repos/tschottdorf/cockroach/branches{/branch}","tags_url":"https://api.github.com/repos/tschottdorf/cockroach/tags","blobs_url":"https://api.github.com/repos/tschottdorf/cockroach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tschottdorf/cockroach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tschottdorf/cockroach/git/refs{/sha}","trees_url":"https://api.github.com/repos/tschottdorf/cockroach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tschottdorf/cockroach/statuses/{sha}","languages_url":"https://api.github.com/repos/tschottdorf/cockroach/languages","stargazers_url":"https://api.github.com/repos/tschottdorf/cockroach/stargazers","contributors_url":"https://api.github.com/repos/tschottdorf/cockroach/contributors","subscribers_url":"https://api.github.com/repos/tschottdorf/cockroach/subscribers","subscription_url":"https://api.github.com/repos/tschottdorf/cockroach/subscription","commits_url":"https://api.github.com/repos/tschottdorf/cockroach/commits{/sha}","git_commits_url":"https://api.github.com/repos/tschottdorf/cockroach/git/commits{/sha}","comments_url":"https://api.github.com/repos/tschottdorf/cockroach/comments{/number}","issue_comment_url":"https://api.github.com/repos/tschottdorf/cockroach/issues/comments{/number}","contents_url":"https://api.github.com/repos/tschottdorf/cockroach/contents/{+path}","compare_url":"https://api.github.com/repos/tschottdorf/cockroach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tschottdorf/cockroach/merges","archive_url":"https://api.github.com/repos/tschottdorf/cockroach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tschottdorf/cockroach/downloads","issues_url":"https://api.github.com/repos/tschottdorf/cockroach/issues{/number}","pulls_url":"https://api.github.com/repos/tschottdorf/cockroach/pulls{/number}","milestones_url":"https://api.github.com/repos/tschottdorf/cockroach/milestones{/number}","notifications_url":"https://api.github.com/repos/tschottdorf/cockroach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tschottdorf/cockroach/labels{/name}","releases_url":"https://api.github.com/repos/tschottdorf/cockroach/releases{/id}","created_at":"2014-09-09T12:26:29Z","updated_at":"2015-04-30T01:27:18Z","pushed_at":"2015-06-03T20:31:21Z","git_url":"git://github.com/tschottdorf/cockroach.git","ssh_url":"git@github.com:tschottdorf/cockroach.git","clone_url":"https://github.com/tschottdorf/cockroach.git","svn_url":"https://github.com/tschottdorf/cockroach","homepage":null,"size":21423,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":16,"forks":0,"open_issues":16,"watchers":0,"default_branch":"master"}},"base":{"label":"cockroachdb:master","ref":"master","sha":"702628c059031be22cee9fbf7201917cd60583be","user":{"login":"cockroachdb","id":6748139,"avatar_url":"https://avatars.githubusercontent.com/u/6748139?v=3","gravatar_id":"","url":"https://api.github.com/users/cockroachdb","html_url":"https://github.com/cockroachdb","followers_url":"https://api.github.com/users/cockroachdb/followers","following_url":"https://api.github.com/users/cockroachdb/following{/other_user}","gists_url":"https://api.github.com/users/cockroachdb/gists{/gist_id}","starred_url":"https://api.github.com/users/cockroachdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cockroachdb/subscriptions","organizations_url":"https://api.github.com/users/cockroachdb/orgs","repos_url":"https://api.github.com/users/cockroachdb/repos","events_url":"https://api.github.com/users/cockroachdb/events{/privacy}","received_events_url":"https://api.github.com/users/cockroachdb/received_events","type":"Organization","site_admin":false},"repo":{"id":16563587,"name":"cockroach","full_name":"cockroachdb/cockroach","owner":{"login":"cockroachdb","id":6748139,"avatar_url":"https://avatars.githubusercontent.com/u/6748139?v=3","gravatar_id":"","url":"https://api.github.com/users/cockroachdb","html_url":"https://github.com/cockroachdb","followers_url":"https://api.github.com/users/cockroachdb/followers","following_url":"https://api.github.com/users/cockroachdb/following{/other_user}","gists_url":"https://api.github.com/users/cockroachdb/gists{/gist_id}","starred_url":"https://api.github.com/users/cockroachdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cockroachdb/subscriptions","organizations_url":"https://api.github.com/users/cockroachdb/orgs","repos_url":"https://api.github.com/users/cockroachdb/repos","events_url":"https://api.github.com/users/cockroachdb/events{/privacy}","received_events_url":"https://api.github.com/users/cockroachdb/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cockroachdb/cockroach","description":"A Scalable, Geo-Replicated, Transactional Datastore","fork":false,"url":"https://api.github.com/repos/cockroachdb/cockroach","forks_url":"https://api.github.com/repos/cockroachdb/cockroach/forks","keys_url":"https://api.github.com/repos/cockroachdb/cockroach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cockroachdb/cockroach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cockroachdb/cockroach/teams","hooks_url":"https://api.github.com/repos/cockroachdb/cockroach/hooks","issue_events_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/events{/number}","events_url":"https://api.github.com/repos/cockroachdb/cockroach/events","assignees_url":"https://api.github.com/repos/cockroachdb/cockroach/assignees{/user}","branches_url":"https://api.github.com/repos/cockroachdb/cockroach/branches{/branch}","tags_url":"https://api.github.com/repos/cockroachdb/cockroach/tags","blobs_url":"https://api.github.com/repos/cockroachdb/cockroach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cockroachdb/cockroach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cockroachdb/cockroach/git/refs{/sha}","trees_url":"https://api.github.com/repos/cockroachdb/cockroach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cockroachdb/cockroach/statuses/{sha}","languages_url":"https://api.github.com/repos/cockroachdb/cockroach/languages","stargazers_url":"https://api.github.com/repos/cockroachdb/cockroach/stargazers","contributors_url":"https://api.github.com/repos/cockroachdb/cockroach/contributors","subscribers_url":"https://api.github.com/repos/cockroachdb/cockroach/subscribers","subscription_url":"https://api.github.com/repos/cockroachdb/cockroach/subscription","commits_url":"https://api.github.com/repos/cockroachdb/cockroach/commits{/sha}","git_commits_url":"https://api.github.com/repos/cockroachdb/cockroach/git/commits{/sha}","comments_url":"https://api.github.com/repos/cockroachdb/cockroach/comments{/number}","issue_comment_url":"https://api.github.com/repos/cockroachdb/cockroach/issues/comments{/number}","contents_url":"https://api.github.com/repos/cockroachdb/cockroach/contents/{+path}","compare_url":"https://api.github.com/repos/cockroachdb/cockroach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cockroachdb/cockroach/merges","archive_url":"https://api.github.com/repos/cockroachdb/cockroach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cockroachdb/cockroach/downloads","issues_url":"https://api.github.com/repos/cockroachdb/cockroach/issues{/number}","pulls_url":"https://api.github.com/repos/cockroachdb/cockroach/pulls{/number}","milestones_url":"https://api.github.com/repos/cockroachdb/cockroach/milestones{/number}","notifications_url":"https://api.github.com/repos/cockroachdb/cockroach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cockroachdb/cockroach/labels{/name}","releases_url":"https://api.github.com/repos/cockroachdb/cockroach/releases{/id}","created_at":"2014-02-06T00:18:47Z","updated_at":"2015-06-03T20:30:05Z","pushed_at":"2015-06-03T20:45:12Z","git_url":"git://github.com/cockroachdb/cockroach.git","ssh_url":"git@github.com:cockroachdb/cockroach.git","clone_url":"https://github.com/cockroachdb/cockroach.git","svn_url":"https://github.com/cockroachdb/cockroach","homepage":null,"size":42808,"stargazers_count":3480,"watchers_count":3480,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":278,"mirror_url":null,"open_issues_count":79,"forks":278,"open_issues":79,"watchers":3480,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1248"},"html":{"href":"https://github.com/cockroachdb/cockroach/pull/1248"},"issue":{"href":"https://api.github.com/repos/cockroachdb/cockroach/issues/1248"},"comments":{"href":"https://api.github.com/repos/cockroachdb/cockroach/issues/1248/comments"},"review_comments":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1248/comments"},"review_comment":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/cockroachdb/cockroach/pulls/1248/commits"},"statuses":{"href":"https://api.github.com/repos/cockroachdb/cockroach/statuses/36763e303525b4f8c43c39d73e456eb9592d9482"}}}}