{"action":"created","comment":{"url":"https://api.github.com/repos/basho/riak_kv/pulls/comments/22899450","id":22899450,"diff_hunk":"@@ -2322,102 +2346,286 @@ new_md_cache(VId) ->\n     %% term format is {TimeStamp, Key, ValueTuple}\n     ets:new(MDCacheName, [ordered_set, {keypos,2}]).\n \n+%% @private increment the per vnode coordinating put counter,\n+%% flushing/leasing if needed\n+-spec update_counter(state()) -> state().\n+update_counter(State=#state{counter=CounterState}) ->\n+    #counter_state{cnt=Counter0} = CounterState,\n+    Counter = Counter0 +  1,\n+    maybe_lease_counter(State#state{counter=CounterState#counter_state{cnt=Counter}}).\n+\n+\n+%% @private we can never use a counter that is greater or equal to the\n+%% one fysnced to disk. If the incremented counter is == to the\n+%% current Lease, we must block until the new Lease is stored. If the\n+%% current counter is 80% through the current lease, and we have not\n+%% asked for a new lease, ask for one.  If we're less than 80% through\n+%% the lease, do nothing.  If not yet blocking(equal) and we already\n+%% asked for a new lease, do nothing.\n+maybe_lease_counter(State=#state{counter=#counter_state{cnt=Lease, lease=Lease,\n+                                                        leasing=true}}) ->\n+    %% Block until we get a new lease, or crash the vnode\n+    {ok, NewState} = blocking_lease_counter(State),\n+    NewState;\n+maybe_lease_counter(State=#state{counter=#counter_state{leasing=true}}) ->\n+    %% not yet at the blocking stage, waiting on a lease\n+    State;\n+maybe_lease_counter(State) ->\n+    #state{status_mgr_pid=MgrPid, counter=CS=#counter_state{cnt=Cnt, lease=Lease,\n+                                                            lease_size=LeaseSize}} = State,\n+    %% @TODO (rdb) configurable??\n+    %% has more than 80% of the lease been used?\n+    CS2 = if (Lease - Cnt) =< 0.2 * LeaseSize  ->\n+                  ok = riak_kv_vnode_status_mgr:lease_counter(MgrPid, LeaseSize),\n+                  CS#counter_state{leasing=true};\n+             ?ELSE ->\n+                  CS\n+          end,\n+    State#state{counter=CS2}.\n+\n+%% @private by now, we have to be waiting for a lease, or an exit,\n+%% from the mgr. Block until we receive a lease. If we get an exit,\n+%% retry a number of times. If we get neither in a reasonable time,\n+%% return an error.\n+-spec blocking_lease_counter(#state{}) ->\n+                                    {ok, #state{}} |\n+                                    counter_lease_error().\n+blocking_lease_counter(State) ->\n+    {MaxErrs, MaxTime} = get_counter_wait_values(),\n+    blocking_lease_counter(State, {0, MaxErrs, MaxTime}).\n+\n+-spec blocking_lease_counter(#state{}, {Errors :: non_neg_integer(),\n+                                        MaxErrors :: non_neg_integer(),\n+                                        TimeRemainingMillis :: non_neg_integer()}\n+                            ) ->\n+                                    {ok, #state{}} |\n+                                    counter_lease_error().\n+blocking_lease_counter(_State, {MaxErrs, MaxErrs, _MaxTime}) ->\n+    {error, counter_lease_max_errors};\n+blocking_lease_counter(State, {ErrCnt, MaxErrors, MaxTime}) ->\n+    #state{idx=Index, vnodeid=VId, status_mgr_pid=Pid, counter=CounterState} = State,\n+    #counter_state{lease_size=LeaseSize} = CounterState,\n+    Start = os:timestamp(),\n+    receive\n+        {'EXIT', Pid, Reason} ->\n+            lager:error(\"Failed to lease counter ~p\", [Reason]),\n+            {ok, NewPid} = riak_kv_vnode_status_mgr:start_link(self(), Index),\n+            ok = riak_kv_vnode_status_mgr:lease_counter(NewPid, LeaseSize),\n+            NewState = State#state{status_mgr_pid=NewPid},\n+            Elapsed = timer:now_diff(os:timestamp(), Start),\n+            blocking_lease_counter(NewState, {ErrCnt+1, MaxErrors, MaxTime - Elapsed});\n+        {counter_lease, {Pid, VId, NewLease}} ->\n+            NewCS = CounterState#counter_state{lease=NewLease, leasing=false},\n+            {ok, State#state{counter=NewCS}};\n+        {counter_lease, {Pid, NewVId, NewLease}} ->\n+            lager:info(\"New Vnode id for ~p. Epoch counter rolled over.\", [Index]),\n+            NewCS = CounterState#counter_state{lease=NewLease, leasing=false, cnt=1},\n+            {ok, State#state{vnodeid=NewVId, counter=NewCS}}\n+    after\n+        MaxTime ->\n+            {error, counter_lease_timeout}\n+    end.\n+\n+%% @private get the configured values for blocking waiting on\n+%% lease/ID. Ensure that non invalid values come in.\n+-spec get_counter_wait_values() ->\n+                                     {MaxErrors :: non_neg_integer(),\n+                                      MaxTime :: non_neg_integer()}.\n+get_counter_wait_values() ->\n+    MaxErrors = non_neg_env(riak_kv, counter_lease_errors, ?DEFAULT_CNTR_LEASE_ERRS),\n+    MaxTime = non_neg_env(riak_kv, counter_lease_timeout, ?DEFAULT_CNTR_LEASE_TO),\n+    {MaxErrors, MaxTime}.\n+\n+%% @private we don't want to crash riak because of a dodgy config\n+%% value, and by this point, cuttlefish be praised, we should have\n+%% sane values. However, if not, ignore negative values for timeout\n+%% and max errors, use the defaults, and log the insanity. Note this\n+%% expects the macro configured defaults to be positive integers!\n+-spec non_neg_env(atom(), atom(), pos_integer()) -> pos_integer().\n+non_neg_env(App, EnvVar, Default) when is_integer(Default),\n+                                       Default > 0 ->\n+    case app_helper:get_env(App, EnvVar, Default) of\n+        N when is_integer(N),\n+               N > 0 ->\n+            N;\n+        X ->\n+            lager:warning(\"Non-integer/Negative integer ~p for vnode counter config ~p.\"\n+                          \" Using default ~p\",\n+                          [X, EnvVar, Default]),\n+            Default\n+    end.\n+\n+%% @private to keep put_merge/6 side effect free (as it is exported\n+%% for testing) introspect the state and local/incoming objects and\n+%% return the actor ID for a put, and possibly updated state. NOTE\n+%% side effects, in that the vnode status on disk can change.\n+%% Why might we need a new key epoch?\n+%% 1. local not found (handled in prepare_put/3\n+%% 2. local found, but never acted on this key before (or don't remember it!)\n+%% 3. local found, local acted, but incoming has greater count or actor epoch\n+%%    This one is tricky, since it indicates some byzantine failure somewhere.\n+-spec maybe_new_key_epoch(boolean(), #state{},\n+                          riak_object:riak_object(),\n+                          riak_object:riak_object()) ->\n+                                 {binary(), #state{}}.\n+maybe_new_key_epoch(false, State, _, _) ->\n+    %% Never add a new key epoch when not coordinating\n+    %% @TODO (rdb) need to mark actor as dirty though.\n+    {State#state.vnodeid, State};\n+maybe_new_key_epoch(true, State, LocalObj, IncomingObj) ->\n+    #state{vnodeid=VId} = State,\n+    %% @TODO (rdb) maybe optimise since highly likey both objects\n+    %% share the majority of actors, maybe a single umerged list of\n+    %% actors, somehow tagged by local | incoming?\n+    case highest_actor(VId, LocalObj) of\n+        {undefined, 0, 0} -> %% Not present locally\n+            %% Never acted on this object before, new epoch.\n+            new_key_epoch(State);\n+        {LocalId, LocalEpoch, LocalCntr} -> %% Present locally\n+            case highest_actor(VId, IncomingObj) of\n+                {_InId, InEpoch, InCntr} when InEpoch > LocalEpoch;\n+                                              InCntr > LocalCntr ->\n+                    %% In coming actor-epoch or counter greater than\n+                    %% local, some byzantine failure, new epoch.\n+                    %%\n+                    %% @TODO (rdb) do we log/stat here?","path":"src/riak_kv_vnode.erl","position":616,"original_position":616,"commit_id":"20d2bd0aea23419ff91be1c6633d31caa2c56562","original_commit_id":"20d2bd0aea23419ff91be1c6633d31caa2c56562","user":{"login":"russelldb","id":56321,"avatar_url":"https://avatars.githubusercontent.com/u/56321?v=3","gravatar_id":"","url":"https://api.github.com/users/russelldb","html_url":"https://github.com/russelldb","followers_url":"https://api.github.com/users/russelldb/followers","following_url":"https://api.github.com/users/russelldb/following{/other_user}","gists_url":"https://api.github.com/users/russelldb/gists{/gist_id}","starred_url":"https://api.github.com/users/russelldb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/russelldb/subscriptions","organizations_url":"https://api.github.com/users/russelldb/orgs","repos_url":"https://api.github.com/users/russelldb/repos","events_url":"https://api.github.com/users/russelldb/events{/privacy}","received_events_url":"https://api.github.com/users/russelldb/received_events","type":"User","site_admin":false},"body":"I’m not sure about “Late\" and \"Old message\"?\r\n\r\nThe vnode has sent a lease message to the manager. It has yet to receive an answer so it enters this blocking receive. What is the old message? Do you mean if we timeout and crash the vnode? There is no old message as the message queue dies with the crashed vnode.\r\n\r\nIs this the sequence you’re thinking of that leads to there being an “old message” for the lease\r\n\r\n1. vnode asks for a lease\r\n2. manager sends a lease message\r\n3. manager crashes\r\n4. vnode in blocking lease receive gets the exit message first (‘cos of the order of receives in statement)\r\n5. vnode starts a new manager and requests a lease\r\n6. vnode pulls the original message (pre crash)\r\n\r\nIs that step 6 the “old message?”\r\n\r\nI don’t think it can happen. The `Pid` in the receive is the new managers `Pid`, the old one will never match. Even if it would I think it is safe. The vnode is getting a new lease! Yay! It can keep working. The lease it gets is smaller than that on disk, yay! Invariant preserved. I assume then there is a step 7\r\n\r\n7. vnode gets new lease message\r\n\r\nAll is still OK. We just have a larger lease/longer until we re-lease. If the vnode crashes it just wasted some integers.\r\n\r\nDoes that seem OK to you? (This is the first time I have thought it through, so I think it was a valid question, if I answered the question you asked, that is.)\r\n\r\nI don’t think there is any way for the vnode to request a new lease before it receives the old lease, and Erlang delivers messages between processes on the same node in order, so they’ll be received in order, meaning the lease always increases. Happy to mumble about it in the morning to be sure, though.\r\n\r\n> On 13 Jan 2015, at 20:55, Andrew J. Stone <notifications@github.com> wrote:\r\n> \r\n> In src/riak_kv_vnode.erl:\r\n> \r\n> > +    #state{vnodeid=VId} = State,\r\n> > +    %% @TODO (rdb) maybe optimise since highly likey both objects\r\n> > +    %% share the majority of actors, maybe a single umerged list of\r\n> > +    %% actors, somehow tagged by local | incoming?\r\n> > +    case highest_actor(VId, LocalObj) of\r\n> > +        {undefined, 0, 0} -> %% Not present locally\r\n> > +            %% Never acted on this object before, new epoch.\r\n> > +            new_key_epoch(State);\r\n> > +        {LocalId, LocalEpoch, LocalCntr} -> %% Present locally\r\n> > +            case highest_actor(VId, IncomingObj) of\r\n> > +                {_InId, InEpoch, InCntr} when InEpoch > LocalEpoch;\r\n> > +                                              InCntr > LocalCntr ->\r\n> > +                    %% In coming actor-epoch or counter greater than\r\n> > +                    %% local, some byzantine failure, new epoch.\r\n> > +                    %%\r\n> > +                    %% @TODO (rdb) do we log/stat here?\r\n> \r\n> I would add at least a warning log here. Either there is a bug in the backend code or the disk is losing things.\r\n> \r\n> —\r\n> Reply to this email directly or view it on GitHub.\r\n> ","created_at":"2015-01-13T22:08:17Z","updated_at":"2015-01-13T22:08:17Z","html_url":"https://github.com/basho/riak_kv/pull/1070#discussion_r22899450","pull_request_url":"https://api.github.com/repos/basho/riak_kv/pulls/1070","_links":{"self":{"href":"https://api.github.com/repos/basho/riak_kv/pulls/comments/22899450"},"html":{"href":"https://github.com/basho/riak_kv/pull/1070#discussion_r22899450"},"pull_request":{"href":"https://api.github.com/repos/basho/riak_kv/pulls/1070"}}},"pull_request":{"url":"https://api.github.com/repos/basho/riak_kv/pulls/1070","id":26693860,"html_url":"https://github.com/basho/riak_kv/pull/1070","diff_url":"https://github.com/basho/riak_kv/pull/1070.diff","patch_url":"https://github.com/basho/riak_kv/pull/1070.patch","issue_url":"https://api.github.com/repos/basho/riak_kv/issues/1070","number":1070,"state":"open","locked":false,"title":"Per-key actor epochs for vnode vclock entries","user":{"login":"russelldb","id":56321,"avatar_url":"https://avatars.githubusercontent.com/u/56321?v=3","gravatar_id":"","url":"https://api.github.com/users/russelldb","html_url":"https://github.com/russelldb","followers_url":"https://api.github.com/users/russelldb/followers","following_url":"https://api.github.com/users/russelldb/following{/other_user}","gists_url":"https://api.github.com/users/russelldb/gists{/gist_id}","starred_url":"https://api.github.com/users/russelldb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/russelldb/subscriptions","organizations_url":"https://api.github.com/users/russelldb/orgs","repos_url":"https://api.github.com/users/russelldb/repos","events_url":"https://api.github.com/users/russelldb/events{/privacy}","received_events_url":"https://api.github.com/users/russelldb/received_events","type":"User","site_admin":false},"body":"See basho/riak_kv#679 and the associated platform_task RFC and summary.\r\n\r\nThis PR addresses the \"doomstone\", backup-restore, and some byzantine flavours of the kv679 bug.\r\n\r\nThe RFC explains the mechanism in detail but briefly:\r\n\r\nadd a persisted to disk vnode counter (persisted with leases, aysnc)\r\nwhen ever a key is written to for the \"first time\" by a vnode, create an epoch actor for the key by concatenating vnodeid+counter (and increment the counter)\r\nThis ensures that a first time write for a key gets a new actor, this is the epoch for the key. It means we don't mix up deleted+re-created keys {a,1} event with the original {a, 1} event for some key, by ensuring an actor per epoch, without causing a keyspace wide actor explosion.\r\n\r\nThere are riak_tests at https://github.com/basho/riak_test/tree/rdb/gh-kv679\r\n\r\n-----\r\n\r\nThis has been long running work, there is a PR here basho/riak_kv#1040 and here basho/riak_kv/pull/1053 that are closed in favour of this.\r\n\r\n\r\n-----\r\n\r\n- [ ] review\r\n- [ ] benchmarking\r\n- [ ] riak_test review\r\n- [ ] add Kelly's EQC test","created_at":"2014-12-30T17:11:50Z","updated_at":"2015-01-13T22:08:17Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0f83f860ee1819ea04c3f3e807b36a07de7244f8","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/basho/riak_kv/pulls/1070/commits","review_comments_url":"https://api.github.com/repos/basho/riak_kv/pulls/1070/comments","review_comment_url":"https://api.github.com/repos/basho/riak_kv/pulls/comments/{number}","comments_url":"https://api.github.com/repos/basho/riak_kv/issues/1070/comments","statuses_url":"https://api.github.com/repos/basho/riak_kv/statuses/20d2bd0aea23419ff91be1c6633d31caa2c56562","head":{"label":"basho:bug/rdb/gh679-actor-epochs-re-rebase2.0","ref":"bug/rdb/gh679-actor-epochs-re-rebase2.0","sha":"20d2bd0aea23419ff91be1c6633d31caa2c56562","user":{"login":"basho","id":176293,"avatar_url":"https://avatars.githubusercontent.com/u/176293?v=3","gravatar_id":"","url":"https://api.github.com/users/basho","html_url":"https://github.com/basho","followers_url":"https://api.github.com/users/basho/followers","following_url":"https://api.github.com/users/basho/following{/other_user}","gists_url":"https://api.github.com/users/basho/gists{/gist_id}","starred_url":"https://api.github.com/users/basho/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/basho/subscriptions","organizations_url":"https://api.github.com/users/basho/orgs","repos_url":"https://api.github.com/users/basho/repos","events_url":"https://api.github.com/users/basho/events{/privacy}","received_events_url":"https://api.github.com/users/basho/received_events","type":"Organization","site_admin":false},"repo":{"id":791522,"name":"riak_kv","full_name":"basho/riak_kv","owner":{"login":"basho","id":176293,"avatar_url":"https://avatars.githubusercontent.com/u/176293?v=3","gravatar_id":"","url":"https://api.github.com/users/basho","html_url":"https://github.com/basho","followers_url":"https://api.github.com/users/basho/followers","following_url":"https://api.github.com/users/basho/following{/other_user}","gists_url":"https://api.github.com/users/basho/gists{/gist_id}","starred_url":"https://api.github.com/users/basho/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/basho/subscriptions","organizations_url":"https://api.github.com/users/basho/orgs","repos_url":"https://api.github.com/users/basho/repos","events_url":"https://api.github.com/users/basho/events{/privacy}","received_events_url":"https://api.github.com/users/basho/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/basho/riak_kv","description":"Riak Key/Value Store","fork":false,"url":"https://api.github.com/repos/basho/riak_kv","forks_url":"https://api.github.com/repos/basho/riak_kv/forks","keys_url":"https://api.github.com/repos/basho/riak_kv/keys{/key_id}","collaborators_url":"https://api.github.com/repos/basho/riak_kv/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/basho/riak_kv/teams","hooks_url":"https://api.github.com/repos/basho/riak_kv/hooks","issue_events_url":"https://api.github.com/repos/basho/riak_kv/issues/events{/number}","events_url":"https://api.github.com/repos/basho/riak_kv/events","assignees_url":"https://api.github.com/repos/basho/riak_kv/assignees{/user}","branches_url":"https://api.github.com/repos/basho/riak_kv/branches{/branch}","tags_url":"https://api.github.com/repos/basho/riak_kv/tags","blobs_url":"https://api.github.com/repos/basho/riak_kv/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/basho/riak_kv/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/basho/riak_kv/git/refs{/sha}","trees_url":"https://api.github.com/repos/basho/riak_kv/git/trees{/sha}","statuses_url":"https://api.github.com/repos/basho/riak_kv/statuses/{sha}","languages_url":"https://api.github.com/repos/basho/riak_kv/languages","stargazers_url":"https://api.github.com/repos/basho/riak_kv/stargazers","contributors_url":"https://api.github.com/repos/basho/riak_kv/contributors","subscribers_url":"https://api.github.com/repos/basho/riak_kv/subscribers","subscription_url":"https://api.github.com/repos/basho/riak_kv/subscription","commits_url":"https://api.github.com/repos/basho/riak_kv/commits{/sha}","git_commits_url":"https://api.github.com/repos/basho/riak_kv/git/commits{/sha}","comments_url":"https://api.github.com/repos/basho/riak_kv/comments{/number}","issue_comment_url":"https://api.github.com/repos/basho/riak_kv/issues/comments/{number}","contents_url":"https://api.github.com/repos/basho/riak_kv/contents/{+path}","compare_url":"https://api.github.com/repos/basho/riak_kv/compare/{base}...{head}","merges_url":"https://api.github.com/repos/basho/riak_kv/merges","archive_url":"https://api.github.com/repos/basho/riak_kv/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/basho/riak_kv/downloads","issues_url":"https://api.github.com/repos/basho/riak_kv/issues{/number}","pulls_url":"https://api.github.com/repos/basho/riak_kv/pulls{/number}","milestones_url":"https://api.github.com/repos/basho/riak_kv/milestones{/number}","notifications_url":"https://api.github.com/repos/basho/riak_kv/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/basho/riak_kv/labels{/name}","releases_url":"https://api.github.com/repos/basho/riak_kv/releases{/id}","created_at":"2010-07-22T17:27:35Z","updated_at":"2014-12-25T09:07:08Z","pushed_at":"2015-01-12T11:35:39Z","git_url":"git://github.com/basho/riak_kv.git","ssh_url":"git@github.com:basho/riak_kv.git","clone_url":"https://github.com/basho/riak_kv.git","svn_url":"https://github.com/basho/riak_kv","homepage":"","size":22529,"stargazers_count":226,"watchers_count":226,"language":"Erlang","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":112,"mirror_url":null,"open_issues_count":129,"forks":112,"open_issues":129,"watchers":226,"default_branch":"develop"}},"base":{"label":"basho:2.0","ref":"2.0","sha":"6823fed40df740e07f678bd0c7b75f29ced74478","user":{"login":"basho","id":176293,"avatar_url":"https://avatars.githubusercontent.com/u/176293?v=3","gravatar_id":"","url":"https://api.github.com/users/basho","html_url":"https://github.com/basho","followers_url":"https://api.github.com/users/basho/followers","following_url":"https://api.github.com/users/basho/following{/other_user}","gists_url":"https://api.github.com/users/basho/gists{/gist_id}","starred_url":"https://api.github.com/users/basho/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/basho/subscriptions","organizations_url":"https://api.github.com/users/basho/orgs","repos_url":"https://api.github.com/users/basho/repos","events_url":"https://api.github.com/users/basho/events{/privacy}","received_events_url":"https://api.github.com/users/basho/received_events","type":"Organization","site_admin":false},"repo":{"id":791522,"name":"riak_kv","full_name":"basho/riak_kv","owner":{"login":"basho","id":176293,"avatar_url":"https://avatars.githubusercontent.com/u/176293?v=3","gravatar_id":"","url":"https://api.github.com/users/basho","html_url":"https://github.com/basho","followers_url":"https://api.github.com/users/basho/followers","following_url":"https://api.github.com/users/basho/following{/other_user}","gists_url":"https://api.github.com/users/basho/gists{/gist_id}","starred_url":"https://api.github.com/users/basho/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/basho/subscriptions","organizations_url":"https://api.github.com/users/basho/orgs","repos_url":"https://api.github.com/users/basho/repos","events_url":"https://api.github.com/users/basho/events{/privacy}","received_events_url":"https://api.github.com/users/basho/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/basho/riak_kv","description":"Riak Key/Value Store","fork":false,"url":"https://api.github.com/repos/basho/riak_kv","forks_url":"https://api.github.com/repos/basho/riak_kv/forks","keys_url":"https://api.github.com/repos/basho/riak_kv/keys{/key_id}","collaborators_url":"https://api.github.com/repos/basho/riak_kv/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/basho/riak_kv/teams","hooks_url":"https://api.github.com/repos/basho/riak_kv/hooks","issue_events_url":"https://api.github.com/repos/basho/riak_kv/issues/events{/number}","events_url":"https://api.github.com/repos/basho/riak_kv/events","assignees_url":"https://api.github.com/repos/basho/riak_kv/assignees{/user}","branches_url":"https://api.github.com/repos/basho/riak_kv/branches{/branch}","tags_url":"https://api.github.com/repos/basho/riak_kv/tags","blobs_url":"https://api.github.com/repos/basho/riak_kv/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/basho/riak_kv/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/basho/riak_kv/git/refs{/sha}","trees_url":"https://api.github.com/repos/basho/riak_kv/git/trees{/sha}","statuses_url":"https://api.github.com/repos/basho/riak_kv/statuses/{sha}","languages_url":"https://api.github.com/repos/basho/riak_kv/languages","stargazers_url":"https://api.github.com/repos/basho/riak_kv/stargazers","contributors_url":"https://api.github.com/repos/basho/riak_kv/contributors","subscribers_url":"https://api.github.com/repos/basho/riak_kv/subscribers","subscription_url":"https://api.github.com/repos/basho/riak_kv/subscription","commits_url":"https://api.github.com/repos/basho/riak_kv/commits{/sha}","git_commits_url":"https://api.github.com/repos/basho/riak_kv/git/commits{/sha}","comments_url":"https://api.github.com/repos/basho/riak_kv/comments{/number}","issue_comment_url":"https://api.github.com/repos/basho/riak_kv/issues/comments/{number}","contents_url":"https://api.github.com/repos/basho/riak_kv/contents/{+path}","compare_url":"https://api.github.com/repos/basho/riak_kv/compare/{base}...{head}","merges_url":"https://api.github.com/repos/basho/riak_kv/merges","archive_url":"https://api.github.com/repos/basho/riak_kv/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/basho/riak_kv/downloads","issues_url":"https://api.github.com/repos/basho/riak_kv/issues{/number}","pulls_url":"https://api.github.com/repos/basho/riak_kv/pulls{/number}","milestones_url":"https://api.github.com/repos/basho/riak_kv/milestones{/number}","notifications_url":"https://api.github.com/repos/basho/riak_kv/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/basho/riak_kv/labels{/name}","releases_url":"https://api.github.com/repos/basho/riak_kv/releases{/id}","created_at":"2010-07-22T17:27:35Z","updated_at":"2014-12-25T09:07:08Z","pushed_at":"2015-01-12T11:35:39Z","git_url":"git://github.com/basho/riak_kv.git","ssh_url":"git@github.com:basho/riak_kv.git","clone_url":"https://github.com/basho/riak_kv.git","svn_url":"https://github.com/basho/riak_kv","homepage":"","size":22529,"stargazers_count":226,"watchers_count":226,"language":"Erlang","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":112,"mirror_url":null,"open_issues_count":129,"forks":112,"open_issues":129,"watchers":226,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/basho/riak_kv/pulls/1070"},"html":{"href":"https://github.com/basho/riak_kv/pull/1070"},"issue":{"href":"https://api.github.com/repos/basho/riak_kv/issues/1070"},"comments":{"href":"https://api.github.com/repos/basho/riak_kv/issues/1070/comments"},"review_comments":{"href":"https://api.github.com/repos/basho/riak_kv/pulls/1070/comments"},"review_comment":{"href":"https://api.github.com/repos/basho/riak_kv/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/basho/riak_kv/pulls/1070/commits"},"statuses":{"href":"https://api.github.com/repos/basho/riak_kv/statuses/20d2bd0aea23419ff91be1c6633d31caa2c56562"}}}}