{"action":"created","comment":{"url":"https://api.github.com/repos/hashicorp/vault/pulls/comments/31532330","id":31532330,"diff_hunk":"@@ -167,3 +179,188 @@ func (b *EtcdBackend) nodePath(key string) string {\n func (b *EtcdBackend) nodePathDir(key string) string {\n \treturn filepath.Join(b.path, key) + \"/\"\n }\n+\n+// nodePathLock returns an etcd directory path used specifically for semaphore\n+// indicies based on the given key.\n+func (b *EtcdBackend) nodePathLock(key string) string {\n+\treturn filepath.Join(b.path, filepath.Dir(key), EtcdNodeLockPrefix+filepath.Base(key)+\"/\")\n+}\n+\n+// Lock is used for mutual exclusion based on the given key.\n+func (c *EtcdBackend) LockWith(key, value string) (Lock, error) {\n+\treturn &EtcdLock{\n+\t\tbackend: c,\n+\t\tkey:     key,\n+\t\tvalue:   value,\n+\t}, nil\n+}\n+\n+// EtcdLock emplements a lock using and etcd backend.\n+type EtcdLock struct {\n+\tbackend                  *EtcdBackend\n+\tkey, value, semaphoreKey string\n+\tlock                     sync.Mutex\n+\tisHeld                   bool\n+\tdone                     chan struct{}\n+}\n+\n+// addSemaphoreKey aquires a new ordered semaphore key.\n+func (c *EtcdLock) addSemaphoreKey() (string, uint64, error) {\n+\n+\t// CreateInOrder is an atomic operation that can be used to enqueue a\n+\t// request onto a semaphore. In the rest of the comments, we refer to the\n+\t// resulting key as a \"semaphore key\".\n+\t// https://coreos.com/etcd/docs/2.0.8/api.html#atomically-creating-in-order-keys\n+\tresponse, err := c.backend.client.CreateInOrder(c.backend.nodePathLock(c.key), \"Vault Lock\", EtcdLockTTL)\n+\tif err != nil {\n+\t\treturn \"\", 0, err\n+\t}\n+\n+\treturn response.Node.Key, response.EtcdIndex, nil\n+}\n+\n+// getSemaphoreKey determines which semaphore key holder has aquired the lock.\n+func (c *EtcdLock) getSemaphoreKey() (string, uint64, error) {\n+\n+\t// Get the list of waiters in order to see if we are next.\n+\tresponse, err := c.backend.client.Get(c.backend.nodePathLock(c.key), true, false)\n+\tif err != nil {\n+\t\treturn \"\", 0, err\n+\t}\n+\n+\t// Make sure the list isn't empty.\n+\tif response.Node.Nodes.Len() == 0 {\n+\t\treturn \"\", response.EtcdIndex, nil\n+\t}\n+\n+\treturn response.Node.Nodes[0].Key, response.EtcdIndex, nil\n+}\n+\n+func (c *EtcdLock) Lock(stopCh <-chan struct{}) (<-chan struct{}, error) {\n+\n+\t// Get the local lock before interacting with etcd.\n+\tc.lock.Lock()\n+\tdefer c.lock.Unlock()\n+\n+\t// Check if the lock is already held.\n+\tif c.isHeld {\n+\t\treturn nil, EtcdLockHeldError\n+\t}\n+\n+\t// Add a new semaphore key that we will track.\n+\tsemaphoreKey, _, err := c.addSemaphoreKey()\n+\tif err != nil {\n+\t\treturn nil, err\n+\t}\n+\tc.semaphoreKey = semaphoreKey\n+\n+\t// Get the current semaphore key.\n+\tcurrentSemaphoreKey, currentEtcdIndex, err := c.getSemaphoreKey()\n+\tif err != nil {\n+\t\treturn nil, err\n+\t}\n+\n+\t// Create an etcd-compatible boolean stop channel from the provided\n+\t// interface stop channel.\n+\tboolStopCh := make(chan bool)\n+\tgo func() {\n+\t\tfor _ = range stopCh {\n+\t\t}\n+\t\tclose(boolStopCh)\n+\t}()\n+\n+\t// Loop until the we current semaphore key matches ours.\n+\tfor semaphoreKey != currentSemaphoreKey {\n+\t\tvar err error\n+\n+\t\t// Start a watch of the entire lock directory, providing the stop channel.\n+\t\tresponse, err := c.backend.client.Watch(c.backend.nodePathLock(c.key), currentEtcdIndex+1, true, nil, boolStopCh)\n+\t\tif err != nil {\n+\n+\t\t\t// If the error is not an etcd error, we can assume it's a notification\n+\t\t\t// of the stop channel having closed. In this scenario, we also want to\n+\t\t\t// remove our semaphore key as we are no longer waiting to aquire the\n+\t\t\t// lock.\n+\t\t\tif _, ok := err.(*etcd.EtcdError); !ok {\n+\t\t\t\t_, err = c.backend.client.Delete(c.semaphoreKey, false)\n+\t\t\t}\n+\n+\t\t\treturn nil, err\n+\t\t}\n+\n+\t\t// Make sure the index we are waiting for has not been removed. If it has,\n+\t\t// this is an error and nothing else needs to be done.\n+\t\tif response.Node.Key == semaphoreKey &&\n+\t\t\t(response.Action == \"delete\" || response.Action == \"expire\") {\n+\t\t\treturn nil, EtcdSemaphoreKeyRemovedError\n+\t\t}\n+\n+\t\t// Get the current semaphore key and etcd index.\n+\t\tcurrentSemaphoreKey, currentEtcdIndex, err = c.getSemaphoreKey()\n+\t\tif err != nil {\n+\t\t\treturn nil, err\n+\t\t}\n+\t}\n+\n+\t// We now hold the lock.\n+\tc.isHeld = true\n+\n+\t// Because we hold the lock, we can set our value for the given key.\n+\tif err := c.backend.Put(&Entry{","path":"physical/etcd.go","position":164,"original_position":164,"commit_id":"2e17df9b4b4087bcda07daf60c68f44b8f24b519","original_commit_id":"2e17df9b4b4087bcda07daf60c68f44b8f24b519","user":{"login":"armon","id":592032,"avatar_url":"https://avatars.githubusercontent.com/u/592032?v=3","gravatar_id":"","url":"https://api.github.com/users/armon","html_url":"https://github.com/armon","followers_url":"https://api.github.com/users/armon/followers","following_url":"https://api.github.com/users/armon/following{/other_user}","gists_url":"https://api.github.com/users/armon/gists{/gist_id}","starred_url":"https://api.github.com/users/armon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/armon/subscriptions","organizations_url":"https://api.github.com/users/armon/orgs","repos_url":"https://api.github.com/users/armon/repos","events_url":"https://api.github.com/users/armon/events{/privacy}","received_events_url":"https://api.github.com/users/armon/received_events","type":"User","site_admin":false},"body":"The race will occur between the other standby instances. E.g. they will see the lock has happened, but not have the associated value which will cause errors on their end.","created_at":"2015-06-02T15:03:55Z","updated_at":"2015-06-02T15:03:55Z","html_url":"https://github.com/hashicorp/vault/pull/297#discussion_r31532330","pull_request_url":"https://api.github.com/repos/hashicorp/vault/pulls/297","_links":{"self":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/comments/31532330"},"html":{"href":"https://github.com/hashicorp/vault/pull/297#discussion_r31532330"},"pull_request":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/297"}}},"pull_request":{"url":"https://api.github.com/repos/hashicorp/vault/pulls/297","id":36678961,"html_url":"https://github.com/hashicorp/vault/pull/297","diff_url":"https://github.com/hashicorp/vault/pull/297.diff","patch_url":"https://github.com/hashicorp/vault/pull/297.patch","issue_url":"https://api.github.com/repos/hashicorp/vault/issues/297","number":297,"state":"open","locked":false,"title":"etcd HA + tests","user":{"login":"buth","id":1918465,"avatar_url":"https://avatars.githubusercontent.com/u/1918465?v=3","gravatar_id":"","url":"https://api.github.com/users/buth","html_url":"https://github.com/buth","followers_url":"https://api.github.com/users/buth/followers","following_url":"https://api.github.com/users/buth/following{/other_user}","gists_url":"https://api.github.com/users/buth/gists{/gist_id}","starred_url":"https://api.github.com/users/buth/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/buth/subscriptions","organizations_url":"https://api.github.com/users/buth/orgs","repos_url":"https://api.github.com/users/buth/repos","events_url":"https://api.github.com/users/buth/events{/privacy}","received_events_url":"https://api.github.com/users/buth/received_events","type":"User","site_admin":false},"body":"Added an HA implementation for the etcd physical backend. The relevant tests that already existed within the `physical` package pass, but there are still a few assumptions made here that I am unsure of.\r\n\r\n* The value provided to the lock *should* be set within the same key/value-store namespace as values set without the lock (i.e. should be accessible with `Get` and overwritable with `Put`).\r\n* The channel returned when the lock is acquired is closed when leadership is lost. The lock *should not* attempt to detect if leadership is lost hostilely (e.g. when a semaphore key expires or is deleted by another client).\r\n\r\nIf either of those are problematic, there's more work to be done here.\r\n\r\nThanks!","created_at":"2015-06-01T22:45:27Z","updated_at":"2015-06-02T15:03:55Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e1dfe1f45c0293580ee61cde2b3febc1040fa360","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/hashicorp/vault/pulls/297/commits","review_comments_url":"https://api.github.com/repos/hashicorp/vault/pulls/297/comments","review_comment_url":"https://api.github.com/repos/hashicorp/vault/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hashicorp/vault/issues/297/comments","statuses_url":"https://api.github.com/repos/hashicorp/vault/statuses/2e17df9b4b4087bcda07daf60c68f44b8f24b519","head":{"label":"buth:etcd-ha","ref":"etcd-ha","sha":"2e17df9b4b4087bcda07daf60c68f44b8f24b519","user":{"login":"buth","id":1918465,"avatar_url":"https://avatars.githubusercontent.com/u/1918465?v=3","gravatar_id":"","url":"https://api.github.com/users/buth","html_url":"https://github.com/buth","followers_url":"https://api.github.com/users/buth/followers","following_url":"https://api.github.com/users/buth/following{/other_user}","gists_url":"https://api.github.com/users/buth/gists{/gist_id}","starred_url":"https://api.github.com/users/buth/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/buth/subscriptions","organizations_url":"https://api.github.com/users/buth/orgs","repos_url":"https://api.github.com/users/buth/repos","events_url":"https://api.github.com/users/buth/events{/privacy}","received_events_url":"https://api.github.com/users/buth/received_events","type":"User","site_admin":false},"repo":{"id":36138343,"name":"vault","full_name":"buth/vault","owner":{"login":"buth","id":1918465,"avatar_url":"https://avatars.githubusercontent.com/u/1918465?v=3","gravatar_id":"","url":"https://api.github.com/users/buth","html_url":"https://github.com/buth","followers_url":"https://api.github.com/users/buth/followers","following_url":"https://api.github.com/users/buth/following{/other_user}","gists_url":"https://api.github.com/users/buth/gists{/gist_id}","starred_url":"https://api.github.com/users/buth/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/buth/subscriptions","organizations_url":"https://api.github.com/users/buth/orgs","repos_url":"https://api.github.com/users/buth/repos","events_url":"https://api.github.com/users/buth/events{/privacy}","received_events_url":"https://api.github.com/users/buth/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/buth/vault","description":"A tool for managing secrets.","fork":true,"url":"https://api.github.com/repos/buth/vault","forks_url":"https://api.github.com/repos/buth/vault/forks","keys_url":"https://api.github.com/repos/buth/vault/keys{/key_id}","collaborators_url":"https://api.github.com/repos/buth/vault/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/buth/vault/teams","hooks_url":"https://api.github.com/repos/buth/vault/hooks","issue_events_url":"https://api.github.com/repos/buth/vault/issues/events{/number}","events_url":"https://api.github.com/repos/buth/vault/events","assignees_url":"https://api.github.com/repos/buth/vault/assignees{/user}","branches_url":"https://api.github.com/repos/buth/vault/branches{/branch}","tags_url":"https://api.github.com/repos/buth/vault/tags","blobs_url":"https://api.github.com/repos/buth/vault/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/buth/vault/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/buth/vault/git/refs{/sha}","trees_url":"https://api.github.com/repos/buth/vault/git/trees{/sha}","statuses_url":"https://api.github.com/repos/buth/vault/statuses/{sha}","languages_url":"https://api.github.com/repos/buth/vault/languages","stargazers_url":"https://api.github.com/repos/buth/vault/stargazers","contributors_url":"https://api.github.com/repos/buth/vault/contributors","subscribers_url":"https://api.github.com/repos/buth/vault/subscribers","subscription_url":"https://api.github.com/repos/buth/vault/subscription","commits_url":"https://api.github.com/repos/buth/vault/commits{/sha}","git_commits_url":"https://api.github.com/repos/buth/vault/git/commits{/sha}","comments_url":"https://api.github.com/repos/buth/vault/comments{/number}","issue_comment_url":"https://api.github.com/repos/buth/vault/issues/comments{/number}","contents_url":"https://api.github.com/repos/buth/vault/contents/{+path}","compare_url":"https://api.github.com/repos/buth/vault/compare/{base}...{head}","merges_url":"https://api.github.com/repos/buth/vault/merges","archive_url":"https://api.github.com/repos/buth/vault/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/buth/vault/downloads","issues_url":"https://api.github.com/repos/buth/vault/issues{/number}","pulls_url":"https://api.github.com/repos/buth/vault/pulls{/number}","milestones_url":"https://api.github.com/repos/buth/vault/milestones{/number}","notifications_url":"https://api.github.com/repos/buth/vault/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/buth/vault/labels{/name}","releases_url":"https://api.github.com/repos/buth/vault/releases{/id}","created_at":"2015-05-23T19:11:51Z","updated_at":"2015-05-23T19:11:52Z","pushed_at":"2015-06-01T22:33:54Z","git_url":"git://github.com/buth/vault.git","ssh_url":"git@github.com:buth/vault.git","clone_url":"https://github.com/buth/vault.git","svn_url":"https://github.com/buth/vault","homepage":"http://vaultproject.io","size":3279,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"hashicorp:master","ref":"master","sha":"669686f654363bbfcdbb31fa56d9ba0b77937c6e","user":{"login":"hashicorp","id":761456,"avatar_url":"https://avatars.githubusercontent.com/u/761456?v=3","gravatar_id":"","url":"https://api.github.com/users/hashicorp","html_url":"https://github.com/hashicorp","followers_url":"https://api.github.com/users/hashicorp/followers","following_url":"https://api.github.com/users/hashicorp/following{/other_user}","gists_url":"https://api.github.com/users/hashicorp/gists{/gist_id}","starred_url":"https://api.github.com/users/hashicorp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hashicorp/subscriptions","organizations_url":"https://api.github.com/users/hashicorp/orgs","repos_url":"https://api.github.com/users/hashicorp/repos","events_url":"https://api.github.com/users/hashicorp/events{/privacy}","received_events_url":"https://api.github.com/users/hashicorp/received_events","type":"Organization","site_admin":false},"repo":{"id":31288958,"name":"vault","full_name":"hashicorp/vault","owner":{"login":"hashicorp","id":761456,"avatar_url":"https://avatars.githubusercontent.com/u/761456?v=3","gravatar_id":"","url":"https://api.github.com/users/hashicorp","html_url":"https://github.com/hashicorp","followers_url":"https://api.github.com/users/hashicorp/followers","following_url":"https://api.github.com/users/hashicorp/following{/other_user}","gists_url":"https://api.github.com/users/hashicorp/gists{/gist_id}","starred_url":"https://api.github.com/users/hashicorp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hashicorp/subscriptions","organizations_url":"https://api.github.com/users/hashicorp/orgs","repos_url":"https://api.github.com/users/hashicorp/repos","events_url":"https://api.github.com/users/hashicorp/events{/privacy}","received_events_url":"https://api.github.com/users/hashicorp/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hashicorp/vault","description":"A tool for managing secrets.","fork":false,"url":"https://api.github.com/repos/hashicorp/vault","forks_url":"https://api.github.com/repos/hashicorp/vault/forks","keys_url":"https://api.github.com/repos/hashicorp/vault/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hashicorp/vault/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hashicorp/vault/teams","hooks_url":"https://api.github.com/repos/hashicorp/vault/hooks","issue_events_url":"https://api.github.com/repos/hashicorp/vault/issues/events{/number}","events_url":"https://api.github.com/repos/hashicorp/vault/events","assignees_url":"https://api.github.com/repos/hashicorp/vault/assignees{/user}","branches_url":"https://api.github.com/repos/hashicorp/vault/branches{/branch}","tags_url":"https://api.github.com/repos/hashicorp/vault/tags","blobs_url":"https://api.github.com/repos/hashicorp/vault/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hashicorp/vault/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hashicorp/vault/git/refs{/sha}","trees_url":"https://api.github.com/repos/hashicorp/vault/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hashicorp/vault/statuses/{sha}","languages_url":"https://api.github.com/repos/hashicorp/vault/languages","stargazers_url":"https://api.github.com/repos/hashicorp/vault/stargazers","contributors_url":"https://api.github.com/repos/hashicorp/vault/contributors","subscribers_url":"https://api.github.com/repos/hashicorp/vault/subscribers","subscription_url":"https://api.github.com/repos/hashicorp/vault/subscription","commits_url":"https://api.github.com/repos/hashicorp/vault/commits{/sha}","git_commits_url":"https://api.github.com/repos/hashicorp/vault/git/commits{/sha}","comments_url":"https://api.github.com/repos/hashicorp/vault/comments{/number}","issue_comment_url":"https://api.github.com/repos/hashicorp/vault/issues/comments{/number}","contents_url":"https://api.github.com/repos/hashicorp/vault/contents/{+path}","compare_url":"https://api.github.com/repos/hashicorp/vault/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hashicorp/vault/merges","archive_url":"https://api.github.com/repos/hashicorp/vault/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hashicorp/vault/downloads","issues_url":"https://api.github.com/repos/hashicorp/vault/issues{/number}","pulls_url":"https://api.github.com/repos/hashicorp/vault/pulls{/number}","milestones_url":"https://api.github.com/repos/hashicorp/vault/milestones{/number}","notifications_url":"https://api.github.com/repos/hashicorp/vault/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hashicorp/vault/labels{/name}","releases_url":"https://api.github.com/repos/hashicorp/vault/releases{/id}","created_at":"2015-02-25T00:15:59Z","updated_at":"2015-06-02T14:32:45Z","pushed_at":"2015-06-02T14:04:10Z","git_url":"git://github.com/hashicorp/vault.git","ssh_url":"git@github.com:hashicorp/vault.git","clone_url":"https://github.com/hashicorp/vault.git","svn_url":"https://github.com/hashicorp/vault","homepage":"http://vaultproject.io","size":7549,"stargazers_count":2511,"watchers_count":2511,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":111,"mirror_url":null,"open_issues_count":40,"forks":111,"open_issues":40,"watchers":2511,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/297"},"html":{"href":"https://github.com/hashicorp/vault/pull/297"},"issue":{"href":"https://api.github.com/repos/hashicorp/vault/issues/297"},"comments":{"href":"https://api.github.com/repos/hashicorp/vault/issues/297/comments"},"review_comments":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/297/comments"},"review_comment":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/297/commits"},"statuses":{"href":"https://api.github.com/repos/hashicorp/vault/statuses/2e17df9b4b4087bcda07daf60c68f44b8f24b519"}}}}