{"action":"created","comment":{"url":"https://api.github.com/repos/control-center/serviced/pulls/comments/38446207","id":38446207,"diff_hunk":"@@ -473,7 +474,51 @@ func (c *Controller) rpcHealthCheck() (chan struct{}, error) {\n \t\t\t}\n \t\t}\n \t}()\n+\treturn gone, nil\n+}\n+\n+func isNFSMountStale(mountpoint string) bool {\n+\tif err := exec.Command(\"/bin/bash\", \"-c\", fmt.Sprintf(\"read -t1 < <(stat -t '%s' 2>&-)\", mountpoint)).Run(); err != nil {\n+\t\tif status, iscode := utils.GetExitStatus(err); iscode && status == 142 {\n+\t\t\t// EREMDEV; wait for NFS to come back\n+\t\t\tglog.Infof(\"Distributed storage temporarily unavailable. Waiting for it to return.\")\n+\t\t\treturn false\n+\n+\t\t}\n+\t\tglog.Errorf(\"Mount point %s check had error (%s); considering stale\", mountpoint, err)\n+\t\treturn true\n+\t}\n+\treturn false\n+}\n \n+// storageHealthCheck returns a channel that will close when the distributed\n+// storage is no longer accessible (i.e., stale NFS mount)\n+func (c *Controller) storageHealthCheck() (chan struct{}, error) {\n+\tgone := make(chan struct{})\n+\tmounts, err := mount.GetMounts()\n+\tif err != nil {\n+\t\treturn nil, err\n+\t}\n+\tnfsMountPoints := []string{}\n+\tfor _, minfo := range mounts {\n+\t\tif strings.HasPrefix(minfo.Fstype, \"nfs\") {\n+\t\t\tnfsMountPoints = append(nfsMountPoints, minfo.Mountpoint)\n+\t\t}\n+\t}\n+\tif len(nfsMountPoints) > 0 {\n+\t\t// Start polling\n+\t\tgo func() {\n+\t\t\tfor {\n+\t\t\t\tfor _, mp := range nfsMountPoints {","path":"container/controller.go","position":47,"original_position":47,"commit_id":"60c636d4d22392bf83599f8028e05027689fd0c6","original_commit_id":"60c636d4d22392bf83599f8028e05027689fd0c6","user":{"login":"iancmcc","id":57971,"avatar_url":"https://avatars.githubusercontent.com/u/57971?v=3","gravatar_id":"","url":"https://api.github.com/users/iancmcc","html_url":"https://github.com/iancmcc","followers_url":"https://api.github.com/users/iancmcc/followers","following_url":"https://api.github.com/users/iancmcc/following{/other_user}","gists_url":"https://api.github.com/users/iancmcc/gists{/gist_id}","starred_url":"https://api.github.com/users/iancmcc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/iancmcc/subscriptions","organizations_url":"https://api.github.com/users/iancmcc/orgs","repos_url":"https://api.github.com/users/iancmcc/repos","events_url":"https://api.github.com/users/iancmcc/events{/privacy}","received_events_url":"https://api.github.com/users/iancmcc/received_events","type":"User","site_admin":false},"body":"What would be an alternative?\nOn Tue, Sep 1, 2015 at 12:22 PM Andrew Lapidas <notifications@github.com>\nwrote:\n\n> In container/controller.go\n> <https://github.com/control-center/serviced/pull/1925#discussion_r38446009>\n> :\n>\n> > +\tgone := make(chan struct{})\n> > +\tmounts, err := mount.GetMounts()\n> > +\tif err != nil {\n> > +\t\treturn nil, err\n> > +\t}\n> > +\tnfsMountPoints := []string{}\n> > +\tfor _, minfo := range mounts {\n> > +\t\tif strings.HasPrefix(minfo.Fstype, \"nfs\") {\n> > +\t\t\tnfsMountPoints = append(nfsMountPoints, minfo.Mountpoint)\n> > +\t\t}\n> > +\t}\n> > +\tif len(nfsMountPoints) > 0 {\n> > +\t\t// Start polling\n> > +\t\tgo func() {\n> > +\t\t\tfor {\n> > +\t\t\t\tfor _, mp := range nfsMountPoints {\n>\n> Do we actually need to check every NFS mount point here?\n>\n> —\n> Reply to this email directly or view it on GitHub\n> <https://github.com/control-center/serviced/pull/1925/files#r38446009>.\n>\n","created_at":"2015-09-01T17:24:22Z","updated_at":"2015-09-01T17:24:22Z","html_url":"https://github.com/control-center/serviced/pull/1925#discussion_r38446207","pull_request_url":"https://api.github.com/repos/control-center/serviced/pulls/1925","_links":{"self":{"href":"https://api.github.com/repos/control-center/serviced/pulls/comments/38446207"},"html":{"href":"https://github.com/control-center/serviced/pull/1925#discussion_r38446207"},"pull_request":{"href":"https://api.github.com/repos/control-center/serviced/pulls/1925"}}},"pull_request":{"url":"https://api.github.com/repos/control-center/serviced/pulls/1925","id":43898893,"html_url":"https://github.com/control-center/serviced/pull/1925","diff_url":"https://github.com/control-center/serviced/pull/1925.diff","patch_url":"https://github.com/control-center/serviced/pull/1925.patch","issue_url":"https://api.github.com/repos/control-center/serviced/issues/1925","number":1925,"state":"open","locked":false,"title":"CC-1217: Services test for stale NFS file handles and commit suicide appropriately.","user":{"login":"iancmcc","id":57971,"avatar_url":"https://avatars.githubusercontent.com/u/57971?v=3","gravatar_id":"","url":"https://api.github.com/users/iancmcc","html_url":"https://github.com/iancmcc","followers_url":"https://api.github.com/users/iancmcc/followers","following_url":"https://api.github.com/users/iancmcc/following{/other_user}","gists_url":"https://api.github.com/users/iancmcc/gists{/gist_id}","starred_url":"https://api.github.com/users/iancmcc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/iancmcc/subscriptions","organizations_url":"https://api.github.com/users/iancmcc/orgs","repos_url":"https://api.github.com/users/iancmcc/repos","events_url":"https://api.github.com/users/iancmcc/events{/privacy}","received_events_url":"https://api.github.com/users/iancmcc/received_events","type":"User","site_admin":false},"body":null,"created_at":"2015-09-01T16:45:51Z","updated_at":"2015-09-01T17:24:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"65d773fa77aafaeb2a0c6c75085fa5e067dbe221","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/control-center/serviced/pulls/1925/commits","review_comments_url":"https://api.github.com/repos/control-center/serviced/pulls/1925/comments","review_comment_url":"https://api.github.com/repos/control-center/serviced/pulls/comments{/number}","comments_url":"https://api.github.com/repos/control-center/serviced/issues/1925/comments","statuses_url":"https://api.github.com/repos/control-center/serviced/statuses/60c636d4d22392bf83599f8028e05027689fd0c6","head":{"label":"control-center:feature/CC-1217","ref":"feature/CC-1217","sha":"60c636d4d22392bf83599f8028e05027689fd0c6","user":{"login":"control-center","id":8160673,"avatar_url":"https://avatars.githubusercontent.com/u/8160673?v=3","gravatar_id":"","url":"https://api.github.com/users/control-center","html_url":"https://github.com/control-center","followers_url":"https://api.github.com/users/control-center/followers","following_url":"https://api.github.com/users/control-center/following{/other_user}","gists_url":"https://api.github.com/users/control-center/gists{/gist_id}","starred_url":"https://api.github.com/users/control-center/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/control-center/subscriptions","organizations_url":"https://api.github.com/users/control-center/orgs","repos_url":"https://api.github.com/users/control-center/repos","events_url":"https://api.github.com/users/control-center/events{/privacy}","received_events_url":"https://api.github.com/users/control-center/received_events","type":"Organization","site_admin":false},"repo":{"id":11929217,"name":"serviced","full_name":"control-center/serviced","owner":{"login":"control-center","id":8160673,"avatar_url":"https://avatars.githubusercontent.com/u/8160673?v=3","gravatar_id":"","url":"https://api.github.com/users/control-center","html_url":"https://github.com/control-center","followers_url":"https://api.github.com/users/control-center/followers","following_url":"https://api.github.com/users/control-center/following{/other_user}","gists_url":"https://api.github.com/users/control-center/gists{/gist_id}","starred_url":"https://api.github.com/users/control-center/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/control-center/subscriptions","organizations_url":"https://api.github.com/users/control-center/orgs","repos_url":"https://api.github.com/users/control-center/repos","events_url":"https://api.github.com/users/control-center/events{/privacy}","received_events_url":"https://api.github.com/users/control-center/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/control-center/serviced","description":"Serviced is a PaaS runtime. It allows users to create, manage and scale services in a uniform way.","fork":false,"url":"https://api.github.com/repos/control-center/serviced","forks_url":"https://api.github.com/repos/control-center/serviced/forks","keys_url":"https://api.github.com/repos/control-center/serviced/keys{/key_id}","collaborators_url":"https://api.github.com/repos/control-center/serviced/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/control-center/serviced/teams","hooks_url":"https://api.github.com/repos/control-center/serviced/hooks","issue_events_url":"https://api.github.com/repos/control-center/serviced/issues/events{/number}","events_url":"https://api.github.com/repos/control-center/serviced/events","assignees_url":"https://api.github.com/repos/control-center/serviced/assignees{/user}","branches_url":"https://api.github.com/repos/control-center/serviced/branches{/branch}","tags_url":"https://api.github.com/repos/control-center/serviced/tags","blobs_url":"https://api.github.com/repos/control-center/serviced/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/control-center/serviced/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/control-center/serviced/git/refs{/sha}","trees_url":"https://api.github.com/repos/control-center/serviced/git/trees{/sha}","statuses_url":"https://api.github.com/repos/control-center/serviced/statuses/{sha}","languages_url":"https://api.github.com/repos/control-center/serviced/languages","stargazers_url":"https://api.github.com/repos/control-center/serviced/stargazers","contributors_url":"https://api.github.com/repos/control-center/serviced/contributors","subscribers_url":"https://api.github.com/repos/control-center/serviced/subscribers","subscription_url":"https://api.github.com/repos/control-center/serviced/subscription","commits_url":"https://api.github.com/repos/control-center/serviced/commits{/sha}","git_commits_url":"https://api.github.com/repos/control-center/serviced/git/commits{/sha}","comments_url":"https://api.github.com/repos/control-center/serviced/comments{/number}","issue_comment_url":"https://api.github.com/repos/control-center/serviced/issues/comments{/number}","contents_url":"https://api.github.com/repos/control-center/serviced/contents/{+path}","compare_url":"https://api.github.com/repos/control-center/serviced/compare/{base}...{head}","merges_url":"https://api.github.com/repos/control-center/serviced/merges","archive_url":"https://api.github.com/repos/control-center/serviced/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/control-center/serviced/downloads","issues_url":"https://api.github.com/repos/control-center/serviced/issues{/number}","pulls_url":"https://api.github.com/repos/control-center/serviced/pulls{/number}","milestones_url":"https://api.github.com/repos/control-center/serviced/milestones{/number}","notifications_url":"https://api.github.com/repos/control-center/serviced/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/control-center/serviced/labels{/name}","releases_url":"https://api.github.com/repos/control-center/serviced/releases{/id}","created_at":"2013-08-06T16:45:00Z","updated_at":"2015-09-01T07:35:19Z","pushed_at":"2015-09-01T16:52:18Z","git_url":"git://github.com/control-center/serviced.git","ssh_url":"git@github.com:control-center/serviced.git","clone_url":"https://github.com/control-center/serviced.git","svn_url":"https://github.com/control-center/serviced","homepage":null,"size":75429,"stargazers_count":68,"watchers_count":68,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":9,"mirror_url":null,"open_issues_count":5,"forks":9,"open_issues":5,"watchers":68,"default_branch":"develop"}},"base":{"label":"control-center:develop","ref":"develop","sha":"ee4809a39db72f1c0ed4384366c6249561000474","user":{"login":"control-center","id":8160673,"avatar_url":"https://avatars.githubusercontent.com/u/8160673?v=3","gravatar_id":"","url":"https://api.github.com/users/control-center","html_url":"https://github.com/control-center","followers_url":"https://api.github.com/users/control-center/followers","following_url":"https://api.github.com/users/control-center/following{/other_user}","gists_url":"https://api.github.com/users/control-center/gists{/gist_id}","starred_url":"https://api.github.com/users/control-center/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/control-center/subscriptions","organizations_url":"https://api.github.com/users/control-center/orgs","repos_url":"https://api.github.com/users/control-center/repos","events_url":"https://api.github.com/users/control-center/events{/privacy}","received_events_url":"https://api.github.com/users/control-center/received_events","type":"Organization","site_admin":false},"repo":{"id":11929217,"name":"serviced","full_name":"control-center/serviced","owner":{"login":"control-center","id":8160673,"avatar_url":"https://avatars.githubusercontent.com/u/8160673?v=3","gravatar_id":"","url":"https://api.github.com/users/control-center","html_url":"https://github.com/control-center","followers_url":"https://api.github.com/users/control-center/followers","following_url":"https://api.github.com/users/control-center/following{/other_user}","gists_url":"https://api.github.com/users/control-center/gists{/gist_id}","starred_url":"https://api.github.com/users/control-center/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/control-center/subscriptions","organizations_url":"https://api.github.com/users/control-center/orgs","repos_url":"https://api.github.com/users/control-center/repos","events_url":"https://api.github.com/users/control-center/events{/privacy}","received_events_url":"https://api.github.com/users/control-center/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/control-center/serviced","description":"Serviced is a PaaS runtime. It allows users to create, manage and scale services in a uniform way.","fork":false,"url":"https://api.github.com/repos/control-center/serviced","forks_url":"https://api.github.com/repos/control-center/serviced/forks","keys_url":"https://api.github.com/repos/control-center/serviced/keys{/key_id}","collaborators_url":"https://api.github.com/repos/control-center/serviced/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/control-center/serviced/teams","hooks_url":"https://api.github.com/repos/control-center/serviced/hooks","issue_events_url":"https://api.github.com/repos/control-center/serviced/issues/events{/number}","events_url":"https://api.github.com/repos/control-center/serviced/events","assignees_url":"https://api.github.com/repos/control-center/serviced/assignees{/user}","branches_url":"https://api.github.com/repos/control-center/serviced/branches{/branch}","tags_url":"https://api.github.com/repos/control-center/serviced/tags","blobs_url":"https://api.github.com/repos/control-center/serviced/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/control-center/serviced/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/control-center/serviced/git/refs{/sha}","trees_url":"https://api.github.com/repos/control-center/serviced/git/trees{/sha}","statuses_url":"https://api.github.com/repos/control-center/serviced/statuses/{sha}","languages_url":"https://api.github.com/repos/control-center/serviced/languages","stargazers_url":"https://api.github.com/repos/control-center/serviced/stargazers","contributors_url":"https://api.github.com/repos/control-center/serviced/contributors","subscribers_url":"https://api.github.com/repos/control-center/serviced/subscribers","subscription_url":"https://api.github.com/repos/control-center/serviced/subscription","commits_url":"https://api.github.com/repos/control-center/serviced/commits{/sha}","git_commits_url":"https://api.github.com/repos/control-center/serviced/git/commits{/sha}","comments_url":"https://api.github.com/repos/control-center/serviced/comments{/number}","issue_comment_url":"https://api.github.com/repos/control-center/serviced/issues/comments{/number}","contents_url":"https://api.github.com/repos/control-center/serviced/contents/{+path}","compare_url":"https://api.github.com/repos/control-center/serviced/compare/{base}...{head}","merges_url":"https://api.github.com/repos/control-center/serviced/merges","archive_url":"https://api.github.com/repos/control-center/serviced/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/control-center/serviced/downloads","issues_url":"https://api.github.com/repos/control-center/serviced/issues{/number}","pulls_url":"https://api.github.com/repos/control-center/serviced/pulls{/number}","milestones_url":"https://api.github.com/repos/control-center/serviced/milestones{/number}","notifications_url":"https://api.github.com/repos/control-center/serviced/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/control-center/serviced/labels{/name}","releases_url":"https://api.github.com/repos/control-center/serviced/releases{/id}","created_at":"2013-08-06T16:45:00Z","updated_at":"2015-09-01T07:35:19Z","pushed_at":"2015-09-01T16:52:18Z","git_url":"git://github.com/control-center/serviced.git","ssh_url":"git@github.com:control-center/serviced.git","clone_url":"https://github.com/control-center/serviced.git","svn_url":"https://github.com/control-center/serviced","homepage":null,"size":75429,"stargazers_count":68,"watchers_count":68,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":9,"mirror_url":null,"open_issues_count":5,"forks":9,"open_issues":5,"watchers":68,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/control-center/serviced/pulls/1925"},"html":{"href":"https://github.com/control-center/serviced/pull/1925"},"issue":{"href":"https://api.github.com/repos/control-center/serviced/issues/1925"},"comments":{"href":"https://api.github.com/repos/control-center/serviced/issues/1925/comments"},"review_comments":{"href":"https://api.github.com/repos/control-center/serviced/pulls/1925/comments"},"review_comment":{"href":"https://api.github.com/repos/control-center/serviced/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/control-center/serviced/pulls/1925/commits"},"statuses":{"href":"https://api.github.com/repos/control-center/serviced/statuses/60c636d4d22392bf83599f8028e05027689fd0c6"}}}}