{"action":"created","comment":{"url":"https://api.github.com/repos/lsst/qserv/pulls/comments/29072013","id":29072013,"diff_hunk":"@@ -0,0 +1,160 @@\n+# LSST Data Management System\n+# Copyright 2015 AURA/LSST.\n+#\n+# This product includes software developed by the\n+# LSST Project (http://www.lsst.org/).\n+#\n+# This program is free software: you can redistribute it and/or modify\n+# it under the terms of the GNU General Public License as published by\n+# the Free Software Foundation, either version 3 of the License, or\n+# (at your option) any later version.\n+#\n+# This program is distributed in the hope that it will be useful,\n+# but WITHOUT ANY WARRANTY; without even the implied warranty of\n+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+# GNU General Public License for more details.\n+#\n+# You should have received a copy of the LSST License Statement and\n+# the GNU General Public License along with this program.  If not,\n+# see <http://www.lsstcorp.org/LegalNotices/>.\n+\n+\"\"\"\n+Module defining nodePool class and related methods.\n+\n+NodePool is a class which allow to launch parallel ssh commands to\n+a set of NodeAdmin instances. Each NodeAdmin instance can be a Qserv\n+master or worker node.\n+\n+It has been used at CC-IN2P3 in spring 2015 to install and manage multiple\n+Qserv development versions for a 50 nodes cluster.\n+\n+@author  Fabrice Jammes, IN2P3\n+\"\"\"\n+\n+#--------------------------------\n+#  Imports of standard modules --\n+#--------------------------------\n+import logging\n+import os\n+from subprocess import Popen, list2cmdline\n+import time\n+\n+#-----------------------------\n+# Imports for other modules --\n+#-----------------------------\n+\n+#----------------------------------\n+# Local non-exported definitions --\n+#----------------------------------\n+_LOG = logging.getLogger(__name__)\n+\n+def cpu_count():\n+    ''' Returns the number of CPUs in the system\n+    '''\n+    try:\n+        return os.sysconf('SC_NPROCESSORS_ONLN')\n+    except (ValueError, OSError, AttributeError):\n+        return 1\n+\n+class NodePool(object):\n+    \"\"\"\n+    Class providing tools to send ssh command to a set of machines.\n+    Use system ssh.\n+    \"\"\"\n+\n+    def __init__(self, nodeAdmins=None, max_task=None):\n+        \"\"\"\n+        Container of nodes which will be managed in parallel using ssh\n+\n+        @param nodeAdmins:  a list of NodeAdmin instances, each node has to be\n+                            contactable via SSH\n+        @param max_task:    number of parallel SSH command which will be run\n+                            concurrently, equal to the number of cpu of the local\n+                            machine by default\n+        \"\"\"\n+\n+        self.nodes = nodeAdmins\n+\n+        if max_task:\n+            self._max_task = max_task\n+        else:\n+            self._max_task = cpu_count()\n+\n+    def execParallel(self, command, stdin=os.devnull):\n+        \"\"\"\n+        Execute one SSH command in parallel on a set of nodes;\n+        fork one process for each node.\n+        Log of each command are stored in $CWD/parallel-cmd-log\n+\n+        @param command: command to run\n+        @param stdin: file redirect to SSH client process standard input\n+\n+        @return the number of nodes in error\n+        \"\"\"\n+        nodes_remaining = list(self.nodes)\n+\n+        if not command:\n+            return  # empty list\n+\n+        logdir = \"parallel-cmd-log\"\n+        try:\n+            os.makedirs(logdir)\n+        except OSError:\n+            if not os.path.isdir(logdir):\n+                raise\n+\n+        def logfile(name):\n+            f = \"{0}-{1}-{2}.txt\".format(node.host, process_id, name)","path":"admin/python/lsst/qserv/admin/nodePool.py","position":null,"original_position":107,"commit_id":"d1f4afd2d471db696f597b5eece64a487c0b544e","original_commit_id":"7af2c92a74fe00f9fdc3a361d8e62c352a738226","user":{"login":"andy-slac","id":4266950,"avatar_url":"https://avatars.githubusercontent.com/u/4266950?v=3","gravatar_id":"","url":"https://api.github.com/users/andy-slac","html_url":"https://github.com/andy-slac","followers_url":"https://api.github.com/users/andy-slac/followers","following_url":"https://api.github.com/users/andy-slac/following{/other_user}","gists_url":"https://api.github.com/users/andy-slac/gists{/gist_id}","starred_url":"https://api.github.com/users/andy-slac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andy-slac/subscriptions","organizations_url":"https://api.github.com/users/andy-slac/orgs","repos_url":"https://api.github.com/users/andy-slac/repos","events_url":"https://api.github.com/users/andy-slac/events{/privacy}","received_events_url":"https://api.github.com/users/andy-slac/received_events","type":"User","site_admin":false},"body":"Why do you want to rely on closures, cannot you just pass the arguments to a function, I think it would make it more reliable.","created_at":"2015-04-24T18:13:15Z","updated_at":"2015-04-24T18:13:15Z","html_url":"https://github.com/lsst/qserv/pull/61#discussion_r29072013","pull_request_url":"https://api.github.com/repos/lsst/qserv/pulls/61","_links":{"self":{"href":"https://api.github.com/repos/lsst/qserv/pulls/comments/29072013"},"html":{"href":"https://github.com/lsst/qserv/pull/61#discussion_r29072013"},"pull_request":{"href":"https://api.github.com/repos/lsst/qserv/pulls/61"}}},"pull_request":{"url":"https://api.github.com/repos/lsst/qserv/pulls/61","id":33774521,"html_url":"https://github.com/lsst/qserv/pull/61","diff_url":"https://github.com/lsst/qserv/pull/61.diff","patch_url":"https://github.com/lsst/qserv/pull/61.patch","issue_url":"https://api.github.com/repos/lsst/qserv/issues/61","number":61,"state":"open","locked":false,"title":"U/fjammes/dm 2341","user":{"login":"jbecla","id":626419,"avatar_url":"https://avatars.githubusercontent.com/u/626419?v=3","gravatar_id":"","url":"https://api.github.com/users/jbecla","html_url":"https://github.com/jbecla","followers_url":"https://api.github.com/users/jbecla/followers","following_url":"https://api.github.com/users/jbecla/following{/other_user}","gists_url":"https://api.github.com/users/jbecla/gists{/gist_id}","starred_url":"https://api.github.com/users/jbecla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbecla/subscriptions","organizations_url":"https://api.github.com/users/jbecla/orgs","repos_url":"https://api.github.com/users/jbecla/repos","events_url":"https://api.github.com/users/jbecla/events{/privacy}","received_events_url":"https://api.github.com/users/jbecla/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-04-21T15:43:15Z","updated_at":"2015-04-24T18:13:15Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9d223a9cb2536cfbc6417fb1fc402f584dae486d","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/lsst/qserv/pulls/61/commits","review_comments_url":"https://api.github.com/repos/lsst/qserv/pulls/61/comments","review_comment_url":"https://api.github.com/repos/lsst/qserv/pulls/comments{/number}","comments_url":"https://api.github.com/repos/lsst/qserv/issues/61/comments","statuses_url":"https://api.github.com/repos/lsst/qserv/statuses/d1f4afd2d471db696f597b5eece64a487c0b544e","head":{"label":"lsst:u/fjammes/DM-2341","ref":"u/fjammes/DM-2341","sha":"d1f4afd2d471db696f597b5eece64a487c0b544e","user":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"repo":{"id":23013460,"name":"qserv","full_name":"lsst/qserv","owner":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/lsst/qserv","description":"LSST Query Services","fork":false,"url":"https://api.github.com/repos/lsst/qserv","forks_url":"https://api.github.com/repos/lsst/qserv/forks","keys_url":"https://api.github.com/repos/lsst/qserv/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lsst/qserv/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lsst/qserv/teams","hooks_url":"https://api.github.com/repos/lsst/qserv/hooks","issue_events_url":"https://api.github.com/repos/lsst/qserv/issues/events{/number}","events_url":"https://api.github.com/repos/lsst/qserv/events","assignees_url":"https://api.github.com/repos/lsst/qserv/assignees{/user}","branches_url":"https://api.github.com/repos/lsst/qserv/branches{/branch}","tags_url":"https://api.github.com/repos/lsst/qserv/tags","blobs_url":"https://api.github.com/repos/lsst/qserv/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lsst/qserv/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lsst/qserv/git/refs{/sha}","trees_url":"https://api.github.com/repos/lsst/qserv/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lsst/qserv/statuses/{sha}","languages_url":"https://api.github.com/repos/lsst/qserv/languages","stargazers_url":"https://api.github.com/repos/lsst/qserv/stargazers","contributors_url":"https://api.github.com/repos/lsst/qserv/contributors","subscribers_url":"https://api.github.com/repos/lsst/qserv/subscribers","subscription_url":"https://api.github.com/repos/lsst/qserv/subscription","commits_url":"https://api.github.com/repos/lsst/qserv/commits{/sha}","git_commits_url":"https://api.github.com/repos/lsst/qserv/git/commits{/sha}","comments_url":"https://api.github.com/repos/lsst/qserv/comments{/number}","issue_comment_url":"https://api.github.com/repos/lsst/qserv/issues/comments{/number}","contents_url":"https://api.github.com/repos/lsst/qserv/contents/{+path}","compare_url":"https://api.github.com/repos/lsst/qserv/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lsst/qserv/merges","archive_url":"https://api.github.com/repos/lsst/qserv/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lsst/qserv/downloads","issues_url":"https://api.github.com/repos/lsst/qserv/issues{/number}","pulls_url":"https://api.github.com/repos/lsst/qserv/pulls{/number}","milestones_url":"https://api.github.com/repos/lsst/qserv/milestones{/number}","notifications_url":"https://api.github.com/repos/lsst/qserv/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lsst/qserv/labels{/name}","releases_url":"https://api.github.com/repos/lsst/qserv/releases{/id}","created_at":"2014-08-16T07:50:36Z","updated_at":"2015-04-24T17:04:59Z","pushed_at":"2015-04-24T17:51:31Z","git_url":"git://github.com/lsst/qserv.git","ssh_url":"git@github.com:lsst/qserv.git","clone_url":"https://github.com/lsst/qserv.git","svn_url":"https://github.com/lsst/qserv","homepage":"http://dm.lsst.org/","size":19838,"stargazers_count":4,"watchers_count":4,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":11,"forks":5,"open_issues":11,"watchers":4,"default_branch":"master"}},"base":{"label":"lsst:master","ref":"master","sha":"7ff5fd648a341ee7687a56e2ef2a6b195f6e0a34","user":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"repo":{"id":23013460,"name":"qserv","full_name":"lsst/qserv","owner":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/lsst/qserv","description":"LSST Query Services","fork":false,"url":"https://api.github.com/repos/lsst/qserv","forks_url":"https://api.github.com/repos/lsst/qserv/forks","keys_url":"https://api.github.com/repos/lsst/qserv/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lsst/qserv/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lsst/qserv/teams","hooks_url":"https://api.github.com/repos/lsst/qserv/hooks","issue_events_url":"https://api.github.com/repos/lsst/qserv/issues/events{/number}","events_url":"https://api.github.com/repos/lsst/qserv/events","assignees_url":"https://api.github.com/repos/lsst/qserv/assignees{/user}","branches_url":"https://api.github.com/repos/lsst/qserv/branches{/branch}","tags_url":"https://api.github.com/repos/lsst/qserv/tags","blobs_url":"https://api.github.com/repos/lsst/qserv/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lsst/qserv/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lsst/qserv/git/refs{/sha}","trees_url":"https://api.github.com/repos/lsst/qserv/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lsst/qserv/statuses/{sha}","languages_url":"https://api.github.com/repos/lsst/qserv/languages","stargazers_url":"https://api.github.com/repos/lsst/qserv/stargazers","contributors_url":"https://api.github.com/repos/lsst/qserv/contributors","subscribers_url":"https://api.github.com/repos/lsst/qserv/subscribers","subscription_url":"https://api.github.com/repos/lsst/qserv/subscription","commits_url":"https://api.github.com/repos/lsst/qserv/commits{/sha}","git_commits_url":"https://api.github.com/repos/lsst/qserv/git/commits{/sha}","comments_url":"https://api.github.com/repos/lsst/qserv/comments{/number}","issue_comment_url":"https://api.github.com/repos/lsst/qserv/issues/comments{/number}","contents_url":"https://api.github.com/repos/lsst/qserv/contents/{+path}","compare_url":"https://api.github.com/repos/lsst/qserv/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lsst/qserv/merges","archive_url":"https://api.github.com/repos/lsst/qserv/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lsst/qserv/downloads","issues_url":"https://api.github.com/repos/lsst/qserv/issues{/number}","pulls_url":"https://api.github.com/repos/lsst/qserv/pulls{/number}","milestones_url":"https://api.github.com/repos/lsst/qserv/milestones{/number}","notifications_url":"https://api.github.com/repos/lsst/qserv/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lsst/qserv/labels{/name}","releases_url":"https://api.github.com/repos/lsst/qserv/releases{/id}","created_at":"2014-08-16T07:50:36Z","updated_at":"2015-04-24T17:04:59Z","pushed_at":"2015-04-24T17:51:31Z","git_url":"git://github.com/lsst/qserv.git","ssh_url":"git@github.com:lsst/qserv.git","clone_url":"https://github.com/lsst/qserv.git","svn_url":"https://github.com/lsst/qserv","homepage":"http://dm.lsst.org/","size":19838,"stargazers_count":4,"watchers_count":4,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":11,"forks":5,"open_issues":11,"watchers":4,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/lsst/qserv/pulls/61"},"html":{"href":"https://github.com/lsst/qserv/pull/61"},"issue":{"href":"https://api.github.com/repos/lsst/qserv/issues/61"},"comments":{"href":"https://api.github.com/repos/lsst/qserv/issues/61/comments"},"review_comments":{"href":"https://api.github.com/repos/lsst/qserv/pulls/61/comments"},"review_comment":{"href":"https://api.github.com/repos/lsst/qserv/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/lsst/qserv/pulls/61/commits"},"statuses":{"href":"https://api.github.com/repos/lsst/qserv/statuses/d1f4afd2d471db696f597b5eece64a487c0b544e"}}}}