{"action":"created","comment":{"url":"https://api.github.com/repos/catkin/catkin_tools/pulls/comments/27349321","id":27349321,"diff_hunk":"@@ -0,0 +1,233 @@\n+# Copyright 2014 Open Source Robotics Foundation, Inc.\n+#\n+# Licensed under the Apache License, Version 2.0 (the \"License\");\n+# you may not use this file except in compliance with the License.\n+# You may obtain a copy of the License at\n+#\n+#     http://www.apache.org/licenses/LICENSE-2.0\n+#\n+# Unless required by applicable law or agreed to in writing, software\n+# distributed under the License is distributed on an \"AS IS\" BASIS,\n+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+# See the License for the specific language governing permissions and\n+# limitations under the License.\n+\n+from __future__ import print_function\n+\n+from multiprocessing import cpu_count\n+from tempfile import mkstemp\n+from termios import FIONREAD\n+\n+import array\n+import fcntl\n+import os\n+import psutil\n+import subprocess\n+import time\n+\n+from .common import log\n+\n+JOBSERVER_SUPPORT_MAKEFILE = b'''\n+all:\n+\\techo $(MAKEFLAGS) | grep -- '--jobserver-fds'\n+'''\n+\n+\n+class _MakeJobServer:\n+    \"\"\"\n+    This class implements a GNU make job server.\n+    \"\"\"\n+\n+    # Singleton jobserver\n+    _singleton = None\n+\n+    def __init__(self, num_jobs=None, max_load=None, max_mem=None):\n+        \"\"\"\n+        :param num_jobs: the maximum number of jobs available\n+        :param max_load: do not dispatch additional jobs if this system load\n+        value is exceeded\n+        :param max_mem: do not dispatch additional jobs if system physical\n+        memory usage exceeds this value\n+        \"\"\"\n+\n+        assert(_MakeJobServer._singleton is None)\n+\n+        if not num_jobs:\n+            try:\n+                num_jobs = cpu_count()\n+            except NotImplementedError:\n+                log('@{yf}WARNING: Failed to determine the cpu_count, falling back to 1 jobs as the default.@|')\n+                num_jobs = 1\n+        else:\n+            num_jobs = int(num_jobs)\n+\n+        self.num_jobs = num_jobs\n+        self.max_load = max_load\n+        self.max_mem = max_mem\n+        self.job_pipe = os.pipe()\n+\n+        # Initialize the pipe with num_jobs tokens\n+        for i in range(num_jobs):\n+            os.write(self.job_pipe[1], b'+')\n+\n+    def _obtain(self):\n+        \"\"\"\n+        Obtain a job server token. Be sure to call release() to avoid\n+        deadlocks.\n+        \"\"\"\n+\n+        while True:\n+            # make sure we're observing load maximums\n+            if self.max_load is not None:\n+                try:\n+                    max_load = 8.0\n+                    load = os.getloadavg()\n+                    if jobserver_running_jobs() > 0 and load[1] > self.max_load:\n+                        time.sleep(0.01)\n+                        continue\n+                except NotImplementedError:\n+                    pass\n+\n+            # make sure we're observing memory maximum\n+            if self.max_mem is not None:\n+                mem_usage = psutil.phymem_usage()\n+                if jobserver_running_jobs() > 0 and mem_usage.percent > self.max_mem:\n+                    time.sleep(0.01)\n+                    continue\n+","path":"catkin_tools/make_jobserver.py","position":144,"original_position":97,"commit_id":"c0ae6e067f9ce375946850f78e04cf71a34de879","original_commit_id":"ffc9fcd8b1a25fc88adaad2d115723163100be1a","user":{"login":"jbohren","id":1253450,"avatar_url":"https://avatars.githubusercontent.com/u/1253450?v=3","gravatar_id":"","url":"https://api.github.com/users/jbohren","html_url":"https://github.com/jbohren","followers_url":"https://api.github.com/users/jbohren/followers","following_url":"https://api.github.com/users/jbohren/following{/other_user}","gists_url":"https://api.github.com/users/jbohren/gists{/gist_id}","starred_url":"https://api.github.com/users/jbohren/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbohren/subscriptions","organizations_url":"https://api.github.com/users/jbohren/orgs","repos_url":"https://api.github.com/users/jbohren/repos","events_url":"https://api.github.com/users/jbohren/events{/privacy}","received_events_url":"https://api.github.com/users/jbohren/received_events","type":"User","site_admin":false},"body":"Yeah, it's already in a single loop.","created_at":"2015-03-28T22:32:45Z","updated_at":"2015-03-28T22:32:45Z","html_url":"https://github.com/catkin/catkin_tools/pull/155#discussion_r27349321","pull_request_url":"https://api.github.com/repos/catkin/catkin_tools/pulls/155","_links":{"self":{"href":"https://api.github.com/repos/catkin/catkin_tools/pulls/comments/27349321"},"html":{"href":"https://github.com/catkin/catkin_tools/pull/155#discussion_r27349321"},"pull_request":{"href":"https://api.github.com/repos/catkin/catkin_tools/pulls/155"}}},"pull_request":{"url":"https://api.github.com/repos/catkin/catkin_tools/pulls/155","id":30821998,"html_url":"https://github.com/catkin/catkin_tools/pull/155","diff_url":"https://github.com/catkin/catkin_tools/pull/155.diff","patch_url":"https://github.com/catkin/catkin_tools/pull/155.patch","issue_url":"https://api.github.com/repos/catkin/catkin_tools/issues/155","number":155,"state":"open","locked":false,"title":"build: GNU Make jobserver to prevent resource glut (make_job_server rebased on master)","user":{"login":"jbohren","id":1253450,"avatar_url":"https://avatars.githubusercontent.com/u/1253450?v=3","gravatar_id":"","url":"https://api.github.com/users/jbohren","html_url":"https://github.com/jbohren","followers_url":"https://api.github.com/users/jbohren/followers","following_url":"https://api.github.com/users/jbohren/following{/other_user}","gists_url":"https://api.github.com/users/jbohren/gists{/gist_id}","starred_url":"https://api.github.com/users/jbohren/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbohren/subscriptions","organizations_url":"https://api.github.com/users/jbohren/orgs","repos_url":"https://api.github.com/users/jbohren/repos","events_url":"https://api.github.com/users/jbohren/events{/privacy}","received_events_url":"https://api.github.com/users/jbohren/received_events","type":"User","site_admin":false},"body":"This is a rebase and extension of @xqms's jobserver prototype in PR #85 which was created to fix #84. Without the jobserver, `catkin build` will use too many resources on machines with multiple cores.\r\n\r\nBehavior using this PR:\r\n- `catkin build --no-jobserver` -- current behavior (maxes out all jobs for all packages unless using another jobserver like distcc)\r\n- `catkin build` -- build using the jobserver and as many jobs as CPUs\r\n- `catkin build --jobserver` -- explicit version of the line above\r\n- `catkin build -jN`-- build using the jobserver and `N` jobs\r\n- `MAKEFLAGS=\"-jN\" catkin build`-- build using the jobserver and `N` jobs\r\n- `catkin config --no-jobserver` -- disable the jobserver for future builds\r\n- `catkin build -lV` -- don't create more than one job at a time if the system load is greater than `V`\r\n- `catkin build --mem-limit P` -- don't create more than one job at a time if more than `P`% of the system memory is used\r\n\r\nOutstanding issues:\r\n- [x] The jobserver behavior isn't documented\r\n- [x] The status line reads `?/0 Jobs` when the jobserver is disabled, it should display something friendlier\r\n- [x] change `internal_make_jobserver` switches `use_internal_make_jobserver`\r\n - https://github.com/catkin/catkin_tools/pull/155/files#r26704312\r\n - https://github.com/catkin/catkin_tools/pull/155/files#r26705185\r\n- [x] [document `configure_make_args`](https://github.com/catkin/catkin_tools/pull/155/files#r26703936)\r\n- [x] clean up jobserver API\r\n - [checking support](https://github.com/catkin/catkin_tools/pull/155/files#r26704509)\r\n\r\n\r\nFuture work:\r\n- The number of jobs (`-jN`) and the number of packages (`-pM`) are still decoupled. The jobserver will limit the number of `Make` instances, but maybe this is ok?\r\n- The current internal jobserver implementation does not monitor load or memory usage. These would be good arguments to support.","created_at":"2015-03-10T04:28:00Z","updated_at":"2015-03-28T22:32:45Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b81209dba47f4b6c71ede9b1bc197bfbdc690c90","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/catkin/catkin_tools/pulls/155/commits","review_comments_url":"https://api.github.com/repos/catkin/catkin_tools/pulls/155/comments","review_comment_url":"https://api.github.com/repos/catkin/catkin_tools/pulls/comments{/number}","comments_url":"https://api.github.com/repos/catkin/catkin_tools/issues/155/comments","statuses_url":"https://api.github.com/repos/catkin/catkin_tools/statuses/c0ae6e067f9ce375946850f78e04cf71a34de879","head":{"label":"jbohren-forks:make_job_server","ref":"make_job_server","sha":"c0ae6e067f9ce375946850f78e04cf71a34de879","user":{"login":"jbohren-forks","id":2775020,"avatar_url":"https://avatars.githubusercontent.com/u/2775020?v=3","gravatar_id":"","url":"https://api.github.com/users/jbohren-forks","html_url":"https://github.com/jbohren-forks","followers_url":"https://api.github.com/users/jbohren-forks/followers","following_url":"https://api.github.com/users/jbohren-forks/following{/other_user}","gists_url":"https://api.github.com/users/jbohren-forks/gists{/gist_id}","starred_url":"https://api.github.com/users/jbohren-forks/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbohren-forks/subscriptions","organizations_url":"https://api.github.com/users/jbohren-forks/orgs","repos_url":"https://api.github.com/users/jbohren-forks/repos","events_url":"https://api.github.com/users/jbohren-forks/events{/privacy}","received_events_url":"https://api.github.com/users/jbohren-forks/received_events","type":"Organization","site_admin":false},"repo":{"id":17716068,"name":"catkin_tools","full_name":"jbohren-forks/catkin_tools","owner":{"login":"jbohren-forks","id":2775020,"avatar_url":"https://avatars.githubusercontent.com/u/2775020?v=3","gravatar_id":"","url":"https://api.github.com/users/jbohren-forks","html_url":"https://github.com/jbohren-forks","followers_url":"https://api.github.com/users/jbohren-forks/followers","following_url":"https://api.github.com/users/jbohren-forks/following{/other_user}","gists_url":"https://api.github.com/users/jbohren-forks/gists{/gist_id}","starred_url":"https://api.github.com/users/jbohren-forks/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbohren-forks/subscriptions","organizations_url":"https://api.github.com/users/jbohren-forks/orgs","repos_url":"https://api.github.com/users/jbohren-forks/repos","events_url":"https://api.github.com/users/jbohren-forks/events{/privacy}","received_events_url":"https://api.github.com/users/jbohren-forks/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/jbohren-forks/catkin_tools","description":"Command line tools for working with catkin","fork":true,"url":"https://api.github.com/repos/jbohren-forks/catkin_tools","forks_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/forks","keys_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/teams","hooks_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/hooks","issue_events_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/issues/events{/number}","events_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/events","assignees_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/assignees{/user}","branches_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/branches{/branch}","tags_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/tags","blobs_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/git/refs{/sha}","trees_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/statuses/{sha}","languages_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/languages","stargazers_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/stargazers","contributors_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/contributors","subscribers_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/subscribers","subscription_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/subscription","commits_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/commits{/sha}","git_commits_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/git/commits{/sha}","comments_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/comments{/number}","issue_comment_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/issues/comments{/number}","contents_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/contents/{+path}","compare_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/merges","archive_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/downloads","issues_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/issues{/number}","pulls_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/pulls{/number}","milestones_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/milestones{/number}","notifications_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/labels{/name}","releases_url":"https://api.github.com/repos/jbohren-forks/catkin_tools/releases{/id}","created_at":"2014-03-13T16:03:30Z","updated_at":"2014-07-03T20:34:24Z","pushed_at":"2015-03-28T22:31:42Z","git_url":"git://github.com/jbohren-forks/catkin_tools.git","ssh_url":"git@github.com:jbohren-forks/catkin_tools.git","clone_url":"https://github.com/jbohren-forks/catkin_tools.git","svn_url":"https://github.com/jbohren-forks/catkin_tools","homepage":null,"size":2754,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"metadata-file"}},"base":{"label":"catkin:master","ref":"master","sha":"c5f659474c9697584642e6103c39a60a926259a3","user":{"login":"catkin","id":5697303,"avatar_url":"https://avatars.githubusercontent.com/u/5697303?v=3","gravatar_id":"","url":"https://api.github.com/users/catkin","html_url":"https://github.com/catkin","followers_url":"https://api.github.com/users/catkin/followers","following_url":"https://api.github.com/users/catkin/following{/other_user}","gists_url":"https://api.github.com/users/catkin/gists{/gist_id}","starred_url":"https://api.github.com/users/catkin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/catkin/subscriptions","organizations_url":"https://api.github.com/users/catkin/orgs","repos_url":"https://api.github.com/users/catkin/repos","events_url":"https://api.github.com/users/catkin/events{/privacy}","received_events_url":"https://api.github.com/users/catkin/received_events","type":"Organization","site_admin":false},"repo":{"id":17157565,"name":"catkin_tools","full_name":"catkin/catkin_tools","owner":{"login":"catkin","id":5697303,"avatar_url":"https://avatars.githubusercontent.com/u/5697303?v=3","gravatar_id":"","url":"https://api.github.com/users/catkin","html_url":"https://github.com/catkin","followers_url":"https://api.github.com/users/catkin/followers","following_url":"https://api.github.com/users/catkin/following{/other_user}","gists_url":"https://api.github.com/users/catkin/gists{/gist_id}","starred_url":"https://api.github.com/users/catkin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/catkin/subscriptions","organizations_url":"https://api.github.com/users/catkin/orgs","repos_url":"https://api.github.com/users/catkin/repos","events_url":"https://api.github.com/users/catkin/events{/privacy}","received_events_url":"https://api.github.com/users/catkin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/catkin/catkin_tools","description":"Command line tools for working with catkin","fork":false,"url":"https://api.github.com/repos/catkin/catkin_tools","forks_url":"https://api.github.com/repos/catkin/catkin_tools/forks","keys_url":"https://api.github.com/repos/catkin/catkin_tools/keys{/key_id}","collaborators_url":"https://api.github.com/repos/catkin/catkin_tools/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/catkin/catkin_tools/teams","hooks_url":"https://api.github.com/repos/catkin/catkin_tools/hooks","issue_events_url":"https://api.github.com/repos/catkin/catkin_tools/issues/events{/number}","events_url":"https://api.github.com/repos/catkin/catkin_tools/events","assignees_url":"https://api.github.com/repos/catkin/catkin_tools/assignees{/user}","branches_url":"https://api.github.com/repos/catkin/catkin_tools/branches{/branch}","tags_url":"https://api.github.com/repos/catkin/catkin_tools/tags","blobs_url":"https://api.github.com/repos/catkin/catkin_tools/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/catkin/catkin_tools/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/catkin/catkin_tools/git/refs{/sha}","trees_url":"https://api.github.com/repos/catkin/catkin_tools/git/trees{/sha}","statuses_url":"https://api.github.com/repos/catkin/catkin_tools/statuses/{sha}","languages_url":"https://api.github.com/repos/catkin/catkin_tools/languages","stargazers_url":"https://api.github.com/repos/catkin/catkin_tools/stargazers","contributors_url":"https://api.github.com/repos/catkin/catkin_tools/contributors","subscribers_url":"https://api.github.com/repos/catkin/catkin_tools/subscribers","subscription_url":"https://api.github.com/repos/catkin/catkin_tools/subscription","commits_url":"https://api.github.com/repos/catkin/catkin_tools/commits{/sha}","git_commits_url":"https://api.github.com/repos/catkin/catkin_tools/git/commits{/sha}","comments_url":"https://api.github.com/repos/catkin/catkin_tools/comments{/number}","issue_comment_url":"https://api.github.com/repos/catkin/catkin_tools/issues/comments{/number}","contents_url":"https://api.github.com/repos/catkin/catkin_tools/contents/{+path}","compare_url":"https://api.github.com/repos/catkin/catkin_tools/compare/{base}...{head}","merges_url":"https://api.github.com/repos/catkin/catkin_tools/merges","archive_url":"https://api.github.com/repos/catkin/catkin_tools/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/catkin/catkin_tools/downloads","issues_url":"https://api.github.com/repos/catkin/catkin_tools/issues{/number}","pulls_url":"https://api.github.com/repos/catkin/catkin_tools/pulls{/number}","milestones_url":"https://api.github.com/repos/catkin/catkin_tools/milestones{/number}","notifications_url":"https://api.github.com/repos/catkin/catkin_tools/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/catkin/catkin_tools/labels{/name}","releases_url":"https://api.github.com/repos/catkin/catkin_tools/releases{/id}","created_at":"2014-02-25T01:43:02Z","updated_at":"2015-03-27T18:29:17Z","pushed_at":"2015-03-28T22:31:43Z","git_url":"git://github.com/catkin/catkin_tools.git","ssh_url":"git@github.com:catkin/catkin_tools.git","clone_url":"https://github.com/catkin/catkin_tools.git","svn_url":"https://github.com/catkin/catkin_tools","homepage":null,"size":2466,"stargazers_count":13,"watchers_count":13,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":13,"mirror_url":null,"open_issues_count":49,"forks":13,"open_issues":49,"watchers":13,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/catkin/catkin_tools/pulls/155"},"html":{"href":"https://github.com/catkin/catkin_tools/pull/155"},"issue":{"href":"https://api.github.com/repos/catkin/catkin_tools/issues/155"},"comments":{"href":"https://api.github.com/repos/catkin/catkin_tools/issues/155/comments"},"review_comments":{"href":"https://api.github.com/repos/catkin/catkin_tools/pulls/155/comments"},"review_comment":{"href":"https://api.github.com/repos/catkin/catkin_tools/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/catkin/catkin_tools/pulls/155/commits"},"statuses":{"href":"https://api.github.com/repos/catkin/catkin_tools/statuses/c0ae6e067f9ce375946850f78e04cf71a34de879"}}}}