{"action":"created","comment":{"url":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/comments/24978042","id":24978042,"diff_hunk":"@@ -0,0 +1,195 @@\n+##\n+# Copyright 2015-2015 Ghent University\n+# Copyright 2015 S3IT, University of Zurich\n+#\n+# This file is part of EasyBuild,\n+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),\n+# with support of Ghent University (http://ugent.be/hpc),\n+# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),\n+# the Hercules foundation (http://www.herculesstichting.be/in_English)\n+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).\n+#\n+# http://github.com/hpcugent/easybuild\n+#\n+# EasyBuild is free software: you can redistribute it and/or modify\n+# it under the terms of the GNU General Public License as published by\n+# the Free Software Foundation v2.\n+#\n+# EasyBuild is distributed in the hope that it will be useful,\n+# but WITHOUT ANY WARRANTY; without even the implied warranty of\n+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+# GNU General Public License for more details.\n+#\n+# You should have received a copy of the GNU General Public License\n+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.\n+##\n+\"\"\"Interface for submitting jobs via GC3Pie\"\"\"\n+\n+\n+import os\n+import time\n+\n+try:\n+    import gc3libs\n+    from gc3libs import Application, Run, create_engine\n+    from gc3libs.core import Engine\n+    from gc3libs.quantity import hours as hr\n+    from gc3libs.workflow import DependentTaskCollection\n+    HAVE_GC3PIE = True\n+except ImportError:\n+    HAVE_GC3PIE = False\n+\n+from easybuild.tools.build_log import print_msg\n+from easybuild.tools.job import JobServer\n+\n+from vsc.utils import fancylogger\n+\n+# inject EasyBuild logger into GC3Pie\n+gc3libs.log = fancylogger.getLogger('gc3pie', fname=False)\n+# make handling of log.error compatible with stdlib logging\n+gc3libs.log.raiseError = False\n+\n+\n+# eb --job --job-backend=GC3Pie\n+class GC3Pie(JobServer):\n+    \"\"\"\n+    Use the GC3Pie__ framework to submit and monitor compilation jobs.\n+\n+    In contrast with acessing an external service, GC3Pie implements\n+    its own workflow manager, which means ``eb --job\n+    --job-backend=GC3Pie`` will keep running until all jobs have\n+    terminated.\n+\n+    .. __: http://gc3pie.googlecode.com/\n+    \"\"\"\n+\n+    USABLE = HAVE_GC3PIE\n+\n+    def begin(self):\n+        \"\"\"\n+        Start a bulk job submission.\n+\n+        Removes any reference to previously-submitted jobs.\n+        \"\"\"\n+        self._jobs = DependentTaskCollection()\n+\n+    def make_job(self, script, name, env_vars=None, hours=None, cores=None):\n+        \"\"\"\n+        Create and return a job object with the given parameters.\n+\n+        First argument `server` is an instance of the corresponding\n+        `JobServer` class, i.e., a `GC3Pie`:class: instance in this case.\n+\n+        Second argument `script` is the content of the job script\n+        itself, i.e., the sequence of shell commands that will be\n+        executed.\n+\n+        Third argument `name` sets the job human-readable name.\n+\n+        Fourth (optional) argument `env_vars` is a dictionary with\n+        key-value pairs of environment variables that should be passed\n+        on to the job.\n+\n+        Fifth and sith (optional) arguments `hours` and `cores` should be\n+        integer values:\n+        * hours must be in the range 1 .. MAX_WALLTIME;\n+        * cores depends on which cluster the job is being run.\n+        \"\"\"\n+        extra_args = {\n+            'jobname': name, # job name in GC3Pie\n+            'name':    name, # same in EasyBuild\n+        }\n+        if env_vars:\n+            extra_args['environment'] = env_vars\n+        if hours:\n+            extra_args['requested_walltime'] = hours*hr\n+        if cores:\n+            extra_args['requested_cores'] = cores\n+        return Application(\n+            # arguments\n+            ['/bin/sh', '-c', script],\n+            # no need to stage files in or out\n+            inputs=[],\n+            outputs=[],\n+            # where should the output (STDOUT/STDERR) files be downloaded to?\n+            output_dir=('%s/easybuild-jobs/%s' % (os.getcwd(), name)),","path":"easybuild/tools/job/gc3pie.py","position":115,"original_position":115,"commit_id":"5aa1c99457338506b7ba86e9b90f8ad733aa92d5","original_commit_id":"5aa1c99457338506b7ba86e9b90f8ad733aa92d5","user":{"login":"riccardomurri","id":447228,"avatar_url":"https://avatars.githubusercontent.com/u/447228?v=3","gravatar_id":"","url":"https://api.github.com/users/riccardomurri","html_url":"https://github.com/riccardomurri","followers_url":"https://api.github.com/users/riccardomurri/followers","following_url":"https://api.github.com/users/riccardomurri/following{/other_user}","gists_url":"https://api.github.com/users/riccardomurri/gists{/gist_id}","starred_url":"https://api.github.com/users/riccardomurri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/riccardomurri/subscriptions","organizations_url":"https://api.github.com/users/riccardomurri/orgs","repos_url":"https://api.github.com/users/riccardomurri/repos","events_url":"https://api.github.com/users/riccardomurri/events{/privacy}","received_events_url":"https://api.github.com/users/riccardomurri/received_events","type":"User","site_admin":false},"body":"how would you like it to be configurable? command-line option or what?","created_at":"2015-02-19T10:26:59Z","updated_at":"2015-02-19T10:26:59Z","html_url":"https://github.com/hpcugent/easybuild-framework/pull/1008#discussion_r24978042","pull_request_url":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/1008","_links":{"self":{"href":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/comments/24978042"},"html":{"href":"https://github.com/hpcugent/easybuild-framework/pull/1008#discussion_r24978042"},"pull_request":{"href":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/1008"}}},"pull_request":{"url":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/1008","id":19749185,"html_url":"https://github.com/hpcugent/easybuild-framework/pull/1008","diff_url":"https://github.com/hpcugent/easybuild-framework/pull/1008.diff","patch_url":"https://github.com/hpcugent/easybuild-framework/pull/1008.patch","issue_url":"https://api.github.com/repos/hpcugent/easybuild-framework/issues/1008","number":1008,"state":"open","locked":false,"title":"Support parallel builds with GC3Pie (#1006, WIP)","user":{"login":"riccardomurri","id":447228,"avatar_url":"https://avatars.githubusercontent.com/u/447228?v=3","gravatar_id":"","url":"https://api.github.com/users/riccardomurri","html_url":"https://github.com/riccardomurri","followers_url":"https://api.github.com/users/riccardomurri/followers","following_url":"https://api.github.com/users/riccardomurri/following{/other_user}","gists_url":"https://api.github.com/users/riccardomurri/gists{/gist_id}","starred_url":"https://api.github.com/users/riccardomurri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/riccardomurri/subscriptions","organizations_url":"https://api.github.com/users/riccardomurri/orgs","repos_url":"https://api.github.com/users/riccardomurri/repos","events_url":"https://api.github.com/users/riccardomurri/events{/privacy}","received_events_url":"https://api.github.com/users/riccardomurri/received_events","type":"User","site_admin":false},"body":"This is meant to be the implementation of #1006.\r\n\r\nIt's currently work-in-progress and in very early stages too.","created_at":"2014-08-13T20:04:26Z","updated_at":"2015-02-19T10:26:59Z","closed_at":null,"merged_at":null,"merge_commit_sha":"641c867aa8e2c0d8e438f9fd96ebf926f7d2a76b","assignee":null,"milestone":{"url":"https://api.github.com/repos/hpcugent/easybuild-framework/milestones/28","labels_url":"https://api.github.com/repos/hpcugent/easybuild-framework/milestones/28/labels","id":780562,"number":28,"title":"v1.16","description":"easybuild-framework version 1.16, bugfixes and enhancements","creator":{"login":"boegel","id":620876,"avatar_url":"https://avatars.githubusercontent.com/u/620876?v=3","gravatar_id":"","url":"https://api.github.com/users/boegel","html_url":"https://github.com/boegel","followers_url":"https://api.github.com/users/boegel/followers","following_url":"https://api.github.com/users/boegel/following{/other_user}","gists_url":"https://api.github.com/users/boegel/gists{/gist_id}","starred_url":"https://api.github.com/users/boegel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/boegel/subscriptions","organizations_url":"https://api.github.com/users/boegel/orgs","repos_url":"https://api.github.com/users/boegel/repos","events_url":"https://api.github.com/users/boegel/events{/privacy}","received_events_url":"https://api.github.com/users/boegel/received_events","type":"User","site_admin":false},"open_issues":8,"closed_issues":3,"state":"open","created_at":"2014-09-06T08:09:05Z","updated_at":"2014-12-10T07:36:46Z","due_on":"2014-10-16T07:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/1008/commits","review_comments_url":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/1008/comments","review_comment_url":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/comments/{number}","comments_url":"https://api.github.com/repos/hpcugent/easybuild-framework/issues/1008/comments","statuses_url":"https://api.github.com/repos/hpcugent/easybuild-framework/statuses/5aa1c99457338506b7ba86e9b90f8ad733aa92d5","head":{"label":"gc3-uzh-ch:gc3pie","ref":"gc3pie","sha":"5aa1c99457338506b7ba86e9b90f8ad733aa92d5","user":{"login":"gc3-uzh-ch","id":3878695,"avatar_url":"https://avatars.githubusercontent.com/u/3878695?v=3","gravatar_id":"","url":"https://api.github.com/users/gc3-uzh-ch","html_url":"https://github.com/gc3-uzh-ch","followers_url":"https://api.github.com/users/gc3-uzh-ch/followers","following_url":"https://api.github.com/users/gc3-uzh-ch/following{/other_user}","gists_url":"https://api.github.com/users/gc3-uzh-ch/gists{/gist_id}","starred_url":"https://api.github.com/users/gc3-uzh-ch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gc3-uzh-ch/subscriptions","organizations_url":"https://api.github.com/users/gc3-uzh-ch/orgs","repos_url":"https://api.github.com/users/gc3-uzh-ch/repos","events_url":"https://api.github.com/users/gc3-uzh-ch/events{/privacy}","received_events_url":"https://api.github.com/users/gc3-uzh-ch/received_events","type":"Organization","site_admin":false},"repo":{"id":22921440,"name":"easybuild-framework","full_name":"gc3-uzh-ch/easybuild-framework","owner":{"login":"gc3-uzh-ch","id":3878695,"avatar_url":"https://avatars.githubusercontent.com/u/3878695?v=3","gravatar_id":"","url":"https://api.github.com/users/gc3-uzh-ch","html_url":"https://github.com/gc3-uzh-ch","followers_url":"https://api.github.com/users/gc3-uzh-ch/followers","following_url":"https://api.github.com/users/gc3-uzh-ch/following{/other_user}","gists_url":"https://api.github.com/users/gc3-uzh-ch/gists{/gist_id}","starred_url":"https://api.github.com/users/gc3-uzh-ch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gc3-uzh-ch/subscriptions","organizations_url":"https://api.github.com/users/gc3-uzh-ch/orgs","repos_url":"https://api.github.com/users/gc3-uzh-ch/repos","events_url":"https://api.github.com/users/gc3-uzh-ch/events{/privacy}","received_events_url":"https://api.github.com/users/gc3-uzh-ch/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/gc3-uzh-ch/easybuild-framework","description":"EasyBuild is a software installation framework in Python that allows you to install software in a structured and robust way. ","fork":true,"url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework","forks_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/forks","keys_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/keys{/key_id}","collaborators_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/teams","hooks_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/hooks","issue_events_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/issues/events{/number}","events_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/events","assignees_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/assignees{/user}","branches_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/branches{/branch}","tags_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/tags","blobs_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/git/refs{/sha}","trees_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/git/trees{/sha}","statuses_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/statuses/{sha}","languages_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/languages","stargazers_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/stargazers","contributors_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/contributors","subscribers_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/subscribers","subscription_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/subscription","commits_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/commits{/sha}","git_commits_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/git/commits{/sha}","comments_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/comments{/number}","issue_comment_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/issues/comments/{number}","contents_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/contents/{+path}","compare_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/compare/{base}...{head}","merges_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/merges","archive_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/downloads","issues_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/issues{/number}","pulls_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/pulls{/number}","milestones_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/milestones{/number}","notifications_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/labels{/name}","releases_url":"https://api.github.com/repos/gc3-uzh-ch/easybuild-framework/releases{/id}","created_at":"2014-08-13T15:51:46Z","updated_at":"2014-08-13T15:51:33Z","pushed_at":"2015-02-16T14:50:09Z","git_url":"git://github.com/gc3-uzh-ch/easybuild-framework.git","ssh_url":"git@github.com:gc3-uzh-ch/easybuild-framework.git","clone_url":"https://github.com/gc3-uzh-ch/easybuild-framework.git","svn_url":"https://github.com/gc3-uzh-ch/easybuild-framework","homepage":"http://hpcugent.github.com/easybuild/","size":8921,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"hpcugent:develop","ref":"develop","sha":"e280ab0a54da189dbd388fa5f2f126039af66ca5","user":{"login":"hpcugent","id":1515263,"avatar_url":"https://avatars.githubusercontent.com/u/1515263?v=3","gravatar_id":"","url":"https://api.github.com/users/hpcugent","html_url":"https://github.com/hpcugent","followers_url":"https://api.github.com/users/hpcugent/followers","following_url":"https://api.github.com/users/hpcugent/following{/other_user}","gists_url":"https://api.github.com/users/hpcugent/gists{/gist_id}","starred_url":"https://api.github.com/users/hpcugent/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hpcugent/subscriptions","organizations_url":"https://api.github.com/users/hpcugent/orgs","repos_url":"https://api.github.com/users/hpcugent/repos","events_url":"https://api.github.com/users/hpcugent/events{/privacy}","received_events_url":"https://api.github.com/users/hpcugent/received_events","type":"Organization","site_admin":false},"repo":{"id":3662176,"name":"easybuild-framework","full_name":"hpcugent/easybuild-framework","owner":{"login":"hpcugent","id":1515263,"avatar_url":"https://avatars.githubusercontent.com/u/1515263?v=3","gravatar_id":"","url":"https://api.github.com/users/hpcugent","html_url":"https://github.com/hpcugent","followers_url":"https://api.github.com/users/hpcugent/followers","following_url":"https://api.github.com/users/hpcugent/following{/other_user}","gists_url":"https://api.github.com/users/hpcugent/gists{/gist_id}","starred_url":"https://api.github.com/users/hpcugent/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hpcugent/subscriptions","organizations_url":"https://api.github.com/users/hpcugent/orgs","repos_url":"https://api.github.com/users/hpcugent/repos","events_url":"https://api.github.com/users/hpcugent/events{/privacy}","received_events_url":"https://api.github.com/users/hpcugent/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hpcugent/easybuild-framework","description":"EasyBuild is a software installation framework in Python that allows you to install software in a structured and robust way. ","fork":false,"url":"https://api.github.com/repos/hpcugent/easybuild-framework","forks_url":"https://api.github.com/repos/hpcugent/easybuild-framework/forks","keys_url":"https://api.github.com/repos/hpcugent/easybuild-framework/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hpcugent/easybuild-framework/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hpcugent/easybuild-framework/teams","hooks_url":"https://api.github.com/repos/hpcugent/easybuild-framework/hooks","issue_events_url":"https://api.github.com/repos/hpcugent/easybuild-framework/issues/events{/number}","events_url":"https://api.github.com/repos/hpcugent/easybuild-framework/events","assignees_url":"https://api.github.com/repos/hpcugent/easybuild-framework/assignees{/user}","branches_url":"https://api.github.com/repos/hpcugent/easybuild-framework/branches{/branch}","tags_url":"https://api.github.com/repos/hpcugent/easybuild-framework/tags","blobs_url":"https://api.github.com/repos/hpcugent/easybuild-framework/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hpcugent/easybuild-framework/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hpcugent/easybuild-framework/git/refs{/sha}","trees_url":"https://api.github.com/repos/hpcugent/easybuild-framework/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hpcugent/easybuild-framework/statuses/{sha}","languages_url":"https://api.github.com/repos/hpcugent/easybuild-framework/languages","stargazers_url":"https://api.github.com/repos/hpcugent/easybuild-framework/stargazers","contributors_url":"https://api.github.com/repos/hpcugent/easybuild-framework/contributors","subscribers_url":"https://api.github.com/repos/hpcugent/easybuild-framework/subscribers","subscription_url":"https://api.github.com/repos/hpcugent/easybuild-framework/subscription","commits_url":"https://api.github.com/repos/hpcugent/easybuild-framework/commits{/sha}","git_commits_url":"https://api.github.com/repos/hpcugent/easybuild-framework/git/commits{/sha}","comments_url":"https://api.github.com/repos/hpcugent/easybuild-framework/comments{/number}","issue_comment_url":"https://api.github.com/repos/hpcugent/easybuild-framework/issues/comments/{number}","contents_url":"https://api.github.com/repos/hpcugent/easybuild-framework/contents/{+path}","compare_url":"https://api.github.com/repos/hpcugent/easybuild-framework/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hpcugent/easybuild-framework/merges","archive_url":"https://api.github.com/repos/hpcugent/easybuild-framework/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hpcugent/easybuild-framework/downloads","issues_url":"https://api.github.com/repos/hpcugent/easybuild-framework/issues{/number}","pulls_url":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls{/number}","milestones_url":"https://api.github.com/repos/hpcugent/easybuild-framework/milestones{/number}","notifications_url":"https://api.github.com/repos/hpcugent/easybuild-framework/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hpcugent/easybuild-framework/labels{/name}","releases_url":"https://api.github.com/repos/hpcugent/easybuild-framework/releases{/id}","created_at":"2012-03-08T17:07:43Z","updated_at":"2015-01-30T14:03:55Z","pushed_at":"2015-02-18T13:12:38Z","git_url":"git://github.com/hpcugent/easybuild-framework.git","ssh_url":"git@github.com:hpcugent/easybuild-framework.git","clone_url":"https://github.com/hpcugent/easybuild-framework.git","svn_url":"https://github.com/hpcugent/easybuild-framework","homepage":"http://hpcugent.github.com/easybuild/","size":14707,"stargazers_count":39,"watchers_count":39,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":54,"mirror_url":null,"open_issues_count":230,"forks":54,"open_issues":230,"watchers":39,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/1008"},"html":{"href":"https://github.com/hpcugent/easybuild-framework/pull/1008"},"issue":{"href":"https://api.github.com/repos/hpcugent/easybuild-framework/issues/1008"},"comments":{"href":"https://api.github.com/repos/hpcugent/easybuild-framework/issues/1008/comments"},"review_comments":{"href":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/1008/comments"},"review_comment":{"href":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/hpcugent/easybuild-framework/pulls/1008/commits"},"statuses":{"href":"https://api.github.com/repos/hpcugent/easybuild-framework/statuses/5aa1c99457338506b7ba86e9b90f8ad733aa92d5"}}}}