{"action":"created","comment":{"url":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/comments/41234766","id":41234766,"diff_hunk":"@@ -93,6 +94,49 @@ def fetch_profiles (sid, dburl=None, client=None, tgt=None, access=None, session\n             print \"Overriding remote sandbox: %s\" % sandbox_url\n \n         sandbox  = saga.filesystem.Directory (sandbox_url, session=session)\n+\n+        # Try to fetch a tarball of profiles, so that we can get them all in one (SAGA) go!\n+        PROFILES_TARBALL = 'profiles.tgz'\n+        tarball_available = False\n+        try:\n+            if sandbox.is_file(PROFILES_TARBALL):\n+                print \"Profiles tarball exists!\"\n+\n+                ftgt = saga.Url('%s/%s' % (tgt_url, PROFILES_TARBALL))\n+\n+                if skip_existing and os.path.isfile(ftgt.path) \\\n+                        and os.stat(ftgt.path).st_size > 0:\n+\n+                    print \"Skipping fetching of '%s/%s' to '%s'.\" % (sandbox_url, PROFILES_TARBALL, tgt_url)\n+                    tarball_available = True\n+                else:\n+\n+                    print \"Fetching '%s%s' to '%s'.\" % (sandbox_url, PROFILES_TARBALL, tgt_url)\n+                    prof_file = saga.filesystem.File(\"%s%s\" % (sandbox_url, PROFILES_TARBALL), session=session)\n+                    prof_file.copy(ftgt, flags=saga.filesystem.CREATE_PARENTS)\n+                    prof_file.close()\n+\n+                    tarball_available = True\n+            else:\n+                print \"Profiles tarball doesnt exists!\"\n+\n+        except saga.DoesNotExist:\n+            print \"exception(TODO): profiles tarball doesnt exists!\"\n+\n+        # We now have a local tarball\n+        if tarball_available:\n+            print \"Extracting tarball %s into '%s'.\" % (ftgt.path, tgt_url.path)\n+            tarball = tarfile.open(ftgt.path)\n+            tarball.extractall(tgt_url.path)\n+\n+            profiles = glob.glob(\"%s/*.prof\" % tgt_url.path)\n+            print \"Tarball %s extracted to '%s'.\" % (ftgt.path, profiles)\n+            ret.extend(profiles)\n+\n+            # If extract succeeded, no need to fetch individual profiles\n+            continue\n+\n+        # If we dont have a tarball (for whichever reason), fetch individual profiles","path":"src/radical/pilot/utils/session.py","position":54,"original_position":54,"commit_id":"f2cad900de33275b7fbf98f085048166e6c2906e","original_commit_id":"f2cad900de33275b7fbf98f085048166e6c2906e","user":{"login":"andre-merzky","id":1319674,"avatar_url":"https://avatars.githubusercontent.com/u/1319674?v=3","gravatar_id":"","url":"https://api.github.com/users/andre-merzky","html_url":"https://github.com/andre-merzky","followers_url":"https://api.github.com/users/andre-merzky/followers","following_url":"https://api.github.com/users/andre-merzky/following{/other_user}","gists_url":"https://api.github.com/users/andre-merzky/gists{/gist_id}","starred_url":"https://api.github.com/users/andre-merzky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andre-merzky/subscriptions","organizations_url":"https://api.github.com/users/andre-merzky/orgs","repos_url":"https://api.github.com/users/andre-merzky/repos","events_url":"https://api.github.com/users/andre-merzky/events{/privacy}","received_events_url":"https://api.github.com/users/andre-merzky/received_events","type":"User","site_admin":false},"body":"In principle, yes.  In practice, I can think of more reasons why a tarball would not get created.  The agent being force-killed due to walltime limits being the obvious one.","created_at":"2015-10-06T08:02:51Z","updated_at":"2015-10-06T08:02:51Z","html_url":"https://github.com/radical-cybertools/radical.pilot/pull/798#discussion_r41234766","pull_request_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/798","_links":{"self":{"href":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/comments/41234766"},"html":{"href":"https://github.com/radical-cybertools/radical.pilot/pull/798#discussion_r41234766"},"pull_request":{"href":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/798"}}},"pull_request":{"url":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/798","id":46797457,"html_url":"https://github.com/radical-cybertools/radical.pilot/pull/798","diff_url":"https://github.com/radical-cybertools/radical.pilot/pull/798.diff","patch_url":"https://github.com/radical-cybertools/radical.pilot/pull/798.patch","issue_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/issues/798","number":798,"state":"open","locked":false,"title":"Feature/proftarball","user":{"login":"marksantcroos","id":383387,"avatar_url":"https://avatars.githubusercontent.com/u/383387?v=3","gravatar_id":"","url":"https://api.github.com/users/marksantcroos","html_url":"https://github.com/marksantcroos","followers_url":"https://api.github.com/users/marksantcroos/followers","following_url":"https://api.github.com/users/marksantcroos/following{/other_user}","gists_url":"https://api.github.com/users/marksantcroos/gists{/gist_id}","starred_url":"https://api.github.com/users/marksantcroos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/marksantcroos/subscriptions","organizations_url":"https://api.github.com/users/marksantcroos/orgs","repos_url":"https://api.github.com/users/marksantcroos/repos","events_url":"https://api.github.com/users/marksantcroos/events{/privacy}","received_events_url":"https://api.github.com/users/marksantcroos/received_events","type":"User","site_admin":false},"body":"With a large number of sub-agents and instances, the number of profile transfers grew out of hand.","created_at":"2015-10-05T15:45:13Z","updated_at":"2015-10-06T08:02:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"8c20cc354085e0ae0de555d62dd73b3e89f30df1","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/798/commits","review_comments_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/798/comments","review_comment_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/comments{/number}","comments_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/issues/798/comments","statuses_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/statuses/f2cad900de33275b7fbf98f085048166e6c2906e","head":{"label":"radical-cybertools:feature/proftarball","ref":"feature/proftarball","sha":"f2cad900de33275b7fbf98f085048166e6c2906e","user":{"login":"radical-cybertools","id":5244521,"avatar_url":"https://avatars.githubusercontent.com/u/5244521?v=3","gravatar_id":"","url":"https://api.github.com/users/radical-cybertools","html_url":"https://github.com/radical-cybertools","followers_url":"https://api.github.com/users/radical-cybertools/followers","following_url":"https://api.github.com/users/radical-cybertools/following{/other_user}","gists_url":"https://api.github.com/users/radical-cybertools/gists{/gist_id}","starred_url":"https://api.github.com/users/radical-cybertools/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/radical-cybertools/subscriptions","organizations_url":"https://api.github.com/users/radical-cybertools/orgs","repos_url":"https://api.github.com/users/radical-cybertools/repos","events_url":"https://api.github.com/users/radical-cybertools/events{/privacy}","received_events_url":"https://api.github.com/users/radical-cybertools/received_events","type":"Organization","site_admin":false},"repo":{"id":11224542,"name":"radical.pilot","full_name":"radical-cybertools/radical.pilot","owner":{"login":"radical-cybertools","id":5244521,"avatar_url":"https://avatars.githubusercontent.com/u/5244521?v=3","gravatar_id":"","url":"https://api.github.com/users/radical-cybertools","html_url":"https://github.com/radical-cybertools","followers_url":"https://api.github.com/users/radical-cybertools/followers","following_url":"https://api.github.com/users/radical-cybertools/following{/other_user}","gists_url":"https://api.github.com/users/radical-cybertools/gists{/gist_id}","starred_url":"https://api.github.com/users/radical-cybertools/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/radical-cybertools/subscriptions","organizations_url":"https://api.github.com/users/radical-cybertools/orgs","repos_url":"https://api.github.com/users/radical-cybertools/repos","events_url":"https://api.github.com/users/radical-cybertools/events{/privacy}","received_events_url":"https://api.github.com/users/radical-cybertools/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/radical-cybertools/radical.pilot","description":"RADICAL-Pilot","fork":false,"url":"https://api.github.com/repos/radical-cybertools/radical.pilot","forks_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/forks","keys_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/teams","hooks_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/hooks","issue_events_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/issues/events{/number}","events_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/events","assignees_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/assignees{/user}","branches_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/branches{/branch}","tags_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/tags","blobs_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/git/refs{/sha}","trees_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/statuses/{sha}","languages_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/languages","stargazers_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/stargazers","contributors_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/contributors","subscribers_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/subscribers","subscription_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/subscription","commits_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/commits{/sha}","git_commits_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/git/commits{/sha}","comments_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/comments{/number}","issue_comment_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/issues/comments{/number}","contents_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/contents/{+path}","compare_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/merges","archive_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/downloads","issues_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/issues{/number}","pulls_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls{/number}","milestones_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/milestones{/number}","notifications_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/labels{/name}","releases_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/releases{/id}","created_at":"2013-07-06T21:14:41Z","updated_at":"2015-09-24T21:30:14Z","pushed_at":"2015-10-05T22:09:59Z","git_url":"git://github.com/radical-cybertools/radical.pilot.git","ssh_url":"git@github.com:radical-cybertools/radical.pilot.git","clone_url":"https://github.com/radical-cybertools/radical.pilot.git","svn_url":"https://github.com/radical-cybertools/radical.pilot","homepage":"http://radical-cybertools.github.io/radical-pilot/index.html","size":123689,"stargazers_count":10,"watchers_count":10,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":149,"forks":3,"open_issues":149,"watchers":10,"default_branch":"devel"}},"base":{"label":"radical-cybertools:devel","ref":"devel","sha":"ecc1fcae3fe3c9552ae4456852c2159832e61785","user":{"login":"radical-cybertools","id":5244521,"avatar_url":"https://avatars.githubusercontent.com/u/5244521?v=3","gravatar_id":"","url":"https://api.github.com/users/radical-cybertools","html_url":"https://github.com/radical-cybertools","followers_url":"https://api.github.com/users/radical-cybertools/followers","following_url":"https://api.github.com/users/radical-cybertools/following{/other_user}","gists_url":"https://api.github.com/users/radical-cybertools/gists{/gist_id}","starred_url":"https://api.github.com/users/radical-cybertools/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/radical-cybertools/subscriptions","organizations_url":"https://api.github.com/users/radical-cybertools/orgs","repos_url":"https://api.github.com/users/radical-cybertools/repos","events_url":"https://api.github.com/users/radical-cybertools/events{/privacy}","received_events_url":"https://api.github.com/users/radical-cybertools/received_events","type":"Organization","site_admin":false},"repo":{"id":11224542,"name":"radical.pilot","full_name":"radical-cybertools/radical.pilot","owner":{"login":"radical-cybertools","id":5244521,"avatar_url":"https://avatars.githubusercontent.com/u/5244521?v=3","gravatar_id":"","url":"https://api.github.com/users/radical-cybertools","html_url":"https://github.com/radical-cybertools","followers_url":"https://api.github.com/users/radical-cybertools/followers","following_url":"https://api.github.com/users/radical-cybertools/following{/other_user}","gists_url":"https://api.github.com/users/radical-cybertools/gists{/gist_id}","starred_url":"https://api.github.com/users/radical-cybertools/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/radical-cybertools/subscriptions","organizations_url":"https://api.github.com/users/radical-cybertools/orgs","repos_url":"https://api.github.com/users/radical-cybertools/repos","events_url":"https://api.github.com/users/radical-cybertools/events{/privacy}","received_events_url":"https://api.github.com/users/radical-cybertools/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/radical-cybertools/radical.pilot","description":"RADICAL-Pilot","fork":false,"url":"https://api.github.com/repos/radical-cybertools/radical.pilot","forks_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/forks","keys_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/teams","hooks_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/hooks","issue_events_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/issues/events{/number}","events_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/events","assignees_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/assignees{/user}","branches_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/branches{/branch}","tags_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/tags","blobs_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/git/refs{/sha}","trees_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/statuses/{sha}","languages_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/languages","stargazers_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/stargazers","contributors_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/contributors","subscribers_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/subscribers","subscription_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/subscription","commits_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/commits{/sha}","git_commits_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/git/commits{/sha}","comments_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/comments{/number}","issue_comment_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/issues/comments{/number}","contents_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/contents/{+path}","compare_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/merges","archive_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/downloads","issues_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/issues{/number}","pulls_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls{/number}","milestones_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/milestones{/number}","notifications_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/labels{/name}","releases_url":"https://api.github.com/repos/radical-cybertools/radical.pilot/releases{/id}","created_at":"2013-07-06T21:14:41Z","updated_at":"2015-09-24T21:30:14Z","pushed_at":"2015-10-05T22:09:59Z","git_url":"git://github.com/radical-cybertools/radical.pilot.git","ssh_url":"git@github.com:radical-cybertools/radical.pilot.git","clone_url":"https://github.com/radical-cybertools/radical.pilot.git","svn_url":"https://github.com/radical-cybertools/radical.pilot","homepage":"http://radical-cybertools.github.io/radical-pilot/index.html","size":123689,"stargazers_count":10,"watchers_count":10,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":149,"forks":3,"open_issues":149,"watchers":10,"default_branch":"devel"}},"_links":{"self":{"href":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/798"},"html":{"href":"https://github.com/radical-cybertools/radical.pilot/pull/798"},"issue":{"href":"https://api.github.com/repos/radical-cybertools/radical.pilot/issues/798"},"comments":{"href":"https://api.github.com/repos/radical-cybertools/radical.pilot/issues/798/comments"},"review_comments":{"href":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/798/comments"},"review_comment":{"href":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/radical-cybertools/radical.pilot/pulls/798/commits"},"statuses":{"href":"https://api.github.com/repos/radical-cybertools/radical.pilot/statuses/f2cad900de33275b7fbf98f085048166e6c2906e"}}}}