{"action":"created","comment":{"url":"https://api.github.com/repos/cylc/cylc/pulls/comments/32604747","id":32604747,"diff_hunk":"@@ -0,0 +1,84 @@\n+#!/usr/bin/env python\n+\n+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.\n+# Copyright (C) 2008-2015 NIWA\n+#\n+# This program is free software: you can redistribute it and/or modify\n+# it under the terms of the GNU General Public License as published by\n+# the Free Software Foundation, either version 3 of the License, or\n+# (at your option) any later version.\n+#\n+# This program is distributed in the hope that it will be useful,\n+# but WITHOUT ANY WARRANTY; without even the implied warranty of\n+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+# GNU General Public License for more details.\n+#\n+# You should have received a copy of the GNU General Public License\n+# along with this program.  If not, see <http://www.gnu.org/licenses/>.\n+\"\"\"cylc [hook] job-logs-retrieve [OPTIONS] HOST:HOST-PATH LOCALHOST-PATH\n+\n+(This command is for internal use.)\n+Retrieve logs from a remote host for a task job.\n+\n+\"\"\"\n+\n+\n+import os\n+from subprocess import check_call\n+import shlex\n+import sys\n+import traceback\n+\n+from cylc.cfgspec.globalcfg import GLOBAL_CFG\n+from cylc.CylcOptionParsers import cop\n+from cylc.wallclock import get_time_string_from_unix_time\n+\n+\n+def main():\n+    \"\"\"CLI main.\"\"\"\n+    # Options and arguments\n+    opt_parser = cop(__doc__, argdoc=[\n+        (\"HOST:HOST-PATH\", \"Path to remote job logs directory\"),\n+        (\"LOCALHOST-PATH\", \"Path to local job logs directory\"),\n+    ])\n+    opt_parser.add_option(\"--max-size\",\n+        help=\"Don't transfer any file larger than SIZE.\",\n+        action=\"store\", default=\"10M\", dest=\"max_size\", metavar=\"SIZE\")\n+    opts, args = opt_parser.parse_args()\n+\n+    # Determine the remote shell template to use\n+    source, target = args\n+    source_auth, source_path = source.split(\":\", 1)\n+    if \"@\" in source_auth:\n+        source_owner, source_host = source_auth.split(\"@\", 1)\n+    else:\n+        source_owner, source_host = (None, source_auth)\n+    ssh_tmpl = str(GLOBAL_CFG.get_host_item(\n+        \"remote shell template\", source_host, source_owner)).replace(\" %s\", \"\")\n+\n+    # Retrieve remote job logs\n+    # N.B. \"scp\" does not have a \"max-size\" option.\n+    check_call([\n+        \"rsync\", \"-a\", \"--rsh=\" + ssh_tmpl, \"--max-size=\" + opts.max_size,\n+        source + \"/\", target])","path":"bin/cylc-job-logs-retrieve","position":63,"original_position":63,"commit_id":"f5ed45f223b4f47b6bfd33ca40c89f3020daafd9","original_commit_id":"f5ed45f223b4f47b6bfd33ca40c89f3020daafd9","user":{"login":"matthewrmshin","id":908613,"avatar_url":"https://avatars.githubusercontent.com/u/908613?v=3","gravatar_id":"","url":"https://api.github.com/users/matthewrmshin","html_url":"https://github.com/matthewrmshin","followers_url":"https://api.github.com/users/matthewrmshin/followers","following_url":"https://api.github.com/users/matthewrmshin/following{/other_user}","gists_url":"https://api.github.com/users/matthewrmshin/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewrmshin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewrmshin/subscriptions","organizations_url":"https://api.github.com/users/matthewrmshin/orgs","repos_url":"https://api.github.com/users/matthewrmshin/repos","events_url":"https://api.github.com/users/matthewrmshin/events{/privacy}","received_events_url":"https://api.github.com/users/matthewrmshin/received_events","type":"User","site_admin":false},"body":"It should be OK with our normal naming conventions.","created_at":"2015-06-17T08:31:51Z","updated_at":"2015-06-17T08:31:51Z","html_url":"https://github.com/cylc/cylc/pull/1503#discussion_r32604747","pull_request_url":"https://api.github.com/repos/cylc/cylc/pulls/1503","_links":{"self":{"href":"https://api.github.com/repos/cylc/cylc/pulls/comments/32604747"},"html":{"href":"https://github.com/cylc/cylc/pull/1503#discussion_r32604747"},"pull_request":{"href":"https://api.github.com/repos/cylc/cylc/pulls/1503"}}},"pull_request":{"url":"https://api.github.com/repos/cylc/cylc/pulls/1503","id":37801355,"html_url":"https://github.com/cylc/cylc/pull/1503","diff_url":"https://github.com/cylc/cylc/pull/1503.diff","patch_url":"https://github.com/cylc/cylc/pull/1503.patch","issue_url":"https://api.github.com/repos/cylc/cylc/issues/1503","number":1503,"state":"open","locked":false,"title":"Event handler enhancements","user":{"login":"matthewrmshin","id":908613,"avatar_url":"https://avatars.githubusercontent.com/u/908613?v=3","gravatar_id":"","url":"https://api.github.com/users/matthewrmshin","html_url":"https://github.com/matthewrmshin","followers_url":"https://api.github.com/users/matthewrmshin/followers","following_url":"https://api.github.com/users/matthewrmshin/following{/other_user}","gists_url":"https://api.github.com/users/matthewrmshin/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewrmshin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewrmshin/subscriptions","organizations_url":"https://api.github.com/users/matthewrmshin/orgs","repos_url":"https://api.github.com/users/matthewrmshin/repos","events_url":"https://api.github.com/users/matthewrmshin/events{/privacy}","received_events_url":"https://api.github.com/users/matthewrmshin/received_events","type":"User","site_admin":false},"body":"New and additional settings to `suite.rc`:\r\n* Configurable remote task job logs retrieval.\r\n* Configurable task event email.\r\n* General task event handlers.\r\n* Task event handler can now be command templates.\r\n* Event handlers can now retry on failure.\r\n\r\nImprove diagnostics to `job-activity.log`.\r\n\r\nStore sub/run and event handler (re)try info in context objects.\r\nStore multiprocess pool command contexts and results in objects\r\nUse table name constants for task proxy DB inserts/updates.\r\n\r\nThis closes #181, addresses bullet points 1 to 3 of #992, and begins the 1st step for #1052.","created_at":"2015-06-16T14:34:02Z","updated_at":"2015-06-17T08:31:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9a00d578a392efde846817ea9c2e8e762fda2f8d","assignee":{"login":"matthewrmshin","id":908613,"avatar_url":"https://avatars.githubusercontent.com/u/908613?v=3","gravatar_id":"","url":"https://api.github.com/users/matthewrmshin","html_url":"https://github.com/matthewrmshin","followers_url":"https://api.github.com/users/matthewrmshin/followers","following_url":"https://api.github.com/users/matthewrmshin/following{/other_user}","gists_url":"https://api.github.com/users/matthewrmshin/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewrmshin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewrmshin/subscriptions","organizations_url":"https://api.github.com/users/matthewrmshin/orgs","repos_url":"https://api.github.com/users/matthewrmshin/repos","events_url":"https://api.github.com/users/matthewrmshin/events{/privacy}","received_events_url":"https://api.github.com/users/matthewrmshin/received_events","type":"User","site_admin":false},"milestone":{"url":"https://api.github.com/repos/cylc/cylc/milestones/5","html_url":"https://github.com/cylc/cylc/milestones/soon","labels_url":"https://api.github.com/repos/cylc/cylc/milestones/5/labels","id":262614,"number":5,"title":"soon","description":"A few months in the future. Top priority issues and easy fixes should initially be marked against this milestone, and then moved to *next-release* if work progresses quickly enough.","creator":{"login":"hjoliver","id":2362137,"avatar_url":"https://avatars.githubusercontent.com/u/2362137?v=3","gravatar_id":"","url":"https://api.github.com/users/hjoliver","html_url":"https://github.com/hjoliver","followers_url":"https://api.github.com/users/hjoliver/followers","following_url":"https://api.github.com/users/hjoliver/following{/other_user}","gists_url":"https://api.github.com/users/hjoliver/gists{/gist_id}","starred_url":"https://api.github.com/users/hjoliver/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hjoliver/subscriptions","organizations_url":"https://api.github.com/users/hjoliver/orgs","repos_url":"https://api.github.com/users/hjoliver/repos","events_url":"https://api.github.com/users/hjoliver/events{/privacy}","received_events_url":"https://api.github.com/users/hjoliver/received_events","type":"User","site_admin":false},"open_issues":62,"closed_issues":0,"state":"open","created_at":"2013-02-08T07:07:13Z","updated_at":"2015-06-17T02:36:38Z","due_on":"2015-08-30T23:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/cylc/cylc/pulls/1503/commits","review_comments_url":"https://api.github.com/repos/cylc/cylc/pulls/1503/comments","review_comment_url":"https://api.github.com/repos/cylc/cylc/pulls/comments{/number}","comments_url":"https://api.github.com/repos/cylc/cylc/issues/1503/comments","statuses_url":"https://api.github.com/repos/cylc/cylc/statuses/f5ed45f223b4f47b6bfd33ca40c89f3020daafd9","head":{"label":"matthewrmshin:hook","ref":"hook","sha":"f5ed45f223b4f47b6bfd33ca40c89f3020daafd9","user":{"login":"matthewrmshin","id":908613,"avatar_url":"https://avatars.githubusercontent.com/u/908613?v=3","gravatar_id":"","url":"https://api.github.com/users/matthewrmshin","html_url":"https://github.com/matthewrmshin","followers_url":"https://api.github.com/users/matthewrmshin/followers","following_url":"https://api.github.com/users/matthewrmshin/following{/other_user}","gists_url":"https://api.github.com/users/matthewrmshin/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewrmshin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewrmshin/subscriptions","organizations_url":"https://api.github.com/users/matthewrmshin/orgs","repos_url":"https://api.github.com/users/matthewrmshin/repos","events_url":"https://api.github.com/users/matthewrmshin/events{/privacy}","received_events_url":"https://api.github.com/users/matthewrmshin/received_events","type":"User","site_admin":false},"repo":{"id":4710539,"name":"cylc","full_name":"matthewrmshin/cylc","owner":{"login":"matthewrmshin","id":908613,"avatar_url":"https://avatars.githubusercontent.com/u/908613?v=3","gravatar_id":"","url":"https://api.github.com/users/matthewrmshin","html_url":"https://github.com/matthewrmshin","followers_url":"https://api.github.com/users/matthewrmshin/followers","following_url":"https://api.github.com/users/matthewrmshin/following{/other_user}","gists_url":"https://api.github.com/users/matthewrmshin/gists{/gist_id}","starred_url":"https://api.github.com/users/matthewrmshin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matthewrmshin/subscriptions","organizations_url":"https://api.github.com/users/matthewrmshin/orgs","repos_url":"https://api.github.com/users/matthewrmshin/repos","events_url":"https://api.github.com/users/matthewrmshin/events{/privacy}","received_events_url":"https://api.github.com/users/matthewrmshin/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/matthewrmshin/cylc","description":"Cylc: an engine for running meteorological suites","fork":true,"url":"https://api.github.com/repos/matthewrmshin/cylc","forks_url":"https://api.github.com/repos/matthewrmshin/cylc/forks","keys_url":"https://api.github.com/repos/matthewrmshin/cylc/keys{/key_id}","collaborators_url":"https://api.github.com/repos/matthewrmshin/cylc/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/matthewrmshin/cylc/teams","hooks_url":"https://api.github.com/repos/matthewrmshin/cylc/hooks","issue_events_url":"https://api.github.com/repos/matthewrmshin/cylc/issues/events{/number}","events_url":"https://api.github.com/repos/matthewrmshin/cylc/events","assignees_url":"https://api.github.com/repos/matthewrmshin/cylc/assignees{/user}","branches_url":"https://api.github.com/repos/matthewrmshin/cylc/branches{/branch}","tags_url":"https://api.github.com/repos/matthewrmshin/cylc/tags","blobs_url":"https://api.github.com/repos/matthewrmshin/cylc/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/matthewrmshin/cylc/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/matthewrmshin/cylc/git/refs{/sha}","trees_url":"https://api.github.com/repos/matthewrmshin/cylc/git/trees{/sha}","statuses_url":"https://api.github.com/repos/matthewrmshin/cylc/statuses/{sha}","languages_url":"https://api.github.com/repos/matthewrmshin/cylc/languages","stargazers_url":"https://api.github.com/repos/matthewrmshin/cylc/stargazers","contributors_url":"https://api.github.com/repos/matthewrmshin/cylc/contributors","subscribers_url":"https://api.github.com/repos/matthewrmshin/cylc/subscribers","subscription_url":"https://api.github.com/repos/matthewrmshin/cylc/subscription","commits_url":"https://api.github.com/repos/matthewrmshin/cylc/commits{/sha}","git_commits_url":"https://api.github.com/repos/matthewrmshin/cylc/git/commits{/sha}","comments_url":"https://api.github.com/repos/matthewrmshin/cylc/comments{/number}","issue_comment_url":"https://api.github.com/repos/matthewrmshin/cylc/issues/comments{/number}","contents_url":"https://api.github.com/repos/matthewrmshin/cylc/contents/{+path}","compare_url":"https://api.github.com/repos/matthewrmshin/cylc/compare/{base}...{head}","merges_url":"https://api.github.com/repos/matthewrmshin/cylc/merges","archive_url":"https://api.github.com/repos/matthewrmshin/cylc/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/matthewrmshin/cylc/downloads","issues_url":"https://api.github.com/repos/matthewrmshin/cylc/issues{/number}","pulls_url":"https://api.github.com/repos/matthewrmshin/cylc/pulls{/number}","milestones_url":"https://api.github.com/repos/matthewrmshin/cylc/milestones{/number}","notifications_url":"https://api.github.com/repos/matthewrmshin/cylc/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/matthewrmshin/cylc/labels{/name}","releases_url":"https://api.github.com/repos/matthewrmshin/cylc/releases{/id}","created_at":"2012-06-19T07:07:09Z","updated_at":"2015-05-08T11:59:22Z","pushed_at":"2015-06-16T22:08:52Z","git_url":"git://github.com/matthewrmshin/cylc.git","ssh_url":"git@github.com:matthewrmshin/cylc.git","clone_url":"https://github.com/matthewrmshin/cylc.git","svn_url":"https://github.com/matthewrmshin/cylc","homepage":"http://cylc.github.io/cylc/","size":72624,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":2,"default_branch":"master"}},"base":{"label":"cylc:master","ref":"master","sha":"e206125ed7cfabba74dc01a6c5aaa14b96e8dad5","user":{"login":"cylc","id":824878,"avatar_url":"https://avatars.githubusercontent.com/u/824878?v=3","gravatar_id":"","url":"https://api.github.com/users/cylc","html_url":"https://github.com/cylc","followers_url":"https://api.github.com/users/cylc/followers","following_url":"https://api.github.com/users/cylc/following{/other_user}","gists_url":"https://api.github.com/users/cylc/gists{/gist_id}","starred_url":"https://api.github.com/users/cylc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cylc/subscriptions","organizations_url":"https://api.github.com/users/cylc/orgs","repos_url":"https://api.github.com/users/cylc/repos","events_url":"https://api.github.com/users/cylc/events{/privacy}","received_events_url":"https://api.github.com/users/cylc/received_events","type":"Organization","site_admin":false},"repo":{"id":1836229,"name":"cylc","full_name":"cylc/cylc","owner":{"login":"cylc","id":824878,"avatar_url":"https://avatars.githubusercontent.com/u/824878?v=3","gravatar_id":"","url":"https://api.github.com/users/cylc","html_url":"https://github.com/cylc","followers_url":"https://api.github.com/users/cylc/followers","following_url":"https://api.github.com/users/cylc/following{/other_user}","gists_url":"https://api.github.com/users/cylc/gists{/gist_id}","starred_url":"https://api.github.com/users/cylc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cylc/subscriptions","organizations_url":"https://api.github.com/users/cylc/orgs","repos_url":"https://api.github.com/users/cylc/repos","events_url":"https://api.github.com/users/cylc/events{/privacy}","received_events_url":"https://api.github.com/users/cylc/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/cylc/cylc","description":"This is the official repository of the cylc suite engine. It will normally contain just the master branch, with completed features merged in from feature branches in developer forks.","fork":false,"url":"https://api.github.com/repos/cylc/cylc","forks_url":"https://api.github.com/repos/cylc/cylc/forks","keys_url":"https://api.github.com/repos/cylc/cylc/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cylc/cylc/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cylc/cylc/teams","hooks_url":"https://api.github.com/repos/cylc/cylc/hooks","issue_events_url":"https://api.github.com/repos/cylc/cylc/issues/events{/number}","events_url":"https://api.github.com/repos/cylc/cylc/events","assignees_url":"https://api.github.com/repos/cylc/cylc/assignees{/user}","branches_url":"https://api.github.com/repos/cylc/cylc/branches{/branch}","tags_url":"https://api.github.com/repos/cylc/cylc/tags","blobs_url":"https://api.github.com/repos/cylc/cylc/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cylc/cylc/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cylc/cylc/git/refs{/sha}","trees_url":"https://api.github.com/repos/cylc/cylc/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cylc/cylc/statuses/{sha}","languages_url":"https://api.github.com/repos/cylc/cylc/languages","stargazers_url":"https://api.github.com/repos/cylc/cylc/stargazers","contributors_url":"https://api.github.com/repos/cylc/cylc/contributors","subscribers_url":"https://api.github.com/repos/cylc/cylc/subscribers","subscription_url":"https://api.github.com/repos/cylc/cylc/subscription","commits_url":"https://api.github.com/repos/cylc/cylc/commits{/sha}","git_commits_url":"https://api.github.com/repos/cylc/cylc/git/commits{/sha}","comments_url":"https://api.github.com/repos/cylc/cylc/comments{/number}","issue_comment_url":"https://api.github.com/repos/cylc/cylc/issues/comments{/number}","contents_url":"https://api.github.com/repos/cylc/cylc/contents/{+path}","compare_url":"https://api.github.com/repos/cylc/cylc/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cylc/cylc/merges","archive_url":"https://api.github.com/repos/cylc/cylc/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cylc/cylc/downloads","issues_url":"https://api.github.com/repos/cylc/cylc/issues{/number}","pulls_url":"https://api.github.com/repos/cylc/cylc/pulls{/number}","milestones_url":"https://api.github.com/repos/cylc/cylc/milestones{/number}","notifications_url":"https://api.github.com/repos/cylc/cylc/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cylc/cylc/labels{/name}","releases_url":"https://api.github.com/repos/cylc/cylc/releases{/id}","created_at":"2011-06-02T08:31:49Z","updated_at":"2015-06-09T12:30:14Z","pushed_at":"2015-06-17T02:36:16Z","git_url":"git://github.com/cylc/cylc.git","ssh_url":"git@github.com:cylc/cylc.git","clone_url":"https://github.com/cylc/cylc.git","svn_url":"https://github.com/cylc/cylc","homepage":"http://cylc.github.com/cylc","size":93674,"stargazers_count":27,"watchers_count":27,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":17,"mirror_url":null,"open_issues_count":156,"forks":17,"open_issues":156,"watchers":27,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/cylc/cylc/pulls/1503"},"html":{"href":"https://github.com/cylc/cylc/pull/1503"},"issue":{"href":"https://api.github.com/repos/cylc/cylc/issues/1503"},"comments":{"href":"https://api.github.com/repos/cylc/cylc/issues/1503/comments"},"review_comments":{"href":"https://api.github.com/repos/cylc/cylc/pulls/1503/comments"},"review_comment":{"href":"https://api.github.com/repos/cylc/cylc/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/cylc/cylc/pulls/1503/commits"},"statuses":{"href":"https://api.github.com/repos/cylc/cylc/statuses/f5ed45f223b4f47b6bfd33ca40c89f3020daafd9"}}}}