{"action":"created","comment":{"url":"https://api.github.com/repos/StackStorm/st2/pulls/comments/45134458","id":45134458,"diff_hunk":"@@ -0,0 +1,141 @@\n+# Licensed to the StackStorm, Inc ('StackStorm') under one or more\n+# contributor license agreements.  See the NOTICE file distributed with\n+# this work for additional information regarding copyright ownership.\n+# The ASF licenses this file to You under the Apache License, Version 2.0\n+# (the \"License\"); you may not use this file except in compliance with\n+# the License.  You may obtain a copy of the License at\n+#\n+#     http://www.apache.org/licenses/LICENSE-2.0\n+#\n+# Unless required by applicable law or agreed to in writing, software\n+# distributed under the License is distributed on an \"AS IS\" BASIS,\n+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+# See the License for the specific language governing permissions and\n+# limitations under the License.\n+\n+import copy\n+\n+from st2common import log as logging\n+from st2common.models.db.liveaction import LiveActionDB\n+import st2common.services.action as action_services\n+from st2common.constants.action import LIVEACTION_STATUS_FAILED\n+from st2common.constants.action import LIVEACTION_STATUS_TIMED_OUT\n+from st2common.util.enum import Enum\n+from st2common.policies.base import ResourcePolicyApplicator\n+\n+__all__ = [\n+    'RetryOnPolicy',\n+    'ExecutionRetryPolicyApplicator'\n+]\n+\n+LOG = logging.getLogger(__name__)\n+\n+VALID_RETRY_STATUSES = [\n+    LIVEACTION_STATUS_FAILED,\n+    LIVEACTION_STATUS_TIMED_OUT\n+]\n+\n+# Maximum value for \"max_retry_count\" parameter\n+MAXIMUM_RETRY_COUNT = 5\n+\n+\n+class RetryOnPolicy(Enum):\n+    FAILURE = 'failure'  # Retry on execution failure\n+    TIMEOUT = 'timeout'  # Retry on execution timeout\n+\n+\n+class ExecutionRetryPolicyApplicator(ResourcePolicyApplicator):\n+    def __init__(self, policy_ref, policy_type, retry_on, max_retry_count=2):\n+        \"\"\"\n+        :param retry_on: Condition to retry the execution on.\n+        :type retry_on: ``str``\n+\n+        :param max_retry_count: Maximum number of times to try to retry an action.\n+        :type max_retry_count: ``int``\n+        \"\"\"\n+        # TODO: Also support sleep delay before retrying\n+        super(ExecutionRetryPolicyApplicator, self).__init__(policy_ref=policy_ref,\n+                                                             policy_type=policy_type)\n+\n+        if retry_on not in RetryOnPolicy.get_valid_values():\n+            raise ValueError('Invalid value for \"retry_on\" parameter')\n+\n+        if max_retry_count > MAXIMUM_RETRY_COUNT:\n+            raise ValueError('Maximum retry count is %s' % (MAXIMUM_RETRY_COUNT))\n+\n+        self.retry_on = retry_on\n+        self.max_retry_count = max_retry_count\n+\n+    def apply_before(self, target):\n+        # Nothing to do here\n+        target = super(ExecutionRetryPolicyApplicator, self).apply_before(target=target)\n+        return target\n+\n+    def apply_after(self, target):\n+        target = super(ExecutionRetryPolicyApplicator, self).apply_before(target=target)\n+\n+        live_action_db = target\n+        retry_count = self._get_live_action_retry_count(live_action_db=live_action_db)\n+\n+        extra = {'live_action_db': live_action_db, 'policy_ref': self._policy_ref,\n+                 'retry_on': self.retry_on, 'max_retry_count': self.max_retry_count,\n+                 'current_retry_count': retry_count}\n+\n+        if live_action_db.status not in VALID_RETRY_STATUSES:\n+            # Currently we only support retrying on failed action\n+            LOG.debug('Liveaction not in a valid retry state, not checking retry policy',\n+                      extra=extra)\n+            return target\n+\n+        if (retry_count + 1) > self.max_retry_count:\n+            LOG.info('Maximum retry count has been reached, not retrying', extra=extra)\n+            return target\n+\n+        has_failed = live_action_db.status == LIVEACTION_STATUS_FAILED\n+        has_timed_out = live_action_db.status == LIVEACTION_STATUS_TIMED_OUT\n+\n+        if has_failed and self.retry_on == RetryOnPolicy.FAILURE:\n+            extra['failure'] = True\n+            LOG.info('Policy matched (failure), retrying action execution...', extra=extra)\n+            self._re_run_live_action(live_action_db=live_action_db)\n+            return target\n+\n+        if has_timed_out and self.retry_on == RetryOnPolicy.TIMEOUT:\n+            extra['timeout'] = True\n+            LOG.info('Policy matched (timeout), retrying action execution...', extra=extra)\n+            self._re_run_live_action(live_action_db=live_action_db)\n+            return target\n+\n+        return target\n+\n+    def _get_live_action_retry_count(self, live_action_db):\n+        \"\"\"\n+        Retrieve current retry count for the provided live action.\n+\n+        :rtype: ``int``\n+        \"\"\"\n+        # TODO: Ideally we would store retry_count in zookeeper or similar and use locking so we\n+        # can run multiple instances of st2notififer\n+        context = getattr(live_action_db, 'context', {})\n+        retry_count = context.get('policies', {}).get('retry_count', 0)\n+\n+        return retry_count\n+\n+    def _re_run_live_action(self, live_action_db):\n+        retry_count = self._get_live_action_retry_count(live_action_db=live_action_db)\n+\n+        # Add additional policy specific info to the context\n+        context = getattr(live_action_db, 'context', {})\n+        new_context = copy.deepcopy(context)\n+        new_context['policies'] = {\n+            'applied_policy': self._policy_ref,","path":"st2actions/st2actions/policies/retry.py","position":131,"original_position":131,"commit_id":"275750c3fb5d7c0a1bca269c05512087e6d6c84d","original_commit_id":"275750c3fb5d7c0a1bca269c05512087e6d6c84d","user":{"login":"Kami","id":125088,"avatar_url":"https://avatars.githubusercontent.com/u/125088?v=3","gravatar_id":"","url":"https://api.github.com/users/Kami","html_url":"https://github.com/Kami","followers_url":"https://api.github.com/users/Kami/followers","following_url":"https://api.github.com/users/Kami/following{/other_user}","gists_url":"https://api.github.com/users/Kami/gists{/gist_id}","starred_url":"https://api.github.com/users/Kami/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Kami/subscriptions","organizations_url":"https://api.github.com/users/Kami/orgs","repos_url":"https://api.github.com/users/Kami/repos","events_url":"https://api.github.com/users/Kami/events{/privacy}","received_events_url":"https://api.github.com/users/Kami/received_events","type":"User","site_admin":false},"body":"Yeah, I was thinking about it, but I couldn't decide on what to do.\r\n\r\nAnother option is `['policies']['retry']`, but this might be too nested?","created_at":"2015-11-17T22:47:39Z","updated_at":"2015-11-17T22:47:39Z","html_url":"https://github.com/StackStorm/st2/pull/2217#discussion_r45134458","pull_request_url":"https://api.github.com/repos/StackStorm/st2/pulls/2217","_links":{"self":{"href":"https://api.github.com/repos/StackStorm/st2/pulls/comments/45134458"},"html":{"href":"https://github.com/StackStorm/st2/pull/2217#discussion_r45134458"},"pull_request":{"href":"https://api.github.com/repos/StackStorm/st2/pulls/2217"}}},"pull_request":{"url":"https://api.github.com/repos/StackStorm/st2/pulls/2217","id":51004095,"html_url":"https://github.com/StackStorm/st2/pull/2217","diff_url":"https://github.com/StackStorm/st2/pull/2217.diff","patch_url":"https://github.com/StackStorm/st2/pull/2217.patch","issue_url":"https://api.github.com/repos/StackStorm/st2/issues/2217","number":2217,"state":"open","locked":false,"title":"[WIP] New \"timeout\" action execution state and retry on failure/timeout policy","user":{"login":"Kami","id":125088,"avatar_url":"https://avatars.githubusercontent.com/u/125088?v=3","gravatar_id":"","url":"https://api.github.com/users/Kami","html_url":"https://github.com/Kami","followers_url":"https://api.github.com/users/Kami/followers","following_url":"https://api.github.com/users/Kami/following{/other_user}","gists_url":"https://api.github.com/users/Kami/gists{/gist_id}","starred_url":"https://api.github.com/users/Kami/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Kami/subscriptions","organizations_url":"https://api.github.com/users/Kami/orgs","repos_url":"https://api.github.com/users/Kami/repos","events_url":"https://api.github.com/users/Kami/events{/privacy}","received_events_url":"https://api.github.com/users/Kami/received_events","type":"User","site_admin":false},"body":"This pull request introduces a new \"timeout\" action execution state. This state is a special type of a failure and represents an execution timeout (previously you couldn't directly identify a timeout - you would need to manually parse the error attribute to figure that out).\r\n\r\nIn addition to that, it also adds a new \"retry\" policy which allows executions to be automatically retried in case of a failure or a timeout.\r\n\r\nTODO:\r\n\r\n- [ ] Tests\r\n- [ ] Examples\r\n- [ ] Documentation","created_at":"2015-11-17T21:03:29Z","updated_at":"2015-11-17T22:47:39Z","closed_at":null,"merged_at":null,"merge_commit_sha":null,"assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/StackStorm/st2/pulls/2217/commits","review_comments_url":"https://api.github.com/repos/StackStorm/st2/pulls/2217/comments","review_comment_url":"https://api.github.com/repos/StackStorm/st2/pulls/comments{/number}","comments_url":"https://api.github.com/repos/StackStorm/st2/issues/2217/comments","statuses_url":"https://api.github.com/repos/StackStorm/st2/statuses/275750c3fb5d7c0a1bca269c05512087e6d6c84d","head":{"label":"StackStorm:retry_policy","ref":"retry_policy","sha":"275750c3fb5d7c0a1bca269c05512087e6d6c84d","user":{"login":"StackStorm","id":4969009,"avatar_url":"https://avatars.githubusercontent.com/u/4969009?v=3","gravatar_id":"","url":"https://api.github.com/users/StackStorm","html_url":"https://github.com/StackStorm","followers_url":"https://api.github.com/users/StackStorm/followers","following_url":"https://api.github.com/users/StackStorm/following{/other_user}","gists_url":"https://api.github.com/users/StackStorm/gists{/gist_id}","starred_url":"https://api.github.com/users/StackStorm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/StackStorm/subscriptions","organizations_url":"https://api.github.com/users/StackStorm/orgs","repos_url":"https://api.github.com/users/StackStorm/repos","events_url":"https://api.github.com/users/StackStorm/events{/privacy}","received_events_url":"https://api.github.com/users/StackStorm/received_events","type":"Organization","site_admin":false},"repo":{"id":19051806,"name":"st2","full_name":"StackStorm/st2","owner":{"login":"StackStorm","id":4969009,"avatar_url":"https://avatars.githubusercontent.com/u/4969009?v=3","gravatar_id":"","url":"https://api.github.com/users/StackStorm","html_url":"https://github.com/StackStorm","followers_url":"https://api.github.com/users/StackStorm/followers","following_url":"https://api.github.com/users/StackStorm/following{/other_user}","gists_url":"https://api.github.com/users/StackStorm/gists{/gist_id}","starred_url":"https://api.github.com/users/StackStorm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/StackStorm/subscriptions","organizations_url":"https://api.github.com/users/StackStorm/orgs","repos_url":"https://api.github.com/users/StackStorm/repos","events_url":"https://api.github.com/users/StackStorm/events{/privacy}","received_events_url":"https://api.github.com/users/StackStorm/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/StackStorm/st2","description":"StackStorm (aka \"IFTTT for Ops\") is event-driven automation commonly used for auto-remediation, security responses, facilitated troubleshooting, complex deployments, and more. Includes rules engine, workflow, 1500+ integrations, native ChatOps and so forth.  Zingy installer available at http://docs.stackstorm.com/install/index.html. Ping us on slack - https://stackstorm-community.slack.com/. ","fork":false,"url":"https://api.github.com/repos/StackStorm/st2","forks_url":"https://api.github.com/repos/StackStorm/st2/forks","keys_url":"https://api.github.com/repos/StackStorm/st2/keys{/key_id}","collaborators_url":"https://api.github.com/repos/StackStorm/st2/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/StackStorm/st2/teams","hooks_url":"https://api.github.com/repos/StackStorm/st2/hooks","issue_events_url":"https://api.github.com/repos/StackStorm/st2/issues/events{/number}","events_url":"https://api.github.com/repos/StackStorm/st2/events","assignees_url":"https://api.github.com/repos/StackStorm/st2/assignees{/user}","branches_url":"https://api.github.com/repos/StackStorm/st2/branches{/branch}","tags_url":"https://api.github.com/repos/StackStorm/st2/tags","blobs_url":"https://api.github.com/repos/StackStorm/st2/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/StackStorm/st2/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/StackStorm/st2/git/refs{/sha}","trees_url":"https://api.github.com/repos/StackStorm/st2/git/trees{/sha}","statuses_url":"https://api.github.com/repos/StackStorm/st2/statuses/{sha}","languages_url":"https://api.github.com/repos/StackStorm/st2/languages","stargazers_url":"https://api.github.com/repos/StackStorm/st2/stargazers","contributors_url":"https://api.github.com/repos/StackStorm/st2/contributors","subscribers_url":"https://api.github.com/repos/StackStorm/st2/subscribers","subscription_url":"https://api.github.com/repos/StackStorm/st2/subscription","commits_url":"https://api.github.com/repos/StackStorm/st2/commits{/sha}","git_commits_url":"https://api.github.com/repos/StackStorm/st2/git/commits{/sha}","comments_url":"https://api.github.com/repos/StackStorm/st2/comments{/number}","issue_comment_url":"https://api.github.com/repos/StackStorm/st2/issues/comments{/number}","contents_url":"https://api.github.com/repos/StackStorm/st2/contents/{+path}","compare_url":"https://api.github.com/repos/StackStorm/st2/compare/{base}...{head}","merges_url":"https://api.github.com/repos/StackStorm/st2/merges","archive_url":"https://api.github.com/repos/StackStorm/st2/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/StackStorm/st2/downloads","issues_url":"https://api.github.com/repos/StackStorm/st2/issues{/number}","pulls_url":"https://api.github.com/repos/StackStorm/st2/pulls{/number}","milestones_url":"https://api.github.com/repos/StackStorm/st2/milestones{/number}","notifications_url":"https://api.github.com/repos/StackStorm/st2/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/StackStorm/st2/labels{/name}","releases_url":"https://api.github.com/repos/StackStorm/st2/releases{/id}","created_at":"2014-04-23T00:51:34Z","updated_at":"2015-11-17T20:01:10Z","pushed_at":"2015-11-17T21:03:29Z","git_url":"git://github.com/StackStorm/st2.git","ssh_url":"git@github.com:StackStorm/st2.git","clone_url":"https://github.com/StackStorm/st2.git","svn_url":"https://github.com/StackStorm/st2","homepage":"http://www.stackstorm.com/","size":14629,"stargazers_count":1059,"watchers_count":1059,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":88,"mirror_url":null,"open_issues_count":75,"forks":88,"open_issues":75,"watchers":1059,"default_branch":"master"}},"base":{"label":"StackStorm:master","ref":"master","sha":"3315a15c2e431650515df7af098fdd3448436bef","user":{"login":"StackStorm","id":4969009,"avatar_url":"https://avatars.githubusercontent.com/u/4969009?v=3","gravatar_id":"","url":"https://api.github.com/users/StackStorm","html_url":"https://github.com/StackStorm","followers_url":"https://api.github.com/users/StackStorm/followers","following_url":"https://api.github.com/users/StackStorm/following{/other_user}","gists_url":"https://api.github.com/users/StackStorm/gists{/gist_id}","starred_url":"https://api.github.com/users/StackStorm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/StackStorm/subscriptions","organizations_url":"https://api.github.com/users/StackStorm/orgs","repos_url":"https://api.github.com/users/StackStorm/repos","events_url":"https://api.github.com/users/StackStorm/events{/privacy}","received_events_url":"https://api.github.com/users/StackStorm/received_events","type":"Organization","site_admin":false},"repo":{"id":19051806,"name":"st2","full_name":"StackStorm/st2","owner":{"login":"StackStorm","id":4969009,"avatar_url":"https://avatars.githubusercontent.com/u/4969009?v=3","gravatar_id":"","url":"https://api.github.com/users/StackStorm","html_url":"https://github.com/StackStorm","followers_url":"https://api.github.com/users/StackStorm/followers","following_url":"https://api.github.com/users/StackStorm/following{/other_user}","gists_url":"https://api.github.com/users/StackStorm/gists{/gist_id}","starred_url":"https://api.github.com/users/StackStorm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/StackStorm/subscriptions","organizations_url":"https://api.github.com/users/StackStorm/orgs","repos_url":"https://api.github.com/users/StackStorm/repos","events_url":"https://api.github.com/users/StackStorm/events{/privacy}","received_events_url":"https://api.github.com/users/StackStorm/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/StackStorm/st2","description":"StackStorm (aka \"IFTTT for Ops\") is event-driven automation commonly used for auto-remediation, security responses, facilitated troubleshooting, complex deployments, and more. Includes rules engine, workflow, 1500+ integrations, native ChatOps and so forth.  Zingy installer available at http://docs.stackstorm.com/install/index.html. Ping us on slack - https://stackstorm-community.slack.com/. ","fork":false,"url":"https://api.github.com/repos/StackStorm/st2","forks_url":"https://api.github.com/repos/StackStorm/st2/forks","keys_url":"https://api.github.com/repos/StackStorm/st2/keys{/key_id}","collaborators_url":"https://api.github.com/repos/StackStorm/st2/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/StackStorm/st2/teams","hooks_url":"https://api.github.com/repos/StackStorm/st2/hooks","issue_events_url":"https://api.github.com/repos/StackStorm/st2/issues/events{/number}","events_url":"https://api.github.com/repos/StackStorm/st2/events","assignees_url":"https://api.github.com/repos/StackStorm/st2/assignees{/user}","branches_url":"https://api.github.com/repos/StackStorm/st2/branches{/branch}","tags_url":"https://api.github.com/repos/StackStorm/st2/tags","blobs_url":"https://api.github.com/repos/StackStorm/st2/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/StackStorm/st2/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/StackStorm/st2/git/refs{/sha}","trees_url":"https://api.github.com/repos/StackStorm/st2/git/trees{/sha}","statuses_url":"https://api.github.com/repos/StackStorm/st2/statuses/{sha}","languages_url":"https://api.github.com/repos/StackStorm/st2/languages","stargazers_url":"https://api.github.com/repos/StackStorm/st2/stargazers","contributors_url":"https://api.github.com/repos/StackStorm/st2/contributors","subscribers_url":"https://api.github.com/repos/StackStorm/st2/subscribers","subscription_url":"https://api.github.com/repos/StackStorm/st2/subscription","commits_url":"https://api.github.com/repos/StackStorm/st2/commits{/sha}","git_commits_url":"https://api.github.com/repos/StackStorm/st2/git/commits{/sha}","comments_url":"https://api.github.com/repos/StackStorm/st2/comments{/number}","issue_comment_url":"https://api.github.com/repos/StackStorm/st2/issues/comments{/number}","contents_url":"https://api.github.com/repos/StackStorm/st2/contents/{+path}","compare_url":"https://api.github.com/repos/StackStorm/st2/compare/{base}...{head}","merges_url":"https://api.github.com/repos/StackStorm/st2/merges","archive_url":"https://api.github.com/repos/StackStorm/st2/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/StackStorm/st2/downloads","issues_url":"https://api.github.com/repos/StackStorm/st2/issues{/number}","pulls_url":"https://api.github.com/repos/StackStorm/st2/pulls{/number}","milestones_url":"https://api.github.com/repos/StackStorm/st2/milestones{/number}","notifications_url":"https://api.github.com/repos/StackStorm/st2/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/StackStorm/st2/labels{/name}","releases_url":"https://api.github.com/repos/StackStorm/st2/releases{/id}","created_at":"2014-04-23T00:51:34Z","updated_at":"2015-11-17T20:01:10Z","pushed_at":"2015-11-17T21:03:29Z","git_url":"git://github.com/StackStorm/st2.git","ssh_url":"git@github.com:StackStorm/st2.git","clone_url":"https://github.com/StackStorm/st2.git","svn_url":"https://github.com/StackStorm/st2","homepage":"http://www.stackstorm.com/","size":14629,"stargazers_count":1059,"watchers_count":1059,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":88,"mirror_url":null,"open_issues_count":75,"forks":88,"open_issues":75,"watchers":1059,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/StackStorm/st2/pulls/2217"},"html":{"href":"https://github.com/StackStorm/st2/pull/2217"},"issue":{"href":"https://api.github.com/repos/StackStorm/st2/issues/2217"},"comments":{"href":"https://api.github.com/repos/StackStorm/st2/issues/2217/comments"},"review_comments":{"href":"https://api.github.com/repos/StackStorm/st2/pulls/2217/comments"},"review_comment":{"href":"https://api.github.com/repos/StackStorm/st2/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/StackStorm/st2/pulls/2217/commits"},"statuses":{"href":"https://api.github.com/repos/StackStorm/st2/statuses/275750c3fb5d7c0a1bca269c05512087e6d6c84d"}}}}