{"action":"created","comment":{"url":"https://api.github.com/repos/wso2/andes/pulls/comments/36942272","id":36942272,"diff_hunk":"@@ -23,129 +23,197 @@\n import org.wso2.andes.server.ClusterResourceHolder;\n import org.wso2.andes.server.cluster.coordination.ClusterCoordinationHandler;\n import org.wso2.andes.server.cluster.coordination.hazelcast.HazelcastAgent;\n+import org.wso2.andes.store.FailureObservingStoreManager;\n+import org.wso2.andes.store.HealthAwareStore;\n+import org.wso2.andes.store.StoreHealthListener;\n \n import java.util.ArrayList;\n import java.util.List;\n+import java.util.concurrent.atomic.AtomicBoolean;\n \n /**\n  * This task will periodically load exchanges,queues,bindings,subscriptions from database\n  * and simulate cluster notifications. This is implemented to bring the node\n  * to the current state of cluster in case some hazlecast notifications are missed\n  */\n-public class AndesRecoveryTask implements Runnable {\n-\n-    private List<QueueListener> queueListeners = new ArrayList<QueueListener>();\n-    private List<ExchangeListener> exchangeListeners = new ArrayList<ExchangeListener>();\n-    private List<BindingListener> bindingListeners = new ArrayList<BindingListener>();\n-\n-    AndesContextStore andesContextStore;\n-    AMQPConstructStore amqpConstructStore;\n-\n-    private static Log log = LogFactory.getLog(AndesRecoveryTask.class);\n-\n-    public AndesRecoveryTask() {\n-        queueListeners.add(new ClusterCoordinationHandler(HazelcastAgent.getInstance()));\n-        exchangeListeners.add(new ClusterCoordinationHandler(HazelcastAgent.getInstance()));\n-        bindingListeners.add(new ClusterCoordinationHandler(HazelcastAgent.getInstance()));\n-\n-        andesContextStore = AndesContext.getInstance().getAndesContextStore();\n-        amqpConstructStore = AndesContext.getInstance().getAMQPConstructStore();\n-    }\n-\n-    @Override\n-    public void run() {\n-        try {\n-            reloadExchangesFromDB();\n-            reloadQueuesFromDB();\n-            reloadBindingsFromDB();\n-            reloadSubscriptions();\n-        } catch (AndesException e) {\n-            log.error(\"Error in running andes recovery task\", e);\n-        }\n-    }\n-\n-    /**\n-     * reload and recover exchanges,\n-     * Queues, Bindings and Subscriptions\n-     *\n-     * @throws AndesException\n-     */\n-    public void recoverExchangesQueuesBindingsSubscriptions() throws AndesException {\n-        reloadExchangesFromDB();\n-        reloadQueuesFromDB();\n-        reloadBindingsFromDB();\n-        reloadSubscriptions();\n-    }\n-\n-    private void reloadExchangesFromDB() throws AndesException {\n-        List<AndesExchange> fromDB = andesContextStore.getAllExchangesStored();\n-        List<AndesExchange> fromMap = amqpConstructStore.getExchanges();\n-        List<AndesExchange> fromDBDuplicated = new ArrayList<AndesExchange>(fromDB);\n-\n-        fromDB.removeAll(fromMap);\n-        for (AndesExchange exchange : fromDB) {\n-            for (ExchangeListener listener : exchangeListeners) {\n-                log.warn(\"Recovering node. Adding exchange \" + exchange.toString());\n-                listener.handleClusterExchangesChanged(exchange, ExchangeListener.ExchangeChange.Added);\n-            }\n-        }\n-\n-        fromMap.removeAll(fromDBDuplicated);\n-        for (AndesExchange exchange : fromMap) {\n-            for (ExchangeListener listener : exchangeListeners) {\n-                log.warn(\"Recovering node. Removing exchange \" + exchange.toString());\n-                listener.handleClusterExchangesChanged(exchange, ExchangeListener.ExchangeChange.Deleted);\n-            }\n-        }\n-    }\n-\n-    private void reloadQueuesFromDB() throws AndesException {\n-        List<AndesQueue> fromDB = andesContextStore.getAllQueuesStored();\n-        List<AndesQueue> fromMap = amqpConstructStore.getQueues();\n-        List<AndesQueue> fromDBDuplicated = new ArrayList<AndesQueue>(fromDB);\n-\n-        fromDB.removeAll(fromMap);\n-        for (AndesQueue queue : fromDB) {\n-            for (QueueListener listener : queueListeners) {\n-                log.warn(\"Recovering node. Adding queue \" + queue.toString());\n-                listener.handleClusterQueuesChanged(queue, QueueListener.QueueEvent.ADDED);\n-            }\n-        }\n-\n-        fromMap.removeAll(fromDBDuplicated);\n-        for (AndesQueue queue : fromMap) {\n-            for (QueueListener listener : queueListeners) {\n-                log.warn(\"Recovering node. Removing queue \" + queue.toString());\n-                listener.handleClusterQueuesChanged(queue, QueueListener.QueueEvent.DELETED);\n-            }\n-        }\n-    }\n-\n-    private void reloadBindingsFromDB() throws AndesException {\n-        List<AndesExchange> exchanges = andesContextStore.getAllExchangesStored();\n-        for (AndesExchange exchange : exchanges) {\n-            List<AndesBinding> fromDB = andesContextStore.getBindingsStoredForExchange(exchange.exchangeName);\n-            List<AndesBinding> fromMap = amqpConstructStore.getBindingsForExchange(exchange.exchangeName);\n-            List<AndesBinding> fromDBDuplicated = new ArrayList<AndesBinding>(fromDB);\n-            fromDB.removeAll(fromMap);\n-            for (AndesBinding binding : fromDB) {\n-                for (BindingListener listener : bindingListeners) {\n-                    log.warn(\"Recovering node. Adding binding \" + binding.toString());\n-                    listener.handleClusterBindingsChanged(binding, BindingListener.BindingEvent.ADDED);\n-                }\n-            }\n-\n-            fromMap.removeAll(fromDBDuplicated);\n-            for (AndesBinding binding : fromMap) {\n-                for (BindingListener listener : bindingListeners) {\n-                    log.warn(\"Recovering node. removing binding \" + binding.toString());\n-                    listener.handleClusterBindingsChanged(binding, BindingListener.BindingEvent.DELETED);\n-                }\n-            }\n-        }\n-    }\n-\n-    private void reloadSubscriptions() throws AndesException {\n-        ClusterResourceHolder.getInstance().getSubscriptionManager().reloadSubscriptionsFromStorage();\n-    }\n+public class AndesRecoveryTask implements Runnable, StoreHealthListener {\n+\n+\tprivate List<QueueListener> queueListeners = new ArrayList<>();\n+\tprivate List<ExchangeListener> exchangeListeners = new ArrayList<>();\n+\tprivate List<BindingListener> bindingListeners = new ArrayList<>();\n+\n+\tAndesContextStore andesContextStore;\n+\tAMQPConstructStore amqpConstructStore;\n+\n+\t// set storeOperational to true since it can be assumed that the store is operational at startup\n+\t// if it is non-operational, the value will be updated immediately\n+\tAtomicBoolean isContextStoreOperational = new AtomicBoolean(true);\n+\n+\tprivate static Log log = LogFactory.getLog(AndesRecoveryTask.class);\n+\n+\tpublic AndesRecoveryTask() {\n+\n+\t\t// Register AndesRecoveryTask class as a StoreHealthListener\n+\t\tFailureObservingStoreManager.registerStoreHealthListener(this);\n+\n+\t\tqueueListeners.add(new ClusterCoordinationHandler(HazelcastAgent.getInstance()));\n+\t\texchangeListeners.add(new ClusterCoordinationHandler(HazelcastAgent.getInstance()));\n+\t\tbindingListeners.add(new ClusterCoordinationHandler(HazelcastAgent.getInstance()));\n+\n+\t\tandesContextStore = AndesContext.getInstance().getAndesContextStore();\n+\t\tamqpConstructStore = AndesContext.getInstance().getAMQPConstructStore();\n+\t}\n+\n+\t@Override\n+\tpublic void run() {\n+\t\tif(isContextStoreOperational.get()) {\n+\t\t\ttry {\n+\t\t\t\treloadExchangesFromDB();\n+\t\t\t\treloadQueuesFromDB();\n+\t\t\t\treloadBindingsFromDB();\n+\t\t\t\treloadSubscriptions();\n+\t\t\t} catch (AndesException e) {\n+\t\t\t\tlog.error(\"Error in running andes recovery task\", e);\n+\t\t\t}\n+\t\t} else {\n+\t\t\tlog.info(\"AndesRecoveryTask was paused due to non-operational context store.\");\n+\t\t}\n+\t}","path":"modules/andes-core/broker/src/main/java/org/wso2/andes/kernel/AndesRecoveryTask.java","position":175,"original_position":175,"commit_id":"253b445daa940a315af6213d8b98cabebb37901c","original_commit_id":"253b445daa940a315af6213d8b98cabebb37901c","user":{"login":"AnujaLK","id":484263,"avatar_url":"https://avatars.githubusercontent.com/u/484263?v=3","gravatar_id":"","url":"https://api.github.com/users/AnujaLK","html_url":"https://github.com/AnujaLK","followers_url":"https://api.github.com/users/AnujaLK/followers","following_url":"https://api.github.com/users/AnujaLK/following{/other_user}","gists_url":"https://api.github.com/users/AnujaLK/gists{/gist_id}","starred_url":"https://api.github.com/users/AnujaLK/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AnujaLK/subscriptions","organizations_url":"https://api.github.com/users/AnujaLK/orgs","repos_url":"https://api.github.com/users/AnujaLK/repos","events_url":"https://api.github.com/users/AnujaLK/events{/privacy}","received_events_url":"https://api.github.com/users/AnujaLK/received_events","type":"User","site_admin":false},"body":"This will run periodically to bring the node to the current state of cluster in case some hazelcast notifications are missed.","created_at":"2015-08-13T05:40:32Z","updated_at":"2015-08-13T05:40:32Z","html_url":"https://github.com/wso2/andes/pull/190#discussion_r36942272","pull_request_url":"https://api.github.com/repos/wso2/andes/pulls/190","_links":{"self":{"href":"https://api.github.com/repos/wso2/andes/pulls/comments/36942272"},"html":{"href":"https://github.com/wso2/andes/pull/190#discussion_r36942272"},"pull_request":{"href":"https://api.github.com/repos/wso2/andes/pulls/190"}}},"pull_request":{"url":"https://api.github.com/repos/wso2/andes/pulls/190","id":39549194,"html_url":"https://github.com/wso2/andes/pull/190","diff_url":"https://github.com/wso2/andes/pull/190.diff","patch_url":"https://github.com/wso2/andes/pull/190.patch","issue_url":"https://api.github.com/repos/wso2/andes/issues/190","number":190,"state":"closed","locked":false,"title":"Implement StoreHealthListener to AndesRecoveryTask","user":{"login":"AnujaLK","id":484263,"avatar_url":"https://avatars.githubusercontent.com/u/484263?v=3","gravatar_id":"","url":"https://api.github.com/users/AnujaLK","html_url":"https://github.com/AnujaLK","followers_url":"https://api.github.com/users/AnujaLK/followers","following_url":"https://api.github.com/users/AnujaLK/following{/other_user}","gists_url":"https://api.github.com/users/AnujaLK/gists{/gist_id}","starred_url":"https://api.github.com/users/AnujaLK/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AnujaLK/subscriptions","organizations_url":"https://api.github.com/users/AnujaLK/orgs","repos_url":"https://api.github.com/users/AnujaLK/repos","events_url":"https://api.github.com/users/AnujaLK/events{/privacy}","received_events_url":"https://api.github.com/users/AnujaLK/received_events","type":"User","site_admin":false},"body":"JIRA - https://wso2.org/jira/browse/MB-1147","created_at":"2015-07-09T05:27:57Z","updated_at":"2015-08-13T05:40:32Z","closed_at":"2015-07-28T06:01:46Z","merged_at":"2015-07-28T06:01:46Z","merge_commit_sha":"944d23da0d6fbd3297db5eff82c76797894db825","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/wso2/andes/pulls/190/commits","review_comments_url":"https://api.github.com/repos/wso2/andes/pulls/190/comments","review_comment_url":"https://api.github.com/repos/wso2/andes/pulls/comments{/number}","comments_url":"https://api.github.com/repos/wso2/andes/issues/190/comments","statuses_url":"https://api.github.com/repos/wso2/andes/statuses/253b445daa940a315af6213d8b98cabebb37901c","head":{"label":"AnujaLK:mb-1147","ref":"mb-1147","sha":"253b445daa940a315af6213d8b98cabebb37901c","user":{"login":"AnujaLK","id":484263,"avatar_url":"https://avatars.githubusercontent.com/u/484263?v=3","gravatar_id":"","url":"https://api.github.com/users/AnujaLK","html_url":"https://github.com/AnujaLK","followers_url":"https://api.github.com/users/AnujaLK/followers","following_url":"https://api.github.com/users/AnujaLK/following{/other_user}","gists_url":"https://api.github.com/users/AnujaLK/gists{/gist_id}","starred_url":"https://api.github.com/users/AnujaLK/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AnujaLK/subscriptions","organizations_url":"https://api.github.com/users/AnujaLK/orgs","repos_url":"https://api.github.com/users/AnujaLK/repos","events_url":"https://api.github.com/users/AnujaLK/events{/privacy}","received_events_url":"https://api.github.com/users/AnujaLK/received_events","type":"User","site_admin":false},"repo":{"id":33649363,"name":"andes","full_name":"AnujaLK/andes","owner":{"login":"AnujaLK","id":484263,"avatar_url":"https://avatars.githubusercontent.com/u/484263?v=3","gravatar_id":"","url":"https://api.github.com/users/AnujaLK","html_url":"https://github.com/AnujaLK","followers_url":"https://api.github.com/users/AnujaLK/followers","following_url":"https://api.github.com/users/AnujaLK/following{/other_user}","gists_url":"https://api.github.com/users/AnujaLK/gists{/gist_id}","starred_url":"https://api.github.com/users/AnujaLK/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AnujaLK/subscriptions","organizations_url":"https://api.github.com/users/AnujaLK/orgs","repos_url":"https://api.github.com/users/AnujaLK/repos","events_url":"https://api.github.com/users/AnujaLK/events{/privacy}","received_events_url":"https://api.github.com/users/AnujaLK/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/AnujaLK/andes","description":"","fork":true,"url":"https://api.github.com/repos/AnujaLK/andes","forks_url":"https://api.github.com/repos/AnujaLK/andes/forks","keys_url":"https://api.github.com/repos/AnujaLK/andes/keys{/key_id}","collaborators_url":"https://api.github.com/repos/AnujaLK/andes/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/AnujaLK/andes/teams","hooks_url":"https://api.github.com/repos/AnujaLK/andes/hooks","issue_events_url":"https://api.github.com/repos/AnujaLK/andes/issues/events{/number}","events_url":"https://api.github.com/repos/AnujaLK/andes/events","assignees_url":"https://api.github.com/repos/AnujaLK/andes/assignees{/user}","branches_url":"https://api.github.com/repos/AnujaLK/andes/branches{/branch}","tags_url":"https://api.github.com/repos/AnujaLK/andes/tags","blobs_url":"https://api.github.com/repos/AnujaLK/andes/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/AnujaLK/andes/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/AnujaLK/andes/git/refs{/sha}","trees_url":"https://api.github.com/repos/AnujaLK/andes/git/trees{/sha}","statuses_url":"https://api.github.com/repos/AnujaLK/andes/statuses/{sha}","languages_url":"https://api.github.com/repos/AnujaLK/andes/languages","stargazers_url":"https://api.github.com/repos/AnujaLK/andes/stargazers","contributors_url":"https://api.github.com/repos/AnujaLK/andes/contributors","subscribers_url":"https://api.github.com/repos/AnujaLK/andes/subscribers","subscription_url":"https://api.github.com/repos/AnujaLK/andes/subscription","commits_url":"https://api.github.com/repos/AnujaLK/andes/commits{/sha}","git_commits_url":"https://api.github.com/repos/AnujaLK/andes/git/commits{/sha}","comments_url":"https://api.github.com/repos/AnujaLK/andes/comments{/number}","issue_comment_url":"https://api.github.com/repos/AnujaLK/andes/issues/comments{/number}","contents_url":"https://api.github.com/repos/AnujaLK/andes/contents/{+path}","compare_url":"https://api.github.com/repos/AnujaLK/andes/compare/{base}...{head}","merges_url":"https://api.github.com/repos/AnujaLK/andes/merges","archive_url":"https://api.github.com/repos/AnujaLK/andes/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/AnujaLK/andes/downloads","issues_url":"https://api.github.com/repos/AnujaLK/andes/issues{/number}","pulls_url":"https://api.github.com/repos/AnujaLK/andes/pulls{/number}","milestones_url":"https://api.github.com/repos/AnujaLK/andes/milestones{/number}","notifications_url":"https://api.github.com/repos/AnujaLK/andes/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/AnujaLK/andes/labels{/name}","releases_url":"https://api.github.com/repos/AnujaLK/andes/releases{/id}","created_at":"2015-04-09T05:01:09Z","updated_at":"2015-04-21T06:44:01Z","pushed_at":"2015-08-12T05:07:12Z","git_url":"git://github.com/AnujaLK/andes.git","ssh_url":"git@github.com:AnujaLK/andes.git","clone_url":"https://github.com/AnujaLK/andes.git","svn_url":"https://github.com/AnujaLK/andes","homepage":null,"size":65799,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"wso2:master","ref":"master","sha":"e185a927b12d81cf8b1225a190aeb6ccd96d0564","user":{"login":"wso2","id":533043,"avatar_url":"https://avatars.githubusercontent.com/u/533043?v=3","gravatar_id":"","url":"https://api.github.com/users/wso2","html_url":"https://github.com/wso2","followers_url":"https://api.github.com/users/wso2/followers","following_url":"https://api.github.com/users/wso2/following{/other_user}","gists_url":"https://api.github.com/users/wso2/gists{/gist_id}","starred_url":"https://api.github.com/users/wso2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wso2/subscriptions","organizations_url":"https://api.github.com/users/wso2/orgs","repos_url":"https://api.github.com/users/wso2/repos","events_url":"https://api.github.com/users/wso2/events{/privacy}","received_events_url":"https://api.github.com/users/wso2/received_events","type":"Organization","site_admin":false},"repo":{"id":20559270,"name":"andes","full_name":"wso2/andes","owner":{"login":"wso2","id":533043,"avatar_url":"https://avatars.githubusercontent.com/u/533043?v=3","gravatar_id":"","url":"https://api.github.com/users/wso2","html_url":"https://github.com/wso2","followers_url":"https://api.github.com/users/wso2/followers","following_url":"https://api.github.com/users/wso2/following{/other_user}","gists_url":"https://api.github.com/users/wso2/gists{/gist_id}","starred_url":"https://api.github.com/users/wso2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wso2/subscriptions","organizations_url":"https://api.github.com/users/wso2/orgs","repos_url":"https://api.github.com/users/wso2/repos","events_url":"https://api.github.com/users/wso2/events{/privacy}","received_events_url":"https://api.github.com/users/wso2/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/wso2/andes","description":"","fork":false,"url":"https://api.github.com/repos/wso2/andes","forks_url":"https://api.github.com/repos/wso2/andes/forks","keys_url":"https://api.github.com/repos/wso2/andes/keys{/key_id}","collaborators_url":"https://api.github.com/repos/wso2/andes/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/wso2/andes/teams","hooks_url":"https://api.github.com/repos/wso2/andes/hooks","issue_events_url":"https://api.github.com/repos/wso2/andes/issues/events{/number}","events_url":"https://api.github.com/repos/wso2/andes/events","assignees_url":"https://api.github.com/repos/wso2/andes/assignees{/user}","branches_url":"https://api.github.com/repos/wso2/andes/branches{/branch}","tags_url":"https://api.github.com/repos/wso2/andes/tags","blobs_url":"https://api.github.com/repos/wso2/andes/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/wso2/andes/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/wso2/andes/git/refs{/sha}","trees_url":"https://api.github.com/repos/wso2/andes/git/trees{/sha}","statuses_url":"https://api.github.com/repos/wso2/andes/statuses/{sha}","languages_url":"https://api.github.com/repos/wso2/andes/languages","stargazers_url":"https://api.github.com/repos/wso2/andes/stargazers","contributors_url":"https://api.github.com/repos/wso2/andes/contributors","subscribers_url":"https://api.github.com/repos/wso2/andes/subscribers","subscription_url":"https://api.github.com/repos/wso2/andes/subscription","commits_url":"https://api.github.com/repos/wso2/andes/commits{/sha}","git_commits_url":"https://api.github.com/repos/wso2/andes/git/commits{/sha}","comments_url":"https://api.github.com/repos/wso2/andes/comments{/number}","issue_comment_url":"https://api.github.com/repos/wso2/andes/issues/comments{/number}","contents_url":"https://api.github.com/repos/wso2/andes/contents/{+path}","compare_url":"https://api.github.com/repos/wso2/andes/compare/{base}...{head}","merges_url":"https://api.github.com/repos/wso2/andes/merges","archive_url":"https://api.github.com/repos/wso2/andes/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/wso2/andes/downloads","issues_url":"https://api.github.com/repos/wso2/andes/issues{/number}","pulls_url":"https://api.github.com/repos/wso2/andes/pulls{/number}","milestones_url":"https://api.github.com/repos/wso2/andes/milestones{/number}","notifications_url":"https://api.github.com/repos/wso2/andes/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/wso2/andes/labels{/name}","releases_url":"https://api.github.com/repos/wso2/andes/releases{/id}","created_at":"2014-06-06T10:04:24Z","updated_at":"2015-05-05T14:31:31Z","pushed_at":"2015-08-12T11:23:56Z","git_url":"git://github.com/wso2/andes.git","ssh_url":"git@github.com:wso2/andes.git","clone_url":"https://github.com/wso2/andes.git","svn_url":"https://github.com/wso2/andes","homepage":null,"size":68352,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":38,"mirror_url":null,"open_issues_count":10,"forks":38,"open_issues":10,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/wso2/andes/pulls/190"},"html":{"href":"https://github.com/wso2/andes/pull/190"},"issue":{"href":"https://api.github.com/repos/wso2/andes/issues/190"},"comments":{"href":"https://api.github.com/repos/wso2/andes/issues/190/comments"},"review_comments":{"href":"https://api.github.com/repos/wso2/andes/pulls/190/comments"},"review_comment":{"href":"https://api.github.com/repos/wso2/andes/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/wso2/andes/pulls/190/commits"},"statuses":{"href":"https://api.github.com/repos/wso2/andes/statuses/253b445daa940a315af6213d8b98cabebb37901c"}}}}