{"action":"created","comment":{"url":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/comments/25847479","id":25847479,"diff_hunk":"@@ -88,38 +92,54 @@ def check(self):\n         logging.info(\"Start check\")\n         assert self.TESTS\n \n-        for category, tests in self.TESTS.items():\n-            yield self.executor.start_env(category)\n-            yield self.executor.set_config(self.get_env_config())\n-            for test in tests:\n-                result_code = yield self.executor.run_code_and_function(\n-                    code=self.user_data['code'],\n-                    function_name=self.FUNCTION_NAME or test[\"function_name\"],\n-                    args=test.get('input', None),\n-                    exec_name=category\n-                )\n-\n-                validator = self.VALIDATOR(test)\n-                validator_result = validator.validate(result_code)\n-\n-                logging.info(\"REFEREE:: check result for category {0}, test {1}: {2}\".format(\n-                    category, tests.index(test), validator_result.test_passed))\n-                if validator_result.additional_data:\n-                    logging.info(\"VALIDATOR:: Additional data: {}\".format(\n-                        validator.additional_data))\n-\n-                if not validator_result.test_passed:\n-                    yield self.executor.kill(category)\n-                    description = \"Category: {0}. Test {1}\".format(category, tests.index(test))\n-                    return (yield self.user.post_check_fail(description))\n-\n-            yield self.executor.kill(category)\n-        return self.check_success()\n+        for category_name, tests in self.TESTS.items():\n+            self.check_category(category_name, tests)\n+\n+        return self.check_success(points=self.points)\n+\n+    @gen.coroutine\n+    def check_category(self, category_name, tests):\n+        yield self.executor.start_env(category_name)\n+        yield self.executor.set_config(self.get_env_config())\n+        for n, test in enumerate(tests):\n+            self.check_test(category_name, test, n)","path":"checkio_referee/base.py","position":68,"original_position":68,"commit_id":"2fbaa5b7c3956a52386901a89bb10973887d225e","original_commit_id":"2fbaa5b7c3956a52386901a89bb10973887d225e","user":{"login":"mastak","id":628807,"avatar_url":"https://avatars.githubusercontent.com/u/628807?v=3","gravatar_id":"","url":"https://api.github.com/users/mastak","html_url":"https://github.com/mastak","followers_url":"https://api.github.com/users/mastak/followers","following_url":"https://api.github.com/users/mastak/following{/other_user}","gists_url":"https://api.github.com/users/mastak/gists{/gist_id}","starred_url":"https://api.github.com/users/mastak/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mastak/subscriptions","organizations_url":"https://api.github.com/users/mastak/orgs","repos_url":"https://api.github.com/users/mastak/repos","events_url":"https://api.github.com/users/mastak/events{/privacy}","received_events_url":"https://api.github.com/users/mastak/received_events","type":"User","site_admin":false},"body":"`yield self.check_test(category_name, test, n)`","created_at":"2015-03-05T08:43:21Z","updated_at":"2015-03-05T08:43:21Z","html_url":"https://github.com/CheckiO/checkio-referee/pull/8#discussion_r25847479","pull_request_url":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/8","_links":{"self":{"href":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/comments/25847479"},"html":{"href":"https://github.com/CheckiO/checkio-referee/pull/8#discussion_r25847479"},"pull_request":{"href":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/8"}}},"pull_request":{"url":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/8","id":30534608,"html_url":"https://github.com/CheckiO/checkio-referee/pull/8","diff_url":"https://github.com/CheckiO/checkio-referee/pull/8.diff","patch_url":"https://github.com/CheckiO/checkio-referee/pull/8.patch","issue_url":"https://api.github.com/repos/CheckiO/checkio-referee/issues/8","number":8,"state":"open","locked":false,"title":"Representations, Decomosition and utils","user":{"login":"Bryukh","id":343120,"avatar_url":"https://avatars.githubusercontent.com/u/343120?v=3","gravatar_id":"","url":"https://api.github.com/users/Bryukh","html_url":"https://github.com/Bryukh","followers_url":"https://api.github.com/users/Bryukh/followers","following_url":"https://api.github.com/users/Bryukh/following{/other_user}","gists_url":"https://api.github.com/users/Bryukh/gists{/gist_id}","starred_url":"https://api.github.com/users/Bryukh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Bryukh/subscriptions","organizations_url":"https://api.github.com/users/Bryukh/orgs","repos_url":"https://api.github.com/users/Bryukh/repos","events_url":"https://api.github.com/users/Bryukh/events{/privacy}","received_events_url":"https://api.github.com/users/Bryukh/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-03-05T07:03:02Z","updated_at":"2015-03-05T08:43:21Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0f2e9afb9bb80301923cf64be6b07d99fcf993fd","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/8/commits","review_comments_url":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/8/comments","review_comment_url":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/comments{/number}","comments_url":"https://api.github.com/repos/CheckiO/checkio-referee/issues/8/comments","statuses_url":"https://api.github.com/repos/CheckiO/checkio-referee/statuses/2fbaa5b7c3956a52386901a89bb10973887d225e","head":{"label":"CheckiO:dev-rank","ref":"dev-rank","sha":"2fbaa5b7c3956a52386901a89bb10973887d225e","user":{"login":"CheckiO","id":3198851,"avatar_url":"https://avatars.githubusercontent.com/u/3198851?v=3","gravatar_id":"","url":"https://api.github.com/users/CheckiO","html_url":"https://github.com/CheckiO","followers_url":"https://api.github.com/users/CheckiO/followers","following_url":"https://api.github.com/users/CheckiO/following{/other_user}","gists_url":"https://api.github.com/users/CheckiO/gists{/gist_id}","starred_url":"https://api.github.com/users/CheckiO/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CheckiO/subscriptions","organizations_url":"https://api.github.com/users/CheckiO/orgs","repos_url":"https://api.github.com/users/CheckiO/repos","events_url":"https://api.github.com/users/CheckiO/events{/privacy}","received_events_url":"https://api.github.com/users/CheckiO/received_events","type":"User","site_admin":false},"repo":{"id":30643416,"name":"checkio-referee","full_name":"CheckiO/checkio-referee","owner":{"login":"CheckiO","id":3198851,"avatar_url":"https://avatars.githubusercontent.com/u/3198851?v=3","gravatar_id":"","url":"https://api.github.com/users/CheckiO","html_url":"https://github.com/CheckiO","followers_url":"https://api.github.com/users/CheckiO/followers","following_url":"https://api.github.com/users/CheckiO/following{/other_user}","gists_url":"https://api.github.com/users/CheckiO/gists{/gist_id}","starred_url":"https://api.github.com/users/CheckiO/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CheckiO/subscriptions","organizations_url":"https://api.github.com/users/CheckiO/orgs","repos_url":"https://api.github.com/users/CheckiO/repos","events_url":"https://api.github.com/users/CheckiO/events{/privacy}","received_events_url":"https://api.github.com/users/CheckiO/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/CheckiO/checkio-referee","description":"","fork":false,"url":"https://api.github.com/repos/CheckiO/checkio-referee","forks_url":"https://api.github.com/repos/CheckiO/checkio-referee/forks","keys_url":"https://api.github.com/repos/CheckiO/checkio-referee/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CheckiO/checkio-referee/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CheckiO/checkio-referee/teams","hooks_url":"https://api.github.com/repos/CheckiO/checkio-referee/hooks","issue_events_url":"https://api.github.com/repos/CheckiO/checkio-referee/issues/events{/number}","events_url":"https://api.github.com/repos/CheckiO/checkio-referee/events","assignees_url":"https://api.github.com/repos/CheckiO/checkio-referee/assignees{/user}","branches_url":"https://api.github.com/repos/CheckiO/checkio-referee/branches{/branch}","tags_url":"https://api.github.com/repos/CheckiO/checkio-referee/tags","blobs_url":"https://api.github.com/repos/CheckiO/checkio-referee/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CheckiO/checkio-referee/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CheckiO/checkio-referee/git/refs{/sha}","trees_url":"https://api.github.com/repos/CheckiO/checkio-referee/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CheckiO/checkio-referee/statuses/{sha}","languages_url":"https://api.github.com/repos/CheckiO/checkio-referee/languages","stargazers_url":"https://api.github.com/repos/CheckiO/checkio-referee/stargazers","contributors_url":"https://api.github.com/repos/CheckiO/checkio-referee/contributors","subscribers_url":"https://api.github.com/repos/CheckiO/checkio-referee/subscribers","subscription_url":"https://api.github.com/repos/CheckiO/checkio-referee/subscription","commits_url":"https://api.github.com/repos/CheckiO/checkio-referee/commits{/sha}","git_commits_url":"https://api.github.com/repos/CheckiO/checkio-referee/git/commits{/sha}","comments_url":"https://api.github.com/repos/CheckiO/checkio-referee/comments{/number}","issue_comment_url":"https://api.github.com/repos/CheckiO/checkio-referee/issues/comments{/number}","contents_url":"https://api.github.com/repos/CheckiO/checkio-referee/contents/{+path}","compare_url":"https://api.github.com/repos/CheckiO/checkio-referee/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CheckiO/checkio-referee/merges","archive_url":"https://api.github.com/repos/CheckiO/checkio-referee/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CheckiO/checkio-referee/downloads","issues_url":"https://api.github.com/repos/CheckiO/checkio-referee/issues{/number}","pulls_url":"https://api.github.com/repos/CheckiO/checkio-referee/pulls{/number}","milestones_url":"https://api.github.com/repos/CheckiO/checkio-referee/milestones{/number}","notifications_url":"https://api.github.com/repos/CheckiO/checkio-referee/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CheckiO/checkio-referee/labels{/name}","releases_url":"https://api.github.com/repos/CheckiO/checkio-referee/releases{/id}","created_at":"2015-02-11T10:48:56Z","updated_at":"2015-03-03T17:23:42Z","pushed_at":"2015-03-05T07:02:06Z","git_url":"git://github.com/CheckiO/checkio-referee.git","ssh_url":"git@github.com:CheckiO/checkio-referee.git","clone_url":"https://github.com/CheckiO/checkio-referee.git","svn_url":"https://github.com/CheckiO/checkio-referee","homepage":null,"size":352,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"base":{"label":"CheckiO:master","ref":"master","sha":"041477bed64455544c67b5856d62000772146104","user":{"login":"CheckiO","id":3198851,"avatar_url":"https://avatars.githubusercontent.com/u/3198851?v=3","gravatar_id":"","url":"https://api.github.com/users/CheckiO","html_url":"https://github.com/CheckiO","followers_url":"https://api.github.com/users/CheckiO/followers","following_url":"https://api.github.com/users/CheckiO/following{/other_user}","gists_url":"https://api.github.com/users/CheckiO/gists{/gist_id}","starred_url":"https://api.github.com/users/CheckiO/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CheckiO/subscriptions","organizations_url":"https://api.github.com/users/CheckiO/orgs","repos_url":"https://api.github.com/users/CheckiO/repos","events_url":"https://api.github.com/users/CheckiO/events{/privacy}","received_events_url":"https://api.github.com/users/CheckiO/received_events","type":"User","site_admin":false},"repo":{"id":30643416,"name":"checkio-referee","full_name":"CheckiO/checkio-referee","owner":{"login":"CheckiO","id":3198851,"avatar_url":"https://avatars.githubusercontent.com/u/3198851?v=3","gravatar_id":"","url":"https://api.github.com/users/CheckiO","html_url":"https://github.com/CheckiO","followers_url":"https://api.github.com/users/CheckiO/followers","following_url":"https://api.github.com/users/CheckiO/following{/other_user}","gists_url":"https://api.github.com/users/CheckiO/gists{/gist_id}","starred_url":"https://api.github.com/users/CheckiO/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CheckiO/subscriptions","organizations_url":"https://api.github.com/users/CheckiO/orgs","repos_url":"https://api.github.com/users/CheckiO/repos","events_url":"https://api.github.com/users/CheckiO/events{/privacy}","received_events_url":"https://api.github.com/users/CheckiO/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/CheckiO/checkio-referee","description":"","fork":false,"url":"https://api.github.com/repos/CheckiO/checkio-referee","forks_url":"https://api.github.com/repos/CheckiO/checkio-referee/forks","keys_url":"https://api.github.com/repos/CheckiO/checkio-referee/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CheckiO/checkio-referee/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CheckiO/checkio-referee/teams","hooks_url":"https://api.github.com/repos/CheckiO/checkio-referee/hooks","issue_events_url":"https://api.github.com/repos/CheckiO/checkio-referee/issues/events{/number}","events_url":"https://api.github.com/repos/CheckiO/checkio-referee/events","assignees_url":"https://api.github.com/repos/CheckiO/checkio-referee/assignees{/user}","branches_url":"https://api.github.com/repos/CheckiO/checkio-referee/branches{/branch}","tags_url":"https://api.github.com/repos/CheckiO/checkio-referee/tags","blobs_url":"https://api.github.com/repos/CheckiO/checkio-referee/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CheckiO/checkio-referee/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CheckiO/checkio-referee/git/refs{/sha}","trees_url":"https://api.github.com/repos/CheckiO/checkio-referee/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CheckiO/checkio-referee/statuses/{sha}","languages_url":"https://api.github.com/repos/CheckiO/checkio-referee/languages","stargazers_url":"https://api.github.com/repos/CheckiO/checkio-referee/stargazers","contributors_url":"https://api.github.com/repos/CheckiO/checkio-referee/contributors","subscribers_url":"https://api.github.com/repos/CheckiO/checkio-referee/subscribers","subscription_url":"https://api.github.com/repos/CheckiO/checkio-referee/subscription","commits_url":"https://api.github.com/repos/CheckiO/checkio-referee/commits{/sha}","git_commits_url":"https://api.github.com/repos/CheckiO/checkio-referee/git/commits{/sha}","comments_url":"https://api.github.com/repos/CheckiO/checkio-referee/comments{/number}","issue_comment_url":"https://api.github.com/repos/CheckiO/checkio-referee/issues/comments{/number}","contents_url":"https://api.github.com/repos/CheckiO/checkio-referee/contents/{+path}","compare_url":"https://api.github.com/repos/CheckiO/checkio-referee/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CheckiO/checkio-referee/merges","archive_url":"https://api.github.com/repos/CheckiO/checkio-referee/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CheckiO/checkio-referee/downloads","issues_url":"https://api.github.com/repos/CheckiO/checkio-referee/issues{/number}","pulls_url":"https://api.github.com/repos/CheckiO/checkio-referee/pulls{/number}","milestones_url":"https://api.github.com/repos/CheckiO/checkio-referee/milestones{/number}","notifications_url":"https://api.github.com/repos/CheckiO/checkio-referee/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CheckiO/checkio-referee/labels{/name}","releases_url":"https://api.github.com/repos/CheckiO/checkio-referee/releases{/id}","created_at":"2015-02-11T10:48:56Z","updated_at":"2015-03-03T17:23:42Z","pushed_at":"2015-03-05T07:02:06Z","git_url":"git://github.com/CheckiO/checkio-referee.git","ssh_url":"git@github.com:CheckiO/checkio-referee.git","clone_url":"https://github.com/CheckiO/checkio-referee.git","svn_url":"https://github.com/CheckiO/checkio-referee","homepage":null,"size":352,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":2,"forks":0,"open_issues":2,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/8"},"html":{"href":"https://github.com/CheckiO/checkio-referee/pull/8"},"issue":{"href":"https://api.github.com/repos/CheckiO/checkio-referee/issues/8"},"comments":{"href":"https://api.github.com/repos/CheckiO/checkio-referee/issues/8/comments"},"review_comments":{"href":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/8/comments"},"review_comment":{"href":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/CheckiO/checkio-referee/pulls/8/commits"},"statuses":{"href":"https://api.github.com/repos/CheckiO/checkio-referee/statuses/2fbaa5b7c3956a52386901a89bb10973887d225e"}}}}