{"action":"created","comment":{"url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/comments/44690057","id":44690057,"diff_hunk":"@@ -0,0 +1,71 @@\n+\"\"\"\n+Tests for flocker base test cases.\n+\"\"\"\n+\n+import errno\n+import shutil\n+\n+from hypothesis import given\n+from hypothesis.strategies import binary\n+from testtools import TestCase\n+from testtools.matchers import (\n+    DirExists,\n+    Not,\n+    PathExists,\n+)\n+from testtools.testresult.doubles import Python27TestResult\n+from twisted.python.filepath import FilePath\n+from twisted.trial import unittest\n+\n+from .._base import AsyncTestCase\n+\n+\n+class AsyncTestCaseTests(TestCase):\n+\n+    @given(binary())\n+    def test_trial_skip_exception(self, reason):\n+        \"\"\"\n+        If tests raise the ``SkipTest`` exported by Trial, then that's\n+        recorded as a skip.\n+        \"\"\"\n+\n+        class SkippingTest(AsyncTestCase):\n+            def test_skip(self):\n+                raise unittest.SkipTest(reason)\n+\n+        test = SkippingTest('test_skip')\n+        result = Python27TestResult()\n+        test.run(result)\n+        self.assertEqual([\n+            ('startTest', test),\n+            ('addSkip', test, reason),\n+            ('stopTest', test),\n+        ], result._events)\n+\n+    def test_mktemp_doesnt_exist(self):\n+        \"\"\"\n+        ``mktemp`` returns a path that doesn't exist inside a directory that\n+        does.\n+        \"\"\"\n+\n+        class SomeTest(AsyncTestCase):\n+            def test_pass(self):\n+                pass\n+\n+        test = SomeTest('test_pass')\n+        temp_path = FilePath(test.mktemp())\n+        self.addCleanup(_remove_dir, temp_path.parent())","path":"flocker/testtools/test/test_base.py","position":81,"original_position":57,"commit_id":"1e9f8b5ad60b31201dde9029fe7606de2196bc1d","original_commit_id":"325ec0cfd2dc8a70ff9b64b08576c5e88a413f32","user":{"login":"jml","id":125674,"avatar_url":"https://avatars.githubusercontent.com/u/125674?v=3","gravatar_id":"","url":"https://api.github.com/users/jml","html_url":"https://github.com/jml","followers_url":"https://api.github.com/users/jml/followers","following_url":"https://api.github.com/users/jml/following{/other_user}","gists_url":"https://api.github.com/users/jml/gists{/gist_id}","starred_url":"https://api.github.com/users/jml/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jml/subscriptions","organizations_url":"https://api.github.com/users/jml/orgs","repos_url":"https://api.github.com/users/jml/repos","events_url":"https://api.github.com/users/jml/events{/privacy}","received_events_url":"https://api.github.com/users/jml/received_events","type":"User","site_admin":false},"body":"Circling back: Now that we don't auto-delete at all, this is even more important. ","created_at":"2015-11-12T17:54:34Z","updated_at":"2015-11-12T17:54:34Z","html_url":"https://github.com/ClusterHQ/flocker/pull/2163#discussion_r44690057","pull_request_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2163","_links":{"self":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/comments/44690057"},"html":{"href":"https://github.com/ClusterHQ/flocker/pull/2163#discussion_r44690057"},"pull_request":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2163"}}},"pull_request":{"url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2163","id":50423809,"html_url":"https://github.com/ClusterHQ/flocker/pull/2163","diff_url":"https://github.com/ClusterHQ/flocker/pull/2163.diff","patch_url":"https://github.com/ClusterHQ/flocker/pull/2163.patch","issue_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/2163","number":2163,"state":"open","locked":false,"title":"[FLOC-3412] New base TestCase ","user":{"login":"jml","id":125674,"avatar_url":"https://avatars.githubusercontent.com/u/125674?v=3","gravatar_id":"","url":"https://api.github.com/users/jml","html_url":"https://github.com/jml","followers_url":"https://api.github.com/users/jml/followers","following_url":"https://api.github.com/users/jml/following{/other_user}","gists_url":"https://api.github.com/users/jml/gists{/gist_id}","starred_url":"https://api.github.com/users/jml/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jml/subscriptions","organizations_url":"https://api.github.com/users/jml/orgs","repos_url":"https://api.github.com/users/jml/repos","events_url":"https://api.github.com/users/jml/events{/privacy}","received_events_url":"https://api.github.com/users/jml/received_events","type":"User","site_admin":false},"body":"In order to implement retries for flaky tests, we're going to have to customize the way tests are run, because we want to retry `setUp`, the test method itself, and `tearDown`.\r\n\r\n[testtools](http://testtools.readthedocs.org/en/latest/) has a built-in mechanism for customizing the behaviour of how a test is run without retrying (using the [RunTest abstraction](http://testtools.readthedocs.org/en/latest/for-framework-folk.html#controlling-test-execution)). Because these are composable, we can combine our (future, not yet implemented) `RetryFlakyTest` runner with the runners that testtools provides for Twisted tests. \r\n\r\nThe implementation strategy here is to create two new \"official\" base classes for flocker tests: `flocker.testtools.TestCase` and `flocker.testtools.AsyncTestCase`. In later branches, all existing tests will be migrated to use these base cases. For now, I've focused only on the acceptance tests, because they are the most flaky ones.\r\n\r\nA later branch will add a runner for retrying flaky tests. \r\n\r\nIn an attempt to actually _add_ value, I've done a little bit of extra work to provide pretty logs with test failures. Unfortunately, I haven't yet got a good example of how it looks. \r\n\r\nSome context:\r\n\r\ntesttools has a feature for attaching arbitrary stuff to a test and having that stuff show up when the test fails. For Twisted tests, it attaches the entire log that occurred while the test was run, e.g.\r\nhttp://build.clusterhq.com/builders/flocker%2Facceptance%2Frackspace%2Fubuntu-14.04%2Frackspace/builds/3324/steps/trial/logs/problems\r\n\r\nBecause we log a lot of stuff to Twisted in our acceptance tests, I thought it would be nice to separate out the eliot logs & pretty print them. This will hopefully make failures easier to debug. When I get an example I'll link it here.\r\n\r\nAs usual, I've added `XXX` comments where I'd particularly like feedback about a trade-off I made or a decision that I'm not entirely comfortable about.\n\n<!-- Reviewable:start -->\n[<img src=\"https://reviewable.io/review_button.png\" height=40 alt=\"Review on Reviewable\"/>](https://reviewable.io/reviews/clusterhq/flocker/2163)\n<!-- Reviewable:end -->\n","created_at":"2015-11-11T19:19:15Z","updated_at":"2015-11-12T17:54:34Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5af38f15d0a448f95ca7dee29bbc330b1650edf9","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2163/commits","review_comments_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2163/comments","review_comment_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/2163/comments","statuses_url":"https://api.github.com/repos/ClusterHQ/flocker/statuses/1e9f8b5ad60b31201dde9029fe7606de2196bc1d","head":{"label":"ClusterHQ:new-base-FLOC-3412","ref":"new-base-FLOC-3412","sha":"1e9f8b5ad60b31201dde9029fe7606de2196bc1d","user":{"login":"ClusterHQ","id":7913449,"avatar_url":"https://avatars.githubusercontent.com/u/7913449?v=3","gravatar_id":"","url":"https://api.github.com/users/ClusterHQ","html_url":"https://github.com/ClusterHQ","followers_url":"https://api.github.com/users/ClusterHQ/followers","following_url":"https://api.github.com/users/ClusterHQ/following{/other_user}","gists_url":"https://api.github.com/users/ClusterHQ/gists{/gist_id}","starred_url":"https://api.github.com/users/ClusterHQ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ClusterHQ/subscriptions","organizations_url":"https://api.github.com/users/ClusterHQ/orgs","repos_url":"https://api.github.com/users/ClusterHQ/repos","events_url":"https://api.github.com/users/ClusterHQ/events{/privacy}","received_events_url":"https://api.github.com/users/ClusterHQ/received_events","type":"Organization","site_admin":false},"repo":{"id":19248542,"name":"flocker","full_name":"ClusterHQ/flocker","owner":{"login":"ClusterHQ","id":7913449,"avatar_url":"https://avatars.githubusercontent.com/u/7913449?v=3","gravatar_id":"","url":"https://api.github.com/users/ClusterHQ","html_url":"https://github.com/ClusterHQ","followers_url":"https://api.github.com/users/ClusterHQ/followers","following_url":"https://api.github.com/users/ClusterHQ/following{/other_user}","gists_url":"https://api.github.com/users/ClusterHQ/gists{/gist_id}","starred_url":"https://api.github.com/users/ClusterHQ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ClusterHQ/subscriptions","organizations_url":"https://api.github.com/users/ClusterHQ/orgs","repos_url":"https://api.github.com/users/ClusterHQ/repos","events_url":"https://api.github.com/users/ClusterHQ/events{/privacy}","received_events_url":"https://api.github.com/users/ClusterHQ/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ClusterHQ/flocker","description":"Container data volume manager for your Dockerized application","fork":false,"url":"https://api.github.com/repos/ClusterHQ/flocker","forks_url":"https://api.github.com/repos/ClusterHQ/flocker/forks","keys_url":"https://api.github.com/repos/ClusterHQ/flocker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ClusterHQ/flocker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ClusterHQ/flocker/teams","hooks_url":"https://api.github.com/repos/ClusterHQ/flocker/hooks","issue_events_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/events{/number}","events_url":"https://api.github.com/repos/ClusterHQ/flocker/events","assignees_url":"https://api.github.com/repos/ClusterHQ/flocker/assignees{/user}","branches_url":"https://api.github.com/repos/ClusterHQ/flocker/branches{/branch}","tags_url":"https://api.github.com/repos/ClusterHQ/flocker/tags","blobs_url":"https://api.github.com/repos/ClusterHQ/flocker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ClusterHQ/flocker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ClusterHQ/flocker/git/refs{/sha}","trees_url":"https://api.github.com/repos/ClusterHQ/flocker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ClusterHQ/flocker/statuses/{sha}","languages_url":"https://api.github.com/repos/ClusterHQ/flocker/languages","stargazers_url":"https://api.github.com/repos/ClusterHQ/flocker/stargazers","contributors_url":"https://api.github.com/repos/ClusterHQ/flocker/contributors","subscribers_url":"https://api.github.com/repos/ClusterHQ/flocker/subscribers","subscription_url":"https://api.github.com/repos/ClusterHQ/flocker/subscription","commits_url":"https://api.github.com/repos/ClusterHQ/flocker/commits{/sha}","git_commits_url":"https://api.github.com/repos/ClusterHQ/flocker/git/commits{/sha}","comments_url":"https://api.github.com/repos/ClusterHQ/flocker/comments{/number}","issue_comment_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/comments{/number}","contents_url":"https://api.github.com/repos/ClusterHQ/flocker/contents/{+path}","compare_url":"https://api.github.com/repos/ClusterHQ/flocker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ClusterHQ/flocker/merges","archive_url":"https://api.github.com/repos/ClusterHQ/flocker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ClusterHQ/flocker/downloads","issues_url":"https://api.github.com/repos/ClusterHQ/flocker/issues{/number}","pulls_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls{/number}","milestones_url":"https://api.github.com/repos/ClusterHQ/flocker/milestones{/number}","notifications_url":"https://api.github.com/repos/ClusterHQ/flocker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ClusterHQ/flocker/labels{/name}","releases_url":"https://api.github.com/repos/ClusterHQ/flocker/releases{/id}","created_at":"2014-04-28T18:02:14Z","updated_at":"2015-11-11T17:41:02Z","pushed_at":"2015-11-12T17:45:35Z","git_url":"git://github.com/ClusterHQ/flocker.git","ssh_url":"git@github.com:ClusterHQ/flocker.git","clone_url":"https://github.com/ClusterHQ/flocker.git","svn_url":"https://github.com/ClusterHQ/flocker","homepage":"https://clusterhq.com","size":71220,"stargazers_count":1945,"watchers_count":1945,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":142,"mirror_url":null,"open_issues_count":31,"forks":142,"open_issues":31,"watchers":1945,"default_branch":"master"}},"base":{"label":"ClusterHQ:master","ref":"master","sha":"83de9ee0884140d4f66a603e1cc0b8f2962d05c7","user":{"login":"ClusterHQ","id":7913449,"avatar_url":"https://avatars.githubusercontent.com/u/7913449?v=3","gravatar_id":"","url":"https://api.github.com/users/ClusterHQ","html_url":"https://github.com/ClusterHQ","followers_url":"https://api.github.com/users/ClusterHQ/followers","following_url":"https://api.github.com/users/ClusterHQ/following{/other_user}","gists_url":"https://api.github.com/users/ClusterHQ/gists{/gist_id}","starred_url":"https://api.github.com/users/ClusterHQ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ClusterHQ/subscriptions","organizations_url":"https://api.github.com/users/ClusterHQ/orgs","repos_url":"https://api.github.com/users/ClusterHQ/repos","events_url":"https://api.github.com/users/ClusterHQ/events{/privacy}","received_events_url":"https://api.github.com/users/ClusterHQ/received_events","type":"Organization","site_admin":false},"repo":{"id":19248542,"name":"flocker","full_name":"ClusterHQ/flocker","owner":{"login":"ClusterHQ","id":7913449,"avatar_url":"https://avatars.githubusercontent.com/u/7913449?v=3","gravatar_id":"","url":"https://api.github.com/users/ClusterHQ","html_url":"https://github.com/ClusterHQ","followers_url":"https://api.github.com/users/ClusterHQ/followers","following_url":"https://api.github.com/users/ClusterHQ/following{/other_user}","gists_url":"https://api.github.com/users/ClusterHQ/gists{/gist_id}","starred_url":"https://api.github.com/users/ClusterHQ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ClusterHQ/subscriptions","organizations_url":"https://api.github.com/users/ClusterHQ/orgs","repos_url":"https://api.github.com/users/ClusterHQ/repos","events_url":"https://api.github.com/users/ClusterHQ/events{/privacy}","received_events_url":"https://api.github.com/users/ClusterHQ/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ClusterHQ/flocker","description":"Container data volume manager for your Dockerized application","fork":false,"url":"https://api.github.com/repos/ClusterHQ/flocker","forks_url":"https://api.github.com/repos/ClusterHQ/flocker/forks","keys_url":"https://api.github.com/repos/ClusterHQ/flocker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ClusterHQ/flocker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ClusterHQ/flocker/teams","hooks_url":"https://api.github.com/repos/ClusterHQ/flocker/hooks","issue_events_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/events{/number}","events_url":"https://api.github.com/repos/ClusterHQ/flocker/events","assignees_url":"https://api.github.com/repos/ClusterHQ/flocker/assignees{/user}","branches_url":"https://api.github.com/repos/ClusterHQ/flocker/branches{/branch}","tags_url":"https://api.github.com/repos/ClusterHQ/flocker/tags","blobs_url":"https://api.github.com/repos/ClusterHQ/flocker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ClusterHQ/flocker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ClusterHQ/flocker/git/refs{/sha}","trees_url":"https://api.github.com/repos/ClusterHQ/flocker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ClusterHQ/flocker/statuses/{sha}","languages_url":"https://api.github.com/repos/ClusterHQ/flocker/languages","stargazers_url":"https://api.github.com/repos/ClusterHQ/flocker/stargazers","contributors_url":"https://api.github.com/repos/ClusterHQ/flocker/contributors","subscribers_url":"https://api.github.com/repos/ClusterHQ/flocker/subscribers","subscription_url":"https://api.github.com/repos/ClusterHQ/flocker/subscription","commits_url":"https://api.github.com/repos/ClusterHQ/flocker/commits{/sha}","git_commits_url":"https://api.github.com/repos/ClusterHQ/flocker/git/commits{/sha}","comments_url":"https://api.github.com/repos/ClusterHQ/flocker/comments{/number}","issue_comment_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/comments{/number}","contents_url":"https://api.github.com/repos/ClusterHQ/flocker/contents/{+path}","compare_url":"https://api.github.com/repos/ClusterHQ/flocker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ClusterHQ/flocker/merges","archive_url":"https://api.github.com/repos/ClusterHQ/flocker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ClusterHQ/flocker/downloads","issues_url":"https://api.github.com/repos/ClusterHQ/flocker/issues{/number}","pulls_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls{/number}","milestones_url":"https://api.github.com/repos/ClusterHQ/flocker/milestones{/number}","notifications_url":"https://api.github.com/repos/ClusterHQ/flocker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ClusterHQ/flocker/labels{/name}","releases_url":"https://api.github.com/repos/ClusterHQ/flocker/releases{/id}","created_at":"2014-04-28T18:02:14Z","updated_at":"2015-11-11T17:41:02Z","pushed_at":"2015-11-12T17:45:35Z","git_url":"git://github.com/ClusterHQ/flocker.git","ssh_url":"git@github.com:ClusterHQ/flocker.git","clone_url":"https://github.com/ClusterHQ/flocker.git","svn_url":"https://github.com/ClusterHQ/flocker","homepage":"https://clusterhq.com","size":71220,"stargazers_count":1945,"watchers_count":1945,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":142,"mirror_url":null,"open_issues_count":31,"forks":142,"open_issues":31,"watchers":1945,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2163"},"html":{"href":"https://github.com/ClusterHQ/flocker/pull/2163"},"issue":{"href":"https://api.github.com/repos/ClusterHQ/flocker/issues/2163"},"comments":{"href":"https://api.github.com/repos/ClusterHQ/flocker/issues/2163/comments"},"review_comments":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2163/comments"},"review_comment":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2163/commits"},"statuses":{"href":"https://api.github.com/repos/ClusterHQ/flocker/statuses/1e9f8b5ad60b31201dde9029fe7606de2196bc1d"}}}}