{"action":"created","comment":{"url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/comments/45069884","id":45069884,"diff_hunk":"@@ -0,0 +1,145 @@\n+# Copyright 2015 ClusterHQ Inc.  See LICENSE file for details.\n+\"\"\"\n+Driver tests for the control service benchmarks.\n+\"\"\"\n+\n+from itertools import count, repeat\n+\n+from zope.interface import implementer\n+\n+from twisted.internet.defer import Deferred, succeed, fail\n+from twisted.trial.unittest import TestCase\n+\n+from benchmark._driver import benchmark\n+from benchmark._interfaces import IProbe, IOperation\n+\n+\n+class FakeMetric:\n+\n+    def __init__(self, measurements):\n+        \"\"\"\n+        :param measurements: An iterator providing measurement to be\n+            returned on each call.\n+        \"\"\"\n+        self.measurements = measurements\n+\n+    def measure(self, f, *a, **kw):\n+        def finished(ignored):\n+            return next(self.measurements)\n+\n+        d = f(*a, **kw)\n+        d.addCallback(finished)\n+        return d\n+\n+\n+@implementer(IProbe)\n+class FakeProbe:\n+    \"\"\"\n+    A probe performs a single operation, which can be timed.\n+    \"\"\"\n+    def __init__(self, good):\n+        self.good = good\n+\n+    def run(self):\n+        if self.good:\n+            return succeed(None)\n+        else:\n+            return fail(RuntimeError('ProbeFailed'))\n+\n+    def cleanup(self):\n+        return succeed(None)\n+\n+\n+@implementer(IOperation)\n+class FakeOperation:\n+\n+    def __init__(self, succeeds):\n+        \"\"\"\n+        :param succeeds: An iterator providing a boolean indicating\n+            whether the probe will succeed.\n+        \"\"\"\n+        self.succeeds = succeeds\n+\n+    def get_probe(self):\n+        return FakeProbe(next(self.succeeds))\n+\n+\n+class FakeScenario:\n+\n+    def __init__(self):\n+        self.scenario_maintained = Deferred()\n+\n+    def start(self):\n+        return succeed(None)\n+\n+    def maintained(self):\n+        return self.scenario_maintained\n+\n+    def stop(self):\n+        return succeed(None)\n+\n+\n+class FakeCollapsingScenario:\n+\n+    def __init__(self):\n+        self.scenario_maintained = fail(RuntimeError('collapse'))\n+\n+    def start(self):\n+        return succeed(None)\n+\n+    def maintained(self):\n+        return self.scenario_maintained\n+\n+    def stop(self):\n+        return succeed(None)\n+\n+\n+class SampleTest(TestCase):\n+\n+    def test_good_probes(self):\n+        \"\"\"\n+        Sampling returns results when probes succeed.\n+        \"\"\"\n+        samples_ready = benchmark(\n+            FakeMetric(count(5)),\n+            FakeOperation(repeat(True)),\n+            FakeScenario(),\n+            3)\n+\n+        def check(samples):\n+            self.assertEqual(\n+                samples, [{'success': True, 'value': x} for x in [5, 6, 7]])\n+        samples_ready.addCallback(check)","path":"benchmark/test/test_driver.py","position":112,"original_position":112,"commit_id":"f105510500e8cf434c5d1aa350a9762374aa02e3","original_commit_id":"f105510500e8cf434c5d1aa350a9762374aa02e3","user":{"login":"jongiddy","id":3735327,"avatar_url":"https://avatars.githubusercontent.com/u/3735327?v=3","gravatar_id":"","url":"https://api.github.com/users/jongiddy","html_url":"https://github.com/jongiddy","followers_url":"https://api.github.com/users/jongiddy/followers","following_url":"https://api.github.com/users/jongiddy/following{/other_user}","gists_url":"https://api.github.com/users/jongiddy/gists{/gist_id}","starred_url":"https://api.github.com/users/jongiddy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jongiddy/subscriptions","organizations_url":"https://api.github.com/users/jongiddy/orgs","repos_url":"https://api.github.com/users/jongiddy/repos","events_url":"https://api.github.com/users/jongiddy/events{/privacy}","received_events_url":"https://api.github.com/users/jongiddy/received_events","type":"User","site_admin":false},"body":"I tried converting to `SynchronousTestCase`, but the `cooperate` function in `benchmark` uses the global reactor: http://twistedmatrix.com/trac/browser/tags/releases/twisted-15.4.0/twisted/internet/task.py#L337","created_at":"2015-11-17T15:06:27Z","updated_at":"2015-11-17T15:06:27Z","html_url":"https://github.com/ClusterHQ/flocker/pull/2162#discussion_r45069884","pull_request_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2162","_links":{"self":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/comments/45069884"},"html":{"href":"https://github.com/ClusterHQ/flocker/pull/2162#discussion_r45069884"},"pull_request":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2162"}}},"pull_request":{"url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2162","id":50416503,"html_url":"https://github.com/ClusterHQ/flocker/pull/2162","diff_url":"https://github.com/ClusterHQ/flocker/pull/2162.diff","patch_url":"https://github.com/ClusterHQ/flocker/pull/2162.patch","issue_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/2162","number":2162,"state":"open","locked":false,"title":"[FLOC-3288] Benchmark driver","user":{"login":"jongiddy","id":3735327,"avatar_url":"https://avatars.githubusercontent.com/u/3735327?v=3","gravatar_id":"","url":"https://api.github.com/users/jongiddy","html_url":"https://github.com/jongiddy","followers_url":"https://api.github.com/users/jongiddy/followers","following_url":"https://api.github.com/users/jongiddy/following{/other_user}","gists_url":"https://api.github.com/users/jongiddy/gists{/gist_id}","starred_url":"https://api.github.com/users/jongiddy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jongiddy/subscriptions","organizations_url":"https://api.github.com/users/jongiddy/orgs","repos_url":"https://api.github.com/users/jongiddy/repos","events_url":"https://api.github.com/users/jongiddy/events{/privacy}","received_events_url":"https://api.github.com/users/jongiddy/received_events","type":"User","site_admin":false},"body":"This PR creates a basic benchmarking command. Using an existing Flocker cluster, the command allows the user to combine a scenario (e.g. specified rate of external load on the control server), an operation to be measured (read request, write request), and a form of measurement (wallclock time, CPU time, log file size).\r\n\r\nThe driver will setup the load scenario, and then run the operation, taking measurements before and after the operation runs.  The measurement is currently sampled 3 times.  The sample results are reported to stdout as a JSON structure.\r\n\r\nThe current PR provides the basic framework and one or two examples of each parameter. The current scenario is \"no load\", the current operations are \"no-op\" and \"read-request\", and the current measurement is \"wallclock\" time.\r\n\r\nAdditional issues in FLOC-2748 will introduce more values for these parameters. Untested versions of these additional values can be seen in the spike branch https://github.com/ClusterHQ/flocker/tree/benchmark-prototype-with-scenarios.\r\n\r\nWith no Flocker cluster, run the benchmark using:\r\n```\r\npython -m benchmark.benchmark_control --operation nop\r\n```\r\n\r\nThis will display a JSON stanza like:\r\n```json\r\n{\"client\": {\"username\": \"jon\", \"flocker_version\": \"1.6.1+111.g4cb5b53\", \"working_directory\": \"/home/jon/projects/clusterhq/flocker\", \"nodename\": \"ecuador\", \"platform\": \"Linux-3.13.0-66-generic-x86_64-with-Ubuntu-14.04-trusty\", \"date\": \"2015-11-11T18:11:19.173954\"}, \"measurement\": \"wallclock\", \"metric_name\": \"nop\", \"samples\": [{\"value\": 1.0967254638671875e-05, \"success\": true}, {\"value\": 6.9141387939453125e-06, \"success\": true}, {\"value\": 7.152557373046875e-06, \"success\": true}], \"server\": {\"flocker_version\": \"1.6.1+111.g4cb5b53\"}}\r\n```\r\n\r\nNote, as well as the final JSON output to stdout, the script displays logging information to stderr.\r\n\r\nIf you have a Flocker cluster handy (and who doesn't), to get a genuine benchmark result, run:\r\n```\r\npython -m benchmark.benchmark_control --control=${FLOCKER_ACCEPTANCE_CONTROL_NODE} --certs=${FLOCKER_ACCEPTANCE_API_CERTIFICATES_PATH}\r\n```\r\nto get a genuine benchmark result.\n\n<!-- Reviewable:start -->\n[<img src=\"https://reviewable.io/review_button.png\" height=40 alt=\"Review on Reviewable\"/>](https://reviewable.io/reviews/clusterhq/flocker/2162)\n<!-- Reviewable:end -->\n","created_at":"2015-11-11T18:15:04Z","updated_at":"2015-11-17T15:06:27Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9dd664dbd3e0d36ff65e1da56ee2bbdaab07e6b8","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2162/commits","review_comments_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2162/comments","review_comment_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/2162/comments","statuses_url":"https://api.github.com/repos/ClusterHQ/flocker/statuses/f105510500e8cf434c5d1aa350a9762374aa02e3","head":{"label":"ClusterHQ:benchmark-driver-2-FLOC-3288","ref":"benchmark-driver-2-FLOC-3288","sha":"f105510500e8cf434c5d1aa350a9762374aa02e3","user":{"login":"ClusterHQ","id":7913449,"avatar_url":"https://avatars.githubusercontent.com/u/7913449?v=3","gravatar_id":"","url":"https://api.github.com/users/ClusterHQ","html_url":"https://github.com/ClusterHQ","followers_url":"https://api.github.com/users/ClusterHQ/followers","following_url":"https://api.github.com/users/ClusterHQ/following{/other_user}","gists_url":"https://api.github.com/users/ClusterHQ/gists{/gist_id}","starred_url":"https://api.github.com/users/ClusterHQ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ClusterHQ/subscriptions","organizations_url":"https://api.github.com/users/ClusterHQ/orgs","repos_url":"https://api.github.com/users/ClusterHQ/repos","events_url":"https://api.github.com/users/ClusterHQ/events{/privacy}","received_events_url":"https://api.github.com/users/ClusterHQ/received_events","type":"Organization","site_admin":false},"repo":{"id":19248542,"name":"flocker","full_name":"ClusterHQ/flocker","owner":{"login":"ClusterHQ","id":7913449,"avatar_url":"https://avatars.githubusercontent.com/u/7913449?v=3","gravatar_id":"","url":"https://api.github.com/users/ClusterHQ","html_url":"https://github.com/ClusterHQ","followers_url":"https://api.github.com/users/ClusterHQ/followers","following_url":"https://api.github.com/users/ClusterHQ/following{/other_user}","gists_url":"https://api.github.com/users/ClusterHQ/gists{/gist_id}","starred_url":"https://api.github.com/users/ClusterHQ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ClusterHQ/subscriptions","organizations_url":"https://api.github.com/users/ClusterHQ/orgs","repos_url":"https://api.github.com/users/ClusterHQ/repos","events_url":"https://api.github.com/users/ClusterHQ/events{/privacy}","received_events_url":"https://api.github.com/users/ClusterHQ/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ClusterHQ/flocker","description":"Container data volume manager for your Dockerized application","fork":false,"url":"https://api.github.com/repos/ClusterHQ/flocker","forks_url":"https://api.github.com/repos/ClusterHQ/flocker/forks","keys_url":"https://api.github.com/repos/ClusterHQ/flocker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ClusterHQ/flocker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ClusterHQ/flocker/teams","hooks_url":"https://api.github.com/repos/ClusterHQ/flocker/hooks","issue_events_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/events{/number}","events_url":"https://api.github.com/repos/ClusterHQ/flocker/events","assignees_url":"https://api.github.com/repos/ClusterHQ/flocker/assignees{/user}","branches_url":"https://api.github.com/repos/ClusterHQ/flocker/branches{/branch}","tags_url":"https://api.github.com/repos/ClusterHQ/flocker/tags","blobs_url":"https://api.github.com/repos/ClusterHQ/flocker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ClusterHQ/flocker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ClusterHQ/flocker/git/refs{/sha}","trees_url":"https://api.github.com/repos/ClusterHQ/flocker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ClusterHQ/flocker/statuses/{sha}","languages_url":"https://api.github.com/repos/ClusterHQ/flocker/languages","stargazers_url":"https://api.github.com/repos/ClusterHQ/flocker/stargazers","contributors_url":"https://api.github.com/repos/ClusterHQ/flocker/contributors","subscribers_url":"https://api.github.com/repos/ClusterHQ/flocker/subscribers","subscription_url":"https://api.github.com/repos/ClusterHQ/flocker/subscription","commits_url":"https://api.github.com/repos/ClusterHQ/flocker/commits{/sha}","git_commits_url":"https://api.github.com/repos/ClusterHQ/flocker/git/commits{/sha}","comments_url":"https://api.github.com/repos/ClusterHQ/flocker/comments{/number}","issue_comment_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/comments{/number}","contents_url":"https://api.github.com/repos/ClusterHQ/flocker/contents/{+path}","compare_url":"https://api.github.com/repos/ClusterHQ/flocker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ClusterHQ/flocker/merges","archive_url":"https://api.github.com/repos/ClusterHQ/flocker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ClusterHQ/flocker/downloads","issues_url":"https://api.github.com/repos/ClusterHQ/flocker/issues{/number}","pulls_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls{/number}","milestones_url":"https://api.github.com/repos/ClusterHQ/flocker/milestones{/number}","notifications_url":"https://api.github.com/repos/ClusterHQ/flocker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ClusterHQ/flocker/labels{/name}","releases_url":"https://api.github.com/repos/ClusterHQ/flocker/releases{/id}","created_at":"2014-04-28T18:02:14Z","updated_at":"2015-11-17T14:34:16Z","pushed_at":"2015-11-17T15:01:53Z","git_url":"git://github.com/ClusterHQ/flocker.git","ssh_url":"git@github.com:ClusterHQ/flocker.git","clone_url":"https://github.com/ClusterHQ/flocker.git","svn_url":"https://github.com/ClusterHQ/flocker","homepage":"https://clusterhq.com","size":44419,"stargazers_count":1952,"watchers_count":1952,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":142,"mirror_url":null,"open_issues_count":33,"forks":142,"open_issues":33,"watchers":1952,"default_branch":"master"}},"base":{"label":"ClusterHQ:master","ref":"master","sha":"83de9ee0884140d4f66a603e1cc0b8f2962d05c7","user":{"login":"ClusterHQ","id":7913449,"avatar_url":"https://avatars.githubusercontent.com/u/7913449?v=3","gravatar_id":"","url":"https://api.github.com/users/ClusterHQ","html_url":"https://github.com/ClusterHQ","followers_url":"https://api.github.com/users/ClusterHQ/followers","following_url":"https://api.github.com/users/ClusterHQ/following{/other_user}","gists_url":"https://api.github.com/users/ClusterHQ/gists{/gist_id}","starred_url":"https://api.github.com/users/ClusterHQ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ClusterHQ/subscriptions","organizations_url":"https://api.github.com/users/ClusterHQ/orgs","repos_url":"https://api.github.com/users/ClusterHQ/repos","events_url":"https://api.github.com/users/ClusterHQ/events{/privacy}","received_events_url":"https://api.github.com/users/ClusterHQ/received_events","type":"Organization","site_admin":false},"repo":{"id":19248542,"name":"flocker","full_name":"ClusterHQ/flocker","owner":{"login":"ClusterHQ","id":7913449,"avatar_url":"https://avatars.githubusercontent.com/u/7913449?v=3","gravatar_id":"","url":"https://api.github.com/users/ClusterHQ","html_url":"https://github.com/ClusterHQ","followers_url":"https://api.github.com/users/ClusterHQ/followers","following_url":"https://api.github.com/users/ClusterHQ/following{/other_user}","gists_url":"https://api.github.com/users/ClusterHQ/gists{/gist_id}","starred_url":"https://api.github.com/users/ClusterHQ/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ClusterHQ/subscriptions","organizations_url":"https://api.github.com/users/ClusterHQ/orgs","repos_url":"https://api.github.com/users/ClusterHQ/repos","events_url":"https://api.github.com/users/ClusterHQ/events{/privacy}","received_events_url":"https://api.github.com/users/ClusterHQ/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ClusterHQ/flocker","description":"Container data volume manager for your Dockerized application","fork":false,"url":"https://api.github.com/repos/ClusterHQ/flocker","forks_url":"https://api.github.com/repos/ClusterHQ/flocker/forks","keys_url":"https://api.github.com/repos/ClusterHQ/flocker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ClusterHQ/flocker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ClusterHQ/flocker/teams","hooks_url":"https://api.github.com/repos/ClusterHQ/flocker/hooks","issue_events_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/events{/number}","events_url":"https://api.github.com/repos/ClusterHQ/flocker/events","assignees_url":"https://api.github.com/repos/ClusterHQ/flocker/assignees{/user}","branches_url":"https://api.github.com/repos/ClusterHQ/flocker/branches{/branch}","tags_url":"https://api.github.com/repos/ClusterHQ/flocker/tags","blobs_url":"https://api.github.com/repos/ClusterHQ/flocker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ClusterHQ/flocker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ClusterHQ/flocker/git/refs{/sha}","trees_url":"https://api.github.com/repos/ClusterHQ/flocker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ClusterHQ/flocker/statuses/{sha}","languages_url":"https://api.github.com/repos/ClusterHQ/flocker/languages","stargazers_url":"https://api.github.com/repos/ClusterHQ/flocker/stargazers","contributors_url":"https://api.github.com/repos/ClusterHQ/flocker/contributors","subscribers_url":"https://api.github.com/repos/ClusterHQ/flocker/subscribers","subscription_url":"https://api.github.com/repos/ClusterHQ/flocker/subscription","commits_url":"https://api.github.com/repos/ClusterHQ/flocker/commits{/sha}","git_commits_url":"https://api.github.com/repos/ClusterHQ/flocker/git/commits{/sha}","comments_url":"https://api.github.com/repos/ClusterHQ/flocker/comments{/number}","issue_comment_url":"https://api.github.com/repos/ClusterHQ/flocker/issues/comments{/number}","contents_url":"https://api.github.com/repos/ClusterHQ/flocker/contents/{+path}","compare_url":"https://api.github.com/repos/ClusterHQ/flocker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ClusterHQ/flocker/merges","archive_url":"https://api.github.com/repos/ClusterHQ/flocker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ClusterHQ/flocker/downloads","issues_url":"https://api.github.com/repos/ClusterHQ/flocker/issues{/number}","pulls_url":"https://api.github.com/repos/ClusterHQ/flocker/pulls{/number}","milestones_url":"https://api.github.com/repos/ClusterHQ/flocker/milestones{/number}","notifications_url":"https://api.github.com/repos/ClusterHQ/flocker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ClusterHQ/flocker/labels{/name}","releases_url":"https://api.github.com/repos/ClusterHQ/flocker/releases{/id}","created_at":"2014-04-28T18:02:14Z","updated_at":"2015-11-17T14:34:16Z","pushed_at":"2015-11-17T15:01:53Z","git_url":"git://github.com/ClusterHQ/flocker.git","ssh_url":"git@github.com:ClusterHQ/flocker.git","clone_url":"https://github.com/ClusterHQ/flocker.git","svn_url":"https://github.com/ClusterHQ/flocker","homepage":"https://clusterhq.com","size":44419,"stargazers_count":1952,"watchers_count":1952,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":142,"mirror_url":null,"open_issues_count":33,"forks":142,"open_issues":33,"watchers":1952,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2162"},"html":{"href":"https://github.com/ClusterHQ/flocker/pull/2162"},"issue":{"href":"https://api.github.com/repos/ClusterHQ/flocker/issues/2162"},"comments":{"href":"https://api.github.com/repos/ClusterHQ/flocker/issues/2162/comments"},"review_comments":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2162/comments"},"review_comment":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ClusterHQ/flocker/pulls/2162/commits"},"statuses":{"href":"https://api.github.com/repos/ClusterHQ/flocker/statuses/f105510500e8cf434c5d1aa350a9762374aa02e3"}}}}