{"action":"created","comment":{"url":"https://api.github.com/repos/ContinuumIO/dask/pulls/comments/34622377","id":34622377,"diff_hunk":"@@ -0,0 +1,281 @@\n+import operator\n+from functools import partial, wraps\n+from itertools import chain\n+from collections import Iterator\n+\n+from toolz import merge, unique\n+\n+from .core import istask\n+from .optimize import cull, fuse\n+from .context import _globals\n+from .compatibility import apply\n+from . import threaded\n+\n+__all__ = ['do', 'value', 'Value']\n+\n+\n+def flat_unique(ls):\n+    \"\"\"Flatten ``ls``, filter by unique id, and return a list\"\"\"\n+    return list(unique(chain.from_iterable(ls), key=id))\n+\n+\n+def unzip(ls, nout):\n+    \"\"\"Unzip a list of lists into ``nout`` outputs.\"\"\"\n+    out = list(zip(*ls))\n+    if not out:\n+        out = [()] * nout\n+    return out\n+\n+\n+def to_task_dasks(task):\n+    \"\"\"Normalize a task and extract all sub-dasks.\n+\n+    - Replace ``Values`` with their keys\n+    - Convert literals to things the schedulers can handle\n+    - Extract dasks from all enclosed values\n+","path":"dask/compose.py","position":36,"original_position":36,"commit_id":"8bbc3322f8faa52f603b1fd1622c30c736266391","original_commit_id":"8bbc3322f8faa52f603b1fd1622c30c736266391","user":{"login":"mrocklin","id":306380,"avatar_url":"https://avatars.githubusercontent.com/u/306380?v=3","gravatar_id":"","url":"https://api.github.com/users/mrocklin","html_url":"https://github.com/mrocklin","followers_url":"https://api.github.com/users/mrocklin/followers","following_url":"https://api.github.com/users/mrocklin/following{/other_user}","gists_url":"https://api.github.com/users/mrocklin/gists{/gist_id}","starred_url":"https://api.github.com/users/mrocklin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrocklin/subscriptions","organizations_url":"https://api.github.com/users/mrocklin/orgs","repos_url":"https://api.github.com/users/mrocklin/repos","events_url":"https://api.github.com/users/mrocklin/events{/privacy}","received_events_url":"https://api.github.com/users/mrocklin/received_events","type":"User","site_admin":false},"body":"This function could use some doctests to show how it works.  I think that it becomes very clear once you see\r\n\r\n```\r\n>>> to_task_dasks((1, 2, 3))\r\n(tuple, [1, 2, 3])\r\n```","created_at":"2015-07-14T21:32:37Z","updated_at":"2015-07-14T21:32:37Z","html_url":"https://github.com/ContinuumIO/dask/pull/408#discussion_r34622377","pull_request_url":"https://api.github.com/repos/ContinuumIO/dask/pulls/408","_links":{"self":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/comments/34622377"},"html":{"href":"https://github.com/ContinuumIO/dask/pull/408#discussion_r34622377"},"pull_request":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/408"}}},"pull_request":{"url":"https://api.github.com/repos/ContinuumIO/dask/pulls/408","id":39640323,"html_url":"https://github.com/ContinuumIO/dask/pull/408","diff_url":"https://github.com/ContinuumIO/dask/pull/408.diff","patch_url":"https://github.com/ContinuumIO/dask/pull/408.patch","issue_url":"https://api.github.com/repos/ContinuumIO/dask/issues/408","number":408,"state":"open","locked":false,"title":"Imperative dask composition","user":{"login":"jcrist","id":2783717,"avatar_url":"https://avatars.githubusercontent.com/u/2783717?v=3","gravatar_id":"","url":"https://api.github.com/users/jcrist","html_url":"https://github.com/jcrist","followers_url":"https://api.github.com/users/jcrist/followers","following_url":"https://api.github.com/users/jcrist/following{/other_user}","gists_url":"https://api.github.com/users/jcrist/gists{/gist_id}","starred_url":"https://api.github.com/users/jcrist/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jcrist/subscriptions","organizations_url":"https://api.github.com/users/jcrist/orgs","repos_url":"https://api.github.com/users/jcrist/repos","events_url":"https://api.github.com/users/jcrist/events{/privacy}","received_events_url":"https://api.github.com/users/jcrist/received_events","type":"User","site_admin":false},"body":"My take on imperative graph composition. Values support operators, and \"bubble-up\" through computation. Similar to #404.\r\n\r\n```python\r\n>>> from dask.compose import value, daskify\r\n\r\n>>> a = value(1)\r\n>>> b = a + 1\r\n>>> c = a * b * 3\r\n>>> c.compute()\r\n6\r\n```\r\n\r\nOne can also wrap functions in a `daskify` decorator, which then returns a `Value` object.\r\n\r\n```python\r\n>>> @daskify\r\n>>> def foo(a, b, c):\r\n...    return a + b + c\r\n\r\n>>> @daskify\r\n>>> def bar(a, b):\r\n...    return a[b] + 4\r\n\r\n>>> dlist = daskify(list)\r\n\r\n>>> a = dlist([1, 2, 3])\r\n>>> b = bar(a, 0)\r\n>>> c = value(3)\r\n>>> d = foo(b, c, 10)\r\n>>> d.compute()\r\n18\r\n```","created_at":"2015-07-09T23:47:58Z","updated_at":"2015-07-14T21:32:37Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1e7b1e88abe5fabfc114ab325cb7cbaf5dce9442","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/ContinuumIO/dask/pulls/408/commits","review_comments_url":"https://api.github.com/repos/ContinuumIO/dask/pulls/408/comments","review_comment_url":"https://api.github.com/repos/ContinuumIO/dask/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ContinuumIO/dask/issues/408/comments","statuses_url":"https://api.github.com/repos/ContinuumIO/dask/statuses/8bbc3322f8faa52f603b1fd1622c30c736266391","head":{"label":"jcrist:compose","ref":"compose","sha":"8bbc3322f8faa52f603b1fd1622c30c736266391","user":{"login":"jcrist","id":2783717,"avatar_url":"https://avatars.githubusercontent.com/u/2783717?v=3","gravatar_id":"","url":"https://api.github.com/users/jcrist","html_url":"https://github.com/jcrist","followers_url":"https://api.github.com/users/jcrist/followers","following_url":"https://api.github.com/users/jcrist/following{/other_user}","gists_url":"https://api.github.com/users/jcrist/gists{/gist_id}","starred_url":"https://api.github.com/users/jcrist/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jcrist/subscriptions","organizations_url":"https://api.github.com/users/jcrist/orgs","repos_url":"https://api.github.com/users/jcrist/repos","events_url":"https://api.github.com/users/jcrist/events{/privacy}","received_events_url":"https://api.github.com/users/jcrist/received_events","type":"User","site_admin":false},"repo":{"id":32761805,"name":"dask","full_name":"jcrist/dask","owner":{"login":"jcrist","id":2783717,"avatar_url":"https://avatars.githubusercontent.com/u/2783717?v=3","gravatar_id":"","url":"https://api.github.com/users/jcrist","html_url":"https://github.com/jcrist","followers_url":"https://api.github.com/users/jcrist/followers","following_url":"https://api.github.com/users/jcrist/following{/other_user}","gists_url":"https://api.github.com/users/jcrist/gists{/gist_id}","starred_url":"https://api.github.com/users/jcrist/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jcrist/subscriptions","organizations_url":"https://api.github.com/users/jcrist/orgs","repos_url":"https://api.github.com/users/jcrist/repos","events_url":"https://api.github.com/users/jcrist/events{/privacy}","received_events_url":"https://api.github.com/users/jcrist/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jcrist/dask","description":"Minimal task scheduling abstraction","fork":true,"url":"https://api.github.com/repos/jcrist/dask","forks_url":"https://api.github.com/repos/jcrist/dask/forks","keys_url":"https://api.github.com/repos/jcrist/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jcrist/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jcrist/dask/teams","hooks_url":"https://api.github.com/repos/jcrist/dask/hooks","issue_events_url":"https://api.github.com/repos/jcrist/dask/issues/events{/number}","events_url":"https://api.github.com/repos/jcrist/dask/events","assignees_url":"https://api.github.com/repos/jcrist/dask/assignees{/user}","branches_url":"https://api.github.com/repos/jcrist/dask/branches{/branch}","tags_url":"https://api.github.com/repos/jcrist/dask/tags","blobs_url":"https://api.github.com/repos/jcrist/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jcrist/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jcrist/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/jcrist/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jcrist/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/jcrist/dask/languages","stargazers_url":"https://api.github.com/repos/jcrist/dask/stargazers","contributors_url":"https://api.github.com/repos/jcrist/dask/contributors","subscribers_url":"https://api.github.com/repos/jcrist/dask/subscribers","subscription_url":"https://api.github.com/repos/jcrist/dask/subscription","commits_url":"https://api.github.com/repos/jcrist/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/jcrist/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/jcrist/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/jcrist/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/jcrist/dask/contents/{+path}","compare_url":"https://api.github.com/repos/jcrist/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jcrist/dask/merges","archive_url":"https://api.github.com/repos/jcrist/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jcrist/dask/downloads","issues_url":"https://api.github.com/repos/jcrist/dask/issues{/number}","pulls_url":"https://api.github.com/repos/jcrist/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/jcrist/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/jcrist/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jcrist/dask/labels{/name}","releases_url":"https://api.github.com/repos/jcrist/dask/releases{/id}","created_at":"2015-03-23T22:11:35Z","updated_at":"2015-03-28T16:10:40Z","pushed_at":"2015-07-14T21:19:48Z","git_url":"git://github.com/jcrist/dask.git","ssh_url":"git@github.com:jcrist/dask.git","clone_url":"https://github.com/jcrist/dask.git","svn_url":"https://github.com/jcrist/dask","homepage":"http://dask.pydata.org","size":7119,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"ContinuumIO:master","ref":"master","sha":"aaf9d1762c87daeec434954806c656415cb08b80","user":{"login":"ContinuumIO","id":1158637,"avatar_url":"https://avatars.githubusercontent.com/u/1158637?v=3","gravatar_id":"","url":"https://api.github.com/users/ContinuumIO","html_url":"https://github.com/ContinuumIO","followers_url":"https://api.github.com/users/ContinuumIO/followers","following_url":"https://api.github.com/users/ContinuumIO/following{/other_user}","gists_url":"https://api.github.com/users/ContinuumIO/gists{/gist_id}","starred_url":"https://api.github.com/users/ContinuumIO/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ContinuumIO/subscriptions","organizations_url":"https://api.github.com/users/ContinuumIO/orgs","repos_url":"https://api.github.com/users/ContinuumIO/repos","events_url":"https://api.github.com/users/ContinuumIO/events{/privacy}","received_events_url":"https://api.github.com/users/ContinuumIO/received_events","type":"Organization","site_admin":false},"repo":{"id":28782747,"name":"dask","full_name":"ContinuumIO/dask","owner":{"login":"ContinuumIO","id":1158637,"avatar_url":"https://avatars.githubusercontent.com/u/1158637?v=3","gravatar_id":"","url":"https://api.github.com/users/ContinuumIO","html_url":"https://github.com/ContinuumIO","followers_url":"https://api.github.com/users/ContinuumIO/followers","following_url":"https://api.github.com/users/ContinuumIO/following{/other_user}","gists_url":"https://api.github.com/users/ContinuumIO/gists{/gist_id}","starred_url":"https://api.github.com/users/ContinuumIO/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ContinuumIO/subscriptions","organizations_url":"https://api.github.com/users/ContinuumIO/orgs","repos_url":"https://api.github.com/users/ContinuumIO/repos","events_url":"https://api.github.com/users/ContinuumIO/events{/privacy}","received_events_url":"https://api.github.com/users/ContinuumIO/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ContinuumIO/dask","description":"Task scheduling and blocked algorithms for parallel processing","fork":false,"url":"https://api.github.com/repos/ContinuumIO/dask","forks_url":"https://api.github.com/repos/ContinuumIO/dask/forks","keys_url":"https://api.github.com/repos/ContinuumIO/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ContinuumIO/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ContinuumIO/dask/teams","hooks_url":"https://api.github.com/repos/ContinuumIO/dask/hooks","issue_events_url":"https://api.github.com/repos/ContinuumIO/dask/issues/events{/number}","events_url":"https://api.github.com/repos/ContinuumIO/dask/events","assignees_url":"https://api.github.com/repos/ContinuumIO/dask/assignees{/user}","branches_url":"https://api.github.com/repos/ContinuumIO/dask/branches{/branch}","tags_url":"https://api.github.com/repos/ContinuumIO/dask/tags","blobs_url":"https://api.github.com/repos/ContinuumIO/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ContinuumIO/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ContinuumIO/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/ContinuumIO/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ContinuumIO/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/ContinuumIO/dask/languages","stargazers_url":"https://api.github.com/repos/ContinuumIO/dask/stargazers","contributors_url":"https://api.github.com/repos/ContinuumIO/dask/contributors","subscribers_url":"https://api.github.com/repos/ContinuumIO/dask/subscribers","subscription_url":"https://api.github.com/repos/ContinuumIO/dask/subscription","commits_url":"https://api.github.com/repos/ContinuumIO/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/ContinuumIO/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/ContinuumIO/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/ContinuumIO/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/ContinuumIO/dask/contents/{+path}","compare_url":"https://api.github.com/repos/ContinuumIO/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ContinuumIO/dask/merges","archive_url":"https://api.github.com/repos/ContinuumIO/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ContinuumIO/dask/downloads","issues_url":"https://api.github.com/repos/ContinuumIO/dask/issues{/number}","pulls_url":"https://api.github.com/repos/ContinuumIO/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/ContinuumIO/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/ContinuumIO/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ContinuumIO/dask/labels{/name}","releases_url":"https://api.github.com/repos/ContinuumIO/dask/releases{/id}","created_at":"2015-01-04T18:50:00Z","updated_at":"2015-07-14T14:56:03Z","pushed_at":"2015-07-14T21:27:30Z","git_url":"git://github.com/ContinuumIO/dask.git","ssh_url":"git@github.com:ContinuumIO/dask.git","clone_url":"https://github.com/ContinuumIO/dask.git","svn_url":"https://github.com/ContinuumIO/dask","homepage":"http://dask.pydata.org","size":10154,"stargazers_count":210,"watchers_count":210,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":35,"mirror_url":null,"open_issues_count":76,"forks":35,"open_issues":76,"watchers":210,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/408"},"html":{"href":"https://github.com/ContinuumIO/dask/pull/408"},"issue":{"href":"https://api.github.com/repos/ContinuumIO/dask/issues/408"},"comments":{"href":"https://api.github.com/repos/ContinuumIO/dask/issues/408/comments"},"review_comments":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/408/comments"},"review_comment":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/408/commits"},"statuses":{"href":"https://api.github.com/repos/ContinuumIO/dask/statuses/8bbc3322f8faa52f603b1fd1622c30c736266391"}}}}