{"action":"created","comment":{"url":"https://api.github.com/repos/spotify/luigi/pulls/comments/25154096","id":25154096,"diff_hunk":"@@ -455,7 +458,78 @@ def test_start_long_before_long_hours_back_and_with_long_hours_forward(self):\n         )\n \n \n+class FilesystemInferenceTest(unittest.TestCase):\n+\n+    def _test_filesystems_and_globs(self, datetime_to_task, datetime_to_re, expected):\n+        actual = list(_get_filesystems_and_globs(datetime_to_task, datetime_to_re))\n+        self.assertEqual(len(actual), len(expected))\n+        for (actual_filesystem, actual_glob), (expected_filesystem, expected_glob) in zip(actual, expected):\n+            self.assertTrue(isinstance(actual_filesystem, expected_filesystem))\n+            self.assertEqual(actual_glob, expected_glob)\n+\n+    def test_date_glob_successfully_inferred(self):\n+        self._test_filesystems_and_globs(\n+            lambda d: CommonDateTask(d),\n+            lambda d: '(%04d).*(%02d).*(%02d)' % (d.year, d.month, d.day),\n+            [\n+                (MockFileSystem, '/n2000y01a05n/[0-9][0-9][0-9][0-9]_[0-9][0-9]-_-[0-9][0-9]aww/21mm01dara21'),\n+            ]\n+        )\n+\n+    def test_datehour_glob_successfully_inferred(self):\n+        self._test_filesystems_and_globs(\n+            lambda d: CommonDateHourTask(d),\n+            lambda d: '(%04d).*(%02d).*(%02d).*(%02d)' % (d.year, d.month, d.day, d.hour),\n+            [\n+                (MockFileSystem, '/n2000y01a05n/[0-9][0-9][0-9][0-9]_[0-9][0-9]-_-[0-9][0-9]aww/21mm[0-9][0-9]dara21'),\n+            ]\n+        )\n+\n+    def test_wrapped_datehour_globs_successfully_inferred(self):\n+        self._test_filesystems_and_globs(\n+            lambda d: CommonWrapperTask(d),\n+            lambda d: '(%04d).*(%02d).*(%02d).*(%02d)' % (d.year, d.month, d.day, d.hour),\n+            [\n+                (MockFileSystem, 'TaskA/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]'),\n+                (MockFileSystem, 'TaskB/no/worries[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]'),\n+            ]\n+        )\n+\n+    def test_inconsistent_output_datehour_glob_not_inferred(self):\n+        class InconsistentlyOutputtingDateHourTask(luigi.Task):\n+            dh = luigi.DateHourParameter()\n+\n+            def output(self):\n+                base = self.dh.strftime('/even/%Y%m%d%H')\n+                if self.dh.hour % 2 == 0:\n+                    return MockFile(base)\n+                else:\n+                    return {\n+                        'spi': MockFile(base + '/something.spi'),\n+                        'spl': MockFile(base + '/something.spl'),\n+                    }\n+\n+        with self.assertRaises(NotImplementedError):\n+            list(_get_filesystems_and_globs(\n+                lambda d: InconsistentlyOutputtingDateHourTask(d),\n+                lambda d: '(%04d).*(%02d).*(%02d).*(%02d)' % (d.year, d.month, d.day, d.hour)))\n+\n+    def test_wrapped_inconsistent_datehour_globs_not_inferred(self):\n+        class InconsistentlyParameterizedWrapperTask(luigi.WrapperTask):\n+            dh = luigi.DateHourParameter()\n+\n+            def requires(self):\n+                yield TaskA(dh=self.dh - datetime.timedelta(days=1))\n+                yield TaskB(dh=self.dh, complicator='no/worries')\n+\n+        with self.assertRaises(NotImplementedError):","path":"test/range_test.py","position":84,"original_position":84,"commit_id":"1aea123433f2dbb80df2ced9a064d4e5d817460b","original_commit_id":"1aea123433f2dbb80df2ced9a064d4e5d817460b","user":{"login":"Tarrasch","id":294349,"avatar_url":"https://avatars.githubusercontent.com/u/294349?v=3","gravatar_id":"","url":"https://api.github.com/users/Tarrasch","html_url":"https://github.com/Tarrasch","followers_url":"https://api.github.com/users/Tarrasch/followers","following_url":"https://api.github.com/users/Tarrasch/following{/other_user}","gists_url":"https://api.github.com/users/Tarrasch/gists{/gist_id}","starred_url":"https://api.github.com/users/Tarrasch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Tarrasch/subscriptions","organizations_url":"https://api.github.com/users/Tarrasch/orgs","repos_url":"https://api.github.com/users/Tarrasch/repos","events_url":"https://api.github.com/users/Tarrasch/events{/privacy}","received_events_url":"https://api.github.com/users/Tarrasch/received_events","type":"User","site_admin":false},"body":"Ah, didn't know this could be a context manager :)","created_at":"2015-02-23T10:25:28Z","updated_at":"2015-02-23T10:25:28Z","html_url":"https://github.com/spotify/luigi/pull/771#discussion_r25154096","pull_request_url":"https://api.github.com/repos/spotify/luigi/pulls/771","_links":{"self":{"href":"https://api.github.com/repos/spotify/luigi/pulls/comments/25154096"},"html":{"href":"https://github.com/spotify/luigi/pull/771#discussion_r25154096"},"pull_request":{"href":"https://api.github.com/repos/spotify/luigi/pulls/771"}}},"pull_request":{"url":"https://api.github.com/repos/spotify/luigi/pulls/771","id":29796565,"html_url":"https://github.com/spotify/luigi/pull/771","diff_url":"https://github.com/spotify/luigi/pull/771.diff","patch_url":"https://github.com/spotify/luigi/pull/771.patch","issue_url":"https://api.github.com/repos/spotify/luigi/issues/771","number":771,"state":"open","locked":false,"title":"make filesystem inference work for RangeDaily","user":{"login":"ulzha","id":84229,"avatar_url":"https://avatars.githubusercontent.com/u/84229?v=3","gravatar_id":"","url":"https://api.github.com/users/ulzha","html_url":"https://github.com/ulzha","followers_url":"https://api.github.com/users/ulzha/followers","following_url":"https://api.github.com/users/ulzha/following{/other_user}","gists_url":"https://api.github.com/users/ulzha/gists{/gist_id}","starred_url":"https://api.github.com/users/ulzha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ulzha/subscriptions","organizations_url":"https://api.github.com/users/ulzha/orgs","repos_url":"https://api.github.com/users/ulzha/repos","events_url":"https://api.github.com/users/ulzha/events{/privacy}","received_events_url":"https://api.github.com/users/ulzha/received_events","type":"User","site_admin":false},"body":"Brings RangeDaily feature set on par with RangeHourly, by\r\ngeneralizing infer_bulk_complete_from_fs implementation.","created_at":"2015-02-23T10:00:06Z","updated_at":"2015-02-23T10:25:28Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6f0198060c38921c21c0d83e173210c3b069248f","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/spotify/luigi/pulls/771/commits","review_comments_url":"https://api.github.com/repos/spotify/luigi/pulls/771/comments","review_comment_url":"https://api.github.com/repos/spotify/luigi/pulls/comments{/number}","comments_url":"https://api.github.com/repos/spotify/luigi/issues/771/comments","statuses_url":"https://api.github.com/repos/spotify/luigi/statuses/1aea123433f2dbb80df2ced9a064d4e5d817460b","head":{"label":"spotify:ulzha/range","ref":"ulzha/range","sha":"1aea123433f2dbb80df2ced9a064d4e5d817460b","user":{"login":"spotify","id":251374,"avatar_url":"https://avatars.githubusercontent.com/u/251374?v=3","gravatar_id":"","url":"https://api.github.com/users/spotify","html_url":"https://github.com/spotify","followers_url":"https://api.github.com/users/spotify/followers","following_url":"https://api.github.com/users/spotify/following{/other_user}","gists_url":"https://api.github.com/users/spotify/gists{/gist_id}","starred_url":"https://api.github.com/users/spotify/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spotify/subscriptions","organizations_url":"https://api.github.com/users/spotify/orgs","repos_url":"https://api.github.com/users/spotify/repos","events_url":"https://api.github.com/users/spotify/events{/privacy}","received_events_url":"https://api.github.com/users/spotify/received_events","type":"Organization","site_admin":false},"repo":{"id":5888353,"name":"luigi","full_name":"spotify/luigi","owner":{"login":"spotify","id":251374,"avatar_url":"https://avatars.githubusercontent.com/u/251374?v=3","gravatar_id":"","url":"https://api.github.com/users/spotify","html_url":"https://github.com/spotify","followers_url":"https://api.github.com/users/spotify/followers","following_url":"https://api.github.com/users/spotify/following{/other_user}","gists_url":"https://api.github.com/users/spotify/gists{/gist_id}","starred_url":"https://api.github.com/users/spotify/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spotify/subscriptions","organizations_url":"https://api.github.com/users/spotify/orgs","repos_url":"https://api.github.com/users/spotify/repos","events_url":"https://api.github.com/users/spotify/events{/privacy}","received_events_url":"https://api.github.com/users/spotify/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/spotify/luigi","description":"Luigi is a Python module that helps you build complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization etc. It also comes with Hadoop support built in. ","fork":false,"url":"https://api.github.com/repos/spotify/luigi","forks_url":"https://api.github.com/repos/spotify/luigi/forks","keys_url":"https://api.github.com/repos/spotify/luigi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/spotify/luigi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/spotify/luigi/teams","hooks_url":"https://api.github.com/repos/spotify/luigi/hooks","issue_events_url":"https://api.github.com/repos/spotify/luigi/issues/events{/number}","events_url":"https://api.github.com/repos/spotify/luigi/events","assignees_url":"https://api.github.com/repos/spotify/luigi/assignees{/user}","branches_url":"https://api.github.com/repos/spotify/luigi/branches{/branch}","tags_url":"https://api.github.com/repos/spotify/luigi/tags","blobs_url":"https://api.github.com/repos/spotify/luigi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/spotify/luigi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/spotify/luigi/git/refs{/sha}","trees_url":"https://api.github.com/repos/spotify/luigi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/spotify/luigi/statuses/{sha}","languages_url":"https://api.github.com/repos/spotify/luigi/languages","stargazers_url":"https://api.github.com/repos/spotify/luigi/stargazers","contributors_url":"https://api.github.com/repos/spotify/luigi/contributors","subscribers_url":"https://api.github.com/repos/spotify/luigi/subscribers","subscription_url":"https://api.github.com/repos/spotify/luigi/subscription","commits_url":"https://api.github.com/repos/spotify/luigi/commits{/sha}","git_commits_url":"https://api.github.com/repos/spotify/luigi/git/commits{/sha}","comments_url":"https://api.github.com/repos/spotify/luigi/comments{/number}","issue_comment_url":"https://api.github.com/repos/spotify/luigi/issues/comments{/number}","contents_url":"https://api.github.com/repos/spotify/luigi/contents/{+path}","compare_url":"https://api.github.com/repos/spotify/luigi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/spotify/luigi/merges","archive_url":"https://api.github.com/repos/spotify/luigi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/spotify/luigi/downloads","issues_url":"https://api.github.com/repos/spotify/luigi/issues{/number}","pulls_url":"https://api.github.com/repos/spotify/luigi/pulls{/number}","milestones_url":"https://api.github.com/repos/spotify/luigi/milestones{/number}","notifications_url":"https://api.github.com/repos/spotify/luigi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/spotify/luigi/labels{/name}","releases_url":"https://api.github.com/repos/spotify/luigi/releases{/id}","created_at":"2012-09-20T15:06:38Z","updated_at":"2015-02-22T21:14:52Z","pushed_at":"2015-02-23T10:02:24Z","git_url":"git://github.com/spotify/luigi.git","ssh_url":"git@github.com:spotify/luigi.git","clone_url":"https://github.com/spotify/luigi.git","svn_url":"https://github.com/spotify/luigi","homepage":null,"size":11428,"stargazers_count":2218,"watchers_count":2218,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":371,"mirror_url":null,"open_issues_count":81,"forks":371,"open_issues":81,"watchers":2218,"default_branch":"master"}},"base":{"label":"spotify:master","ref":"master","sha":"f004c3527951fa53b95e504139a32d562f6efd6f","user":{"login":"spotify","id":251374,"avatar_url":"https://avatars.githubusercontent.com/u/251374?v=3","gravatar_id":"","url":"https://api.github.com/users/spotify","html_url":"https://github.com/spotify","followers_url":"https://api.github.com/users/spotify/followers","following_url":"https://api.github.com/users/spotify/following{/other_user}","gists_url":"https://api.github.com/users/spotify/gists{/gist_id}","starred_url":"https://api.github.com/users/spotify/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spotify/subscriptions","organizations_url":"https://api.github.com/users/spotify/orgs","repos_url":"https://api.github.com/users/spotify/repos","events_url":"https://api.github.com/users/spotify/events{/privacy}","received_events_url":"https://api.github.com/users/spotify/received_events","type":"Organization","site_admin":false},"repo":{"id":5888353,"name":"luigi","full_name":"spotify/luigi","owner":{"login":"spotify","id":251374,"avatar_url":"https://avatars.githubusercontent.com/u/251374?v=3","gravatar_id":"","url":"https://api.github.com/users/spotify","html_url":"https://github.com/spotify","followers_url":"https://api.github.com/users/spotify/followers","following_url":"https://api.github.com/users/spotify/following{/other_user}","gists_url":"https://api.github.com/users/spotify/gists{/gist_id}","starred_url":"https://api.github.com/users/spotify/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spotify/subscriptions","organizations_url":"https://api.github.com/users/spotify/orgs","repos_url":"https://api.github.com/users/spotify/repos","events_url":"https://api.github.com/users/spotify/events{/privacy}","received_events_url":"https://api.github.com/users/spotify/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/spotify/luigi","description":"Luigi is a Python module that helps you build complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization etc. It also comes with Hadoop support built in. ","fork":false,"url":"https://api.github.com/repos/spotify/luigi","forks_url":"https://api.github.com/repos/spotify/luigi/forks","keys_url":"https://api.github.com/repos/spotify/luigi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/spotify/luigi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/spotify/luigi/teams","hooks_url":"https://api.github.com/repos/spotify/luigi/hooks","issue_events_url":"https://api.github.com/repos/spotify/luigi/issues/events{/number}","events_url":"https://api.github.com/repos/spotify/luigi/events","assignees_url":"https://api.github.com/repos/spotify/luigi/assignees{/user}","branches_url":"https://api.github.com/repos/spotify/luigi/branches{/branch}","tags_url":"https://api.github.com/repos/spotify/luigi/tags","blobs_url":"https://api.github.com/repos/spotify/luigi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/spotify/luigi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/spotify/luigi/git/refs{/sha}","trees_url":"https://api.github.com/repos/spotify/luigi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/spotify/luigi/statuses/{sha}","languages_url":"https://api.github.com/repos/spotify/luigi/languages","stargazers_url":"https://api.github.com/repos/spotify/luigi/stargazers","contributors_url":"https://api.github.com/repos/spotify/luigi/contributors","subscribers_url":"https://api.github.com/repos/spotify/luigi/subscribers","subscription_url":"https://api.github.com/repos/spotify/luigi/subscription","commits_url":"https://api.github.com/repos/spotify/luigi/commits{/sha}","git_commits_url":"https://api.github.com/repos/spotify/luigi/git/commits{/sha}","comments_url":"https://api.github.com/repos/spotify/luigi/comments{/number}","issue_comment_url":"https://api.github.com/repos/spotify/luigi/issues/comments{/number}","contents_url":"https://api.github.com/repos/spotify/luigi/contents/{+path}","compare_url":"https://api.github.com/repos/spotify/luigi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/spotify/luigi/merges","archive_url":"https://api.github.com/repos/spotify/luigi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/spotify/luigi/downloads","issues_url":"https://api.github.com/repos/spotify/luigi/issues{/number}","pulls_url":"https://api.github.com/repos/spotify/luigi/pulls{/number}","milestones_url":"https://api.github.com/repos/spotify/luigi/milestones{/number}","notifications_url":"https://api.github.com/repos/spotify/luigi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/spotify/luigi/labels{/name}","releases_url":"https://api.github.com/repos/spotify/luigi/releases{/id}","created_at":"2012-09-20T15:06:38Z","updated_at":"2015-02-22T21:14:52Z","pushed_at":"2015-02-23T10:02:24Z","git_url":"git://github.com/spotify/luigi.git","ssh_url":"git@github.com:spotify/luigi.git","clone_url":"https://github.com/spotify/luigi.git","svn_url":"https://github.com/spotify/luigi","homepage":null,"size":11428,"stargazers_count":2218,"watchers_count":2218,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":371,"mirror_url":null,"open_issues_count":81,"forks":371,"open_issues":81,"watchers":2218,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/spotify/luigi/pulls/771"},"html":{"href":"https://github.com/spotify/luigi/pull/771"},"issue":{"href":"https://api.github.com/repos/spotify/luigi/issues/771"},"comments":{"href":"https://api.github.com/repos/spotify/luigi/issues/771/comments"},"review_comments":{"href":"https://api.github.com/repos/spotify/luigi/pulls/771/comments"},"review_comment":{"href":"https://api.github.com/repos/spotify/luigi/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/spotify/luigi/pulls/771/commits"},"statuses":{"href":"https://api.github.com/repos/spotify/luigi/statuses/1aea123433f2dbb80df2ced9a064d4e5d817460b"}}}}