{"action":"created","comment":{"url":"https://api.github.com/repos/twitter/summingbird/pulls/comments/26994412","id":26994412,"diff_hunk":"@@ -0,0 +1,153 @@\n+/*\n+ Copyright 2013 Twitter, Inc.\n+\n+ Licensed under the Apache License, Version 2.0 (the \"License\");\n+ you may not use this file except in compliance with the License.\n+ You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+ Unless required by applicable law or agreed to in writing, software\n+ distributed under the License is distributed on an \"AS IS\" BASIS,\n+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ See the License for the specific language governing permissions and\n+ limitations under the License.\n+ */\n+\n+package com.twitter.summingbird.scalding\n+\n+import cascading.flow.{ Flow, FlowDef }\n+\n+import com.twitter.algebird._\n+import com.twitter.algebird.monad._\n+import com.twitter.summingbird.batch._\n+import com.twitter.summingbird.option.Commutative\n+import com.twitter.scalding.{ Source => ScaldingSource, Test => TestMode, _ }\n+\n+import org.scalacheck._\n+import org.scalacheck.Prop._\n+import org.scalacheck.Properties\n+\n+/**\n+ * Tests for BatchedStore\n+ */\n+object BatchedStoreProperties extends Properties(\"BatchedStore's Properties\") {\n+\n+  implicit def intersectionArb[T: Arbitrary: Ordering]: Arbitrary[Intersection[InclusiveLower, ExclusiveUpper, T]] =\n+    Arbitrary {\n+      for {\n+        l <- Arbitrary.arbitrary[T]\n+        u <- Arbitrary.arbitrary[T]\n+        if implicitly[Ordering[T]].lt(l, u)\n+      } yield Intersection(InclusiveLower(l), ExclusiveUpper(u))\n+    }\n+\n+  implicit val arbTimestamp: Arbitrary[Timestamp] = Arbitrary {\n+    Gen.choose(1L, 100000L)\n+      .map { Timestamp(_) }\n+  }\n+\n+  def sample[T: Arbitrary]: T = Arbitrary.arbitrary[T].sample.get\n+\n+  implicit val arbitraryPipeFactory: Arbitrary[PipeFactory[(Int, Int)]] = {\n+    Arbitrary {\n+      Gen.const {\n+        StateWithError[(Interval[Timestamp], Mode), List[FailureReason], FlowToPipe[(Int, Int)]] {\n+          (timeMode: (Interval[Timestamp], Mode)) =>\n+            {\n+              val (time: Interval[Timestamp], mode: Mode) = timeMode\n+              val a: FlowToPipe[(Int, Int)] = Reader { (fdM: (FlowDef, Mode)) => TypedPipe.from[(Timestamp, (Int, Int))](Seq((Timestamp(10), (2, 3)))) }\n+              Right((timeMode, a))\n+            }\n+        }\n+      }\n+    }\n+  }\n+\n+  implicit def timeExtractor[T <: (Long, Any)] = TestUtil.simpleTimeExtractor[T]\n+\n+  implicit val arbitraryInputWithTimeStampAndBatcher: Arbitrary[(List[(Long, Int)], Batcher, TestStore[Int, Int])] = Arbitrary {\n+    Arbitrary.arbitrary[List[Int]]\n+      .map { in => in.zipWithIndex.map { case (item: Int, time: Int) => (time.toLong, item) } }\n+      .map { in =>\n+        val batcher: Batcher = TestUtil.randomBatcher(in)\n+        val lastTimeStamp = in.size\n+        val testStore = TestStore[Int, Int](\"test\", batcher, sample[Map[Int, Int]], lastTimeStamp)\n+        (in, batcher, testStore)\n+      }\n+  }\n+\n+  implicit def arbitraryLocalMode: Arbitrary[Mode] = Arbitrary { Gen.const(Local(true)) }\n+\n+  property(\"readAfterLastBatch should return interval starting from the last batch written\") = {\n+    forAll {\n+      (diskPipeFactory: PipeFactory[(Int, Int)],\n+      interval: Intersection[InclusiveLower, ExclusiveUpper, Timestamp],\n+      inputWithTimeStampAndBatcherAndStore: (List[(Long, Int)], Batcher, TestStore[Int, Int]),\n+      mode: Mode) =>\n+        val (inputWithTimeStamp, batcher, testStore) = inputWithTimeStampAndBatcherAndStore\n+        val result = testStore.readAfterLastBatch(diskPipeFactory)((interval, mode))\n+\n+        result match {\n+          case Right(((readInterval: Intersection[InclusiveLower, ExclusiveUpper, Timestamp], _), _)) => {","path":"summingbird-scalding-test/src/test/scala/com/twitter/summingbird/scalding/BatchedStoreProperties.scala","position":92,"original_position":92,"commit_id":"c935dd8a352b758534fb6d98cdf213dcd63fb145","original_commit_id":"c935dd8a352b758534fb6d98cdf213dcd63fb145","user":{"login":"johnynek","id":67958,"avatar_url":"https://avatars.githubusercontent.com/u/67958?v=3","gravatar_id":"","url":"https://api.github.com/users/johnynek","html_url":"https://github.com/johnynek","followers_url":"https://api.github.com/users/johnynek/followers","following_url":"https://api.github.com/users/johnynek/following{/other_user}","gists_url":"https://api.github.com/users/johnynek/gists{/gist_id}","starred_url":"https://api.github.com/users/johnynek/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnynek/subscriptions","organizations_url":"https://api.github.com/users/johnynek/orgs","repos_url":"https://api.github.com/users/johnynek/repos","events_url":"https://api.github.com/users/johnynek/events{/privacy}","received_events_url":"https://api.github.com/users/johnynek/received_events","type":"User","site_admin":false},"body":"Can we also test the upper bound? What is the property there? Same as interval upper bound? <= that?","created_at":"2015-03-24T00:21:12Z","updated_at":"2015-03-24T00:21:12Z","html_url":"https://github.com/twitter/summingbird/pull/613#discussion_r26994412","pull_request_url":"https://api.github.com/repos/twitter/summingbird/pulls/613","_links":{"self":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/comments/26994412"},"html":{"href":"https://github.com/twitter/summingbird/pull/613#discussion_r26994412"},"pull_request":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/613"}}},"pull_request":{"url":"https://api.github.com/repos/twitter/summingbird/pulls/613","id":31646049,"html_url":"https://github.com/twitter/summingbird/pull/613","diff_url":"https://github.com/twitter/summingbird/pull/613.diff","patch_url":"https://github.com/twitter/summingbird/pull/613.patch","issue_url":"https://api.github.com/repos/twitter/summingbird/issues/613","number":613,"state":"open","locked":false,"title":"ensure at least one batch before merging","user":{"login":"tsdeng","id":169678,"avatar_url":"https://avatars.githubusercontent.com/u/169678?v=3","gravatar_id":"","url":"https://api.github.com/users/tsdeng","html_url":"https://github.com/tsdeng","followers_url":"https://api.github.com/users/tsdeng/followers","following_url":"https://api.github.com/users/tsdeng/following{/other_user}","gists_url":"https://api.github.com/users/tsdeng/gists{/gist_id}","starred_url":"https://api.github.com/users/tsdeng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tsdeng/subscriptions","organizations_url":"https://api.github.com/users/tsdeng/orgs","repos_url":"https://api.github.com/users/tsdeng/repos","events_url":"https://api.github.com/users/tsdeng/events{/privacy}","received_events_url":"https://api.github.com/users/tsdeng/received_events","type":"User","site_admin":false},"body":"retry of\r\nhttps://github.com/twitter/summingbird/pull/612\r\n\r\nthe check of \"at least one batch is covered by timespan\" only applies to merging.","created_at":"2015-03-20T19:49:32Z","updated_at":"2015-03-24T00:21:12Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ae681d4e6b6011960846ff91fac023c443702b45","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/twitter/summingbird/pulls/613/commits","review_comments_url":"https://api.github.com/repos/twitter/summingbird/pulls/613/comments","review_comment_url":"https://api.github.com/repos/twitter/summingbird/pulls/comments{/number}","comments_url":"https://api.github.com/repos/twitter/summingbird/issues/613/comments","statuses_url":"https://api.github.com/repos/twitter/summingbird/statuses/c935dd8a352b758534fb6d98cdf213dcd63fb145","head":{"label":"tsdeng:ensure_at_least_one_batch_before_merging","ref":"ensure_at_least_one_batch_before_merging","sha":"c935dd8a352b758534fb6d98cdf213dcd63fb145","user":{"login":"tsdeng","id":169678,"avatar_url":"https://avatars.githubusercontent.com/u/169678?v=3","gravatar_id":"","url":"https://api.github.com/users/tsdeng","html_url":"https://github.com/tsdeng","followers_url":"https://api.github.com/users/tsdeng/followers","following_url":"https://api.github.com/users/tsdeng/following{/other_user}","gists_url":"https://api.github.com/users/tsdeng/gists{/gist_id}","starred_url":"https://api.github.com/users/tsdeng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tsdeng/subscriptions","organizations_url":"https://api.github.com/users/tsdeng/orgs","repos_url":"https://api.github.com/users/tsdeng/repos","events_url":"https://api.github.com/users/tsdeng/events{/privacy}","received_events_url":"https://api.github.com/users/tsdeng/received_events","type":"User","site_admin":false},"repo":{"id":19541913,"name":"summingbird","full_name":"tsdeng/summingbird","owner":{"login":"tsdeng","id":169678,"avatar_url":"https://avatars.githubusercontent.com/u/169678?v=3","gravatar_id":"","url":"https://api.github.com/users/tsdeng","html_url":"https://github.com/tsdeng","followers_url":"https://api.github.com/users/tsdeng/followers","following_url":"https://api.github.com/users/tsdeng/following{/other_user}","gists_url":"https://api.github.com/users/tsdeng/gists{/gist_id}","starred_url":"https://api.github.com/users/tsdeng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tsdeng/subscriptions","organizations_url":"https://api.github.com/users/tsdeng/orgs","repos_url":"https://api.github.com/users/tsdeng/repos","events_url":"https://api.github.com/users/tsdeng/events{/privacy}","received_events_url":"https://api.github.com/users/tsdeng/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tsdeng/summingbird","description":"Streaming MapReduce with Scalding and Storm","fork":true,"url":"https://api.github.com/repos/tsdeng/summingbird","forks_url":"https://api.github.com/repos/tsdeng/summingbird/forks","keys_url":"https://api.github.com/repos/tsdeng/summingbird/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tsdeng/summingbird/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tsdeng/summingbird/teams","hooks_url":"https://api.github.com/repos/tsdeng/summingbird/hooks","issue_events_url":"https://api.github.com/repos/tsdeng/summingbird/issues/events{/number}","events_url":"https://api.github.com/repos/tsdeng/summingbird/events","assignees_url":"https://api.github.com/repos/tsdeng/summingbird/assignees{/user}","branches_url":"https://api.github.com/repos/tsdeng/summingbird/branches{/branch}","tags_url":"https://api.github.com/repos/tsdeng/summingbird/tags","blobs_url":"https://api.github.com/repos/tsdeng/summingbird/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tsdeng/summingbird/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tsdeng/summingbird/git/refs{/sha}","trees_url":"https://api.github.com/repos/tsdeng/summingbird/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tsdeng/summingbird/statuses/{sha}","languages_url":"https://api.github.com/repos/tsdeng/summingbird/languages","stargazers_url":"https://api.github.com/repos/tsdeng/summingbird/stargazers","contributors_url":"https://api.github.com/repos/tsdeng/summingbird/contributors","subscribers_url":"https://api.github.com/repos/tsdeng/summingbird/subscribers","subscription_url":"https://api.github.com/repos/tsdeng/summingbird/subscription","commits_url":"https://api.github.com/repos/tsdeng/summingbird/commits{/sha}","git_commits_url":"https://api.github.com/repos/tsdeng/summingbird/git/commits{/sha}","comments_url":"https://api.github.com/repos/tsdeng/summingbird/comments{/number}","issue_comment_url":"https://api.github.com/repos/tsdeng/summingbird/issues/comments{/number}","contents_url":"https://api.github.com/repos/tsdeng/summingbird/contents/{+path}","compare_url":"https://api.github.com/repos/tsdeng/summingbird/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tsdeng/summingbird/merges","archive_url":"https://api.github.com/repos/tsdeng/summingbird/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tsdeng/summingbird/downloads","issues_url":"https://api.github.com/repos/tsdeng/summingbird/issues{/number}","pulls_url":"https://api.github.com/repos/tsdeng/summingbird/pulls{/number}","milestones_url":"https://api.github.com/repos/tsdeng/summingbird/milestones{/number}","notifications_url":"https://api.github.com/repos/tsdeng/summingbird/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tsdeng/summingbird/labels{/name}","releases_url":"https://api.github.com/repos/tsdeng/summingbird/releases{/id}","created_at":"2014-05-07T16:32:20Z","updated_at":"2015-03-23T17:38:12Z","pushed_at":"2015-03-23T23:58:54Z","git_url":"git://github.com/tsdeng/summingbird.git","ssh_url":"git@github.com:tsdeng/summingbird.git","clone_url":"https://github.com/tsdeng/summingbird.git","svn_url":"https://github.com/tsdeng/summingbird","homepage":"https://twitter.com/summingbird","size":4705,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"develop"}},"base":{"label":"twitter:develop","ref":"develop","sha":"a772a6491a60deefd8e623d059346dbadea515dc","user":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"repo":{"id":5957708,"name":"summingbird","full_name":"twitter/summingbird","owner":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/twitter/summingbird","description":"Streaming MapReduce with Scalding and Storm","fork":false,"url":"https://api.github.com/repos/twitter/summingbird","forks_url":"https://api.github.com/repos/twitter/summingbird/forks","keys_url":"https://api.github.com/repos/twitter/summingbird/keys{/key_id}","collaborators_url":"https://api.github.com/repos/twitter/summingbird/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/twitter/summingbird/teams","hooks_url":"https://api.github.com/repos/twitter/summingbird/hooks","issue_events_url":"https://api.github.com/repos/twitter/summingbird/issues/events{/number}","events_url":"https://api.github.com/repos/twitter/summingbird/events","assignees_url":"https://api.github.com/repos/twitter/summingbird/assignees{/user}","branches_url":"https://api.github.com/repos/twitter/summingbird/branches{/branch}","tags_url":"https://api.github.com/repos/twitter/summingbird/tags","blobs_url":"https://api.github.com/repos/twitter/summingbird/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/twitter/summingbird/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/twitter/summingbird/git/refs{/sha}","trees_url":"https://api.github.com/repos/twitter/summingbird/git/trees{/sha}","statuses_url":"https://api.github.com/repos/twitter/summingbird/statuses/{sha}","languages_url":"https://api.github.com/repos/twitter/summingbird/languages","stargazers_url":"https://api.github.com/repos/twitter/summingbird/stargazers","contributors_url":"https://api.github.com/repos/twitter/summingbird/contributors","subscribers_url":"https://api.github.com/repos/twitter/summingbird/subscribers","subscription_url":"https://api.github.com/repos/twitter/summingbird/subscription","commits_url":"https://api.github.com/repos/twitter/summingbird/commits{/sha}","git_commits_url":"https://api.github.com/repos/twitter/summingbird/git/commits{/sha}","comments_url":"https://api.github.com/repos/twitter/summingbird/comments{/number}","issue_comment_url":"https://api.github.com/repos/twitter/summingbird/issues/comments{/number}","contents_url":"https://api.github.com/repos/twitter/summingbird/contents/{+path}","compare_url":"https://api.github.com/repos/twitter/summingbird/compare/{base}...{head}","merges_url":"https://api.github.com/repos/twitter/summingbird/merges","archive_url":"https://api.github.com/repos/twitter/summingbird/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/twitter/summingbird/downloads","issues_url":"https://api.github.com/repos/twitter/summingbird/issues{/number}","pulls_url":"https://api.github.com/repos/twitter/summingbird/pulls{/number}","milestones_url":"https://api.github.com/repos/twitter/summingbird/milestones{/number}","notifications_url":"https://api.github.com/repos/twitter/summingbird/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/twitter/summingbird/labels{/name}","releases_url":"https://api.github.com/repos/twitter/summingbird/releases{/id}","created_at":"2012-09-25T22:38:35Z","updated_at":"2015-03-23T23:17:22Z","pushed_at":"2015-03-23T23:58:54Z","git_url":"git://github.com/twitter/summingbird.git","ssh_url":"git@github.com:twitter/summingbird.git","clone_url":"https://github.com/twitter/summingbird.git","svn_url":"https://github.com/twitter/summingbird","homepage":"https://twitter.com/summingbird","size":12034,"stargazers_count":1450,"watchers_count":1450,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":185,"mirror_url":null,"open_issues_count":144,"forks":185,"open_issues":144,"watchers":1450,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/613"},"html":{"href":"https://github.com/twitter/summingbird/pull/613"},"issue":{"href":"https://api.github.com/repos/twitter/summingbird/issues/613"},"comments":{"href":"https://api.github.com/repos/twitter/summingbird/issues/613/comments"},"review_comments":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/613/comments"},"review_comment":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/613/commits"},"statuses":{"href":"https://api.github.com/repos/twitter/summingbird/statuses/c935dd8a352b758534fb6d98cdf213dcd63fb145"}}}}