{"action":"created","comment":{"url":"https://api.github.com/repos/twitter/scalding/pulls/comments/26357709","id":26357709,"diff_hunk":"@@ -0,0 +1,299 @@\n+/*\n+ Copyright 2014 Twitter, Inc.\n+\n+ Licensed under the Apache License, Version 2.0 (the \"License\");\n+ you may not use this file except in compliance with the License.\n+ You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+ Unless required by applicable law or agreed to in writing, software\n+ distributed under the License is distributed on an \"AS IS\" BASIS,\n+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ See the License for the specific language governing permissions and\n+ limitations under the License.\n+ */\n+package com.twitter.scalding.macros.impl.ordered_serialization\n+\n+import com.twitter.scalding._\n+import com.twitter.scalding.serialization.OrderedSerialization\n+import com.twitter.scalding.serialization.JavaStreamEnrichments\n+import java.io.InputStream\n+import scala.reflect.macros.Context\n+import scala.language.experimental.macros\n+import scala.util.control.NonFatal\n+\n+object CommonCompareBinary {\n+  import com.twitter.scalding.serialization.JavaStreamEnrichments._\n+\n+  // If the lengths are equal and greater than this number\n+  // we will compare on all the containing bytes\n+  val minSizeForFulBinaryCompare = 24\n+\n+  /**\n+   * This method will compare two InputStreams of given lengths\n+   * If the inputsteam supports mark/reset (such as those backed by Array[Byte]),\n+   * and the lengths are equal and longer than minSizeForFulBinaryCompare we first\n+   * check if they are byte-for-byte identical, which is a cheap way to avoid doing\n+   * potentially complex logic in binary comparators\n+   */\n+  final def earlyEqual(inputStreamA: InputStream,\n+    lenA: Int,\n+    inputStreamB: InputStream,\n+    lenB: Int): Boolean =\n+    (lenA > minSizeForFulBinaryCompare &&\n+      (lenA == lenB) &&\n+      inputStreamA.markSupported &&\n+      inputStreamB.markSupported) && {\n+        inputStreamA.mark(lenA)\n+        inputStreamB.mark(lenB)\n+\n+        var pos: Int = 0\n+        while (pos < lenA) {\n+          val a = inputStreamA.read\n+          val b = inputStreamB.read\n+          pos += 1\n+          if (a != b) {\n+            inputStreamA.reset()\n+            inputStreamB.reset()\n+            // yeah, return sucks, but trying to optimize here\n+            return false\n+          }\n+          // a == b, but may be eof\n+          if (a < 0) return JavaStreamEnrichments.eof\n+        }\n+        // we consumed all the bytes, and they were all equal\n+        true\n+      }\n+}\n+object TreeOrderedBuf {\n+  import CompileTimeLengthTypes._\n+  def toOrderedSerialization[T](c: Context)(t: TreeOrderedBuf[c.type])(implicit T: t.ctx.WeakTypeTag[T]): t.ctx.Expr[OrderedSerialization[T]] = {\n+    import t.ctx.universe._\n+    def freshT(id: String) = newTermName(c.fresh(s\"fresh_$id\"))\n+    val outputLength = freshT(\"outputLength\")\n+\n+    val innerLengthFn: Tree = {\n+      val element = freshT(\"element\")\n+\n+      val fnBodyOpt = t.length(q\"$element\") match {\n+        case _: NoLengthCalculationAvailable[_] => None\n+        case const: ConstantLengthCalculation[_] => None\n+        case f: FastLengthCalculation[_] => Some(q\"\"\"\n+        _root_.com.twitter.scalding.macros.impl.ordered_serialization.runtime_helpers.DynamicLen(${f.asInstanceOf[FastLengthCalculation[c.type]].t})\n+        \"\"\")\n+        case m: MaybeLengthCalculation[_] => Some(m.asInstanceOf[MaybeLengthCalculation[c.type]].t)\n+      }\n+\n+      fnBodyOpt.map { fnBody =>\n+        q\"\"\"\n+        @inline private[this] def payloadLength($element: $T): _root_.com.twitter.scalding.macros.impl.ordered_serialization.runtime_helpers.MaybeLength = {\n+          $fnBody\n+        }\n+        \"\"\"\n+      }.getOrElse(q\"()\")\n+    }\n+\n+    def binaryLengthGen(typeName: Tree): (Tree, Tree) = {\n+      val tempLen = freshT(\"tempLen\")\n+      val lensLen = freshT(\"lensLen\")\n+      val element = freshT(\"element\")\n+      val callDynamic = (q\"\"\"override def staticSize: Option[Int] = None\"\"\",\n+        q\"\"\"\n+\n+      override def dynamicSize($element: $typeName): Option[Int] = {\n+        val $tempLen = payloadLength($element) match {\n+          case _root_.com.twitter.scalding.macros.impl.ordered_serialization.runtime_helpers.NoLengthCalculation => None\n+          case _root_.com.twitter.scalding.macros.impl.ordered_serialization.runtime_helpers.ConstLen(l) => Some(l)\n+          case _root_.com.twitter.scalding.macros.impl.ordered_serialization.runtime_helpers.DynamicLen(l) => Some(l)\n+        }\n+        (if ($tempLen.isDefined) {\n+          // Avoid a closure here while we are geeking out\n+          val innerLen = $tempLen.get\n+          val $lensLen = posVarIntSize(innerLen)\n+          Some(innerLen + $lensLen)\n+       } else None): Option[Int]\n+     }\n+      \"\"\")\n+      t.length(q\"$element\") match {\n+        case _: NoLengthCalculationAvailable[_] => (q\"\"\"\n+          override def staticSize: Option[Int] = None\"\"\", q\"\"\"\n+          override def dynamicSize($element: $typeName): Option[Int] = None\"\"\")\n+        case const: ConstantLengthCalculation[_] => (q\"\"\"\n+          override val staticSize: Option[Int] = Some(${const.toInt})\"\"\", q\"\"\"\n+          override def dynamicSize($element: $typeName): Option[Int] = staticSize\"\"\")\n+        case f: FastLengthCalculation[_] => callDynamic\n+        case m: MaybeLengthCalculation[_] => callDynamic\n+      }\n+    }\n+\n+    def putFnGen(outerbaos: TermName, element: TermName) = {\n+      val baos = freshT(\"baos\")\n+      val len = freshT(\"len\")\n+      val oldPos = freshT(\"oldPos\")\n+\n+      /**\n+       * This is the worst case: we have to serialize in a side buffer\n+       * and then see how large it actually is. This happens for cases, like\n+       * string, where the cost to see the serialized size is not cheaper than\n+       * directly serializing.\n+       */\n+      val noLenCalc = q\"\"\"\n+      // Start with pretty big buffers because reallocation will be expensive\n+      val $baos = new _root_.java.io.ByteArrayOutputStream(256)\n+      ${t.put(baos, element)}\n+      val $len = $baos.size\n+      $outerbaos.writePosVarInt($len)\n+      $baos.writeTo($outerbaos)","path":"scalding-macros/src/main/scala/com/twitter/scalding/macros/impl/ordered_serialization/TreeOrderedBuf.scala","position":147,"original_position":147,"commit_id":"6104de58a0c68c7f5d487abd5d40faf8b416f724","original_commit_id":"6104de58a0c68c7f5d487abd5d40faf8b416f724","user":{"login":"isnotinvain","id":372969,"avatar_url":"https://avatars.githubusercontent.com/u/372969?v=3","gravatar_id":"","url":"https://api.github.com/users/isnotinvain","html_url":"https://github.com/isnotinvain","followers_url":"https://api.github.com/users/isnotinvain/followers","following_url":"https://api.github.com/users/isnotinvain/following{/other_user}","gists_url":"https://api.github.com/users/isnotinvain/gists{/gist_id}","starred_url":"https://api.github.com/users/isnotinvain/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/isnotinvain/subscriptions","organizations_url":"https://api.github.com/users/isnotinvain/orgs","repos_url":"https://api.github.com/users/isnotinvain/repos","events_url":"https://api.github.com/users/isnotinvain/events{/privacy}","received_events_url":"https://api.github.com/users/isnotinvain/received_events","type":"User","site_admin":false},"body":"we dono't have control over baos, so moot point","created_at":"2015-03-13T00:40:07Z","updated_at":"2015-03-13T00:40:07Z","html_url":"https://github.com/twitter/scalding/pull/1209#discussion_r26357709","pull_request_url":"https://api.github.com/repos/twitter/scalding/pulls/1209","_links":{"self":{"href":"https://api.github.com/repos/twitter/scalding/pulls/comments/26357709"},"html":{"href":"https://github.com/twitter/scalding/pull/1209#discussion_r26357709"},"pull_request":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1209"}}},"pull_request":{"url":"https://api.github.com/repos/twitter/scalding/pulls/1209","id":30044035,"html_url":"https://github.com/twitter/scalding/pull/1209","diff_url":"https://github.com/twitter/scalding/pull/1209.diff","patch_url":"https://github.com/twitter/scalding/pull/1209.patch","issue_url":"https://api.github.com/repos/twitter/scalding/issues/1209","number":1209,"state":"open","locked":false,"title":"Macros for OrderedSerialization","user":{"login":"johnynek","id":67958,"avatar_url":"https://avatars.githubusercontent.com/u/67958?v=3","gravatar_id":"","url":"https://api.github.com/users/johnynek","html_url":"https://github.com/johnynek","followers_url":"https://api.github.com/users/johnynek/followers","following_url":"https://api.github.com/users/johnynek/following{/other_user}","gists_url":"https://api.github.com/users/johnynek/gists{/gist_id}","starred_url":"https://api.github.com/users/johnynek/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnynek/subscriptions","organizations_url":"https://api.github.com/users/johnynek/orgs","repos_url":"https://api.github.com/users/johnynek/repos","events_url":"https://api.github.com/users/johnynek/events{/privacy}","received_events_url":"https://api.github.com/users/johnynek/received_events","type":"User","site_admin":false},"body":"These are the macros on top of #1208.\r\n\r\nWe should only merge this after #1208 is merged and probably after we have deployed this and tested in detail at Twitter.","created_at":"2015-02-26T01:47:14Z","updated_at":"2015-03-13T00:40:07Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b38fe5d24afd6f29f2979d18ffafe3305a26af87","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/twitter/scalding/pulls/1209/commits","review_comments_url":"https://api.github.com/repos/twitter/scalding/pulls/1209/comments","review_comment_url":"https://api.github.com/repos/twitter/scalding/pulls/comments{/number}","comments_url":"https://api.github.com/repos/twitter/scalding/issues/1209/comments","statuses_url":"https://api.github.com/repos/twitter/scalding/statuses/6104de58a0c68c7f5d487abd5d40faf8b416f724","head":{"label":"twitter:fbs-staging-macros","ref":"fbs-staging-macros","sha":"6104de58a0c68c7f5d487abd5d40faf8b416f724","user":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"repo":{"id":3146558,"name":"scalding","full_name":"twitter/scalding","owner":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/twitter/scalding","description":"A Scala API for Cascading","fork":false,"url":"https://api.github.com/repos/twitter/scalding","forks_url":"https://api.github.com/repos/twitter/scalding/forks","keys_url":"https://api.github.com/repos/twitter/scalding/keys{/key_id}","collaborators_url":"https://api.github.com/repos/twitter/scalding/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/twitter/scalding/teams","hooks_url":"https://api.github.com/repos/twitter/scalding/hooks","issue_events_url":"https://api.github.com/repos/twitter/scalding/issues/events{/number}","events_url":"https://api.github.com/repos/twitter/scalding/events","assignees_url":"https://api.github.com/repos/twitter/scalding/assignees{/user}","branches_url":"https://api.github.com/repos/twitter/scalding/branches{/branch}","tags_url":"https://api.github.com/repos/twitter/scalding/tags","blobs_url":"https://api.github.com/repos/twitter/scalding/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/twitter/scalding/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/twitter/scalding/git/refs{/sha}","trees_url":"https://api.github.com/repos/twitter/scalding/git/trees{/sha}","statuses_url":"https://api.github.com/repos/twitter/scalding/statuses/{sha}","languages_url":"https://api.github.com/repos/twitter/scalding/languages","stargazers_url":"https://api.github.com/repos/twitter/scalding/stargazers","contributors_url":"https://api.github.com/repos/twitter/scalding/contributors","subscribers_url":"https://api.github.com/repos/twitter/scalding/subscribers","subscription_url":"https://api.github.com/repos/twitter/scalding/subscription","commits_url":"https://api.github.com/repos/twitter/scalding/commits{/sha}","git_commits_url":"https://api.github.com/repos/twitter/scalding/git/commits{/sha}","comments_url":"https://api.github.com/repos/twitter/scalding/comments{/number}","issue_comment_url":"https://api.github.com/repos/twitter/scalding/issues/comments{/number}","contents_url":"https://api.github.com/repos/twitter/scalding/contents/{+path}","compare_url":"https://api.github.com/repos/twitter/scalding/compare/{base}...{head}","merges_url":"https://api.github.com/repos/twitter/scalding/merges","archive_url":"https://api.github.com/repos/twitter/scalding/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/twitter/scalding/downloads","issues_url":"https://api.github.com/repos/twitter/scalding/issues{/number}","pulls_url":"https://api.github.com/repos/twitter/scalding/pulls{/number}","milestones_url":"https://api.github.com/repos/twitter/scalding/milestones{/number}","notifications_url":"https://api.github.com/repos/twitter/scalding/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/twitter/scalding/labels{/name}","releases_url":"https://api.github.com/repos/twitter/scalding/releases{/id}","created_at":"2012-01-10T16:22:08Z","updated_at":"2015-03-12T23:40:07Z","pushed_at":"2015-03-10T20:29:52Z","git_url":"git://github.com/twitter/scalding.git","ssh_url":"git@github.com:twitter/scalding.git","clone_url":"https://github.com/twitter/scalding.git","svn_url":"https://github.com/twitter/scalding","homepage":"http://twitter.com/scalding","size":21327,"stargazers_count":1985,"watchers_count":1985,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":444,"mirror_url":null,"open_issues_count":224,"forks":444,"open_issues":224,"watchers":1985,"default_branch":"develop"}},"base":{"label":"twitter:develop","ref":"develop","sha":"ff79f900e797767c727dd78d4ed9b50a80bae249","user":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"repo":{"id":3146558,"name":"scalding","full_name":"twitter/scalding","owner":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/twitter/scalding","description":"A Scala API for Cascading","fork":false,"url":"https://api.github.com/repos/twitter/scalding","forks_url":"https://api.github.com/repos/twitter/scalding/forks","keys_url":"https://api.github.com/repos/twitter/scalding/keys{/key_id}","collaborators_url":"https://api.github.com/repos/twitter/scalding/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/twitter/scalding/teams","hooks_url":"https://api.github.com/repos/twitter/scalding/hooks","issue_events_url":"https://api.github.com/repos/twitter/scalding/issues/events{/number}","events_url":"https://api.github.com/repos/twitter/scalding/events","assignees_url":"https://api.github.com/repos/twitter/scalding/assignees{/user}","branches_url":"https://api.github.com/repos/twitter/scalding/branches{/branch}","tags_url":"https://api.github.com/repos/twitter/scalding/tags","blobs_url":"https://api.github.com/repos/twitter/scalding/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/twitter/scalding/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/twitter/scalding/git/refs{/sha}","trees_url":"https://api.github.com/repos/twitter/scalding/git/trees{/sha}","statuses_url":"https://api.github.com/repos/twitter/scalding/statuses/{sha}","languages_url":"https://api.github.com/repos/twitter/scalding/languages","stargazers_url":"https://api.github.com/repos/twitter/scalding/stargazers","contributors_url":"https://api.github.com/repos/twitter/scalding/contributors","subscribers_url":"https://api.github.com/repos/twitter/scalding/subscribers","subscription_url":"https://api.github.com/repos/twitter/scalding/subscription","commits_url":"https://api.github.com/repos/twitter/scalding/commits{/sha}","git_commits_url":"https://api.github.com/repos/twitter/scalding/git/commits{/sha}","comments_url":"https://api.github.com/repos/twitter/scalding/comments{/number}","issue_comment_url":"https://api.github.com/repos/twitter/scalding/issues/comments{/number}","contents_url":"https://api.github.com/repos/twitter/scalding/contents/{+path}","compare_url":"https://api.github.com/repos/twitter/scalding/compare/{base}...{head}","merges_url":"https://api.github.com/repos/twitter/scalding/merges","archive_url":"https://api.github.com/repos/twitter/scalding/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/twitter/scalding/downloads","issues_url":"https://api.github.com/repos/twitter/scalding/issues{/number}","pulls_url":"https://api.github.com/repos/twitter/scalding/pulls{/number}","milestones_url":"https://api.github.com/repos/twitter/scalding/milestones{/number}","notifications_url":"https://api.github.com/repos/twitter/scalding/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/twitter/scalding/labels{/name}","releases_url":"https://api.github.com/repos/twitter/scalding/releases{/id}","created_at":"2012-01-10T16:22:08Z","updated_at":"2015-03-12T23:40:07Z","pushed_at":"2015-03-10T20:29:52Z","git_url":"git://github.com/twitter/scalding.git","ssh_url":"git@github.com:twitter/scalding.git","clone_url":"https://github.com/twitter/scalding.git","svn_url":"https://github.com/twitter/scalding","homepage":"http://twitter.com/scalding","size":21327,"stargazers_count":1985,"watchers_count":1985,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":444,"mirror_url":null,"open_issues_count":224,"forks":444,"open_issues":224,"watchers":1985,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1209"},"html":{"href":"https://github.com/twitter/scalding/pull/1209"},"issue":{"href":"https://api.github.com/repos/twitter/scalding/issues/1209"},"comments":{"href":"https://api.github.com/repos/twitter/scalding/issues/1209/comments"},"review_comments":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1209/comments"},"review_comment":{"href":"https://api.github.com/repos/twitter/scalding/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1209/commits"},"statuses":{"href":"https://api.github.com/repos/twitter/scalding/statuses/6104de58a0c68c7f5d487abd5d40faf8b416f724"}}}}