{"action":"created","comment":{"url":"https://api.github.com/repos/twitter/scalding/pulls/comments/23245704","id":23245704,"diff_hunk":"@@ -0,0 +1,165 @@\n+/*\n+ Copyright 2014 Twitter, Inc.\n+\n+ Licensed under the Apache License, Version 2.0 (the \"License\");\n+ you may not use this file except in compliance with the License.\n+ You may obtain a copy of the License at\n+\n+ http://www.apache.org/licenses/LICENSE-2.0\n+\n+ Unless required by applicable law or agreed to in writing, software\n+ distributed under the License is distributed on an \"AS IS\" BASIS,\n+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ See the License for the specific language governing permissions and\n+ limitations under the License.\n+ */\n+package com.twitter.scalding.macros.impl.ordser\n+\n+import scala.language.experimental.macros\n+import scala.reflect.macros.Context\n+\n+import com.twitter.scalding._\n+\n+object ProductLike {\n+  def compareBinary(c: Context)(inputStreamA: c.TermName, inputStreamB: c.TermName)(elementData: List[(c.universe.Type, c.universe.TermName, TreeOrderedBuf[c.type])]): c.Tree = {\n+    import c.universe._\n+    def freshT(id: String) = newTermName(c.fresh(id))\n+\n+    elementData.foldLeft(Option.empty[Tree]) {\n+      case (existingTreeOpt, (tpe, accessorSymbol, tBuf)) =>\n+        existingTreeOpt match {\n+          case Some(t) =>\n+            val lastCmp = freshT(\"lastCmp\")\n+            Some(q\"\"\"\n+              val $lastCmp = $t\n+              if($lastCmp != 0) {\n+                $lastCmp\n+              } else {\n+                ${tBuf.compareBinary(inputStreamA, inputStreamB)}\n+              }\n+              \"\"\")\n+          case None =>\n+            Some(tBuf.compareBinary(inputStreamA, inputStreamB))\n+        }\n+    }.getOrElse(q\"0\")\n+  }\n+  def put(c: Context)(inputStream: c.TermName, element: c.TermName)(elementData: List[(c.universe.Type, c.universe.TermName, TreeOrderedBuf[c.type])]): c.Tree = {\n+    import c.universe._\n+    def freshT(id: String) = newTermName(c.fresh(id))\n+    val innerElement = freshT(\"innerElement\")\n+\n+    elementData.foldLeft(q\"\") {\n+      case (existingTree, (tpe, accessorSymbol, tBuf)) =>\n+        q\"\"\"\n+          $existingTree\n+          val $innerElement = $element.$accessorSymbol\n+          ${tBuf.put(inputStream, innerElement)}\n+          \"\"\"\n+    }\n+  }\n+\n+  def length(c: Context)(element: c.Tree)(elementData: List[(c.universe.Type, c.universe.TermName, TreeOrderedBuf[c.type])]): LengthTypes[c.type] = {\n+    import c.universe._\n+\n+    val (constSize, dynamicFunctions, maybeLength, noLength) =\n+      elementData.foldLeft((0, List[c.Tree](), List[c.Tree](), 0)) {\n+        case ((constantLength, dynamicLength, maybeLength, noLength), (tpe, accessorSymbol, tBuf)) =>\n+\n+          tBuf.length(q\"$element.$accessorSymbol\") match {\n+            case const: ConstantLengthCalculation[_] => (constantLength + const.asInstanceOf[ConstantLengthCalculation[c.type]].toInt, dynamicLength, maybeLength, noLength)\n+            case f: FastLengthCalculation[_] => (constantLength, dynamicLength :+ f.asInstanceOf[FastLengthCalculation[c.type]].t, maybeLength, noLength)\n+            case m: MaybeLengthCalculation[_] => (constantLength, dynamicLength, maybeLength :+ m.asInstanceOf[MaybeLengthCalculation[c.type]].t, noLength)\n+            case _: NoLengthCalculationAvailable[_] => (constantLength, dynamicLength, maybeLength, noLength + 1)\n+          }\n+      }\n+\n+    val combinedDynamic = dynamicFunctions.foldLeft(q\"\"\"$constSize\"\"\") {\n+      case (prev, t) =>\n+        q\"$prev + $t\"\n+    }\n+\n+    if (noLength > 0) {\n+      NoLengthCalculationAvailable(c)\n+    } else {\n+      if (maybeLength.size == 0 && dynamicFunctions.size == 0) {\n+        ConstantLengthCalculation(c)(constSize)\n+      } else {\n+        if (maybeLength.size == 0) {\n+          FastLengthCalculation(c)(combinedDynamic)\n+        } else {\n+\n+          // Contains an Option[Either[Int, Int]]\n+          val combinedMaybe: Tree = maybeLength.tail.foldLeft(maybeLength.head) {\n+            case (hOpt, nxtOpt) =>\n+              q\"\"\"\n+              $hOpt.flatMap { last =>\n+                $nxtOpt.map { nxt =>\n+                  (last, nxt) match {","path":"scalding-macros/src/main/scala/com/twitter/scalding/macros/impl/ordser/ProductLike.scala","position":97,"original_position":97,"commit_id":"ef9dc243386ec41c9b4c4df6ff0e62cc83080a2a","original_commit_id":"ef9dc243386ec41c9b4c4df6ff0e62cc83080a2a","user":{"login":"johnynek","id":67958,"avatar_url":"https://avatars.githubusercontent.com/u/67958?v=3","gravatar_id":"","url":"https://api.github.com/users/johnynek","html_url":"https://github.com/johnynek","followers_url":"https://api.github.com/users/johnynek/followers","following_url":"https://api.github.com/users/johnynek/following{/other_user}","gists_url":"https://api.github.com/users/johnynek/gists{/gist_id}","starred_url":"https://api.github.com/users/johnynek/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnynek/subscriptions","organizations_url":"https://api.github.com/users/johnynek/orgs","repos_url":"https://api.github.com/users/johnynek/repos","events_url":"https://api.github.com/users/johnynek/events{/privacy}","received_events_url":"https://api.github.com/users/johnynek/received_events","type":"User","site_admin":false},"body":"seems like we need a type other than Either here, and use `+` method on that type to implement this logic. I don't think it needs to be coded into the macro.","created_at":"2015-01-20T18:59:44Z","updated_at":"2015-01-20T18:59:44Z","html_url":"https://github.com/twitter/scalding/pull/1165#discussion_r23245704","pull_request_url":"https://api.github.com/repos/twitter/scalding/pulls/1165","_links":{"self":{"href":"https://api.github.com/repos/twitter/scalding/pulls/comments/23245704"},"html":{"href":"https://github.com/twitter/scalding/pull/1165#discussion_r23245704"},"pull_request":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1165"}}},"pull_request":{"url":"https://api.github.com/repos/twitter/scalding/pulls/1165","id":27577439,"html_url":"https://github.com/twitter/scalding/pull/1165","diff_url":"https://github.com/twitter/scalding/pull/1165.diff","patch_url":"https://github.com/twitter/scalding/pull/1165.patch","issue_url":"https://api.github.com/repos/twitter/scalding/issues/1165","number":1165,"state":"open","locked":false,"title":"Macros for ordered serialization","user":{"login":"ianoc","id":446652,"avatar_url":"https://avatars.githubusercontent.com/u/446652?v=3","gravatar_id":"","url":"https://api.github.com/users/ianoc","html_url":"https://github.com/ianoc","followers_url":"https://api.github.com/users/ianoc/followers","following_url":"https://api.github.com/users/ianoc/following{/other_user}","gists_url":"https://api.github.com/users/ianoc/gists{/gist_id}","starred_url":"https://api.github.com/users/ianoc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ianoc/subscriptions","organizations_url":"https://api.github.com/users/ianoc/orgs","repos_url":"https://api.github.com/users/ianoc/repos","events_url":"https://api.github.com/users/ianoc/events{/privacy}","received_events_url":"https://api.github.com/users/ianoc/received_events","type":"User","site_admin":false},"body":"Currently clocking in at:\r\n\r\n1min 35sec / 6min 45sec in our internal benchmark\r\n\r\nThe TreeOrderedBuf object could do with some more cleaning up really, but to get it out there for @johnynek to have a look at really as is. \r\n\r\nIn some tests the StringOrderedSerialization geeking out code doesn't seem to make any difference on or off in our benchmark, so I'm not sure if we want to keep the more complicated code. But it works and is tested, so I don't really mind either way. \r\n\r\nAn alternative would be just:\r\n```\r\n  final def binaryIntCompare(leftSize: Int, seekingLeft: InputStream, rightSize: Int, seekingRight: InputStream): Int = {\r\n    val minLength = Math.min(leftSize, rightSize)\r\n    var i: Int = 0\r\n    var comparator = 0\r\n    while (i < minLength && comparator == 0) {\r\n      comparator = Integer.compare(seekingLeft.readUnsignedByte, seekingRight.readUnsignedByte)\r\n      i = i + 1\r\n    }\r\n    if (comparator == 0) {\r\n      leftSize - rightSize\r\n    } else {\r\n      comparator\r\n    }\r\n  }\r\n```\r\n\r\nWe don't have micro benchmarks around this directly so our outer benchmarks might not be too telling on this really.","created_at":"2015-01-18T21:35:50Z","updated_at":"2015-01-20T18:59:44Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e8487ae9d0a11c048c063c6ca4457c0c47bf526a","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/twitter/scalding/pulls/1165/commits","review_comments_url":"https://api.github.com/repos/twitter/scalding/pulls/1165/comments","review_comment_url":"https://api.github.com/repos/twitter/scalding/pulls/comments/{number}","comments_url":"https://api.github.com/repos/twitter/scalding/issues/1165/comments","statuses_url":"https://api.github.com/repos/twitter/scalding/statuses/ef9dc243386ec41c9b4c4df6ff0e62cc83080a2a","head":{"label":"twitter:ianoc/macros2","ref":"ianoc/macros2","sha":"ef9dc243386ec41c9b4c4df6ff0e62cc83080a2a","user":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"repo":{"id":3146558,"name":"scalding","full_name":"twitter/scalding","owner":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/twitter/scalding","description":"A Scala API for Cascading","fork":false,"url":"https://api.github.com/repos/twitter/scalding","forks_url":"https://api.github.com/repos/twitter/scalding/forks","keys_url":"https://api.github.com/repos/twitter/scalding/keys{/key_id}","collaborators_url":"https://api.github.com/repos/twitter/scalding/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/twitter/scalding/teams","hooks_url":"https://api.github.com/repos/twitter/scalding/hooks","issue_events_url":"https://api.github.com/repos/twitter/scalding/issues/events{/number}","events_url":"https://api.github.com/repos/twitter/scalding/events","assignees_url":"https://api.github.com/repos/twitter/scalding/assignees{/user}","branches_url":"https://api.github.com/repos/twitter/scalding/branches{/branch}","tags_url":"https://api.github.com/repos/twitter/scalding/tags","blobs_url":"https://api.github.com/repos/twitter/scalding/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/twitter/scalding/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/twitter/scalding/git/refs{/sha}","trees_url":"https://api.github.com/repos/twitter/scalding/git/trees{/sha}","statuses_url":"https://api.github.com/repos/twitter/scalding/statuses/{sha}","languages_url":"https://api.github.com/repos/twitter/scalding/languages","stargazers_url":"https://api.github.com/repos/twitter/scalding/stargazers","contributors_url":"https://api.github.com/repos/twitter/scalding/contributors","subscribers_url":"https://api.github.com/repos/twitter/scalding/subscribers","subscription_url":"https://api.github.com/repos/twitter/scalding/subscription","commits_url":"https://api.github.com/repos/twitter/scalding/commits{/sha}","git_commits_url":"https://api.github.com/repos/twitter/scalding/git/commits{/sha}","comments_url":"https://api.github.com/repos/twitter/scalding/comments{/number}","issue_comment_url":"https://api.github.com/repos/twitter/scalding/issues/comments/{number}","contents_url":"https://api.github.com/repos/twitter/scalding/contents/{+path}","compare_url":"https://api.github.com/repos/twitter/scalding/compare/{base}...{head}","merges_url":"https://api.github.com/repos/twitter/scalding/merges","archive_url":"https://api.github.com/repos/twitter/scalding/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/twitter/scalding/downloads","issues_url":"https://api.github.com/repos/twitter/scalding/issues{/number}","pulls_url":"https://api.github.com/repos/twitter/scalding/pulls{/number}","milestones_url":"https://api.github.com/repos/twitter/scalding/milestones{/number}","notifications_url":"https://api.github.com/repos/twitter/scalding/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/twitter/scalding/labels{/name}","releases_url":"https://api.github.com/repos/twitter/scalding/releases{/id}","created_at":"2012-01-10T16:22:08Z","updated_at":"2015-01-18T13:37:52Z","pushed_at":"2015-01-19T00:34:06Z","git_url":"git://github.com/twitter/scalding.git","ssh_url":"git@github.com:twitter/scalding.git","clone_url":"https://github.com/twitter/scalding.git","svn_url":"https://github.com/twitter/scalding","homepage":"http://twitter.com/scalding","size":19504,"stargazers_count":1897,"watchers_count":1897,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":424,"mirror_url":null,"open_issues_count":205,"forks":424,"open_issues":205,"watchers":1897,"default_branch":"develop"}},"base":{"label":"twitter:ianoc/stringBugFix","ref":"ianoc/stringBugFix","sha":"6fa7575837b06732caaad9efb0742f2e3fc269a9","user":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"repo":{"id":3146558,"name":"scalding","full_name":"twitter/scalding","owner":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/twitter/scalding","description":"A Scala API for Cascading","fork":false,"url":"https://api.github.com/repos/twitter/scalding","forks_url":"https://api.github.com/repos/twitter/scalding/forks","keys_url":"https://api.github.com/repos/twitter/scalding/keys{/key_id}","collaborators_url":"https://api.github.com/repos/twitter/scalding/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/twitter/scalding/teams","hooks_url":"https://api.github.com/repos/twitter/scalding/hooks","issue_events_url":"https://api.github.com/repos/twitter/scalding/issues/events{/number}","events_url":"https://api.github.com/repos/twitter/scalding/events","assignees_url":"https://api.github.com/repos/twitter/scalding/assignees{/user}","branches_url":"https://api.github.com/repos/twitter/scalding/branches{/branch}","tags_url":"https://api.github.com/repos/twitter/scalding/tags","blobs_url":"https://api.github.com/repos/twitter/scalding/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/twitter/scalding/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/twitter/scalding/git/refs{/sha}","trees_url":"https://api.github.com/repos/twitter/scalding/git/trees{/sha}","statuses_url":"https://api.github.com/repos/twitter/scalding/statuses/{sha}","languages_url":"https://api.github.com/repos/twitter/scalding/languages","stargazers_url":"https://api.github.com/repos/twitter/scalding/stargazers","contributors_url":"https://api.github.com/repos/twitter/scalding/contributors","subscribers_url":"https://api.github.com/repos/twitter/scalding/subscribers","subscription_url":"https://api.github.com/repos/twitter/scalding/subscription","commits_url":"https://api.github.com/repos/twitter/scalding/commits{/sha}","git_commits_url":"https://api.github.com/repos/twitter/scalding/git/commits{/sha}","comments_url":"https://api.github.com/repos/twitter/scalding/comments{/number}","issue_comment_url":"https://api.github.com/repos/twitter/scalding/issues/comments/{number}","contents_url":"https://api.github.com/repos/twitter/scalding/contents/{+path}","compare_url":"https://api.github.com/repos/twitter/scalding/compare/{base}...{head}","merges_url":"https://api.github.com/repos/twitter/scalding/merges","archive_url":"https://api.github.com/repos/twitter/scalding/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/twitter/scalding/downloads","issues_url":"https://api.github.com/repos/twitter/scalding/issues{/number}","pulls_url":"https://api.github.com/repos/twitter/scalding/pulls{/number}","milestones_url":"https://api.github.com/repos/twitter/scalding/milestones{/number}","notifications_url":"https://api.github.com/repos/twitter/scalding/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/twitter/scalding/labels{/name}","releases_url":"https://api.github.com/repos/twitter/scalding/releases{/id}","created_at":"2012-01-10T16:22:08Z","updated_at":"2015-01-18T13:37:52Z","pushed_at":"2015-01-19T00:34:06Z","git_url":"git://github.com/twitter/scalding.git","ssh_url":"git@github.com:twitter/scalding.git","clone_url":"https://github.com/twitter/scalding.git","svn_url":"https://github.com/twitter/scalding","homepage":"http://twitter.com/scalding","size":19504,"stargazers_count":1897,"watchers_count":1897,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":424,"mirror_url":null,"open_issues_count":205,"forks":424,"open_issues":205,"watchers":1897,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1165"},"html":{"href":"https://github.com/twitter/scalding/pull/1165"},"issue":{"href":"https://api.github.com/repos/twitter/scalding/issues/1165"},"comments":{"href":"https://api.github.com/repos/twitter/scalding/issues/1165/comments"},"review_comments":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1165/comments"},"review_comment":{"href":"https://api.github.com/repos/twitter/scalding/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1165/commits"},"statuses":{"href":"https://api.github.com/repos/twitter/scalding/statuses/ef9dc243386ec41c9b4c4df6ff0e62cc83080a2a"}}}}