{"action":"created","comment":{"url":"https://api.github.com/repos/uwescience/myria/pulls/comments/46223232","id":46223232,"diff_hunk":"@@ -0,0 +1,214 @@\n+package edu.washington.escience.myria.operator;\n+\n+import java.util.ArrayList;\n+import java.util.Arrays;\n+import java.util.BitSet;\n+import java.util.List;\n+\n+import jersey.repackaged.com.google.common.base.Preconditions;\n+\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+import com.google.common.collect.ImmutableMap;\n+import com.gs.collections.api.block.procedure.primitive.IntProcedure;\n+import com.gs.collections.impl.list.mutable.primitive.IntArrayList;\n+import com.gs.collections.impl.map.mutable.primitive.IntObjectHashMap;\n+\n+import edu.washington.escience.myria.Schema;\n+import edu.washington.escience.myria.Type;\n+import edu.washington.escience.myria.column.Column;\n+import edu.washington.escience.myria.column.IntArrayColumn;\n+import edu.washington.escience.myria.storage.MutableTupleBuffer;\n+import edu.washington.escience.myria.storage.TupleBatch;\n+import edu.washington.escience.myria.storage.TupleUtils;\n+import edu.washington.escience.myria.util.HashUtils;\n+\n+/**\n+ * Keeps distinct tuples with their counts and only emits a tuple at the first time when its count hits the threshold.\n+ * */\n+public final class CountFilter extends StreamingState {\n+\n+  /** Required for Java serialization. */\n+  private static final long serialVersionUID = 1L;\n+\n+  /**\n+   * The logger for this class.\n+   * */\n+  static final Logger LOGGER = LoggerFactory.getLogger(CountFilter.class);\n+\n+  /**\n+   * Indices to unique tuples.\n+   * */\n+  private transient IntObjectHashMap<IntArrayList> uniqueTupleIndices;\n+\n+  /**\n+   * The buffer for storing unique tuples.\n+   * */\n+  private transient MutableTupleBuffer uniqueTuples = null;\n+  /**\n+   * The count of each unique tuple.\n+   * */\n+  private transient MutableTupleBuffer tupleCounts = null;\n+\n+  /** threshold of the count. */\n+  public final int threshold;\n+\n+  /** column indices of the key. */\n+  private final int[] keyColIndices;\n+\n+  /**\n+   * @param threshold threshold\n+   * @param keyColIndices key column indices.\n+   */\n+  public CountFilter(final int threshold, final int[] keyColIndices) {\n+    Preconditions.checkArgument(threshold >= 0, \"threshold needs to be greater than or equal to 0\");\n+    this.threshold = threshold;\n+    this.keyColIndices = Arrays.copyOf(keyColIndices, keyColIndices.length);\n+  }\n+\n+  @Override\n+  public void cleanup() {\n+    uniqueTuples = null;\n+    uniqueTupleIndices = null;\n+    tupleCounts = null;\n+  }\n+\n+  /**\n+   * Do duplicate elimination for tb.\n+   * \n+   * @param tb the TupleBatch for performing DupElim.\n+   * @return the duplicate eliminated TB.\n+   * */\n+  protected TupleBatch countFilter(final TupleBatch tb) {\n+    final int numTuples = tb.numTuples();\n+    if (numTuples <= 0) {\n+      return tb;\n+    }\n+    doCount.inputTB = tb;\n+    final List<? extends Column<?>> columns = tb.getDataColumns();\n+    final BitSet toRemove = new BitSet(numTuples);\n+    for (int i = 0; i < numTuples; ++i) {\n+      final int nextIndex = uniqueTuples.numTuples();\n+      final int cntHashCode = HashUtils.hashSubRow(tb, keyColIndices, i);\n+      IntArrayList tupleIndexList = uniqueTupleIndices.get(cntHashCode);\n+      if (tupleIndexList == null) {\n+        tupleIndexList = new IntArrayList();\n+        uniqueTupleIndices.put(cntHashCode, tupleIndexList);\n+      }\n+      doCount.found = false;\n+      doCount.meet = false;\n+      doCount.sourceRow = i;\n+      tupleIndexList.forEach(doCount);\n+      if (!doCount.found) {\n+        for (int j : keyColIndices) {\n+          uniqueTuples.put(j, columns.get(j), i);\n+        }\n+        tupleCounts.put(0, new IntArrayColumn(new int[] { 1 }, 1), 0);\n+        tupleIndexList.add(nextIndex);\n+        if (threshold <= 1) {\n+          doCount.meet = true;\n+        }\n+      }\n+      if (!doCount.meet) {\n+        toRemove.set(i);\n+      }\n+    }\n+    return tb.filterOut(toRemove).selectColumns(keyColIndices);\n+  }\n+\n+  @Override\n+  public Schema getSchema() {\n+    Schema schema = getOp().getInputSchema();\n+    if (schema == null) {\n+      return null;\n+    }\n+    return schema.getSubSchema(keyColIndices);\n+  }\n+\n+  @Override\n+  public void init(final ImmutableMap<String, Object> execEnvVars) {\n+    uniqueTupleIndices = new IntObjectHashMap<IntArrayList>();","path":"src/edu/washington/escience/myria/operator/CountFilter.java","position":131,"original_position":131,"commit_id":"3ed82cf9dbff557324ca28cad38225d7b51eacb1","original_commit_id":"3ed82cf9dbff557324ca28cad38225d7b51eacb1","user":{"login":"senderista","id":131425,"avatar_url":"https://avatars.githubusercontent.com/u/131425?v=3","gravatar_id":"","url":"https://api.github.com/users/senderista","html_url":"https://github.com/senderista","followers_url":"https://api.github.com/users/senderista/followers","following_url":"https://api.github.com/users/senderista/following{/other_user}","gists_url":"https://api.github.com/users/senderista/gists{/gist_id}","starred_url":"https://api.github.com/users/senderista/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/senderista/subscriptions","organizations_url":"https://api.github.com/users/senderista/orgs","repos_url":"https://api.github.com/users/senderista/repos","events_url":"https://api.github.com/users/senderista/events{/privacy}","received_events_url":"https://api.github.com/users/senderista/received_events","type":"User","site_admin":false},"body":"You can ignore these comments for now; I opened https://github.com/uwescience/myria/issues/787 to address this issue.","created_at":"2015-12-01T00:01:37Z","updated_at":"2015-12-01T00:01:37Z","html_url":"https://github.com/uwescience/myria/pull/776#discussion_r46223232","pull_request_url":"https://api.github.com/repos/uwescience/myria/pulls/776","_links":{"self":{"href":"https://api.github.com/repos/uwescience/myria/pulls/comments/46223232"},"html":{"href":"https://github.com/uwescience/myria/pull/776#discussion_r46223232"},"pull_request":{"href":"https://api.github.com/repos/uwescience/myria/pulls/776"}}},"pull_request":{"url":"https://api.github.com/repos/uwescience/myria/pulls/776","id":45524361,"html_url":"https://github.com/uwescience/myria/pull/776","diff_url":"https://github.com/uwescience/myria/pull/776.diff","patch_url":"https://github.com/uwescience/myria/pull/776.patch","issue_url":"https://api.github.com/repos/uwescience/myria/issues/776","number":776,"state":"open","locked":false,"title":"Add count filter","user":{"login":"jingjingwang","id":2280236,"avatar_url":"https://avatars.githubusercontent.com/u/2280236?v=3","gravatar_id":"","url":"https://api.github.com/users/jingjingwang","html_url":"https://github.com/jingjingwang","followers_url":"https://api.github.com/users/jingjingwang/followers","following_url":"https://api.github.com/users/jingjingwang/following{/other_user}","gists_url":"https://api.github.com/users/jingjingwang/gists{/gist_id}","starred_url":"https://api.github.com/users/jingjingwang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jingjingwang/subscriptions","organizations_url":"https://api.github.com/users/jingjingwang/orgs","repos_url":"https://api.github.com/users/jingjingwang/repos","events_url":"https://api.github.com/users/jingjingwang/events{/privacy}","received_events_url":"https://api.github.com/users/jingjingwang/received_events","type":"User","site_admin":false},"body":"A CountFilter StreamingState keeps distinct tuples with their counts and only emits a tuple at the first time when its count hits the threshold. It's designed for serving an IDB with `count` as the aggregate function and a `>= threshold` condition. ","created_at":"2015-09-21T08:37:52Z","updated_at":"2015-12-01T00:01:38Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bc293efc9272b325f139c5ae22586a4a14aa9c95","assignee":{"login":"senderista","id":131425,"avatar_url":"https://avatars.githubusercontent.com/u/131425?v=3","gravatar_id":"","url":"https://api.github.com/users/senderista","html_url":"https://github.com/senderista","followers_url":"https://api.github.com/users/senderista/followers","following_url":"https://api.github.com/users/senderista/following{/other_user}","gists_url":"https://api.github.com/users/senderista/gists{/gist_id}","starred_url":"https://api.github.com/users/senderista/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/senderista/subscriptions","organizations_url":"https://api.github.com/users/senderista/orgs","repos_url":"https://api.github.com/users/senderista/repos","events_url":"https://api.github.com/users/senderista/events{/privacy}","received_events_url":"https://api.github.com/users/senderista/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/uwescience/myria/pulls/776/commits","review_comments_url":"https://api.github.com/repos/uwescience/myria/pulls/776/comments","review_comment_url":"https://api.github.com/repos/uwescience/myria/pulls/comments{/number}","comments_url":"https://api.github.com/repos/uwescience/myria/issues/776/comments","statuses_url":"https://api.github.com/repos/uwescience/myria/statuses/3ed82cf9dbff557324ca28cad38225d7b51eacb1","head":{"label":"uwescience:add_count_filter","ref":"add_count_filter","sha":"3ed82cf9dbff557324ca28cad38225d7b51eacb1","user":{"login":"uwescience","id":2243113,"avatar_url":"https://avatars.githubusercontent.com/u/2243113?v=3","gravatar_id":"","url":"https://api.github.com/users/uwescience","html_url":"https://github.com/uwescience","followers_url":"https://api.github.com/users/uwescience/followers","following_url":"https://api.github.com/users/uwescience/following{/other_user}","gists_url":"https://api.github.com/users/uwescience/gists{/gist_id}","starred_url":"https://api.github.com/users/uwescience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uwescience/subscriptions","organizations_url":"https://api.github.com/users/uwescience/orgs","repos_url":"https://api.github.com/users/uwescience/repos","events_url":"https://api.github.com/users/uwescience/events{/privacy}","received_events_url":"https://api.github.com/users/uwescience/received_events","type":"Organization","site_admin":false},"repo":{"id":5612317,"name":"myria","full_name":"uwescience/myria","owner":{"login":"uwescience","id":2243113,"avatar_url":"https://avatars.githubusercontent.com/u/2243113?v=3","gravatar_id":"","url":"https://api.github.com/users/uwescience","html_url":"https://github.com/uwescience","followers_url":"https://api.github.com/users/uwescience/followers","following_url":"https://api.github.com/users/uwescience/following{/other_user}","gists_url":"https://api.github.com/users/uwescience/gists{/gist_id}","starred_url":"https://api.github.com/users/uwescience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uwescience/subscriptions","organizations_url":"https://api.github.com/users/uwescience/orgs","repos_url":"https://api.github.com/users/uwescience/repos","events_url":"https://api.github.com/users/uwescience/events{/privacy}","received_events_url":"https://api.github.com/users/uwescience/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/uwescience/myria","description":"Myria is a scalable Analytics-as-a-Service platform based on relational algebra.","fork":false,"url":"https://api.github.com/repos/uwescience/myria","forks_url":"https://api.github.com/repos/uwescience/myria/forks","keys_url":"https://api.github.com/repos/uwescience/myria/keys{/key_id}","collaborators_url":"https://api.github.com/repos/uwescience/myria/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/uwescience/myria/teams","hooks_url":"https://api.github.com/repos/uwescience/myria/hooks","issue_events_url":"https://api.github.com/repos/uwescience/myria/issues/events{/number}","events_url":"https://api.github.com/repos/uwescience/myria/events","assignees_url":"https://api.github.com/repos/uwescience/myria/assignees{/user}","branches_url":"https://api.github.com/repos/uwescience/myria/branches{/branch}","tags_url":"https://api.github.com/repos/uwescience/myria/tags","blobs_url":"https://api.github.com/repos/uwescience/myria/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/uwescience/myria/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/uwescience/myria/git/refs{/sha}","trees_url":"https://api.github.com/repos/uwescience/myria/git/trees{/sha}","statuses_url":"https://api.github.com/repos/uwescience/myria/statuses/{sha}","languages_url":"https://api.github.com/repos/uwescience/myria/languages","stargazers_url":"https://api.github.com/repos/uwescience/myria/stargazers","contributors_url":"https://api.github.com/repos/uwescience/myria/contributors","subscribers_url":"https://api.github.com/repos/uwescience/myria/subscribers","subscription_url":"https://api.github.com/repos/uwescience/myria/subscription","commits_url":"https://api.github.com/repos/uwescience/myria/commits{/sha}","git_commits_url":"https://api.github.com/repos/uwescience/myria/git/commits{/sha}","comments_url":"https://api.github.com/repos/uwescience/myria/comments{/number}","issue_comment_url":"https://api.github.com/repos/uwescience/myria/issues/comments{/number}","contents_url":"https://api.github.com/repos/uwescience/myria/contents/{+path}","compare_url":"https://api.github.com/repos/uwescience/myria/compare/{base}...{head}","merges_url":"https://api.github.com/repos/uwescience/myria/merges","archive_url":"https://api.github.com/repos/uwescience/myria/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/uwescience/myria/downloads","issues_url":"https://api.github.com/repos/uwescience/myria/issues{/number}","pulls_url":"https://api.github.com/repos/uwescience/myria/pulls{/number}","milestones_url":"https://api.github.com/repos/uwescience/myria/milestones{/number}","notifications_url":"https://api.github.com/repos/uwescience/myria/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/uwescience/myria/labels{/name}","releases_url":"https://api.github.com/repos/uwescience/myria/releases{/id}","created_at":"2012-08-30T08:27:36Z","updated_at":"2015-11-26T08:01:36Z","pushed_at":"2015-11-30T19:29:29Z","git_url":"git://github.com/uwescience/myria.git","ssh_url":"git@github.com:uwescience/myria.git","clone_url":"https://github.com/uwescience/myria.git","svn_url":"https://github.com/uwescience/myria","homepage":"myria.cs.washington.edu","size":51848,"stargazers_count":31,"watchers_count":31,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":20,"mirror_url":null,"open_issues_count":83,"forks":20,"open_issues":83,"watchers":31,"default_branch":"master"}},"base":{"label":"uwescience:master","ref":"master","sha":"666b3adca58a81ff32c3a59b97357f7692274620","user":{"login":"uwescience","id":2243113,"avatar_url":"https://avatars.githubusercontent.com/u/2243113?v=3","gravatar_id":"","url":"https://api.github.com/users/uwescience","html_url":"https://github.com/uwescience","followers_url":"https://api.github.com/users/uwescience/followers","following_url":"https://api.github.com/users/uwescience/following{/other_user}","gists_url":"https://api.github.com/users/uwescience/gists{/gist_id}","starred_url":"https://api.github.com/users/uwescience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uwescience/subscriptions","organizations_url":"https://api.github.com/users/uwescience/orgs","repos_url":"https://api.github.com/users/uwescience/repos","events_url":"https://api.github.com/users/uwescience/events{/privacy}","received_events_url":"https://api.github.com/users/uwescience/received_events","type":"Organization","site_admin":false},"repo":{"id":5612317,"name":"myria","full_name":"uwescience/myria","owner":{"login":"uwescience","id":2243113,"avatar_url":"https://avatars.githubusercontent.com/u/2243113?v=3","gravatar_id":"","url":"https://api.github.com/users/uwescience","html_url":"https://github.com/uwescience","followers_url":"https://api.github.com/users/uwescience/followers","following_url":"https://api.github.com/users/uwescience/following{/other_user}","gists_url":"https://api.github.com/users/uwescience/gists{/gist_id}","starred_url":"https://api.github.com/users/uwescience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uwescience/subscriptions","organizations_url":"https://api.github.com/users/uwescience/orgs","repos_url":"https://api.github.com/users/uwescience/repos","events_url":"https://api.github.com/users/uwescience/events{/privacy}","received_events_url":"https://api.github.com/users/uwescience/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/uwescience/myria","description":"Myria is a scalable Analytics-as-a-Service platform based on relational algebra.","fork":false,"url":"https://api.github.com/repos/uwescience/myria","forks_url":"https://api.github.com/repos/uwescience/myria/forks","keys_url":"https://api.github.com/repos/uwescience/myria/keys{/key_id}","collaborators_url":"https://api.github.com/repos/uwescience/myria/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/uwescience/myria/teams","hooks_url":"https://api.github.com/repos/uwescience/myria/hooks","issue_events_url":"https://api.github.com/repos/uwescience/myria/issues/events{/number}","events_url":"https://api.github.com/repos/uwescience/myria/events","assignees_url":"https://api.github.com/repos/uwescience/myria/assignees{/user}","branches_url":"https://api.github.com/repos/uwescience/myria/branches{/branch}","tags_url":"https://api.github.com/repos/uwescience/myria/tags","blobs_url":"https://api.github.com/repos/uwescience/myria/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/uwescience/myria/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/uwescience/myria/git/refs{/sha}","trees_url":"https://api.github.com/repos/uwescience/myria/git/trees{/sha}","statuses_url":"https://api.github.com/repos/uwescience/myria/statuses/{sha}","languages_url":"https://api.github.com/repos/uwescience/myria/languages","stargazers_url":"https://api.github.com/repos/uwescience/myria/stargazers","contributors_url":"https://api.github.com/repos/uwescience/myria/contributors","subscribers_url":"https://api.github.com/repos/uwescience/myria/subscribers","subscription_url":"https://api.github.com/repos/uwescience/myria/subscription","commits_url":"https://api.github.com/repos/uwescience/myria/commits{/sha}","git_commits_url":"https://api.github.com/repos/uwescience/myria/git/commits{/sha}","comments_url":"https://api.github.com/repos/uwescience/myria/comments{/number}","issue_comment_url":"https://api.github.com/repos/uwescience/myria/issues/comments{/number}","contents_url":"https://api.github.com/repos/uwescience/myria/contents/{+path}","compare_url":"https://api.github.com/repos/uwescience/myria/compare/{base}...{head}","merges_url":"https://api.github.com/repos/uwescience/myria/merges","archive_url":"https://api.github.com/repos/uwescience/myria/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/uwescience/myria/downloads","issues_url":"https://api.github.com/repos/uwescience/myria/issues{/number}","pulls_url":"https://api.github.com/repos/uwescience/myria/pulls{/number}","milestones_url":"https://api.github.com/repos/uwescience/myria/milestones{/number}","notifications_url":"https://api.github.com/repos/uwescience/myria/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/uwescience/myria/labels{/name}","releases_url":"https://api.github.com/repos/uwescience/myria/releases{/id}","created_at":"2012-08-30T08:27:36Z","updated_at":"2015-11-26T08:01:36Z","pushed_at":"2015-11-30T19:29:29Z","git_url":"git://github.com/uwescience/myria.git","ssh_url":"git@github.com:uwescience/myria.git","clone_url":"https://github.com/uwescience/myria.git","svn_url":"https://github.com/uwescience/myria","homepage":"myria.cs.washington.edu","size":51848,"stargazers_count":31,"watchers_count":31,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":20,"mirror_url":null,"open_issues_count":83,"forks":20,"open_issues":83,"watchers":31,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/uwescience/myria/pulls/776"},"html":{"href":"https://github.com/uwescience/myria/pull/776"},"issue":{"href":"https://api.github.com/repos/uwescience/myria/issues/776"},"comments":{"href":"https://api.github.com/repos/uwescience/myria/issues/776/comments"},"review_comments":{"href":"https://api.github.com/repos/uwescience/myria/pulls/776/comments"},"review_comment":{"href":"https://api.github.com/repos/uwescience/myria/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/uwescience/myria/pulls/776/commits"},"statuses":{"href":"https://api.github.com/repos/uwescience/myria/statuses/3ed82cf9dbff557324ca28cad38225d7b51eacb1"}}}}