{"action":"created","comment":{"url":"https://api.github.com/repos/tuplejump/FiloDB/pulls/comments/45644522","id":45644522,"diff_hunk":"@@ -80,3 +80,24 @@ case class DoubleKeyHelper(segmentLen: Double) extends SingleSortKeyHelper[Doubl\n     java.lang.Double.longBitsToDouble(bytes.toLong(true, ByteOrdering.BigEndian))\n }\n \n+/**\n+ * Right now, you have to specify a prefixLen for the string key helper.\n+ * All string keys with identical prefixes (characters 0 to prefixLen - 1) will then be\n+ * bucketed into the same segment.\n+ * Thus one needs to look through your string sort key data and make sure to bucketize\n+ * correctly.\n+ *\n+ * TODO: perhaps combine prefixLen with a range of chars the last char is allowed to vary on\n+ */\n+case class StringKeyHelper(prefixLen: Int) extends SingleSortKeyHelper[String] {","path":"core/src/main/scala/filodb.core/SortKeyHelper.scala","position":22,"original_position":22,"commit_id":"76c5a7ee5e7f3b5b7306e464303642862c948a68","original_commit_id":"76c5a7ee5e7f3b5b7306e464303642862c948a68","user":{"login":"helena","id":406103,"avatar_url":"https://avatars.githubusercontent.com/u/406103?v=3","gravatar_id":"","url":"https://api.github.com/users/helena","html_url":"https://github.com/helena","followers_url":"https://api.github.com/users/helena/followers","following_url":"https://api.github.com/users/helena/following{/other_user}","gists_url":"https://api.github.com/users/helena/gists{/gist_id}","starred_url":"https://api.github.com/users/helena/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/helena/subscriptions","organizations_url":"https://api.github.com/users/helena/orgs","repos_url":"https://api.github.com/users/helena/repos","events_url":"https://api.github.com/users/helena/events{/privacy}","received_events_url":"https://api.github.com/users/helena/received_events","type":"User","site_admin":false},"body":"Might be re-usable around the filodb codbase though and belong outside this type.","created_at":"2015-11-23T18:59:50Z","updated_at":"2015-11-23T18:59:50Z","html_url":"https://github.com/tuplejump/FiloDB/pull/46#discussion_r45644522","pull_request_url":"https://api.github.com/repos/tuplejump/FiloDB/pulls/46","_links":{"self":{"href":"https://api.github.com/repos/tuplejump/FiloDB/pulls/comments/45644522"},"html":{"href":"https://github.com/tuplejump/FiloDB/pull/46#discussion_r45644522"},"pull_request":{"href":"https://api.github.com/repos/tuplejump/FiloDB/pulls/46"}}},"pull_request":{"url":"https://api.github.com/repos/tuplejump/FiloDB/pulls/46","id":51050280,"html_url":"https://github.com/tuplejump/FiloDB/pull/46","diff_url":"https://github.com/tuplejump/FiloDB/pull/46.diff","patch_url":"https://github.com/tuplejump/FiloDB/pull/46.patch","issue_url":"https://api.github.com/repos/tuplejump/FiloDB/issues/46","number":46,"state":"open","locked":false,"title":"Performance and Ingestion Improvements","user":{"login":"velvia","id":1062875,"avatar_url":"https://avatars.githubusercontent.com/u/1062875?v=3","gravatar_id":"","url":"https://api.github.com/users/velvia","html_url":"https://github.com/velvia","followers_url":"https://api.github.com/users/velvia/followers","following_url":"https://api.github.com/users/velvia/following{/other_user}","gists_url":"https://api.github.com/users/velvia/gists{/gist_id}","starred_url":"https://api.github.com/users/velvia/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/velvia/subscriptions","organizations_url":"https://api.github.com/users/velvia/orgs","repos_url":"https://api.github.com/users/velvia/repos","events_url":"https://api.github.com/users/velvia/events{/privacy}","received_events_url":"https://api.github.com/users/velvia/received_events","type":"User","site_admin":false},"body":"This includes a bunch of important performance and ingestion improvements;  in short - ingestion is MUCH faster, stable, and exact;  query is faster due to concurrency working.\r\n\r\nThe biggest change, towards the end, is a new `FiloMemTable` which uses Filo vectors to store data in the memtable; it is significantly faster than the MapDB-based MemTable for both writing and reading.\r\n\r\nOther changes:\r\n- Added a `StringKeyHelper` and allow String as a sort key type.  The segment key is derived as the first `segment_size` characters of the sort key.\r\n- A new CLI command, analyze exists to analyze the distribution of segments and chunks in a dataset\r\n- Reading efficiency of segment indexes has been improved\r\n- splits_per_node has been fixed and defaults to 4 splits, improving most query times; before it was limited to effectively 1 partition/thread per node\r\n- Fix for #38 \r\n- Two fixes for `RowSource` so that Spark ingestion no longer hangs at the end, and is at-least-once by design","created_at":"2015-11-18T07:18:06Z","updated_at":"2015-11-23T18:59:50Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bf77c9777a121538094261f15a388a5645bedf96","assignee":{"login":"helena","id":406103,"avatar_url":"https://avatars.githubusercontent.com/u/406103?v=3","gravatar_id":"","url":"https://api.github.com/users/helena","html_url":"https://github.com/helena","followers_url":"https://api.github.com/users/helena/followers","following_url":"https://api.github.com/users/helena/following{/other_user}","gists_url":"https://api.github.com/users/helena/gists{/gist_id}","starred_url":"https://api.github.com/users/helena/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/helena/subscriptions","organizations_url":"https://api.github.com/users/helena/orgs","repos_url":"https://api.github.com/users/helena/repos","events_url":"https://api.github.com/users/helena/events{/privacy}","received_events_url":"https://api.github.com/users/helena/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/tuplejump/FiloDB/pulls/46/commits","review_comments_url":"https://api.github.com/repos/tuplejump/FiloDB/pulls/46/comments","review_comment_url":"https://api.github.com/repos/tuplejump/FiloDB/pulls/comments{/number}","comments_url":"https://api.github.com/repos/tuplejump/FiloDB/issues/46/comments","statuses_url":"https://api.github.com/repos/tuplejump/FiloDB/statuses/76c5a7ee5e7f3b5b7306e464303642862c948a68","head":{"label":"tuplejump:velvia/misc-perf-changes","ref":"velvia/misc-perf-changes","sha":"76c5a7ee5e7f3b5b7306e464303642862c948a68","user":{"login":"tuplejump","id":3493976,"avatar_url":"https://avatars.githubusercontent.com/u/3493976?v=3","gravatar_id":"","url":"https://api.github.com/users/tuplejump","html_url":"https://github.com/tuplejump","followers_url":"https://api.github.com/users/tuplejump/followers","following_url":"https://api.github.com/users/tuplejump/following{/other_user}","gists_url":"https://api.github.com/users/tuplejump/gists{/gist_id}","starred_url":"https://api.github.com/users/tuplejump/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tuplejump/subscriptions","organizations_url":"https://api.github.com/users/tuplejump/orgs","repos_url":"https://api.github.com/users/tuplejump/repos","events_url":"https://api.github.com/users/tuplejump/events{/privacy}","received_events_url":"https://api.github.com/users/tuplejump/received_events","type":"Organization","site_admin":false},"repo":{"id":29259281,"name":"FiloDB","full_name":"tuplejump/FiloDB","owner":{"login":"tuplejump","id":3493976,"avatar_url":"https://avatars.githubusercontent.com/u/3493976?v=3","gravatar_id":"","url":"https://api.github.com/users/tuplejump","html_url":"https://github.com/tuplejump","followers_url":"https://api.github.com/users/tuplejump/followers","following_url":"https://api.github.com/users/tuplejump/following{/other_user}","gists_url":"https://api.github.com/users/tuplejump/gists{/gist_id}","starred_url":"https://api.github.com/users/tuplejump/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tuplejump/subscriptions","organizations_url":"https://api.github.com/users/tuplejump/orgs","repos_url":"https://api.github.com/users/tuplejump/repos","events_url":"https://api.github.com/users/tuplejump/events{/privacy}","received_events_url":"https://api.github.com/users/tuplejump/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/tuplejump/FiloDB","description":"Distributed.  Columnar.  Versioned.  Streaming.  SQL.","fork":false,"url":"https://api.github.com/repos/tuplejump/FiloDB","forks_url":"https://api.github.com/repos/tuplejump/FiloDB/forks","keys_url":"https://api.github.com/repos/tuplejump/FiloDB/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tuplejump/FiloDB/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tuplejump/FiloDB/teams","hooks_url":"https://api.github.com/repos/tuplejump/FiloDB/hooks","issue_events_url":"https://api.github.com/repos/tuplejump/FiloDB/issues/events{/number}","events_url":"https://api.github.com/repos/tuplejump/FiloDB/events","assignees_url":"https://api.github.com/repos/tuplejump/FiloDB/assignees{/user}","branches_url":"https://api.github.com/repos/tuplejump/FiloDB/branches{/branch}","tags_url":"https://api.github.com/repos/tuplejump/FiloDB/tags","blobs_url":"https://api.github.com/repos/tuplejump/FiloDB/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tuplejump/FiloDB/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tuplejump/FiloDB/git/refs{/sha}","trees_url":"https://api.github.com/repos/tuplejump/FiloDB/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tuplejump/FiloDB/statuses/{sha}","languages_url":"https://api.github.com/repos/tuplejump/FiloDB/languages","stargazers_url":"https://api.github.com/repos/tuplejump/FiloDB/stargazers","contributors_url":"https://api.github.com/repos/tuplejump/FiloDB/contributors","subscribers_url":"https://api.github.com/repos/tuplejump/FiloDB/subscribers","subscription_url":"https://api.github.com/repos/tuplejump/FiloDB/subscription","commits_url":"https://api.github.com/repos/tuplejump/FiloDB/commits{/sha}","git_commits_url":"https://api.github.com/repos/tuplejump/FiloDB/git/commits{/sha}","comments_url":"https://api.github.com/repos/tuplejump/FiloDB/comments{/number}","issue_comment_url":"https://api.github.com/repos/tuplejump/FiloDB/issues/comments{/number}","contents_url":"https://api.github.com/repos/tuplejump/FiloDB/contents/{+path}","compare_url":"https://api.github.com/repos/tuplejump/FiloDB/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tuplejump/FiloDB/merges","archive_url":"https://api.github.com/repos/tuplejump/FiloDB/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tuplejump/FiloDB/downloads","issues_url":"https://api.github.com/repos/tuplejump/FiloDB/issues{/number}","pulls_url":"https://api.github.com/repos/tuplejump/FiloDB/pulls{/number}","milestones_url":"https://api.github.com/repos/tuplejump/FiloDB/milestones{/number}","notifications_url":"https://api.github.com/repos/tuplejump/FiloDB/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tuplejump/FiloDB/labels{/name}","releases_url":"https://api.github.com/repos/tuplejump/FiloDB/releases{/id}","created_at":"2015-01-14T18:35:32Z","updated_at":"2015-11-22T01:04:28Z","pushed_at":"2015-11-23T09:17:16Z","git_url":"git://github.com/tuplejump/FiloDB.git","ssh_url":"git@github.com:tuplejump/FiloDB.git","clone_url":"https://github.com/tuplejump/FiloDB.git","svn_url":"https://github.com/tuplejump/FiloDB","homepage":"","size":4638,"stargazers_count":226,"watchers_count":226,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":42,"mirror_url":null,"open_issues_count":16,"forks":42,"open_issues":16,"watchers":226,"default_branch":"master"}},"base":{"label":"tuplejump:master","ref":"master","sha":"ff0d67a345292094d1bcde51ce622a964d531276","user":{"login":"tuplejump","id":3493976,"avatar_url":"https://avatars.githubusercontent.com/u/3493976?v=3","gravatar_id":"","url":"https://api.github.com/users/tuplejump","html_url":"https://github.com/tuplejump","followers_url":"https://api.github.com/users/tuplejump/followers","following_url":"https://api.github.com/users/tuplejump/following{/other_user}","gists_url":"https://api.github.com/users/tuplejump/gists{/gist_id}","starred_url":"https://api.github.com/users/tuplejump/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tuplejump/subscriptions","organizations_url":"https://api.github.com/users/tuplejump/orgs","repos_url":"https://api.github.com/users/tuplejump/repos","events_url":"https://api.github.com/users/tuplejump/events{/privacy}","received_events_url":"https://api.github.com/users/tuplejump/received_events","type":"Organization","site_admin":false},"repo":{"id":29259281,"name":"FiloDB","full_name":"tuplejump/FiloDB","owner":{"login":"tuplejump","id":3493976,"avatar_url":"https://avatars.githubusercontent.com/u/3493976?v=3","gravatar_id":"","url":"https://api.github.com/users/tuplejump","html_url":"https://github.com/tuplejump","followers_url":"https://api.github.com/users/tuplejump/followers","following_url":"https://api.github.com/users/tuplejump/following{/other_user}","gists_url":"https://api.github.com/users/tuplejump/gists{/gist_id}","starred_url":"https://api.github.com/users/tuplejump/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tuplejump/subscriptions","organizations_url":"https://api.github.com/users/tuplejump/orgs","repos_url":"https://api.github.com/users/tuplejump/repos","events_url":"https://api.github.com/users/tuplejump/events{/privacy}","received_events_url":"https://api.github.com/users/tuplejump/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/tuplejump/FiloDB","description":"Distributed.  Columnar.  Versioned.  Streaming.  SQL.","fork":false,"url":"https://api.github.com/repos/tuplejump/FiloDB","forks_url":"https://api.github.com/repos/tuplejump/FiloDB/forks","keys_url":"https://api.github.com/repos/tuplejump/FiloDB/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tuplejump/FiloDB/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tuplejump/FiloDB/teams","hooks_url":"https://api.github.com/repos/tuplejump/FiloDB/hooks","issue_events_url":"https://api.github.com/repos/tuplejump/FiloDB/issues/events{/number}","events_url":"https://api.github.com/repos/tuplejump/FiloDB/events","assignees_url":"https://api.github.com/repos/tuplejump/FiloDB/assignees{/user}","branches_url":"https://api.github.com/repos/tuplejump/FiloDB/branches{/branch}","tags_url":"https://api.github.com/repos/tuplejump/FiloDB/tags","blobs_url":"https://api.github.com/repos/tuplejump/FiloDB/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tuplejump/FiloDB/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tuplejump/FiloDB/git/refs{/sha}","trees_url":"https://api.github.com/repos/tuplejump/FiloDB/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tuplejump/FiloDB/statuses/{sha}","languages_url":"https://api.github.com/repos/tuplejump/FiloDB/languages","stargazers_url":"https://api.github.com/repos/tuplejump/FiloDB/stargazers","contributors_url":"https://api.github.com/repos/tuplejump/FiloDB/contributors","subscribers_url":"https://api.github.com/repos/tuplejump/FiloDB/subscribers","subscription_url":"https://api.github.com/repos/tuplejump/FiloDB/subscription","commits_url":"https://api.github.com/repos/tuplejump/FiloDB/commits{/sha}","git_commits_url":"https://api.github.com/repos/tuplejump/FiloDB/git/commits{/sha}","comments_url":"https://api.github.com/repos/tuplejump/FiloDB/comments{/number}","issue_comment_url":"https://api.github.com/repos/tuplejump/FiloDB/issues/comments{/number}","contents_url":"https://api.github.com/repos/tuplejump/FiloDB/contents/{+path}","compare_url":"https://api.github.com/repos/tuplejump/FiloDB/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tuplejump/FiloDB/merges","archive_url":"https://api.github.com/repos/tuplejump/FiloDB/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tuplejump/FiloDB/downloads","issues_url":"https://api.github.com/repos/tuplejump/FiloDB/issues{/number}","pulls_url":"https://api.github.com/repos/tuplejump/FiloDB/pulls{/number}","milestones_url":"https://api.github.com/repos/tuplejump/FiloDB/milestones{/number}","notifications_url":"https://api.github.com/repos/tuplejump/FiloDB/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tuplejump/FiloDB/labels{/name}","releases_url":"https://api.github.com/repos/tuplejump/FiloDB/releases{/id}","created_at":"2015-01-14T18:35:32Z","updated_at":"2015-11-22T01:04:28Z","pushed_at":"2015-11-23T09:17:16Z","git_url":"git://github.com/tuplejump/FiloDB.git","ssh_url":"git@github.com:tuplejump/FiloDB.git","clone_url":"https://github.com/tuplejump/FiloDB.git","svn_url":"https://github.com/tuplejump/FiloDB","homepage":"","size":4638,"stargazers_count":226,"watchers_count":226,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":42,"mirror_url":null,"open_issues_count":16,"forks":42,"open_issues":16,"watchers":226,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/tuplejump/FiloDB/pulls/46"},"html":{"href":"https://github.com/tuplejump/FiloDB/pull/46"},"issue":{"href":"https://api.github.com/repos/tuplejump/FiloDB/issues/46"},"comments":{"href":"https://api.github.com/repos/tuplejump/FiloDB/issues/46/comments"},"review_comments":{"href":"https://api.github.com/repos/tuplejump/FiloDB/pulls/46/comments"},"review_comment":{"href":"https://api.github.com/repos/tuplejump/FiloDB/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/tuplejump/FiloDB/pulls/46/commits"},"statuses":{"href":"https://api.github.com/repos/tuplejump/FiloDB/statuses/76c5a7ee5e7f3b5b7306e464303642862c948a68"}}}}