{"action":"created","comment":{"url":"https://api.github.com/repos/influxdb/influxdb/pulls/comments/41669649","id":41669649,"diff_hunk":"@@ -0,0 +1,204 @@\n+package subscriber\n+\n+import (\n+\t\"expvar\"\n+\t\"fmt\"\n+\t\"log\"\n+\t\"net/url\"\n+\t\"os\"\n+\t\"time\"\n+\n+\t\"github.com/influxdb/influxdb\"\n+\t\"github.com/influxdb/influxdb/cluster\"\n+\t\"github.com/influxdb/influxdb/meta\"\n+)\n+\n+// Statistics for the CQ service.\n+const (\n+\tstatPointsWritten = \"points_written\"\n+)\n+\n+type PointsWriter interface {\n+\tWritePoints(p *cluster.WritePointsRequest) error\n+}\n+\n+// unique set that identifies a given subscription\n+type subEntry struct {\n+\tdb   string\n+\trp   string\n+\tname string\n+}\n+\n+type Service struct {\n+\tsubs      map[subEntry]PointsWriter\n+\tMetaStore interface {\n+\t\tDatabases() ([]meta.DatabaseInfo, error)\n+\t}\n+\tNewPointsWriter func(u url.URL) (PointsWriter, error)\n+\tLogger          *log.Logger\n+\tstatMap         *expvar.Map\n+\tticker          *time.Ticker\n+}\n+\n+func NewService() *Service {\n+\treturn &Service{\n+\t\tsubs:            make(map[subEntry]PointsWriter),\n+\t\tNewPointsWriter: newPointsWriter,\n+\t\tLogger:          log.New(os.Stderr, \"[subscriber] \", log.LstdFlags),\n+\t\tstatMap:         influxdb.NewStatistics(\"subscriber\", \"subscriber\", nil),\n+\t}\n+}\n+\n+func (s *Service) Open() error {\n+\tif s.MetaStore == nil {\n+\t\tpanic(\"no meta store\")\n+\t}\n+\ts.Update()\n+\tgo s.pollForMetaUpdates()\n+\n+\treturn nil\n+}\n+\n+func (s *Service) Close() error {\n+\tif s.ticker != nil {\n+\t\ts.ticker.Stop()\n+\t}\n+\treturn nil\n+}\n+\n+func (s *Service) pollForMetaUpdates() {\n+\ts.ticker = time.NewTicker(time.Minute)\n+\tfor range s.ticker.C {\n+\t\ts.Update()\n+\t}\n+}\n+\n+// start new and stop deleted subscriptions.\n+func (s *Service) Update() error {\n+\tdbis, err := s.MetaStore.Databases()\n+\tif err != nil {\n+\t\treturn err\n+\t}\n+\tallEntries := make(map[subEntry]bool, 0)\n+\t// Add in new subscriptions\n+\tfor _, dbi := range dbis {\n+\t\tfor _, rpi := range dbi.RetentionPolicies {\n+\t\t\tfor _, si := range rpi.Subscriptions {\n+\t\t\t\tse := subEntry{\n+\t\t\t\t\tdb:   dbi.Name,\n+\t\t\t\t\trp:   rpi.Name,\n+\t\t\t\t\tname: si.Name,\n+\t\t\t\t}\n+\t\t\t\tallEntries[se] = true\n+\t\t\t\tif _, ok := s.subs[se]; ok {\n+\t\t\t\t\tcontinue\n+\t\t\t\t}\n+\t\t\t\tvar bm BalanceMode\n+\t\t\t\tswitch si.Mode {\n+\t\t\t\tcase \"ALL\":\n+\t\t\t\t\tbm = ALL\n+\t\t\t\tcase \"ANY\":\n+\t\t\t\t\tbm = ANY\n+\t\t\t\tdefault:\n+\t\t\t\t\treturn fmt.Errorf(\"unknown balance mode %q\", si.Mode)\n+\t\t\t\t}\n+\t\t\t\twriters := make([]PointsWriter, len(si.Destinations))\n+\t\t\t\tstatMaps := make([]*expvar.Map, len(writers))\n+\t\t\t\tfor i, dest := range si.Destinations {\n+\t\t\t\t\tu, err := url.Parse(dest)\n+\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\treturn err\n+\t\t\t\t\t}\n+\t\t\t\t\tw, err := s.NewPointsWriter(*u)\n+\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\treturn err\n+\t\t\t\t\t}\n+\t\t\t\t\twriters[i] = w\n+\t\t\t\t\ttags := map[string]string{\"database\": se.db, \"retention_policy\": se.rp, \"name\": se.name, \"mode\": si.Mode}\n+\t\t\t\t\tstatMaps[i] = influxdb.NewStatistics(dest, \"subscriber\", tags)","path":"services/subscriber/service.go","position":118,"original_position":118,"commit_id":"f79e1e532001b76a8e477efa69f86b3967e83e19","original_commit_id":"f79e1e532001b76a8e477efa69f86b3967e83e19","user":{"login":"nathanielc","id":3771906,"avatar_url":"https://avatars.githubusercontent.com/u/3771906?v=3","gravatar_id":"","url":"https://api.github.com/users/nathanielc","html_url":"https://github.com/nathanielc","followers_url":"https://api.github.com/users/nathanielc/followers","following_url":"https://api.github.com/users/nathanielc/following{/other_user}","gists_url":"https://api.github.com/users/nathanielc/gists{/gist_id}","starred_url":"https://api.github.com/users/nathanielc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nathanielc/subscriptions","organizations_url":"https://api.github.com/users/nathanielc/orgs","repos_url":"https://api.github.com/users/nathanielc/repos","events_url":"https://api.github.com/users/nathanielc/events{/privacy}","received_events_url":"https://api.github.com/users/nathanielc/received_events","type":"User","site_admin":false},"body":"@otoolep Does this look good for the stats? I didn't add it to the UDP object since here we can tag the stats with more useful information.","created_at":"2015-10-09T19:34:50Z","updated_at":"2015-10-09T19:34:50Z","html_url":"https://github.com/influxdb/influxdb/pull/4375#discussion_r41669649","pull_request_url":"https://api.github.com/repos/influxdb/influxdb/pulls/4375","_links":{"self":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/comments/41669649"},"html":{"href":"https://github.com/influxdb/influxdb/pull/4375#discussion_r41669649"},"pull_request":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/4375"}}},"pull_request":{"url":"https://api.github.com/repos/influxdb/influxdb/pulls/4375","id":47207147,"html_url":"https://github.com/influxdb/influxdb/pull/4375","diff_url":"https://github.com/influxdb/influxdb/pull/4375.diff","patch_url":"https://github.com/influxdb/influxdb/pull/4375.patch","issue_url":"https://api.github.com/repos/influxdb/influxdb/issues/4375","number":4375,"state":"open","locked":false,"title":"Feature add subscriber service for creating/dropping subscriptions [WIP]","user":{"login":"nathanielc","id":3771906,"avatar_url":"https://avatars.githubusercontent.com/u/3771906?v=3","gravatar_id":"","url":"https://api.github.com/users/nathanielc","html_url":"https://github.com/nathanielc","followers_url":"https://api.github.com/users/nathanielc/followers","following_url":"https://api.github.com/users/nathanielc/following{/other_user}","gists_url":"https://api.github.com/users/nathanielc/gists{/gist_id}","starred_url":"https://api.github.com/users/nathanielc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nathanielc/subscriptions","organizations_url":"https://api.github.com/users/nathanielc/orgs","repos_url":"https://api.github.com/users/nathanielc/repos","events_url":"https://api.github.com/users/nathanielc/events{/privacy}","received_events_url":"https://api.github.com/users/nathanielc/received_events","type":"User","site_admin":false},"body":"Basic idea is to allow for forks or subscriptions of the data stream to be defined. A subscription is tied to database and retention policy and will send all data received to the subscription endpoint. Currently the only implementation is a UDP protocol. We do not want to create back-pressure within InfluxDB hence UDP.\r\n\r\nUsage:\r\n\r\nThe following query creates a subscription called `sub0` that forks all incoming data to database `mydb` and policy `default` to the hosts `localhost:9090` and `remotehost:9090` in a round robin fashion. If you want to send all data to both local and remote hosts create two subscriptions.\r\n\r\n`CREATE SUBSCRIPTION sub0 ON mydb.default HOSTS 'localhost:9090', 'remotehost:9090' TYPE 'udp'`\r\n\r\nThe following query drops the subscriptions:\r\n\r\n`DROP SUBSCRIPTION sub0 ON mydb.default`\r\n\r\nThe following query lists all subscriptions:\r\n\r\n`SHOW SUBSCRIPTIONS`\r\n\r\nThese subscriptions are stored in the raft meta data so the whole cluster will fork its data at each node respectively. \r\n\r\n\r\nTODO:\r\n\r\n~~Need to update the QL docs and write some basic dos on how to use subcriptions.~~\r\n~~Detect new or deleted subscriptions and start/stop them.~~\r\n","created_at":"2015-10-08T21:12:24Z","updated_at":"2015-10-09T19:34:50Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a4958ee3fe40bced5172fee40349d4a12d5a2896","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/influxdb/influxdb/pulls/4375/commits","review_comments_url":"https://api.github.com/repos/influxdb/influxdb/pulls/4375/comments","review_comment_url":"https://api.github.com/repos/influxdb/influxdb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/influxdb/influxdb/issues/4375/comments","statuses_url":"https://api.github.com/repos/influxdb/influxdb/statuses/f79e1e532001b76a8e477efa69f86b3967e83e19","head":{"label":"influxdb:subscriptions","ref":"subscriptions","sha":"f79e1e532001b76a8e477efa69f86b3967e83e19","user":{"login":"influxdb","id":5713248,"avatar_url":"https://avatars.githubusercontent.com/u/5713248?v=3","gravatar_id":"","url":"https://api.github.com/users/influxdb","html_url":"https://github.com/influxdb","followers_url":"https://api.github.com/users/influxdb/followers","following_url":"https://api.github.com/users/influxdb/following{/other_user}","gists_url":"https://api.github.com/users/influxdb/gists{/gist_id}","starred_url":"https://api.github.com/users/influxdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/influxdb/subscriptions","organizations_url":"https://api.github.com/users/influxdb/orgs","repos_url":"https://api.github.com/users/influxdb/repos","events_url":"https://api.github.com/users/influxdb/events{/privacy}","received_events_url":"https://api.github.com/users/influxdb/received_events","type":"Organization","site_admin":false},"repo":{"id":13124802,"name":"influxdb","full_name":"influxdb/influxdb","owner":{"login":"influxdb","id":5713248,"avatar_url":"https://avatars.githubusercontent.com/u/5713248?v=3","gravatar_id":"","url":"https://api.github.com/users/influxdb","html_url":"https://github.com/influxdb","followers_url":"https://api.github.com/users/influxdb/followers","following_url":"https://api.github.com/users/influxdb/following{/other_user}","gists_url":"https://api.github.com/users/influxdb/gists{/gist_id}","starred_url":"https://api.github.com/users/influxdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/influxdb/subscriptions","organizations_url":"https://api.github.com/users/influxdb/orgs","repos_url":"https://api.github.com/users/influxdb/repos","events_url":"https://api.github.com/users/influxdb/events{/privacy}","received_events_url":"https://api.github.com/users/influxdb/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/influxdb/influxdb","description":"Scalable datastore for metrics, events, and real-time analytics","fork":false,"url":"https://api.github.com/repos/influxdb/influxdb","forks_url":"https://api.github.com/repos/influxdb/influxdb/forks","keys_url":"https://api.github.com/repos/influxdb/influxdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/influxdb/influxdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/influxdb/influxdb/teams","hooks_url":"https://api.github.com/repos/influxdb/influxdb/hooks","issue_events_url":"https://api.github.com/repos/influxdb/influxdb/issues/events{/number}","events_url":"https://api.github.com/repos/influxdb/influxdb/events","assignees_url":"https://api.github.com/repos/influxdb/influxdb/assignees{/user}","branches_url":"https://api.github.com/repos/influxdb/influxdb/branches{/branch}","tags_url":"https://api.github.com/repos/influxdb/influxdb/tags","blobs_url":"https://api.github.com/repos/influxdb/influxdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/influxdb/influxdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/influxdb/influxdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/influxdb/influxdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/influxdb/influxdb/statuses/{sha}","languages_url":"https://api.github.com/repos/influxdb/influxdb/languages","stargazers_url":"https://api.github.com/repos/influxdb/influxdb/stargazers","contributors_url":"https://api.github.com/repos/influxdb/influxdb/contributors","subscribers_url":"https://api.github.com/repos/influxdb/influxdb/subscribers","subscription_url":"https://api.github.com/repos/influxdb/influxdb/subscription","commits_url":"https://api.github.com/repos/influxdb/influxdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/influxdb/influxdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/influxdb/influxdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/influxdb/influxdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/influxdb/influxdb/contents/{+path}","compare_url":"https://api.github.com/repos/influxdb/influxdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/influxdb/influxdb/merges","archive_url":"https://api.github.com/repos/influxdb/influxdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/influxdb/influxdb/downloads","issues_url":"https://api.github.com/repos/influxdb/influxdb/issues{/number}","pulls_url":"https://api.github.com/repos/influxdb/influxdb/pulls{/number}","milestones_url":"https://api.github.com/repos/influxdb/influxdb/milestones{/number}","notifications_url":"https://api.github.com/repos/influxdb/influxdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/influxdb/influxdb/labels{/name}","releases_url":"https://api.github.com/repos/influxdb/influxdb/releases{/id}","created_at":"2013-09-26T14:31:10Z","updated_at":"2015-10-09T19:30:02Z","pushed_at":"2015-10-09T19:31:52Z","git_url":"git://github.com/influxdb/influxdb.git","ssh_url":"git@github.com:influxdb/influxdb.git","clone_url":"https://github.com/influxdb/influxdb.git","svn_url":"https://github.com/influxdb/influxdb","homepage":"http://influxdb.com","size":60771,"stargazers_count":6462,"watchers_count":6462,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":718,"mirror_url":null,"open_issues_count":378,"forks":718,"open_issues":378,"watchers":6462,"default_branch":"master"}},"base":{"label":"influxdb:master","ref":"master","sha":"f2e1dfed4fabe4b57e998e31e80a673022c5f6ba","user":{"login":"influxdb","id":5713248,"avatar_url":"https://avatars.githubusercontent.com/u/5713248?v=3","gravatar_id":"","url":"https://api.github.com/users/influxdb","html_url":"https://github.com/influxdb","followers_url":"https://api.github.com/users/influxdb/followers","following_url":"https://api.github.com/users/influxdb/following{/other_user}","gists_url":"https://api.github.com/users/influxdb/gists{/gist_id}","starred_url":"https://api.github.com/users/influxdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/influxdb/subscriptions","organizations_url":"https://api.github.com/users/influxdb/orgs","repos_url":"https://api.github.com/users/influxdb/repos","events_url":"https://api.github.com/users/influxdb/events{/privacy}","received_events_url":"https://api.github.com/users/influxdb/received_events","type":"Organization","site_admin":false},"repo":{"id":13124802,"name":"influxdb","full_name":"influxdb/influxdb","owner":{"login":"influxdb","id":5713248,"avatar_url":"https://avatars.githubusercontent.com/u/5713248?v=3","gravatar_id":"","url":"https://api.github.com/users/influxdb","html_url":"https://github.com/influxdb","followers_url":"https://api.github.com/users/influxdb/followers","following_url":"https://api.github.com/users/influxdb/following{/other_user}","gists_url":"https://api.github.com/users/influxdb/gists{/gist_id}","starred_url":"https://api.github.com/users/influxdb/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/influxdb/subscriptions","organizations_url":"https://api.github.com/users/influxdb/orgs","repos_url":"https://api.github.com/users/influxdb/repos","events_url":"https://api.github.com/users/influxdb/events{/privacy}","received_events_url":"https://api.github.com/users/influxdb/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/influxdb/influxdb","description":"Scalable datastore for metrics, events, and real-time analytics","fork":false,"url":"https://api.github.com/repos/influxdb/influxdb","forks_url":"https://api.github.com/repos/influxdb/influxdb/forks","keys_url":"https://api.github.com/repos/influxdb/influxdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/influxdb/influxdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/influxdb/influxdb/teams","hooks_url":"https://api.github.com/repos/influxdb/influxdb/hooks","issue_events_url":"https://api.github.com/repos/influxdb/influxdb/issues/events{/number}","events_url":"https://api.github.com/repos/influxdb/influxdb/events","assignees_url":"https://api.github.com/repos/influxdb/influxdb/assignees{/user}","branches_url":"https://api.github.com/repos/influxdb/influxdb/branches{/branch}","tags_url":"https://api.github.com/repos/influxdb/influxdb/tags","blobs_url":"https://api.github.com/repos/influxdb/influxdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/influxdb/influxdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/influxdb/influxdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/influxdb/influxdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/influxdb/influxdb/statuses/{sha}","languages_url":"https://api.github.com/repos/influxdb/influxdb/languages","stargazers_url":"https://api.github.com/repos/influxdb/influxdb/stargazers","contributors_url":"https://api.github.com/repos/influxdb/influxdb/contributors","subscribers_url":"https://api.github.com/repos/influxdb/influxdb/subscribers","subscription_url":"https://api.github.com/repos/influxdb/influxdb/subscription","commits_url":"https://api.github.com/repos/influxdb/influxdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/influxdb/influxdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/influxdb/influxdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/influxdb/influxdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/influxdb/influxdb/contents/{+path}","compare_url":"https://api.github.com/repos/influxdb/influxdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/influxdb/influxdb/merges","archive_url":"https://api.github.com/repos/influxdb/influxdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/influxdb/influxdb/downloads","issues_url":"https://api.github.com/repos/influxdb/influxdb/issues{/number}","pulls_url":"https://api.github.com/repos/influxdb/influxdb/pulls{/number}","milestones_url":"https://api.github.com/repos/influxdb/influxdb/milestones{/number}","notifications_url":"https://api.github.com/repos/influxdb/influxdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/influxdb/influxdb/labels{/name}","releases_url":"https://api.github.com/repos/influxdb/influxdb/releases{/id}","created_at":"2013-09-26T14:31:10Z","updated_at":"2015-10-09T19:30:02Z","pushed_at":"2015-10-09T19:31:52Z","git_url":"git://github.com/influxdb/influxdb.git","ssh_url":"git@github.com:influxdb/influxdb.git","clone_url":"https://github.com/influxdb/influxdb.git","svn_url":"https://github.com/influxdb/influxdb","homepage":"http://influxdb.com","size":60771,"stargazers_count":6462,"watchers_count":6462,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":718,"mirror_url":null,"open_issues_count":378,"forks":718,"open_issues":378,"watchers":6462,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/4375"},"html":{"href":"https://github.com/influxdb/influxdb/pull/4375"},"issue":{"href":"https://api.github.com/repos/influxdb/influxdb/issues/4375"},"comments":{"href":"https://api.github.com/repos/influxdb/influxdb/issues/4375/comments"},"review_comments":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/4375/comments"},"review_comment":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/influxdb/influxdb/pulls/4375/commits"},"statuses":{"href":"https://api.github.com/repos/influxdb/influxdb/statuses/f79e1e532001b76a8e477efa69f86b3967e83e19"}}}}