{"action":"created","comment":{"url":"https://api.github.com/repos/coreos/etcd/pulls/comments/40943109","id":40943109,"diff_hunk":"@@ -0,0 +1,210 @@\n+package main\n+\n+import (\n+\t\"bufio\"\n+\t\"bytes\"\n+\t\"flag\"\n+\t\"fmt\"\n+\t\"math\"\n+\t\"net/http\"\n+\t\"os\"\n+\t\"runtime\"\n+\t\"sort\"\n+\t\"strconv\"\n+\t\"strings\"\n+\t\"time\"\n+\n+\t\"github.com/coreos/etcd/tools/etcd-top/Godeps/_workspace/src/github.com/akrennmair/gopcap\"\n+\t\"github.com/coreos/etcd/tools/etcd-top/Godeps/_workspace/src/github.com/spacejam/loghisto\"\n+)\n+\n+type nameSum struct {\n+\tName string\n+\tSum  float64\n+\tRate float64\n+}\n+\n+type nameSums []nameSum\n+\n+func (n nameSums) Len() int {\n+\treturn len(n)\n+}\n+func (n nameSums) Less(i, j int) bool {\n+\treturn n[i].Sum > n[j].Sum\n+}\n+func (n nameSums) Swap(i, j int) {\n+\tn[i], n[j] = n[j], n[i]\n+}\n+\n+// This function listens for periodic metrics from the loghisto metric system,\n+// and upon receipt of a batch of them it will print out the desired topK.\n+func statPrinter(metricStream chan *loghisto.ProcessedMetricSet, topK, period uint) {\n+\tfor m := range metricStream {\n+\t\trequestCounter := float64(0)\n+\t\tnvs := nameSums{}\n+\t\tfor k, v := range m.Metrics {\n+\t\t\t// loghisto adds _rate suffixed metrics for counters and histograms\n+\t\t\tif strings.HasSuffix(k, \"_rate\") && !strings.HasSuffix(k, \"_rate_rate\") {\n+\t\t\t\tcontinue\n+\t\t\t}\n+\t\t\tnvs = append(nvs, nameSum{\n+\t\t\t\tName: k,\n+\t\t\t\tSum:  v,\n+\t\t\t\tRate: m.Metrics[k+\"_rate\"],\n+\t\t\t})\n+\t\t\trequestCounter += m.Metrics[k+\"_rate\"]\n+\t\t}\n+\n+\t\tfmt.Printf(\"\\n%d sniffed %d requests over last %d seconds\\n\\n\", time.Now().Unix(),\n+\t\t\tuint(requestCounter), period)\n+\t\tif len(nvs) == 0 {\n+\t\t\tcontinue\n+\t\t}\n+\t\tsort.Sort(nvs)\n+\t\tfmt.Printf(\"Top %d most popular http requests:\\n\", topK)\n+\t\tfmt.Println(\"Total Sum  Period Sum Verb Path\")\n+\t\tfor _, nv := range nvs[0:int(math.Min(float64(len(nvs)), float64(topK)))] {\n+\t\t\tfmt.Printf(\"%9.1d %7.1d %s\\n\", int(nv.Sum), int(nv.Rate), nv.Name)\n+\t\t}\n+\t}\n+}\n+\n+// packetDecoder decodes packets and hands them off to the streamRouter\n+func packetDecoder(packetsIn chan *pcap.Packet, packetsOut chan *pcap.Packet) {\n+\tfor pkt := range packetsIn {\n+\t\tpkt.Decode()\n+\t\tselect {\n+\t\tcase packetsOut <- pkt:\n+\t\tdefault:\n+\t\t\tfmt.Println(\"shedding at decoder!\")\n+\t\t}\n+\t}\n+}\n+\n+// processor tries to parse an http request from each packet, and if\n+// successful it records metrics about it in the loghisto metric system.\n+func processor(ms *loghisto.MetricSystem, packetsIn chan *pcap.Packet) {\n+\tfor pkt := range packetsIn {\n+\t\treq, reqErr := http.ReadRequest(bufio.NewReader(bytes.NewReader(pkt.Payload)))\n+\t\tif reqErr == nil {\n+\t\t\tms.Counter(req.Method+\" \"+req.URL.Path, 1)\n+\t\t}\n+\t}\n+}\n+\n+// streamRouter takes a decoded packet and routes it to\n+// a processor that can deal with all requests and responses\n+// for this particular TCP connection.  This allows the\n+// processor to own a local map of requests so that it\n+// can avoid coordinating with other goroutines to perform\n+// analysis.\n+func streamRouter(\n+\tports []uint16,\n+\tparsedPackets chan *pcap.Packet,\n+\tprocessors []chan *pcap.Packet,\n+) {\n+\tfor pkt := range parsedPackets {\n+\t\tclientPort := uint16(0)\n+\t\tfor _, p := range ports {\n+\t\t\tif pkt.TCP == nil {\n+\t\t\t\tbreak\n+\t\t\t}\n+\t\t\tif pkt.TCP.SrcPort == p {\n+\t\t\t\tclientPort = pkt.TCP.DestPort\n+\t\t\t\tbreak\n+\t\t\t}\n+\t\t\tif pkt.TCP.DestPort == p {\n+\t\t\t\tclientPort = pkt.TCP.SrcPort\n+\t\t\t\tbreak\n+\t\t\t}\n+\t\t}\n+\t\tif clientPort != 0 {\n+\t\t\t// client Port can be assumed to have sufficient entropy for\n+\t\t\t// distribution among processors, and we want the same\n+\t\t\t// tcp stream to go to the same processor every time\n+\t\t\t// so that if we do proper packet reconstruction it will\n+\t\t\t// be easier.\n+\t\t\tselect {\n+\t\t\tcase processors[int(clientPort)%len(processors)] <- pkt:\n+\t\t\tdefault:\n+\t\t\t\tfmt.Println(\"Shedding load at router!\")\n+\t\t\t}\n+\t\t}\n+\t}\n+}\n+\n+// 1. parse args\n+// 2. start the loghisto metric system\n+// 3. start the processing and printing goroutines\n+// 4. open the pcap handler\n+// 5. hand off packets from the handler to the decoder\n+func main() {\n+\tportsArg := flag.String(\"ports\", \"2379,4001\", \"etcd listening ports\")\n+\tiface := flag.String(\"iface\", \"eth0\", \"interface for sniffing traffic on\")\n+\tpromisc := flag.Bool(\"promiscuous\", true, \"promiscuous mode\")\n+\tperiod := flag.Uint(\"period\", 1, \"seconds between submissions\")\n+\ttopK := flag.Uint(\"topk\", 10, \"submit stats for the top <K> sniffed paths\")\n+\tflag.Parse()\n+\n+\tnumCPU := runtime.NumCPU()\n+\truntime.GOMAXPROCS(numCPU)\n+\n+\tms := loghisto.NewMetricSystem(time.Duration(*period)*time.Second, false)\n+\tms.Start()\n+\tmetricStream := make(chan *loghisto.ProcessedMetricSet, 2)\n+\tms.SubscribeToProcessedMetrics(metricStream)\n+\tdefer ms.UnsubscribeFromProcessedMetrics(metricStream)\n+\n+\tgo statPrinter(metricStream, *topK, *period)\n+\n+\tports := []uint16{}\n+\tfor _, p := range strings.Split(*portsArg, \",\") {\n+\t\tp, err := strconv.Atoi(p)\n+\t\tif err == nil {\n+\t\t\tports = append(ports, uint16(p))\n+\t\t}\n+\t}\n+\n+\th, err := pcap.Openlive(*iface, 1518, *promisc, 1000)\n+\tif err != nil {\n+\t\tfmt.Println(err)\n+\t\tos.Exit(1)\n+\t}\n+\tdefer h.Close()\n+\n+\tportArray := strings.Split(*portsArg, \",\")\n+\tdst := strings.Join(portArray, \" or dst port \")\n+\tsrc := strings.Join(portArray, \" or src port \")\n+\tfilter := fmt.Sprintf(\"tcp and (dst port %s or src port %s)\", dst, src)\n+\tfmt.Println(\"using bpf filter: \", filter)\n+\tif err := h.Setfilter(filter); err != nil {\n+\t\tfmt.Println(err)\n+\t\tos.Exit(1)\n+\t}\n+\n+\tunparsedPackets := make(chan *pcap.Packet, 16384)\n+\tparsedPackets := make(chan *pcap.Packet, 16384)\n+\tfor i := 0; i < 5; i++ {","path":"tools/etcd-top/etcd-top.go","position":187,"original_position":187,"commit_id":"13858059aef0e552aa7789752885d3f75033b64f","original_commit_id":"13858059aef0e552aa7789752885d3f75033b64f","user":{"login":"xiang90","id":4479947,"avatar_url":"https://avatars.githubusercontent.com/u/4479947?v=3","gravatar_id":"","url":"https://api.github.com/users/xiang90","html_url":"https://github.com/xiang90","followers_url":"https://api.github.com/users/xiang90/followers","following_url":"https://api.github.com/users/xiang90/following{/other_user}","gists_url":"https://api.github.com/users/xiang90/gists{/gist_id}","starred_url":"https://api.github.com/users/xiang90/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xiang90/subscriptions","organizations_url":"https://api.github.com/users/xiang90/orgs","repos_url":"https://api.github.com/users/xiang90/repos","events_url":"https://api.github.com/users/xiang90/events{/privacy}","received_events_url":"https://api.github.com/users/xiang90/received_events","type":"User","site_admin":false},"body":"why 5 routines?","created_at":"2015-10-01T17:43:03Z","updated_at":"2015-10-01T17:43:03Z","html_url":"https://github.com/coreos/etcd/pull/3615#discussion_r40943109","pull_request_url":"https://api.github.com/repos/coreos/etcd/pulls/3615","_links":{"self":{"href":"https://api.github.com/repos/coreos/etcd/pulls/comments/40943109"},"html":{"href":"https://github.com/coreos/etcd/pull/3615#discussion_r40943109"},"pull_request":{"href":"https://api.github.com/repos/coreos/etcd/pulls/3615"}}},"pull_request":{"url":"https://api.github.com/repos/coreos/etcd/pulls/3615","id":46359084,"html_url":"https://github.com/coreos/etcd/pull/3615","diff_url":"https://github.com/coreos/etcd/pull/3615.diff","patch_url":"https://github.com/coreos/etcd/pull/3615.patch","issue_url":"https://api.github.com/repos/coreos/etcd/issues/3615","number":3615,"state":"open","locked":false,"title":"libpcap tool for analyzing popular commands on a production etcd server.","user":{"login":"spacejam","id":1505488,"avatar_url":"https://avatars.githubusercontent.com/u/1505488?v=3","gravatar_id":"","url":"https://api.github.com/users/spacejam","html_url":"https://github.com/spacejam","followers_url":"https://api.github.com/users/spacejam/followers","following_url":"https://api.github.com/users/spacejam/following{/other_user}","gists_url":"https://api.github.com/users/spacejam/gists{/gist_id}","starred_url":"https://api.github.com/users/spacejam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spacejam/subscriptions","organizations_url":"https://api.github.com/users/spacejam/orgs","repos_url":"https://api.github.com/users/spacejam/repos","events_url":"https://api.github.com/users/spacejam/events{/privacy}","received_events_url":"https://api.github.com/users/spacejam/received_events","type":"User","site_admin":false},"body":"@xiang90 \r\nI've stripped out the latency and prometheus components that are present in the original etcd-top tool because they are pretty lossy, and make the code totally unreadable.  I've kept the connection detecting packet router to help with possible future adaptation.","created_at":"2015-09-30T03:54:26Z","updated_at":"2015-10-01T17:43:03Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1de8cef51acc34d4192fc3ff2f3d46e6f7bf1203","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/coreos/etcd/pulls/3615/commits","review_comments_url":"https://api.github.com/repos/coreos/etcd/pulls/3615/comments","review_comment_url":"https://api.github.com/repos/coreos/etcd/pulls/comments{/number}","comments_url":"https://api.github.com/repos/coreos/etcd/issues/3615/comments","statuses_url":"https://api.github.com/repos/coreos/etcd/statuses/13858059aef0e552aa7789752885d3f75033b64f","head":{"label":"spacejam:master","ref":"master","sha":"13858059aef0e552aa7789752885d3f75033b64f","user":{"login":"spacejam","id":1505488,"avatar_url":"https://avatars.githubusercontent.com/u/1505488?v=3","gravatar_id":"","url":"https://api.github.com/users/spacejam","html_url":"https://github.com/spacejam","followers_url":"https://api.github.com/users/spacejam/followers","following_url":"https://api.github.com/users/spacejam/following{/other_user}","gists_url":"https://api.github.com/users/spacejam/gists{/gist_id}","starred_url":"https://api.github.com/users/spacejam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spacejam/subscriptions","organizations_url":"https://api.github.com/users/spacejam/orgs","repos_url":"https://api.github.com/users/spacejam/repos","events_url":"https://api.github.com/users/spacejam/events{/privacy}","received_events_url":"https://api.github.com/users/spacejam/received_events","type":"User","site_admin":false},"repo":{"id":41083243,"name":"etcd","full_name":"spacejam/etcd","owner":{"login":"spacejam","id":1505488,"avatar_url":"https://avatars.githubusercontent.com/u/1505488?v=3","gravatar_id":"","url":"https://api.github.com/users/spacejam","html_url":"https://github.com/spacejam","followers_url":"https://api.github.com/users/spacejam/followers","following_url":"https://api.github.com/users/spacejam/following{/other_user}","gists_url":"https://api.github.com/users/spacejam/gists{/gist_id}","starred_url":"https://api.github.com/users/spacejam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spacejam/subscriptions","organizations_url":"https://api.github.com/users/spacejam/orgs","repos_url":"https://api.github.com/users/spacejam/repos","events_url":"https://api.github.com/users/spacejam/events{/privacy}","received_events_url":"https://api.github.com/users/spacejam/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/spacejam/etcd","description":"A distributed consistent key-value store for shared configuration and service discovery","fork":true,"url":"https://api.github.com/repos/spacejam/etcd","forks_url":"https://api.github.com/repos/spacejam/etcd/forks","keys_url":"https://api.github.com/repos/spacejam/etcd/keys{/key_id}","collaborators_url":"https://api.github.com/repos/spacejam/etcd/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/spacejam/etcd/teams","hooks_url":"https://api.github.com/repos/spacejam/etcd/hooks","issue_events_url":"https://api.github.com/repos/spacejam/etcd/issues/events{/number}","events_url":"https://api.github.com/repos/spacejam/etcd/events","assignees_url":"https://api.github.com/repos/spacejam/etcd/assignees{/user}","branches_url":"https://api.github.com/repos/spacejam/etcd/branches{/branch}","tags_url":"https://api.github.com/repos/spacejam/etcd/tags","blobs_url":"https://api.github.com/repos/spacejam/etcd/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/spacejam/etcd/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/spacejam/etcd/git/refs{/sha}","trees_url":"https://api.github.com/repos/spacejam/etcd/git/trees{/sha}","statuses_url":"https://api.github.com/repos/spacejam/etcd/statuses/{sha}","languages_url":"https://api.github.com/repos/spacejam/etcd/languages","stargazers_url":"https://api.github.com/repos/spacejam/etcd/stargazers","contributors_url":"https://api.github.com/repos/spacejam/etcd/contributors","subscribers_url":"https://api.github.com/repos/spacejam/etcd/subscribers","subscription_url":"https://api.github.com/repos/spacejam/etcd/subscription","commits_url":"https://api.github.com/repos/spacejam/etcd/commits{/sha}","git_commits_url":"https://api.github.com/repos/spacejam/etcd/git/commits{/sha}","comments_url":"https://api.github.com/repos/spacejam/etcd/comments{/number}","issue_comment_url":"https://api.github.com/repos/spacejam/etcd/issues/comments{/number}","contents_url":"https://api.github.com/repos/spacejam/etcd/contents/{+path}","compare_url":"https://api.github.com/repos/spacejam/etcd/compare/{base}...{head}","merges_url":"https://api.github.com/repos/spacejam/etcd/merges","archive_url":"https://api.github.com/repos/spacejam/etcd/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/spacejam/etcd/downloads","issues_url":"https://api.github.com/repos/spacejam/etcd/issues{/number}","pulls_url":"https://api.github.com/repos/spacejam/etcd/pulls{/number}","milestones_url":"https://api.github.com/repos/spacejam/etcd/milestones{/number}","notifications_url":"https://api.github.com/repos/spacejam/etcd/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/spacejam/etcd/labels{/name}","releases_url":"https://api.github.com/repos/spacejam/etcd/releases{/id}","created_at":"2015-08-20T08:01:10Z","updated_at":"2015-08-20T08:01:12Z","pushed_at":"2015-09-30T04:34:39Z","git_url":"git://github.com/spacejam/etcd.git","ssh_url":"git@github.com:spacejam/etcd.git","clone_url":"https://github.com/spacejam/etcd.git","svn_url":"https://github.com/spacejam/etcd","homepage":"","size":17630,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"coreos:master","ref":"master","sha":"533e728b64ac519f33b40bd693a9145534f8c01e","user":{"login":"coreos","id":3730757,"avatar_url":"https://avatars.githubusercontent.com/u/3730757?v=3","gravatar_id":"","url":"https://api.github.com/users/coreos","html_url":"https://github.com/coreos","followers_url":"https://api.github.com/users/coreos/followers","following_url":"https://api.github.com/users/coreos/following{/other_user}","gists_url":"https://api.github.com/users/coreos/gists{/gist_id}","starred_url":"https://api.github.com/users/coreos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/coreos/subscriptions","organizations_url":"https://api.github.com/users/coreos/orgs","repos_url":"https://api.github.com/users/coreos/repos","events_url":"https://api.github.com/users/coreos/events{/privacy}","received_events_url":"https://api.github.com/users/coreos/received_events","type":"Organization","site_admin":false},"repo":{"id":11225014,"name":"etcd","full_name":"coreos/etcd","owner":{"login":"coreos","id":3730757,"avatar_url":"https://avatars.githubusercontent.com/u/3730757?v=3","gravatar_id":"","url":"https://api.github.com/users/coreos","html_url":"https://github.com/coreos","followers_url":"https://api.github.com/users/coreos/followers","following_url":"https://api.github.com/users/coreos/following{/other_user}","gists_url":"https://api.github.com/users/coreos/gists{/gist_id}","starred_url":"https://api.github.com/users/coreos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/coreos/subscriptions","organizations_url":"https://api.github.com/users/coreos/orgs","repos_url":"https://api.github.com/users/coreos/repos","events_url":"https://api.github.com/users/coreos/events{/privacy}","received_events_url":"https://api.github.com/users/coreos/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/coreos/etcd","description":"A distributed consistent key-value store for shared configuration and service discovery","fork":false,"url":"https://api.github.com/repos/coreos/etcd","forks_url":"https://api.github.com/repos/coreos/etcd/forks","keys_url":"https://api.github.com/repos/coreos/etcd/keys{/key_id}","collaborators_url":"https://api.github.com/repos/coreos/etcd/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/coreos/etcd/teams","hooks_url":"https://api.github.com/repos/coreos/etcd/hooks","issue_events_url":"https://api.github.com/repos/coreos/etcd/issues/events{/number}","events_url":"https://api.github.com/repos/coreos/etcd/events","assignees_url":"https://api.github.com/repos/coreos/etcd/assignees{/user}","branches_url":"https://api.github.com/repos/coreos/etcd/branches{/branch}","tags_url":"https://api.github.com/repos/coreos/etcd/tags","blobs_url":"https://api.github.com/repos/coreos/etcd/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/coreos/etcd/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/coreos/etcd/git/refs{/sha}","trees_url":"https://api.github.com/repos/coreos/etcd/git/trees{/sha}","statuses_url":"https://api.github.com/repos/coreos/etcd/statuses/{sha}","languages_url":"https://api.github.com/repos/coreos/etcd/languages","stargazers_url":"https://api.github.com/repos/coreos/etcd/stargazers","contributors_url":"https://api.github.com/repos/coreos/etcd/contributors","subscribers_url":"https://api.github.com/repos/coreos/etcd/subscribers","subscription_url":"https://api.github.com/repos/coreos/etcd/subscription","commits_url":"https://api.github.com/repos/coreos/etcd/commits{/sha}","git_commits_url":"https://api.github.com/repos/coreos/etcd/git/commits{/sha}","comments_url":"https://api.github.com/repos/coreos/etcd/comments{/number}","issue_comment_url":"https://api.github.com/repos/coreos/etcd/issues/comments{/number}","contents_url":"https://api.github.com/repos/coreos/etcd/contents/{+path}","compare_url":"https://api.github.com/repos/coreos/etcd/compare/{base}...{head}","merges_url":"https://api.github.com/repos/coreos/etcd/merges","archive_url":"https://api.github.com/repos/coreos/etcd/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/coreos/etcd/downloads","issues_url":"https://api.github.com/repos/coreos/etcd/issues{/number}","pulls_url":"https://api.github.com/repos/coreos/etcd/pulls{/number}","milestones_url":"https://api.github.com/repos/coreos/etcd/milestones{/number}","notifications_url":"https://api.github.com/repos/coreos/etcd/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/coreos/etcd/labels{/name}","releases_url":"https://api.github.com/repos/coreos/etcd/releases{/id}","created_at":"2013-07-06T21:57:21Z","updated_at":"2015-10-01T17:16:27Z","pushed_at":"2015-10-01T17:11:12Z","git_url":"git://github.com/coreos/etcd.git","ssh_url":"git@github.com:coreos/etcd.git","clone_url":"https://github.com/coreos/etcd.git","svn_url":"https://github.com/coreos/etcd","homepage":"","size":36637,"stargazers_count":7634,"watchers_count":7634,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1129,"mirror_url":null,"open_issues_count":131,"forks":1129,"open_issues":131,"watchers":7634,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/coreos/etcd/pulls/3615"},"html":{"href":"https://github.com/coreos/etcd/pull/3615"},"issue":{"href":"https://api.github.com/repos/coreos/etcd/issues/3615"},"comments":{"href":"https://api.github.com/repos/coreos/etcd/issues/3615/comments"},"review_comments":{"href":"https://api.github.com/repos/coreos/etcd/pulls/3615/comments"},"review_comment":{"href":"https://api.github.com/repos/coreos/etcd/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/coreos/etcd/pulls/3615/commits"},"statuses":{"href":"https://api.github.com/repos/coreos/etcd/statuses/13858059aef0e552aa7789752885d3f75033b64f"}}}}