{"action":"created","comment":{"url":"https://api.github.com/repos/square/metrics/pulls/comments/39354565","id":39354565,"diff_hunk":"@@ -72,132 +105,295 @@ func main() {\n \tif err != nil {\n \t\tcommon.ExitWithMessage(fmt.Sprintf(\"Error while reading rules: %s\", err.Error()))\n \t}\n-\t_ = util.RuleBasedGraphiteConverter{Ruleset: ruleset}\n+\tgraphiteConverter := util.RuleBasedGraphiteConverter{Ruleset: ruleset}\n+\n+\t// metricFile, err := os.Open(*metricsFile)\n+\tmetrics, err := ReadMetricsFile(*metricsFile)\n+\tif err != nil {\n+\t\tfmt.Printf(\"Fatal error reading metrics file %s\\n\", err)\n+\t\tos.Exit(1)\n+\t}\n+\n+\tDoAnalysis(metrics, graphiteConverter)\n+\n+\tfmt.Printf(\"Total metric count %d\\n\", len(metrics))\n+\n+\t// matched := 0\n+\t// unmatched := 0\n+\t// reverse_convert_failed := 0\n+\n+\t// fmt.Printf(\"Matched: %d\\n\", matched)\n+\t// fmt.Printf(\"Unmatched: %d\\n\", unmatched)\n+\t// fmt.Printf(\"Reverse convert failed: %d\\n\", reverse_convert_failed)\n \n-\tmetricFile, err := os.Open(*metricsFile)\n \tif err != nil {\n \t\tcommon.ExitWithMessage(\"No metric file.\")\n \t}\n-\tscanner := bufio.NewScanner(metricFile)\n+\t// scanner := bufio.NewScanner(metricFile)\n \tcassandraConfig := cassandra.CassandraMetricMetadataConfig{\n \t\tHosts:    config.MetricMetadataConfig.Hosts,\n \t\tKeyspace: config.MetricMetadataConfig.Keyspace,\n \t}\n-\tapiInstance := common.NewMetricMetadataAPI(cassandraConfig)\n+\t_ = common.NewMetricMetadataAPI(cassandraConfig)\n \n-\tvar output *os.File\n-\tif *unmatchedFile != \"\" {\n-\t\toutput, err = os.Create(*unmatchedFile)\n-\t\tif err != nil {\n-\t\t\tcommon.ExitWithMessage(fmt.Sprintf(\"Error creating the output file: %s\", err.Error()))\n-\t\t}\n-\t}\n-\tstat := run(ruleset, scanner, apiInstance, output)\n-\treport(stat)\n+\t// var output *os.File\n+\t// if *unmatchedFile != \"\" {\n+\t// \toutput, err = os.Create(*unmatchedFile)\n+\t// \tif err != nil {\n+\t// \t\tcommon.ExitWithMessage(fmt.Sprintf(\"Error creating the output file: %s\", err.Error()))\n+\t// \t}\n+\t// }\n+\t// stat := run(ruleset, scanner, apiInstance, output)\n+\t// report(stat)\n }\n \n-func run(ruleset util.RuleSet, scanner *bufio.Scanner, apiInstance api.MetricMetadataAPI, unmatched *os.File) Statistics {\n+type ChunkResult struct {\n+\tmatched                int\n+\tunmatched              int\n+\treverse_convert_failed int\n+}\n+\n+func DoAnalysis(metrics []string, graphiteConverter util.RuleBasedGraphiteConverter) {\n+\tgraphiteConverter.EnableStats()\n+\n+\tworkQueue := make(chan []string, len(metrics)/25+1)\n+\tresultQueue := make(chan ChunkResult, len(metrics))\n+\tunmatchedQueue := make(chan string, len(metrics))\n \tvar wg sync.WaitGroup\n-\tstat := Statistics{\n-\t\tperMetric: make(map[api.MetricKey]PerMetricStatistics),\n+\n+\ti := 0\n+\tfor i = 0; i+25 < len(metrics); i = i + 25 {\n+\t\tworkQueue <- metrics[i : i+25]\n \t}\n-\ttype result struct {\n-\t\tinput   string\n-\t\tresult  api.TaggedMetric\n-\t\tsuccess bool\n+\tif i < len(metrics) {\n+\t\tworkQueue <- metrics[i:]\n \t}\n-\tinputBuffer := make(chan string, 10)\n-\toutputBuffer := make(chan result, 10)\n-\tdone := make(chan struct{})\n-\tfor id := 0; id < runtime.NumCPU(); id++ {\n+\n+\tclose(workQueue)\n+\n+\tfmt.Printf(\"Starting work...\\n\")\n+\tfor j := 0; j < 10; j++ {\n+\t\twg.Add(1)\n \t\tgo func() {\n+\t\t\tcounter := 0\n+\t\t\tdefer wg.Done()\n \t\t\tfor {\n \t\t\t\tselect {\n-\t\t\t\tcase <-done:\n-\t\t\t\t\treturn\n-\t\t\t\tcase input := <-inputBuffer:\n-\t\t\t\t\tmetric, matched := ruleset.MatchRule(input)\n-\t\t\t\t\toutputBuffer <- result{\n-\t\t\t\t\t\tinput,\n-\t\t\t\t\t\tmetric,\n-\t\t\t\t\t\tmatched,\n+\t\t\t\tcase metrics, more := <-workQueue:\n+\t\t\t\t\tcounter++\n+\t\t\t\t\tif counter%100 == 0 && counter != 0 {\n+\t\t\t\t\t\tfmt.Printf(\".\")\n+\t\t\t\t\t}\n+\t\t\t\t\tchunk_result := ChunkResult{}\n+\t\t\t\t\tfor _, metric := range metrics {\n+\t\t\t\t\t\tgraphiteMetric := util.GraphiteMetric(metric)\n+\t\t\t\t\t\ttaggedMetric, err := graphiteConverter.ToTaggedName(graphiteMetric)\n+\t\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\t\t_, err := graphiteConverter.ToGraphiteName(taggedMetric)\n+\t\t\t\t\t\t\tif err != nil {\n+\t\t\t\t\t\t\t\tchunk_result.reverse_convert_failed++\n+\t\t\t\t\t\t\t} else {\n+\n+\t\t\t\t\t\t\t}\n+\t\t\t\t\t\t\tchunk_result.matched++\n+\t\t\t\t\t\t} else {\n+\t\t\t\t\t\t\tunmatchedQueue <- metric\n+\t\t\t\t\t\t\tchunk_result.unmatched++\n+\t\t\t\t\t\t}\n+\t\t\t\t\t}\n+\t\t\t\t\tresultQueue <- chunk_result\n+\n+\t\t\t\t\tif !more {","path":"main/ruletester/ruletester.go","position":209,"original_position":209,"commit_id":"66a6fb5d6288dcf94cdca5eb7a21d7bcbdcba320","original_commit_id":"66a6fb5d6288dcf94cdca5eb7a21d7bcbdcba320","user":{"login":"Nathan-Fenner","id":6179181,"avatar_url":"https://avatars.githubusercontent.com/u/6179181?v=3","gravatar_id":"","url":"https://api.github.com/users/Nathan-Fenner","html_url":"https://github.com/Nathan-Fenner","followers_url":"https://api.github.com/users/Nathan-Fenner/followers","following_url":"https://api.github.com/users/Nathan-Fenner/following{/other_user}","gists_url":"https://api.github.com/users/Nathan-Fenner/gists{/gist_id}","starred_url":"https://api.github.com/users/Nathan-Fenner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Nathan-Fenner/subscriptions","organizations_url":"https://api.github.com/users/Nathan-Fenner/orgs","repos_url":"https://api.github.com/users/Nathan-Fenner/repos","events_url":"https://api.github.com/users/Nathan-Fenner/events{/privacy}","received_events_url":"https://api.github.com/users/Nathan-Fenner/received_events","type":"User","site_admin":false},"body":"Should be at the top of this `case`. If `more` is false, then the value `metrics` will be the zero value (nil), so looping over it is pointless.\r\n\r\n","created_at":"2015-09-14T00:21:35Z","updated_at":"2015-09-14T00:21:35Z","html_url":"https://github.com/square/metrics/pull/201#discussion_r39354565","pull_request_url":"https://api.github.com/repos/square/metrics/pulls/201","_links":{"self":{"href":"https://api.github.com/repos/square/metrics/pulls/comments/39354565"},"html":{"href":"https://github.com/square/metrics/pull/201#discussion_r39354565"},"pull_request":{"href":"https://api.github.com/repos/square/metrics/pulls/201"}}},"pull_request":{"url":"https://api.github.com/repos/square/metrics/pulls/201","id":44609411,"html_url":"https://github.com/square/metrics/pull/201","diff_url":"https://github.com/square/metrics/pull/201.diff","patch_url":"https://github.com/square/metrics/pull/201.patch","issue_url":"https://api.github.com/repos/square/metrics/issues/201","number":201,"state":"open","locked":false,"title":"Build txt file reports from what matches and what doesn't","user":{"login":"cchandler","id":26321,"avatar_url":"https://avatars.githubusercontent.com/u/26321?v=3","gravatar_id":"","url":"https://api.github.com/users/cchandler","html_url":"https://github.com/cchandler","followers_url":"https://api.github.com/users/cchandler/followers","following_url":"https://api.github.com/users/cchandler/following{/other_user}","gists_url":"https://api.github.com/users/cchandler/gists{/gist_id}","starred_url":"https://api.github.com/users/cchandler/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cchandler/subscriptions","organizations_url":"https://api.github.com/users/cchandler/orgs","repos_url":"https://api.github.com/users/cchandler/repos","events_url":"https://api.github.com/users/cchandler/events{/privacy}","received_events_url":"https://api.github.com/users/cchandler/received_events","type":"User","site_admin":false},"body":"I've modified the old ruletester file to be able to read the new zlib compressed gob encoded cache file we generate from the metrics-indexer upstream. Yes, the ruletester file is a bit of a mess right now, but I think that's fine. We need to refine this and get more data out of it most likely, so I'm leaving it there for now for the next person :-).\r\n\r\nI've also exported a lot of the data from the util.GraphiteConverter stuff so we can get per-rule statistics from within the converter itself. There's probably more we want to track.","created_at":"2015-09-09T22:40:34Z","updated_at":"2015-09-14T00:21:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"223c6d560fc0921e274df5d69acae5ab009732e2","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/square/metrics/pulls/201/commits","review_comments_url":"https://api.github.com/repos/square/metrics/pulls/201/comments","review_comment_url":"https://api.github.com/repos/square/metrics/pulls/comments{/number}","comments_url":"https://api.github.com/repos/square/metrics/issues/201/comments","statuses_url":"https://api.github.com/repos/square/metrics/statuses/66a6fb5d6288dcf94cdca5eb7a21d7bcbdcba320","head":{"label":"square:cchandler-read-cache","ref":"cchandler-read-cache","sha":"66a6fb5d6288dcf94cdca5eb7a21d7bcbdcba320","user":{"login":"square","id":82592,"avatar_url":"https://avatars.githubusercontent.com/u/82592?v=3","gravatar_id":"","url":"https://api.github.com/users/square","html_url":"https://github.com/square","followers_url":"https://api.github.com/users/square/followers","following_url":"https://api.github.com/users/square/following{/other_user}","gists_url":"https://api.github.com/users/square/gists{/gist_id}","starred_url":"https://api.github.com/users/square/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/square/subscriptions","organizations_url":"https://api.github.com/users/square/orgs","repos_url":"https://api.github.com/users/square/repos","events_url":"https://api.github.com/users/square/events{/privacy}","received_events_url":"https://api.github.com/users/square/received_events","type":"Organization","site_admin":false},"repo":{"id":34279409,"name":"metrics","full_name":"square/metrics","owner":{"login":"square","id":82592,"avatar_url":"https://avatars.githubusercontent.com/u/82592?v=3","gravatar_id":"","url":"https://api.github.com/users/square","html_url":"https://github.com/square","followers_url":"https://api.github.com/users/square/followers","following_url":"https://api.github.com/users/square/following{/other_user}","gists_url":"https://api.github.com/users/square/gists{/gist_id}","starred_url":"https://api.github.com/users/square/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/square/subscriptions","organizations_url":"https://api.github.com/users/square/orgs","repos_url":"https://api.github.com/users/square/repos","events_url":"https://api.github.com/users/square/events{/privacy}","received_events_url":"https://api.github.com/users/square/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/square/metrics","description":"Metrics Query Engine","fork":false,"url":"https://api.github.com/repos/square/metrics","forks_url":"https://api.github.com/repos/square/metrics/forks","keys_url":"https://api.github.com/repos/square/metrics/keys{/key_id}","collaborators_url":"https://api.github.com/repos/square/metrics/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/square/metrics/teams","hooks_url":"https://api.github.com/repos/square/metrics/hooks","issue_events_url":"https://api.github.com/repos/square/metrics/issues/events{/number}","events_url":"https://api.github.com/repos/square/metrics/events","assignees_url":"https://api.github.com/repos/square/metrics/assignees{/user}","branches_url":"https://api.github.com/repos/square/metrics/branches{/branch}","tags_url":"https://api.github.com/repos/square/metrics/tags","blobs_url":"https://api.github.com/repos/square/metrics/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/square/metrics/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/square/metrics/git/refs{/sha}","trees_url":"https://api.github.com/repos/square/metrics/git/trees{/sha}","statuses_url":"https://api.github.com/repos/square/metrics/statuses/{sha}","languages_url":"https://api.github.com/repos/square/metrics/languages","stargazers_url":"https://api.github.com/repos/square/metrics/stargazers","contributors_url":"https://api.github.com/repos/square/metrics/contributors","subscribers_url":"https://api.github.com/repos/square/metrics/subscribers","subscription_url":"https://api.github.com/repos/square/metrics/subscription","commits_url":"https://api.github.com/repos/square/metrics/commits{/sha}","git_commits_url":"https://api.github.com/repos/square/metrics/git/commits{/sha}","comments_url":"https://api.github.com/repos/square/metrics/comments{/number}","issue_comment_url":"https://api.github.com/repos/square/metrics/issues/comments{/number}","contents_url":"https://api.github.com/repos/square/metrics/contents/{+path}","compare_url":"https://api.github.com/repos/square/metrics/compare/{base}...{head}","merges_url":"https://api.github.com/repos/square/metrics/merges","archive_url":"https://api.github.com/repos/square/metrics/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/square/metrics/downloads","issues_url":"https://api.github.com/repos/square/metrics/issues{/number}","pulls_url":"https://api.github.com/repos/square/metrics/pulls{/number}","milestones_url":"https://api.github.com/repos/square/metrics/milestones{/number}","notifications_url":"https://api.github.com/repos/square/metrics/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/square/metrics/labels{/name}","releases_url":"https://api.github.com/repos/square/metrics/releases{/id}","created_at":"2015-04-20T18:43:35Z","updated_at":"2015-09-01T11:07:37Z","pushed_at":"2015-09-13T23:10:17Z","git_url":"git://github.com/square/metrics.git","ssh_url":"git@github.com:square/metrics.git","clone_url":"https://github.com/square/metrics.git","svn_url":"https://github.com/square/metrics","homepage":"","size":3710,"stargazers_count":22,"watchers_count":22,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":6,"mirror_url":null,"open_issues_count":7,"forks":6,"open_issues":7,"watchers":22,"default_branch":"master"}},"base":{"label":"square:master","ref":"master","sha":"b9535c9e40d2c3e366144292a48d77fdb269fe00","user":{"login":"square","id":82592,"avatar_url":"https://avatars.githubusercontent.com/u/82592?v=3","gravatar_id":"","url":"https://api.github.com/users/square","html_url":"https://github.com/square","followers_url":"https://api.github.com/users/square/followers","following_url":"https://api.github.com/users/square/following{/other_user}","gists_url":"https://api.github.com/users/square/gists{/gist_id}","starred_url":"https://api.github.com/users/square/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/square/subscriptions","organizations_url":"https://api.github.com/users/square/orgs","repos_url":"https://api.github.com/users/square/repos","events_url":"https://api.github.com/users/square/events{/privacy}","received_events_url":"https://api.github.com/users/square/received_events","type":"Organization","site_admin":false},"repo":{"id":34279409,"name":"metrics","full_name":"square/metrics","owner":{"login":"square","id":82592,"avatar_url":"https://avatars.githubusercontent.com/u/82592?v=3","gravatar_id":"","url":"https://api.github.com/users/square","html_url":"https://github.com/square","followers_url":"https://api.github.com/users/square/followers","following_url":"https://api.github.com/users/square/following{/other_user}","gists_url":"https://api.github.com/users/square/gists{/gist_id}","starred_url":"https://api.github.com/users/square/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/square/subscriptions","organizations_url":"https://api.github.com/users/square/orgs","repos_url":"https://api.github.com/users/square/repos","events_url":"https://api.github.com/users/square/events{/privacy}","received_events_url":"https://api.github.com/users/square/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/square/metrics","description":"Metrics Query Engine","fork":false,"url":"https://api.github.com/repos/square/metrics","forks_url":"https://api.github.com/repos/square/metrics/forks","keys_url":"https://api.github.com/repos/square/metrics/keys{/key_id}","collaborators_url":"https://api.github.com/repos/square/metrics/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/square/metrics/teams","hooks_url":"https://api.github.com/repos/square/metrics/hooks","issue_events_url":"https://api.github.com/repos/square/metrics/issues/events{/number}","events_url":"https://api.github.com/repos/square/metrics/events","assignees_url":"https://api.github.com/repos/square/metrics/assignees{/user}","branches_url":"https://api.github.com/repos/square/metrics/branches{/branch}","tags_url":"https://api.github.com/repos/square/metrics/tags","blobs_url":"https://api.github.com/repos/square/metrics/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/square/metrics/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/square/metrics/git/refs{/sha}","trees_url":"https://api.github.com/repos/square/metrics/git/trees{/sha}","statuses_url":"https://api.github.com/repos/square/metrics/statuses/{sha}","languages_url":"https://api.github.com/repos/square/metrics/languages","stargazers_url":"https://api.github.com/repos/square/metrics/stargazers","contributors_url":"https://api.github.com/repos/square/metrics/contributors","subscribers_url":"https://api.github.com/repos/square/metrics/subscribers","subscription_url":"https://api.github.com/repos/square/metrics/subscription","commits_url":"https://api.github.com/repos/square/metrics/commits{/sha}","git_commits_url":"https://api.github.com/repos/square/metrics/git/commits{/sha}","comments_url":"https://api.github.com/repos/square/metrics/comments{/number}","issue_comment_url":"https://api.github.com/repos/square/metrics/issues/comments{/number}","contents_url":"https://api.github.com/repos/square/metrics/contents/{+path}","compare_url":"https://api.github.com/repos/square/metrics/compare/{base}...{head}","merges_url":"https://api.github.com/repos/square/metrics/merges","archive_url":"https://api.github.com/repos/square/metrics/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/square/metrics/downloads","issues_url":"https://api.github.com/repos/square/metrics/issues{/number}","pulls_url":"https://api.github.com/repos/square/metrics/pulls{/number}","milestones_url":"https://api.github.com/repos/square/metrics/milestones{/number}","notifications_url":"https://api.github.com/repos/square/metrics/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/square/metrics/labels{/name}","releases_url":"https://api.github.com/repos/square/metrics/releases{/id}","created_at":"2015-04-20T18:43:35Z","updated_at":"2015-09-01T11:07:37Z","pushed_at":"2015-09-13T23:10:17Z","git_url":"git://github.com/square/metrics.git","ssh_url":"git@github.com:square/metrics.git","clone_url":"https://github.com/square/metrics.git","svn_url":"https://github.com/square/metrics","homepage":"","size":3710,"stargazers_count":22,"watchers_count":22,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":6,"mirror_url":null,"open_issues_count":7,"forks":6,"open_issues":7,"watchers":22,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/square/metrics/pulls/201"},"html":{"href":"https://github.com/square/metrics/pull/201"},"issue":{"href":"https://api.github.com/repos/square/metrics/issues/201"},"comments":{"href":"https://api.github.com/repos/square/metrics/issues/201/comments"},"review_comments":{"href":"https://api.github.com/repos/square/metrics/pulls/201/comments"},"review_comment":{"href":"https://api.github.com/repos/square/metrics/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/square/metrics/pulls/201/commits"},"statuses":{"href":"https://api.github.com/repos/square/metrics/statuses/66a6fb5d6288dcf94cdca5eb7a21d7bcbdcba320"}}}}