{"action":"created","comment":{"url":"https://api.github.com/repos/prometheus/prometheus/pulls/comments/36836122","id":36836122,"diff_hunk":"@@ -0,0 +1,593 @@\n+// Copyright 2015 The Prometheus Authors\n+// Licensed under the Apache License, Version 2.0 (the \"License\");\n+// you may not use this file except in compliance with the License.\n+// You may obtain a copy of the License at\n+//\n+// http://www.apache.org/licenses/LICENSE-2.0\n+//\n+// Unless required by applicable law or agreed to in writing, software\n+// distributed under the License is distributed on an \"AS IS\" BASIS,\n+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+// See the License for the specific language governing permissions and\n+// limitations under the License.\n+\n+package discovery\n+\n+import (\n+\t\"fmt\"\n+\t\"io/ioutil\"\n+\t\"net\"\n+\t\"sync\"\n+\n+\t\"crypto/tls\"\n+\t\"crypto/x509\"\n+\t\"encoding/json\"\n+\t\"net/http\"\n+\t\"os\"\n+\t\"strconv\"\n+\t\"time\"\n+\n+\tclientmodel \"github.com/prometheus/client_golang/model\"\n+\t\"github.com/prometheus/log\"\n+\t\"github.com/prometheus/prometheus/config\"\n+\t\"github.com/prometheus/prometheus/retrieval/discovery/kubernetes\"\n+\t\"github.com/prometheus/prometheus/util/httputil\"\n+)\n+\n+const (\n+\tkubernetesSourceServicePrefix = \"services\"\n+\n+\t// KubernetesClusterNameLabel is the name for the this kubernetes cluster.\n+\tkubernetesClusterNameLabel = clientmodel.MetaLabelPrefix + \"kubernetes_cluster_name\"\n+\t// KubernetesNodeLabel is the name for the label containing a target's node name.\n+\tkubernetesNodeLabel = clientmodel.MetaLabelPrefix + \"kubernetes_node\"\n+\t// KubernetesServiceNamespaceLabel is the name for the label containing a target's service namespace.\n+\tkubernetesServiceNamespaceLabel = clientmodel.MetaLabelPrefix + \"kubernetes_service_namespace\"\n+\t// KubernetesServiceNameLabel is the name for the label containing a target's service name.\n+\tkubernetesServiceNameLabel = clientmodel.MetaLabelPrefix + \"kubernetes_service_name\"\n+\t// KubernetesNodesTargetGroupName is the name given to the target group for nodes.\n+\tkubernetesNodesTargetGroupName = \"nodes\"\n+\n+\tkubernetesServiceAccountToken  = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n+\tkubernetesServiceAccountCACert = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n+\n+\tkubernetesAPIVersion          = \"v1\"\n+\tkubernetesAPIPrefix           = \"api/\" + kubernetesAPIVersion\n+\tkubernetesNodesURL            = kubernetesAPIPrefix + \"/nodes\"\n+\tkubernetesServicesURL         = kubernetesAPIPrefix + \"/services\"\n+\tkubernetesEndpointsURL        = kubernetesAPIPrefix + \"/endpoints\"\n+\tkubernetesServiceEndpointsURL = kubernetesAPIPrefix + \"/namespaces/%s/endpoints/%s\"\n+)\n+\n+type KubernetesDiscovery struct {\n+\tclient *http.Client\n+\tconf   *config.KubernetesSDConfig\n+\n+\tnodesResourceVersion     string\n+\tservicesResourceVersion  string\n+\tendpointsResourceVersion string\n+\tnodes                    map[string]*kubernetes.Node\n+\tservices                 map[string]map[string]*kubernetes.Service\n+\tnodesMu                  sync.RWMutex\n+\tservicesMu               sync.RWMutex\n+\trunDone                  chan struct{}\n+}\n+\n+func NewKubernetesDiscovery(conf *config.KubernetesSDConfig) *KubernetesDiscovery {\n+\tclient, err := newKubernetesHTTPClient(conf)\n+\n+\tif err != nil {\n+\t\tlog.Errorf(\"discovery.kubernetes.NewKubernetesDiscovery: %s\", err)\n+\t\treturn nil\n+\t}\n+\n+\tkd := &KubernetesDiscovery{\n+\t\tclient:   client,\n+\t\tconf:     conf,\n+\t\tnodes:    make(map[string]*kubernetes.Node),\n+\t\tservices: make(map[string]map[string]*kubernetes.Service),\n+\t\trunDone:  make(chan struct{}),\n+\t}\n+\n+\treturn kd\n+}\n+\n+// Sources implements the TargetProvider interface.\n+func (kd *KubernetesDiscovery) Sources() []string {\n+\tres, err := kd.client.Get(kd.conf.Server + kubernetesNodesURL)\n+\tif err != nil {\n+\t\t// If we can't list nodes then we can't watch them. Assume this is a misconfiguration\n+\t\t// & log & return empty.\n+\t\tlog.Errorf(\"discovery.kubernetes.Sources: %s\", err)\n+\t\treturn []string{}\n+\t}\n+\tif res.StatusCode != http.StatusOK {\n+\t\tlog.Errorf(\"discovery.kubernetes.Sources: %s\", res.Body)\n+\t\treturn []string{}\n+\t}\n+\n+\tvar nodes kubernetes.NodeList\n+\tif err := json.NewDecoder(res.Body).Decode(&nodes); err != nil {\n+\t\tlog.Errorf(\"discovery.kubernetes.Sources: %s\", err)\n+\t\treturn []string{}\n+\t}\n+\n+\tkd.nodesMu.Lock()\n+\tdefer kd.nodesMu.Unlock()\n+\n+\tsourceNames := make([]string, 0, len(nodes.Items))\n+\n+\tkd.nodesResourceVersion = nodes.ResourceVersion\n+\tfor idx, node := range nodes.Items {\n+\t\tsourceNames = append(sourceNames, kubernetesNodesTargetGroupName+\":\"+node.ObjectMeta.Name)\n+\t\tkd.nodes[node.ObjectMeta.Name] = &nodes.Items[idx]\n+\t}\n+\n+\tif kd.conf.DiscoverServices {\n+\t\tres, err := kd.client.Get(kd.conf.Server + kubernetesServicesURL)\n+\t\tif err != nil {\n+\t\t\t// If we can't list services then we can't watch them. Assume this is a misconfiguration\n+\t\t\t// & log & return empty.\n+\t\t\tlog.Errorf(\"discovery.kubernetes.Sources: %s\", err)\n+\t\t\treturn []string{}\n+\t\t}\n+\t\tif res.StatusCode != http.StatusOK {\n+\t\t\tlog.Errorf(\"discovery.kubernetes.Sources: %s\", res.Body)\n+\t\t\treturn []string{}\n+\t\t}\n+\t\tvar services kubernetes.ServiceList\n+\t\tif err := json.NewDecoder(res.Body).Decode(&services); err != nil {\n+\t\t\tlog.Errorf(\"discovery.kubernetes.Sources: %s\", err)\n+\t\t\treturn []string{}\n+\t\t}\n+\t\tif err != nil {\n+\t\t\t// If we can't list nodes then we can't watch them. Assume this is a misconfiguration\n+\t\t\t// & log & return empty.\n+\t\t\tlog.Errorf(\"discovery.kubernetes.Sources: %s\", err)\n+\t\t\treturn []string{}\n+\t\t}\n+\t\tkd.servicesMu.Lock()\n+\t\tdefer kd.servicesMu.Unlock()\n+\n+\t\tkd.servicesResourceVersion = services.ResourceVersion\n+\t\tfor idx, service := range services.Items {\n+\t\t\tif kd.checkScrape(&service) {\n+\t\t\t\tsourceNames = append(sourceNames, kubernetesSourceServicePrefix+\":\"+service.ObjectMeta.Namespace+\"/\"+service.ObjectMeta.Name)\n+\t\t\t\tnamespace, ok := kd.services[service.ObjectMeta.Namespace]\n+\t\t\t\tif !ok {\n+\t\t\t\t\tnamespace = make(map[string]*kubernetes.Service)\n+\t\t\t\t\tkd.services[service.ObjectMeta.Namespace] = namespace\n+\t\t\t\t}\n+\t\t\t\tnamespace[service.ObjectMeta.Name] = &services.Items[idx]\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\treturn sourceNames\n+}\n+\n+func (kd *KubernetesDiscovery) checkScrape(s *kubernetes.Service) bool {\n+\tif scrape, ok := s.ObjectMeta.Annotations[kd.conf.ScrapeKey]; ok {\n+\t\tshouldScrape, err := strconv.ParseBool(scrape)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Invalid value (%s) for %s annotation on service %s/%s: %s\", scrape, kd.conf.ScrapeKey, s.ObjectMeta.Namespace, s.ObjectMeta.Name, err)\n+\t\t\treturn false\n+\t\t}\n+\t\tif shouldScrape {\n+\t\t\treturn shouldScrape\n+\t\t}\n+\t}\n+\treturn false\n+}\n+\n+// Run implements the TargetProvider interface.\n+func (kd *KubernetesDiscovery) Run(ch chan<- *config.TargetGroup) {\n+\tdefer close(ch)\n+\tdefer close(kd.runDone)\n+\n+\tkd.updateNodesTargetGroup(ch)\n+\n+\tfor _, ns := range kd.services {\n+\t\tfor _, service := range ns {\n+\t\t\tkd.addService(service, ch)\n+\t\t}\n+\t}\n+\n+\tretryDelay := time.Duration(kd.conf.RetryInterval)\n+\n+\tupdate := make(chan interface{}, 10)\n+\tdefer close(update)\n+\tgo kd.watchNodes(update, retryDelay)\n+\n+\tif kd.conf.DiscoverServices {\n+\t\tgo kd.watchServices(update, retryDelay)\n+\t\tgo kd.watchServiceEndpoints(update, retryDelay)\n+\t}\n+\n+\tfor {\n+\t\tselect {\n+\t\tcase <-kd.runDone:\n+\t\t\treturn\n+\t\tcase event := <-update:\n+\t\t\tswitch obj := event.(type) {\n+\t\t\tcase *kubernetes.NodeEvent:\n+\t\t\t\tkd.updateNode(obj.Node, obj.EventType)\n+\t\t\t\tkd.updateNodesTargetGroup(ch)\n+\t\t\tcase *kubernetes.ServiceEvent:\n+\t\t\t\tkd.updateService(obj.Service, obj.EventType, ch)\n+\t\t\tcase *kubernetes.EndpointsEvent:\n+\t\t\t\tkd.updateServiceEndpoints(obj.Endpoints, obj.EventType, ch)\n+\t\t\t}\n+\t\t}\n+\t}\n+}\n+\n+// Stop implements the TargetProvider interface.\n+func (kd *KubernetesDiscovery) Stop() {\n+\tlog.Debugf(\"Stopping Kubernetes discovery for %s\", kd.conf.Server)\n+\n+\t// The lock prevents Run from terminating while the watchers attempt\n+\t// to send on their channels.\n+\tkd.nodesMu.Lock()\n+\tdefer kd.nodesMu.Unlock()\n+\tkd.servicesMu.Lock()\n+\tdefer kd.servicesMu.Unlock()\n+\n+\t// Terminate Run.\n+\tkd.runDone <- struct{}{}\n+\n+\tlog.Debugf(\"Kubernetes discovery for %s stopped.\", kd.conf.Server)\n+}\n+\n+func (kd *KubernetesDiscovery) updateNodesTargetGroup(ch chan<- *config.TargetGroup) {\n+\tkd.nodesMu.Lock()\n+\tdefer kd.nodesMu.Unlock()\n+\ttg := &config.TargetGroup{Source: kubernetesNodesTargetGroupName}\n+\n+\t// Now let's loop through the nodes & add them to the target group with appropriate labels.\n+\tfor nodeName, node := range kd.nodes {\n+\t\taddress := fmt.Sprintf(\"%s:%d\", node.Status.Addresses[0].Address, kd.conf.NodeMetricsPort)\n+\n+\t\tt := clientmodel.LabelSet{\n+\t\t\tclientmodel.AddressLabel: clientmodel.LabelValue(address),\n+\t\t\tkubernetesNodeLabel:      clientmodel.LabelValue(nodeName),\n+\t\t}\n+\t\tfor k, v := range node.ObjectMeta.Labels {\n+\t\t\tt[clientmodel.LabelName(k)] = clientmodel.LabelValue(v)\n+\t\t}\n+\t\ttg.Targets = append(tg.Targets, t)\n+\t}\n+\n+\tch <- tg\n+}\n+\n+func (kd *KubernetesDiscovery) updateNode(node *kubernetes.Node, eventType kubernetes.EventType) {\n+\tkd.nodesMu.Lock()\n+\tdefer kd.nodesMu.Unlock()\n+\tupdatedNodeName := node.ObjectMeta.Name\n+\tswitch eventType {\n+\tcase \"Deleted\":\n+\t\t// Deleted - remove from nodes map\n+\t\tdelete(kd.nodes, updatedNodeName)\n+\tcase \"Added\", \"Modified\":\n+\t\t// Added/Modified - update the node in the nodes map\n+\t\tkd.nodes[updatedNodeName] = node\n+\t}\n+}\n+\n+// watchNodes watches nodes as they come & go.\n+func (kd *KubernetesDiscovery) watchNodes(events chan interface{}, retryDelay time.Duration) {\n+\tuntil(func() {\n+\t\treq, err := http.NewRequest(\"GET\", kd.conf.Server+kubernetesNodesURL, nil)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Failed to watch nodes: %s\", err)\n+\t\t\treturn\n+\t\t}\n+\t\tvalues := req.URL.Query()\n+\t\tvalues.Add(\"watch\", \"true\")\n+\t\tvalues.Add(\"resourceVersion\", kd.nodesResourceVersion)\n+\t\treq.URL.RawQuery = values.Encode()\n+\t\tres, err := kd.client.Do(req)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Failed to watch nodes: %s\", err)\n+\t\t\treturn\n+\t\t}\n+\t\tif res.StatusCode != http.StatusOK {\n+\t\t\tlog.Errorf(\"Failed to watch nodes: %s\", res.StatusCode)\n+\t\t\treturn\n+\t\t}\n+\n+\t\td := json.NewDecoder(res.Body)\n+\n+\t\tfor {\n+\t\t\tvar event kubernetes.NodeEvent\n+\t\t\tif err := d.Decode(&event); err != nil {\n+\t\t\t\tlog.Errorf(\"Failed to watch nodes: %s\", err)\n+\t\t\t\treturn\n+\t\t\t}\n+\t\t\tkd.nodesResourceVersion = event.Node.ObjectMeta.ResourceVersion\n+\t\t\tevents <- &event\n+\t\t}\n+\t}, retryDelay, kd.runDone)\n+}\n+\n+// watchServices watches services as they come & go.\n+func (kd *KubernetesDiscovery) watchServices(events chan interface{}, retryDelay time.Duration) {\n+\tuntil(func() {\n+\t\treq, err := http.NewRequest(\"GET\", kd.conf.Server+kubernetesServicesURL, nil)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Failed to watch services: %s\", err)\n+\t\t\treturn\n+\t\t}\n+\t\tvalues := req.URL.Query()\n+\t\tvalues.Add(\"watch\", \"true\")\n+\t\tvalues.Add(\"resourceVersion\", kd.servicesResourceVersion)\n+\t\treq.URL.RawQuery = values.Encode()\n+\n+\t\tres, err := kd.client.Do(req)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Failed to watch services: %s\", err)\n+\t\t\treturn\n+\t\t}\n+\t\tif res.StatusCode != http.StatusOK {\n+\t\t\tlog.Errorf(\"Failed to watch services: %s\", res.StatusCode)\n+\t\t\treturn\n+\t\t}\n+\n+\t\td := json.NewDecoder(res.Body)\n+\n+\t\tfor {\n+\t\t\tvar event kubernetes.ServiceEvent\n+\t\t\tif err := d.Decode(&event); err != nil {\n+\t\t\t\tlog.Errorf(\"Unable to watch services: %s\", err)\n+\t\t\t\treturn\n+\t\t\t}\n+\t\t\tkd.servicesResourceVersion = event.Service.ObjectMeta.ResourceVersion\n+\t\t\tevents <- &event\n+\t\t}\n+\t}, retryDelay, kd.runDone)\n+}\n+\n+func (kd *KubernetesDiscovery) updateService(service *kubernetes.Service, eventType kubernetes.EventType, ch chan<- *config.TargetGroup) {\n+\tkd.servicesMu.Lock()\n+\tdefer kd.servicesMu.Unlock()\n+\tname := service.ObjectMeta.Name\n+\tnamespace := service.ObjectMeta.Namespace\n+\tscrape := kd.checkScrape(service)\n+\t_, ok := kd.services[namespace][name]\n+\tswitch eventType {\n+\tcase kubernetes.Deleted:\n+\t\tif ok {\n+\t\t\tkd.deleteService(service, ch)\n+\t\t}\n+\tcase kubernetes.Modified:\n+\t\t// Exists & no scrape - delete.\n+\t\tif ok && !scrape {\n+\t\t\tkd.deleteService(service, ch)\n+\t\t\treturn\n+\t\t}\n+\n+\t\tif scrape {\n+\t\t\t// Must be scrapable so let's add it to ensure labels are updated properly at least.\n+\t\t\tkd.addService(service, ch)\n+\t\t}\n+\tcase kubernetes.Added:\n+\t\tif scrape {\n+\t\t\tkd.addService(service, ch)\n+\t\t}\n+\t}\n+}\n+\n+func (kd *KubernetesDiscovery) deleteService(service *kubernetes.Service, ch chan<- *config.TargetGroup) {\n+\ttg := &config.TargetGroup{Source: kubernetesSourceServicePrefix + \":\" + service.ObjectMeta.Namespace + \"/\" + service.ObjectMeta.Name}\n+\tch <- tg\n+\tdelete(kd.services[service.ObjectMeta.Namespace], service.ObjectMeta.Name)\n+\tif len(kd.services[service.ObjectMeta.Namespace]) == 0 {\n+\t\tdelete(kd.services, service.ObjectMeta.Namespace)\n+\t}\n+}\n+\n+func (kd *KubernetesDiscovery) addService(service *kubernetes.Service, ch chan<- *config.TargetGroup) {\n+\tnamespace, ok := kd.services[service.ObjectMeta.Namespace]\n+\tif !ok {\n+\t\tnamespace = make(map[string]*kubernetes.Service)\n+\t\tkd.services[service.ObjectMeta.Namespace] = namespace\n+\t}\n+\tnamespace[service.ObjectMeta.Name] = service\n+\tendpointURL := fmt.Sprintf(kubernetesServiceEndpointsURL, service.ObjectMeta.Namespace, service.ObjectMeta.Name)\n+\tres, err := kd.client.Get(kd.conf.Server + endpointURL)\n+\tif err != nil {\n+\t\tlog.Errorf(\"Error getting service endpoints: %s\", err)\n+\t\treturn\n+\t}\n+\tif res.StatusCode != http.StatusOK {\n+\t\tlog.Errorf(\"Failed to get service endpoints: %s\", res.StatusCode)\n+\t\treturn\n+\t}\n+\n+\tvar endpoints kubernetes.Endpoints\n+\tif err := json.NewDecoder(res.Body).Decode(&endpoints); err != nil {\n+\t\tlog.Errorf(\"Error getting service endpoints: %s\", err)\n+\t\treturn\n+\t}\n+\n+\tkd.updateServiceTargetGroup(service, &endpoints, ch)\n+}\n+\n+func (kd *KubernetesDiscovery) updateServiceTargetGroup(service *kubernetes.Service, endpoints *kubernetes.Endpoints, ch chan<- *config.TargetGroup) {\n+\ttg := &config.TargetGroup{\n+\t\tSource: kubernetesSourceServicePrefix + \":\" + service.ObjectMeta.Namespace + \"/\" + service.ObjectMeta.Name,\n+\t\tLabels: clientmodel.LabelSet{\n+\t\t\tkubernetesServiceNamespaceLabel: clientmodel.LabelValue(service.ObjectMeta.Namespace),\n+\t\t\tkubernetesServiceNameLabel:      clientmodel.LabelValue(service.ObjectMeta.Name),\n+\t\t},\n+\t}\n+\n+\tfor k, v := range service.ObjectMeta.Labels {","path":"retrieval/discovery/kubernetes.go","position":422,"original_position":426,"commit_id":"68dd401cc1c17c8538bad3e753cd44e08eec9f7b","original_commit_id":"388dd9cd2ec8c070ccce7dccb6a1db82c83264fb","user":{"login":"brian-brazil","id":7115638,"avatar_url":"https://avatars.githubusercontent.com/u/7115638?v=3","gravatar_id":"","url":"https://api.github.com/users/brian-brazil","html_url":"https://github.com/brian-brazil","followers_url":"https://api.github.com/users/brian-brazil/followers","following_url":"https://api.github.com/users/brian-brazil/following{/other_user}","gists_url":"https://api.github.com/users/brian-brazil/gists{/gist_id}","starred_url":"https://api.github.com/users/brian-brazil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brian-brazil/subscriptions","organizations_url":"https://api.github.com/users/brian-brazil/orgs","repos_url":"https://api.github.com/users/brian-brazil/repos","events_url":"https://api.github.com/users/brian-brazil/events{/privacy}","received_events_url":"https://api.github.com/users/brian-brazil/received_events","type":"User","site_admin":false},"body":"> How about a label whitelist that could be a slice of regexes? All non-matching would go to meta labels, otherwise \"proper\" labels?\r\n\r\nThat could be an option if it's completely opt-in, but we're lacking all the other support needed to make that manageable for the user on the other end as we presume you know the names of the labels you want to work with. There's a lot of scope here for shooting yourself in the foot, as non-constant target labels are a big anti-pattern. In the extreme you end up with every gauge as a label on the target - which sounds useful for grouping but doesn't work out in practice at all.\r\n\r\nThe usual way to solve this is to have a single timeseries with all these labels and the value of 1 that you can join to things. Then the rest of your timeseries don't break, and you can still make use of the labels. This'd be the first time that'd be coming from SD rather than the target itself so that'd need plumbing.\r\n\r\n> I'm not sure I follow. What do you mean by general case? IMO in this case users are not going to want to label everything separately, relabel extensively & correlate those labels to resources somehow. \r\n\r\nLabels in Kubernetes aren't just for resource management, they're also for thing like selecting what goes into a load balancer. One potential use case is that every client of your service had it's own load balancer for each client, with it's own label for selecting pods. That'd completely break monitoring with the setup you propose here as the target labels could change every few minutes.","created_at":"2015-08-12T08:20:30Z","updated_at":"2015-08-12T08:20:30Z","html_url":"https://github.com/prometheus/prometheus/pull/905#discussion_r36836122","pull_request_url":"https://api.github.com/repos/prometheus/prometheus/pulls/905","_links":{"self":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/comments/36836122"},"html":{"href":"https://github.com/prometheus/prometheus/pull/905#discussion_r36836122"},"pull_request":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/905"}}},"pull_request":{"url":"https://api.github.com/repos/prometheus/prometheus/pulls/905","id":40298852,"html_url":"https://github.com/prometheus/prometheus/pull/905","diff_url":"https://github.com/prometheus/prometheus/pull/905.diff","patch_url":"https://github.com/prometheus/prometheus/pull/905.patch","issue_url":"https://api.github.com/repos/prometheus/prometheus/issues/905","number":905,"state":"open","locked":false,"title":"Kubernetes discovery","user":{"login":"jimmidyson","id":464659,"avatar_url":"https://avatars.githubusercontent.com/u/464659?v=3","gravatar_id":"","url":"https://api.github.com/users/jimmidyson","html_url":"https://github.com/jimmidyson","followers_url":"https://api.github.com/users/jimmidyson/followers","following_url":"https://api.github.com/users/jimmidyson/following{/other_user}","gists_url":"https://api.github.com/users/jimmidyson/gists{/gist_id}","starred_url":"https://api.github.com/users/jimmidyson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jimmidyson/subscriptions","organizations_url":"https://api.github.com/users/jimmidyson/orgs","repos_url":"https://api.github.com/users/jimmidyson/repos","events_url":"https://api.github.com/users/jimmidyson/events{/privacy}","received_events_url":"https://api.github.com/users/jimmidyson/received_events","type":"User","site_admin":false},"body":"As discussed in #904. This initial implementation adds in node discovery which scrapes pod/container metrics from all nodes in the cluster.\r\n\r\nWould love feedback please. Missing tests, although I guess only place that I can add them easily is to the config parsing? Oh & documentation.\r\n\r\nSorry about the amount of godeps required - not sure of how to avoid that if that's a problem? Certainly makes the PR harder to review. If you have a better way to add these deps in please let me know.","created_at":"2015-07-18T21:30:50Z","updated_at":"2015-08-12T08:20:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4654c3e469deba2187f7baa4a8c85ed8bb9cedc3","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/prometheus/prometheus/pulls/905/commits","review_comments_url":"https://api.github.com/repos/prometheus/prometheus/pulls/905/comments","review_comment_url":"https://api.github.com/repos/prometheus/prometheus/pulls/comments{/number}","comments_url":"https://api.github.com/repos/prometheus/prometheus/issues/905/comments","statuses_url":"https://api.github.com/repos/prometheus/prometheus/statuses/68dd401cc1c17c8538bad3e753cd44e08eec9f7b","head":{"label":"fabric8io:kubernetes-discovery","ref":"kubernetes-discovery","sha":"68dd401cc1c17c8538bad3e753cd44e08eec9f7b","user":{"login":"fabric8io","id":6074349,"avatar_url":"https://avatars.githubusercontent.com/u/6074349?v=3","gravatar_id":"","url":"https://api.github.com/users/fabric8io","html_url":"https://github.com/fabric8io","followers_url":"https://api.github.com/users/fabric8io/followers","following_url":"https://api.github.com/users/fabric8io/following{/other_user}","gists_url":"https://api.github.com/users/fabric8io/gists{/gist_id}","starred_url":"https://api.github.com/users/fabric8io/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabric8io/subscriptions","organizations_url":"https://api.github.com/users/fabric8io/orgs","repos_url":"https://api.github.com/users/fabric8io/repos","events_url":"https://api.github.com/users/fabric8io/events{/privacy}","received_events_url":"https://api.github.com/users/fabric8io/received_events","type":"Organization","site_admin":false},"repo":{"id":39276484,"name":"prometheus","full_name":"fabric8io/prometheus","owner":{"login":"fabric8io","id":6074349,"avatar_url":"https://avatars.githubusercontent.com/u/6074349?v=3","gravatar_id":"","url":"https://api.github.com/users/fabric8io","html_url":"https://github.com/fabric8io","followers_url":"https://api.github.com/users/fabric8io/followers","following_url":"https://api.github.com/users/fabric8io/following{/other_user}","gists_url":"https://api.github.com/users/fabric8io/gists{/gist_id}","starred_url":"https://api.github.com/users/fabric8io/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabric8io/subscriptions","organizations_url":"https://api.github.com/users/fabric8io/orgs","repos_url":"https://api.github.com/users/fabric8io/repos","events_url":"https://api.github.com/users/fabric8io/events{/privacy}","received_events_url":"https://api.github.com/users/fabric8io/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/fabric8io/prometheus","description":"The Prometheus monitoring system and time series database.","fork":true,"url":"https://api.github.com/repos/fabric8io/prometheus","forks_url":"https://api.github.com/repos/fabric8io/prometheus/forks","keys_url":"https://api.github.com/repos/fabric8io/prometheus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fabric8io/prometheus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fabric8io/prometheus/teams","hooks_url":"https://api.github.com/repos/fabric8io/prometheus/hooks","issue_events_url":"https://api.github.com/repos/fabric8io/prometheus/issues/events{/number}","events_url":"https://api.github.com/repos/fabric8io/prometheus/events","assignees_url":"https://api.github.com/repos/fabric8io/prometheus/assignees{/user}","branches_url":"https://api.github.com/repos/fabric8io/prometheus/branches{/branch}","tags_url":"https://api.github.com/repos/fabric8io/prometheus/tags","blobs_url":"https://api.github.com/repos/fabric8io/prometheus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fabric8io/prometheus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fabric8io/prometheus/git/refs{/sha}","trees_url":"https://api.github.com/repos/fabric8io/prometheus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fabric8io/prometheus/statuses/{sha}","languages_url":"https://api.github.com/repos/fabric8io/prometheus/languages","stargazers_url":"https://api.github.com/repos/fabric8io/prometheus/stargazers","contributors_url":"https://api.github.com/repos/fabric8io/prometheus/contributors","subscribers_url":"https://api.github.com/repos/fabric8io/prometheus/subscribers","subscription_url":"https://api.github.com/repos/fabric8io/prometheus/subscription","commits_url":"https://api.github.com/repos/fabric8io/prometheus/commits{/sha}","git_commits_url":"https://api.github.com/repos/fabric8io/prometheus/git/commits{/sha}","comments_url":"https://api.github.com/repos/fabric8io/prometheus/comments{/number}","issue_comment_url":"https://api.github.com/repos/fabric8io/prometheus/issues/comments{/number}","contents_url":"https://api.github.com/repos/fabric8io/prometheus/contents/{+path}","compare_url":"https://api.github.com/repos/fabric8io/prometheus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fabric8io/prometheus/merges","archive_url":"https://api.github.com/repos/fabric8io/prometheus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fabric8io/prometheus/downloads","issues_url":"https://api.github.com/repos/fabric8io/prometheus/issues{/number}","pulls_url":"https://api.github.com/repos/fabric8io/prometheus/pulls{/number}","milestones_url":"https://api.github.com/repos/fabric8io/prometheus/milestones{/number}","notifications_url":"https://api.github.com/repos/fabric8io/prometheus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fabric8io/prometheus/labels{/name}","releases_url":"https://api.github.com/repos/fabric8io/prometheus/releases{/id}","created_at":"2015-07-17T21:40:55Z","updated_at":"2015-08-05T10:45:38Z","pushed_at":"2015-08-11T20:39:24Z","git_url":"git://github.com/fabric8io/prometheus.git","ssh_url":"git@github.com:fabric8io/prometheus.git","clone_url":"https://github.com/fabric8io/prometheus.git","svn_url":"https://github.com/fabric8io/prometheus","homepage":"http://prometheus.io/","size":8491,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"prometheus:master","ref":"master","sha":"a8605825a0af7d97c232a9135ba478954d0666e9","user":{"login":"prometheus","id":3380462,"avatar_url":"https://avatars.githubusercontent.com/u/3380462?v=3","gravatar_id":"","url":"https://api.github.com/users/prometheus","html_url":"https://github.com/prometheus","followers_url":"https://api.github.com/users/prometheus/followers","following_url":"https://api.github.com/users/prometheus/following{/other_user}","gists_url":"https://api.github.com/users/prometheus/gists{/gist_id}","starred_url":"https://api.github.com/users/prometheus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prometheus/subscriptions","organizations_url":"https://api.github.com/users/prometheus/orgs","repos_url":"https://api.github.com/users/prometheus/repos","events_url":"https://api.github.com/users/prometheus/events{/privacy}","received_events_url":"https://api.github.com/users/prometheus/received_events","type":"Organization","site_admin":false},"repo":{"id":6838921,"name":"prometheus","full_name":"prometheus/prometheus","owner":{"login":"prometheus","id":3380462,"avatar_url":"https://avatars.githubusercontent.com/u/3380462?v=3","gravatar_id":"","url":"https://api.github.com/users/prometheus","html_url":"https://github.com/prometheus","followers_url":"https://api.github.com/users/prometheus/followers","following_url":"https://api.github.com/users/prometheus/following{/other_user}","gists_url":"https://api.github.com/users/prometheus/gists{/gist_id}","starred_url":"https://api.github.com/users/prometheus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prometheus/subscriptions","organizations_url":"https://api.github.com/users/prometheus/orgs","repos_url":"https://api.github.com/users/prometheus/repos","events_url":"https://api.github.com/users/prometheus/events{/privacy}","received_events_url":"https://api.github.com/users/prometheus/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/prometheus/prometheus","description":"The Prometheus monitoring system and time series database.","fork":false,"url":"https://api.github.com/repos/prometheus/prometheus","forks_url":"https://api.github.com/repos/prometheus/prometheus/forks","keys_url":"https://api.github.com/repos/prometheus/prometheus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/prometheus/prometheus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/prometheus/prometheus/teams","hooks_url":"https://api.github.com/repos/prometheus/prometheus/hooks","issue_events_url":"https://api.github.com/repos/prometheus/prometheus/issues/events{/number}","events_url":"https://api.github.com/repos/prometheus/prometheus/events","assignees_url":"https://api.github.com/repos/prometheus/prometheus/assignees{/user}","branches_url":"https://api.github.com/repos/prometheus/prometheus/branches{/branch}","tags_url":"https://api.github.com/repos/prometheus/prometheus/tags","blobs_url":"https://api.github.com/repos/prometheus/prometheus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/prometheus/prometheus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/prometheus/prometheus/git/refs{/sha}","trees_url":"https://api.github.com/repos/prometheus/prometheus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/prometheus/prometheus/statuses/{sha}","languages_url":"https://api.github.com/repos/prometheus/prometheus/languages","stargazers_url":"https://api.github.com/repos/prometheus/prometheus/stargazers","contributors_url":"https://api.github.com/repos/prometheus/prometheus/contributors","subscribers_url":"https://api.github.com/repos/prometheus/prometheus/subscribers","subscription_url":"https://api.github.com/repos/prometheus/prometheus/subscription","commits_url":"https://api.github.com/repos/prometheus/prometheus/commits{/sha}","git_commits_url":"https://api.github.com/repos/prometheus/prometheus/git/commits{/sha}","comments_url":"https://api.github.com/repos/prometheus/prometheus/comments{/number}","issue_comment_url":"https://api.github.com/repos/prometheus/prometheus/issues/comments{/number}","contents_url":"https://api.github.com/repos/prometheus/prometheus/contents/{+path}","compare_url":"https://api.github.com/repos/prometheus/prometheus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/prometheus/prometheus/merges","archive_url":"https://api.github.com/repos/prometheus/prometheus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/prometheus/prometheus/downloads","issues_url":"https://api.github.com/repos/prometheus/prometheus/issues{/number}","pulls_url":"https://api.github.com/repos/prometheus/prometheus/pulls{/number}","milestones_url":"https://api.github.com/repos/prometheus/prometheus/milestones{/number}","notifications_url":"https://api.github.com/repos/prometheus/prometheus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/prometheus/prometheus/labels{/name}","releases_url":"https://api.github.com/repos/prometheus/prometheus/releases{/id}","created_at":"2012-11-24T11:14:12Z","updated_at":"2015-08-12T07:31:16Z","pushed_at":"2015-08-11T22:55:09Z","git_url":"git://github.com/prometheus/prometheus.git","ssh_url":"git@github.com:prometheus/prometheus.git","clone_url":"https://github.com/prometheus/prometheus.git","svn_url":"https://github.com/prometheus/prometheus","homepage":"http://prometheus.io/","size":18012,"stargazers_count":2740,"watchers_count":2740,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":171,"mirror_url":null,"open_issues_count":94,"forks":171,"open_issues":94,"watchers":2740,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/905"},"html":{"href":"https://github.com/prometheus/prometheus/pull/905"},"issue":{"href":"https://api.github.com/repos/prometheus/prometheus/issues/905"},"comments":{"href":"https://api.github.com/repos/prometheus/prometheus/issues/905/comments"},"review_comments":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/905/comments"},"review_comment":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/905/commits"},"statuses":{"href":"https://api.github.com/repos/prometheus/prometheus/statuses/68dd401cc1c17c8538bad3e753cd44e08eec9f7b"}}}}