{"action":"created","comment":{"url":"https://api.github.com/repos/prometheus/prometheus/pulls/comments/36950053","id":36950053,"diff_hunk":"@@ -0,0 +1,552 @@\n+// Copyright 2015 The Prometheus Authors\n+// Licensed under the Apache License, Version 2.0 (the \"License\");\n+// you may not use this file except in compliance with the License.\n+// You may obtain a copy of the License at\n+//\n+// http://www.apache.org/licenses/LICENSE-2.0\n+//\n+// Unless required by applicable law or agreed to in writing, software\n+// distributed under the License is distributed on an \"AS IS\" BASIS,\n+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+// See the License for the specific language governing permissions and\n+// limitations under the License.\n+\n+package kubernetes\n+\n+import (\n+\t\"crypto/tls\"\n+\t\"crypto/x509\"\n+\t\"encoding/json\"\n+\t\"fmt\"\n+\t\"io/ioutil\"\n+\t\"net\"\n+\t\"net/http\"\n+\t\"os\"\n+\t\"sync\"\n+\t\"time\"\n+\n+\tclientmodel \"github.com/prometheus/client_golang/model\"\n+\t\"github.com/prometheus/log\"\n+\t\"github.com/prometheus/prometheus/config\"\n+\t\"github.com/prometheus/prometheus/util/httputil\"\n+\t\"github.com/prometheus/prometheus/util/strutil\"\n+)\n+\n+const (\n+\tsourceServicePrefix = \"services\"\n+\n+\t// kubernetesMetaLabelPrefix is the meta prefix used for all meta labels\n+\t// in this discovery.\n+\tmetaLabelPrefix = clientmodel.MetaLabelPrefix + \"kubernetes_\"\n+\t// nodeLabel is the name for the label containing a target's node name.\n+\tnodeLabel = metaLabelPrefix + \"node\"\n+\t// serviceNamespaceLabel is the name for the label containing a target's service namespace.\n+\tserviceNamespaceLabel = metaLabelPrefix + \"service_namespace\"\n+\t// serviceNameLabel is the name for the label containing a target's service name.\n+\tserviceNameLabel = metaLabelPrefix + \"service_name\"\n+\t// nodeLabelPrefix is the prefix for the node labels\n+\tnodeLabelPrefix = metaLabelPrefix + \"node_label_\"\n+\t// serviceLabelPrefix is the prefix for the service labels\n+\tserviceLabelPrefix = metaLabelPrefix + \"service_label_\"\n+\t// serviceAnnotationPrefix is the prefix for the service annotations\n+\tserviceAnnotationPrefix = metaLabelPrefix + \"service_annotation_\"\n+\t// nodesTargetGroupName is the name given to the target group for nodes.\n+\tnodesTargetGroupName = \"nodes\"\n+\n+\tserviceAccountToken  = \"/var/run/secrets/kubernetes.io/serviceaccount/token\"\n+\tserviceAccountCACert = \"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\"\n+\n+\tapiVersion          = \"v1\"\n+\tapiPrefix           = \"api/\" + apiVersion\n+\tnodesURL            = apiPrefix + \"/nodes\"\n+\tservicesURL         = apiPrefix + \"/services\"\n+\tendpointsURL        = apiPrefix + \"/endpoints\"\n+\tserviceEndpointsURL = apiPrefix + \"/namespaces/%s/endpoints/%s\"\n+)\n+\n+type KubernetesDiscovery struct {\n+\tclient *http.Client\n+\tConf   *config.KubernetesSDConfig\n+\n+\tnodesResourceVersion     string\n+\tservicesResourceVersion  string\n+\tendpointsResourceVersion string\n+\tnodes                    map[string]*Node\n+\tservices                 map[string]map[string]*Service\n+\tnodesMu                  sync.RWMutex\n+\tservicesMu               sync.RWMutex\n+\trunDone                  chan struct{}\n+}\n+\n+func (kd *KubernetesDiscovery) Initialize() error {\n+\tclient, err := newKubernetesHTTPClient(kd.Conf)\n+\n+\tif err != nil {\n+\t\treturn err\n+\t}\n+\n+\tkd.client = client\n+\tkd.nodes = make(map[string]*Node)\n+\tkd.services = make(map[string]map[string]*Service)\n+\tkd.runDone = make(chan struct{})\n+\n+\treturn nil\n+}\n+\n+// Sources implements the TargetProvider interface.\n+func (kd *KubernetesDiscovery) Sources() []string {\n+\tres, err := kd.client.Get(kd.Conf.Server + nodesURL)\n+\tif err != nil {\n+\t\t// If we can't list nodes then we can't watch them. Assume this is a misconfiguration\n+\t\t// & log & return empty.\n+\t\tlog.Errorf(\"discovery.Sources: %s\", err)\n+\t\treturn []string{}\n+\t}\n+\tif res.StatusCode != http.StatusOK {\n+\t\tlog.Errorf(\"discovery.Sources: %s\", res.Body)\n+\t\treturn []string{}\n+\t}\n+\n+\tvar nodes NodeList\n+\tif err := json.NewDecoder(res.Body).Decode(&nodes); err != nil {\n+\t\tlog.Errorf(\"discovery.Sources: %s\", err)\n+\t\treturn []string{}\n+\t}\n+\n+\tkd.nodesMu.Lock()\n+\tdefer kd.nodesMu.Unlock()\n+\n+\tsourceNames := make([]string, 0, len(nodes.Items))\n+\n+\tkd.nodesResourceVersion = nodes.ResourceVersion\n+\tfor idx, node := range nodes.Items {\n+\t\tsourceNames = append(sourceNames, nodesTargetGroupName+\":\"+node.ObjectMeta.Name)\n+\t\tkd.nodes[node.ObjectMeta.Name] = &nodes.Items[idx]\n+\t}\n+\n+\tres, err = kd.client.Get(kd.Conf.Server + servicesURL)\n+\tif err != nil {\n+\t\t// If we can't list services then we can't watch them. Assume this is a misconfiguration\n+\t\t// & log & return empty.\n+\t\tlog.Errorf(\"discovery.Sources: %s\", err)\n+\t\treturn []string{}\n+\t}\n+\tif res.StatusCode != http.StatusOK {\n+\t\tlog.Errorf(\"discovery.Sources: %s\", res.Body)\n+\t\treturn []string{}\n+\t}\n+\tvar services ServiceList\n+\tif err := json.NewDecoder(res.Body).Decode(&services); err != nil {\n+\t\tlog.Errorf(\"discovery.Sources: %s\", err)\n+\t\treturn []string{}\n+\t}\n+\tif err != nil {\n+\t\t// If we can't list nodes then we can't watch them. Assume this is a misconfiguration\n+\t\t// & log & return empty.\n+\t\tlog.Errorf(\"discovery.Sources: %s\", err)\n+\t\treturn []string{}\n+\t}\n+\tkd.servicesMu.Lock()\n+\tdefer kd.servicesMu.Unlock()\n+\n+\tkd.servicesResourceVersion = services.ResourceVersion\n+\tfor idx, service := range services.Items {\n+\t\tsourceNames = append(sourceNames, sourceServicePrefix+\":\"+service.ObjectMeta.Namespace+\"/\"+service.ObjectMeta.Name)\n+\t\tnamespace, ok := kd.services[service.ObjectMeta.Namespace]\n+\t\tif !ok {\n+\t\t\tnamespace = make(map[string]*Service)\n+\t\t\tkd.services[service.ObjectMeta.Namespace] = namespace\n+\t\t}\n+\t\tnamespace[service.ObjectMeta.Name] = &services.Items[idx]\n+\t}\n+\n+\treturn sourceNames\n+}\n+\n+// Run implements the TargetProvider interface.\n+func (kd *KubernetesDiscovery) Run(ch chan<- *config.TargetGroup) {\n+\tdefer close(ch)\n+\n+\tkd.updateNodesTargetGroup(ch)\n+\n+\tfor _, ns := range kd.services {\n+\t\tfor _, service := range ns {\n+\t\t\tkd.addService(service, ch)\n+\t\t}\n+\t}\n+\n+\tretryDelay := time.Duration(kd.Conf.RetryInterval)\n+\n+\tupdate := make(chan interface{}, 10)\n+\tdefer close(update)\n+\n+\tgo kd.watchNodes(update, retryDelay)\n+\tgo kd.watchServices(update, retryDelay)\n+\tgo kd.watchServiceEndpoints(update, retryDelay)\n+\n+\tfor {\n+\t\tselect {\n+\t\tcase <-kd.runDone:\n+\t\t\treturn\n+\t\tcase event := <-update:\n+\t\t\tswitch obj := event.(type) {\n+\t\t\tcase *nodeEvent:\n+\t\t\t\tkd.updateNode(obj.Node, obj.EventType)\n+\t\t\t\tkd.updateNodesTargetGroup(ch)\n+\t\t\tcase *serviceEvent:\n+\t\t\t\tkd.updateService(obj.Service, obj.EventType, ch)\n+\t\t\tcase *endpointsEvent:\n+\t\t\t\tkd.updateServiceEndpoints(obj.Endpoints, obj.EventType, ch)\n+\t\t\t}\n+\t\t}\n+\t}\n+}\n+\n+// Stop implements the TargetProvider interface.\n+func (kd *KubernetesDiscovery) Stop() {\n+\tlog.Debugf(\"Stopping Kubernetes discovery for %s\", kd.Conf.Server)\n+\n+\t// The lock prevents Run from terminating while the watchers attempt\n+\t// to send on their channels.\n+\tkd.nodesMu.Lock()\n+\tdefer kd.nodesMu.Unlock()\n+\tkd.servicesMu.Lock()\n+\tdefer kd.servicesMu.Unlock()\n+\n+\t// Terminate Run.\n+\tkd.runDone <- struct{}{}\n+\n+\tlog.Debugf(\"Kubernetes discovery for %s stopped.\", kd.Conf.Server)\n+}\n+\n+func (kd *KubernetesDiscovery) updateNodesTargetGroup(ch chan<- *config.TargetGroup) {\n+\tkd.nodesMu.Lock()\n+\tdefer kd.nodesMu.Unlock()\n+\ttg := &config.TargetGroup{Source: nodesTargetGroupName}\n+\n+\t// Now let's loop through the nodes & add them to the target group with appropriate labels.\n+\tfor nodeName, node := range kd.nodes {\n+\t\taddress := fmt.Sprintf(\"%s:%d\", node.Status.Addresses[0].Address, kd.Conf.KubeletPort)\n+\n+\t\tt := clientmodel.LabelSet{\n+\t\t\tclientmodel.AddressLabel: clientmodel.LabelValue(address),\n+\t\t\tnodeLabel:                clientmodel.LabelValue(nodeName),\n+\t\t}\n+\t\tfor k, v := range node.ObjectMeta.Labels {\n+\t\t\tlabelName := strutil.SanitizeLabelName(nodeLabelPrefix + k)\n+\t\t\tt[clientmodel.LabelName(labelName)] = clientmodel.LabelValue(v)\n+\t\t}\n+\t\ttg.Targets = append(tg.Targets, t)\n+\t}\n+\n+\tch <- tg\n+}\n+\n+func (kd *KubernetesDiscovery) updateNode(node *Node, eventType EventType) {\n+\tkd.nodesMu.Lock()\n+\tdefer kd.nodesMu.Unlock()\n+\tupdatedNodeName := node.ObjectMeta.Name\n+\tswitch eventType {\n+\tcase \"Deleted\":\n+\t\t// Deleted - remove from nodes map\n+\t\tdelete(kd.nodes, updatedNodeName)\n+\tcase \"Added\", \"Modified\":\n+\t\t// Added/Modified - update the node in the nodes map\n+\t\tkd.nodes[updatedNodeName] = node\n+\t}\n+}\n+\n+// watchNodes watches nodes as they come & go.\n+func (kd *KubernetesDiscovery) watchNodes(events chan interface{}, retryDelay time.Duration) {\n+\tuntil(func() {\n+\t\treq, err := http.NewRequest(\"GET\", kd.Conf.Server+nodesURL, nil)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Failed to watch nodes: %s\", err)\n+\t\t\treturn\n+\t\t}\n+\t\tvalues := req.URL.Query()\n+\t\tvalues.Add(\"watch\", \"true\")\n+\t\tvalues.Add(\"resourceVersion\", kd.nodesResourceVersion)\n+\t\treq.URL.RawQuery = values.Encode()\n+\t\tres, err := kd.client.Do(req)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Failed to watch nodes: %s\", err)\n+\t\t\treturn\n+\t\t}\n+\t\tif res.StatusCode != http.StatusOK {\n+\t\t\tlog.Errorf(\"Failed to watch nodes: %s\", res.StatusCode)\n+\t\t\treturn\n+\t\t}\n+\n+\t\td := json.NewDecoder(res.Body)\n+\n+\t\tfor {\n+\t\t\tvar event nodeEvent\n+\t\t\tif err := d.Decode(&event); err != nil {\n+\t\t\t\tlog.Errorf(\"Failed to watch nodes: %s\", err)\n+\t\t\t\treturn\n+\t\t\t}\n+\t\t\tkd.nodesResourceVersion = event.Node.ObjectMeta.ResourceVersion\n+\t\t\tevents <- &event\n+\t\t}\n+\t}, retryDelay, kd.runDone)\n+}\n+\n+// watchServices watches services as they come & go.\n+func (kd *KubernetesDiscovery) watchServices(events chan interface{}, retryDelay time.Duration) {\n+\tuntil(func() {\n+\t\treq, err := http.NewRequest(\"GET\", kd.Conf.Server+servicesURL, nil)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Failed to watch services: %s\", err)\n+\t\t\treturn\n+\t\t}\n+\t\tvalues := req.URL.Query()\n+\t\tvalues.Add(\"watch\", \"true\")\n+\t\tvalues.Add(\"resourceVersion\", kd.servicesResourceVersion)\n+\t\treq.URL.RawQuery = values.Encode()\n+\n+\t\tres, err := kd.client.Do(req)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Failed to watch services: %s\", err)\n+\t\t\treturn\n+\t\t}\n+\t\tif res.StatusCode != http.StatusOK {\n+\t\t\tlog.Errorf(\"Failed to watch services: %s\", res.StatusCode)\n+\t\t\treturn\n+\t\t}\n+\n+\t\td := json.NewDecoder(res.Body)\n+\n+\t\tfor {\n+\t\t\tvar event serviceEvent\n+\t\t\tif err := d.Decode(&event); err != nil {\n+\t\t\t\tlog.Errorf(\"Unable to watch services: %s\", err)\n+\t\t\t\treturn\n+\t\t\t}\n+\t\t\tkd.servicesResourceVersion = event.Service.ObjectMeta.ResourceVersion\n+\t\t\tevents <- &event\n+\t\t}\n+\t}, retryDelay, kd.runDone)\n+}\n+\n+func (kd *KubernetesDiscovery) updateService(service *Service, eventType EventType, ch chan<- *config.TargetGroup) {\n+\tkd.servicesMu.Lock()\n+\tdefer kd.servicesMu.Unlock()\n+\tname := service.ObjectMeta.Name\n+\tnamespace := service.ObjectMeta.Namespace\n+\t_, ok := kd.services[namespace][name]\n+\tswitch eventType {\n+\tcase deleted:\n+\t\tif ok {\n+\t\t\tkd.deleteService(service, ch)\n+\t\t}\n+\tcase added, modified:\n+\t\tkd.addService(service, ch)\n+\t}\n+}\n+\n+func (kd *KubernetesDiscovery) deleteService(service *Service, ch chan<- *config.TargetGroup) {\n+\ttg := &config.TargetGroup{Source: sourceServicePrefix + \":\" + service.ObjectMeta.Namespace + \"/\" + service.ObjectMeta.Name}\n+\tch <- tg\n+\tdelete(kd.services[service.ObjectMeta.Namespace], service.ObjectMeta.Name)\n+\tif len(kd.services[service.ObjectMeta.Namespace]) == 0 {\n+\t\tdelete(kd.services, service.ObjectMeta.Namespace)\n+\t}\n+}\n+\n+func (kd *KubernetesDiscovery) addService(service *Service, ch chan<- *config.TargetGroup) {\n+\tnamespace, ok := kd.services[service.ObjectMeta.Namespace]\n+\tif !ok {\n+\t\tnamespace = make(map[string]*Service)\n+\t\tkd.services[service.ObjectMeta.Namespace] = namespace\n+\t}\n+\tnamespace[service.ObjectMeta.Name] = service\n+\tendpointURL := fmt.Sprintf(serviceEndpointsURL, service.ObjectMeta.Namespace, service.ObjectMeta.Name)\n+\tres, err := kd.client.Get(kd.Conf.Server + endpointURL)\n+\tif err != nil {\n+\t\tlog.Errorf(\"Error getting service endpoints: %s\", err)\n+\t\treturn\n+\t}\n+\tif res.StatusCode != http.StatusOK {\n+\t\tlog.Errorf(\"Failed to get service endpoints: %s\", res.StatusCode)\n+\t\treturn\n+\t}\n+\n+\tvar endpoints Endpoints\n+\tif err := json.NewDecoder(res.Body).Decode(&endpoints); err != nil {\n+\t\tlog.Errorf(\"Error getting service endpoints: %s\", err)\n+\t\treturn\n+\t}\n+\n+\tkd.updateServiceTargetGroup(service, &endpoints, ch)\n+}\n+\n+func (kd *KubernetesDiscovery) updateServiceTargetGroup(service *Service, endpoints *Endpoints, ch chan<- *config.TargetGroup) {\n+\ttg := &config.TargetGroup{\n+\t\tSource: sourceServicePrefix + \":\" + service.ObjectMeta.Namespace + \"/\" + service.ObjectMeta.Name,\n+\t\tLabels: clientmodel.LabelSet{\n+\t\t\tserviceNamespaceLabel: clientmodel.LabelValue(service.ObjectMeta.Namespace),\n+\t\t\tserviceNameLabel:      clientmodel.LabelValue(service.ObjectMeta.Name),\n+\t\t},\n+\t}\n+\n+\tfor k, v := range service.ObjectMeta.Labels {\n+\t\tlabelName := strutil.SanitizeLabelName(serviceLabelPrefix + k)\n+\t\ttg.Labels[clientmodel.LabelName(labelName)] = clientmodel.LabelValue(v)\n+\t}\n+\n+\tfor k, v := range service.ObjectMeta.Annotations {\n+\t\tlabelName := strutil.SanitizeLabelName(serviceAnnotationPrefix + k)\n+\t\ttg.Labels[clientmodel.LabelName(labelName)] = clientmodel.LabelValue(v)\n+\t}\n+\n+\t// Now let's loop through the endpoints & add them to the target group with appropriate labels\n+\tfor _, eps := range endpoints.Subsets {\n+\t\tepPort := eps.Ports[0].Port\n+\n+\t\tfor _, addr := range eps.Addresses {\n+\t\t\tipAddr := addr.IP\n+\t\t\tif len(ipAddr) == net.IPv6len {\n+\t\t\t\tipAddr = \"[\" + ipAddr + \"]\"\n+\t\t\t}\n+\t\t\taddress := fmt.Sprintf(\"%s:%d\", ipAddr, epPort)\n+\n+\t\t\tt := clientmodel.LabelSet{clientmodel.AddressLabel: clientmodel.LabelValue(address)}\n+\n+\t\t\ttg.Targets = append(tg.Targets, t)\n+\t\t}\n+\t}\n+\n+\tch <- tg\n+}\n+\n+// watchServiceEndpoints watches service endpoints as they come & go\n+func (kd *KubernetesDiscovery) watchServiceEndpoints(events chan interface{}, retryDelay time.Duration) {\n+\tuntil(func() {\n+\t\treq, err := http.NewRequest(\"GET\", kd.Conf.Server+endpointsURL, nil)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Failed to watch service endpoints: %s\", err)\n+\t\t\treturn\n+\t\t}\n+\t\tvalues := req.URL.Query()\n+\t\tvalues.Add(\"watch\", \"true\")\n+\t\tvalues.Add(\"resourceVersion\", kd.servicesResourceVersion)\n+\t\treq.URL.RawQuery = values.Encode()\n+\n+\t\tres, err := kd.client.Do(req)\n+\t\tif err != nil {\n+\t\t\tlog.Errorf(\"Failed to watch service endpoints: %s\", err)\n+\t\t\treturn\n+\t\t}\n+\t\tif res.StatusCode != http.StatusOK {\n+\t\t\tlog.Errorf(\"Failed to watch service endpoints: %s\", res.StatusCode)\n+\t\t\treturn\n+\t\t}\n+\n+\t\td := json.NewDecoder(res.Body)\n+\n+\t\tfor {\n+\t\t\tvar event endpointsEvent\n+\t\t\tif err := d.Decode(&event); err != nil {\n+\t\t\t\tlog.Errorf(\"Unable to watch service endpoints: %s\", err)\n+\t\t\t\treturn\n+\t\t\t}\n+\t\t\tkd.servicesResourceVersion = event.Endpoints.ObjectMeta.ResourceVersion\n+\t\t\tevents <- &event\n+\t\t}\n+\t}, retryDelay, kd.runDone)\n+}\n+\n+func (kd *KubernetesDiscovery) updateServiceEndpoints(endpoints *Endpoints, eventType EventType, ch chan<- *config.TargetGroup) {\n+\tkd.servicesMu.Lock()\n+\tdefer kd.servicesMu.Unlock()\n+\tserviceNamespace := endpoints.ObjectMeta.Namespace\n+\tserviceName := endpoints.ObjectMeta.Name\n+\tif service, ok := kd.services[serviceNamespace][serviceName]; ok {\n+\t\tkd.updateServiceTargetGroup(service, endpoints, ch)\n+\t}\n+}\n+\n+func newKubernetesHTTPClient(conf *config.KubernetesSDConfig) (*http.Client, error) {\n+\tbearerTokenFile := conf.BearerTokenFile\n+\tcaFile := conf.CAFile\n+\tif conf.InCluster {\n+\t\tif len(bearerTokenFile) == 0 {\n+\t\t\tbearerTokenFile = serviceAccountToken\n+\t\t}\n+\t\tif len(caFile) == 0 {\n+\t\t\t// With recent versions, the CA certificate is provided as a token\n+\t\t\t// but we need to handle older versions too. In this case, don't\n+\t\t\t// set the CAFile & the configuration will have to use Insecure\n+\t\t\tif _, err := os.Stat(serviceAccountCACert); err == nil {\n+\t\t\t\tcaFile = serviceAccountCACert\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\ttlsConfig := &tls.Config{InsecureSkipVerify: conf.Insecure}\n+\n+\t// Load client cert if specified\n+\tif len(conf.CertFile) > 0 && len(conf.KeyFile) > 0 {\n+\t\tcert, err := tls.LoadX509KeyPair(conf.CertFile, conf.KeyFile)\n+\t\tif err != nil {\n+\t\t\treturn nil, err\n+\t\t}\n+\t\ttlsConfig.Certificates = []tls.Certificate{cert}\n+\t}\n+\n+\tcaCertPool := x509.NewCertPool()\n+\tif len(caFile) > 0 {\n+\t\t// Load CA cert\n+\t\tcaCert, err := ioutil.ReadFile(caFile)\n+\t\tif err != nil {\n+\t\t\treturn nil, err\n+\t\t}\n+\t\tcaCertPool.AppendCertsFromPEM(caCert)\n+\t}\n+\ttlsConfig.RootCAs = caCertPool\n+\n+\ttlsConfig.BuildNameToCertificate()\n+\n+\ttr := &http.Transport{\n+\t\tDial: func(netw, addr string) (c net.Conn, err error) {\n+\t\t\tc, err = net.DialTimeout(netw, addr, time.Duration(conf.RequestTimeout))\n+\t\t\treturn\n+\t\t},\n+\t}\n+\ttr.TLSClientConfig = tlsConfig\n+\tvar rt http.RoundTripper\n+\trt = tr\n+\n+\tbearerToken, err := ioutil.ReadFile(bearerTokenFile)\n+\tif err != nil {\n+\t\treturn nil, err\n+\t}\n+\n+\tif len(bearerToken) > 0 {\n+\t\trt = httputil.NewBearerAuthRoundTripper(string(bearerToken), rt)\n+\t}\n+\tif len(conf.Username) > 0 && len(conf.Password) > 0 {\n+\t\trt = httputil.NewBasicAuthRoundTripper(conf.Username, conf.Password, rt)\n+\t}\n+\n+\treturn &http.Client{\n+\t\tTransport: rt,\n+\t}, nil\n+}\n+\n+// Until loops until stop channel is closed, running f every period.\n+// f may not be invoked if stop channel is already closed.\n+func until(f func(), period time.Duration, stopCh <-chan struct{}) {\n+\tfor {\n+\t\tselect {\n+\t\tcase <-stopCh:\n+\t\t\treturn\n+\t\tdefault:\n+\t\t}\n+\t\tfunc() {","path":"retrieval/discovery/kubernetes/discovery.go","position":547,"original_position":547,"commit_id":"1df67691f7d56a7943820f8084f7e5d33c46f4db","original_commit_id":"dc2c80b5ef88a7171c66ccf28d65ad055dd547ff","user":{"login":"jimmidyson","id":464659,"avatar_url":"https://avatars.githubusercontent.com/u/464659?v=3","gravatar_id":"","url":"https://api.github.com/users/jimmidyson","html_url":"https://github.com/jimmidyson","followers_url":"https://api.github.com/users/jimmidyson/followers","following_url":"https://api.github.com/users/jimmidyson/following{/other_user}","gists_url":"https://api.github.com/users/jimmidyson/gists{/gist_id}","starred_url":"https://api.github.com/users/jimmidyson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jimmidyson/subscriptions","organizations_url":"https://api.github.com/users/jimmidyson/orgs","repos_url":"https://api.github.com/users/jimmidyson/repos","events_url":"https://api.github.com/users/jimmidyson/events{/privacy}","received_events_url":"https://api.github.com/users/jimmidyson/received_events","type":"User","site_admin":false},"body":"Removing initial `f()` in your example would mean waiting for `period` for first invocation - again not desirable.","created_at":"2015-08-13T08:18:05Z","updated_at":"2015-08-13T08:18:05Z","html_url":"https://github.com/prometheus/prometheus/pull/905#discussion_r36950053","pull_request_url":"https://api.github.com/repos/prometheus/prometheus/pulls/905","_links":{"self":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/comments/36950053"},"html":{"href":"https://github.com/prometheus/prometheus/pull/905#discussion_r36950053"},"pull_request":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/905"}}},"pull_request":{"url":"https://api.github.com/repos/prometheus/prometheus/pulls/905","id":40298852,"html_url":"https://github.com/prometheus/prometheus/pull/905","diff_url":"https://github.com/prometheus/prometheus/pull/905.diff","patch_url":"https://github.com/prometheus/prometheus/pull/905.patch","issue_url":"https://api.github.com/repos/prometheus/prometheus/issues/905","number":905,"state":"open","locked":false,"title":"Kubernetes discovery","user":{"login":"jimmidyson","id":464659,"avatar_url":"https://avatars.githubusercontent.com/u/464659?v=3","gravatar_id":"","url":"https://api.github.com/users/jimmidyson","html_url":"https://github.com/jimmidyson","followers_url":"https://api.github.com/users/jimmidyson/followers","following_url":"https://api.github.com/users/jimmidyson/following{/other_user}","gists_url":"https://api.github.com/users/jimmidyson/gists{/gist_id}","starred_url":"https://api.github.com/users/jimmidyson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jimmidyson/subscriptions","organizations_url":"https://api.github.com/users/jimmidyson/orgs","repos_url":"https://api.github.com/users/jimmidyson/repos","events_url":"https://api.github.com/users/jimmidyson/events{/privacy}","received_events_url":"https://api.github.com/users/jimmidyson/received_events","type":"User","site_admin":false},"body":"As discussed in #904. This initial implementation adds in node discovery which scrapes pod/container metrics from all nodes in the cluster.\r\n\r\nWould love feedback please. Missing tests, although I guess only place that I can add them easily is to the config parsing? Oh & documentation.\r\n\r\nSorry about the amount of godeps required - not sure of how to avoid that if that's a problem? Certainly makes the PR harder to review. If you have a better way to add these deps in please let me know.","created_at":"2015-07-18T21:30:50Z","updated_at":"2015-08-13T08:18:05Z","closed_at":null,"merged_at":null,"merge_commit_sha":"88f3093d54075d5bb03a7d8160e8c9ca25822566","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/prometheus/prometheus/pulls/905/commits","review_comments_url":"https://api.github.com/repos/prometheus/prometheus/pulls/905/comments","review_comment_url":"https://api.github.com/repos/prometheus/prometheus/pulls/comments{/number}","comments_url":"https://api.github.com/repos/prometheus/prometheus/issues/905/comments","statuses_url":"https://api.github.com/repos/prometheus/prometheus/statuses/1df67691f7d56a7943820f8084f7e5d33c46f4db","head":{"label":"fabric8io:kubernetes-discovery","ref":"kubernetes-discovery","sha":"1df67691f7d56a7943820f8084f7e5d33c46f4db","user":{"login":"fabric8io","id":6074349,"avatar_url":"https://avatars.githubusercontent.com/u/6074349?v=3","gravatar_id":"","url":"https://api.github.com/users/fabric8io","html_url":"https://github.com/fabric8io","followers_url":"https://api.github.com/users/fabric8io/followers","following_url":"https://api.github.com/users/fabric8io/following{/other_user}","gists_url":"https://api.github.com/users/fabric8io/gists{/gist_id}","starred_url":"https://api.github.com/users/fabric8io/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabric8io/subscriptions","organizations_url":"https://api.github.com/users/fabric8io/orgs","repos_url":"https://api.github.com/users/fabric8io/repos","events_url":"https://api.github.com/users/fabric8io/events{/privacy}","received_events_url":"https://api.github.com/users/fabric8io/received_events","type":"Organization","site_admin":false},"repo":{"id":39276484,"name":"prometheus","full_name":"fabric8io/prometheus","owner":{"login":"fabric8io","id":6074349,"avatar_url":"https://avatars.githubusercontent.com/u/6074349?v=3","gravatar_id":"","url":"https://api.github.com/users/fabric8io","html_url":"https://github.com/fabric8io","followers_url":"https://api.github.com/users/fabric8io/followers","following_url":"https://api.github.com/users/fabric8io/following{/other_user}","gists_url":"https://api.github.com/users/fabric8io/gists{/gist_id}","starred_url":"https://api.github.com/users/fabric8io/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabric8io/subscriptions","organizations_url":"https://api.github.com/users/fabric8io/orgs","repos_url":"https://api.github.com/users/fabric8io/repos","events_url":"https://api.github.com/users/fabric8io/events{/privacy}","received_events_url":"https://api.github.com/users/fabric8io/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/fabric8io/prometheus","description":"The Prometheus monitoring system and time series database.","fork":true,"url":"https://api.github.com/repos/fabric8io/prometheus","forks_url":"https://api.github.com/repos/fabric8io/prometheus/forks","keys_url":"https://api.github.com/repos/fabric8io/prometheus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fabric8io/prometheus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fabric8io/prometheus/teams","hooks_url":"https://api.github.com/repos/fabric8io/prometheus/hooks","issue_events_url":"https://api.github.com/repos/fabric8io/prometheus/issues/events{/number}","events_url":"https://api.github.com/repos/fabric8io/prometheus/events","assignees_url":"https://api.github.com/repos/fabric8io/prometheus/assignees{/user}","branches_url":"https://api.github.com/repos/fabric8io/prometheus/branches{/branch}","tags_url":"https://api.github.com/repos/fabric8io/prometheus/tags","blobs_url":"https://api.github.com/repos/fabric8io/prometheus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fabric8io/prometheus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fabric8io/prometheus/git/refs{/sha}","trees_url":"https://api.github.com/repos/fabric8io/prometheus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fabric8io/prometheus/statuses/{sha}","languages_url":"https://api.github.com/repos/fabric8io/prometheus/languages","stargazers_url":"https://api.github.com/repos/fabric8io/prometheus/stargazers","contributors_url":"https://api.github.com/repos/fabric8io/prometheus/contributors","subscribers_url":"https://api.github.com/repos/fabric8io/prometheus/subscribers","subscription_url":"https://api.github.com/repos/fabric8io/prometheus/subscription","commits_url":"https://api.github.com/repos/fabric8io/prometheus/commits{/sha}","git_commits_url":"https://api.github.com/repos/fabric8io/prometheus/git/commits{/sha}","comments_url":"https://api.github.com/repos/fabric8io/prometheus/comments{/number}","issue_comment_url":"https://api.github.com/repos/fabric8io/prometheus/issues/comments{/number}","contents_url":"https://api.github.com/repos/fabric8io/prometheus/contents/{+path}","compare_url":"https://api.github.com/repos/fabric8io/prometheus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fabric8io/prometheus/merges","archive_url":"https://api.github.com/repos/fabric8io/prometheus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fabric8io/prometheus/downloads","issues_url":"https://api.github.com/repos/fabric8io/prometheus/issues{/number}","pulls_url":"https://api.github.com/repos/fabric8io/prometheus/pulls{/number}","milestones_url":"https://api.github.com/repos/fabric8io/prometheus/milestones{/number}","notifications_url":"https://api.github.com/repos/fabric8io/prometheus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fabric8io/prometheus/labels{/name}","releases_url":"https://api.github.com/repos/fabric8io/prometheus/releases{/id}","created_at":"2015-07-17T21:40:55Z","updated_at":"2015-08-05T10:45:38Z","pushed_at":"2015-08-13T08:06:45Z","git_url":"git://github.com/fabric8io/prometheus.git","ssh_url":"git@github.com:fabric8io/prometheus.git","clone_url":"https://github.com/fabric8io/prometheus.git","svn_url":"https://github.com/fabric8io/prometheus","homepage":"http://prometheus.io/","size":8491,"stargazers_count":0,"watchers_count":0,"language":"Go","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"prometheus:master","ref":"master","sha":"b5861b02aca916543df66d3b6436a30907249a12","user":{"login":"prometheus","id":3380462,"avatar_url":"https://avatars.githubusercontent.com/u/3380462?v=3","gravatar_id":"","url":"https://api.github.com/users/prometheus","html_url":"https://github.com/prometheus","followers_url":"https://api.github.com/users/prometheus/followers","following_url":"https://api.github.com/users/prometheus/following{/other_user}","gists_url":"https://api.github.com/users/prometheus/gists{/gist_id}","starred_url":"https://api.github.com/users/prometheus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prometheus/subscriptions","organizations_url":"https://api.github.com/users/prometheus/orgs","repos_url":"https://api.github.com/users/prometheus/repos","events_url":"https://api.github.com/users/prometheus/events{/privacy}","received_events_url":"https://api.github.com/users/prometheus/received_events","type":"Organization","site_admin":false},"repo":{"id":6838921,"name":"prometheus","full_name":"prometheus/prometheus","owner":{"login":"prometheus","id":3380462,"avatar_url":"https://avatars.githubusercontent.com/u/3380462?v=3","gravatar_id":"","url":"https://api.github.com/users/prometheus","html_url":"https://github.com/prometheus","followers_url":"https://api.github.com/users/prometheus/followers","following_url":"https://api.github.com/users/prometheus/following{/other_user}","gists_url":"https://api.github.com/users/prometheus/gists{/gist_id}","starred_url":"https://api.github.com/users/prometheus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/prometheus/subscriptions","organizations_url":"https://api.github.com/users/prometheus/orgs","repos_url":"https://api.github.com/users/prometheus/repos","events_url":"https://api.github.com/users/prometheus/events{/privacy}","received_events_url":"https://api.github.com/users/prometheus/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/prometheus/prometheus","description":"The Prometheus monitoring system and time series database.","fork":false,"url":"https://api.github.com/repos/prometheus/prometheus","forks_url":"https://api.github.com/repos/prometheus/prometheus/forks","keys_url":"https://api.github.com/repos/prometheus/prometheus/keys{/key_id}","collaborators_url":"https://api.github.com/repos/prometheus/prometheus/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/prometheus/prometheus/teams","hooks_url":"https://api.github.com/repos/prometheus/prometheus/hooks","issue_events_url":"https://api.github.com/repos/prometheus/prometheus/issues/events{/number}","events_url":"https://api.github.com/repos/prometheus/prometheus/events","assignees_url":"https://api.github.com/repos/prometheus/prometheus/assignees{/user}","branches_url":"https://api.github.com/repos/prometheus/prometheus/branches{/branch}","tags_url":"https://api.github.com/repos/prometheus/prometheus/tags","blobs_url":"https://api.github.com/repos/prometheus/prometheus/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/prometheus/prometheus/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/prometheus/prometheus/git/refs{/sha}","trees_url":"https://api.github.com/repos/prometheus/prometheus/git/trees{/sha}","statuses_url":"https://api.github.com/repos/prometheus/prometheus/statuses/{sha}","languages_url":"https://api.github.com/repos/prometheus/prometheus/languages","stargazers_url":"https://api.github.com/repos/prometheus/prometheus/stargazers","contributors_url":"https://api.github.com/repos/prometheus/prometheus/contributors","subscribers_url":"https://api.github.com/repos/prometheus/prometheus/subscribers","subscription_url":"https://api.github.com/repos/prometheus/prometheus/subscription","commits_url":"https://api.github.com/repos/prometheus/prometheus/commits{/sha}","git_commits_url":"https://api.github.com/repos/prometheus/prometheus/git/commits{/sha}","comments_url":"https://api.github.com/repos/prometheus/prometheus/comments{/number}","issue_comment_url":"https://api.github.com/repos/prometheus/prometheus/issues/comments{/number}","contents_url":"https://api.github.com/repos/prometheus/prometheus/contents/{+path}","compare_url":"https://api.github.com/repos/prometheus/prometheus/compare/{base}...{head}","merges_url":"https://api.github.com/repos/prometheus/prometheus/merges","archive_url":"https://api.github.com/repos/prometheus/prometheus/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/prometheus/prometheus/downloads","issues_url":"https://api.github.com/repos/prometheus/prometheus/issues{/number}","pulls_url":"https://api.github.com/repos/prometheus/prometheus/pulls{/number}","milestones_url":"https://api.github.com/repos/prometheus/prometheus/milestones{/number}","notifications_url":"https://api.github.com/repos/prometheus/prometheus/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/prometheus/prometheus/labels{/name}","releases_url":"https://api.github.com/repos/prometheus/prometheus/releases{/id}","created_at":"2012-11-24T11:14:12Z","updated_at":"2015-08-13T03:54:27Z","pushed_at":"2015-08-13T08:06:46Z","git_url":"git://github.com/prometheus/prometheus.git","ssh_url":"git@github.com:prometheus/prometheus.git","clone_url":"https://github.com/prometheus/prometheus.git","svn_url":"https://github.com/prometheus/prometheus","homepage":"http://prometheus.io/","size":18012,"stargazers_count":2750,"watchers_count":2750,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":171,"mirror_url":null,"open_issues_count":97,"forks":171,"open_issues":97,"watchers":2750,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/905"},"html":{"href":"https://github.com/prometheus/prometheus/pull/905"},"issue":{"href":"https://api.github.com/repos/prometheus/prometheus/issues/905"},"comments":{"href":"https://api.github.com/repos/prometheus/prometheus/issues/905/comments"},"review_comments":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/905/comments"},"review_comment":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/prometheus/prometheus/pulls/905/commits"},"statuses":{"href":"https://api.github.com/repos/prometheus/prometheus/statuses/1df67691f7d56a7943820f8084f7e5d33c46f4db"}}}}