{"action":"created","comment":{"url":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/comments/31600165","id":31600165,"diff_hunk":"@@ -166,6 +271,254 @@ AwsManager.prototype._filterSpotRequests = function (spotReqs) {\n };\n \n /**\n+ * Compare two state objects to find the instances and requests which are no\n+ * longer in the new state object.  The assumption here is that the items that\n+ * are no longer in the state are those which have been terminated.  This\n+ * method returns those instances and request which are no longer present in\n+ * state.  You'll need to have another data source to find the resolution of\n+ * the now missing resources\n+ */\n+AwsManager.prototype._compareStates = function (newState, previousState, deadState) {\n+  assert(newState);\n+  assert(previousState);\n+  assert(deadState);\n+\n+  var missingIds = {\n+    instances: [],\n+    requests: [],\n+  };\n+\n+  // to make comparison of states easier, we create a list of all the ids of\n+  // both spot requests and instances in each of the two compared states\n+  var allInstancesInPreviousState = previousState.instances.map(function (instance) {\n+    return instance.InstanceId;\n+  });\n+  var allRequestsInPreviousState = previousState.requests.map(function (request) {\n+    return request.SpotInstanceRequestId;\n+  });\n+  var allInstancesInNewState = newState.instances.map(function (instance) {\n+    return instance.InstanceId;\n+  });\n+  var allRequestsInNewState = newState.requests.map(function (request) {\n+    return request.SpotInstanceRequestId;\n+  });\n+\n+  // find all the instances and request ids which were in the previous state\n+  // but not in the new state\n+  missingIds.instances = allInstancesInPreviousState.filter(function (id) {\n+    return !allInstancesInNewState.includes(id);\n+  });\n+  missingIds.requests = allRequestsInPreviousState.filter(function (id) {\n+    return !allRequestsInNewState.includes(id);\n+  });\n+\n+  // Now let's grab those instances and requests which are absent, but instead\n+  // let's use their new state object instead of the old one.  This is to avoid\n+  // the problem of getting the stale state info in the later methods which\n+  // need information about why the state change occured\n+  return {\n+    instances: deadState.instances.filter(function (instance) {\n+      return missingIds.instances.includes(instance.InstanceId);\n+    }),\n+    requests: deadState.requests.filter(function (request) {\n+      return missingIds.requests.includes(request.SpotInstanceRequestId);\n+    }),\n+  };\n+};\n+\n+/**\n+ * Take a list of instances which are now absent and figure out why they are gone.\n+ * This will submit Influx points.  Differences are those objects which no longer\n+ * show up in the 'live' set of ec2 queries, instead show\n+ *\n+ * Remember that we're only looking at the differences between iteration intervals.\n+ * This is how we ensure that we don't look the same spot request twice.\n+ */\n+AwsManager.prototype._reconcileStateDifferences = function (differences, deadState, apiState) {\n+  var that = this;\n+  assert(differences);\n+  assert(deadState);\n+  assert(apiState);\n+\n+  function plotSpotFulfilment (request) {\n+    // Once we go from open -> active with a status of fulfilled we can log this\n+    // spot request as successfully fulfilled.  This does not imply that\n+    that.reportSpotRequestsFulfilled({\n+      provisionerId: that.provisionerId,\n+      region: request.Region,\n+      az: request.LaunchSpecification.Placement.AvailabilityZone,\n+      instanceType: request.LaunchSpecification.InstanceType,\n+      workerType: request.WorkerType,\n+      id: request.SpotInstanceRequestId,\n+      instanceId: request.InstanceId,\n+      time: dateForInflux(request.Status.UpdateTime),\n+    });\n+    debug('spot request %j fulfilled!', request);\n+  }\n+\n+  // We want to figure out what happened to each of the spot requests which are\n+  // no longer showing up as pending.  These can either be fulfilled or\n+  // rejected.  FOr those which are fulfilled we want to create a data point\n+  // which will let us trend how long spot request fulfilment takes.  For those\n+  // which fail we want stats on why they failed so we can later analyze this\n+  differences.requests.forEach(function (request) {\n+    if (request.State === 'active' && request.Status.Code === 'fulfilled') {\n+      plotSpotFulfilment(request);\n+    } else if (request.State === 'open') {\n+      // Here we have those spot requests which are no longer unfulfilled but\n+      // do not have an api state which reflects that\n+      that.__awaitingSpotFulfilmentStatus.push({\n+        id: request.SpotInstanceRequestId,\n+        time: new Date(),\n+        iterationCount: 0,\n+      });\n+    } else {\n+      // I wonder if we should be reporting spot requests which failed because\n+      // of bid amount here in their own special series for use in trying to\n+      // figure out what to bid.  We could use refused spot bids as a lower\n+      // limit for bids\n+      that.reportSpotRequestsDied({\n+        provisionerId: that.provisionerId,\n+        region: request.Region,\n+        az: request.LaunchSpecification.Placement.AvailabilityZone,\n+        instanceType: request.LaunchSpecification.InstanceType,\n+        workerType: request.WorkerType,\n+        id: request.SpotInstanceRequestId,\n+        time: dateForInflux(request.Status.UpdateTime),\n+        bid: parseFloat(request.SpotPrice, 10),\n+        state: request.State,\n+        statusCode: request.Status.Code,\n+        statusMsg: request.Status.Message,\n+      });\n+      debug('spot request %j did something else', request);\n+    }\n+  });\n+\n+  this.__awaitingSpotFulfilmentStatus = this.__awaitingSpotFulfilmentStatus.filter(function (requestAwaiting) {\n+    var keepInTheList = true;\n+\n+    deadState.requests.forEach(function (requestMightHave) {\n+      if (requestMightHave.SpotInstanceRequestId === requestAwaiting) {\n+        if (requestMightHave.State === 'active' && requestMightHave.Status.Code === 'filfilled') {","path":"provisioner/aws-manager.js","position":326,"original_position":326,"commit_id":"501a5c7f2ddad8ed6a667ed7b77699e9cf0ec431","original_commit_id":"501a5c7f2ddad8ed6a667ed7b77699e9cf0ec431","user":{"login":"jonasfj","id":149732,"avatar_url":"https://avatars.githubusercontent.com/u/149732?v=3","gravatar_id":"","url":"https://api.github.com/users/jonasfj","html_url":"https://github.com/jonasfj","followers_url":"https://api.github.com/users/jonasfj/followers","following_url":"https://api.github.com/users/jonasfj/following{/other_user}","gists_url":"https://api.github.com/users/jonasfj/gists{/gist_id}","starred_url":"https://api.github.com/users/jonasfj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jonasfj/subscriptions","organizations_url":"https://api.github.com/users/jonasfj/orgs","repos_url":"https://api.github.com/users/jonasfj/repos","events_url":"https://api.github.com/users/jonasfj/events{/privacy}","received_events_url":"https://api.github.com/users/jonasfj/received_events","type":"User","site_admin":false},"body":"Is `filfilled` correct?","created_at":"2015-06-03T07:44:16Z","updated_at":"2015-06-03T07:44:16Z","html_url":"https://github.com/taskcluster/aws-provisioner/pull/24#discussion_r31600165","pull_request_url":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/24","_links":{"self":{"href":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/comments/31600165"},"html":{"href":"https://github.com/taskcluster/aws-provisioner/pull/24#discussion_r31600165"},"pull_request":{"href":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/24"}}},"pull_request":{"url":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/24","id":36724491,"html_url":"https://github.com/taskcluster/aws-provisioner/pull/24","diff_url":"https://github.com/taskcluster/aws-provisioner/pull/24.diff","patch_url":"https://github.com/taskcluster/aws-provisioner/pull/24.patch","issue_url":"https://api.github.com/repos/taskcluster/aws-provisioner/issues/24","number":24,"state":"open","locked":false,"title":"Stats for provisioner","user":{"login":"jhford","id":607353,"avatar_url":"https://avatars.githubusercontent.com/u/607353?v=3","gravatar_id":"","url":"https://api.github.com/users/jhford","html_url":"https://github.com/jhford","followers_url":"https://api.github.com/users/jhford/followers","following_url":"https://api.github.com/users/jhford/following{/other_user}","gists_url":"https://api.github.com/users/jhford/gists{/gist_id}","starred_url":"https://api.github.com/users/jhford/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jhford/subscriptions","organizations_url":"https://api.github.com/users/jhford/orgs","repos_url":"https://api.github.com/users/jhford/repos","events_url":"https://api.github.com/users/jhford/events{/privacy}","received_events_url":"https://api.github.com/users/jhford/received_events","type":"User","site_admin":false},"body":"Submit relelvant stats regarding provisioning to influx.","created_at":"2015-06-02T12:49:55Z","updated_at":"2015-06-03T07:44:16Z","closed_at":null,"merged_at":null,"merge_commit_sha":"51e7e07e356db6bdba9cb570c3984555667650a5","assignee":{"login":"jonasfj","id":149732,"avatar_url":"https://avatars.githubusercontent.com/u/149732?v=3","gravatar_id":"","url":"https://api.github.com/users/jonasfj","html_url":"https://github.com/jonasfj","followers_url":"https://api.github.com/users/jonasfj/followers","following_url":"https://api.github.com/users/jonasfj/following{/other_user}","gists_url":"https://api.github.com/users/jonasfj/gists{/gist_id}","starred_url":"https://api.github.com/users/jonasfj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jonasfj/subscriptions","organizations_url":"https://api.github.com/users/jonasfj/orgs","repos_url":"https://api.github.com/users/jonasfj/repos","events_url":"https://api.github.com/users/jonasfj/events{/privacy}","received_events_url":"https://api.github.com/users/jonasfj/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/24/commits","review_comments_url":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/24/comments","review_comment_url":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/comments{/number}","comments_url":"https://api.github.com/repos/taskcluster/aws-provisioner/issues/24/comments","statuses_url":"https://api.github.com/repos/taskcluster/aws-provisioner/statuses/501a5c7f2ddad8ed6a667ed7b77699e9cf0ec431","head":{"label":"taskcluster:stats","ref":"stats","sha":"501a5c7f2ddad8ed6a667ed7b77699e9cf0ec431","user":{"login":"taskcluster","id":6257436,"avatar_url":"https://avatars.githubusercontent.com/u/6257436?v=3","gravatar_id":"","url":"https://api.github.com/users/taskcluster","html_url":"https://github.com/taskcluster","followers_url":"https://api.github.com/users/taskcluster/followers","following_url":"https://api.github.com/users/taskcluster/following{/other_user}","gists_url":"https://api.github.com/users/taskcluster/gists{/gist_id}","starred_url":"https://api.github.com/users/taskcluster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/taskcluster/subscriptions","organizations_url":"https://api.github.com/users/taskcluster/orgs","repos_url":"https://api.github.com/users/taskcluster/repos","events_url":"https://api.github.com/users/taskcluster/events{/privacy}","received_events_url":"https://api.github.com/users/taskcluster/received_events","type":"Organization","site_admin":false},"repo":{"id":16123958,"name":"aws-provisioner","full_name":"taskcluster/aws-provisioner","owner":{"login":"taskcluster","id":6257436,"avatar_url":"https://avatars.githubusercontent.com/u/6257436?v=3","gravatar_id":"","url":"https://api.github.com/users/taskcluster","html_url":"https://github.com/taskcluster","followers_url":"https://api.github.com/users/taskcluster/followers","following_url":"https://api.github.com/users/taskcluster/following{/other_user}","gists_url":"https://api.github.com/users/taskcluster/gists{/gist_id}","starred_url":"https://api.github.com/users/taskcluster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/taskcluster/subscriptions","organizations_url":"https://api.github.com/users/taskcluster/orgs","repos_url":"https://api.github.com/users/taskcluster/repos","events_url":"https://api.github.com/users/taskcluster/events{/privacy}","received_events_url":"https://api.github.com/users/taskcluster/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/taskcluster/aws-provisioner","description":"AWS Provisioner for the TaskCluster","fork":false,"url":"https://api.github.com/repos/taskcluster/aws-provisioner","forks_url":"https://api.github.com/repos/taskcluster/aws-provisioner/forks","keys_url":"https://api.github.com/repos/taskcluster/aws-provisioner/keys{/key_id}","collaborators_url":"https://api.github.com/repos/taskcluster/aws-provisioner/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/taskcluster/aws-provisioner/teams","hooks_url":"https://api.github.com/repos/taskcluster/aws-provisioner/hooks","issue_events_url":"https://api.github.com/repos/taskcluster/aws-provisioner/issues/events{/number}","events_url":"https://api.github.com/repos/taskcluster/aws-provisioner/events","assignees_url":"https://api.github.com/repos/taskcluster/aws-provisioner/assignees{/user}","branches_url":"https://api.github.com/repos/taskcluster/aws-provisioner/branches{/branch}","tags_url":"https://api.github.com/repos/taskcluster/aws-provisioner/tags","blobs_url":"https://api.github.com/repos/taskcluster/aws-provisioner/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/taskcluster/aws-provisioner/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/taskcluster/aws-provisioner/git/refs{/sha}","trees_url":"https://api.github.com/repos/taskcluster/aws-provisioner/git/trees{/sha}","statuses_url":"https://api.github.com/repos/taskcluster/aws-provisioner/statuses/{sha}","languages_url":"https://api.github.com/repos/taskcluster/aws-provisioner/languages","stargazers_url":"https://api.github.com/repos/taskcluster/aws-provisioner/stargazers","contributors_url":"https://api.github.com/repos/taskcluster/aws-provisioner/contributors","subscribers_url":"https://api.github.com/repos/taskcluster/aws-provisioner/subscribers","subscription_url":"https://api.github.com/repos/taskcluster/aws-provisioner/subscription","commits_url":"https://api.github.com/repos/taskcluster/aws-provisioner/commits{/sha}","git_commits_url":"https://api.github.com/repos/taskcluster/aws-provisioner/git/commits{/sha}","comments_url":"https://api.github.com/repos/taskcluster/aws-provisioner/comments{/number}","issue_comment_url":"https://api.github.com/repos/taskcluster/aws-provisioner/issues/comments{/number}","contents_url":"https://api.github.com/repos/taskcluster/aws-provisioner/contents/{+path}","compare_url":"https://api.github.com/repos/taskcluster/aws-provisioner/compare/{base}...{head}","merges_url":"https://api.github.com/repos/taskcluster/aws-provisioner/merges","archive_url":"https://api.github.com/repos/taskcluster/aws-provisioner/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/taskcluster/aws-provisioner/downloads","issues_url":"https://api.github.com/repos/taskcluster/aws-provisioner/issues{/number}","pulls_url":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls{/number}","milestones_url":"https://api.github.com/repos/taskcluster/aws-provisioner/milestones{/number}","notifications_url":"https://api.github.com/repos/taskcluster/aws-provisioner/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/taskcluster/aws-provisioner/labels{/name}","releases_url":"https://api.github.com/repos/taskcluster/aws-provisioner/releases{/id}","created_at":"2014-01-22T00:44:52Z","updated_at":"2015-05-20T05:19:58Z","pushed_at":"2015-06-02T13:00:11Z","git_url":"git://github.com/taskcluster/aws-provisioner.git","ssh_url":"git@github.com:taskcluster/aws-provisioner.git","clone_url":"https://github.com/taskcluster/aws-provisioner.git","svn_url":"https://github.com/taskcluster/aws-provisioner","homepage":null,"size":12090,"stargazers_count":2,"watchers_count":2,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":1,"forks":2,"open_issues":1,"watchers":2,"default_branch":"master"}},"base":{"label":"taskcluster:master","ref":"master","sha":"121dbd132256a0f160e80ff223e351b391d6bf4f","user":{"login":"taskcluster","id":6257436,"avatar_url":"https://avatars.githubusercontent.com/u/6257436?v=3","gravatar_id":"","url":"https://api.github.com/users/taskcluster","html_url":"https://github.com/taskcluster","followers_url":"https://api.github.com/users/taskcluster/followers","following_url":"https://api.github.com/users/taskcluster/following{/other_user}","gists_url":"https://api.github.com/users/taskcluster/gists{/gist_id}","starred_url":"https://api.github.com/users/taskcluster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/taskcluster/subscriptions","organizations_url":"https://api.github.com/users/taskcluster/orgs","repos_url":"https://api.github.com/users/taskcluster/repos","events_url":"https://api.github.com/users/taskcluster/events{/privacy}","received_events_url":"https://api.github.com/users/taskcluster/received_events","type":"Organization","site_admin":false},"repo":{"id":16123958,"name":"aws-provisioner","full_name":"taskcluster/aws-provisioner","owner":{"login":"taskcluster","id":6257436,"avatar_url":"https://avatars.githubusercontent.com/u/6257436?v=3","gravatar_id":"","url":"https://api.github.com/users/taskcluster","html_url":"https://github.com/taskcluster","followers_url":"https://api.github.com/users/taskcluster/followers","following_url":"https://api.github.com/users/taskcluster/following{/other_user}","gists_url":"https://api.github.com/users/taskcluster/gists{/gist_id}","starred_url":"https://api.github.com/users/taskcluster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/taskcluster/subscriptions","organizations_url":"https://api.github.com/users/taskcluster/orgs","repos_url":"https://api.github.com/users/taskcluster/repos","events_url":"https://api.github.com/users/taskcluster/events{/privacy}","received_events_url":"https://api.github.com/users/taskcluster/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/taskcluster/aws-provisioner","description":"AWS Provisioner for the TaskCluster","fork":false,"url":"https://api.github.com/repos/taskcluster/aws-provisioner","forks_url":"https://api.github.com/repos/taskcluster/aws-provisioner/forks","keys_url":"https://api.github.com/repos/taskcluster/aws-provisioner/keys{/key_id}","collaborators_url":"https://api.github.com/repos/taskcluster/aws-provisioner/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/taskcluster/aws-provisioner/teams","hooks_url":"https://api.github.com/repos/taskcluster/aws-provisioner/hooks","issue_events_url":"https://api.github.com/repos/taskcluster/aws-provisioner/issues/events{/number}","events_url":"https://api.github.com/repos/taskcluster/aws-provisioner/events","assignees_url":"https://api.github.com/repos/taskcluster/aws-provisioner/assignees{/user}","branches_url":"https://api.github.com/repos/taskcluster/aws-provisioner/branches{/branch}","tags_url":"https://api.github.com/repos/taskcluster/aws-provisioner/tags","blobs_url":"https://api.github.com/repos/taskcluster/aws-provisioner/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/taskcluster/aws-provisioner/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/taskcluster/aws-provisioner/git/refs{/sha}","trees_url":"https://api.github.com/repos/taskcluster/aws-provisioner/git/trees{/sha}","statuses_url":"https://api.github.com/repos/taskcluster/aws-provisioner/statuses/{sha}","languages_url":"https://api.github.com/repos/taskcluster/aws-provisioner/languages","stargazers_url":"https://api.github.com/repos/taskcluster/aws-provisioner/stargazers","contributors_url":"https://api.github.com/repos/taskcluster/aws-provisioner/contributors","subscribers_url":"https://api.github.com/repos/taskcluster/aws-provisioner/subscribers","subscription_url":"https://api.github.com/repos/taskcluster/aws-provisioner/subscription","commits_url":"https://api.github.com/repos/taskcluster/aws-provisioner/commits{/sha}","git_commits_url":"https://api.github.com/repos/taskcluster/aws-provisioner/git/commits{/sha}","comments_url":"https://api.github.com/repos/taskcluster/aws-provisioner/comments{/number}","issue_comment_url":"https://api.github.com/repos/taskcluster/aws-provisioner/issues/comments{/number}","contents_url":"https://api.github.com/repos/taskcluster/aws-provisioner/contents/{+path}","compare_url":"https://api.github.com/repos/taskcluster/aws-provisioner/compare/{base}...{head}","merges_url":"https://api.github.com/repos/taskcluster/aws-provisioner/merges","archive_url":"https://api.github.com/repos/taskcluster/aws-provisioner/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/taskcluster/aws-provisioner/downloads","issues_url":"https://api.github.com/repos/taskcluster/aws-provisioner/issues{/number}","pulls_url":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls{/number}","milestones_url":"https://api.github.com/repos/taskcluster/aws-provisioner/milestones{/number}","notifications_url":"https://api.github.com/repos/taskcluster/aws-provisioner/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/taskcluster/aws-provisioner/labels{/name}","releases_url":"https://api.github.com/repos/taskcluster/aws-provisioner/releases{/id}","created_at":"2014-01-22T00:44:52Z","updated_at":"2015-05-20T05:19:58Z","pushed_at":"2015-06-02T13:00:11Z","git_url":"git://github.com/taskcluster/aws-provisioner.git","ssh_url":"git@github.com:taskcluster/aws-provisioner.git","clone_url":"https://github.com/taskcluster/aws-provisioner.git","svn_url":"https://github.com/taskcluster/aws-provisioner","homepage":null,"size":12090,"stargazers_count":2,"watchers_count":2,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":1,"forks":2,"open_issues":1,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/24"},"html":{"href":"https://github.com/taskcluster/aws-provisioner/pull/24"},"issue":{"href":"https://api.github.com/repos/taskcluster/aws-provisioner/issues/24"},"comments":{"href":"https://api.github.com/repos/taskcluster/aws-provisioner/issues/24/comments"},"review_comments":{"href":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/24/comments"},"review_comment":{"href":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/taskcluster/aws-provisioner/pulls/24/commits"},"statuses":{"href":"https://api.github.com/repos/taskcluster/aws-provisioner/statuses/501a5c7f2ddad8ed6a667ed7b77699e9cf0ec431"}}}}