{"action":"created","comment":{"url":"https://api.github.com/repos/taskcluster/docker-worker/pulls/comments/43738585","id":43738585,"diff_hunk":"@@ -1,45 +1,81 @@\n import https from 'https';\n import url from 'url';\n+import fs from 'fs';\n+import temporary from 'temporary';\n+import promiseRetry from 'promise-retry';\n \n export default async function uploadToS3 (task, sourceStream, artifactName, expiration, httpsHeaders) {\n-  var queue = task.runtime.queue;\n-\n-  var artifact = await queue.createArtifact(\n-    task.status.taskId,\n-    task.runId,\n-    artifactName,\n-    {\n-      // Why s3? It's currently cheaper to store data in s3 this could easily\n-      // be used with azure simply by changing s3 -> azure.\n-      storageType: 's3',\n-      expires: new Date(Math.min(expiration, new Date(task.task.expires))),\n-      contentType: httpsHeaders['content-type']\n-    }\n-  );\n-\n-  let parsedUrl = url.parse(artifact.putUrl);\n-  let options = {\n-    hostname: parsedUrl.hostname,\n-    path: parsedUrl.path,\n-    method: 'PUT',\n-    headers: httpsHeaders\n-  };\n-\n-  let req = https.request(options);\n-  sourceStream.pipe(req);\n-\n-  let response = await new Promise((accept, reject) => {\n-    req.on('response', accept);\n-    req.on('error', reject);\n-    req.setTimeout(5 * 60 * 1000, reject);\n+  let queue = task.runtime.queue;\n+\n+  var artifact;\n+  var parsedUrl;\n+  var options;\n+\n+  let tmp = new temporary.File();\n+  await new Promise((accept, reject) => {\n+    let stream = fs.createWriteStream(tmp.path);\n+    stream.on('error', reject);\n+    stream.on('finish', accept);\n+    sourceStream.pipe(stream);\n   });\n \n-  // Flush the data from the reponse so it's not held in memory\n-  response.resume();\n+  try {\n+    await promiseRetry((retry, number) => {\n+      if (number > 1) {\n+        task.runtime.log('retrying artifact upload');\n+      }\n+\n+      return new Promise(async (accept, reject) => {\n+        new Promise((accept, reject) => {","path":"lib/upload_to_s3.js","position":61,"original_position":61,"commit_id":"bd1ebf843902e7be2a6197b09e418bda9a8cf7bc","original_commit_id":"bd1ebf843902e7be2a6197b09e418bda9a8cf7bc","user":{"login":"walac","id":611309,"avatar_url":"https://avatars.githubusercontent.com/u/611309?v=3","gravatar_id":"","url":"https://api.github.com/users/walac","html_url":"https://github.com/walac","followers_url":"https://api.github.com/users/walac/followers","following_url":"https://api.github.com/users/walac/following{/other_user}","gists_url":"https://api.github.com/users/walac/gists{/gist_id}","starred_url":"https://api.github.com/users/walac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/walac/subscriptions","organizations_url":"https://api.github.com/users/walac/orgs","repos_url":"https://api.github.com/users/walac/repos","events_url":"https://api.github.com/users/walac/events{/privacy}","received_events_url":"https://api.github.com/users/walac/received_events","type":"User","site_admin":false},"body":"I didn't understand your comment. Anyway, I updated the code to use `await` for `createArtifact`.","created_at":"2015-11-03T11:38:19Z","updated_at":"2015-11-03T11:38:19Z","html_url":"https://github.com/taskcluster/docker-worker/pull/177#discussion_r43738585","pull_request_url":"https://api.github.com/repos/taskcluster/docker-worker/pulls/177","_links":{"self":{"href":"https://api.github.com/repos/taskcluster/docker-worker/pulls/comments/43738585"},"html":{"href":"https://github.com/taskcluster/docker-worker/pull/177#discussion_r43738585"},"pull_request":{"href":"https://api.github.com/repos/taskcluster/docker-worker/pulls/177"}}},"pull_request":{"url":"https://api.github.com/repos/taskcluster/docker-worker/pulls/177","id":49070422,"html_url":"https://github.com/taskcluster/docker-worker/pull/177","diff_url":"https://github.com/taskcluster/docker-worker/pull/177.diff","patch_url":"https://github.com/taskcluster/docker-worker/pull/177.patch","issue_url":"https://api.github.com/repos/taskcluster/docker-worker/issues/177","number":177,"state":"open","locked":false,"title":"Bug 1186619: Retry artifact upload.","user":{"login":"walac","id":611309,"avatar_url":"https://avatars.githubusercontent.com/u/611309?v=3","gravatar_id":"","url":"https://api.github.com/users/walac","html_url":"https://github.com/walac","followers_url":"https://api.github.com/users/walac/followers","following_url":"https://api.github.com/users/walac/following{/other_user}","gists_url":"https://api.github.com/users/walac/gists{/gist_id}","starred_url":"https://api.github.com/users/walac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/walac/subscriptions","organizations_url":"https://api.github.com/users/walac/orgs","repos_url":"https://api.github.com/users/walac/repos","events_url":"https://api.github.com/users/walac/events{/privacy}","received_events_url":"https://api.github.com/users/walac/received_events","type":"User","site_admin":false},"body":"When an artifact upload fails, we try to upload a limited number of\r\ntimes before failing it.","created_at":"2015-10-28T21:35:58Z","updated_at":"2015-11-03T11:38:19Z","closed_at":null,"merged_at":null,"merge_commit_sha":"127bbe609a80e687c53225498556d75c369942fb","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/taskcluster/docker-worker/pulls/177/commits","review_comments_url":"https://api.github.com/repos/taskcluster/docker-worker/pulls/177/comments","review_comment_url":"https://api.github.com/repos/taskcluster/docker-worker/pulls/comments{/number}","comments_url":"https://api.github.com/repos/taskcluster/docker-worker/issues/177/comments","statuses_url":"https://api.github.com/repos/taskcluster/docker-worker/statuses/bd1ebf843902e7be2a6197b09e418bda9a8cf7bc","head":{"label":"walac:master","ref":"master","sha":"bd1ebf843902e7be2a6197b09e418bda9a8cf7bc","user":{"login":"walac","id":611309,"avatar_url":"https://avatars.githubusercontent.com/u/611309?v=3","gravatar_id":"","url":"https://api.github.com/users/walac","html_url":"https://github.com/walac","followers_url":"https://api.github.com/users/walac/followers","following_url":"https://api.github.com/users/walac/following{/other_user}","gists_url":"https://api.github.com/users/walac/gists{/gist_id}","starred_url":"https://api.github.com/users/walac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/walac/subscriptions","organizations_url":"https://api.github.com/users/walac/orgs","repos_url":"https://api.github.com/users/walac/repos","events_url":"https://api.github.com/users/walac/events{/privacy}","received_events_url":"https://api.github.com/users/walac/received_events","type":"User","site_admin":false},"repo":{"id":43084633,"name":"docker-worker","full_name":"walac/docker-worker","owner":{"login":"walac","id":611309,"avatar_url":"https://avatars.githubusercontent.com/u/611309?v=3","gravatar_id":"","url":"https://api.github.com/users/walac","html_url":"https://github.com/walac","followers_url":"https://api.github.com/users/walac/followers","following_url":"https://api.github.com/users/walac/following{/other_user}","gists_url":"https://api.github.com/users/walac/gists{/gist_id}","starred_url":"https://api.github.com/users/walac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/walac/subscriptions","organizations_url":"https://api.github.com/users/walac/orgs","repos_url":"https://api.github.com/users/walac/repos","events_url":"https://api.github.com/users/walac/events{/privacy}","received_events_url":"https://api.github.com/users/walac/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/walac/docker-worker","description":"Docker worker for the cluster","fork":true,"url":"https://api.github.com/repos/walac/docker-worker","forks_url":"https://api.github.com/repos/walac/docker-worker/forks","keys_url":"https://api.github.com/repos/walac/docker-worker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/walac/docker-worker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/walac/docker-worker/teams","hooks_url":"https://api.github.com/repos/walac/docker-worker/hooks","issue_events_url":"https://api.github.com/repos/walac/docker-worker/issues/events{/number}","events_url":"https://api.github.com/repos/walac/docker-worker/events","assignees_url":"https://api.github.com/repos/walac/docker-worker/assignees{/user}","branches_url":"https://api.github.com/repos/walac/docker-worker/branches{/branch}","tags_url":"https://api.github.com/repos/walac/docker-worker/tags","blobs_url":"https://api.github.com/repos/walac/docker-worker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/walac/docker-worker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/walac/docker-worker/git/refs{/sha}","trees_url":"https://api.github.com/repos/walac/docker-worker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/walac/docker-worker/statuses/{sha}","languages_url":"https://api.github.com/repos/walac/docker-worker/languages","stargazers_url":"https://api.github.com/repos/walac/docker-worker/stargazers","contributors_url":"https://api.github.com/repos/walac/docker-worker/contributors","subscribers_url":"https://api.github.com/repos/walac/docker-worker/subscribers","subscription_url":"https://api.github.com/repos/walac/docker-worker/subscription","commits_url":"https://api.github.com/repos/walac/docker-worker/commits{/sha}","git_commits_url":"https://api.github.com/repos/walac/docker-worker/git/commits{/sha}","comments_url":"https://api.github.com/repos/walac/docker-worker/comments{/number}","issue_comment_url":"https://api.github.com/repos/walac/docker-worker/issues/comments{/number}","contents_url":"https://api.github.com/repos/walac/docker-worker/contents/{+path}","compare_url":"https://api.github.com/repos/walac/docker-worker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/walac/docker-worker/merges","archive_url":"https://api.github.com/repos/walac/docker-worker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/walac/docker-worker/downloads","issues_url":"https://api.github.com/repos/walac/docker-worker/issues{/number}","pulls_url":"https://api.github.com/repos/walac/docker-worker/pulls{/number}","milestones_url":"https://api.github.com/repos/walac/docker-worker/milestones{/number}","notifications_url":"https://api.github.com/repos/walac/docker-worker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/walac/docker-worker/labels{/name}","releases_url":"https://api.github.com/repos/walac/docker-worker/releases{/id}","created_at":"2015-09-24T18:17:51Z","updated_at":"2015-09-24T18:17:53Z","pushed_at":"2015-11-02T19:42:01Z","git_url":"git://github.com/walac/docker-worker.git","ssh_url":"git@github.com:walac/docker-worker.git","clone_url":"https://github.com/walac/docker-worker.git","svn_url":"https://github.com/walac/docker-worker","homepage":"","size":22486,"stargazers_count":0,"watchers_count":0,"language":"JavaScript","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"taskcluster:master","ref":"master","sha":"f84c8800757b8f729048e9da31b6a7b1b678fc6f","user":{"login":"taskcluster","id":6257436,"avatar_url":"https://avatars.githubusercontent.com/u/6257436?v=3","gravatar_id":"","url":"https://api.github.com/users/taskcluster","html_url":"https://github.com/taskcluster","followers_url":"https://api.github.com/users/taskcluster/followers","following_url":"https://api.github.com/users/taskcluster/following{/other_user}","gists_url":"https://api.github.com/users/taskcluster/gists{/gist_id}","starred_url":"https://api.github.com/users/taskcluster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/taskcluster/subscriptions","organizations_url":"https://api.github.com/users/taskcluster/orgs","repos_url":"https://api.github.com/users/taskcluster/repos","events_url":"https://api.github.com/users/taskcluster/events{/privacy}","received_events_url":"https://api.github.com/users/taskcluster/received_events","type":"Organization","site_admin":false},"repo":{"id":15545665,"name":"docker-worker","full_name":"taskcluster/docker-worker","owner":{"login":"taskcluster","id":6257436,"avatar_url":"https://avatars.githubusercontent.com/u/6257436?v=3","gravatar_id":"","url":"https://api.github.com/users/taskcluster","html_url":"https://github.com/taskcluster","followers_url":"https://api.github.com/users/taskcluster/followers","following_url":"https://api.github.com/users/taskcluster/following{/other_user}","gists_url":"https://api.github.com/users/taskcluster/gists{/gist_id}","starred_url":"https://api.github.com/users/taskcluster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/taskcluster/subscriptions","organizations_url":"https://api.github.com/users/taskcluster/orgs","repos_url":"https://api.github.com/users/taskcluster/repos","events_url":"https://api.github.com/users/taskcluster/events{/privacy}","received_events_url":"https://api.github.com/users/taskcluster/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/taskcluster/docker-worker","description":"Docker worker for the cluster","fork":false,"url":"https://api.github.com/repos/taskcluster/docker-worker","forks_url":"https://api.github.com/repos/taskcluster/docker-worker/forks","keys_url":"https://api.github.com/repos/taskcluster/docker-worker/keys{/key_id}","collaborators_url":"https://api.github.com/repos/taskcluster/docker-worker/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/taskcluster/docker-worker/teams","hooks_url":"https://api.github.com/repos/taskcluster/docker-worker/hooks","issue_events_url":"https://api.github.com/repos/taskcluster/docker-worker/issues/events{/number}","events_url":"https://api.github.com/repos/taskcluster/docker-worker/events","assignees_url":"https://api.github.com/repos/taskcluster/docker-worker/assignees{/user}","branches_url":"https://api.github.com/repos/taskcluster/docker-worker/branches{/branch}","tags_url":"https://api.github.com/repos/taskcluster/docker-worker/tags","blobs_url":"https://api.github.com/repos/taskcluster/docker-worker/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/taskcluster/docker-worker/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/taskcluster/docker-worker/git/refs{/sha}","trees_url":"https://api.github.com/repos/taskcluster/docker-worker/git/trees{/sha}","statuses_url":"https://api.github.com/repos/taskcluster/docker-worker/statuses/{sha}","languages_url":"https://api.github.com/repos/taskcluster/docker-worker/languages","stargazers_url":"https://api.github.com/repos/taskcluster/docker-worker/stargazers","contributors_url":"https://api.github.com/repos/taskcluster/docker-worker/contributors","subscribers_url":"https://api.github.com/repos/taskcluster/docker-worker/subscribers","subscription_url":"https://api.github.com/repos/taskcluster/docker-worker/subscription","commits_url":"https://api.github.com/repos/taskcluster/docker-worker/commits{/sha}","git_commits_url":"https://api.github.com/repos/taskcluster/docker-worker/git/commits{/sha}","comments_url":"https://api.github.com/repos/taskcluster/docker-worker/comments{/number}","issue_comment_url":"https://api.github.com/repos/taskcluster/docker-worker/issues/comments{/number}","contents_url":"https://api.github.com/repos/taskcluster/docker-worker/contents/{+path}","compare_url":"https://api.github.com/repos/taskcluster/docker-worker/compare/{base}...{head}","merges_url":"https://api.github.com/repos/taskcluster/docker-worker/merges","archive_url":"https://api.github.com/repos/taskcluster/docker-worker/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/taskcluster/docker-worker/downloads","issues_url":"https://api.github.com/repos/taskcluster/docker-worker/issues{/number}","pulls_url":"https://api.github.com/repos/taskcluster/docker-worker/pulls{/number}","milestones_url":"https://api.github.com/repos/taskcluster/docker-worker/milestones{/number}","notifications_url":"https://api.github.com/repos/taskcluster/docker-worker/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/taskcluster/docker-worker/labels{/name}","releases_url":"https://api.github.com/repos/taskcluster/docker-worker/releases{/id}","created_at":"2013-12-31T07:38:22Z","updated_at":"2015-10-23T01:43:49Z","pushed_at":"2015-11-02T19:42:02Z","git_url":"git://github.com/taskcluster/docker-worker.git","ssh_url":"git@github.com:taskcluster/docker-worker.git","clone_url":"https://github.com/taskcluster/docker-worker.git","svn_url":"https://github.com/taskcluster/docker-worker","homepage":"","size":26156,"stargazers_count":18,"watchers_count":18,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":16,"mirror_url":null,"open_issues_count":3,"forks":16,"open_issues":3,"watchers":18,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/taskcluster/docker-worker/pulls/177"},"html":{"href":"https://github.com/taskcluster/docker-worker/pull/177"},"issue":{"href":"https://api.github.com/repos/taskcluster/docker-worker/issues/177"},"comments":{"href":"https://api.github.com/repos/taskcluster/docker-worker/issues/177/comments"},"review_comments":{"href":"https://api.github.com/repos/taskcluster/docker-worker/pulls/177/comments"},"review_comment":{"href":"https://api.github.com/repos/taskcluster/docker-worker/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/taskcluster/docker-worker/pulls/177/commits"},"statuses":{"href":"https://api.github.com/repos/taskcluster/docker-worker/statuses/bd1ebf843902e7be2a6197b09e418bda9a8cf7bc"}}}}