{"action":"created","comment":{"url":"https://api.github.com/repos/ambiata/notion/pulls/comments/26553276","id":26553276,"diff_hunk":"@@ -0,0 +1,92 @@\n+package com.ambiata.notion.distcopy\n+\n+import com.ambiata.mundane.control.RIO\n+import com.ambiata.mundane.io._\n+import com.ambiata.notion.core._\n+import com.ambiata.poacher.hdfs.Hdfs\n+import com.ambiata.saws.s3.{S3Address, S3Prefix, _}\n+import org.apache.hadoop.fs._\n+\n+import scalaz.Scalaz._\n+\n+/**\n+ * Copy functions between Hdfs and S3\n+ */\n+object FileDistCopy {\n+\n+  /**\n+   * Download a large file by doing a distcopy from s3 to hdfs\n+   * Don't do anything if the file already exists\n+   */\n+  def distCopyDownloadFile(from: S3Address, to: Location, locationIO: LocationIO): RIO[Unit] =\n+    for {\n+      toUri  <- hadoopPath(to)\n+      exists <- pathExist(new Path(toUri), locationIO)\n+      _      <- DistCopyJob.run(Mappings(Vector(DownloadMapping(from, new Path(toUri)))), distCopyConfiguration(locationIO)).whenM(exists)\n+    } yield ()\n+\n+  /**\n+   * Download a large directory by doing a distcopy from s3 to a hdfs directory\n+   *\n+   * Don't copy files which have already been copied\n+   */\n+  def distCopyDownloadDirectory(from: S3Prefix, to: Location, locationIO: LocationIO): RIO[Unit] =\n+    for {\n+      toUri         <- hadoopPath(to)\n+      fromAddresses <- from.listAddress.execute(locationIO.s3Client)\n+      mappings      <- fromAddresses.traverseU(createDownloadMapping(HdfsLocation(toUri), locationIO))\n+      _             <- DistCopyJob.run(Mappings(mappings.toVector.flatten), distCopyConfiguration(locationIO))\n+    } yield ()\n+\n+  /** upload a large file by doing a distcopy from hdfs to s3 */\n+  def distCopyUploadFile(from: Location, to: S3Pattern, locationIO: LocationIO): RIO[Unit] =\n+    for {\n+      fromUri <- hadoopPath(from)\n+      _       <- DistCopyJob.run(Mappings(Vector(UploadMapping(new Path(fromUri), S3Address(to.bucket, to.unknown)))), distCopyConfiguration(locationIO))\n+    } yield ()\n+\n+  /** upload a large file by doing a distcopy from hdfs to s3 */\n+  def distCopyUploadDirectory(from: Location, to: S3Location, locationIO: LocationIO): RIO[Unit] =\n+    for {\n+      fromUri   <- hadoopPath(from)\n+      fromPaths <- Hdfs.globFiles(new Path(fromUri, \"*\")).run(locationIO.configuration)\n+      mappings  <- fromPaths.traverseU(createUploadMapping(to, locationIO))\n+      _         <- DistCopyJob.run(Mappings(mappings.toVector.flatten), distCopyConfiguration(locationIO))\n+    } yield ()","path":"notion-distcopy/src/main/scala/com/ambiata/notion/distcopy/FileDistCopy.scala","position":55,"original_position":55,"commit_id":"a1043c7c76c2707423b3c3c0caab03e096588b65","original_commit_id":"a1043c7c76c2707423b3c3c0caab03e096588b65","user":{"login":"markhibberd","id":22832,"avatar_url":"https://avatars.githubusercontent.com/u/22832?v=3","gravatar_id":"","url":"https://api.github.com/users/markhibberd","html_url":"https://github.com/markhibberd","followers_url":"https://api.github.com/users/markhibberd/followers","following_url":"https://api.github.com/users/markhibberd/following{/other_user}","gists_url":"https://api.github.com/users/markhibberd/gists{/gist_id}","starred_url":"https://api.github.com/users/markhibberd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/markhibberd/subscriptions","organizations_url":"https://api.github.com/users/markhibberd/orgs","repos_url":"https://api.github.com/users/markhibberd/repos","events_url":"https://api.github.com/users/markhibberd/events{/privacy}","received_events_url":"https://api.github.com/users/markhibberd/received_events","type":"User","site_admin":false},"body":"Yeh no worries, I don't think it is way off or anything - I just think we can tighten up the types a little bit to stop some potential bugs - probably easier to discuss with pen and paper though.","created_at":"2015-03-17T07:18:33Z","updated_at":"2015-03-17T07:18:33Z","html_url":"https://github.com/ambiata/notion/pull/58#discussion_r26553276","pull_request_url":"https://api.github.com/repos/ambiata/notion/pulls/58","_links":{"self":{"href":"https://api.github.com/repos/ambiata/notion/pulls/comments/26553276"},"html":{"href":"https://github.com/ambiata/notion/pull/58#discussion_r26553276"},"pull_request":{"href":"https://api.github.com/repos/ambiata/notion/pulls/58"}}},"pull_request":{"url":"https://api.github.com/repos/ambiata/notion/pulls/58","id":31011083,"html_url":"https://github.com/ambiata/notion/pull/58","diff_url":"https://github.com/ambiata/notion/pull/58.diff","patch_url":"https://github.com/ambiata/notion/pull/58.patch","issue_url":"https://api.github.com/repos/ambiata/notion/issues/58","number":58,"state":"open","locked":false,"title":"Added classes to copy files, copy directories","user":{"login":"etorreborre","id":10988,"avatar_url":"https://avatars.githubusercontent.com/u/10988?v=3","gravatar_id":"","url":"https://api.github.com/users/etorreborre","html_url":"https://github.com/etorreborre","followers_url":"https://api.github.com/users/etorreborre/followers","following_url":"https://api.github.com/users/etorreborre/following{/other_user}","gists_url":"https://api.github.com/users/etorreborre/gists{/gist_id}","starred_url":"https://api.github.com/users/etorreborre/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/etorreborre/subscriptions","organizations_url":"https://api.github.com/users/etorreborre/orgs","repos_url":"https://api.github.com/users/etorreborre/repos","events_url":"https://api.github.com/users/etorreborre/events{/privacy}","received_events_url":"https://api.github.com/users/etorreborre/received_events","type":"User","site_admin":false},"body":"WIP - 3 contributions here (could be split in more PRs):\r\n\r\n - a `copyFile` method on `LocationIO` (test missing)\r\n - a `FileDistCopy` object with functions using distcopy to download / upload files or directories\r\n - a `ShadowInputsOutputs` object with functions to download files before executing local or cluster applications if necessary\r\n\r\nA lot more testing is necessary but it is going to be very helpful to have such code being shared to build \"end-to-end\" applications.","created_at":"2015-03-12T06:23:38Z","updated_at":"2015-03-17T07:18:33Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ca0b95b575a4955e86f5d2924306ade55c97aaa7","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/ambiata/notion/pulls/58/commits","review_comments_url":"https://api.github.com/repos/ambiata/notion/pulls/58/comments","review_comment_url":"https://api.github.com/repos/ambiata/notion/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ambiata/notion/issues/58/comments","statuses_url":"https://api.github.com/repos/ambiata/notion/statuses/a1043c7c76c2707423b3c3c0caab03e096588b65","head":{"label":"ambiata:topic/location-file-copy","ref":"topic/location-file-copy","sha":"a1043c7c76c2707423b3c3c0caab03e096588b65","user":{"login":"ambiata","id":1691023,"avatar_url":"https://avatars.githubusercontent.com/u/1691023?v=3","gravatar_id":"","url":"https://api.github.com/users/ambiata","html_url":"https://github.com/ambiata","followers_url":"https://api.github.com/users/ambiata/followers","following_url":"https://api.github.com/users/ambiata/following{/other_user}","gists_url":"https://api.github.com/users/ambiata/gists{/gist_id}","starred_url":"https://api.github.com/users/ambiata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ambiata/subscriptions","organizations_url":"https://api.github.com/users/ambiata/orgs","repos_url":"https://api.github.com/users/ambiata/repos","events_url":"https://api.github.com/users/ambiata/events{/privacy}","received_events_url":"https://api.github.com/users/ambiata/received_events","type":"Organization","site_admin":false},"repo":{"id":25012772,"name":"notion","full_name":"ambiata/notion","owner":{"login":"ambiata","id":1691023,"avatar_url":"https://avatars.githubusercontent.com/u/1691023?v=3","gravatar_id":"","url":"https://api.github.com/users/ambiata","html_url":"https://github.com/ambiata","followers_url":"https://api.github.com/users/ambiata/followers","following_url":"https://api.github.com/users/ambiata/following{/other_user}","gists_url":"https://api.github.com/users/ambiata/gists{/gist_id}","starred_url":"https://api.github.com/users/ambiata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ambiata/subscriptions","organizations_url":"https://api.github.com/users/ambiata/orgs","repos_url":"https://api.github.com/users/ambiata/repos","events_url":"https://api.github.com/users/ambiata/events{/privacy}","received_events_url":"https://api.github.com/users/ambiata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ambiata/notion","description":"a general understanding; vague or imperfect conception or idea of something - in this case of file system like data storage abstractions","fork":false,"url":"https://api.github.com/repos/ambiata/notion","forks_url":"https://api.github.com/repos/ambiata/notion/forks","keys_url":"https://api.github.com/repos/ambiata/notion/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ambiata/notion/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ambiata/notion/teams","hooks_url":"https://api.github.com/repos/ambiata/notion/hooks","issue_events_url":"https://api.github.com/repos/ambiata/notion/issues/events{/number}","events_url":"https://api.github.com/repos/ambiata/notion/events","assignees_url":"https://api.github.com/repos/ambiata/notion/assignees{/user}","branches_url":"https://api.github.com/repos/ambiata/notion/branches{/branch}","tags_url":"https://api.github.com/repos/ambiata/notion/tags","blobs_url":"https://api.github.com/repos/ambiata/notion/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ambiata/notion/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ambiata/notion/git/refs{/sha}","trees_url":"https://api.github.com/repos/ambiata/notion/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ambiata/notion/statuses/{sha}","languages_url":"https://api.github.com/repos/ambiata/notion/languages","stargazers_url":"https://api.github.com/repos/ambiata/notion/stargazers","contributors_url":"https://api.github.com/repos/ambiata/notion/contributors","subscribers_url":"https://api.github.com/repos/ambiata/notion/subscribers","subscription_url":"https://api.github.com/repos/ambiata/notion/subscription","commits_url":"https://api.github.com/repos/ambiata/notion/commits{/sha}","git_commits_url":"https://api.github.com/repos/ambiata/notion/git/commits{/sha}","comments_url":"https://api.github.com/repos/ambiata/notion/comments{/number}","issue_comment_url":"https://api.github.com/repos/ambiata/notion/issues/comments{/number}","contents_url":"https://api.github.com/repos/ambiata/notion/contents/{+path}","compare_url":"https://api.github.com/repos/ambiata/notion/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ambiata/notion/merges","archive_url":"https://api.github.com/repos/ambiata/notion/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ambiata/notion/downloads","issues_url":"https://api.github.com/repos/ambiata/notion/issues{/number}","pulls_url":"https://api.github.com/repos/ambiata/notion/pulls{/number}","milestones_url":"https://api.github.com/repos/ambiata/notion/milestones{/number}","notifications_url":"https://api.github.com/repos/ambiata/notion/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ambiata/notion/labels{/name}","releases_url":"https://api.github.com/repos/ambiata/notion/releases{/id}","created_at":"2014-10-09T23:32:38Z","updated_at":"2015-03-10T04:47:21Z","pushed_at":"2015-03-17T05:20:43Z","git_url":"git://github.com/ambiata/notion.git","ssh_url":"git@github.com:ambiata/notion.git","clone_url":"https://github.com/ambiata/notion.git","svn_url":"https://github.com/ambiata/notion","homepage":null,"size":1623,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":9,"forks":0,"open_issues":9,"watchers":0,"default_branch":"master"}},"base":{"label":"ambiata:master","ref":"master","sha":"e74e4d92d389e89eef4379cf276637b94dc01922","user":{"login":"ambiata","id":1691023,"avatar_url":"https://avatars.githubusercontent.com/u/1691023?v=3","gravatar_id":"","url":"https://api.github.com/users/ambiata","html_url":"https://github.com/ambiata","followers_url":"https://api.github.com/users/ambiata/followers","following_url":"https://api.github.com/users/ambiata/following{/other_user}","gists_url":"https://api.github.com/users/ambiata/gists{/gist_id}","starred_url":"https://api.github.com/users/ambiata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ambiata/subscriptions","organizations_url":"https://api.github.com/users/ambiata/orgs","repos_url":"https://api.github.com/users/ambiata/repos","events_url":"https://api.github.com/users/ambiata/events{/privacy}","received_events_url":"https://api.github.com/users/ambiata/received_events","type":"Organization","site_admin":false},"repo":{"id":25012772,"name":"notion","full_name":"ambiata/notion","owner":{"login":"ambiata","id":1691023,"avatar_url":"https://avatars.githubusercontent.com/u/1691023?v=3","gravatar_id":"","url":"https://api.github.com/users/ambiata","html_url":"https://github.com/ambiata","followers_url":"https://api.github.com/users/ambiata/followers","following_url":"https://api.github.com/users/ambiata/following{/other_user}","gists_url":"https://api.github.com/users/ambiata/gists{/gist_id}","starred_url":"https://api.github.com/users/ambiata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ambiata/subscriptions","organizations_url":"https://api.github.com/users/ambiata/orgs","repos_url":"https://api.github.com/users/ambiata/repos","events_url":"https://api.github.com/users/ambiata/events{/privacy}","received_events_url":"https://api.github.com/users/ambiata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ambiata/notion","description":"a general understanding; vague or imperfect conception or idea of something - in this case of file system like data storage abstractions","fork":false,"url":"https://api.github.com/repos/ambiata/notion","forks_url":"https://api.github.com/repos/ambiata/notion/forks","keys_url":"https://api.github.com/repos/ambiata/notion/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ambiata/notion/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ambiata/notion/teams","hooks_url":"https://api.github.com/repos/ambiata/notion/hooks","issue_events_url":"https://api.github.com/repos/ambiata/notion/issues/events{/number}","events_url":"https://api.github.com/repos/ambiata/notion/events","assignees_url":"https://api.github.com/repos/ambiata/notion/assignees{/user}","branches_url":"https://api.github.com/repos/ambiata/notion/branches{/branch}","tags_url":"https://api.github.com/repos/ambiata/notion/tags","blobs_url":"https://api.github.com/repos/ambiata/notion/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ambiata/notion/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ambiata/notion/git/refs{/sha}","trees_url":"https://api.github.com/repos/ambiata/notion/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ambiata/notion/statuses/{sha}","languages_url":"https://api.github.com/repos/ambiata/notion/languages","stargazers_url":"https://api.github.com/repos/ambiata/notion/stargazers","contributors_url":"https://api.github.com/repos/ambiata/notion/contributors","subscribers_url":"https://api.github.com/repos/ambiata/notion/subscribers","subscription_url":"https://api.github.com/repos/ambiata/notion/subscription","commits_url":"https://api.github.com/repos/ambiata/notion/commits{/sha}","git_commits_url":"https://api.github.com/repos/ambiata/notion/git/commits{/sha}","comments_url":"https://api.github.com/repos/ambiata/notion/comments{/number}","issue_comment_url":"https://api.github.com/repos/ambiata/notion/issues/comments{/number}","contents_url":"https://api.github.com/repos/ambiata/notion/contents/{+path}","compare_url":"https://api.github.com/repos/ambiata/notion/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ambiata/notion/merges","archive_url":"https://api.github.com/repos/ambiata/notion/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ambiata/notion/downloads","issues_url":"https://api.github.com/repos/ambiata/notion/issues{/number}","pulls_url":"https://api.github.com/repos/ambiata/notion/pulls{/number}","milestones_url":"https://api.github.com/repos/ambiata/notion/milestones{/number}","notifications_url":"https://api.github.com/repos/ambiata/notion/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ambiata/notion/labels{/name}","releases_url":"https://api.github.com/repos/ambiata/notion/releases{/id}","created_at":"2014-10-09T23:32:38Z","updated_at":"2015-03-10T04:47:21Z","pushed_at":"2015-03-17T05:20:43Z","git_url":"git://github.com/ambiata/notion.git","ssh_url":"git@github.com:ambiata/notion.git","clone_url":"https://github.com/ambiata/notion.git","svn_url":"https://github.com/ambiata/notion","homepage":null,"size":1623,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":9,"forks":0,"open_issues":9,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ambiata/notion/pulls/58"},"html":{"href":"https://github.com/ambiata/notion/pull/58"},"issue":{"href":"https://api.github.com/repos/ambiata/notion/issues/58"},"comments":{"href":"https://api.github.com/repos/ambiata/notion/issues/58/comments"},"review_comments":{"href":"https://api.github.com/repos/ambiata/notion/pulls/58/comments"},"review_comment":{"href":"https://api.github.com/repos/ambiata/notion/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ambiata/notion/pulls/58/commits"},"statuses":{"href":"https://api.github.com/repos/ambiata/notion/statuses/a1043c7c76c2707423b3c3c0caab03e096588b65"}}}}