{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/43202843","id":43202843,"diff_hunk":"@@ -0,0 +1,166 @@\n+/*\n+ * Copyright (C) 2014-2015 LinkedIn Corp. All rights reserved.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use\n+ * this file except in compliance with the License. You may obtain a copy of the\n+ * License at  http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software distributed\n+ * under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n+ * CONDITIONS OF ANY KIND, either express or implied.\n+ */\n+\n+package gobblin.data.management.copy;\n+\n+import gobblin.configuration.ConfigurationKeys;\n+import gobblin.data.management.partition.File;\n+\n+import java.io.ByteArrayInputStream;\n+import java.io.ByteArrayOutputStream;\n+import java.io.DataInput;\n+import java.io.DataInputStream;\n+import java.io.DataOutput;\n+import java.io.DataOutputStream;\n+import java.io.IOException;\n+import java.util.List;\n+import java.util.Properties;\n+\n+import lombok.AccessLevel;\n+import lombok.AllArgsConstructor;\n+import lombok.EqualsAndHashCode;\n+import lombok.Getter;\n+import lombok.NoArgsConstructor;\n+import lombok.Setter;\n+\n+import org.apache.commons.codec.DecoderException;\n+import org.apache.commons.codec.binary.Hex;\n+import org.apache.hadoop.fs.FileStatus;\n+import org.apache.hadoop.fs.Path;\n+import org.apache.hadoop.io.Text;\n+import org.apache.hadoop.io.Writable;\n+\n+import com.google.common.collect.Lists;\n+\n+\n+/**\n+ * Abstraction for a file to copy from {@link #origin} to {@link #destination}.\n+ */\n+@Getter\n+@Setter\n+@AllArgsConstructor\n+@NoArgsConstructor(access = AccessLevel.PRIVATE)\n+@EqualsAndHashCode\n+public class CopyableFile implements File, Writable {\n+\n+  public static final String SERIALIZED_COPYABLE_FILE = \"gobblin.copy.serialized.copyable.file\";\n+\n+  /** {@link FileStatus} of the existing origin file. */\n+  private FileStatus origin;\n+  /** Destination {@link Path} of the file. */\n+  private Path destination;\n+  /** {@link Path} to the file relative to the final publish directory {@link ConfigurationKeys#DATA_PUBLISHER_FINAL_DIR} */\n+  private Path relativeDestination;\n+  /** Desired {@link OwnerAndPermission} of the destination path. */\n+  private OwnerAndPermission destinationOwnerAndPermission;\n+  /**\n+   * Desired {@link OwnerAndPermission} of the ancestor directories of the destination path. The list is ordered from\n+   * deepest to highest directory.\n+   *\n+   * <p>\n+   * For example, if {@link #destination} is /a/b/c/file, then the first element of this list is the desired owner and\n+   * permission for directory /a/b/c, the second is the desired owner and permission for directory /a/b, and so on.\n+   * </p>\n+   *\n+   * <p>\n+   * If there are fewer elements in the list than ancestor directories in {@link #destination}, it is understood that\n+   * extra directories are allowed to have any owner and permission.\n+   * </p>\n+   */\n+  private List<OwnerAndPermission> ancestorsOwnerAndPermission;\n+  /** Checksum of the origin file. */\n+  private byte[] checksum;\n+\n+  @Override\n+  public FileStatus getFileStatus() {\n+    return this.origin;\n+  }\n+\n+  @Override\n+  public void write(DataOutput dataOutput) throws IOException {\n+    this.origin.write(dataOutput);\n+    Text.writeString(dataOutput, this.destination.toString());\n+    Text.writeString(dataOutput, this.relativeDestination.toString());\n+    this.destinationOwnerAndPermission.write(dataOutput);\n+    dataOutput.writeInt(this.ancestorsOwnerAndPermission.size());\n+    for (OwnerAndPermission oap : this.ancestorsOwnerAndPermission) {\n+      oap.write(dataOutput);\n+    }\n+    dataOutput.writeInt(this.checksum.length);\n+    dataOutput.write(this.checksum);\n+  }\n+\n+  @Override\n+  public void readFields(DataInput dataInput) throws IOException {\n+    this.origin = new FileStatus();\n+    this.origin.readFields(dataInput);\n+    this.destination = new Path(Text.readString(dataInput));\n+    this.relativeDestination = new Path(Text.readString(dataInput));\n+    this.destinationOwnerAndPermission = OwnerAndPermission.read(dataInput);\n+    int ancestors = dataInput.readInt();\n+    this.ancestorsOwnerAndPermission = Lists.newArrayList();\n+    for (int i = 0; i < ancestors; i++) {\n+      this.ancestorsOwnerAndPermission.add(OwnerAndPermission.read(dataInput));\n+    }\n+    int checksumSize = dataInput.readInt();\n+    this.checksum = new byte[checksumSize];\n+    dataInput.readFully(this.checksum);\n+  }\n+\n+  /**\n+   * Read a {@link gobblin.data.management.copy.CopyableFile} from a {@link java.io.DataInput}.\n+   *\n+   * @throws IOException\n+   */\n+  public static CopyableFile read(DataInput dataInput) throws IOException {\n+    CopyableFile copyableFile = new CopyableFile();\n+    copyableFile.readFields(dataInput);\n+    return copyableFile;\n+  }\n+\n+  /**\n+   * Serialize an instance of {@link CopyableFile} into a {@link String}. Usually use to store the copyableFile in state\n+   * at key {@link #SERIALIZED_COPYABLE_FILE}\n+   *\n+   * @param copyableFile to be serialized\n+   * @return serialized string\n+   */\n+  public static String serializeCopyableFile(CopyableFile copyableFile) throws IOException {","path":"gobblin-data-management/src/main/java/gobblin/data/management/copy/CopyableFile.java","position":137,"original_position":137,"commit_id":"64d84d423e34c83850943c8a70512d4654d1ab16","original_commit_id":"64d84d423e34c83850943c8a70512d4654d1ab16","user":{"login":"chavdar","id":928795,"avatar_url":"https://avatars.githubusercontent.com/u/928795?v=3","gravatar_id":"","url":"https://api.github.com/users/chavdar","html_url":"https://github.com/chavdar","followers_url":"https://api.github.com/users/chavdar/followers","following_url":"https://api.github.com/users/chavdar/following{/other_user}","gists_url":"https://api.github.com/users/chavdar/gists{/gist_id}","starred_url":"https://api.github.com/users/chavdar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chavdar/subscriptions","organizations_url":"https://api.github.com/users/chavdar/orgs","repos_url":"https://api.github.com/users/chavdar/repos","events_url":"https://api.github.com/users/chavdar/events{/privacy}","received_events_url":"https://api.github.com/users/chavdar/received_events","type":"User","site_admin":false},"body":"I think if we are serializing to a String, we better use something human-readable like json.","created_at":"2015-10-27T23:35:46Z","updated_at":"2015-10-27T23:35:46Z","html_url":"https://github.com/linkedin/gobblin/pull/411#discussion_r43202843","pull_request_url":"https://api.github.com/repos/linkedin/gobblin/pulls/411","_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/43202843"},"html":{"href":"https://github.com/linkedin/gobblin/pull/411#discussion_r43202843"},"pull_request":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/411"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/411","id":48913442,"html_url":"https://github.com/linkedin/gobblin/pull/411","diff_url":"https://github.com/linkedin/gobblin/pull/411.diff","patch_url":"https://github.com/linkedin/gobblin/pull/411.patch","issue_url":"https://api.github.com/repos/linkedin/gobblin/issues/411","number":411,"state":"open","locked":false,"title":"Dataset and copyable dataset changes for gobblin distcp","user":{"login":"pcadabam","id":1166372,"avatar_url":"https://avatars.githubusercontent.com/u/1166372?v=3","gravatar_id":"","url":"https://api.github.com/users/pcadabam","html_url":"https://github.com/pcadabam","followers_url":"https://api.github.com/users/pcadabam/followers","following_url":"https://api.github.com/users/pcadabam/following{/other_user}","gists_url":"https://api.github.com/users/pcadabam/gists{/gist_id}","starred_url":"https://api.github.com/users/pcadabam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pcadabam/subscriptions","organizations_url":"https://api.github.com/users/pcadabam/orgs","repos_url":"https://api.github.com/users/pcadabam/repos","events_url":"https://api.github.com/users/pcadabam/events{/privacy}","received_events_url":"https://api.github.com/users/pcadabam/received_events","type":"User","site_admin":false},"body":"@ibuenros and @chavdar please take a look at this.\r\n\r\nSplit this PR https://github.com/linkedin/gobblin/pull/403 into 2 parts. It was difficult to split part 1 as they were closely related.\r\n\r\nPart 1 - Dataset and CopyableDataset changes \r\nPart 2 - SFTP changes\r\n\r\nThis PR is for part 1. I will send out another PR for part 2 after merging this.","created_at":"2015-10-27T18:36:35Z","updated_at":"2015-10-27T23:35:46Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e04c25413411c725519b951cae717492cb3fc032","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/gobblin/pulls/411/commits","review_comments_url":"https://api.github.com/repos/linkedin/gobblin/pulls/411/comments","review_comment_url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/gobblin/issues/411/comments","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/64d84d423e34c83850943c8a70512d4654d1ab16","head":{"label":"pcadabam:master","ref":"master","sha":"64d84d423e34c83850943c8a70512d4654d1ab16","user":{"login":"pcadabam","id":1166372,"avatar_url":"https://avatars.githubusercontent.com/u/1166372?v=3","gravatar_id":"","url":"https://api.github.com/users/pcadabam","html_url":"https://github.com/pcadabam","followers_url":"https://api.github.com/users/pcadabam/followers","following_url":"https://api.github.com/users/pcadabam/following{/other_user}","gists_url":"https://api.github.com/users/pcadabam/gists{/gist_id}","starred_url":"https://api.github.com/users/pcadabam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pcadabam/subscriptions","organizations_url":"https://api.github.com/users/pcadabam/orgs","repos_url":"https://api.github.com/users/pcadabam/repos","events_url":"https://api.github.com/users/pcadabam/events{/privacy}","received_events_url":"https://api.github.com/users/pcadabam/received_events","type":"User","site_admin":false},"repo":{"id":36956709,"name":"gobblin","full_name":"pcadabam/gobblin","owner":{"login":"pcadabam","id":1166372,"avatar_url":"https://avatars.githubusercontent.com/u/1166372?v=3","gravatar_id":"","url":"https://api.github.com/users/pcadabam","html_url":"https://github.com/pcadabam","followers_url":"https://api.github.com/users/pcadabam/followers","following_url":"https://api.github.com/users/pcadabam/following{/other_user}","gists_url":"https://api.github.com/users/pcadabam/gists{/gist_id}","starred_url":"https://api.github.com/users/pcadabam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pcadabam/subscriptions","organizations_url":"https://api.github.com/users/pcadabam/orgs","repos_url":"https://api.github.com/users/pcadabam/repos","events_url":"https://api.github.com/users/pcadabam/events{/privacy}","received_events_url":"https://api.github.com/users/pcadabam/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/pcadabam/gobblin","description":"","fork":true,"url":"https://api.github.com/repos/pcadabam/gobblin","forks_url":"https://api.github.com/repos/pcadabam/gobblin/forks","keys_url":"https://api.github.com/repos/pcadabam/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pcadabam/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pcadabam/gobblin/teams","hooks_url":"https://api.github.com/repos/pcadabam/gobblin/hooks","issue_events_url":"https://api.github.com/repos/pcadabam/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/pcadabam/gobblin/events","assignees_url":"https://api.github.com/repos/pcadabam/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/pcadabam/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/pcadabam/gobblin/tags","blobs_url":"https://api.github.com/repos/pcadabam/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pcadabam/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pcadabam/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/pcadabam/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pcadabam/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/pcadabam/gobblin/languages","stargazers_url":"https://api.github.com/repos/pcadabam/gobblin/stargazers","contributors_url":"https://api.github.com/repos/pcadabam/gobblin/contributors","subscribers_url":"https://api.github.com/repos/pcadabam/gobblin/subscribers","subscription_url":"https://api.github.com/repos/pcadabam/gobblin/subscription","commits_url":"https://api.github.com/repos/pcadabam/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/pcadabam/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/pcadabam/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/pcadabam/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/pcadabam/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/pcadabam/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pcadabam/gobblin/merges","archive_url":"https://api.github.com/repos/pcadabam/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pcadabam/gobblin/downloads","issues_url":"https://api.github.com/repos/pcadabam/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/pcadabam/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/pcadabam/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/pcadabam/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pcadabam/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/pcadabam/gobblin/releases{/id}","created_at":"2015-06-05T21:39:40Z","updated_at":"2015-06-05T21:39:42Z","pushed_at":"2015-10-27T18:32:31Z","git_url":"git://github.com/pcadabam/gobblin.git","ssh_url":"git@github.com:pcadabam/gobblin.git","clone_url":"https://github.com/pcadabam/gobblin.git","svn_url":"https://github.com/pcadabam/gobblin","homepage":null,"size":80265,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"f6c52075732215809c2028ddf1477ba164d64ff5","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":27392543,"name":"gobblin","full_name":"linkedin/gobblin","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":false,"url":"https://api.github.com/repos/linkedin/gobblin","forks_url":"https://api.github.com/repos/linkedin/gobblin/forks","keys_url":"https://api.github.com/repos/linkedin/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/gobblin/teams","hooks_url":"https://api.github.com/repos/linkedin/gobblin/hooks","issue_events_url":"https://api.github.com/repos/linkedin/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/gobblin/events","assignees_url":"https://api.github.com/repos/linkedin/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/gobblin/tags","blobs_url":"https://api.github.com/repos/linkedin/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/gobblin/languages","stargazers_url":"https://api.github.com/repos/linkedin/gobblin/stargazers","contributors_url":"https://api.github.com/repos/linkedin/gobblin/contributors","subscribers_url":"https://api.github.com/repos/linkedin/gobblin/subscribers","subscription_url":"https://api.github.com/repos/linkedin/gobblin/subscription","commits_url":"https://api.github.com/repos/linkedin/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/gobblin/merges","archive_url":"https://api.github.com/repos/linkedin/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/gobblin/downloads","issues_url":"https://api.github.com/repos/linkedin/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/gobblin/releases{/id}","created_at":"2014-12-01T18:10:50Z","updated_at":"2015-10-27T15:23:56Z","pushed_at":"2015-10-27T22:22:03Z","git_url":"git://github.com/linkedin/gobblin.git","ssh_url":"git@github.com:linkedin/gobblin.git","clone_url":"https://github.com/linkedin/gobblin.git","svn_url":"https://github.com/linkedin/gobblin","homepage":"https://github.com/linkedin/gobblin/wiki","size":161931,"stargazers_count":261,"watchers_count":261,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":85,"mirror_url":null,"open_issues_count":12,"forks":85,"open_issues":12,"watchers":261,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/411"},"html":{"href":"https://github.com/linkedin/gobblin/pull/411"},"issue":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/411"},"comments":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/411/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/411/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/411/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/gobblin/statuses/64d84d423e34c83850943c8a70512d4654d1ab16"}}}}