{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/35606375","id":35606375,"diff_hunk":"@@ -0,0 +1,79 @@\n+/*\n+* Copyright (C) 2014-2015 LinkedIn Corp. All rights reserved.\n+*\n+* Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use\n+* this file except in compliance with the License. You may obtain a copy of the\n+* License at  http://www.apache.org/licenses/LICENSE-2.0\n+*\n+* Unless required by applicable law or agreed to in writing, software distributed\n+* under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n+* CONDITIONS OF ANY KIND, either express or implied.\n+*/\n+\n+package gobblin.data.management.retention.policy;\n+\n+import java.util.Collection;\n+import java.util.List;\n+\n+import azkaban.utils.Props;\n+\n+import com.google.common.base.Predicate;\n+import com.google.common.collect.Iterables;\n+import com.google.common.collect.Lists;\n+\n+import lombok.AllArgsConstructor;\n+\n+import org.apache.hadoop.fs.Path;\n+\n+import gobblin.configuration.ConfigurationKeys;\n+import gobblin.data.management.retention.version.DatasetVersion;\n+import gobblin.data.management.retention.version.StringDatasetVersion;\n+import gobblin.util.ProxiedFileSystemUtils;\n+\n+/**\n+ * Implementation of {@link RetentionPolicy} that uses a {@link StringDatasetVersion} and checks if each name of a\n+ * {@link StringDatasetVersion} corresponds to a proxyable user.\n+ *\n+ * <p>\n+ *  A proxyable user is defined by the {@link ProxiedFileSystemUtils#canProxyAs(String, String, Path)} method.\n+ * <p>\n+ */\n+@AllArgsConstructor\n+public class ProxyableFolderRetentionPolicy implements RetentionPolicy<StringDatasetVersion> {\n+\n+  private final String userNameToProxyAs;\n+  private final String superUserName;\n+  private final Path superUserKeytabLocation;\n+\n+  public ProxyableFolderRetentionPolicy(Props props) {\n+    this(props.getString(ConfigurationKeys.FS_PROXY_AS_USER_NAME), props\n+        .getString(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS), new Path(\n+        props.getString(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION)));\n+  }\n+\n+  @Override\n+  public Class<? extends DatasetVersion> versionClass() {\n+    return StringDatasetVersion.class;\n+  }\n+\n+  @Override\n+  public Collection<StringDatasetVersion> listDeletableVersions(List<StringDatasetVersion> allVersions) {\n+    return Lists.newArrayList(Iterables.filter(allVersions, new ProxyableVersion()));\n+  }\n+\n+  private class ProxyableVersion implements Predicate<StringDatasetVersion> {\n+\n+    @Override\n+    public boolean apply(StringDatasetVersion input) {\n+      return Iterables.all(input.getPathsToDelete(), new ProxyablePath());\n+    }\n+  }\n+\n+  private class ProxyablePath implements Predicate<Path> {\n+\n+    @Override\n+    public boolean apply(Path input) {\n+      return ProxiedFileSystemUtils.canProxyAs(userNameToProxyAs, superUserName, superUserKeytabLocation);","path":"gobblin-data-management/src/main/java/gobblin/data/management/retention/policy/ProxyableFolderRetentionPolicy.java","position":76,"original_position":76,"commit_id":"54cf9ae52dda118e35798cf065659a69ead6fc03","original_commit_id":"54cf9ae52dda118e35798cf065659a69ead6fc03","user":{"login":"ydailinkedin","id":10764942,"avatar_url":"https://avatars.githubusercontent.com/u/10764942?v=3","gravatar_id":"","url":"https://api.github.com/users/ydailinkedin","html_url":"https://github.com/ydailinkedin","followers_url":"https://api.github.com/users/ydailinkedin/followers","following_url":"https://api.github.com/users/ydailinkedin/following{/other_user}","gists_url":"https://api.github.com/users/ydailinkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/ydailinkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ydailinkedin/subscriptions","organizations_url":"https://api.github.com/users/ydailinkedin/orgs","repos_url":"https://api.github.com/users/ydailinkedin/repos","events_url":"https://api.github.com/users/ydailinkedin/events{/privacy}","received_events_url":"https://api.github.com/users/ydailinkedin/received_events","type":"User","site_admin":false},"body":"For each `input`, this check will require a log-in action of the super user. I am not sure if there is any pre-check in filesystem cache for the fileowner of `input`. A potential problem is that for 1000 files of a same owner under a same hdfs dir, `apply` will log in 1000 times, which could cause performance issue.","created_at":"2015-07-28T01:14:29Z","updated_at":"2015-07-28T01:14:29Z","html_url":"https://github.com/linkedin/gobblin/pull/249#discussion_r35606375","pull_request_url":"https://api.github.com/repos/linkedin/gobblin/pulls/249","_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/35606375"},"html":{"href":"https://github.com/linkedin/gobblin/pull/249#discussion_r35606375"},"pull_request":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/249"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/249","id":40954396,"html_url":"https://github.com/linkedin/gobblin/pull/249","diff_url":"https://github.com/linkedin/gobblin/pull/249.diff","patch_url":"https://github.com/linkedin/gobblin/pull/249.patch","issue_url":"https://api.github.com/repos/linkedin/gobblin/issues/249","number":249,"state":"open","locked":false,"title":"Adding new data-management policies, updating proxying code, creating PublisherUtils","user":{"login":"sahilTakiar","id":1282583,"avatar_url":"https://avatars.githubusercontent.com/u/1282583?v=3","gravatar_id":"","url":"https://api.github.com/users/sahilTakiar","html_url":"https://github.com/sahilTakiar","followers_url":"https://api.github.com/users/sahilTakiar/followers","following_url":"https://api.github.com/users/sahilTakiar/following{/other_user}","gists_url":"https://api.github.com/users/sahilTakiar/gists{/gist_id}","starred_url":"https://api.github.com/users/sahilTakiar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sahilTakiar/subscriptions","organizations_url":"https://api.github.com/users/sahilTakiar/orgs","repos_url":"https://api.github.com/users/sahilTakiar/repos","events_url":"https://api.github.com/users/sahilTakiar/events{/privacy}","received_events_url":"https://api.github.com/users/sahilTakiar/received_events","type":"User","site_admin":false},"body":"Changes:\r\n\r\n* Creating a few new data-management policies\r\n* Creating new classes `ProxiedFileSystemUtils` and `ProxiedFileSystemCache` in order to replace `ProxiedFileSystemWrapper`. The main reason I did this was the clean up some of the code and to make a cache around all proxied filesystem objects created\r\n* Creating a new class called `PublisherUtils` to store utilities for the `DataPublisher`s","created_at":"2015-07-27T21:58:33Z","updated_at":"2015-07-28T01:14:29Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f1466d6a7075b9de22691e57db634203b81336ef","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/gobblin/pulls/249/commits","review_comments_url":"https://api.github.com/repos/linkedin/gobblin/pulls/249/comments","review_comment_url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/gobblin/issues/249/comments","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/54cf9ae52dda118e35798cf065659a69ead6fc03","head":{"label":"sahilTakiar:retention","ref":"retention","sha":"54cf9ae52dda118e35798cf065659a69ead6fc03","user":{"login":"sahilTakiar","id":1282583,"avatar_url":"https://avatars.githubusercontent.com/u/1282583?v=3","gravatar_id":"","url":"https://api.github.com/users/sahilTakiar","html_url":"https://github.com/sahilTakiar","followers_url":"https://api.github.com/users/sahilTakiar/followers","following_url":"https://api.github.com/users/sahilTakiar/following{/other_user}","gists_url":"https://api.github.com/users/sahilTakiar/gists{/gist_id}","starred_url":"https://api.github.com/users/sahilTakiar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sahilTakiar/subscriptions","organizations_url":"https://api.github.com/users/sahilTakiar/orgs","repos_url":"https://api.github.com/users/sahilTakiar/repos","events_url":"https://api.github.com/users/sahilTakiar/events{/privacy}","received_events_url":"https://api.github.com/users/sahilTakiar/received_events","type":"User","site_admin":false},"repo":{"id":38903584,"name":"gobblin","full_name":"sahilTakiar/gobblin","owner":{"login":"sahilTakiar","id":1282583,"avatar_url":"https://avatars.githubusercontent.com/u/1282583?v=3","gravatar_id":"","url":"https://api.github.com/users/sahilTakiar","html_url":"https://github.com/sahilTakiar","followers_url":"https://api.github.com/users/sahilTakiar/followers","following_url":"https://api.github.com/users/sahilTakiar/following{/other_user}","gists_url":"https://api.github.com/users/sahilTakiar/gists{/gist_id}","starred_url":"https://api.github.com/users/sahilTakiar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sahilTakiar/subscriptions","organizations_url":"https://api.github.com/users/sahilTakiar/orgs","repos_url":"https://api.github.com/users/sahilTakiar/repos","events_url":"https://api.github.com/users/sahilTakiar/events{/privacy}","received_events_url":"https://api.github.com/users/sahilTakiar/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/sahilTakiar/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":true,"url":"https://api.github.com/repos/sahilTakiar/gobblin","forks_url":"https://api.github.com/repos/sahilTakiar/gobblin/forks","keys_url":"https://api.github.com/repos/sahilTakiar/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/sahilTakiar/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/sahilTakiar/gobblin/teams","hooks_url":"https://api.github.com/repos/sahilTakiar/gobblin/hooks","issue_events_url":"https://api.github.com/repos/sahilTakiar/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/sahilTakiar/gobblin/events","assignees_url":"https://api.github.com/repos/sahilTakiar/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/sahilTakiar/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/sahilTakiar/gobblin/tags","blobs_url":"https://api.github.com/repos/sahilTakiar/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/sahilTakiar/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/sahilTakiar/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/sahilTakiar/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/sahilTakiar/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/sahilTakiar/gobblin/languages","stargazers_url":"https://api.github.com/repos/sahilTakiar/gobblin/stargazers","contributors_url":"https://api.github.com/repos/sahilTakiar/gobblin/contributors","subscribers_url":"https://api.github.com/repos/sahilTakiar/gobblin/subscribers","subscription_url":"https://api.github.com/repos/sahilTakiar/gobblin/subscription","commits_url":"https://api.github.com/repos/sahilTakiar/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/sahilTakiar/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/sahilTakiar/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/sahilTakiar/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/sahilTakiar/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/sahilTakiar/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/sahilTakiar/gobblin/merges","archive_url":"https://api.github.com/repos/sahilTakiar/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/sahilTakiar/gobblin/downloads","issues_url":"https://api.github.com/repos/sahilTakiar/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/sahilTakiar/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/sahilTakiar/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/sahilTakiar/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/sahilTakiar/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/sahilTakiar/gobblin/releases{/id}","created_at":"2015-07-10T21:50:38Z","updated_at":"2015-07-10T21:50:39Z","pushed_at":"2015-07-28T00:41:39Z","git_url":"git://github.com/sahilTakiar/gobblin.git","ssh_url":"git@github.com:sahilTakiar/gobblin.git","clone_url":"https://github.com/sahilTakiar/gobblin.git","svn_url":"https://github.com/sahilTakiar/gobblin","homepage":"","size":77930,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"2e5237ecbee984dc0751d9381a19ec0292709040","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":27392543,"name":"gobblin","full_name":"linkedin/gobblin","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":false,"url":"https://api.github.com/repos/linkedin/gobblin","forks_url":"https://api.github.com/repos/linkedin/gobblin/forks","keys_url":"https://api.github.com/repos/linkedin/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/gobblin/teams","hooks_url":"https://api.github.com/repos/linkedin/gobblin/hooks","issue_events_url":"https://api.github.com/repos/linkedin/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/gobblin/events","assignees_url":"https://api.github.com/repos/linkedin/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/gobblin/tags","blobs_url":"https://api.github.com/repos/linkedin/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/gobblin/languages","stargazers_url":"https://api.github.com/repos/linkedin/gobblin/stargazers","contributors_url":"https://api.github.com/repos/linkedin/gobblin/contributors","subscribers_url":"https://api.github.com/repos/linkedin/gobblin/subscribers","subscription_url":"https://api.github.com/repos/linkedin/gobblin/subscription","commits_url":"https://api.github.com/repos/linkedin/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/gobblin/merges","archive_url":"https://api.github.com/repos/linkedin/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/gobblin/downloads","issues_url":"https://api.github.com/repos/linkedin/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/gobblin/releases{/id}","created_at":"2014-12-01T18:10:50Z","updated_at":"2015-07-27T23:26:35Z","pushed_at":"2015-07-28T00:41:39Z","git_url":"git://github.com/linkedin/gobblin.git","ssh_url":"git@github.com:linkedin/gobblin.git","clone_url":"https://github.com/linkedin/gobblin.git","svn_url":"https://github.com/linkedin/gobblin","homepage":"","size":157766,"stargazers_count":97,"watchers_count":97,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":39,"mirror_url":null,"open_issues_count":3,"forks":39,"open_issues":3,"watchers":97,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/249"},"html":{"href":"https://github.com/linkedin/gobblin/pull/249"},"issue":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/249"},"comments":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/249/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/249/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/249/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/gobblin/statuses/54cf9ae52dda118e35798cf065659a69ead6fc03"}}}}