{"action":"created","comment":{"url":"https://api.github.com/repos/kite-sdk/kite/pulls/comments/28343287","id":28343287,"diff_hunk":"@@ -0,0 +1,229 @@\n+/**\n+ * Copyright 2015 Cloudera Inc.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\");\n+ * you may not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package org.kitesdk.data.spi.filesystem;\n+\n+import java.io.FileNotFoundException;\n+import java.io.IOException;\n+import java.util.ArrayList;\n+import java.util.EnumSet;\n+import java.util.List;\n+import java.util.Map;\n+import java.util.TreeMap;\n+\n+import org.apache.hadoop.conf.Configuration;\n+import org.apache.hadoop.fs.CreateFlag;\n+import org.apache.hadoop.fs.FSDataInputStream;\n+import org.apache.hadoop.fs.FSDataOutputStream;\n+import org.apache.hadoop.fs.FileContext;\n+import org.apache.hadoop.fs.FileStatus;\n+import org.apache.hadoop.fs.FileSystem;\n+import org.apache.hadoop.fs.Options.CreateOpts;\n+import org.apache.hadoop.fs.Options.Rename;\n+import org.apache.hadoop.fs.Path;\n+import org.apache.hadoop.fs.permission.FsPermission;\n+import org.kitesdk.data.DatasetException;\n+import org.kitesdk.data.DatasetIOException;\n+import org.kitesdk.data.spi.Constraints;\n+\n+import com.google.common.base.Joiner;\n+\n+/**\n+ * Manager for creating, and checking {@link View#isReady() ready} signals.\n+ * Stored in a filesystem, typically HDFS.\n+ */\n+public class SignalManager {\n+\n+  private final Path signalDirectory;\n+  private final Configuration conf;\n+\n+  private static final String UNBOUNDED_CONSTRAINT = \"unbounded\";\n+\n+  private static final String STAGED_SIGNAL_POSTFIX = \".tmp\";\n+\n+  /**\n+   * Creates a new signal manager using the given signal directory.\n+   *\n+   * @param conf the Hadoop configuration\n+   * @param signalDirectory directory in which the manager\n+   *                        stores signals.\n+   *\n+   * @return a signal manager instance.\n+   */\n+  public static SignalManager create(Configuration conf, Path signalDirectory) {\n+    try {\n+      FileSystem rootFileSystem = signalDirectory.getFileSystem(conf);\n+\n+      rootFileSystem.mkdirs(signalDirectory);\n+\n+      return new SignalManager(signalDirectory, conf);\n+    } catch (IOException e) {\n+      throw new DatasetIOException(\"Unable to create signal manager directory: \"\n+              + signalDirectory, e);\n+    }\n+  }\n+\n+  /**\n+   * Loads a signal manager that stores metadata data under the given signal\n+   * metadata directory if it exists. Returns <code>null</code> if it does not.\n+   *\n+   * @param conf the Hadoop configuration\n+   * @param signalDirectory directory in which the manager stores signals.\n+   *\n+   * @return a signal manager instance, or <code>null</code> if the given\n+   * directory does not exist.\n+   */\n+  public static SignalManager load(Configuration conf, Path signalDirectory) {\n+    try {\n+      FileSystem rootFileSystem = signalDirectory.getFileSystem(conf);\n+\n+      if (rootFileSystem.exists(signalDirectory)) {\n+        return new SignalManager(signalDirectory, conf);\n+      } else {\n+        return null;\n+      }\n+    } catch (IOException e) {\n+      throw new DatasetIOException (\"Cannot load signal manager at:\"\n+              + signalDirectory, e);\n+    }\n+  }\n+\n+  private SignalManager(Path signalDirectory, Configuration conf)\n+          throws IOException  {\n+    this.signalDirectory = signalDirectory;\n+    this.conf = conf;\n+  }\n+\n+  /**\n+   * Create a signal for the specified constraints.\n+   *\n+   * @param viewConstraints The constraints to create a signal for.\n+   *\n+   * @throws DatasetException if the signal could not be created.\n+   */\n+  public void signalReady(Constraints viewConstraints) {\n+    String normalizedConstraints = getNormalizedConstraints(viewConstraints);\n+\n+    Path signalPath = new Path(signalDirectory, normalizedConstraints);\n+    Path stagedSignalPath = new Path(signalDirectory, normalizedConstraints + STAGED_SIGNAL_POSTFIX);\n+\n+    try{\n+      // create the output stream to overwrite the current contents, if the directory or file\n+      // does not exist it will be created\n+      FileContext context = FileContext.getFileContext(conf);\n+      final FSDataOutputStream os = context.create(stagedSignalPath,\n+              EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), CreateOpts.perms(FsPermission.getDefault()));","path":"kite-data/kite-data-core/src/main/java/org/kitesdk/data/spi/filesystem/SignalManager.java","position":127,"original_position":127,"commit_id":"cc934fc8527bfed51751d6afa2d28b3a0390f289","original_commit_id":"cc934fc8527bfed51751d6afa2d28b3a0390f289","user":{"login":"joey","id":249392,"avatar_url":"https://avatars.githubusercontent.com/u/249392?v=3","gravatar_id":"","url":"https://api.github.com/users/joey","html_url":"https://github.com/joey","followers_url":"https://api.github.com/users/joey/followers","following_url":"https://api.github.com/users/joey/following{/other_user}","gists_url":"https://api.github.com/users/joey/gists{/gist_id}","starred_url":"https://api.github.com/users/joey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joey/subscriptions","organizations_url":"https://api.github.com/users/joey/orgs","repos_url":"https://api.github.com/users/joey/repos","events_url":"https://api.github.com/users/joey/events{/privacy}","received_events_url":"https://api.github.com/users/joey/received_events","type":"User","site_admin":false},"body":"We should discuss dropping Hadoop1 support on the mailing list. I'm in favor of it, but I do think it would be considered a backwards incompatible change so it would require a version number bump.","created_at":"2015-04-14T16:23:37Z","updated_at":"2015-04-14T16:23:37Z","html_url":"https://github.com/kite-sdk/kite/pull/348#discussion_r28343287","pull_request_url":"https://api.github.com/repos/kite-sdk/kite/pulls/348","_links":{"self":{"href":"https://api.github.com/repos/kite-sdk/kite/pulls/comments/28343287"},"html":{"href":"https://github.com/kite-sdk/kite/pull/348#discussion_r28343287"},"pull_request":{"href":"https://api.github.com/repos/kite-sdk/kite/pulls/348"}}},"pull_request":{"url":"https://api.github.com/repos/kite-sdk/kite/pulls/348","id":33185622,"html_url":"https://github.com/kite-sdk/kite/pull/348","diff_url":"https://github.com/kite-sdk/kite/pull/348.diff","patch_url":"https://github.com/kite-sdk/kite/pull/348.patch","issue_url":"https://api.github.com/repos/kite-sdk/kite/issues/348","number":348,"state":"open","locked":false,"title":"CDK-451 signal ready views","user":{"login":"bbrownz","id":1099854,"avatar_url":"https://avatars.githubusercontent.com/u/1099854?v=3","gravatar_id":"","url":"https://api.github.com/users/bbrownz","html_url":"https://github.com/bbrownz","followers_url":"https://api.github.com/users/bbrownz/followers","following_url":"https://api.github.com/users/bbrownz/following{/other_user}","gists_url":"https://api.github.com/users/bbrownz/gists{/gist_id}","starred_url":"https://api.github.com/users/bbrownz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bbrownz/subscriptions","organizations_url":"https://api.github.com/users/bbrownz/orgs","repos_url":"https://api.github.com/users/bbrownz/repos","events_url":"https://api.github.com/users/bbrownz/events{/privacy}","received_events_url":"https://api.github.com/users/bbrownz/received_events","type":"User","site_admin":false},"body":"Implementation for hdfs and hive based views, hbase is currently unimplemented.\r\n\r\nIncludes updates to interfaces for views for the basic signalReady/isReady concept\r\n\r\nUpdates were also made to the dataset repository interface to create these signals for datasets based on constraints, and to determine the last time those constraints were signaled.\r\n\r\nMapReduce was updated to create these signals (if possible) when a write to a dataset is successful. Crunch was updated to support using these signals for WriteMode.CHECKPOINT (with a 'ready' view essentially indicating a previous success)","created_at":"2015-04-13T17:39:52Z","updated_at":"2015-04-14T16:23:37Z","closed_at":null,"merged_at":null,"merge_commit_sha":"99d3b7651bec216b008f80d36dcb9ce401e13b0b","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/kite-sdk/kite/pulls/348/commits","review_comments_url":"https://api.github.com/repos/kite-sdk/kite/pulls/348/comments","review_comment_url":"https://api.github.com/repos/kite-sdk/kite/pulls/comments{/number}","comments_url":"https://api.github.com/repos/kite-sdk/kite/issues/348/comments","statuses_url":"https://api.github.com/repos/kite-sdk/kite/statuses/cc934fc8527bfed51751d6afa2d28b3a0390f289","head":{"label":"bbrownz:CDK-451","ref":"CDK-451","sha":"cc934fc8527bfed51751d6afa2d28b3a0390f289","user":{"login":"bbrownz","id":1099854,"avatar_url":"https://avatars.githubusercontent.com/u/1099854?v=3","gravatar_id":"","url":"https://api.github.com/users/bbrownz","html_url":"https://github.com/bbrownz","followers_url":"https://api.github.com/users/bbrownz/followers","following_url":"https://api.github.com/users/bbrownz/following{/other_user}","gists_url":"https://api.github.com/users/bbrownz/gists{/gist_id}","starred_url":"https://api.github.com/users/bbrownz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bbrownz/subscriptions","organizations_url":"https://api.github.com/users/bbrownz/orgs","repos_url":"https://api.github.com/users/bbrownz/repos","events_url":"https://api.github.com/users/bbrownz/events{/privacy}","received_events_url":"https://api.github.com/users/bbrownz/received_events","type":"User","site_admin":false},"repo":{"id":33877632,"name":"kite","full_name":"bbrownz/kite","owner":{"login":"bbrownz","id":1099854,"avatar_url":"https://avatars.githubusercontent.com/u/1099854?v=3","gravatar_id":"","url":"https://api.github.com/users/bbrownz","html_url":"https://github.com/bbrownz","followers_url":"https://api.github.com/users/bbrownz/followers","following_url":"https://api.github.com/users/bbrownz/following{/other_user}","gists_url":"https://api.github.com/users/bbrownz/gists{/gist_id}","starred_url":"https://api.github.com/users/bbrownz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bbrownz/subscriptions","organizations_url":"https://api.github.com/users/bbrownz/orgs","repos_url":"https://api.github.com/users/bbrownz/repos","events_url":"https://api.github.com/users/bbrownz/events{/privacy}","received_events_url":"https://api.github.com/users/bbrownz/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/bbrownz/kite","description":"Kite SDK","fork":true,"url":"https://api.github.com/repos/bbrownz/kite","forks_url":"https://api.github.com/repos/bbrownz/kite/forks","keys_url":"https://api.github.com/repos/bbrownz/kite/keys{/key_id}","collaborators_url":"https://api.github.com/repos/bbrownz/kite/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/bbrownz/kite/teams","hooks_url":"https://api.github.com/repos/bbrownz/kite/hooks","issue_events_url":"https://api.github.com/repos/bbrownz/kite/issues/events{/number}","events_url":"https://api.github.com/repos/bbrownz/kite/events","assignees_url":"https://api.github.com/repos/bbrownz/kite/assignees{/user}","branches_url":"https://api.github.com/repos/bbrownz/kite/branches{/branch}","tags_url":"https://api.github.com/repos/bbrownz/kite/tags","blobs_url":"https://api.github.com/repos/bbrownz/kite/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/bbrownz/kite/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/bbrownz/kite/git/refs{/sha}","trees_url":"https://api.github.com/repos/bbrownz/kite/git/trees{/sha}","statuses_url":"https://api.github.com/repos/bbrownz/kite/statuses/{sha}","languages_url":"https://api.github.com/repos/bbrownz/kite/languages","stargazers_url":"https://api.github.com/repos/bbrownz/kite/stargazers","contributors_url":"https://api.github.com/repos/bbrownz/kite/contributors","subscribers_url":"https://api.github.com/repos/bbrownz/kite/subscribers","subscription_url":"https://api.github.com/repos/bbrownz/kite/subscription","commits_url":"https://api.github.com/repos/bbrownz/kite/commits{/sha}","git_commits_url":"https://api.github.com/repos/bbrownz/kite/git/commits{/sha}","comments_url":"https://api.github.com/repos/bbrownz/kite/comments{/number}","issue_comment_url":"https://api.github.com/repos/bbrownz/kite/issues/comments{/number}","contents_url":"https://api.github.com/repos/bbrownz/kite/contents/{+path}","compare_url":"https://api.github.com/repos/bbrownz/kite/compare/{base}...{head}","merges_url":"https://api.github.com/repos/bbrownz/kite/merges","archive_url":"https://api.github.com/repos/bbrownz/kite/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/bbrownz/kite/downloads","issues_url":"https://api.github.com/repos/bbrownz/kite/issues{/number}","pulls_url":"https://api.github.com/repos/bbrownz/kite/pulls{/number}","milestones_url":"https://api.github.com/repos/bbrownz/kite/milestones{/number}","notifications_url":"https://api.github.com/repos/bbrownz/kite/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/bbrownz/kite/labels{/name}","releases_url":"https://api.github.com/repos/bbrownz/kite/releases{/id}","created_at":"2015-04-13T15:45:33Z","updated_at":"2015-04-13T15:45:38Z","pushed_at":"2015-04-14T14:35:43Z","git_url":"git://github.com/bbrownz/kite.git","ssh_url":"git@github.com:bbrownz/kite.git","clone_url":"https://github.com/bbrownz/kite.git","svn_url":"https://github.com/bbrownz/kite","homepage":"http://kitesdk.org/docs/current/","size":76931,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"kite-sdk:master","ref":"master","sha":"738b397cf1f8b1fbea4fbe9bda017839081d9f37","user":{"login":"kite-sdk","id":5732728,"avatar_url":"https://avatars.githubusercontent.com/u/5732728?v=3","gravatar_id":"","url":"https://api.github.com/users/kite-sdk","html_url":"https://github.com/kite-sdk","followers_url":"https://api.github.com/users/kite-sdk/followers","following_url":"https://api.github.com/users/kite-sdk/following{/other_user}","gists_url":"https://api.github.com/users/kite-sdk/gists{/gist_id}","starred_url":"https://api.github.com/users/kite-sdk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kite-sdk/subscriptions","organizations_url":"https://api.github.com/users/kite-sdk/orgs","repos_url":"https://api.github.com/users/kite-sdk/repos","events_url":"https://api.github.com/users/kite-sdk/events{/privacy}","received_events_url":"https://api.github.com/users/kite-sdk/received_events","type":"Organization","site_admin":false},"repo":{"id":14829187,"name":"kite","full_name":"kite-sdk/kite","owner":{"login":"kite-sdk","id":5732728,"avatar_url":"https://avatars.githubusercontent.com/u/5732728?v=3","gravatar_id":"","url":"https://api.github.com/users/kite-sdk","html_url":"https://github.com/kite-sdk","followers_url":"https://api.github.com/users/kite-sdk/followers","following_url":"https://api.github.com/users/kite-sdk/following{/other_user}","gists_url":"https://api.github.com/users/kite-sdk/gists{/gist_id}","starred_url":"https://api.github.com/users/kite-sdk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kite-sdk/subscriptions","organizations_url":"https://api.github.com/users/kite-sdk/orgs","repos_url":"https://api.github.com/users/kite-sdk/repos","events_url":"https://api.github.com/users/kite-sdk/events{/privacy}","received_events_url":"https://api.github.com/users/kite-sdk/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/kite-sdk/kite","description":"Kite SDK","fork":false,"url":"https://api.github.com/repos/kite-sdk/kite","forks_url":"https://api.github.com/repos/kite-sdk/kite/forks","keys_url":"https://api.github.com/repos/kite-sdk/kite/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kite-sdk/kite/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kite-sdk/kite/teams","hooks_url":"https://api.github.com/repos/kite-sdk/kite/hooks","issue_events_url":"https://api.github.com/repos/kite-sdk/kite/issues/events{/number}","events_url":"https://api.github.com/repos/kite-sdk/kite/events","assignees_url":"https://api.github.com/repos/kite-sdk/kite/assignees{/user}","branches_url":"https://api.github.com/repos/kite-sdk/kite/branches{/branch}","tags_url":"https://api.github.com/repos/kite-sdk/kite/tags","blobs_url":"https://api.github.com/repos/kite-sdk/kite/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kite-sdk/kite/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kite-sdk/kite/git/refs{/sha}","trees_url":"https://api.github.com/repos/kite-sdk/kite/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kite-sdk/kite/statuses/{sha}","languages_url":"https://api.github.com/repos/kite-sdk/kite/languages","stargazers_url":"https://api.github.com/repos/kite-sdk/kite/stargazers","contributors_url":"https://api.github.com/repos/kite-sdk/kite/contributors","subscribers_url":"https://api.github.com/repos/kite-sdk/kite/subscribers","subscription_url":"https://api.github.com/repos/kite-sdk/kite/subscription","commits_url":"https://api.github.com/repos/kite-sdk/kite/commits{/sha}","git_commits_url":"https://api.github.com/repos/kite-sdk/kite/git/commits{/sha}","comments_url":"https://api.github.com/repos/kite-sdk/kite/comments{/number}","issue_comment_url":"https://api.github.com/repos/kite-sdk/kite/issues/comments{/number}","contents_url":"https://api.github.com/repos/kite-sdk/kite/contents/{+path}","compare_url":"https://api.github.com/repos/kite-sdk/kite/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kite-sdk/kite/merges","archive_url":"https://api.github.com/repos/kite-sdk/kite/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kite-sdk/kite/downloads","issues_url":"https://api.github.com/repos/kite-sdk/kite/issues{/number}","pulls_url":"https://api.github.com/repos/kite-sdk/kite/pulls{/number}","milestones_url":"https://api.github.com/repos/kite-sdk/kite/milestones{/number}","notifications_url":"https://api.github.com/repos/kite-sdk/kite/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kite-sdk/kite/labels{/name}","releases_url":"https://api.github.com/repos/kite-sdk/kite/releases{/id}","created_at":"2013-11-30T22:06:02Z","updated_at":"2015-04-11T00:15:50Z","pushed_at":"2015-04-14T14:35:44Z","git_url":"git://github.com/kite-sdk/kite.git","ssh_url":"git@github.com:kite-sdk/kite.git","clone_url":"https://github.com/kite-sdk/kite.git","svn_url":"https://github.com/kite-sdk/kite","homepage":"http://kitesdk.org/docs/current/","size":76931,"stargazers_count":124,"watchers_count":124,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":89,"mirror_url":null,"open_issues_count":13,"forks":89,"open_issues":13,"watchers":124,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/kite-sdk/kite/pulls/348"},"html":{"href":"https://github.com/kite-sdk/kite/pull/348"},"issue":{"href":"https://api.github.com/repos/kite-sdk/kite/issues/348"},"comments":{"href":"https://api.github.com/repos/kite-sdk/kite/issues/348/comments"},"review_comments":{"href":"https://api.github.com/repos/kite-sdk/kite/pulls/348/comments"},"review_comment":{"href":"https://api.github.com/repos/kite-sdk/kite/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/kite-sdk/kite/pulls/348/commits"},"statuses":{"href":"https://api.github.com/repos/kite-sdk/kite/statuses/cc934fc8527bfed51751d6afa2d28b3a0390f289"}}}}