{"action":"created","comment":{"url":"https://api.github.com/repos/joey/flume/pulls/comments/23483736","id":23483736,"diff_hunk":"@@ -0,0 +1,79 @@\n+/**\n+ * Licensed to the Apache Software Foundation (ASF) under one\n+ * or more contributor license agreements.  See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership.  The ASF licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License.  You may obtain a copy of the License at\n+ *\n+ *     http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.flume.sink.kite.policy;\n+\n+import org.apache.flume.Context;\n+import org.apache.flume.Event;\n+import org.apache.flume.EventDeliveryException;\n+\n+/**\n+ * A policy for dealing with non-recoverable event delivery failures.\n+ * \n+ * Non-recoverable event delivery failures include:\n+ * \n+ * 1. Error parsing the event body thrown from the {@link EntityParser}\n+ * 2. A schema mismatch between the schema of an event and the schema of the\n+ *    destination dataset.\n+ * 3. A missing schema from the Event header when using the\n+ *    {@link AvroEntityParser}.\n+ */\n+public interface FailurePolicy {\n+\n+  /**\n+   * Handle a non-recoverable event.\n+   * \n+   * @param event The event\n+   * @param cause The cause of the failure\n+   * @throws EventDeliveryException The policy failed to handle the event. When\n+   *                                this is thrown, the Flume transaction will\n+   *                                be rolled back and the event will be retried\n+   *                                along with the rest of the batch.\n+   */\n+  public void handle(Event event, Throwable cause)\n+      throws EventDeliveryException;\n+\n+  /**\n+   * Called before the Flume transaction is committed.\n+   * \n+   * This allows the policy implementation to flush any data that it may not\n+   * have fully handled.\n+   * \n+   * @throws EventDeliveryException The policy failed while ending the batch.\n+   *                                When this is thrown, the Flume transaction\n+   *                                will be rolled back and the batch will be\n+   *                                retried.\n+   */\n+  public void endBatch() throws EventDeliveryException;","path":"flume-ng-sinks/flume-dataset-sink/src/main/java/org/apache/flume/sink/kite/policy/FailurePolicy.java","position":62,"original_position":62,"commit_id":"c2705578455e849f7f4ce4dc8117bf1bccbbcf32","original_commit_id":"c2705578455e849f7f4ce4dc8117bf1bccbbcf32","user":{"login":"joey","id":249392,"avatar_url":"https://avatars.githubusercontent.com/u/249392?v=3","gravatar_id":"","url":"https://api.github.com/users/joey","html_url":"https://github.com/joey","followers_url":"https://api.github.com/users/joey/followers","following_url":"https://api.github.com/users/joey/following{/other_user}","gists_url":"https://api.github.com/users/joey/gists{/gist_id}","starred_url":"https://api.github.com/users/joey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joey/subscriptions","organizations_url":"https://api.github.com/users/joey/orgs","repos_url":"https://api.github.com/users/joey/repos","events_url":"https://api.github.com/users/joey/events{/privacy}","received_events_url":"https://api.github.com/users/joey/received_events","type":"User","site_admin":false},"body":"If I do that, then I need to make sure that the configured error dataset supports flush since if the user already created it, then it could be a Parquet dataset.","created_at":"2015-01-23T22:00:15Z","updated_at":"2015-01-23T22:00:15Z","html_url":"https://github.com/joey/flume/pull/1#discussion_r23483736","pull_request_url":"https://api.github.com/repos/joey/flume/pulls/1","_links":{"self":{"href":"https://api.github.com/repos/joey/flume/pulls/comments/23483736"},"html":{"href":"https://github.com/joey/flume/pull/1#discussion_r23483736"},"pull_request":{"href":"https://api.github.com/repos/joey/flume/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/joey/flume/pulls/1","id":27947209,"html_url":"https://github.com/joey/flume/pull/1","diff_url":"https://github.com/joey/flume/pull/1.diff","patch_url":"https://github.com/joey/flume/pull/1.patch","issue_url":"https://api.github.com/repos/joey/flume/issues/1","number":1,"state":"open","locked":false,"title":"Flume 2591 datasetsink 2.0","user":{"login":"joey","id":249392,"avatar_url":"https://avatars.githubusercontent.com/u/249392?v=3","gravatar_id":"","url":"https://api.github.com/users/joey","html_url":"https://github.com/joey","followers_url":"https://api.github.com/users/joey/followers","following_url":"https://api.github.com/users/joey/following{/other_user}","gists_url":"https://api.github.com/users/joey/gists{/gist_id}","starred_url":"https://api.github.com/users/joey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joey/subscriptions","organizations_url":"https://api.github.com/users/joey/orgs","repos_url":"https://api.github.com/users/joey/repos","events_url":"https://api.github.com/users/joey/events{/privacy}","received_events_url":"https://api.github.com/users/joey/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-01-23T19:10:32Z","updated_at":"2015-01-23T22:00:15Z","closed_at":null,"merged_at":null,"merge_commit_sha":"38f58eebc9cf1f47d230ee5a1e4ae8b866aa1869","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/joey/flume/pulls/1/commits","review_comments_url":"https://api.github.com/repos/joey/flume/pulls/1/comments","review_comment_url":"https://api.github.com/repos/joey/flume/pulls/comments/{number}","comments_url":"https://api.github.com/repos/joey/flume/issues/1/comments","statuses_url":"https://api.github.com/repos/joey/flume/statuses/c2705578455e849f7f4ce4dc8117bf1bccbbcf32","head":{"label":"joey:FLUME-2591-datasetsink-2.0","ref":"FLUME-2591-datasetsink-2.0","sha":"c2705578455e849f7f4ce4dc8117bf1bccbbcf32","user":{"login":"joey","id":249392,"avatar_url":"https://avatars.githubusercontent.com/u/249392?v=3","gravatar_id":"","url":"https://api.github.com/users/joey","html_url":"https://github.com/joey","followers_url":"https://api.github.com/users/joey/followers","following_url":"https://api.github.com/users/joey/following{/other_user}","gists_url":"https://api.github.com/users/joey/gists{/gist_id}","starred_url":"https://api.github.com/users/joey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joey/subscriptions","organizations_url":"https://api.github.com/users/joey/orgs","repos_url":"https://api.github.com/users/joey/repos","events_url":"https://api.github.com/users/joey/events{/privacy}","received_events_url":"https://api.github.com/users/joey/received_events","type":"User","site_admin":false},"repo":{"id":4204676,"name":"flume","full_name":"joey/flume","owner":{"login":"joey","id":249392,"avatar_url":"https://avatars.githubusercontent.com/u/249392?v=3","gravatar_id":"","url":"https://api.github.com/users/joey","html_url":"https://github.com/joey","followers_url":"https://api.github.com/users/joey/followers","following_url":"https://api.github.com/users/joey/following{/other_user}","gists_url":"https://api.github.com/users/joey/gists{/gist_id}","starred_url":"https://api.github.com/users/joey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joey/subscriptions","organizations_url":"https://api.github.com/users/joey/orgs","repos_url":"https://api.github.com/users/joey/repos","events_url":"https://api.github.com/users/joey/events{/privacy}","received_events_url":"https://api.github.com/users/joey/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/joey/flume","description":"Mirror of Apache Flume","fork":true,"url":"https://api.github.com/repos/joey/flume","forks_url":"https://api.github.com/repos/joey/flume/forks","keys_url":"https://api.github.com/repos/joey/flume/keys{/key_id}","collaborators_url":"https://api.github.com/repos/joey/flume/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/joey/flume/teams","hooks_url":"https://api.github.com/repos/joey/flume/hooks","issue_events_url":"https://api.github.com/repos/joey/flume/issues/events{/number}","events_url":"https://api.github.com/repos/joey/flume/events","assignees_url":"https://api.github.com/repos/joey/flume/assignees{/user}","branches_url":"https://api.github.com/repos/joey/flume/branches{/branch}","tags_url":"https://api.github.com/repos/joey/flume/tags","blobs_url":"https://api.github.com/repos/joey/flume/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/joey/flume/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/joey/flume/git/refs{/sha}","trees_url":"https://api.github.com/repos/joey/flume/git/trees{/sha}","statuses_url":"https://api.github.com/repos/joey/flume/statuses/{sha}","languages_url":"https://api.github.com/repos/joey/flume/languages","stargazers_url":"https://api.github.com/repos/joey/flume/stargazers","contributors_url":"https://api.github.com/repos/joey/flume/contributors","subscribers_url":"https://api.github.com/repos/joey/flume/subscribers","subscription_url":"https://api.github.com/repos/joey/flume/subscription","commits_url":"https://api.github.com/repos/joey/flume/commits{/sha}","git_commits_url":"https://api.github.com/repos/joey/flume/git/commits{/sha}","comments_url":"https://api.github.com/repos/joey/flume/comments{/number}","issue_comment_url":"https://api.github.com/repos/joey/flume/issues/comments/{number}","contents_url":"https://api.github.com/repos/joey/flume/contents/{+path}","compare_url":"https://api.github.com/repos/joey/flume/compare/{base}...{head}","merges_url":"https://api.github.com/repos/joey/flume/merges","archive_url":"https://api.github.com/repos/joey/flume/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/joey/flume/downloads","issues_url":"https://api.github.com/repos/joey/flume/issues{/number}","pulls_url":"https://api.github.com/repos/joey/flume/pulls{/number}","milestones_url":"https://api.github.com/repos/joey/flume/milestones{/number}","notifications_url":"https://api.github.com/repos/joey/flume/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/joey/flume/labels{/name}","releases_url":"https://api.github.com/repos/joey/flume/releases{/id}","created_at":"2012-05-02T15:49:57Z","updated_at":"2015-01-23T19:08:28Z","pushed_at":"2015-01-23T19:09:41Z","git_url":"git://github.com/joey/flume.git","ssh_url":"git@github.com:joey/flume.git","clone_url":"https://github.com/joey/flume.git","svn_url":"https://github.com/joey/flume","homepage":"","size":39645,"stargazers_count":1,"watchers_count":1,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":1,"default_branch":"trunk"}},"base":{"label":"joey:trunk","ref":"trunk","sha":"82631f811a7c6a4d8e6ec886f80d1a21876947e5","user":{"login":"joey","id":249392,"avatar_url":"https://avatars.githubusercontent.com/u/249392?v=3","gravatar_id":"","url":"https://api.github.com/users/joey","html_url":"https://github.com/joey","followers_url":"https://api.github.com/users/joey/followers","following_url":"https://api.github.com/users/joey/following{/other_user}","gists_url":"https://api.github.com/users/joey/gists{/gist_id}","starred_url":"https://api.github.com/users/joey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joey/subscriptions","organizations_url":"https://api.github.com/users/joey/orgs","repos_url":"https://api.github.com/users/joey/repos","events_url":"https://api.github.com/users/joey/events{/privacy}","received_events_url":"https://api.github.com/users/joey/received_events","type":"User","site_admin":false},"repo":{"id":4204676,"name":"flume","full_name":"joey/flume","owner":{"login":"joey","id":249392,"avatar_url":"https://avatars.githubusercontent.com/u/249392?v=3","gravatar_id":"","url":"https://api.github.com/users/joey","html_url":"https://github.com/joey","followers_url":"https://api.github.com/users/joey/followers","following_url":"https://api.github.com/users/joey/following{/other_user}","gists_url":"https://api.github.com/users/joey/gists{/gist_id}","starred_url":"https://api.github.com/users/joey/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joey/subscriptions","organizations_url":"https://api.github.com/users/joey/orgs","repos_url":"https://api.github.com/users/joey/repos","events_url":"https://api.github.com/users/joey/events{/privacy}","received_events_url":"https://api.github.com/users/joey/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/joey/flume","description":"Mirror of Apache Flume","fork":true,"url":"https://api.github.com/repos/joey/flume","forks_url":"https://api.github.com/repos/joey/flume/forks","keys_url":"https://api.github.com/repos/joey/flume/keys{/key_id}","collaborators_url":"https://api.github.com/repos/joey/flume/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/joey/flume/teams","hooks_url":"https://api.github.com/repos/joey/flume/hooks","issue_events_url":"https://api.github.com/repos/joey/flume/issues/events{/number}","events_url":"https://api.github.com/repos/joey/flume/events","assignees_url":"https://api.github.com/repos/joey/flume/assignees{/user}","branches_url":"https://api.github.com/repos/joey/flume/branches{/branch}","tags_url":"https://api.github.com/repos/joey/flume/tags","blobs_url":"https://api.github.com/repos/joey/flume/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/joey/flume/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/joey/flume/git/refs{/sha}","trees_url":"https://api.github.com/repos/joey/flume/git/trees{/sha}","statuses_url":"https://api.github.com/repos/joey/flume/statuses/{sha}","languages_url":"https://api.github.com/repos/joey/flume/languages","stargazers_url":"https://api.github.com/repos/joey/flume/stargazers","contributors_url":"https://api.github.com/repos/joey/flume/contributors","subscribers_url":"https://api.github.com/repos/joey/flume/subscribers","subscription_url":"https://api.github.com/repos/joey/flume/subscription","commits_url":"https://api.github.com/repos/joey/flume/commits{/sha}","git_commits_url":"https://api.github.com/repos/joey/flume/git/commits{/sha}","comments_url":"https://api.github.com/repos/joey/flume/comments{/number}","issue_comment_url":"https://api.github.com/repos/joey/flume/issues/comments/{number}","contents_url":"https://api.github.com/repos/joey/flume/contents/{+path}","compare_url":"https://api.github.com/repos/joey/flume/compare/{base}...{head}","merges_url":"https://api.github.com/repos/joey/flume/merges","archive_url":"https://api.github.com/repos/joey/flume/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/joey/flume/downloads","issues_url":"https://api.github.com/repos/joey/flume/issues{/number}","pulls_url":"https://api.github.com/repos/joey/flume/pulls{/number}","milestones_url":"https://api.github.com/repos/joey/flume/milestones{/number}","notifications_url":"https://api.github.com/repos/joey/flume/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/joey/flume/labels{/name}","releases_url":"https://api.github.com/repos/joey/flume/releases{/id}","created_at":"2012-05-02T15:49:57Z","updated_at":"2015-01-23T19:08:28Z","pushed_at":"2015-01-23T19:09:41Z","git_url":"git://github.com/joey/flume.git","ssh_url":"git@github.com:joey/flume.git","clone_url":"https://github.com/joey/flume.git","svn_url":"https://github.com/joey/flume","homepage":"","size":39645,"stargazers_count":1,"watchers_count":1,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":1,"default_branch":"trunk"}},"_links":{"self":{"href":"https://api.github.com/repos/joey/flume/pulls/1"},"html":{"href":"https://github.com/joey/flume/pull/1"},"issue":{"href":"https://api.github.com/repos/joey/flume/issues/1"},"comments":{"href":"https://api.github.com/repos/joey/flume/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/joey/flume/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/joey/flume/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/joey/flume/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/joey/flume/statuses/c2705578455e849f7f4ce4dc8117bf1bccbbcf32"}}}}