{"action":"created","comment":{"url":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments/28911178","id":28911178,"diff_hunk":"@@ -0,0 +1,141 @@\n+/*\n+ * Licensed to Elasticsearch under one or more contributor\n+ * license agreements. See the NOTICE file distributed with\n+ * this work for additional information regarding copyright\n+ * ownership. Elasticsearch licenses this file to you under\n+ * the Apache License, Version 2.0 (the \"License\"); you may\n+ * not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an\n+ * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ * KIND, either express or implied.  See the License for the\n+ * specific language governing permissions and limitations\n+ * under the License.\n+ */\n+\n+package org.elasticsearch.index.translog.fs;\n+\n+import org.elasticsearch.ElasticsearchException;\n+import org.elasticsearch.common.bytes.BytesArray;\n+import org.elasticsearch.index.translog.Translog;\n+\n+import java.io.Closeable;\n+import java.io.IOException;\n+import java.nio.ByteBuffer;\n+import java.nio.channels.FileChannel;\n+import java.util.concurrent.atomic.AtomicBoolean;\n+import java.util.concurrent.atomic.AtomicLong;\n+\n+/**\n+ * A base class for all classes that allows reading ops from translog files\n+ */\n+public abstract class FsChannelReader implements Closeable {\n+\n+    public static final int UNKNOWN = -1;\n+\n+    protected final long id;\n+    protected final ChannelReference channelReference;\n+    protected final FileChannel channel;\n+    protected final AtomicBoolean closed = new AtomicBoolean(false);\n+\n+    public FsChannelReader(long id, ChannelReference channelReference) {\n+        this.id = id;\n+        this.channelReference = channelReference;\n+        this.channel = channelReference.channel();\n+        assert channelReference.assertAttach(this);\n+    }\n+\n+    public long translogId() {\n+        return this.id;\n+    }\n+\n+    abstract public long sizeInBytes();\n+\n+    /** the position the first operation is written at */\n+    public long firstPosition() {\n+        return channelReference.stream().headerSize();\n+    }\n+\n+\n+    abstract public int totalOperations();\n+\n+    public Translog.Operation readOp(Translog.Location location) throws IOException {\n+        assert location.translogId == id : \"read location's translog id [\" + location.translogId + \"] is not [\" + id + \"]\";\n+        ByteBuffer buffer = ByteBuffer.allocate(location.size);\n+        return readOp(buffer, location.translogLocation, location.size);\n+    }\n+\n+    /**\n+     * reads an operation from the given position and changes to point at the start of the next\n+     * operation\n+     */\n+    public Translog.Operation readOpAndAdvancePosition(ByteBuffer reusableBuffer, AtomicLong position) throws IOException {\n+        final long pos = position.get();\n+        int opSize = readOpSize(reusableBuffer, pos);\n+        Translog.Operation op = readOp(reusableBuffer, pos, opSize);\n+        position.addAndGet(opSize);\n+        return op;\n+    }\n+\n+    protected int readOpSize(ByteBuffer reusableBuffer, long position) {\n+        // read op size from disk\n+        assert reusableBuffer.capacity() >= 4 : \"reusable buffer must have capacity >=4 when reading opSize. got [\" + reusableBuffer.capacity() + \"]\";\n+        try {\n+            reusableBuffer.clear();\n+            reusableBuffer.limit(4);\n+            readBytes(reusableBuffer, position);\n+            reusableBuffer.flip();\n+            // Add an extra 4 to account for the operation size integer itself\n+            return reusableBuffer.getInt() + 4;\n+        } catch (IOException e) {\n+            throw new ElasticsearchException(\"unexpected exception reading from translog snapshot of \" + this.channelReference.file(), e);\n+        }\n+    }\n+\n+    /**\n+     * reads an operation at the given position and returns it. The buffer length is equal to the number\n+     * of bytes reads.\n+     */\n+    protected Translog.Operation readOp(ByteBuffer reusableBuffer, long position, int opSize) throws IOException {","path":"src/main/java/org/elasticsearch/index/translog/fs/FsChannelReader.java","position":103,"original_position":103,"commit_id":"68bb6555b88ebd7a01e323943430905799b17c2d","original_commit_id":"68bb6555b88ebd7a01e323943430905799b17c2d","user":{"login":"s1monw","id":973334,"avatar_url":"https://avatars.githubusercontent.com/u/973334?v=3","gravatar_id":"","url":"https://api.github.com/users/s1monw","html_url":"https://github.com/s1monw","followers_url":"https://api.github.com/users/s1monw/followers","following_url":"https://api.github.com/users/s1monw/following{/other_user}","gists_url":"https://api.github.com/users/s1monw/gists{/gist_id}","starred_url":"https://api.github.com/users/s1monw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s1monw/subscriptions","organizations_url":"https://api.github.com/users/s1monw/orgs","repos_url":"https://api.github.com/users/s1monw/repos","events_url":"https://api.github.com/users/s1monw/events{/privacy}","received_events_url":"https://api.github.com/users/s1monw/received_events","type":"User","site_admin":false},"body":"just name it `read`","created_at":"2015-04-22T20:24:51Z","updated_at":"2015-04-22T20:24:51Z","html_url":"https://github.com/elastic/elasticsearch/pull/10624#discussion_r28911178","pull_request_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/10624","_links":{"self":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments/28911178"},"html":{"href":"https://github.com/elastic/elasticsearch/pull/10624#discussion_r28911178"},"pull_request":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/10624"}}},"pull_request":{"url":"https://api.github.com/repos/elastic/elasticsearch/pulls/10624","id":33427575,"html_url":"https://github.com/elastic/elasticsearch/pull/10624","diff_url":"https://github.com/elastic/elasticsearch/pull/10624.diff","patch_url":"https://github.com/elastic/elasticsearch/pull/10624.patch","issue_url":"https://api.github.com/repos/elastic/elasticsearch/issues/10624","number":10624,"state":"open","locked":false,"title":"Decouple recoveries from engine flush","user":{"login":"bleskes","id":1006375,"avatar_url":"https://avatars.githubusercontent.com/u/1006375?v=3","gravatar_id":"","url":"https://api.github.com/users/bleskes","html_url":"https://github.com/bleskes","followers_url":"https://api.github.com/users/bleskes/followers","following_url":"https://api.github.com/users/bleskes/following{/other_user}","gists_url":"https://api.github.com/users/bleskes/gists{/gist_id}","starred_url":"https://api.github.com/users/bleskes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bleskes/subscriptions","organizations_url":"https://api.github.com/users/bleskes/orgs","repos_url":"https://api.github.com/users/bleskes/repos","events_url":"https://api.github.com/users/bleskes/events{/privacy}","received_events_url":"https://api.github.com/users/bleskes/received_events","type":"User","site_admin":false},"body":"In order to safely complete recoveries / relocations we have to keep all operation done since the recovery start at available for replay. At the moment we do so by preventing the engine from flushing and thus making sure that the operations are kept in the translog. A side effect of this is that the translog keeps on growing until the recovery is done. This is not a problem as we do need these operations but if the another recovery starts concurrently it may have an unneededly long translog to replay. Also, if we shutdown the engine for some reason at this point (like when a node is restarted)  we have to recover a long translog when we come back.\r\n\r\nTo void this, the translog is changed to be based on multiple files instead of a single one. This allows recoveries to keep hold to the files they need while allowing the engine to flush and do a lucene commit (which will create a new translog files bellow the hood).\r\n\r\nChange highlights:\r\n- Refactor Translog file management to allow for multiple files.\r\n- Translog maintains a list of referenced files, both by outstanding recoveries and files containing operations not yet committed to Lucene.\r\n- A new Translog.View concept is introduced, allowing recoveries to get a reference to all currently uncommitted translog files plus all future translog files created until the view is closed. They can use this view to iterate over operations.\r\n- Recovery phase3 is removed. That phase was replaying operations while preventing new writes to the engine. This is unneeded as standard indexing also send all operations from the start of the recovery  to the recovering shard. Replay all ops in the view acquired in recovery start is enough to guarantee no operation is lost.\r\n- Opening and closing the translog is now the responsibility of the IndexShard. ShadowIndexShards do not open the translog.\r\n\r\n\r\nThere are a still some no commits and some open issues around the fact that ShadowIndexShards doesn't have a translog (I have some ideas for solutions but I want to discuss before making the PR even bigger). Finally  `testConcurrentWriteViewsAndSnapshot` has a concurrency issue (test bug) I need to solve but I think we can start the review cycles. @s1monw @dakrone can you have a look?\r\n\r\n","created_at":"2015-04-16T08:13:05Z","updated_at":"2015-04-22T20:24:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":null,"assignee":{"login":"s1monw","id":973334,"avatar_url":"https://avatars.githubusercontent.com/u/973334?v=3","gravatar_id":"","url":"https://api.github.com/users/s1monw","html_url":"https://github.com/s1monw","followers_url":"https://api.github.com/users/s1monw/followers","following_url":"https://api.github.com/users/s1monw/following{/other_user}","gists_url":"https://api.github.com/users/s1monw/gists{/gist_id}","starred_url":"https://api.github.com/users/s1monw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s1monw/subscriptions","organizations_url":"https://api.github.com/users/s1monw/orgs","repos_url":"https://api.github.com/users/s1monw/repos","events_url":"https://api.github.com/users/s1monw/events{/privacy}","received_events_url":"https://api.github.com/users/s1monw/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/10624/commits","review_comments_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/10624/comments","review_comment_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments{/number}","comments_url":"https://api.github.com/repos/elastic/elasticsearch/issues/10624/comments","statuses_url":"https://api.github.com/repos/elastic/elasticsearch/statuses/68bb6555b88ebd7a01e323943430905799b17c2d","head":{"label":"bleskes:gen_translog","ref":"gen_translog","sha":"68bb6555b88ebd7a01e323943430905799b17c2d","user":{"login":"bleskes","id":1006375,"avatar_url":"https://avatars.githubusercontent.com/u/1006375?v=3","gravatar_id":"","url":"https://api.github.com/users/bleskes","html_url":"https://github.com/bleskes","followers_url":"https://api.github.com/users/bleskes/followers","following_url":"https://api.github.com/users/bleskes/following{/other_user}","gists_url":"https://api.github.com/users/bleskes/gists{/gist_id}","starred_url":"https://api.github.com/users/bleskes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bleskes/subscriptions","organizations_url":"https://api.github.com/users/bleskes/orgs","repos_url":"https://api.github.com/users/bleskes/repos","events_url":"https://api.github.com/users/bleskes/events{/privacy}","received_events_url":"https://api.github.com/users/bleskes/received_events","type":"User","site_admin":false},"repo":{"id":9219509,"name":"elasticsearch","full_name":"bleskes/elasticsearch","owner":{"login":"bleskes","id":1006375,"avatar_url":"https://avatars.githubusercontent.com/u/1006375?v=3","gravatar_id":"","url":"https://api.github.com/users/bleskes","html_url":"https://github.com/bleskes","followers_url":"https://api.github.com/users/bleskes/followers","following_url":"https://api.github.com/users/bleskes/following{/other_user}","gists_url":"https://api.github.com/users/bleskes/gists{/gist_id}","starred_url":"https://api.github.com/users/bleskes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bleskes/subscriptions","organizations_url":"https://api.github.com/users/bleskes/orgs","repos_url":"https://api.github.com/users/bleskes/repos","events_url":"https://api.github.com/users/bleskes/events{/privacy}","received_events_url":"https://api.github.com/users/bleskes/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/bleskes/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":true,"url":"https://api.github.com/repos/bleskes/elasticsearch","forks_url":"https://api.github.com/repos/bleskes/elasticsearch/forks","keys_url":"https://api.github.com/repos/bleskes/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/bleskes/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/bleskes/elasticsearch/teams","hooks_url":"https://api.github.com/repos/bleskes/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/bleskes/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/bleskes/elasticsearch/events","assignees_url":"https://api.github.com/repos/bleskes/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/bleskes/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/bleskes/elasticsearch/tags","blobs_url":"https://api.github.com/repos/bleskes/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/bleskes/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/bleskes/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/bleskes/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/bleskes/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/bleskes/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/bleskes/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/bleskes/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/bleskes/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/bleskes/elasticsearch/subscription","commits_url":"https://api.github.com/repos/bleskes/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/bleskes/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/bleskes/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/bleskes/elasticsearch/issues/comments{/number}","contents_url":"https://api.github.com/repos/bleskes/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/bleskes/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/bleskes/elasticsearch/merges","archive_url":"https://api.github.com/repos/bleskes/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/bleskes/elasticsearch/downloads","issues_url":"https://api.github.com/repos/bleskes/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/bleskes/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/bleskes/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/bleskes/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/bleskes/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/bleskes/elasticsearch/releases{/id}","created_at":"2013-04-04T14:06:17Z","updated_at":"2015-04-08T13:40:19Z","pushed_at":"2015-04-22T20:16:36Z","git_url":"git://github.com/bleskes/elasticsearch.git","ssh_url":"git@github.com:bleskes/elasticsearch.git","clone_url":"https://github.com/bleskes/elasticsearch.git","svn_url":"https://github.com/bleskes/elasticsearch","homepage":"http://elasticsearch.org","size":123042,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"elastic:master","ref":"master","sha":"410f1790d1d5cb4cce6c23643d889cc12cd1326f","user":{"login":"elastic","id":6764390,"avatar_url":"https://avatars.githubusercontent.com/u/6764390?v=3","gravatar_id":"","url":"https://api.github.com/users/elastic","html_url":"https://github.com/elastic","followers_url":"https://api.github.com/users/elastic/followers","following_url":"https://api.github.com/users/elastic/following{/other_user}","gists_url":"https://api.github.com/users/elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elastic/subscriptions","organizations_url":"https://api.github.com/users/elastic/orgs","repos_url":"https://api.github.com/users/elastic/repos","events_url":"https://api.github.com/users/elastic/events{/privacy}","received_events_url":"https://api.github.com/users/elastic/received_events","type":"Organization","site_admin":false},"repo":{"id":507775,"name":"elasticsearch","full_name":"elastic/elasticsearch","owner":{"login":"elastic","id":6764390,"avatar_url":"https://avatars.githubusercontent.com/u/6764390?v=3","gravatar_id":"","url":"https://api.github.com/users/elastic","html_url":"https://github.com/elastic","followers_url":"https://api.github.com/users/elastic/followers","following_url":"https://api.github.com/users/elastic/following{/other_user}","gists_url":"https://api.github.com/users/elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elastic/subscriptions","organizations_url":"https://api.github.com/users/elastic/orgs","repos_url":"https://api.github.com/users/elastic/repos","events_url":"https://api.github.com/users/elastic/events{/privacy}","received_events_url":"https://api.github.com/users/elastic/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/elastic/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":false,"url":"https://api.github.com/repos/elastic/elasticsearch","forks_url":"https://api.github.com/repos/elastic/elasticsearch/forks","keys_url":"https://api.github.com/repos/elastic/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/elastic/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/elastic/elasticsearch/teams","hooks_url":"https://api.github.com/repos/elastic/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/elastic/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/elastic/elasticsearch/events","assignees_url":"https://api.github.com/repos/elastic/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/elastic/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/elastic/elasticsearch/tags","blobs_url":"https://api.github.com/repos/elastic/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/elastic/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/elastic/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/elastic/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/elastic/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/elastic/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/elastic/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/elastic/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/elastic/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/elastic/elasticsearch/subscription","commits_url":"https://api.github.com/repos/elastic/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/elastic/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/elastic/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/elastic/elasticsearch/issues/comments{/number}","contents_url":"https://api.github.com/repos/elastic/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/elastic/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/elastic/elasticsearch/merges","archive_url":"https://api.github.com/repos/elastic/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/elastic/elasticsearch/downloads","issues_url":"https://api.github.com/repos/elastic/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/elastic/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/elastic/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/elastic/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/elastic/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/elastic/elasticsearch/releases{/id}","created_at":"2010-02-08T13:20:56Z","updated_at":"2015-04-22T19:13:02Z","pushed_at":"2015-04-22T20:24:20Z","git_url":"git://github.com/elastic/elasticsearch.git","ssh_url":"git@github.com:elastic/elasticsearch.git","clone_url":"https://github.com/elastic/elasticsearch.git","svn_url":"https://github.com/elastic/elasticsearch","homepage":"http://elasticsearch.org","size":337165,"stargazers_count":10902,"watchers_count":10902,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3422,"mirror_url":null,"open_issues_count":1120,"forks":3422,"open_issues":1120,"watchers":10902,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/10624"},"html":{"href":"https://github.com/elastic/elasticsearch/pull/10624"},"issue":{"href":"https://api.github.com/repos/elastic/elasticsearch/issues/10624"},"comments":{"href":"https://api.github.com/repos/elastic/elasticsearch/issues/10624/comments"},"review_comments":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/10624/comments"},"review_comment":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/10624/commits"},"statuses":{"href":"https://api.github.com/repos/elastic/elasticsearch/statuses/68bb6555b88ebd7a01e323943430905799b17c2d"}}}}