{"action":"created","comment":{"url":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/comments/22410103","id":22410103,"diff_hunk":"@@ -0,0 +1,144 @@\n+/*\n+ * Licensed to Elasticsearch under one or more contributor\n+ * license agreements. See the NOTICE file distributed with\n+ * this work for additional information regarding copyright\n+ * ownership. Elasticsearch licenses this file to you under\n+ * the Apache License, Version 2.0 (the \"License\"); you may\n+ * not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an\n+ * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ * KIND, either express or implied.  See the License for the\n+ * specific language governing permissions and limitations\n+ * under the License.\n+ */\n+package org.elasticsearch.index.store;\n+\n+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;\n+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;\n+import org.elasticsearch.action.bulk.BulkItemResponse;\n+import org.elasticsearch.action.bulk.BulkRequestBuilder;\n+import org.elasticsearch.action.bulk.BulkResponse;\n+import org.elasticsearch.action.search.SearchResponse;\n+import org.elasticsearch.cluster.node.DiscoveryNode;\n+import org.elasticsearch.common.settings.ImmutableSettings;\n+import org.elasticsearch.common.settings.Settings;\n+import org.elasticsearch.common.xcontent.XContentBuilder;\n+import org.elasticsearch.discovery.Discovery;\n+import org.elasticsearch.rest.RestStatus;\n+import org.elasticsearch.search.SearchHit;\n+import org.elasticsearch.test.ElasticsearchIntegrationTest;\n+import org.elasticsearch.test.transport.MockTransportService;\n+import org.elasticsearch.transport.*;\n+import org.junit.Test;\n+\n+import java.io.IOException;\n+import java.util.*;\n+import java.util.concurrent.ExecutionException;\n+\n+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;\n+import static org.elasticsearch.index.query.QueryBuilders.termQuery;\n+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*;\n+import static org.hamcrest.CoreMatchers.equalTo;\n+import static org.hamcrest.Matchers.greaterThan;\n+\n+@ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE)\n+public class ExceptionRetryTests extends ElasticsearchIntegrationTest {\n+\n+    volatile boolean exceptionThrown = false;\n+\n+    @Override\n+    protected Settings nodeSettings(int nodeOrdinal) {\n+        return ImmutableSettings.builder()\n+                .put(super.nodeSettings(nodeOrdinal)).put(\"gateway.type\", \"local\")\n+                .put(TransportModule.TRANSPORT_SERVICE_TYPE_KEY, MockTransportService.class.getName())\n+                .build();\n+    }\n+\n+    /**\n+     * Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally failing.\n+     * If auto generated ids are used this must not lead to duplicate ids\n+     * see https://github.com/elasticsearch/elasticsearch/issues/8788\n+     */\n+    @Test\n+    public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {\n+        int numDocs = scaledRandomIntBetween(100, 1000);\n+        NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();\n+        List<NodeStats> dataNodeStats = new ArrayList<>();\n+        for (NodeStats stat : nodeStats.getNodes()) {\n+            dataNodeStats.add(stat);\n+        }\n+\n+        Collections.shuffle(dataNodeStats, getRandom());\n+        NodeStats unluckyNode = dataNodeStats.get(1);\n+\n+        assertAcked(client().admin().indices().prepareCreate(\"index\"));\n+        ensureGreen(\"index\");\n+\n+        //create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.\n+        for (NodeStats dataNode : dataNodeStats) {\n+            MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().name()));\n+            mockTransportService.addDelegate(internalCluster().getInstance(Discovery.class, unluckyNode.getNode().name()).localNode(), new MockTransportService.DelegateTransport(mockTransportService.original()) {\n+\n+                @Override\n+                public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {\n+                    super.sendRequest(node, requestId, action, request, options);\n+                    if (action.equals(\"indices:data/write/bulk[s]\") && !exceptionThrown) {\n+                        logger.info(\"Throw disconnected exception\");\n+                        exceptionThrown = true;\n+                        throw new ConnectTransportException(node, action);\n+                    }\n+\n+                }\n+            });\n+        }\n+\n+        BulkRequestBuilder bulkBuilder = client().prepareBulk();\n+        for (int i = 0; i < numDocs; i++) {\n+            XContentBuilder doc = null;\n+            doc = jsonBuilder().startObject().field(\"foo\", \"bar\").endObject();\n+            bulkBuilder.add(client().prepareIndex(\"index\", \"type\").setSource(doc));\n+        }\n+\n+        BulkResponse response = bulkBuilder.get();\n+        if (response.hasFailures()) {\n+            for (BulkItemResponse singleIndexRespons : response.getItems()) {\n+                if (singleIndexRespons.isFailed()) {\n+                    if (singleIndexRespons.getFailure().getStatus() != RestStatus.CONFLICT) {\n+                        //This measn that we accept to get a conflict when adding a doucument for the first time which seems odd.\n+                        fail();","path":"src/test/java/org/elasticsearch/index/store/ExceptionRetryTests.java","position":113,"original_position":113,"commit_id":"30fac69e1a5ae704845094b6e8f8bf194d2b5e0a","original_commit_id":"30fac69e1a5ae704845094b6e8f8bf194d2b5e0a","user":{"login":"s1monw","id":973334,"avatar_url":"https://avatars.githubusercontent.com/u/973334?v=3","gravatar_id":"","url":"https://api.github.com/users/s1monw","html_url":"https://github.com/s1monw","followers_url":"https://api.github.com/users/s1monw/followers","following_url":"https://api.github.com/users/s1monw/following{/other_user}","gists_url":"https://api.github.com/users/s1monw/gists{/gist_id}","starred_url":"https://api.github.com/users/s1monw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s1monw/subscriptions","organizations_url":"https://api.github.com/users/s1monw/orgs","repos_url":"https://api.github.com/users/s1monw/repos","events_url":"https://api.github.com/users/s1monw/events{/privacy}","received_events_url":"https://api.github.com/users/s1monw/received_events","type":"User","site_admin":false},"body":"can we but some more info into the fail message here?","created_at":"2015-01-02T13:24:35Z","updated_at":"2015-01-02T13:24:35Z","html_url":"https://github.com/elasticsearch/elasticsearch/pull/9125#discussion_r22410103","pull_request_url":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/9125","_links":{"self":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/comments/22410103"},"html":{"href":"https://github.com/elasticsearch/elasticsearch/pull/9125#discussion_r22410103"},"pull_request":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/9125"}}},"pull_request":{"url":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/9125","id":26755547,"html_url":"https://github.com/elasticsearch/elasticsearch/pull/9125","diff_url":"https://github.com/elasticsearch/elasticsearch/pull/9125.diff","patch_url":"https://github.com/elasticsearch/elasticsearch/pull/9125.patch","issue_url":"https://api.github.com/repos/elasticsearch/elasticsearch/issues/9125","number":9125,"state":"open","locked":false,"title":"Prevent creation of duplicates on bulk indexing with auto generated ids and retry","user":{"login":"brwe","id":4320215,"avatar_url":"https://avatars.githubusercontent.com/u/4320215?v=3","gravatar_id":"","url":"https://api.github.com/users/brwe","html_url":"https://github.com/brwe","followers_url":"https://api.github.com/users/brwe/followers","following_url":"https://api.github.com/users/brwe/following{/other_user}","gists_url":"https://api.github.com/users/brwe/gists{/gist_id}","starred_url":"https://api.github.com/users/brwe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brwe/subscriptions","organizations_url":"https://api.github.com/users/brwe/orgs","repos_url":"https://api.github.com/users/brwe/repos","events_url":"https://api.github.com/users/brwe/events{/privacy}","received_events_url":"https://api.github.com/users/brwe/received_events","type":"User","site_admin":false},"body":"If bulk index request fails due to a disconnect, unavailable shard etc, the request is\r\nretried before actually failing. However, even in case of failure the documents\r\nmight already be indexed. For autogenerated ids the request must not add the\r\ndocuments again and therefore canHaveDuplicates must be set to true.\r\n\r\nThere are two options to fix this in this pull request. \r\n1. Just add the flag canHaveDuplicates to the request (30fac69). The user then gets a version conflict in the bulk response (see https://github.com/brwe/elasticsearch/commit/30fac69e1a5ae704845094b6e8f8bf194d2b5e0a#diff-38d197a0e6b168e3ca10ca0553b238adR111) which is confusing since the document cannot have been there before if the id was autogenerated in case the user only uses auto generated ids.\r\n2. We additionally prevent the version conflict by checking in InternalEngine for versions and autogenerated ids (9802466, see https://github.com/brwe/elasticsearch/commit/980246615725d2420c56a31171d4a1e93f47b722#diff-5666f4a211c1b7bbaf76addf0c27451bR504). However, this means that if there is an actual version conflict because a document was added before with this id, then the user will not be notified. \r\n\r\nI am unsure which of the two is the lesser evil. \r\n\r\ncloses #8788","created_at":"2015-01-02T12:03:23Z","updated_at":"2015-01-02T13:24:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5f967ec3d09aa6bbefe9d96688f4b8b1ad34dcf4","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/9125/commits","review_comments_url":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/9125/comments","review_comment_url":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/comments/{number}","comments_url":"https://api.github.com/repos/elasticsearch/elasticsearch/issues/9125/comments","statuses_url":"https://api.github.com/repos/elasticsearch/elasticsearch/statuses/30fac69e1a5ae704845094b6e8f8bf194d2b5e0a","head":{"label":"brwe:dup-ids-light","ref":"dup-ids-light","sha":"30fac69e1a5ae704845094b6e8f8bf194d2b5e0a","user":{"login":"brwe","id":4320215,"avatar_url":"https://avatars.githubusercontent.com/u/4320215?v=3","gravatar_id":"","url":"https://api.github.com/users/brwe","html_url":"https://github.com/brwe","followers_url":"https://api.github.com/users/brwe/followers","following_url":"https://api.github.com/users/brwe/following{/other_user}","gists_url":"https://api.github.com/users/brwe/gists{/gist_id}","starred_url":"https://api.github.com/users/brwe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brwe/subscriptions","organizations_url":"https://api.github.com/users/brwe/orgs","repos_url":"https://api.github.com/users/brwe/repos","events_url":"https://api.github.com/users/brwe/events{/privacy}","received_events_url":"https://api.github.com/users/brwe/received_events","type":"User","site_admin":false},"repo":{"id":9810286,"name":"elasticsearch","full_name":"brwe/elasticsearch","owner":{"login":"brwe","id":4320215,"avatar_url":"https://avatars.githubusercontent.com/u/4320215?v=3","gravatar_id":"","url":"https://api.github.com/users/brwe","html_url":"https://github.com/brwe","followers_url":"https://api.github.com/users/brwe/followers","following_url":"https://api.github.com/users/brwe/following{/other_user}","gists_url":"https://api.github.com/users/brwe/gists{/gist_id}","starred_url":"https://api.github.com/users/brwe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brwe/subscriptions","organizations_url":"https://api.github.com/users/brwe/orgs","repos_url":"https://api.github.com/users/brwe/repos","events_url":"https://api.github.com/users/brwe/events{/privacy}","received_events_url":"https://api.github.com/users/brwe/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/brwe/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":true,"url":"https://api.github.com/repos/brwe/elasticsearch","forks_url":"https://api.github.com/repos/brwe/elasticsearch/forks","keys_url":"https://api.github.com/repos/brwe/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/brwe/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/brwe/elasticsearch/teams","hooks_url":"https://api.github.com/repos/brwe/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/brwe/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/brwe/elasticsearch/events","assignees_url":"https://api.github.com/repos/brwe/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/brwe/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/brwe/elasticsearch/tags","blobs_url":"https://api.github.com/repos/brwe/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/brwe/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/brwe/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/brwe/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/brwe/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/brwe/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/brwe/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/brwe/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/brwe/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/brwe/elasticsearch/subscription","commits_url":"https://api.github.com/repos/brwe/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/brwe/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/brwe/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/brwe/elasticsearch/issues/comments/{number}","contents_url":"https://api.github.com/repos/brwe/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/brwe/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/brwe/elasticsearch/merges","archive_url":"https://api.github.com/repos/brwe/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/brwe/elasticsearch/downloads","issues_url":"https://api.github.com/repos/brwe/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/brwe/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/brwe/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/brwe/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/brwe/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/brwe/elasticsearch/releases{/id}","created_at":"2013-05-02T09:49:12Z","updated_at":"2014-08-14T12:47:29Z","pushed_at":"2015-01-02T11:44:21Z","git_url":"git://github.com/brwe/elasticsearch.git","ssh_url":"git@github.com:brwe/elasticsearch.git","clone_url":"https://github.com/brwe/elasticsearch.git","svn_url":"https://github.com/brwe/elasticsearch","homepage":"http://elasticsearch.org","size":77089,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"elasticsearch:master","ref":"master","sha":"3e37c89932bca3af580b476b0181a7da851f64e6","user":{"login":"elasticsearch","id":141304,"avatar_url":"https://avatars.githubusercontent.com/u/141304?v=3","gravatar_id":"","url":"https://api.github.com/users/elasticsearch","html_url":"https://github.com/elasticsearch","followers_url":"https://api.github.com/users/elasticsearch/followers","following_url":"https://api.github.com/users/elasticsearch/following{/other_user}","gists_url":"https://api.github.com/users/elasticsearch/gists{/gist_id}","starred_url":"https://api.github.com/users/elasticsearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elasticsearch/subscriptions","organizations_url":"https://api.github.com/users/elasticsearch/orgs","repos_url":"https://api.github.com/users/elasticsearch/repos","events_url":"https://api.github.com/users/elasticsearch/events{/privacy}","received_events_url":"https://api.github.com/users/elasticsearch/received_events","type":"Organization","site_admin":false},"repo":{"id":507775,"name":"elasticsearch","full_name":"elasticsearch/elasticsearch","owner":{"login":"elasticsearch","id":141304,"avatar_url":"https://avatars.githubusercontent.com/u/141304?v=3","gravatar_id":"","url":"https://api.github.com/users/elasticsearch","html_url":"https://github.com/elasticsearch","followers_url":"https://api.github.com/users/elasticsearch/followers","following_url":"https://api.github.com/users/elasticsearch/following{/other_user}","gists_url":"https://api.github.com/users/elasticsearch/gists{/gist_id}","starred_url":"https://api.github.com/users/elasticsearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elasticsearch/subscriptions","organizations_url":"https://api.github.com/users/elasticsearch/orgs","repos_url":"https://api.github.com/users/elasticsearch/repos","events_url":"https://api.github.com/users/elasticsearch/events{/privacy}","received_events_url":"https://api.github.com/users/elasticsearch/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/elasticsearch/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":false,"url":"https://api.github.com/repos/elasticsearch/elasticsearch","forks_url":"https://api.github.com/repos/elasticsearch/elasticsearch/forks","keys_url":"https://api.github.com/repos/elasticsearch/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/elasticsearch/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/elasticsearch/elasticsearch/teams","hooks_url":"https://api.github.com/repos/elasticsearch/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/elasticsearch/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/elasticsearch/elasticsearch/events","assignees_url":"https://api.github.com/repos/elasticsearch/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/elasticsearch/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/elasticsearch/elasticsearch/tags","blobs_url":"https://api.github.com/repos/elasticsearch/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/elasticsearch/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/elasticsearch/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/elasticsearch/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/elasticsearch/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/elasticsearch/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/elasticsearch/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/elasticsearch/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/elasticsearch/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/elasticsearch/elasticsearch/subscription","commits_url":"https://api.github.com/repos/elasticsearch/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/elasticsearch/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/elasticsearch/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/elasticsearch/elasticsearch/issues/comments/{number}","contents_url":"https://api.github.com/repos/elasticsearch/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/elasticsearch/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/elasticsearch/elasticsearch/merges","archive_url":"https://api.github.com/repos/elasticsearch/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/elasticsearch/elasticsearch/downloads","issues_url":"https://api.github.com/repos/elasticsearch/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/elasticsearch/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/elasticsearch/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/elasticsearch/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/elasticsearch/elasticsearch/releases{/id}","created_at":"2010-02-08T13:20:56Z","updated_at":"2015-01-02T12:24:52Z","pushed_at":"2015-01-02T12:15:18Z","git_url":"git://github.com/elasticsearch/elasticsearch.git","ssh_url":"git@github.com:elasticsearch/elasticsearch.git","clone_url":"https://github.com/elasticsearch/elasticsearch.git","svn_url":"https://github.com/elasticsearch/elasticsearch","homepage":"http://elasticsearch.org","size":256624,"stargazers_count":9415,"watchers_count":9415,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2886,"mirror_url":null,"open_issues_count":841,"forks":2886,"open_issues":841,"watchers":9415,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/9125"},"html":{"href":"https://github.com/elasticsearch/elasticsearch/pull/9125"},"issue":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/issues/9125"},"comments":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/issues/9125/comments"},"review_comments":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/9125/comments"},"review_comment":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/pulls/9125/commits"},"statuses":{"href":"https://api.github.com/repos/elasticsearch/elasticsearch/statuses/30fac69e1a5ae704845094b6e8f8bf194d2b5e0a"}}}}