{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/29190170","id":29190170,"diff_hunk":"@@ -0,0 +1,260 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.network.sasl;\n+\n+import java.io.IOException;\n+import java.nio.ByteBuffer;\n+import java.nio.channels.WritableByteChannel;\n+import java.util.List;\n+\n+import com.google.common.annotations.VisibleForTesting;\n+import com.google.common.base.Preconditions;\n+import io.netty.buffer.ByteBuf;\n+import io.netty.buffer.Unpooled;\n+import io.netty.channel.Channel;\n+import io.netty.channel.ChannelHandlerContext;\n+import io.netty.channel.ChannelOutboundHandlerAdapter;\n+import io.netty.channel.ChannelPromise;\n+import io.netty.channel.FileRegion;\n+import io.netty.handler.codec.MessageToMessageDecoder;\n+import io.netty.util.AbstractReferenceCounted;\n+import io.netty.util.ReferenceCountUtil;\n+\n+import org.apache.spark.network.util.ByteArrayWritableChannel;\n+import org.apache.spark.network.util.NettyUtils;\n+\n+class SaslEncryption {\n+\n+  @VisibleForTesting\n+  static final String ENCRYPTION_HANDLER_NAME = \"saslEncryption\";\n+\n+  /**\n+   * Adds channel handlers that perform encryption / decryption of data using SASL.\n+   *\n+   * @param channel The channel.\n+   * @param backend The SASL backend.\n+   * @param maxOutboundBlockSize Max size in bytes of outgoing encrypted blocks, to control\n+   *                             memory usage.\n+   */\n+  static void addToChannel(\n+      Channel channel,\n+      SaslEncryptionBackend backend,\n+      int maxOutboundBlockSize) {\n+    channel.pipeline()\n+      .addFirst(ENCRYPTION_HANDLER_NAME, new EncryptionHandler(backend, maxOutboundBlockSize))\n+      .addFirst(\"saslDecryption\", new DecryptionHandler(backend))\n+      .addFirst(\"saslFrameDecoder\", NettyUtils.createFrameDecoder());\n+  }\n+\n+  private static class EncryptionHandler extends ChannelOutboundHandlerAdapter {\n+\n+    private final int maxOutboundBlockSize;\n+    private final SaslEncryptionBackend backend;\n+\n+    EncryptionHandler(SaslEncryptionBackend backend, int maxOutboundBlockSize) {\n+      this.backend = backend;\n+      this.maxOutboundBlockSize = maxOutboundBlockSize;\n+    }\n+\n+    /**\n+     * Wrap the incoming message in an implementation that will perform encryption lazily. This is\n+     * needed to guarantee ordering of the outgoing encrypted packets - they need to be decrypted in\n+     * the same order, and netty doesn't have an atomic ChannelHandlerContext.write() API, so it\n+     * does not guarantee any ordering.\n+     */\n+    @Override\n+    public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise)\n+      throws Exception {\n+\n+      ctx.write(new EncryptedMessage(backend, msg, maxOutboundBlockSize), promise);\n+    }\n+\n+    @Override\n+    public void handlerRemoved(ChannelHandlerContext ctx) throws Exception {\n+      try {\n+        backend.dispose();\n+      } finally {\n+        super.handlerRemoved(ctx);\n+      }\n+    }\n+\n+  }\n+\n+  private static class DecryptionHandler extends MessageToMessageDecoder<ByteBuf> {\n+\n+    private final SaslEncryptionBackend backend;\n+\n+    DecryptionHandler(SaslEncryptionBackend backend) {\n+      this.backend = backend;\n+    }\n+\n+    @Override\n+    protected void decode(ChannelHandlerContext ctx, ByteBuf msg, List<Object> out)\n+      throws Exception {\n+\n+      byte[] data;\n+      int offset;\n+      int length = msg.readableBytes();\n+      if (msg.hasArray()) {\n+        data = msg.array();\n+        offset = msg.arrayOffset();\n+      } else {\n+        data = new byte[length];\n+        msg.readBytes(data);\n+        offset = 0;\n+      }\n+\n+      out.add(Unpooled.wrappedBuffer(backend.unwrap(data, offset, length)));\n+    }\n+\n+  }\n+\n+  private static class EncryptedMessage extends AbstractReferenceCounted implements FileRegion {\n+\n+    private final SaslEncryptionBackend backend;\n+    private final boolean isByteBuf;\n+    private final ByteBuf buf;\n+    private final FileRegion region;\n+    private final ByteArrayWritableChannel byteChannel;\n+\n+    private ByteBuf currentHeader;\n+    private ByteBuffer currentChunk;\n+    private long currentChunkSize;\n+    private long unencryptedChunkSize;\n+    private long transferred;\n+\n+    EncryptedMessage(SaslEncryptionBackend backend, Object msg, int maxOutboundBlockSize) {\n+      Preconditions.checkArgument(msg instanceof ByteBuf || msg instanceof FileRegion,\n+        \"Unrecognized message type: %s\", msg.getClass().getName());\n+      this.backend = backend;\n+      this.isByteBuf = msg instanceof ByteBuf;\n+      this.buf = isByteBuf ? (ByteBuf) msg : null;\n+      this.region = isByteBuf ? null : (FileRegion) msg;\n+      this.byteChannel = new ByteArrayWritableChannel(maxOutboundBlockSize);\n+    }\n+\n+    /**\n+     * Returns the size of the original (unencrypted) message.\n+     *\n+     * This makes assumptions about how netty treats FileRegion instances, because there's no way\n+     * to know beforehand what will be the size of the encrypted message. Namely, it assumes\n+     * that netty will try to transfer data from this message while\n+     * <code>transfered() < count()</code>. So these two methods return, technically, wrong data,\n+     * but netty doesn't know better.\n+     */\n+    @Override\n+    public long count() {\n+      return isByteBuf ? buf.readableBytes() : region.count();\n+    }\n+\n+    @Override\n+    public long position() {\n+      return 0;\n+    }\n+\n+    /**\n+     * Returns an approximation of the amount of data transferred. See {@link #count()}.\n+     */\n+    @Override\n+    public long transfered() {\n+      return transferred;\n+    }\n+\n+    /**\n+     * Transfers data from the original message to the channel, encrypting it in the process.\n+     *\n+     * This method also breaks down the original message into smaller chunks when needed. This\n+     * is done to keep memory usage under control. This avoids having to copy the whole message\n+     * data into memory at once, and can avoid ballooning memory usage when transferring large\n+     * messages such as shuffle blocks.\n+     *\n+     * The {@link #transfered()} counter also behaves a little funny, in that it won't go forward\n+     * until a whole chunk has been written. This is done because the code can't use the actual\n+     * number of bytes written to the channel as the transferred count (see {@link #count()}).\n+     * Instead, once an encrypted chunk is written to the output (including its header), the\n+     * size of the original block will be added to the {@link #transfered()} amount.\n+     */\n+    @Override\n+    public long transferTo(final WritableByteChannel target, final long position)\n+      throws IOException {\n+\n+      Preconditions.checkArgument(position == transfered(), \"Invalid position.\");\n+\n+      long written = 0;\n+      do {\n+        if (currentChunk == null) {\n+          nextChunk();\n+        }\n+\n+        if (currentHeader.readableBytes() > 0) {\n+          int bytesWritten = target.write(currentHeader.nioBuffer());\n+          currentHeader.skipBytes(bytesWritten);\n+          if (currentHeader.readableBytes() > 0) {\n+            // Break out of loop if there are still header bytes left to write.\n+            break;\n+          }\n+        }\n+\n+        target.write(currentChunk);\n+        if (!currentChunk.hasRemaining()) {\n+          // Only update the count of written bytes once a full chunk has been written.","path":"network/common/src/main/java/org/apache/spark/network/sasl/SaslEncryption.java","position":215,"original_position":215,"commit_id":"4797519b16c237c2573b235b49b656650eb9d0ae","original_commit_id":"4797519b16c237c2573b235b49b656650eb9d0ae","user":{"login":"vanzin","id":1694083,"avatar_url":"https://avatars.githubusercontent.com/u/1694083?v=3","gravatar_id":"","url":"https://api.github.com/users/vanzin","html_url":"https://github.com/vanzin","followers_url":"https://api.github.com/users/vanzin/followers","following_url":"https://api.github.com/users/vanzin/following{/other_user}","gists_url":"https://api.github.com/users/vanzin/gists{/gist_id}","starred_url":"https://api.github.com/users/vanzin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vanzin/subscriptions","organizations_url":"https://api.github.com/users/vanzin/orgs","repos_url":"https://api.github.com/users/vanzin/repos","events_url":"https://api.github.com/users/vanzin/events{/privacy}","received_events_url":"https://api.github.com/users/vanzin/received_events","type":"User","site_admin":false},"body":"I see. I could return `1` instead, and keep the counts synchronized, but some pathological case might still require returning `0`. That should be rare, though, so let me do this to avoid the performance hit in the common case.","created_at":"2015-04-27T20:56:30Z","updated_at":"2015-04-27T20:56:30Z","html_url":"https://github.com/apache/spark/pull/5377#discussion_r29190170","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/5377","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/29190170"},"html":{"href":"https://github.com/apache/spark/pull/5377#discussion_r29190170"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/5377"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/5377","id":32719999,"html_url":"https://github.com/apache/spark/pull/5377","diff_url":"https://github.com/apache/spark/pull/5377.diff","patch_url":"https://github.com/apache/spark/pull/5377.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/5377","number":5377,"state":"open","locked":false,"title":"[SPARK-6229] Add SASL encryption to network library.","user":{"login":"vanzin","id":1694083,"avatar_url":"https://avatars.githubusercontent.com/u/1694083?v=3","gravatar_id":"","url":"https://api.github.com/users/vanzin","html_url":"https://github.com/vanzin","followers_url":"https://api.github.com/users/vanzin/followers","following_url":"https://api.github.com/users/vanzin/following{/other_user}","gists_url":"https://api.github.com/users/vanzin/gists{/gist_id}","starred_url":"https://api.github.com/users/vanzin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vanzin/subscriptions","organizations_url":"https://api.github.com/users/vanzin/orgs","repos_url":"https://api.github.com/users/vanzin/repos","events_url":"https://api.github.com/users/vanzin/events{/privacy}","received_events_url":"https://api.github.com/users/vanzin/received_events","type":"User","site_admin":false},"body":"There are two main parts of this change:\r\n\r\n- Extending the bootstrap mechanism in the network library to add a server-side\r\n  bootstrap (which works a little bit differently than the client-side bootstrap), and\r\n  to allow the  bootstraps to modify the underlying channel.\r\n\r\n- Use SASL to encrypt data going through the RPC channel.\r\n\r\nThe second item requires some non-optimal code to be able to work around the\r\nfact that the outbound path in netty is not thread-safe, and ordering is very important\r\nwhen encryption is in the picture.\r\n\r\nA lot of the changes outside the network/common library are just to adjust to the\r\nchanged API for initializing the RPC server.","created_at":"2015-04-06T23:05:21Z","updated_at":"2015-04-27T20:56:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5e216111eceeb7235e62f0506ca6dbb67ed094d2","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/5377/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/5377/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/5377/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/4797519b16c237c2573b235b49b656650eb9d0ae","head":{"label":"vanzin:SPARK-6229","ref":"SPARK-6229","sha":"4797519b16c237c2573b235b49b656650eb9d0ae","user":{"login":"vanzin","id":1694083,"avatar_url":"https://avatars.githubusercontent.com/u/1694083?v=3","gravatar_id":"","url":"https://api.github.com/users/vanzin","html_url":"https://github.com/vanzin","followers_url":"https://api.github.com/users/vanzin/followers","following_url":"https://api.github.com/users/vanzin/following{/other_user}","gists_url":"https://api.github.com/users/vanzin/gists{/gist_id}","starred_url":"https://api.github.com/users/vanzin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vanzin/subscriptions","organizations_url":"https://api.github.com/users/vanzin/orgs","repos_url":"https://api.github.com/users/vanzin/repos","events_url":"https://api.github.com/users/vanzin/events{/privacy}","received_events_url":"https://api.github.com/users/vanzin/received_events","type":"User","site_admin":false},"repo":{"id":18302272,"name":"spark","full_name":"vanzin/spark","owner":{"login":"vanzin","id":1694083,"avatar_url":"https://avatars.githubusercontent.com/u/1694083?v=3","gravatar_id":"","url":"https://api.github.com/users/vanzin","html_url":"https://github.com/vanzin","followers_url":"https://api.github.com/users/vanzin/followers","following_url":"https://api.github.com/users/vanzin/following{/other_user}","gists_url":"https://api.github.com/users/vanzin/gists{/gist_id}","starred_url":"https://api.github.com/users/vanzin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vanzin/subscriptions","organizations_url":"https://api.github.com/users/vanzin/orgs","repos_url":"https://api.github.com/users/vanzin/repos","events_url":"https://api.github.com/users/vanzin/events{/privacy}","received_events_url":"https://api.github.com/users/vanzin/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/vanzin/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/vanzin/spark","forks_url":"https://api.github.com/repos/vanzin/spark/forks","keys_url":"https://api.github.com/repos/vanzin/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/vanzin/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/vanzin/spark/teams","hooks_url":"https://api.github.com/repos/vanzin/spark/hooks","issue_events_url":"https://api.github.com/repos/vanzin/spark/issues/events{/number}","events_url":"https://api.github.com/repos/vanzin/spark/events","assignees_url":"https://api.github.com/repos/vanzin/spark/assignees{/user}","branches_url":"https://api.github.com/repos/vanzin/spark/branches{/branch}","tags_url":"https://api.github.com/repos/vanzin/spark/tags","blobs_url":"https://api.github.com/repos/vanzin/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/vanzin/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/vanzin/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/vanzin/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/vanzin/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/vanzin/spark/languages","stargazers_url":"https://api.github.com/repos/vanzin/spark/stargazers","contributors_url":"https://api.github.com/repos/vanzin/spark/contributors","subscribers_url":"https://api.github.com/repos/vanzin/spark/subscribers","subscription_url":"https://api.github.com/repos/vanzin/spark/subscription","commits_url":"https://api.github.com/repos/vanzin/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/vanzin/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/vanzin/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/vanzin/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/vanzin/spark/contents/{+path}","compare_url":"https://api.github.com/repos/vanzin/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/vanzin/spark/merges","archive_url":"https://api.github.com/repos/vanzin/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/vanzin/spark/downloads","issues_url":"https://api.github.com/repos/vanzin/spark/issues{/number}","pulls_url":"https://api.github.com/repos/vanzin/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/vanzin/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/vanzin/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/vanzin/spark/labels{/name}","releases_url":"https://api.github.com/repos/vanzin/spark/releases{/id}","created_at":"2014-03-31T17:49:25Z","updated_at":"2014-06-25T00:58:09Z","pushed_at":"2015-04-24T23:49:21Z","git_url":"git://github.com/vanzin/spark.git","ssh_url":"git@github.com:vanzin/spark.git","clone_url":"https://github.com/vanzin/spark.git","svn_url":"https://github.com/vanzin/spark","homepage":null,"size":98260,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"30363ede8635f2548e444697dbcf60a795b61a84","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-04-27T20:46:38Z","pushed_at":"2015-04-27T20:40:06Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1546961,"stargazers_count":3766,"watchers_count":3766,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3282,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":313,"forks":3282,"open_issues":313,"watchers":3766,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/5377"},"html":{"href":"https://github.com/apache/spark/pull/5377"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/5377"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/5377/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/5377/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/5377/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/4797519b16c237c2573b235b49b656650eb9d0ae"}}}}