{"action":"created","comment":{"url":"https://api.github.com/repos/addthis/hydra/pulls/comments/27070170","id":27070170,"diff_hunk":"@@ -0,0 +1,160 @@\n+/*\n+ * Licensed under the Apache License, Version 2.0 (the \"License\");\n+ * you may not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package com.addthis.hydra.task.map;\n+\n+\n+import java.util.NoSuchElementException;\n+import java.util.concurrent.CompletableFuture;\n+import java.util.concurrent.ForkJoinPool;\n+import java.util.concurrent.RecursiveAction;\n+import java.util.concurrent.Semaphore;\n+import java.util.concurrent.atomic.AtomicBoolean;\n+\n+import com.addthis.basis.util.Parameter;\n+\n+import com.addthis.bundle.core.Bundle;\n+import com.addthis.hydra.task.source.TaskDataSource;\n+\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+public final class MapFeederForkJoin implements Runnable {\n+    private static final Logger log = LoggerFactory.getLogger(MapFeederForkJoin.class);\n+\n+    private static final int QUEUE_DEPTH = Parameter.intValue(\"task.queue.depth\", 100);\n+\n+    // state control\n+    private final AtomicBoolean errored = new AtomicBoolean(false);\n+    private boolean hasClosedStreams = false; // not shared with MapperTasks\n+\n+    // enclosing task\n+    private final StreamMapper task;\n+    private final TaskDataSource source;\n+\n+    // mapper task controls\n+    private final ForkJoinPool mapperPool;\n+    private final int parallelism;\n+    private final Semaphore enqueuePermits;\n+\n+    private long start = System.currentTimeMillis();\n+    private long totalBundles = 0;\n+\n+    public MapFeederForkJoin(StreamMapper task, TaskDataSource source, int parallelism) {\n+        this.source = source;\n+        this.task = task;\n+        this.parallelism = parallelism;\n+        this.mapperPool = new ForkJoinPool(parallelism);\n+        this.enqueuePermits = new Semaphore(QUEUE_DEPTH);\n+    }\n+\n+    @Override public void run() {\n+        log.info(\"starting {} thread(s) for src={}\", parallelism, source);\n+        try {\n+            if (source.isEnabled()) {\n+                while (fillBuffer()) {\n+                    if (Thread.interrupted()) {\n+                        closeSourceIfNeeded();\n+                    }\n+                }\n+            }\n+            closeSourceIfNeeded();\n+            joinProcessors();\n+            log.info(\"all ({}) task threads exited; sending taskComplete\", parallelism);\n+            logBundleThroughput();\n+            // run in different threads to isolate them from interrupts. ie. \"taskCompleteUninterruptibly\"\n+            // join awaits completion, is uninterruptible, and will propagate any exception\n+            CompletableFuture.runAsync(task::taskComplete).join();\n+        } catch (Throwable t) {\n+            logBundleThroughput();\n+            handleUncaughtThrowable(t);\n+        }\n+        log.debug(\"task feeder exited\");\n+    }\n+\n+    private void logBundleThroughput() {\n+        long elapse = (System.currentTimeMillis() - start) / 1000;\n+        log.info(\"{} bundles processed in {} seconds (avg rate={}/s)\", totalBundles, elapse, totalBundles / elapse);\n+    }\n+\n+    /**\n+     * Immediately halting is the best way we have of ensuring errors are reported without a significantly\n+     * more involved system.\n+     */\n+    private void handleUncaughtThrowable(Throwable t) {\n+        if (errored.compareAndSet(false, true)) {\n+            log.error(\"unrecoverable error in task feeder or one of its mapper threads. immediately halting jvm\", t);\n+            Runtime.getRuntime().halt(1);","path":"hydra-task/src/main/java/com/addthis/hydra/task/map/MapFeederForkJoin.java","position":97,"original_position":97,"commit_id":"46aa65edd5c7e269a3e8488eba22a43614578ec8","original_commit_id":"46aa65edd5c7e269a3e8488eba22a43614578ec8","user":{"login":"yuesong","id":104592,"avatar_url":"https://avatars.githubusercontent.com/u/104592?v=3","gravatar_id":"","url":"https://api.github.com/users/yuesong","html_url":"https://github.com/yuesong","followers_url":"https://api.github.com/users/yuesong/followers","following_url":"https://api.github.com/users/yuesong/following{/other_user}","gists_url":"https://api.github.com/users/yuesong/gists{/gist_id}","starred_url":"https://api.github.com/users/yuesong/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yuesong/subscriptions","organizations_url":"https://api.github.com/users/yuesong/orgs","repos_url":"https://api.github.com/users/yuesong/repos","events_url":"https://api.github.com/users/yuesong/events{/privacy}","received_events_url":"https://api.github.com/users/yuesong/received_events","type":"User","site_admin":false},"body":"what method?","created_at":"2015-03-24T20:36:49Z","updated_at":"2015-03-24T20:36:49Z","html_url":"https://github.com/addthis/hydra/pull/167#discussion_r27070170","pull_request_url":"https://api.github.com/repos/addthis/hydra/pulls/167","_links":{"self":{"href":"https://api.github.com/repos/addthis/hydra/pulls/comments/27070170"},"html":{"href":"https://github.com/addthis/hydra/pull/167#discussion_r27070170"},"pull_request":{"href":"https://api.github.com/repos/addthis/hydra/pulls/167"}}},"pull_request":{"url":"https://api.github.com/repos/addthis/hydra/pulls/167","id":31827067,"html_url":"https://github.com/addthis/hydra/pull/167","diff_url":"https://github.com/addthis/hydra/pull/167.diff","patch_url":"https://github.com/addthis/hydra/pull/167.patch","issue_url":"https://api.github.com/repos/addthis/hydra/issues/167","number":167,"state":"open","locked":false,"title":"Added MapFeederForkJoin and a StreamMapper flag to use it. Ref T53505","user":{"login":"yuesong","id":104592,"avatar_url":"https://avatars.githubusercontent.com/u/104592?v=3","gravatar_id":"","url":"https://api.github.com/users/yuesong","html_url":"https://github.com/yuesong","followers_url":"https://api.github.com/users/yuesong/followers","following_url":"https://api.github.com/users/yuesong/following{/other_user}","gists_url":"https://api.github.com/users/yuesong/gists{/gist_id}","starred_url":"https://api.github.com/users/yuesong/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yuesong/subscriptions","organizations_url":"https://api.github.com/users/yuesong/orgs","repos_url":"https://api.github.com/users/yuesong/repos","events_url":"https://api.github.com/users/yuesong/events{/privacy}","received_events_url":"https://api.github.com/users/yuesong/received_events","type":"User","site_admin":false},"body":"In my local synthetic tests that simulates a few corner cases, the new MapFeederForkJoin achieved better bundle processing throughput than the current impl. In a test where I copied real source data and job config, it did the same.","created_at":"2015-03-24T13:45:08Z","updated_at":"2015-03-24T20:36:49Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3d41303fa1f1c33489c271f29860e75b838775dd","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/addthis/hydra/pulls/167/commits","review_comments_url":"https://api.github.com/repos/addthis/hydra/pulls/167/comments","review_comment_url":"https://api.github.com/repos/addthis/hydra/pulls/comments{/number}","comments_url":"https://api.github.com/repos/addthis/hydra/issues/167/comments","statuses_url":"https://api.github.com/repos/addthis/hydra/statuses/46aa65edd5c7e269a3e8488eba22a43614578ec8","head":{"label":"addthis:forkjoin-mapfeeder","ref":"forkjoin-mapfeeder","sha":"46aa65edd5c7e269a3e8488eba22a43614578ec8","user":{"login":"addthis","id":2325903,"avatar_url":"https://avatars.githubusercontent.com/u/2325903?v=3","gravatar_id":"","url":"https://api.github.com/users/addthis","html_url":"https://github.com/addthis","followers_url":"https://api.github.com/users/addthis/followers","following_url":"https://api.github.com/users/addthis/following{/other_user}","gists_url":"https://api.github.com/users/addthis/gists{/gist_id}","starred_url":"https://api.github.com/users/addthis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/addthis/subscriptions","organizations_url":"https://api.github.com/users/addthis/orgs","repos_url":"https://api.github.com/users/addthis/repos","events_url":"https://api.github.com/users/addthis/events{/privacy}","received_events_url":"https://api.github.com/users/addthis/received_events","type":"Organization","site_admin":false},"repo":{"id":15475337,"name":"hydra","full_name":"addthis/hydra","owner":{"login":"addthis","id":2325903,"avatar_url":"https://avatars.githubusercontent.com/u/2325903?v=3","gravatar_id":"","url":"https://api.github.com/users/addthis","html_url":"https://github.com/addthis","followers_url":"https://api.github.com/users/addthis/followers","following_url":"https://api.github.com/users/addthis/following{/other_user}","gists_url":"https://api.github.com/users/addthis/gists{/gist_id}","starred_url":"https://api.github.com/users/addthis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/addthis/subscriptions","organizations_url":"https://api.github.com/users/addthis/orgs","repos_url":"https://api.github.com/users/addthis/repos","events_url":"https://api.github.com/users/addthis/events{/privacy}","received_events_url":"https://api.github.com/users/addthis/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/addthis/hydra","description":"","fork":false,"url":"https://api.github.com/repos/addthis/hydra","forks_url":"https://api.github.com/repos/addthis/hydra/forks","keys_url":"https://api.github.com/repos/addthis/hydra/keys{/key_id}","collaborators_url":"https://api.github.com/repos/addthis/hydra/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/addthis/hydra/teams","hooks_url":"https://api.github.com/repos/addthis/hydra/hooks","issue_events_url":"https://api.github.com/repos/addthis/hydra/issues/events{/number}","events_url":"https://api.github.com/repos/addthis/hydra/events","assignees_url":"https://api.github.com/repos/addthis/hydra/assignees{/user}","branches_url":"https://api.github.com/repos/addthis/hydra/branches{/branch}","tags_url":"https://api.github.com/repos/addthis/hydra/tags","blobs_url":"https://api.github.com/repos/addthis/hydra/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/addthis/hydra/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/addthis/hydra/git/refs{/sha}","trees_url":"https://api.github.com/repos/addthis/hydra/git/trees{/sha}","statuses_url":"https://api.github.com/repos/addthis/hydra/statuses/{sha}","languages_url":"https://api.github.com/repos/addthis/hydra/languages","stargazers_url":"https://api.github.com/repos/addthis/hydra/stargazers","contributors_url":"https://api.github.com/repos/addthis/hydra/contributors","subscribers_url":"https://api.github.com/repos/addthis/hydra/subscribers","subscription_url":"https://api.github.com/repos/addthis/hydra/subscription","commits_url":"https://api.github.com/repos/addthis/hydra/commits{/sha}","git_commits_url":"https://api.github.com/repos/addthis/hydra/git/commits{/sha}","comments_url":"https://api.github.com/repos/addthis/hydra/comments{/number}","issue_comment_url":"https://api.github.com/repos/addthis/hydra/issues/comments{/number}","contents_url":"https://api.github.com/repos/addthis/hydra/contents/{+path}","compare_url":"https://api.github.com/repos/addthis/hydra/compare/{base}...{head}","merges_url":"https://api.github.com/repos/addthis/hydra/merges","archive_url":"https://api.github.com/repos/addthis/hydra/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/addthis/hydra/downloads","issues_url":"https://api.github.com/repos/addthis/hydra/issues{/number}","pulls_url":"https://api.github.com/repos/addthis/hydra/pulls{/number}","milestones_url":"https://api.github.com/repos/addthis/hydra/milestones{/number}","notifications_url":"https://api.github.com/repos/addthis/hydra/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/addthis/hydra/labels{/name}","releases_url":"https://api.github.com/repos/addthis/hydra/releases{/id}","created_at":"2013-12-27T14:31:31Z","updated_at":"2015-03-24T15:47:53Z","pushed_at":"2015-03-24T15:47:53Z","git_url":"git://github.com/addthis/hydra.git","ssh_url":"git@github.com:addthis/hydra.git","clone_url":"https://github.com/addthis/hydra.git","svn_url":"https://github.com/addthis/hydra","homepage":null,"size":29069,"stargazers_count":271,"watchers_count":271,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":45,"mirror_url":null,"open_issues_count":2,"forks":45,"open_issues":2,"watchers":271,"default_branch":"master"}},"base":{"label":"addthis:master","ref":"master","sha":"1b0ef8776fbb6f6a30d6eca7b0d41a463d49056a","user":{"login":"addthis","id":2325903,"avatar_url":"https://avatars.githubusercontent.com/u/2325903?v=3","gravatar_id":"","url":"https://api.github.com/users/addthis","html_url":"https://github.com/addthis","followers_url":"https://api.github.com/users/addthis/followers","following_url":"https://api.github.com/users/addthis/following{/other_user}","gists_url":"https://api.github.com/users/addthis/gists{/gist_id}","starred_url":"https://api.github.com/users/addthis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/addthis/subscriptions","organizations_url":"https://api.github.com/users/addthis/orgs","repos_url":"https://api.github.com/users/addthis/repos","events_url":"https://api.github.com/users/addthis/events{/privacy}","received_events_url":"https://api.github.com/users/addthis/received_events","type":"Organization","site_admin":false},"repo":{"id":15475337,"name":"hydra","full_name":"addthis/hydra","owner":{"login":"addthis","id":2325903,"avatar_url":"https://avatars.githubusercontent.com/u/2325903?v=3","gravatar_id":"","url":"https://api.github.com/users/addthis","html_url":"https://github.com/addthis","followers_url":"https://api.github.com/users/addthis/followers","following_url":"https://api.github.com/users/addthis/following{/other_user}","gists_url":"https://api.github.com/users/addthis/gists{/gist_id}","starred_url":"https://api.github.com/users/addthis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/addthis/subscriptions","organizations_url":"https://api.github.com/users/addthis/orgs","repos_url":"https://api.github.com/users/addthis/repos","events_url":"https://api.github.com/users/addthis/events{/privacy}","received_events_url":"https://api.github.com/users/addthis/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/addthis/hydra","description":"","fork":false,"url":"https://api.github.com/repos/addthis/hydra","forks_url":"https://api.github.com/repos/addthis/hydra/forks","keys_url":"https://api.github.com/repos/addthis/hydra/keys{/key_id}","collaborators_url":"https://api.github.com/repos/addthis/hydra/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/addthis/hydra/teams","hooks_url":"https://api.github.com/repos/addthis/hydra/hooks","issue_events_url":"https://api.github.com/repos/addthis/hydra/issues/events{/number}","events_url":"https://api.github.com/repos/addthis/hydra/events","assignees_url":"https://api.github.com/repos/addthis/hydra/assignees{/user}","branches_url":"https://api.github.com/repos/addthis/hydra/branches{/branch}","tags_url":"https://api.github.com/repos/addthis/hydra/tags","blobs_url":"https://api.github.com/repos/addthis/hydra/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/addthis/hydra/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/addthis/hydra/git/refs{/sha}","trees_url":"https://api.github.com/repos/addthis/hydra/git/trees{/sha}","statuses_url":"https://api.github.com/repos/addthis/hydra/statuses/{sha}","languages_url":"https://api.github.com/repos/addthis/hydra/languages","stargazers_url":"https://api.github.com/repos/addthis/hydra/stargazers","contributors_url":"https://api.github.com/repos/addthis/hydra/contributors","subscribers_url":"https://api.github.com/repos/addthis/hydra/subscribers","subscription_url":"https://api.github.com/repos/addthis/hydra/subscription","commits_url":"https://api.github.com/repos/addthis/hydra/commits{/sha}","git_commits_url":"https://api.github.com/repos/addthis/hydra/git/commits{/sha}","comments_url":"https://api.github.com/repos/addthis/hydra/comments{/number}","issue_comment_url":"https://api.github.com/repos/addthis/hydra/issues/comments{/number}","contents_url":"https://api.github.com/repos/addthis/hydra/contents/{+path}","compare_url":"https://api.github.com/repos/addthis/hydra/compare/{base}...{head}","merges_url":"https://api.github.com/repos/addthis/hydra/merges","archive_url":"https://api.github.com/repos/addthis/hydra/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/addthis/hydra/downloads","issues_url":"https://api.github.com/repos/addthis/hydra/issues{/number}","pulls_url":"https://api.github.com/repos/addthis/hydra/pulls{/number}","milestones_url":"https://api.github.com/repos/addthis/hydra/milestones{/number}","notifications_url":"https://api.github.com/repos/addthis/hydra/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/addthis/hydra/labels{/name}","releases_url":"https://api.github.com/repos/addthis/hydra/releases{/id}","created_at":"2013-12-27T14:31:31Z","updated_at":"2015-03-24T15:47:53Z","pushed_at":"2015-03-24T15:47:53Z","git_url":"git://github.com/addthis/hydra.git","ssh_url":"git@github.com:addthis/hydra.git","clone_url":"https://github.com/addthis/hydra.git","svn_url":"https://github.com/addthis/hydra","homepage":null,"size":29069,"stargazers_count":271,"watchers_count":271,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":45,"mirror_url":null,"open_issues_count":2,"forks":45,"open_issues":2,"watchers":271,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/addthis/hydra/pulls/167"},"html":{"href":"https://github.com/addthis/hydra/pull/167"},"issue":{"href":"https://api.github.com/repos/addthis/hydra/issues/167"},"comments":{"href":"https://api.github.com/repos/addthis/hydra/issues/167/comments"},"review_comments":{"href":"https://api.github.com/repos/addthis/hydra/pulls/167/comments"},"review_comment":{"href":"https://api.github.com/repos/addthis/hydra/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/addthis/hydra/pulls/167/commits"},"statuses":{"href":"https://api.github.com/repos/addthis/hydra/statuses/46aa65edd5c7e269a3e8488eba22a43614578ec8"}}}}