{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/46746127","id":46746127,"diff_hunk":"@@ -0,0 +1,168 @@\n+/*\n+ * Copyright (C) 2014-2015 LinkedIn Corp. All rights reserved.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use\n+ * this file except in compliance with the License. You may obtain a copy of the\n+ * License at  http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software distributed\n+ * under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n+ * CONDITIONS OF ANY KIND, either express or implied.\n+ */\n+\n+package gobblin.yarn;\n+\n+import java.io.IOException;\n+import java.net.URL;\n+import java.util.concurrent.TimeoutException;\n+\n+import org.apache.curator.framework.CuratorFramework;\n+import org.apache.curator.framework.CuratorFrameworkFactory;\n+import org.apache.curator.retry.RetryOneTime;\n+import org.apache.curator.test.TestingServer;\n+import org.apache.hadoop.yarn.api.records.ApplicationId;\n+import org.apache.hadoop.yarn.client.api.YarnClient;\n+import org.apache.hadoop.yarn.conf.YarnConfiguration;\n+import org.apache.hadoop.yarn.server.MiniYARNCluster;\n+import org.apache.helix.HelixManager;\n+import org.apache.helix.HelixManagerFactory;\n+import org.apache.helix.InstanceType;\n+import org.apache.helix.model.Message;\n+import org.testng.Assert;\n+import org.testng.annotations.AfterClass;\n+import org.testng.annotations.BeforeClass;\n+import org.testng.annotations.Test;\n+\n+import com.typesafe.config.Config;\n+import com.typesafe.config.ConfigFactory;\n+\n+import com.google.common.io.Closer;\n+\n+\n+/**\n+ * Unit tests for {@link GobblinYarnAppLauncher}.\n+ *\n+ * @author ynli\n+ */\n+@Test(groups = { \"gobblin.yarn\" })\n+public class GobblinYarnAppLauncherTest implements HelixMessageTestBase {\n+\n+  private static final int TEST_ZK_PORT = 3181;\n+\n+  private YarnClient yarnClient;\n+\n+  private CuratorFramework curatorFramework;\n+\n+  private Config config;\n+\n+  private HelixManager helixManager;\n+\n+  private GobblinYarnAppLauncher gobblinYarnAppLauncher;\n+  private ApplicationId applicationId;\n+\n+  private final Closer closer = Closer.create();\n+\n+  @BeforeClass\n+  public void setUp() throws Exception {\n+    YarnConfiguration clusterConf = new YarnConfiguration();\n+\n+    MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster(\"TestCluster\", 1, 1, 1));\n+    miniYARNCluster.init(clusterConf);\n+    miniYARNCluster.start();\n+\n+    this.yarnClient = this.closer.register(YarnClient.createYarnClient());\n+    this.yarnClient.init(clusterConf);\n+    this.yarnClient.start();\n+\n+    TestingServer testingZKServer = this.closer.register(new TestingServer(TEST_ZK_PORT));\n+    this.curatorFramework = this.closer.register(\n+        CuratorFrameworkFactory.newClient(testingZKServer.getConnectString(), new RetryOneTime(2000)));\n+    this.curatorFramework.start();\n+\n+    URL url = GobblinYarnAppLauncherTest.class.getClassLoader().getResource(\n+        GobblinYarnAppLauncherTest.class.getSimpleName() + \".conf\");\n+    Assert.assertNotNull(url, \"Could not find resource \" + url);\n+\n+    this.config = ConfigFactory.parseURL(url).resolve();\n+\n+    String zkConnectionString = this.config.getString(GobblinYarnConfigurationKeys.ZK_CONNECTION_STRING_KEY);\n+    this.helixManager = HelixManagerFactory\n+        .getZKHelixManager(this.config.getString(GobblinYarnConfigurationKeys.HELIX_CLUSTER_NAME_KEY),\n+            TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString);\n+\n+    this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf);\n+  }\n+\n+  @Test\n+  public void testCreateHelixCluster() throws Exception {\n+    // This is tested here instead of in YarnHelixUtilsTest to avoid setting up yet another testing ZooKeeper server.\n+    YarnHelixUtils.createGobblinYarnHelixCluster(\n+        this.config.getString(GobblinYarnConfigurationKeys.ZK_CONNECTION_STRING_KEY),\n+        this.config.getString(GobblinYarnConfigurationKeys.HELIX_CLUSTER_NAME_KEY));\n+\n+    Assert.assertEquals(this.curatorFramework.checkExists().forPath(\n+        String.format(\"/%s\", GobblinYarnAppLauncherTest.class.getSimpleName())).getVersion(), 0);\n+    Assert.assertEquals(this.curatorFramework.checkExists().forPath(\n+        String.format(\"/%s/CONTROLLER\", GobblinYarnAppLauncherTest.class.getSimpleName())).getVersion(), 0);\n+  }\n+\n+  /**\n+   * For some yet unknown reason, hostname resolution for the ResourceManager in {@link MiniYARNCluster}\n+   * has some issue that causes the {@link YarnClient} not be able to connect and submit the Gobblin Yarn\n+   * application successfully. This works fine on local machine though. So disabling this and the test\n+   * below that depends on it on Travis-CI.\n+   *\n+   * @throws Exception\n+   */\n+  @Test(groups = { \"disabledOnTravis\" }, dependsOnMethods = \"testCreateHelixCluster\")\n+  public void testSetupAndSubmitApplication() throws Exception {\n+    this.applicationId = this.gobblinYarnAppLauncher.setupAndSubmitApplication();\n+  }\n+\n+  @Test(groups = { \"disabledOnTravis\" }, dependsOnMethods = \"testSetupAndSubmitApplication\")\n+  public void testGetReconnectableApplicationId() throws Exception {\n+    Assert.assertEquals(this.gobblinYarnAppLauncher.getReconnectableApplicationId().get(), this.applicationId);\n+    this.yarnClient.killApplication(this.applicationId);\n+  }\n+\n+  @Test(dependsOnMethods = \"testCreateHelixCluster\")\n+  public void testSendShutdownRequest() throws Exception {\n+    this.helixManager.connect();\n+    this.helixManager.getMessagingService().registerMessageHandlerFactory(Message.MessageType.SHUTDOWN.toString(),\n+        new TestShutdownMessageHandlerFactory(this));\n+\n+    this.gobblinYarnAppLauncher.connectHelixManager();\n+    this.gobblinYarnAppLauncher.sendShutdownRequest();\n+\n+    Assert.assertEquals(this.curatorFramework.checkExists().forPath(\n+        String.format(\"/%s/CONTROLLER/MESSAGES\", GobblinYarnAppLauncherTest.class.getSimpleName())).getVersion(), 0);\n+    Thread.sleep(500);\n+    Assert.assertEquals(this.curatorFramework.getChildren().forPath(String.format(\"/%s/CONTROLLER/MESSAGES\",\n+        GobblinYarnAppLauncherTest.class.getSimpleName())).size(), 1);\n+    // Give Helix sometime to handle the message\n+    Thread.sleep(2000);\n+    Assert.assertEquals(this.curatorFramework.getChildren().forPath(String.format(\"/%s/CONTROLLER/MESSAGES\",\n+        GobblinYarnAppLauncherTest.class.getSimpleName())).size(), 0);\n+  }\n+\n+  @AfterClass\n+  public void tearDown() throws IOException, TimeoutException {\n+    try {\n+      if (this.helixManager.isConnected()) {\n+        this.helixManager.disconnect();\n+      }\n+      this.gobblinYarnAppLauncher.disconnectHelixManager();\n+    } catch (Throwable t) {","path":"gobblin-yarn/src/test/java/gobblin/yarn/GobblinYarnAppLauncherTest.java","position":155,"original_position":155,"commit_id":"910fbcd73f3e7f78974c6fff2adb33c0be04db33","original_commit_id":"910fbcd73f3e7f78974c6fff2adb33c0be04db33","user":{"login":"liyinan926","id":3709137,"avatar_url":"https://avatars.githubusercontent.com/u/3709137?v=3","gravatar_id":"","url":"https://api.github.com/users/liyinan926","html_url":"https://github.com/liyinan926","followers_url":"https://api.github.com/users/liyinan926/followers","following_url":"https://api.github.com/users/liyinan926/following{/other_user}","gists_url":"https://api.github.com/users/liyinan926/gists{/gist_id}","starred_url":"https://api.github.com/users/liyinan926/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liyinan926/subscriptions","organizations_url":"https://api.github.com/users/liyinan926/orgs","repos_url":"https://api.github.com/users/liyinan926/repos","events_url":"https://api.github.com/users/liyinan926/events{/privacy}","received_events_url":"https://api.github.com/users/liyinan926/received_events","type":"User","site_admin":false},"body":"Fixed.","created_at":"2015-12-04T23:42:43Z","updated_at":"2015-12-04T23:42:43Z","html_url":"https://github.com/linkedin/gobblin/pull/501#discussion_r46746127","pull_request_url":"https://api.github.com/repos/linkedin/gobblin/pulls/501","_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/46746127"},"html":{"href":"https://github.com/linkedin/gobblin/pull/501#discussion_r46746127"},"pull_request":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/501"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/501","id":52564906,"html_url":"https://github.com/linkedin/gobblin/pull/501","diff_url":"https://github.com/linkedin/gobblin/pull/501.diff","patch_url":"https://github.com/linkedin/gobblin/pull/501.patch","issue_url":"https://api.github.com/repos/linkedin/gobblin/issues/501","number":501,"state":"open","locked":false,"title":"Unit tests for Gobblin Yarn part I","user":{"login":"liyinan926","id":3709137,"avatar_url":"https://avatars.githubusercontent.com/u/3709137?v=3","gravatar_id":"","url":"https://api.github.com/users/liyinan926","html_url":"https://github.com/liyinan926","followers_url":"https://api.github.com/users/liyinan926/followers","following_url":"https://api.github.com/users/liyinan926/following{/other_user}","gists_url":"https://api.github.com/users/liyinan926/gists{/gist_id}","starred_url":"https://api.github.com/users/liyinan926/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liyinan926/subscriptions","organizations_url":"https://api.github.com/users/liyinan926/orgs","repos_url":"https://api.github.com/users/liyinan926/repos","events_url":"https://api.github.com/users/liyinan926/events{/privacy}","received_events_url":"https://api.github.com/users/liyinan926/received_events","type":"User","site_admin":false},"body":"This is part I of unit tests for Gobblin on Yarn. There are still a few classes that has dependency on Yarn that I'm trying to figure out the best way to test. Will create a PR for part II for those classes later.","created_at":"2015-12-03T17:23:05Z","updated_at":"2015-12-04T23:42:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"50d404ea0b04a46c42c0a7159d376f5a2ee38dab","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/gobblin/pulls/501/commits","review_comments_url":"https://api.github.com/repos/linkedin/gobblin/pulls/501/comments","review_comment_url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/gobblin/issues/501/comments","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/910fbcd73f3e7f78974c6fff2adb33c0be04db33","head":{"label":"liyinan926:gobblin-yarn","ref":"gobblin-yarn","sha":"910fbcd73f3e7f78974c6fff2adb33c0be04db33","user":{"login":"liyinan926","id":3709137,"avatar_url":"https://avatars.githubusercontent.com/u/3709137?v=3","gravatar_id":"","url":"https://api.github.com/users/liyinan926","html_url":"https://github.com/liyinan926","followers_url":"https://api.github.com/users/liyinan926/followers","following_url":"https://api.github.com/users/liyinan926/following{/other_user}","gists_url":"https://api.github.com/users/liyinan926/gists{/gist_id}","starred_url":"https://api.github.com/users/liyinan926/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liyinan926/subscriptions","organizations_url":"https://api.github.com/users/liyinan926/orgs","repos_url":"https://api.github.com/users/liyinan926/repos","events_url":"https://api.github.com/users/liyinan926/events{/privacy}","received_events_url":"https://api.github.com/users/liyinan926/received_events","type":"User","site_admin":false},"repo":{"id":31117112,"name":"gobblin","full_name":"liyinan926/gobblin","owner":{"login":"liyinan926","id":3709137,"avatar_url":"https://avatars.githubusercontent.com/u/3709137?v=3","gravatar_id":"","url":"https://api.github.com/users/liyinan926","html_url":"https://github.com/liyinan926","followers_url":"https://api.github.com/users/liyinan926/followers","following_url":"https://api.github.com/users/liyinan926/following{/other_user}","gists_url":"https://api.github.com/users/liyinan926/gists{/gist_id}","starred_url":"https://api.github.com/users/liyinan926/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/liyinan926/subscriptions","organizations_url":"https://api.github.com/users/liyinan926/orgs","repos_url":"https://api.github.com/users/liyinan926/repos","events_url":"https://api.github.com/users/liyinan926/events{/privacy}","received_events_url":"https://api.github.com/users/liyinan926/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/liyinan926/gobblin","description":"","fork":true,"url":"https://api.github.com/repos/liyinan926/gobblin","forks_url":"https://api.github.com/repos/liyinan926/gobblin/forks","keys_url":"https://api.github.com/repos/liyinan926/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/liyinan926/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/liyinan926/gobblin/teams","hooks_url":"https://api.github.com/repos/liyinan926/gobblin/hooks","issue_events_url":"https://api.github.com/repos/liyinan926/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/liyinan926/gobblin/events","assignees_url":"https://api.github.com/repos/liyinan926/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/liyinan926/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/liyinan926/gobblin/tags","blobs_url":"https://api.github.com/repos/liyinan926/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/liyinan926/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/liyinan926/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/liyinan926/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/liyinan926/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/liyinan926/gobblin/languages","stargazers_url":"https://api.github.com/repos/liyinan926/gobblin/stargazers","contributors_url":"https://api.github.com/repos/liyinan926/gobblin/contributors","subscribers_url":"https://api.github.com/repos/liyinan926/gobblin/subscribers","subscription_url":"https://api.github.com/repos/liyinan926/gobblin/subscription","commits_url":"https://api.github.com/repos/liyinan926/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/liyinan926/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/liyinan926/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/liyinan926/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/liyinan926/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/liyinan926/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/liyinan926/gobblin/merges","archive_url":"https://api.github.com/repos/liyinan926/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/liyinan926/gobblin/downloads","issues_url":"https://api.github.com/repos/liyinan926/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/liyinan926/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/liyinan926/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/liyinan926/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/liyinan926/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/liyinan926/gobblin/releases{/id}","created_at":"2015-02-21T06:07:55Z","updated_at":"2015-05-05T00:31:33Z","pushed_at":"2015-12-04T07:18:22Z","git_url":"git://github.com/liyinan926/gobblin.git","ssh_url":"git@github.com:liyinan926/gobblin.git","clone_url":"https://github.com/liyinan926/gobblin.git","svn_url":"https://github.com/liyinan926/gobblin","homepage":null,"size":81297,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"e5a67b7b277c0215f3b383f739daf7f1abdc1e44","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":27392543,"name":"gobblin","full_name":"linkedin/gobblin","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":false,"url":"https://api.github.com/repos/linkedin/gobblin","forks_url":"https://api.github.com/repos/linkedin/gobblin/forks","keys_url":"https://api.github.com/repos/linkedin/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/gobblin/teams","hooks_url":"https://api.github.com/repos/linkedin/gobblin/hooks","issue_events_url":"https://api.github.com/repos/linkedin/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/gobblin/events","assignees_url":"https://api.github.com/repos/linkedin/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/gobblin/tags","blobs_url":"https://api.github.com/repos/linkedin/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/gobblin/languages","stargazers_url":"https://api.github.com/repos/linkedin/gobblin/stargazers","contributors_url":"https://api.github.com/repos/linkedin/gobblin/contributors","subscribers_url":"https://api.github.com/repos/linkedin/gobblin/subscribers","subscription_url":"https://api.github.com/repos/linkedin/gobblin/subscription","commits_url":"https://api.github.com/repos/linkedin/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/gobblin/merges","archive_url":"https://api.github.com/repos/linkedin/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/gobblin/downloads","issues_url":"https://api.github.com/repos/linkedin/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/gobblin/releases{/id}","created_at":"2014-12-01T18:10:50Z","updated_at":"2015-12-04T23:24:03Z","pushed_at":"2015-12-04T23:11:40Z","git_url":"git://github.com/linkedin/gobblin.git","ssh_url":"git@github.com:linkedin/gobblin.git","clone_url":"https://github.com/linkedin/gobblin.git","svn_url":"https://github.com/linkedin/gobblin","homepage":"https://github.com/linkedin/gobblin/wiki","size":156828,"stargazers_count":317,"watchers_count":317,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":121,"mirror_url":null,"open_issues_count":19,"forks":121,"open_issues":19,"watchers":317,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/501"},"html":{"href":"https://github.com/linkedin/gobblin/pull/501"},"issue":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/501"},"comments":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/501/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/501/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/501/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/gobblin/statuses/910fbcd73f3e7f78974c6fff2adb33c0be04db33"}}}}