{"action":"created","comment":{"url":"https://api.github.com/repos/amplab/tachyon/pulls/comments/45137176","id":45137176,"diff_hunk":"@@ -0,0 +1,159 @@\n+/*\n+ * Licensed to the University of California, Berkeley under one or more contributor license\n+ * agreements. See the NOTICE file distributed with this work for additional information regarding\n+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance with the License. You may obtain a\n+ * copy of the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software distributed under the License\n+ * is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n+ * or implied. See the License for the specific language governing permissions and limitations under\n+ * the License.\n+ */\n+\n+package tachyon.master.lineage;\n+\n+import java.util.List;\n+import java.util.concurrent.TimeUnit;\n+\n+import org.junit.Assert;\n+import org.junit.Before;\n+import org.junit.BeforeClass;\n+import org.junit.Rule;\n+import org.junit.Test;\n+\n+import com.google.common.collect.Lists;\n+\n+import tachyon.Constants;\n+import tachyon.IntegrationTestConstants;\n+import tachyon.LocalTachyonClusterResource;\n+import tachyon.TachyonURI;\n+import tachyon.client.TachyonStorageType;\n+import tachyon.client.UnderStorageType;\n+import tachyon.client.file.FileOutStream;\n+import tachyon.client.file.options.OutStreamOptions;\n+import tachyon.client.lineage.LineageMasterClient;\n+import tachyon.client.lineage.TachyonLineageFileSystem;\n+import tachyon.conf.TachyonConf;\n+import tachyon.heartbeat.HeartbeatContext;\n+import tachyon.heartbeat.HeartbeatScheduler;\n+import tachyon.job.CommandLineJob;\n+import tachyon.job.JobConf;\n+import tachyon.master.lineage.meta.LineageFileState;\n+import tachyon.thrift.LineageInfo;\n+\n+/**\n+ * Integration tests for the lineage module.\n+ */\n+public final class LineageMastertIntegrationsTest {\n+  private static final int BLOCK_SIZE_BYTES = 128;\n+  private static final long WORKER_CAPACITY_BYTES = Constants.GB;\n+  private static final int QUOTA_UNIT_BYTES = 128;\n+  private static final int BUFFER_BYTES = 100;\n+\n+  @Rule\n+  public LocalTachyonClusterResource mLocalTachyonClusterResource =\n+      new LocalTachyonClusterResource(WORKER_CAPACITY_BYTES, QUOTA_UNIT_BYTES, BLOCK_SIZE_BYTES,\n+          Constants.USER_FILE_BUFFER_BYTES, String.valueOf(BUFFER_BYTES),\n+          Constants.WORKER_DATA_SERVER, IntegrationTestConstants.NETTY_DATA_SERVER);\n+\n+  private static final String OUT_FILE = \"/test\";\n+  private TachyonConf mTestConf;\n+  private CommandLineJob mJob;\n+\n+  @BeforeClass\n+  public static void beforeClass() {\n+    HeartbeatContext.setTimerClass(HeartbeatContext.MASTER_CHECKPOINT_SCHEDULING,\n+        HeartbeatContext.SCHEDULED_TIMER_CLASS);\n+    HeartbeatContext.setTimerClass(HeartbeatContext.WORKER_LINEAGE_SYNC,\n+        HeartbeatContext.SCHEDULED_TIMER_CLASS);\n+  }\n+\n+  @Before\n+  public void before() throws Exception {\n+    mJob = new CommandLineJob(\"test\", new JobConf(\"output\"));\n+    mTestConf = mLocalTachyonClusterResource.get().getMasterTachyonConf();\n+  }\n+\n+  @Test\n+  @LocalTachyonClusterResource.Config(tachyonConfParams = {Constants.USER_LINEAGE_ENABLED, \"true\"})\n+  public void lineageCreationTest() throws Exception {\n+    LineageMasterClient lineageMasterClient = getLineageMasterClient();\n+\n+    try {\n+      lineageMasterClient.createLineage(Lists.<String>newArrayList(), Lists.newArrayList(OUT_FILE),\n+          mJob);\n+\n+      List<LineageInfo> infos = lineageMasterClient.getLineageInfoList();\n+      Assert.assertEquals(1, infos.size());\n+      Assert.assertEquals(LineageFileState.CREATED.toString(),\n+          infos.get(0).outputFiles.get(0).state);\n+    } finally {\n+      lineageMasterClient.close();\n+    }\n+  }\n+\n+  @Test\n+  @LocalTachyonClusterResource.Config(tachyonConfParams = {Constants.USER_LINEAGE_ENABLED, \"true\"})\n+  public void lineageCompleteAndAsyncPersistTest() throws Exception {\n+    LineageMasterClient lineageMasterClient = getLineageMasterClient();\n+\n+    Assert.assertTrue(HeartbeatScheduler.await(HeartbeatContext.MASTER_CHECKPOINT_SCHEDULING, 5,\n+        TimeUnit.SECONDS));\n+    Assert.assertTrue(\n+        HeartbeatScheduler.await(HeartbeatContext.WORKER_LINEAGE_SYNC, 5, TimeUnit.SECONDS));\n+\n+    try {\n+      lineageMasterClient.createLineage(Lists.<String>newArrayList(), Lists.newArrayList(OUT_FILE),\n+          mJob);\n+\n+      OutStreamOptions options =\n+          new OutStreamOptions.Builder(mTestConf).setTachyonStorageType(TachyonStorageType.STORE)\n+              .setUnderStorageType(UnderStorageType.NO_PERSIST).setBlockSizeBytes(BLOCK_SIZE_BYTES)\n+              .build();\n+      TachyonLineageFileSystem tfs =\n+          (TachyonLineageFileSystem) mLocalTachyonClusterResource.get().getClient();\n+      FileOutStream outputStream = tfs.getOutStream(new TachyonURI(OUT_FILE), options);\n+      outputStream.write(1);\n+      outputStream.close();\n+\n+      List<LineageInfo> infos = lineageMasterClient.getLineageInfoList();\n+      Assert.assertEquals(LineageFileState.COMPLETED.toString(),\n+          infos.get(0).outputFiles.get(0).state);\n+\n+      // Execute the checkpoint scheduler for async checkpoint\n+      HeartbeatScheduler.schedule(HeartbeatContext.MASTER_CHECKPOINT_SCHEDULING);\n+      HeartbeatScheduler.schedule(HeartbeatContext.WORKER_LINEAGE_SYNC);\n+      Assert.assertTrue(HeartbeatScheduler.await(HeartbeatContext.MASTER_CHECKPOINT_SCHEDULING, 5,\n+          TimeUnit.SECONDS));\n+      Assert.assertTrue(\n+          HeartbeatScheduler.await(HeartbeatContext.WORKER_LINEAGE_SYNC, 5, TimeUnit.SECONDS));\n+\n+      infos = lineageMasterClient.getLineageInfoList();\n+      Assert.assertEquals(LineageFileState.PERSISENCE_REQUESTED.toString(),\n+          infos.get(0).outputFiles.get(0).state);\n+\n+      // sleep and wait for worker to persist the file\n+      Thread.sleep(5);\n+\n+      // worker notifies the master\n+      HeartbeatScheduler.schedule(HeartbeatContext.WORKER_LINEAGE_SYNC);\n+      Assert.assertTrue(\n+          HeartbeatScheduler.await(HeartbeatContext.WORKER_LINEAGE_SYNC, 5000, TimeUnit.SECONDS));","path":"integration-tests/src/test/java/tachyon/master/lineage/LineageMastertIntegrationsTest.java","position":144,"original_position":144,"commit_id":"914b2a9b0e74c368fbb9558a7ee82600c26bce6f","original_commit_id":"914b2a9b0e74c368fbb9558a7ee82600c26bce6f","user":{"login":"yupeng9","id":13425258,"avatar_url":"https://avatars.githubusercontent.com/u/13425258?v=3","gravatar_id":"","url":"https://api.github.com/users/yupeng9","html_url":"https://github.com/yupeng9","followers_url":"https://api.github.com/users/yupeng9/followers","following_url":"https://api.github.com/users/yupeng9/following{/other_user}","gists_url":"https://api.github.com/users/yupeng9/gists{/gist_id}","starred_url":"https://api.github.com/users/yupeng9/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yupeng9/subscriptions","organizations_url":"https://api.github.com/users/yupeng9/orgs","repos_url":"https://api.github.com/users/yupeng9/repos","events_url":"https://api.github.com/users/yupeng9/events{/privacy}","received_events_url":"https://api.github.com/users/yupeng9/received_events","type":"User","site_admin":false},"body":"Good catch.","created_at":"2015-11-17T23:12:53Z","updated_at":"2015-11-17T23:12:53Z","html_url":"https://github.com/amplab/tachyon/pull/2086#discussion_r45137176","pull_request_url":"https://api.github.com/repos/amplab/tachyon/pulls/2086","_links":{"self":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/comments/45137176"},"html":{"href":"https://github.com/amplab/tachyon/pull/2086#discussion_r45137176"},"pull_request":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/2086"}}},"pull_request":{"url":"https://api.github.com/repos/amplab/tachyon/pulls/2086","id":50910727,"html_url":"https://github.com/amplab/tachyon/pull/2086","diff_url":"https://github.com/amplab/tachyon/pull/2086.diff","patch_url":"https://github.com/amplab/tachyon/pull/2086.patch","issue_url":"https://api.github.com/repos/amplab/tachyon/issues/2086","number":2086,"state":"open","locked":false,"title":"[TACHYON-1207] Added integration tests for lineage","user":{"login":"yupeng9","id":13425258,"avatar_url":"https://avatars.githubusercontent.com/u/13425258?v=3","gravatar_id":"","url":"https://api.github.com/users/yupeng9","html_url":"https://github.com/yupeng9","followers_url":"https://api.github.com/users/yupeng9/followers","following_url":"https://api.github.com/users/yupeng9/following{/other_user}","gists_url":"https://api.github.com/users/yupeng9/gists{/gist_id}","starred_url":"https://api.github.com/users/yupeng9/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yupeng9/subscriptions","organizations_url":"https://api.github.com/users/yupeng9/orgs","repos_url":"https://api.github.com/users/yupeng9/repos","events_url":"https://api.github.com/users/yupeng9/events{/privacy}","received_events_url":"https://api.github.com/users/yupeng9/received_events","type":"User","site_admin":false},"body":"https://tachyon.atlassian.net/browse/TACHYON-1207","created_at":"2015-11-17T06:55:10Z","updated_at":"2015-11-17T23:12:53Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a9ed298e26a8e337d3f2ca866c787e0e9d619da7","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/amplab/tachyon/pulls/2086/commits","review_comments_url":"https://api.github.com/repos/amplab/tachyon/pulls/2086/comments","review_comment_url":"https://api.github.com/repos/amplab/tachyon/pulls/comments{/number}","comments_url":"https://api.github.com/repos/amplab/tachyon/issues/2086/comments","statuses_url":"https://api.github.com/repos/amplab/tachyon/statuses/914b2a9b0e74c368fbb9558a7ee82600c26bce6f","head":{"label":"yupeng9:lineageExample","ref":"lineageExample","sha":"914b2a9b0e74c368fbb9558a7ee82600c26bce6f","user":{"login":"yupeng9","id":13425258,"avatar_url":"https://avatars.githubusercontent.com/u/13425258?v=3","gravatar_id":"","url":"https://api.github.com/users/yupeng9","html_url":"https://github.com/yupeng9","followers_url":"https://api.github.com/users/yupeng9/followers","following_url":"https://api.github.com/users/yupeng9/following{/other_user}","gists_url":"https://api.github.com/users/yupeng9/gists{/gist_id}","starred_url":"https://api.github.com/users/yupeng9/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yupeng9/subscriptions","organizations_url":"https://api.github.com/users/yupeng9/orgs","repos_url":"https://api.github.com/users/yupeng9/repos","events_url":"https://api.github.com/users/yupeng9/events{/privacy}","received_events_url":"https://api.github.com/users/yupeng9/received_events","type":"User","site_admin":false},"repo":{"id":39411244,"name":"tachyon","full_name":"yupeng9/tachyon","owner":{"login":"yupeng9","id":13425258,"avatar_url":"https://avatars.githubusercontent.com/u/13425258?v=3","gravatar_id":"","url":"https://api.github.com/users/yupeng9","html_url":"https://github.com/yupeng9","followers_url":"https://api.github.com/users/yupeng9/followers","following_url":"https://api.github.com/users/yupeng9/following{/other_user}","gists_url":"https://api.github.com/users/yupeng9/gists{/gist_id}","starred_url":"https://api.github.com/users/yupeng9/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yupeng9/subscriptions","organizations_url":"https://api.github.com/users/yupeng9/orgs","repos_url":"https://api.github.com/users/yupeng9/repos","events_url":"https://api.github.com/users/yupeng9/events{/privacy}","received_events_url":"https://api.github.com/users/yupeng9/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/yupeng9/tachyon","description":" A Reliable Memory Centric Distributed Storage System","fork":true,"url":"https://api.github.com/repos/yupeng9/tachyon","forks_url":"https://api.github.com/repos/yupeng9/tachyon/forks","keys_url":"https://api.github.com/repos/yupeng9/tachyon/keys{/key_id}","collaborators_url":"https://api.github.com/repos/yupeng9/tachyon/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/yupeng9/tachyon/teams","hooks_url":"https://api.github.com/repos/yupeng9/tachyon/hooks","issue_events_url":"https://api.github.com/repos/yupeng9/tachyon/issues/events{/number}","events_url":"https://api.github.com/repos/yupeng9/tachyon/events","assignees_url":"https://api.github.com/repos/yupeng9/tachyon/assignees{/user}","branches_url":"https://api.github.com/repos/yupeng9/tachyon/branches{/branch}","tags_url":"https://api.github.com/repos/yupeng9/tachyon/tags","blobs_url":"https://api.github.com/repos/yupeng9/tachyon/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/yupeng9/tachyon/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/yupeng9/tachyon/git/refs{/sha}","trees_url":"https://api.github.com/repos/yupeng9/tachyon/git/trees{/sha}","statuses_url":"https://api.github.com/repos/yupeng9/tachyon/statuses/{sha}","languages_url":"https://api.github.com/repos/yupeng9/tachyon/languages","stargazers_url":"https://api.github.com/repos/yupeng9/tachyon/stargazers","contributors_url":"https://api.github.com/repos/yupeng9/tachyon/contributors","subscribers_url":"https://api.github.com/repos/yupeng9/tachyon/subscribers","subscription_url":"https://api.github.com/repos/yupeng9/tachyon/subscription","commits_url":"https://api.github.com/repos/yupeng9/tachyon/commits{/sha}","git_commits_url":"https://api.github.com/repos/yupeng9/tachyon/git/commits{/sha}","comments_url":"https://api.github.com/repos/yupeng9/tachyon/comments{/number}","issue_comment_url":"https://api.github.com/repos/yupeng9/tachyon/issues/comments{/number}","contents_url":"https://api.github.com/repos/yupeng9/tachyon/contents/{+path}","compare_url":"https://api.github.com/repos/yupeng9/tachyon/compare/{base}...{head}","merges_url":"https://api.github.com/repos/yupeng9/tachyon/merges","archive_url":"https://api.github.com/repos/yupeng9/tachyon/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/yupeng9/tachyon/downloads","issues_url":"https://api.github.com/repos/yupeng9/tachyon/issues{/number}","pulls_url":"https://api.github.com/repos/yupeng9/tachyon/pulls{/number}","milestones_url":"https://api.github.com/repos/yupeng9/tachyon/milestones{/number}","notifications_url":"https://api.github.com/repos/yupeng9/tachyon/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/yupeng9/tachyon/labels{/name}","releases_url":"https://api.github.com/repos/yupeng9/tachyon/releases{/id}","created_at":"2015-07-20T22:20:20Z","updated_at":"2015-07-20T22:20:22Z","pushed_at":"2015-11-17T07:43:28Z","git_url":"git://github.com/yupeng9/tachyon.git","ssh_url":"git@github.com:yupeng9/tachyon.git","clone_url":"https://github.com/yupeng9/tachyon.git","svn_url":"https://github.com/yupeng9/tachyon","homepage":"http://tachyon-project.org","size":29681,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"amplab:master","ref":"master","sha":"1153fe552c411cc3617684aeb1974ed69db42f86","user":{"login":"amplab","id":1438454,"avatar_url":"https://avatars.githubusercontent.com/u/1438454?v=3","gravatar_id":"","url":"https://api.github.com/users/amplab","html_url":"https://github.com/amplab","followers_url":"https://api.github.com/users/amplab/followers","following_url":"https://api.github.com/users/amplab/following{/other_user}","gists_url":"https://api.github.com/users/amplab/gists{/gist_id}","starred_url":"https://api.github.com/users/amplab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplab/subscriptions","organizations_url":"https://api.github.com/users/amplab/orgs","repos_url":"https://api.github.com/users/amplab/repos","events_url":"https://api.github.com/users/amplab/events{/privacy}","received_events_url":"https://api.github.com/users/amplab/received_events","type":"Organization","site_admin":false},"repo":{"id":7276954,"name":"tachyon","full_name":"amplab/tachyon","owner":{"login":"amplab","id":1438454,"avatar_url":"https://avatars.githubusercontent.com/u/1438454?v=3","gravatar_id":"","url":"https://api.github.com/users/amplab","html_url":"https://github.com/amplab","followers_url":"https://api.github.com/users/amplab/followers","following_url":"https://api.github.com/users/amplab/following{/other_user}","gists_url":"https://api.github.com/users/amplab/gists{/gist_id}","starred_url":"https://api.github.com/users/amplab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplab/subscriptions","organizations_url":"https://api.github.com/users/amplab/orgs","repos_url":"https://api.github.com/users/amplab/repos","events_url":"https://api.github.com/users/amplab/events{/privacy}","received_events_url":"https://api.github.com/users/amplab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/amplab/tachyon","description":"Memory Centric Reliable Distributed Storage","fork":false,"url":"https://api.github.com/repos/amplab/tachyon","forks_url":"https://api.github.com/repos/amplab/tachyon/forks","keys_url":"https://api.github.com/repos/amplab/tachyon/keys{/key_id}","collaborators_url":"https://api.github.com/repos/amplab/tachyon/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/amplab/tachyon/teams","hooks_url":"https://api.github.com/repos/amplab/tachyon/hooks","issue_events_url":"https://api.github.com/repos/amplab/tachyon/issues/events{/number}","events_url":"https://api.github.com/repos/amplab/tachyon/events","assignees_url":"https://api.github.com/repos/amplab/tachyon/assignees{/user}","branches_url":"https://api.github.com/repos/amplab/tachyon/branches{/branch}","tags_url":"https://api.github.com/repos/amplab/tachyon/tags","blobs_url":"https://api.github.com/repos/amplab/tachyon/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/amplab/tachyon/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/amplab/tachyon/git/refs{/sha}","trees_url":"https://api.github.com/repos/amplab/tachyon/git/trees{/sha}","statuses_url":"https://api.github.com/repos/amplab/tachyon/statuses/{sha}","languages_url":"https://api.github.com/repos/amplab/tachyon/languages","stargazers_url":"https://api.github.com/repos/amplab/tachyon/stargazers","contributors_url":"https://api.github.com/repos/amplab/tachyon/contributors","subscribers_url":"https://api.github.com/repos/amplab/tachyon/subscribers","subscription_url":"https://api.github.com/repos/amplab/tachyon/subscription","commits_url":"https://api.github.com/repos/amplab/tachyon/commits{/sha}","git_commits_url":"https://api.github.com/repos/amplab/tachyon/git/commits{/sha}","comments_url":"https://api.github.com/repos/amplab/tachyon/comments{/number}","issue_comment_url":"https://api.github.com/repos/amplab/tachyon/issues/comments{/number}","contents_url":"https://api.github.com/repos/amplab/tachyon/contents/{+path}","compare_url":"https://api.github.com/repos/amplab/tachyon/compare/{base}...{head}","merges_url":"https://api.github.com/repos/amplab/tachyon/merges","archive_url":"https://api.github.com/repos/amplab/tachyon/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/amplab/tachyon/downloads","issues_url":"https://api.github.com/repos/amplab/tachyon/issues{/number}","pulls_url":"https://api.github.com/repos/amplab/tachyon/pulls{/number}","milestones_url":"https://api.github.com/repos/amplab/tachyon/milestones{/number}","notifications_url":"https://api.github.com/repos/amplab/tachyon/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/amplab/tachyon/labels{/name}","releases_url":"https://api.github.com/repos/amplab/tachyon/releases{/id}","created_at":"2012-12-21T17:43:46Z","updated_at":"2015-11-17T22:35:47Z","pushed_at":"2015-11-17T22:22:55Z","git_url":"git://github.com/amplab/tachyon.git","ssh_url":"git@github.com:amplab/tachyon.git","clone_url":"https://github.com/amplab/tachyon.git","svn_url":"https://github.com/amplab/tachyon","homepage":"http://tachyon-project.org","size":38420,"stargazers_count":1742,"watchers_count":1742,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":687,"mirror_url":null,"open_issues_count":28,"forks":687,"open_issues":28,"watchers":1742,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/2086"},"html":{"href":"https://github.com/amplab/tachyon/pull/2086"},"issue":{"href":"https://api.github.com/repos/amplab/tachyon/issues/2086"},"comments":{"href":"https://api.github.com/repos/amplab/tachyon/issues/2086/comments"},"review_comments":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/2086/comments"},"review_comment":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/amplab/tachyon/pulls/2086/commits"},"statuses":{"href":"https://api.github.com/repos/amplab/tachyon/statuses/914b2a9b0e74c368fbb9558a7ee82600c26bce6f"}}}}