{"action":"created","comment":{"url":"https://api.github.com/repos/allenai/pipeline/pulls/comments/31542564","id":31542564,"diff_hunk":"@@ -0,0 +1,364 @@\n+package org.allenai.pipeline\n+\n+import java.io.File\n+import java.net.URI\n+import java.text.SimpleDateFormat\n+import java.util.Date\n+\n+import com.typesafe.config.Config\n+import org.allenai.common.Config._\n+import org.allenai.common.Logging\n+import org.allenai.pipeline.IoHelpers._\n+import spray.json.DefaultJsonProtocol._\n+import spray.json.JsonFormat\n+\n+import scala.reflect.ClassTag\n+import scala.util.control.NonFatal\n+\n+/** A top-level data flow pipeline.\n+  * Provides methods for persisting Producers in a consistent location,\n+  * running Producers,\n+  * and producing a visualization of the end-to-end data flow\n+  */\n+trait Pipeline extends Logging {\n+\n+  def rootOutputUrl: URI\n+\n+  /** Run the pipeline.  All steps that have been persisted will be computed, along with any upstream dependencies */\n+  def run(title: String) = {\n+    runPipelineReturnResults(title, persistedSteps)\n+  }\n+\n+  def persistedSteps = steps.toMap\n+\n+  /** Common-case methods for persisting Producers */\n+  object Persist {\n+\n+    /** Persist a collection */\n+    object Collection {\n+      def asText[T: StringSerializable: ClassTag](\n+        step: Producer[Iterable[T]],\n+        stepName: String = null,\n+        suffix: String = \".txt\"\n+      )(): PersistedProducer[Iterable[T], FlatArtifact] =\n+        persist(step, LineCollectionIo.text[T], stepName, suffix)\n+\n+      def asJson[T: JsonFormat: ClassTag](\n+        step: Producer[Iterable[T]],\n+        stepName: String = null,\n+        suffix: String = \".json\"\n+      )(): PersistedProducer[Iterable[T], FlatArtifact] =\n+        persist(step, LineCollectionIo.json[T], stepName, suffix)\n+    }\n+\n+    /** Persist a single object */\n+    object Singleton {\n+      def asText[T: StringSerializable: ClassTag](\n+        step: Producer[T],\n+        stepName: String = null,\n+        suffix: String = \".txt\"\n+      )(): PersistedProducer[T, FlatArtifact] =\n+        persist(step, SingletonIo.text[T], stepName, suffix)\n+\n+      def asJson[T: JsonFormat: ClassTag](\n+        step: Producer[T],\n+        stepName: String = null,\n+        suffix: String = \".json\"\n+      )(): PersistedProducer[T, FlatArtifact] =\n+        persist(step, SingletonIo.json[T], stepName, suffix)\n+    }\n+\n+    /** Persist an Iterator */\n+    object Iterator {\n+      def asText[T: StringSerializable: ClassTag](\n+        step: Producer[Iterator[T]],\n+        stepName: String = null,\n+        suffix: String = \".txt\"\n+      ): PersistedProducer[Iterator[T], FlatArtifact] =\n+        persist(step, LineIteratorIo.text[T], stepName, suffix)\n+\n+      def asJson[T: JsonFormat: ClassTag](\n+        step: Producer[Iterator[T]],\n+        stepName: String = null,\n+        suffix: String = \".json\"\n+      )(): PersistedProducer[Iterator[T], FlatArtifact] =\n+        persist(step, LineIteratorIo.json[T], stepName, suffix)\n+    }\n+\n+  }\n+\n+  /** Create a persisted version of the given Producer\n+    * The producer is registered as a target of the pipeline, and will be computed\n+    * when the pipeline is run.\n+    * See Persist.Collection, Persist.Singleton, etc. utility methods above\n+    * @param original the non-persisted Producer\n+    * @param io the serialization format\n+    * @param suffix a file suffix\n+    * @return the persisted Producer\n+    */\n+  def persist[T, A <: Artifact: ClassTag](\n+    original: Producer[T],\n+    io: Serializer[T, A] with Deserializer[T, A],\n+    name: String = null,\n+    suffix: String = \"\"\n+  ): PersistedProducer[T, A] = {\n+    val stepName = Option(name).getOrElse(original.stepInfo.className)\n+    val path = s\"data/$stepName.${hashId(original, io)}$suffix\"\n+    persistToArtifact(original, io, createOutputArtifact[A](path), name)\n+  }\n+\n+  def persistToUrl[T, A <: Artifact: ClassTag](\n+    original: Producer[T],\n+    io: Serializer[T, A] with Deserializer[T, A],\n+    url: URI,\n+    name: String = null\n+  ): PersistedProducer[T, A] = {\n+    persistToArtifact(original, io, artifactFactory.createArtifact[A](url), name)\n+  }\n+\n+  /** Persist this Producer and add it to list of targets that will be computed when the pipeline is run */\n+  def persistToArtifact[T, A <: Artifact](\n+    original: Producer[T],\n+    io: Serializer[T, A] with Deserializer[T, A],\n+    artifact: A,\n+    name: String = null\n+  ): PersistedProducer[T, A] = {\n+    val persisted = new ProducerWithPersistence(original, io, artifact)\n+    val stepNameBase = Option(name).getOrElse(original.stepInfo.className)\n+    var i = 1\n+    var stepName = stepNameBase\n+    while (steps.contains(stepName)) {\n+      stepName = s\"$stepNameBase.$i\"\n+      i += 1\n+    }\n+    addTarget(stepName, persisted)\n+  }\n+\n+  def persistCustom[T, P <: Producer[T], A <: Artifact : ClassTag](\n+    original: P,\n+    makePersisted: (P, A) => PersistedProducer[T, A],\n+    name: String = null,\n+    suffix: String = \"\"\n+    ): PersistedProducer[T, A] = {\n+    val stepName = Option(name).getOrElse(original.stepInfo.className)\n+    val path = s\"data/$stepName.${original.stepInfo.signature.id}$suffix\"\n+    val artifact = createOutputArtifact[A](path)\n+    addTarget(stepName, makePersisted(original, artifact))\n+  }\n+\n+  def addTarget[T, A <: Artifact](stepName: String, target: PersistedProducer[T, A]) = {\n+    steps(stepName) = target\n+    target\n+  }\n+\n+  protected[this] def urlToArtifact = CreateCoreArtifacts.fromFileUrls\n+\n+  def artifactFactory = ArtifactFactory(urlToArtifact)\n+\n+  /** Create an Artifact at the given path, relative to the rootOutputUrl\n+    * (path may also be an absolute URL)\n+    */\n+  def createOutputArtifact[A <: Artifact: ClassTag](path: String): A =\n+    artifactFactory.createArtifact[A](rootOutputUrl, path)\n+\n+  def getStepsByName(targetNames: Iterable[String]) = {\n+    val targets = targetNames.flatMap(s => steps.get(s).map(p => (s, p)))\n+    if (targets.size != targetNames.size) {\n+      val unresolveNames = targetNames.filterNot(steps.contains)\n+      sys.error(s\"Step names not found: ${unresolveNames.mkString(\"[\", \",\", \"]\")}\")\n+    }\n+    targets\n+  }\n+\n+  /** Run only specified steps in the pipeline.  Upstream dependencies must exist already.  They will not be computed */\n+  def runOnly(title: String, targetNames: String*): Iterable[(String, Any)] = {\n+    runOnly(title, getStepsByName(targetNames))\n+  }\n+\n+  def runOnly(title: String, targets: Iterable[(String, Producer[_])]): Iterable[(String, Any)] = {\n+    val targetStepInfo = targets.map(_._2.stepInfo).toSet\n+    val allDependencies = targets.flatMap { case (s, p) => Workflow.upstreamDependencies(p) }\n+    val nonExistentDependencies =\n+      for {\n+        p <- allDependencies if p.isInstanceOf[PersistedProducer[_, _]]\n+        pp = p.asInstanceOf[PersistedProducer[_, _ <: Artifact]]\n+        if !targetStepInfo(pp.stepInfo)\n+        if !pp.artifact.exists\n+      } yield pp.stepInfo\n+    require(nonExistentDependencies.isEmpty, {\n+      val targetNames = targetStepInfo.map(_.className).mkString(\",\")\n+      val dependencyNames = nonExistentDependencies.map(_.className).mkString(\",\")\n+      s\"Cannot run steps [$targetNames]. Upstream dependencies [$dependencyNames] have not been computed\"\n+    })\n+    runPipelineReturnResults(title, targets)\n+  }\n+\n+  protected[this] def runPipelineReturnResults(rawTitle: String, targets: Iterable[(String, Producer[_])]): Iterable[(String, Any)] = {\n+    // Order the outputs so that the ones with the fewest dependencies are executed first\n+    val outputs = targets.toVector.map {\n+      case (name, p) => (Workflow.upstreamDependencies(p).size, name, p)\n+    }.sortBy(_._1).map { case (count, name, p) => (name, p) }\n+    val result: Seq[(String, Any)] = try {\n+      val start = System.currentTimeMillis\n+      val result = outputs.map { case (name, p) => (name, p.get) }\n+      val duration = (System.currentTimeMillis - start) / 1000.0\n+      logger.info(f\"Ran pipeline in $duration%.3f s\")\n+      result\n+    } catch {\n+      case NonFatal(e) =>\n+        logger.error(\"Untrapped exception\", e)\n+        List()\n+    }\n+\n+    val title = rawTitle.replaceAll(\"\"\"\\s+\"\"\", \"-\")\n+    val today = new SimpleDateFormat(\"yyyy-MM-dd-HH-mm-ss\").format(new Date())\n+\n+    val workflowArtifact = createOutputArtifact[FlatArtifact](s\"summary/$title-$today.workflow.json\")\n+    val workflow = Workflow.forPipeline(targets.toMap)\n+    SingletonIo.json[Workflow].write(workflow, workflowArtifact)\n+\n+    val htmlArtifact = createOutputArtifact[FlatArtifact](s\"summary/$title-$today.html\")\n+    SingletonIo.text[String].write(workflow.renderHtml, htmlArtifact)\n+\n+    val signatureArtifact = createOutputArtifact[FlatArtifact](s\"summary/$title-$today.signatures.json\")\n+    val signatureFormat = Signature.jsonWriter\n+    val signatures = targets.map { case (s, p) => signatureFormat.write(p.stepInfo.signature) }.toList.toJson\n+    signatureArtifact.write { writer => writer.write(signatures.prettyPrint) }\n+\n+    logger.info(s\"Summary written to ${toHttpUrl(htmlArtifact.url)}\")\n+    result\n+  }\n+\n+  // Generate a hash unique to this Producer/Serialization combination\n+  protected def hashId[T, A <: Artifact](\n+    p: Producer[T],\n+    io: Serializer[T, A] with Deserializer[T, A]\n+  ) =\n+    p.stepInfo.copy(\n+      dependencies = p.stepInfo.dependencies + (\"io\" -> io)\n+    ).signature.id\n+\n+  // Convert S3 URLs to an http: URL viewable in a browser\n+  def toHttpUrl(url: URI): URI = {\n+    url.getScheme match {\n+      case \"s3\" | \"s3n\" =>\n+        new java.net.URI(\"http\", s\"${\n+          url.getHost\n+        }.s3.amazonaws.com\", url.getPath, null)\n+      case \"file\" =>\n+        new java.net.URI(null, null, url.getPath, null)\n+      case _ => url\n+    }\n+  }\n+\n+  def dryRun(outputDir: File, rawTitle: String, targets: Iterable[(String, Producer[_])] = persistedSteps.toList): Unit = {\n+    val title = s\"${\n+      rawTitle.replaceAll(\"\"\"\\s+\"\"\", \"-\")\n+    }-dryRun\"\n+    val workflowArtifact = new FileArtifact(new File(outputDir, s\"$title.workflow.json\"))\n+    val workflow = Workflow.forPipeline(targets)\n+    SingletonIo.json[Workflow].write(workflow, workflowArtifact)\n+\n+    val htmlArtifact = new FileArtifact(new File(outputDir, s\"$title.html\"))\n+    SingletonIo.text[String].write(workflow.renderHtml, htmlArtifact)\n+\n+    val signatureArtifact = new FileArtifact(new File(outputDir, s\"$title.signatures.json\"))\n+    val signatureFormat = Signature.jsonWriter\n+    val signatures = targets.map { case (s, p) => signatureFormat.write(p.stepInfo.signature) }.toJson\n+    signatureArtifact.write {\n+      writer => writer.write(signatures.prettyPrint)\n+    }\n+\n+    logger.info(s\"Summary written to $outputDir\")\n+  }\n+\n+  protected[this] val steps =\n+    scala.collection.mutable.Map.empty[String, PersistedProducer[_, _ <: Artifact]]\n+}\n+\n+object Pipeline {\n+  // Create a Pipeline that writes output to the given directory\n+  def apply(rootDir: File) =\n+    new Pipeline {\n+      def rootOutputUrl = rootDir.toURI\n+    }\n+\n+  def configured(cfg: Config) =\n+    new ConfiguredPipeline {\n+      val config = cfg\n+    }\n+}\n+\n+trait ConfiguredPipeline extends Pipeline {\n+  val config: Config\n+\n+  override def rootOutputUrl =\n+    config.get[String](\"output.dir\").map(s => new URI(s))\n+      .getOrElse(new File(System.getProperty(\"user.dir\")).toURI)\n+\n+  protected[this] def getStringList(key: String) =\n+    config.get[String](key) match {\n+      case Some(s) => List(s)\n+      case None => config.get[Seq[String]](key) match {\n+        case Some(sList) => sList\n+        case None => List()\n+      }\n+    }\n+\n+  override def run(rawTitle: String) = {\n+    val (targets, isRunOnly) =\n+      getStringList(\"runOnly\") match {\n+        case seq if seq.nonEmpty =>\n+          (getStepsByName(seq), true)\n+        case _ =>\n+          getStringList(\"runUntil\") match {\n+            case seq if seq.nonEmpty =>\n+              (getStepsByName(seq), false)\n+            case _ =>\n+              (persistedSteps.toList, false)\n+          }\n+      }\n+\n+    config.get[Boolean](\"dryRun\") match {\n+      case Some(true) =>\n+        val outputDir = config.get[String](\"dryRunOutput\")\n+          .getOrElse(System.getProperty(\"user.dir\"))\n+        dryRun(new File(outputDir), rawTitle, targets)\n+        List()\n+      case _ =>\n+        if (isRunOnly) {\n+          runOnly(rawTitle, targets)\n+        } else {\n+          runPipelineReturnResults(rawTitle, targets)\n+        }\n+    }\n+  }\n+\n+  override def persist[T, A <: Artifact: ClassTag](","path":"core/src/main/scala/org/allenai/pipeline/Pipeline.scala","position":337,"original_position":337,"commit_id":"e63ea47f47c4e8a5153ad3e018ace229a86884f2","original_commit_id":"e63ea47f47c4e8a5153ad3e018ace229a86884f2","user":{"login":"rodneykinney","id":3767011,"avatar_url":"https://avatars.githubusercontent.com/u/3767011?v=3","gravatar_id":"","url":"https://api.github.com/users/rodneykinney","html_url":"https://github.com/rodneykinney","followers_url":"https://api.github.com/users/rodneykinney/followers","following_url":"https://api.github.com/users/rodneykinney/following{/other_user}","gists_url":"https://api.github.com/users/rodneykinney/gists{/gist_id}","starred_url":"https://api.github.com/users/rodneykinney/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rodneykinney/subscriptions","organizations_url":"https://api.github.com/users/rodneykinney/orgs","repos_url":"https://api.github.com/users/rodneykinney/repos","events_url":"https://api.github.com/users/rodneykinney/events{/privacy}","received_events_url":"https://api.github.com/users/rodneykinney/received_events","type":"User","site_admin":false},"body":"As per email thread:\r\n\r\n- Remove optionallyPersist() method, replace with persist()\r\n- Behavior of persist() is to check config for an overridden output location\r\n- If no config is found, behavior is to persist in the default location\r\n- No longer necessary to set output.persist.XXX=true to enable persistence\r\n- Can still disable persistence by setting output.persist.XXX=false in the config","created_at":"2015-06-02T16:35:59Z","updated_at":"2015-06-02T16:35:59Z","html_url":"https://github.com/allenai/pipeline/pull/31#discussion_r31542564","pull_request_url":"https://api.github.com/repos/allenai/pipeline/pulls/31","_links":{"self":{"href":"https://api.github.com/repos/allenai/pipeline/pulls/comments/31542564"},"html":{"href":"https://github.com/allenai/pipeline/pull/31#discussion_r31542564"},"pull_request":{"href":"https://api.github.com/repos/allenai/pipeline/pulls/31"}}},"pull_request":{"url":"https://api.github.com/repos/allenai/pipeline/pulls/31","id":36744907,"html_url":"https://github.com/allenai/pipeline/pull/31","diff_url":"https://github.com/allenai/pipeline/pull/31.diff","patch_url":"https://github.com/allenai/pipeline/pull/31.patch","issue_url":"https://api.github.com/repos/allenai/pipeline/issues/31","number":31,"state":"open","locked":false,"title":"Refactor into sub-projects","user":{"login":"rodneykinney","id":3767011,"avatar_url":"https://avatars.githubusercontent.com/u/3767011?v=3","gravatar_id":"","url":"https://api.github.com/users/rodneykinney","html_url":"https://github.com/rodneykinney","followers_url":"https://api.github.com/users/rodneykinney/followers","following_url":"https://api.github.com/users/rodneykinney/following{/other_user}","gists_url":"https://api.github.com/users/rodneykinney/gists{/gist_id}","starred_url":"https://api.github.com/users/rodneykinney/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rodneykinney/subscriptions","organizations_url":"https://api.github.com/users/rodneykinney/orgs","repos_url":"https://api.github.com/users/rodneykinney/repos","events_url":"https://api.github.com/users/rodneykinney/events{/privacy}","received_events_url":"https://api.github.com/users/rodneykinney/received_events","type":"User","site_admin":false},"body":"Breaks up the pipeline project into the following sub-projects:\r\n\r\n - core (no cloud computing dependencies)\r\n - s3 (support for reading/writing to S3)\r\n - spark (support for running Spark-based pipelines)\r\n\r\nThe spark project contains much of the code from the scholar/offline project that was specific to Spark but not specific to S2.\r\n\r\nThe core project has been enhanced to support the naming of pipeline steps.","created_at":"2015-06-02T16:00:26Z","updated_at":"2015-06-02T16:35:59Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c38e9a0f792085fda3d9e4f2c0116e3123e47e02","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/allenai/pipeline/pulls/31/commits","review_comments_url":"https://api.github.com/repos/allenai/pipeline/pulls/31/comments","review_comment_url":"https://api.github.com/repos/allenai/pipeline/pulls/comments{/number}","comments_url":"https://api.github.com/repos/allenai/pipeline/issues/31/comments","statuses_url":"https://api.github.com/repos/allenai/pipeline/statuses/e63ea47f47c4e8a5153ad3e018ace229a86884f2","head":{"label":"rodneykinney:master","ref":"master","sha":"e63ea47f47c4e8a5153ad3e018ace229a86884f2","user":{"login":"rodneykinney","id":3767011,"avatar_url":"https://avatars.githubusercontent.com/u/3767011?v=3","gravatar_id":"","url":"https://api.github.com/users/rodneykinney","html_url":"https://github.com/rodneykinney","followers_url":"https://api.github.com/users/rodneykinney/followers","following_url":"https://api.github.com/users/rodneykinney/following{/other_user}","gists_url":"https://api.github.com/users/rodneykinney/gists{/gist_id}","starred_url":"https://api.github.com/users/rodneykinney/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rodneykinney/subscriptions","organizations_url":"https://api.github.com/users/rodneykinney/orgs","repos_url":"https://api.github.com/users/rodneykinney/repos","events_url":"https://api.github.com/users/rodneykinney/events{/privacy}","received_events_url":"https://api.github.com/users/rodneykinney/received_events","type":"User","site_admin":false},"repo":{"id":32085607,"name":"pipeline","full_name":"rodneykinney/pipeline","owner":{"login":"rodneykinney","id":3767011,"avatar_url":"https://avatars.githubusercontent.com/u/3767011?v=3","gravatar_id":"","url":"https://api.github.com/users/rodneykinney","html_url":"https://github.com/rodneykinney","followers_url":"https://api.github.com/users/rodneykinney/followers","following_url":"https://api.github.com/users/rodneykinney/following{/other_user}","gists_url":"https://api.github.com/users/rodneykinney/gists{/gist_id}","starred_url":"https://api.github.com/users/rodneykinney/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rodneykinney/subscriptions","organizations_url":"https://api.github.com/users/rodneykinney/orgs","repos_url":"https://api.github.com/users/rodneykinney/repos","events_url":"https://api.github.com/users/rodneykinney/events{/privacy}","received_events_url":"https://api.github.com/users/rodneykinney/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/rodneykinney/pipeline","description":"Library for building reproducible data pipelines to support experimentation","fork":true,"url":"https://api.github.com/repos/rodneykinney/pipeline","forks_url":"https://api.github.com/repos/rodneykinney/pipeline/forks","keys_url":"https://api.github.com/repos/rodneykinney/pipeline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rodneykinney/pipeline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rodneykinney/pipeline/teams","hooks_url":"https://api.github.com/repos/rodneykinney/pipeline/hooks","issue_events_url":"https://api.github.com/repos/rodneykinney/pipeline/issues/events{/number}","events_url":"https://api.github.com/repos/rodneykinney/pipeline/events","assignees_url":"https://api.github.com/repos/rodneykinney/pipeline/assignees{/user}","branches_url":"https://api.github.com/repos/rodneykinney/pipeline/branches{/branch}","tags_url":"https://api.github.com/repos/rodneykinney/pipeline/tags","blobs_url":"https://api.github.com/repos/rodneykinney/pipeline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rodneykinney/pipeline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rodneykinney/pipeline/git/refs{/sha}","trees_url":"https://api.github.com/repos/rodneykinney/pipeline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rodneykinney/pipeline/statuses/{sha}","languages_url":"https://api.github.com/repos/rodneykinney/pipeline/languages","stargazers_url":"https://api.github.com/repos/rodneykinney/pipeline/stargazers","contributors_url":"https://api.github.com/repos/rodneykinney/pipeline/contributors","subscribers_url":"https://api.github.com/repos/rodneykinney/pipeline/subscribers","subscription_url":"https://api.github.com/repos/rodneykinney/pipeline/subscription","commits_url":"https://api.github.com/repos/rodneykinney/pipeline/commits{/sha}","git_commits_url":"https://api.github.com/repos/rodneykinney/pipeline/git/commits{/sha}","comments_url":"https://api.github.com/repos/rodneykinney/pipeline/comments{/number}","issue_comment_url":"https://api.github.com/repos/rodneykinney/pipeline/issues/comments{/number}","contents_url":"https://api.github.com/repos/rodneykinney/pipeline/contents/{+path}","compare_url":"https://api.github.com/repos/rodneykinney/pipeline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rodneykinney/pipeline/merges","archive_url":"https://api.github.com/repos/rodneykinney/pipeline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rodneykinney/pipeline/downloads","issues_url":"https://api.github.com/repos/rodneykinney/pipeline/issues{/number}","pulls_url":"https://api.github.com/repos/rodneykinney/pipeline/pulls{/number}","milestones_url":"https://api.github.com/repos/rodneykinney/pipeline/milestones{/number}","notifications_url":"https://api.github.com/repos/rodneykinney/pipeline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rodneykinney/pipeline/labels{/name}","releases_url":"https://api.github.com/repos/rodneykinney/pipeline/releases{/id}","created_at":"2015-03-12T15:46:16Z","updated_at":"2015-03-30T21:54:04Z","pushed_at":"2015-06-02T15:47:36Z","git_url":"git://github.com/rodneykinney/pipeline.git","ssh_url":"git@github.com:rodneykinney/pipeline.git","clone_url":"https://github.com/rodneykinney/pipeline.git","svn_url":"https://github.com/rodneykinney/pipeline","homepage":"","size":828,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"allenai:master","ref":"master","sha":"53bc10a7d9d5b1ee125a6d688e476daed805d358","user":{"login":"allenai","id":5667695,"avatar_url":"https://avatars.githubusercontent.com/u/5667695?v=3","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"repo":{"id":31559978,"name":"pipeline","full_name":"allenai/pipeline","owner":{"login":"allenai","id":5667695,"avatar_url":"https://avatars.githubusercontent.com/u/5667695?v=3","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/allenai/pipeline","description":"Library for building reproducible data pipelines to support experimentation","fork":false,"url":"https://api.github.com/repos/allenai/pipeline","forks_url":"https://api.github.com/repos/allenai/pipeline/forks","keys_url":"https://api.github.com/repos/allenai/pipeline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/allenai/pipeline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/allenai/pipeline/teams","hooks_url":"https://api.github.com/repos/allenai/pipeline/hooks","issue_events_url":"https://api.github.com/repos/allenai/pipeline/issues/events{/number}","events_url":"https://api.github.com/repos/allenai/pipeline/events","assignees_url":"https://api.github.com/repos/allenai/pipeline/assignees{/user}","branches_url":"https://api.github.com/repos/allenai/pipeline/branches{/branch}","tags_url":"https://api.github.com/repos/allenai/pipeline/tags","blobs_url":"https://api.github.com/repos/allenai/pipeline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/allenai/pipeline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/allenai/pipeline/git/refs{/sha}","trees_url":"https://api.github.com/repos/allenai/pipeline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/allenai/pipeline/statuses/{sha}","languages_url":"https://api.github.com/repos/allenai/pipeline/languages","stargazers_url":"https://api.github.com/repos/allenai/pipeline/stargazers","contributors_url":"https://api.github.com/repos/allenai/pipeline/contributors","subscribers_url":"https://api.github.com/repos/allenai/pipeline/subscribers","subscription_url":"https://api.github.com/repos/allenai/pipeline/subscription","commits_url":"https://api.github.com/repos/allenai/pipeline/commits{/sha}","git_commits_url":"https://api.github.com/repos/allenai/pipeline/git/commits{/sha}","comments_url":"https://api.github.com/repos/allenai/pipeline/comments{/number}","issue_comment_url":"https://api.github.com/repos/allenai/pipeline/issues/comments{/number}","contents_url":"https://api.github.com/repos/allenai/pipeline/contents/{+path}","compare_url":"https://api.github.com/repos/allenai/pipeline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/allenai/pipeline/merges","archive_url":"https://api.github.com/repos/allenai/pipeline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/allenai/pipeline/downloads","issues_url":"https://api.github.com/repos/allenai/pipeline/issues{/number}","pulls_url":"https://api.github.com/repos/allenai/pipeline/pulls{/number}","milestones_url":"https://api.github.com/repos/allenai/pipeline/milestones{/number}","notifications_url":"https://api.github.com/repos/allenai/pipeline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/allenai/pipeline/labels{/name}","releases_url":"https://api.github.com/repos/allenai/pipeline/releases{/id}","created_at":"2015-03-02T19:57:02Z","updated_at":"2015-04-09T03:00:49Z","pushed_at":"2015-06-02T16:00:27Z","git_url":"git://github.com/allenai/pipeline.git","ssh_url":"git@github.com:allenai/pipeline.git","clone_url":"https://github.com/allenai/pipeline.git","svn_url":"https://github.com/allenai/pipeline","homepage":"","size":1304,"stargazers_count":2,"watchers_count":2,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":4,"forks":5,"open_issues":4,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/allenai/pipeline/pulls/31"},"html":{"href":"https://github.com/allenai/pipeline/pull/31"},"issue":{"href":"https://api.github.com/repos/allenai/pipeline/issues/31"},"comments":{"href":"https://api.github.com/repos/allenai/pipeline/issues/31/comments"},"review_comments":{"href":"https://api.github.com/repos/allenai/pipeline/pulls/31/comments"},"review_comment":{"href":"https://api.github.com/repos/allenai/pipeline/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/allenai/pipeline/pulls/31/commits"},"statuses":{"href":"https://api.github.com/repos/allenai/pipeline/statuses/e63ea47f47c4e8a5153ad3e018ace229a86884f2"}}}}