{"action":"created","comment":{"url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/comments/38489635","id":38489635,"diff_hunk":"@@ -0,0 +1,409 @@\n+/*\n+ * Copyright 2012 LinkedIn Corp.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n+ * use this file except in compliance with the License. You may obtain a copy of\n+ * the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n+ * License for the specific language governing permissions and limitations under\n+ * the License.\n+ */\n+\n+package azkaban.jobtype;\n+\n+import java.io.BufferedReader;\n+import java.io.File;\n+import java.io.FileNotFoundException;\n+import java.io.FileReader;\n+import java.io.FilenameFilter;\n+import java.io.IOException;\n+import java.lang.reflect.InvocationTargetException;\n+import java.lang.reflect.Method;\n+import java.util.HashSet;\n+import java.util.Set;\n+import java.util.regex.Matcher;\n+import java.util.regex.Pattern;\n+\n+import org.apache.hadoop.yarn.api.records.ApplicationId;\n+import org.apache.hadoop.yarn.client.api.YarnClient;\n+import org.apache.hadoop.yarn.conf.YarnConfiguration;\n+import org.apache.log4j.Logger;\n+\n+import azkaban.security.commons.HadoopSecurityManager;\n+import azkaban.security.commons.HadoopSecurityManagerException;\n+import azkaban.utils.Props;\n+\n+/**\n+ * There are many common methods that's required by the Hadoop*Job.java's. They are all consolidated\n+ * here.\n+ * \n+ * @see azkaban.jobtype.HadoopSparkJob\n+ * @see azkaban.jobtype.HadoopHiveJob\n+ * @see azkaban.jobtype.HadoopPigJob\n+ * @see azkaban.jobtype.HadoopJavaJob\n+ */\n+\n+public class HadoopJobUtils {\n+\n+  public static final String HADOOP_SECURITY_MANAGER_CLASS_PARAM = \"hadoop.security.manager.class\";\n+\n+  public static void cancelHadoopTokens(HadoopSecurityManager hadoopSecurityManager,\n+          String userToProxy, File tokenFile, Logger log) {\n+    try {\n+      hadoopSecurityManager.cancelTokens(tokenFile, userToProxy, log);\n+    } catch (HadoopSecurityManagerException e) {\n+      e.printStackTrace();\n+      log.error(e.getCause() + e.getMessage());\n+    } catch (Exception e) {\n+      e.printStackTrace();\n+      log.error(e.getCause() + e.getMessage());\n+    }\n+  }\n+\n+  /**\n+   * Based on the HADOOP_SECURITY_MANAGER_CLASS_PARAM setting in the incoming props, finds the\n+   * correct HadoopSecurityManager Java class\n+   * \n+   * @param props\n+   * @param log\n+   * @return a HadoopSecurityManager object. Will throw exception if any errors occur (including not\n+   *         finding a class)\n+   * @throws RuntimeException\n+   *           : If any errors happen along the way.\n+   */\n+  public static HadoopSecurityManager loadHadoopSecurityManager(Props props, Logger log)\n+          throws RuntimeException {\n+\n+    Class<?> hadoopSecurityManagerClass = props.getClass(HADOOP_SECURITY_MANAGER_CLASS_PARAM, true,\n+            HadoopJobUtils.class.getClassLoader());\n+    log.info(\"Loading hadoop security manager \" + hadoopSecurityManagerClass.getName());\n+    HadoopSecurityManager hadoopSecurityManager = null;\n+\n+    try {\n+      Method getInstanceMethod = hadoopSecurityManagerClass.getMethod(\"getInstance\", Props.class);\n+      hadoopSecurityManager = (HadoopSecurityManager) getInstanceMethod.invoke(\n+              hadoopSecurityManagerClass, props);\n+    } catch (InvocationTargetException e) {\n+      log.error(\"Could not instantiate Hadoop Security Manager \"\n+              + hadoopSecurityManagerClass.getName() + e.getCause());\n+      throw new RuntimeException(e.getCause());\n+    } catch (Exception e) {\n+      e.printStackTrace();\n+      throw new RuntimeException(e.getCause());\n+    }\n+\n+    return hadoopSecurityManager;\n+\n+  }\n+\n+  /**\n+   * Fetching token with the Azkaban user\n+   * \n+   * @param hadoopSecurityManager\n+   * @param props\n+   * @param log\n+   * @return\n+   * @throws HadoopSecurityManagerException\n+   */\n+  public static File getHadoopTokens(HadoopSecurityManager hadoopSecurityManager, Props props,\n+          Logger log) throws HadoopSecurityManagerException {\n+\n+    File tokenFile = null;\n+    try {\n+      tokenFile = File.createTempFile(\"mr-azkaban\", \".token\");\n+    } catch (Exception e) {\n+      e.printStackTrace();\n+      throw new HadoopSecurityManagerException(\"Failed to create the token file.\", e);\n+    }\n+\n+    hadoopSecurityManager.prefetchToken(tokenFile, props, log);\n+\n+    return tokenFile;\n+  }\n+\n+  /**\n+   * <pre>\n+   * If there's a * specification in the \"jar\" argument (e.g. jar=./lib/*,./lib2/*),\n+   * this method helps to resolve the * into actual jar names inside the folder, and in order.\n+   * This is due to the requirement that Spark 14 doesn't seem to do the resolution for users\n+   * \n+   * TODO: test case required: one replacement, two replacement.\n+   * </pre>\n+   * \n+   * @param unresolvedJarSpec\n+   * @return jar file list, comma separated, all .../* expanded into actual jar names in order\n+   * \n+   */\n+  public static String resolveWildCardForJarSpec(String workingDirectory, String unresolvedJarSpec,\n+          Logger log) {\n+\n+    log.info(\"resolveWildCardForJarSpec: unresolved jar specification: \" + unresolvedJarSpec);\n+    log.info(\"working directory: \" + workingDirectory);\n+\n+    if (unresolvedJarSpec == null || unresolvedJarSpec.isEmpty())\n+      return null;\n+\n+    StringBuilder resolvedJarSpec = new StringBuilder();\n+\n+    String[] unresolvedJarSpecList = unresolvedJarSpec.split(\",\");\n+    for (String s : unresolvedJarSpecList) {\n+      // if need resolution\n+      if (s.endsWith(\"*\")) {\n+        // remove last 2 characters to get to the folder\n+        String dirName = String.format(\"%s/%s\", workingDirectory, s.substring(0, s.length() - 2));\n+\n+        File[] jars = null;\n+        try {\n+          jars = getFilesInFolderByRegex(new File(dirName), \".*jar\");\n+        } catch (FileNotFoundException fnfe) {\n+          // if folder doesn't exist, just ignore\n+          continue;\n+        }\n+\n+        // if the folder is there, add them to the jar list\n+        for (File jar : jars) {\n+          resolvedJarSpec.append(jar.toString() + \",\");\n+        }\n+      } else { // no need for resolution\n+        resolvedJarSpec.append(s + \",\");\n+      }\n+    }\n+\n+    log.debug(\"resolveWildCardForJarSpec: resolvedJarSpec: \" + resolvedJarSpec);\n+\n+    // remove the trailing comma\n+    int lastCharIndex = resolvedJarSpec.length() - 1;\n+    if (lastCharIndex >= 0 && resolvedJarSpec.charAt(lastCharIndex) == ',') {\n+      resolvedJarSpec.deleteCharAt(lastCharIndex);\n+    }\n+\n+    return resolvedJarSpec.toString();\n+  }\n+\n+  /**\n+   * <pre>\n+   * This method looks for the proper user execution jar.  \n+   * The user input is expected in the following 2 formats:\n+   *   1. ./lib/abc\n+   *   2. ./lib/abc.jar\n+   * \n+   * This method will use prefix matching to find any jar that is the form of abc*.jar,\n+   * so that users can bump jar versions without doing modifications to their Hadoop DSL.\n+   * \n+   * This method will throw an Exception if more than one jar that matches the prefix is found\n+   * \n+   * @param workingDirectory\n+   * @param userSpecifiedJarName\n+   * @return the resolved actual jar file name to execute\n+   */\n+  public static String resolveExecutionJarName(String workingDirectory,\n+          String userSpecifiedJarName, Logger log) {\n+\n+    log.debug(\"Resolving execution jar name: working directory: \" + workingDirectory\n+            + \", user specified name: \" + userSpecifiedJarName);\n+\n+    // in case user decides to specify with abc.jar, instead of only abc\n+    if (userSpecifiedJarName.endsWith(\".jar\"))\n+      userSpecifiedJarName = userSpecifiedJarName.replace(\".jar\", \"\");\n+\n+    // can't use java 1.7 stuff, reverting to a slightly ugly implementation\n+    String userSpecifiedJarPath = String.format(\"%s/%s\", workingDirectory, userSpecifiedJarName);\n+    int lastIndexOfSlash = userSpecifiedJarPath.lastIndexOf(\"/\");\n+    final String jarPrefix = userSpecifiedJarPath.substring(lastIndexOfSlash + 1);\n+    final String dirName = userSpecifiedJarPath.substring(0, lastIndexOfSlash);\n+    log.debug(\"Resolving execution jar name: dirname: \" + dirName + \", jar name:  \" + jarPrefix);\n+\n+    File[] potentialExecutionJarList;\n+    try {\n+      potentialExecutionJarList = getFilesInFolderByRegex(new File(dirName), jarPrefix + \".*jar\");\n+    } catch (FileNotFoundException e) {\n+      throw new IllegalStateException(\n+              \"execution jar is suppose to be in this folder, but the folder doesn't exist: \"\n+                      + dirName);\n+    }\n+\n+    if (potentialExecutionJarList.length == 0) {\n+      throw new IllegalStateException(\"unable to find execution jar for Spark at path: \"\n+              + userSpecifiedJarPath + \"*.jar\");\n+    } else if (potentialExecutionJarList.length > 1) {\n+      throw new IllegalStateException(\n+              \"I find more than one matching instance of the execution jar at the path, don't know which one to use: \"\n+                      + userSpecifiedJarPath + \"*.jar\");\n+    }\n+\n+    String resolvedJarName = potentialExecutionJarList[0].toString();\n+    log.debug(\"Resolving execution jar name: resolvedJarName: \" + resolvedJarName);\n+    return resolvedJarName;\n+  }\n+\n+  /**\n+   * \n+   * @return a list of files in the given folder that matches the regex. It may be empty, but will\n+   *         never return a null\n+   * @throws FileNotFoundException\n+   */\n+  private static File[] getFilesInFolderByRegex(File folder, final String regex)\n+          throws FileNotFoundException {\n+    // sanity check\n+\n+    if (!folder.exists()) {\n+      throw new FileNotFoundException();\n+\n+    }\n+    if (!folder.isDirectory()) {\n+      throw new IllegalStateException(\n+              \"execution jar is suppose to be in this folder, but the object present is not a directory: \"\n+                      + folder);\n+    }\n+\n+    File[] matchingFiles = folder.listFiles(new FilenameFilter() {\n+      @Override\n+      public boolean accept(File dir, String name) {\n+        if (name.matches(regex))\n+          return true;\n+        else\n+          return false;\n+      }\n+    });\n+\n+    if (matchingFiles == null) {\n+      throw new IllegalStateException(\n+              \"the ls command in ResolveExecutionJarName threw an IOException\");\n+    }\n+\n+    return matchingFiles;\n+  }\n+\n+  /**\n+   * Pass in a log file, this method will find all the hadoop jobs it has launched, and kills it\n+   * \n+   * Only works with Hadoop2\n+   * \n+   * @param logFilePath\n+   * @param log\n+   */\n+  public static void killAllSpawnedHadoopJobs(String logFilePath, Logger log) {\n+    Set<String> allSpawnedJobs = findApplicationIdFromLog(logFilePath, log);\n+    log.info(\"applicationIds to kill: \" + allSpawnedJobs);\n+  \n+    for (String appId : allSpawnedJobs) {\n+      try {\n+        killJobOnCluster(appId, log);\n+      } catch (Throwable t) {\n+        log.warn(\"something happened while trying to kill this job: \" + appId, t);\n+      }\n+    }\n+  }\n+\n+  /**\n+   * <pre>\n+   * Takes in a log file, will grep every line to look for the application_id pattern.\n+   * If it finds multiple, it will return all of them, de-duped (this is possible in the case of pig jobs)\n+   * This can be used in conjunction with the @killJobOnCluster method in this file.\n+   * </pre>\n+   * \n+   * @param logFilePath\n+   * @return a Set. May be empty, but will never be null\n+   */\n+  public static Set<String> findApplicationIdFromLog(String logFilePath, Logger log) {\n+  \n+    File logFile = new File(logFilePath);\n+  \n+    if (!logFile.exists()) {\n+      throw new IllegalArgumentException(\"the logFilePath does not exist: \" + logFilePath);\n+    }\n+    if (!logFile.isFile()) {\n+      throw new IllegalArgumentException(\"the logFilePath specified  is not a valid file: \"\n+              + logFilePath);\n+    }\n+    if (!logFile.canRead()) {\n+      throw new IllegalArgumentException(\"unable to read the logFilePath specified: \" + logFilePath);\n+    }\n+  \n+    BufferedReader br = null;\n+    Set<String> applicationIds = new HashSet<String>();\n+    Pattern p = Pattern.compile(\".* (application_\\\\d+_\\\\d+).*\");\n+  \n+    try {\n+      br = new BufferedReader(new FileReader(logFile));\n+      String input;\n+  \n+      // finds all the application IDs\n+      while ((input = br.readLine()) != null) {\n+        Matcher m = p.matcher(input);\n+        if (m.find()) {\n+          String appId = m.group(1);\n+          if (!applicationIds.contains(appId)) {\n+            applicationIds.add(appId);\n+          }\n+        }\n+      }\n+    } catch (IOException e) {\n+      e.printStackTrace();\n+      log.info(\"Error while trying to find applicationId for Spark log\", e);\n+    } finally {\n+      try {\n+        if (br != null)\n+          br.close();\n+      } catch (Exception e) {\n+        // do nothing\n+      }\n+    }\n+    return applicationIds;\n+  }\n+\n+  /**\n+   * <pre>\n+   * Uses YarnClient to kill the job on HDFS.\n+   * Using JobClient only works partially:\n+   *   If yarn container has started but spark job haven't, it will kill\n+   *   If spark job has started, the cancel will hang until the spark job is complete\n+   *   If the spark job is complete, it will return immediately, with a job not found on job tracker\n+   * </pre>\n+   * \n+   * @param applicationId\n+   */\n+  public static void killJobOnCluster(String applicationId, Logger log) {\n+    try {\n+      YarnConfiguration yarnConf = new YarnConfiguration();\n+      YarnClient yarnClient = YarnClient.createYarnClient();\n+      yarnClient.init(yarnConf);\n+      yarnClient.start();\n+\n+      String[] split = applicationId.split(\"_\");\n+      ApplicationId aid = ApplicationId.newInstance(Long.parseLong(split[1]),\n+              Integer.parseInt(split[2]));\n+\n+      log.info(\"start klling application: \" + aid);\n+      yarnClient.killApplication(aid);\n+      log.info(\"successfully killed application: \" + aid);\n+    } catch (Exception e) {\n+      log.error(\"Failure while try to kill the hadoop job.  Will skip and continue\", e);","path":"plugins/jobtype/src/azkaban/jobtype/HadoopJobUtils.java","position":386,"original_position":386,"commit_id":"212f8238167e424e949f40d39e48ea6b77270cd4","original_commit_id":"212f8238167e424e949f40d39e48ea6b77270cd4","user":{"login":"johnyu0520","id":12088335,"avatar_url":"https://avatars.githubusercontent.com/u/12088335?v=3","gravatar_id":"","url":"https://api.github.com/users/johnyu0520","html_url":"https://github.com/johnyu0520","followers_url":"https://api.github.com/users/johnyu0520/followers","following_url":"https://api.github.com/users/johnyu0520/following{/other_user}","gists_url":"https://api.github.com/users/johnyu0520/gists{/gist_id}","starred_url":"https://api.github.com/users/johnyu0520/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnyu0520/subscriptions","organizations_url":"https://api.github.com/users/johnyu0520/orgs","repos_url":"https://api.github.com/users/johnyu0520/repos","events_url":"https://api.github.com/users/johnyu0520/events{/privacy}","received_events_url":"https://api.github.com/users/johnyu0520/received_events","type":"User","site_admin":false},"body":"I was debating what happens if the job that is suppose to kill a job fails.\r\nIgnoring it will make it like current behavior\r\nFailing a killed job sounds kind of weird to me\r\n\r\nThoughts?","created_at":"2015-09-02T00:44:30Z","updated_at":"2015-09-02T00:44:30Z","html_url":"https://github.com/azkaban/azkaban-plugins/pull/195#discussion_r38489635","pull_request_url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/195","_links":{"self":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/comments/38489635"},"html":{"href":"https://github.com/azkaban/azkaban-plugins/pull/195#discussion_r38489635"},"pull_request":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/195"}}},"pull_request":{"url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/195","id":41741679,"html_url":"https://github.com/azkaban/azkaban-plugins/pull/195","diff_url":"https://github.com/azkaban/azkaban-plugins/pull/195.diff","patch_url":"https://github.com/azkaban/azkaban-plugins/pull/195.patch","issue_url":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/195","number":195,"state":"open","locked":false,"title":"Azkaban Spark job type plugin","user":{"login":"johnyu0520","id":12088335,"avatar_url":"https://avatars.githubusercontent.com/u/12088335?v=3","gravatar_id":"","url":"https://api.github.com/users/johnyu0520","html_url":"https://github.com/johnyu0520","followers_url":"https://api.github.com/users/johnyu0520/followers","following_url":"https://api.github.com/users/johnyu0520/following{/other_user}","gists_url":"https://api.github.com/users/johnyu0520/gists{/gist_id}","starred_url":"https://api.github.com/users/johnyu0520/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnyu0520/subscriptions","organizations_url":"https://api.github.com/users/johnyu0520/orgs","repos_url":"https://api.github.com/users/johnyu0520/repos","events_url":"https://api.github.com/users/johnyu0520/events{/privacy}","received_events_url":"https://api.github.com/users/johnyu0520/received_events","type":"User","site_admin":false},"body":"initial commit for Azkaban Spark job type.  Currently works for David Leung's example-spark-azkaban-report","created_at":"2015-08-06T00:36:29Z","updated_at":"2015-09-02T00:44:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"eaa832b72841f2f5b8dedcd0152593564c88e707","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/195/commits","review_comments_url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/195/comments","review_comment_url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/comments{/number}","comments_url":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/195/comments","statuses_url":"https://api.github.com/repos/azkaban/azkaban-plugins/statuses/212f8238167e424e949f40d39e48ea6b77270cd4","head":{"label":"johnyu0520:master","ref":"master","sha":"212f8238167e424e949f40d39e48ea6b77270cd4","user":{"login":"johnyu0520","id":12088335,"avatar_url":"https://avatars.githubusercontent.com/u/12088335?v=3","gravatar_id":"","url":"https://api.github.com/users/johnyu0520","html_url":"https://github.com/johnyu0520","followers_url":"https://api.github.com/users/johnyu0520/followers","following_url":"https://api.github.com/users/johnyu0520/following{/other_user}","gists_url":"https://api.github.com/users/johnyu0520/gists{/gist_id}","starred_url":"https://api.github.com/users/johnyu0520/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnyu0520/subscriptions","organizations_url":"https://api.github.com/users/johnyu0520/orgs","repos_url":"https://api.github.com/users/johnyu0520/repos","events_url":"https://api.github.com/users/johnyu0520/events{/privacy}","received_events_url":"https://api.github.com/users/johnyu0520/received_events","type":"User","site_admin":false},"repo":{"id":39963122,"name":"azkaban-plugins","full_name":"johnyu0520/azkaban-plugins","owner":{"login":"johnyu0520","id":12088335,"avatar_url":"https://avatars.githubusercontent.com/u/12088335?v=3","gravatar_id":"","url":"https://api.github.com/users/johnyu0520","html_url":"https://github.com/johnyu0520","followers_url":"https://api.github.com/users/johnyu0520/followers","following_url":"https://api.github.com/users/johnyu0520/following{/other_user}","gists_url":"https://api.github.com/users/johnyu0520/gists{/gist_id}","starred_url":"https://api.github.com/users/johnyu0520/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnyu0520/subscriptions","organizations_url":"https://api.github.com/users/johnyu0520/orgs","repos_url":"https://api.github.com/users/johnyu0520/repos","events_url":"https://api.github.com/users/johnyu0520/events{/privacy}","received_events_url":"https://api.github.com/users/johnyu0520/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/johnyu0520/azkaban-plugins","description":"Plugins for Azkaban.","fork":true,"url":"https://api.github.com/repos/johnyu0520/azkaban-plugins","forks_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/forks","keys_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/keys{/key_id}","collaborators_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/teams","hooks_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/hooks","issue_events_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/issues/events{/number}","events_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/events","assignees_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/assignees{/user}","branches_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/branches{/branch}","tags_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/tags","blobs_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/git/refs{/sha}","trees_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/git/trees{/sha}","statuses_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/statuses/{sha}","languages_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/languages","stargazers_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/stargazers","contributors_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/contributors","subscribers_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/subscribers","subscription_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/subscription","commits_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/commits{/sha}","git_commits_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/git/commits{/sha}","comments_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/comments{/number}","issue_comment_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/issues/comments{/number}","contents_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/contents/{+path}","compare_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/compare/{base}...{head}","merges_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/merges","archive_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/downloads","issues_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/issues{/number}","pulls_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/pulls{/number}","milestones_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/milestones{/number}","notifications_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/labels{/name}","releases_url":"https://api.github.com/repos/johnyu0520/azkaban-plugins/releases{/id}","created_at":"2015-07-30T17:24:05Z","updated_at":"2015-07-30T17:24:07Z","pushed_at":"2015-08-29T00:08:34Z","git_url":"git://github.com/johnyu0520/azkaban-plugins.git","ssh_url":"git@github.com:johnyu0520/azkaban-plugins.git","clone_url":"https://github.com/johnyu0520/azkaban-plugins.git","svn_url":"https://github.com/johnyu0520/azkaban-plugins","homepage":"https://azkaban.github.io","size":146639,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"azkaban:master","ref":"master","sha":"f802cd615085e63c6de4853518f9a8108902f129","user":{"login":"azkaban","id":213618,"avatar_url":"https://avatars.githubusercontent.com/u/213618?v=3","gravatar_id":"","url":"https://api.github.com/users/azkaban","html_url":"https://github.com/azkaban","followers_url":"https://api.github.com/users/azkaban/followers","following_url":"https://api.github.com/users/azkaban/following{/other_user}","gists_url":"https://api.github.com/users/azkaban/gists{/gist_id}","starred_url":"https://api.github.com/users/azkaban/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/azkaban/subscriptions","organizations_url":"https://api.github.com/users/azkaban/orgs","repos_url":"https://api.github.com/users/azkaban/repos","events_url":"https://api.github.com/users/azkaban/events{/privacy}","received_events_url":"https://api.github.com/users/azkaban/received_events","type":"Organization","site_admin":false},"repo":{"id":7566859,"name":"azkaban-plugins","full_name":"azkaban/azkaban-plugins","owner":{"login":"azkaban","id":213618,"avatar_url":"https://avatars.githubusercontent.com/u/213618?v=3","gravatar_id":"","url":"https://api.github.com/users/azkaban","html_url":"https://github.com/azkaban","followers_url":"https://api.github.com/users/azkaban/followers","following_url":"https://api.github.com/users/azkaban/following{/other_user}","gists_url":"https://api.github.com/users/azkaban/gists{/gist_id}","starred_url":"https://api.github.com/users/azkaban/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/azkaban/subscriptions","organizations_url":"https://api.github.com/users/azkaban/orgs","repos_url":"https://api.github.com/users/azkaban/repos","events_url":"https://api.github.com/users/azkaban/events{/privacy}","received_events_url":"https://api.github.com/users/azkaban/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/azkaban/azkaban-plugins","description":"Plugins for Azkaban.","fork":false,"url":"https://api.github.com/repos/azkaban/azkaban-plugins","forks_url":"https://api.github.com/repos/azkaban/azkaban-plugins/forks","keys_url":"https://api.github.com/repos/azkaban/azkaban-plugins/keys{/key_id}","collaborators_url":"https://api.github.com/repos/azkaban/azkaban-plugins/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/azkaban/azkaban-plugins/teams","hooks_url":"https://api.github.com/repos/azkaban/azkaban-plugins/hooks","issue_events_url":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/events{/number}","events_url":"https://api.github.com/repos/azkaban/azkaban-plugins/events","assignees_url":"https://api.github.com/repos/azkaban/azkaban-plugins/assignees{/user}","branches_url":"https://api.github.com/repos/azkaban/azkaban-plugins/branches{/branch}","tags_url":"https://api.github.com/repos/azkaban/azkaban-plugins/tags","blobs_url":"https://api.github.com/repos/azkaban/azkaban-plugins/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/azkaban/azkaban-plugins/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/azkaban/azkaban-plugins/git/refs{/sha}","trees_url":"https://api.github.com/repos/azkaban/azkaban-plugins/git/trees{/sha}","statuses_url":"https://api.github.com/repos/azkaban/azkaban-plugins/statuses/{sha}","languages_url":"https://api.github.com/repos/azkaban/azkaban-plugins/languages","stargazers_url":"https://api.github.com/repos/azkaban/azkaban-plugins/stargazers","contributors_url":"https://api.github.com/repos/azkaban/azkaban-plugins/contributors","subscribers_url":"https://api.github.com/repos/azkaban/azkaban-plugins/subscribers","subscription_url":"https://api.github.com/repos/azkaban/azkaban-plugins/subscription","commits_url":"https://api.github.com/repos/azkaban/azkaban-plugins/commits{/sha}","git_commits_url":"https://api.github.com/repos/azkaban/azkaban-plugins/git/commits{/sha}","comments_url":"https://api.github.com/repos/azkaban/azkaban-plugins/comments{/number}","issue_comment_url":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/comments{/number}","contents_url":"https://api.github.com/repos/azkaban/azkaban-plugins/contents/{+path}","compare_url":"https://api.github.com/repos/azkaban/azkaban-plugins/compare/{base}...{head}","merges_url":"https://api.github.com/repos/azkaban/azkaban-plugins/merges","archive_url":"https://api.github.com/repos/azkaban/azkaban-plugins/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/azkaban/azkaban-plugins/downloads","issues_url":"https://api.github.com/repos/azkaban/azkaban-plugins/issues{/number}","pulls_url":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls{/number}","milestones_url":"https://api.github.com/repos/azkaban/azkaban-plugins/milestones{/number}","notifications_url":"https://api.github.com/repos/azkaban/azkaban-plugins/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/azkaban/azkaban-plugins/labels{/name}","releases_url":"https://api.github.com/repos/azkaban/azkaban-plugins/releases{/id}","created_at":"2013-01-11T20:53:36Z","updated_at":"2015-07-13T01:49:15Z","pushed_at":"2015-08-29T00:08:35Z","git_url":"git://github.com/azkaban/azkaban-plugins.git","ssh_url":"git@github.com:azkaban/azkaban-plugins.git","clone_url":"https://github.com/azkaban/azkaban-plugins.git","svn_url":"https://github.com/azkaban/azkaban-plugins","homepage":"https://azkaban.github.io","size":221786,"stargazers_count":29,"watchers_count":29,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":64,"mirror_url":null,"open_issues_count":19,"forks":64,"open_issues":19,"watchers":29,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/195"},"html":{"href":"https://github.com/azkaban/azkaban-plugins/pull/195"},"issue":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/195"},"comments":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/issues/195/comments"},"review_comments":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/195/comments"},"review_comment":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/pulls/195/commits"},"statuses":{"href":"https://api.github.com/repos/azkaban/azkaban-plugins/statuses/212f8238167e424e949f40d39e48ea6b77270cd4"}}}}