{"action":"created","comment":{"url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/comments/33602373","id":33602373,"diff_hunk":"@@ -42,33 +42,34 @@\n      data may linger in the database. We periodically sweep the\n      database, compacting it and performing regular cleanup so we can\n      maintain acceptable performance.\"\n-  (:require [puppetlabs.puppetdb.scf.storage :as scf-store]\n-            [puppetlabs.puppetdb.command.dlo :as dlo]\n-            [puppetlabs.puppetdb.query.population :as pop]\n-            [puppetlabs.puppetdb.jdbc :as pl-jdbc]\n-            [puppetlabs.puppetdb.mq :as mq]\n+  (:require [clj-time.core :refer [ago]]\n+            [clojure.java.io :as io]\n             [clojure.java.jdbc :as sql]\n             [clojure.tools.logging :as log]\n-            [puppetlabs.puppetdb.http.server :as server]\n-            [puppetlabs.puppetdb.config :as conf]\n-            [puppetlabs.kitchensink.core :as kitchensink]\n-            [robert.hooke :as rh]\n-            [puppetlabs.trapperkeeper.core :refer [defservice] :as tk]\n-            [puppetlabs.trapperkeeper.services :refer [service-id service-context]]\n             [compojure.core :as compojure]\n-            [clojure.java.io :refer [file]]\n-            [clj-time.core :refer [ago]]\n+            [me.raynes.fs :as fs]\n             [overtone.at-at :refer [mk-pool interspaced]]\n-            [slingshot.slingshot :refer [throw+ try+]]\n-            [puppetlabs.puppetdb.time :refer [to-seconds to-millis parse-period\n-                                              format-period period?]]\n+            [puppetlabs.kitchensink.core :as kitchensink]\n+            [puppetlabs.puppetdb.cheshire :as json]\n+            [puppetlabs.puppetdb.command.constants :refer [command-names]]\n+            [puppetlabs.puppetdb.command.dlo :as dlo]\n+            [puppetlabs.puppetdb.config :as conf]\n+            [puppetlabs.puppetdb.http.server :as server]\n+            [puppetlabs.puppetdb.jdbc :as pl-jdbc]\n             [puppetlabs.puppetdb.jdbc :refer [with-transacted-connection]]\n-            [puppetlabs.puppetdb.scf.migrate :refer [migrate! indexes!]]\n             [puppetlabs.puppetdb.meta.version :refer [version update-info]]\n-            [puppetlabs.puppetdb.command.constants :refer [command-names]]\n-            [puppetlabs.puppetdb.cheshire :as json]\n+            [puppetlabs.puppetdb.mq :as mq]\n             [puppetlabs.puppetdb.query-eng :as qeng]\n-            [puppetlabs.puppetdb.utils :as utils])\n+            [puppetlabs.puppetdb.query.population :as pop]\n+            [puppetlabs.puppetdb.scf.migrate :refer [migrate! indexes!]]\n+            [puppetlabs.puppetdb.scf.storage :as scf-store]\n+            [puppetlabs.puppetdb.time :refer [to-seconds to-millis parse-period\n+                                              format-period period?]]\n+            [puppetlabs.puppetdb.utils :as utils]\n+            [puppetlabs.trapperkeeper.core :refer [defservice] :as tk]\n+            [puppetlabs.trapperkeeper.services :refer [service-id service-context]]\n+            [robert.hooke :as rh]\n+            [slingshot.slingshot :refer [throw+ try+]])","path":"src/puppetlabs/puppetdb/cli/services.clj","position":51,"original_position":51,"commit_id":"6e0ed7c17a57bdce84fa178731e6dcac3d8be64c","original_commit_id":"6e0ed7c17a57bdce84fa178731e6dcac3d8be64c","user":{"login":"ajroetker","id":3198124,"avatar_url":"https://avatars.githubusercontent.com/u/3198124?v=3","gravatar_id":"","url":"https://api.github.com/users/ajroetker","html_url":"https://github.com/ajroetker","followers_url":"https://api.github.com/users/ajroetker/followers","following_url":"https://api.github.com/users/ajroetker/following{/other_user}","gists_url":"https://api.github.com/users/ajroetker/gists{/gist_id}","starred_url":"https://api.github.com/users/ajroetker/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ajroetker/subscriptions","organizations_url":"https://api.github.com/users/ajroetker/orgs","repos_url":"https://api.github.com/users/ajroetker/repos","events_url":"https://api.github.com/users/ajroetker/events{/privacy}","received_events_url":"https://api.github.com/users/ajroetker/received_events","type":"User","site_admin":false},"body":"The only changes were `clojure.java.io :as io` and requiring the `me.raynes.fs :as fs`","created_at":"2015-06-30T17:52:15Z","updated_at":"2015-06-30T17:52:15Z","html_url":"https://github.com/puppetlabs/puppetdb/pull/1490#discussion_r33602373","pull_request_url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/1490","_links":{"self":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/comments/33602373"},"html":{"href":"https://github.com/puppetlabs/puppetdb/pull/1490#discussion_r33602373"},"pull_request":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/1490"}}},"pull_request":{"url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/1490","id":38500961,"html_url":"https://github.com/puppetlabs/puppetdb/pull/1490","diff_url":"https://github.com/puppetlabs/puppetdb/pull/1490.diff","patch_url":"https://github.com/puppetlabs/puppetdb/pull/1490.patch","issue_url":"https://api.github.com/repos/puppetlabs/puppetdb/issues/1490","number":1490,"state":"open","locked":false,"title":"(PDB-1631) master/add import and export endpoints","user":{"login":"ajroetker","id":3198124,"avatar_url":"https://avatars.githubusercontent.com/u/3198124?v=3","gravatar_id":"","url":"https://api.github.com/users/ajroetker","html_url":"https://github.com/ajroetker","followers_url":"https://api.github.com/users/ajroetker/followers","following_url":"https://api.github.com/users/ajroetker/following{/other_user}","gists_url":"https://api.github.com/users/ajroetker/gists{/gist_id}","starred_url":"https://api.github.com/users/ajroetker/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ajroetker/subscriptions","organizations_url":"https://api.github.com/users/ajroetker/orgs","repos_url":"https://api.github.com/users/ajroetker/repos","events_url":"https://api.github.com/users/ajroetker/events{/privacy}","received_events_url":"https://api.github.com/users/ajroetker/received_events","type":"User","site_admin":false},"body":"This PR adds a `puppetlabs.puppetdb.helpers` namespace.\r\nThe `helpers` namespace contains a trapperkeeper service which serves a handler of the new import/export endpoints.\r\n\r\nThis PR also creates a new subdirectory of the `vardir`, namely `backups`. This directory holds all the PuppetDB backup archives and is essentially the root directory of the `import` and `export` endpoints.\r\n\r\nThe new endpoints are served under the `/pdb/cli/v1` api root prefix.\r\n\r\nThe `export` endpoint will either stream you the backup tarball `GET`, or will trigger an export to the local `backups` directory `PUT`.\r\nThe streaming `GET` version takes an `archive` param which is useful for things like a browser, where the value supplied to the param will be the name of the file downloaded, for `curl` or Clojure calls to the endpoint, the param isn't necessary.\r\nThe local `PUT` version takes a param (`archive`) which is the name of the local backup tarball file you wish to make (in the future, this might be a timestamped default).\r\n\r\nThe import endpoints are `upload` and `import`.\r\nThe `upload` endpoint takes a multipart header (`curl ... -F \"archive=@foo_backup\"`) one of the params should be the `archive` file which you wish to upload to the PuppetDB backups directory.\r\nThe `import` endpoint takes a single `archive` param specifies which archive in the `backups` dir to import into PuppetDB, and executes the import.\r\n\r\nThe CLI tools (import/export) functionality has been moved to these endpoints and the tools themselves essentially just make calls to these endpoints. Import is an upload, followed by an import.\r\n\r\nTODO and followups for this PR:\r\nMove the import/export logic out of the `cli` namespaces, maybe their own namespaces.\r\nAs to not be overreaching, this PR tried to allow the old import/export code to live next to the new code, as such there are a lot of `partial` floating around and anonymous functions be passed through the new/old code, this should be moved to just assume in-process command-submission and querying.\r\nMore validation around the compojure app.\r\nMore granular testing around the endpoints would be nice.\r\nAdd a benchmarking endpoint for data-loading, there is code duplication with the import tool there.\r\nDocumentation for the new endpoints.\r\n","created_at":"2015-06-25T00:10:49Z","updated_at":"2015-06-30T17:52:15Z","closed_at":null,"merged_at":null,"merge_commit_sha":"223b1fa47e0f5cb60a22a8fc1d0a2d3c4dda1f97","assignee":null,"milestone":{"url":"https://api.github.com/repos/puppetlabs/puppetdb/milestones/1","html_url":"https://github.com/puppetlabs/puppetdb/milestones/3.1.0%20(master)","labels_url":"https://api.github.com/repos/puppetlabs/puppetdb/milestones/1/labels","id":1187036,"number":1,"title":"3.1.0 (master)","description":"","creator":{"login":"kbarber","id":80595,"avatar_url":"https://avatars.githubusercontent.com/u/80595?v=3","gravatar_id":"","url":"https://api.github.com/users/kbarber","html_url":"https://github.com/kbarber","followers_url":"https://api.github.com/users/kbarber/followers","following_url":"https://api.github.com/users/kbarber/following{/other_user}","gists_url":"https://api.github.com/users/kbarber/gists{/gist_id}","starred_url":"https://api.github.com/users/kbarber/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kbarber/subscriptions","organizations_url":"https://api.github.com/users/kbarber/orgs","repos_url":"https://api.github.com/users/kbarber/repos","events_url":"https://api.github.com/users/kbarber/events{/privacy}","received_events_url":"https://api.github.com/users/kbarber/received_events","type":"User","site_admin":false},"open_issues":5,"closed_issues":0,"state":"open","created_at":"2015-06-30T12:56:42Z","updated_at":"2015-06-30T16:45:13Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/1490/commits","review_comments_url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/1490/comments","review_comment_url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/puppetlabs/puppetdb/issues/1490/comments","statuses_url":"https://api.github.com/repos/puppetlabs/puppetdb/statuses/6e0ed7c17a57bdce84fa178731e6dcac3d8be64c","head":{"label":"ajroetker:ticket/pdb-1631/master/add_import_export_endpoints","ref":"ticket/pdb-1631/master/add_import_export_endpoints","sha":"6e0ed7c17a57bdce84fa178731e6dcac3d8be64c","user":{"login":"ajroetker","id":3198124,"avatar_url":"https://avatars.githubusercontent.com/u/3198124?v=3","gravatar_id":"","url":"https://api.github.com/users/ajroetker","html_url":"https://github.com/ajroetker","followers_url":"https://api.github.com/users/ajroetker/followers","following_url":"https://api.github.com/users/ajroetker/following{/other_user}","gists_url":"https://api.github.com/users/ajroetker/gists{/gist_id}","starred_url":"https://api.github.com/users/ajroetker/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ajroetker/subscriptions","organizations_url":"https://api.github.com/users/ajroetker/orgs","repos_url":"https://api.github.com/users/ajroetker/repos","events_url":"https://api.github.com/users/ajroetker/events{/privacy}","received_events_url":"https://api.github.com/users/ajroetker/received_events","type":"User","site_admin":false},"repo":{"id":10415180,"name":"puppetdb","full_name":"ajroetker/puppetdb","owner":{"login":"ajroetker","id":3198124,"avatar_url":"https://avatars.githubusercontent.com/u/3198124?v=3","gravatar_id":"","url":"https://api.github.com/users/ajroetker","html_url":"https://github.com/ajroetker","followers_url":"https://api.github.com/users/ajroetker/followers","following_url":"https://api.github.com/users/ajroetker/following{/other_user}","gists_url":"https://api.github.com/users/ajroetker/gists{/gist_id}","starred_url":"https://api.github.com/users/ajroetker/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ajroetker/subscriptions","organizations_url":"https://api.github.com/users/ajroetker/orgs","repos_url":"https://api.github.com/users/ajroetker/repos","events_url":"https://api.github.com/users/ajroetker/events{/privacy}","received_events_url":"https://api.github.com/users/ajroetker/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ajroetker/puppetdb","description":"Centralized Puppet Storage","fork":true,"url":"https://api.github.com/repos/ajroetker/puppetdb","forks_url":"https://api.github.com/repos/ajroetker/puppetdb/forks","keys_url":"https://api.github.com/repos/ajroetker/puppetdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ajroetker/puppetdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ajroetker/puppetdb/teams","hooks_url":"https://api.github.com/repos/ajroetker/puppetdb/hooks","issue_events_url":"https://api.github.com/repos/ajroetker/puppetdb/issues/events{/number}","events_url":"https://api.github.com/repos/ajroetker/puppetdb/events","assignees_url":"https://api.github.com/repos/ajroetker/puppetdb/assignees{/user}","branches_url":"https://api.github.com/repos/ajroetker/puppetdb/branches{/branch}","tags_url":"https://api.github.com/repos/ajroetker/puppetdb/tags","blobs_url":"https://api.github.com/repos/ajroetker/puppetdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ajroetker/puppetdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ajroetker/puppetdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/ajroetker/puppetdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ajroetker/puppetdb/statuses/{sha}","languages_url":"https://api.github.com/repos/ajroetker/puppetdb/languages","stargazers_url":"https://api.github.com/repos/ajroetker/puppetdb/stargazers","contributors_url":"https://api.github.com/repos/ajroetker/puppetdb/contributors","subscribers_url":"https://api.github.com/repos/ajroetker/puppetdb/subscribers","subscription_url":"https://api.github.com/repos/ajroetker/puppetdb/subscription","commits_url":"https://api.github.com/repos/ajroetker/puppetdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/ajroetker/puppetdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/ajroetker/puppetdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/ajroetker/puppetdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/ajroetker/puppetdb/contents/{+path}","compare_url":"https://api.github.com/repos/ajroetker/puppetdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ajroetker/puppetdb/merges","archive_url":"https://api.github.com/repos/ajroetker/puppetdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ajroetker/puppetdb/downloads","issues_url":"https://api.github.com/repos/ajroetker/puppetdb/issues{/number}","pulls_url":"https://api.github.com/repos/ajroetker/puppetdb/pulls{/number}","milestones_url":"https://api.github.com/repos/ajroetker/puppetdb/milestones{/number}","notifications_url":"https://api.github.com/repos/ajroetker/puppetdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ajroetker/puppetdb/labels{/name}","releases_url":"https://api.github.com/repos/ajroetker/puppetdb/releases{/id}","created_at":"2013-05-31T23:59:00Z","updated_at":"2015-04-29T22:51:50Z","pushed_at":"2015-06-30T17:01:31Z","git_url":"git://github.com/ajroetker/puppetdb.git","ssh_url":"git@github.com:ajroetker/puppetdb.git","clone_url":"https://github.com/ajroetker/puppetdb.git","svn_url":"https://github.com/ajroetker/puppetdb","homepage":"http://docs.puppetlabs.com/puppetdb","size":14355,"stargazers_count":0,"watchers_count":0,"language":"Clojure","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"puppetlabs:master","ref":"master","sha":"ec6f8e75762f5462eb2a14a96a8433ad9c49bfbc","user":{"login":"puppetlabs","id":234268,"avatar_url":"https://avatars.githubusercontent.com/u/234268?v=3","gravatar_id":"","url":"https://api.github.com/users/puppetlabs","html_url":"https://github.com/puppetlabs","followers_url":"https://api.github.com/users/puppetlabs/followers","following_url":"https://api.github.com/users/puppetlabs/following{/other_user}","gists_url":"https://api.github.com/users/puppetlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/puppetlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/puppetlabs/subscriptions","organizations_url":"https://api.github.com/users/puppetlabs/orgs","repos_url":"https://api.github.com/users/puppetlabs/repos","events_url":"https://api.github.com/users/puppetlabs/events{/privacy}","received_events_url":"https://api.github.com/users/puppetlabs/received_events","type":"Organization","site_admin":false},"repo":{"id":2193032,"name":"puppetdb","full_name":"puppetlabs/puppetdb","owner":{"login":"puppetlabs","id":234268,"avatar_url":"https://avatars.githubusercontent.com/u/234268?v=3","gravatar_id":"","url":"https://api.github.com/users/puppetlabs","html_url":"https://github.com/puppetlabs","followers_url":"https://api.github.com/users/puppetlabs/followers","following_url":"https://api.github.com/users/puppetlabs/following{/other_user}","gists_url":"https://api.github.com/users/puppetlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/puppetlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/puppetlabs/subscriptions","organizations_url":"https://api.github.com/users/puppetlabs/orgs","repos_url":"https://api.github.com/users/puppetlabs/repos","events_url":"https://api.github.com/users/puppetlabs/events{/privacy}","received_events_url":"https://api.github.com/users/puppetlabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/puppetlabs/puppetdb","description":"Centralized Puppet Storage","fork":false,"url":"https://api.github.com/repos/puppetlabs/puppetdb","forks_url":"https://api.github.com/repos/puppetlabs/puppetdb/forks","keys_url":"https://api.github.com/repos/puppetlabs/puppetdb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/puppetlabs/puppetdb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/puppetlabs/puppetdb/teams","hooks_url":"https://api.github.com/repos/puppetlabs/puppetdb/hooks","issue_events_url":"https://api.github.com/repos/puppetlabs/puppetdb/issues/events{/number}","events_url":"https://api.github.com/repos/puppetlabs/puppetdb/events","assignees_url":"https://api.github.com/repos/puppetlabs/puppetdb/assignees{/user}","branches_url":"https://api.github.com/repos/puppetlabs/puppetdb/branches{/branch}","tags_url":"https://api.github.com/repos/puppetlabs/puppetdb/tags","blobs_url":"https://api.github.com/repos/puppetlabs/puppetdb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/puppetlabs/puppetdb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/puppetlabs/puppetdb/git/refs{/sha}","trees_url":"https://api.github.com/repos/puppetlabs/puppetdb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/puppetlabs/puppetdb/statuses/{sha}","languages_url":"https://api.github.com/repos/puppetlabs/puppetdb/languages","stargazers_url":"https://api.github.com/repos/puppetlabs/puppetdb/stargazers","contributors_url":"https://api.github.com/repos/puppetlabs/puppetdb/contributors","subscribers_url":"https://api.github.com/repos/puppetlabs/puppetdb/subscribers","subscription_url":"https://api.github.com/repos/puppetlabs/puppetdb/subscription","commits_url":"https://api.github.com/repos/puppetlabs/puppetdb/commits{/sha}","git_commits_url":"https://api.github.com/repos/puppetlabs/puppetdb/git/commits{/sha}","comments_url":"https://api.github.com/repos/puppetlabs/puppetdb/comments{/number}","issue_comment_url":"https://api.github.com/repos/puppetlabs/puppetdb/issues/comments{/number}","contents_url":"https://api.github.com/repos/puppetlabs/puppetdb/contents/{+path}","compare_url":"https://api.github.com/repos/puppetlabs/puppetdb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/puppetlabs/puppetdb/merges","archive_url":"https://api.github.com/repos/puppetlabs/puppetdb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/puppetlabs/puppetdb/downloads","issues_url":"https://api.github.com/repos/puppetlabs/puppetdb/issues{/number}","pulls_url":"https://api.github.com/repos/puppetlabs/puppetdb/pulls{/number}","milestones_url":"https://api.github.com/repos/puppetlabs/puppetdb/milestones{/number}","notifications_url":"https://api.github.com/repos/puppetlabs/puppetdb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/puppetlabs/puppetdb/labels{/name}","releases_url":"https://api.github.com/repos/puppetlabs/puppetdb/releases{/id}","created_at":"2011-08-11T18:54:46Z","updated_at":"2015-06-24T10:48:49Z","pushed_at":"2015-06-30T17:29:16Z","git_url":"git://github.com/puppetlabs/puppetdb.git","ssh_url":"git@github.com:puppetlabs/puppetdb.git","clone_url":"https://github.com/puppetlabs/puppetdb.git","svn_url":"https://github.com/puppetlabs/puppetdb","homepage":"http://docs.puppetlabs.com/puppetdb","size":21631,"stargazers_count":164,"watchers_count":164,"language":"Clojure","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":131,"mirror_url":null,"open_issues_count":8,"forks":131,"open_issues":8,"watchers":164,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/1490"},"html":{"href":"https://github.com/puppetlabs/puppetdb/pull/1490"},"issue":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/issues/1490"},"comments":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/issues/1490/comments"},"review_comments":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/1490/comments"},"review_comment":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/pulls/1490/commits"},"statuses":{"href":"https://api.github.com/repos/puppetlabs/puppetdb/statuses/6e0ed7c17a57bdce84fa178731e6dcac3d8be64c"}}}}