{"action":"created","comment":{"url":"https://api.github.com/repos/chef/chef-server/pulls/comments/35777436","id":35777436,"diff_hunk":"@@ -0,0 +1,227 @@\n+-module(bksw_sync).\n+%%\n+%% bksw_sync: Asyncronous syncing of uploaded files to a remote node.\n+%%\n+%% This module provides handles asyncronous requests sent by the\n+%% application anytime a file is created or delete. It calls the\n+%% external `rsync` command to ensure that a remote host also has a\n+%% copy of these files.\n+%%\n+%% On startup, a full_sync message is added to our queue. This ensures\n+%% that any missed messages when we were down (say, because of a\n+%% failure) are correctly handled.\n+%%\n+%% Everytime a file is created or deleted, bksw_sync:new or\n+%% bksw_sync:deleted should be called with the path of the created or\n+%% deleted file. bksw_sync will add this path to a list of unsynced\n+%% paths.\n+%%\n+%% If no new paths have been added for MAX_UNFLUSHED_TIME\n+%% milliseconds, we will call rsync on any unsynced paths.\n+%% Alternatively, if the unsynced list grows to MAX_UNFLUSHED_SIZE, we\n+%% will call rsync on the unsynced paths.\n+%%\n+%% The calls to rsync are done syncronously such that only one rsync\n+%% process is working at a time. Paths created or deleted during a\n+%% running sync will queue up in the processes mailbox for later\n+%% handling.\n+%%\n+%% If a full_sync or partial_sync succeeds, we clear the\n+%% unsynced_paths list and wait for more updates.\n+%%\n+%% If a sync fails, we do not clear the list, and try again in 2\n+%% seconds.\n+%%\n+-behavior(gen_server).\n+\n+-export([start_link/2,\n+         new/1,\n+         delete/1,\n+         sync/0,\n+         full_sync/0,\n+         status/0,\n+         init/1,\n+         handle_info/2,\n+         handle_call/3,\n+         handle_cast/2,\n+         terminate/2,\n+         code_change/3\n+        ]).\n+\n+-define(MAX_UNFLUSHED_SIZE, 250).\n+-define(MAX_UNFLUSHED_TIME, 2000).\n+-define(RSYNC_EXE, \"rsync\").\n+-define(RSYNC_ARGS, \"-r --delete --delete-missing-args -vv\").\n+\n+-record(sync_state, {\n+          % A list of paths that have not yet been synced.\n+          % These paths are stored relative to the base_directory.\n+          unsynced_paths,\n+          % The base bookshelf data directory.  We initialize this to\n+          % bksw_conf:data_store() in init/1,\n+          base_directory,\n+          % A valid rsync remote URI.  We expect the user to supply\n+          % this explicitly as part of the configuration.\n+          rsync_remote}).\n+\n+\n+%%\n+%% Public API\n+%%\n+start_link(BaseDir, RemoteURI) ->\n+    gen_server:start_link({local, ?MODULE}, ?MODULE, [BaseDir, RemoteURI], []).\n+\n+new(Path) ->\n+    gen_server:cast(?MODULE, {created, Path}).\n+\n+delete(Path) ->\n+    gen_server:cast(?MODULE, {deleted, Path}).\n+\n+sync() ->\n+    gen_server:cast(?MODULE, sync).\n+\n+full_sync() ->\n+    gen_server:cast(?MODULE, full_sync).\n+\n+status() ->\n+    gen_server:call(?MODULE, status).\n+\n+%%\n+%% gen_server callbacks\n+%%\n+init([BaseDir, RemoteURI]) ->\n+    %% Send ourselves a full_sync to ensure that we always do a\n+    %% full_sync on startup. Note: this doesn't block init() since it\n+    %% is a gen_server:cast underneath.\n+    full_sync(),\n+    lager:info(\"starting bksw_rsync for ~p ~p\", [BaseDir, RemoteURI]),\n+    {ok, #sync_state{unsynced_paths = [],\n+                     base_directory = to_binary(BaseDir),\n+                     rsync_remote = RemoteURI}}.\n+\n+handle_cast({created, Path}, State) ->\n+    add_to_unsynced(Path, State);\n+handle_cast({deleted, Path}, State) ->\n+    add_to_unsynced(Path, State);\n+handle_cast(full_sync, State) ->\n+    lager:info(\"starting full rsync run\"),\n+    RsyncCmd = exec:run(mk_rsync_cmd(full, State), [stdout, stderr, sync]),\n+    case RsyncCmd of\n+        {ok, _Rest} ->\n+            lager:info(\"full rsync run successful.\"),\n+            {noreply, State};\n+        {error, Result} ->\n+            lager:error(\"full rsync run failed, retrying in ~p ms\", [?MAX_UNFLUSHED_TIME]),\n+            log_rsync_error(Result),\n+            timer:apply_after(?MAX_UNFLUSHED_TIME, ?MODULE, full_sync, []),\n+            {noreply, State}\n+    end;\n+handle_cast(sync, State = #sync_state{unsynced_paths = []}) ->\n+    lager:info(\"sync message recieved but no data to sync\"),\n+    {noreply, State};\n+handle_cast(sync, State = #sync_state{unsynced_paths = UP}) ->\n+    lager:info(\"starting partial sync on ~p paths\", [length(UP)]),\n+    {ok, File, Filename} = tempfile(),\n+    [file:write(File, iolist_to_binary([P, $\\n])) || P <- UP ],\n+    file:close(File),\n+    Cmd = mk_rsync_cmd(partial, State, Filename),\n+    CmdStatus = exec:run(Cmd, [stdout, stderr, sync]),\n+    file:delete(Filename),\n+    case CmdStatus of\n+        {ok, _Rest} ->\n+            lager:info(\"partial rsync run successful.\"),\n+            {noreply, State#sync_state{unsynced_paths = []}};\n+        {error, Result} ->\n+            lager:error(\"partial rsync run failed, retrying in ~p ms\", [?MAX_UNFLUSHED_TIME]),\n+            log_rsync_error(Result),\n+            timer:apply_after(?MAX_UNFLUSHED_TIME, ?MODULE, sync, []),\n+            {noreply, State}\n+    end.\n+\n+handle_call(status, _From, State) ->\n+    % Use a timeout here to prevent status calls from stranding\n+    % unsynced paths for longer than expected.\n+    {reply, State, State, ?MAX_UNFLUSHED_TIME}.\n+\n+handle_info(timeout, State = #sync_state{unsynced_paths = UP}) ->\n+    lager:info(\"No updates for ~p ms\", [?MAX_UNFLUSHED_TIME]),\n+    case UP of\n+        [] ->\n+            lager:info(\"Up to date. Nothing to do.\"),\n+            {noreply, State};\n+        _Paths ->\n+            sync(),\n+            {noreply, State}\n+    end.\n+\n+code_change(_OldVsn, State, _Extra) ->\n+    {ok, State}.\n+\n+terminate(_Reason, _State) ->\n+    ok.\n+\n+%%\n+%% Internal Functions\n+%%\n+\n+log_rsync_error(Result) ->","path":"src/bookshelf/src/bksw_sync.erl","position":167,"original_position":167,"commit_id":"5539252a5a1588a5adf986728736d6b86f11e536","original_commit_id":"5539252a5a1588a5adf986728736d6b86f11e536","user":{"login":"stevendanna","id":852371,"avatar_url":"https://avatars.githubusercontent.com/u/852371?v=3","gravatar_id":"","url":"https://api.github.com/users/stevendanna","html_url":"https://github.com/stevendanna","followers_url":"https://api.github.com/users/stevendanna/followers","following_url":"https://api.github.com/users/stevendanna/following{/other_user}","gists_url":"https://api.github.com/users/stevendanna/gists{/gist_id}","starred_url":"https://api.github.com/users/stevendanna/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stevendanna/subscriptions","organizations_url":"https://api.github.com/users/stevendanna/orgs","repos_url":"https://api.github.com/users/stevendanna/repos","events_url":"https://api.github.com/users/stevendanna/events{/privacy}","received_events_url":"https://api.github.com/users/stevendanna/received_events","type":"User","site_admin":false},"body":"But we can still add a guard and then a fall-through case.","created_at":"2015-07-29T16:06:30Z","updated_at":"2015-07-29T16:06:30Z","html_url":"https://github.com/chef/chef-server/pull/456#discussion_r35777436","pull_request_url":"https://api.github.com/repos/chef/chef-server/pulls/456","_links":{"self":{"href":"https://api.github.com/repos/chef/chef-server/pulls/comments/35777436"},"html":{"href":"https://github.com/chef/chef-server/pull/456#discussion_r35777436"},"pull_request":{"href":"https://api.github.com/repos/chef/chef-server/pulls/456"}}},"pull_request":{"url":"https://api.github.com/repos/chef/chef-server/pulls/456","id":41115775,"html_url":"https://github.com/chef/chef-server/pull/456","diff_url":"https://github.com/chef/chef-server/pull/456.diff","patch_url":"https://github.com/chef/chef-server/pull/456.patch","issue_url":"https://api.github.com/repos/chef/chef-server/issues/456","number":456,"state":"open","locked":false,"title":"[bookshelf] Add bksw_sync module","user":{"login":"stevendanna","id":852371,"avatar_url":"https://avatars.githubusercontent.com/u/852371?v=3","gravatar_id":"","url":"https://api.github.com/users/stevendanna","html_url":"https://github.com/stevendanna","followers_url":"https://api.github.com/users/stevendanna/followers","following_url":"https://api.github.com/users/stevendanna/following{/other_user}","gists_url":"https://api.github.com/users/stevendanna/gists{/gist_id}","starred_url":"https://api.github.com/users/stevendanna/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stevendanna/subscriptions","organizations_url":"https://api.github.com/users/stevendanna/orgs","repos_url":"https://api.github.com/users/stevendanna/repos","events_url":"https://api.github.com/users/stevendanna/events{/privacy}","received_events_url":"https://api.github.com/users/stevendanna/received_events","type":"User","site_admin":false},"body":"The bksw_sync module handles asyncronous notifications of file uploads\r\nor deletions, using rsync to replicate the files to a remote host.","created_at":"2015-07-29T12:11:05Z","updated_at":"2015-07-29T16:06:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b6cf09c286b1eafe4a11e74e2b7adc2fcaeef9b2","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/chef/chef-server/pulls/456/commits","review_comments_url":"https://api.github.com/repos/chef/chef-server/pulls/456/comments","review_comment_url":"https://api.github.com/repos/chef/chef-server/pulls/comments{/number}","comments_url":"https://api.github.com/repos/chef/chef-server/issues/456/comments","statuses_url":"https://api.github.com/repos/chef/chef-server/statuses/5539252a5a1588a5adf986728736d6b86f11e536","head":{"label":"chef:ssd/bookshelf-internal-sync","ref":"ssd/bookshelf-internal-sync","sha":"5539252a5a1588a5adf986728736d6b86f11e536","user":{"login":"chef","id":29740,"avatar_url":"https://avatars.githubusercontent.com/u/29740?v=3","gravatar_id":"","url":"https://api.github.com/users/chef","html_url":"https://github.com/chef","followers_url":"https://api.github.com/users/chef/followers","following_url":"https://api.github.com/users/chef/following{/other_user}","gists_url":"https://api.github.com/users/chef/gists{/gist_id}","starred_url":"https://api.github.com/users/chef/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chef/subscriptions","organizations_url":"https://api.github.com/users/chef/orgs","repos_url":"https://api.github.com/users/chef/repos","events_url":"https://api.github.com/users/chef/events{/privacy}","received_events_url":"https://api.github.com/users/chef/received_events","type":"Organization","site_admin":false},"repo":{"id":24732433,"name":"chef-server","full_name":"chef/chef-server","owner":{"login":"chef","id":29740,"avatar_url":"https://avatars.githubusercontent.com/u/29740?v=3","gravatar_id":"","url":"https://api.github.com/users/chef","html_url":"https://github.com/chef","followers_url":"https://api.github.com/users/chef/followers","following_url":"https://api.github.com/users/chef/following{/other_user}","gists_url":"https://api.github.com/users/chef/gists{/gist_id}","starred_url":"https://api.github.com/users/chef/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chef/subscriptions","organizations_url":"https://api.github.com/users/chef/orgs","repos_url":"https://api.github.com/users/chef/repos","events_url":"https://api.github.com/users/chef/events{/privacy}","received_events_url":"https://api.github.com/users/chef/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/chef/chef-server","description":"The Chef Server ","fork":false,"url":"https://api.github.com/repos/chef/chef-server","forks_url":"https://api.github.com/repos/chef/chef-server/forks","keys_url":"https://api.github.com/repos/chef/chef-server/keys{/key_id}","collaborators_url":"https://api.github.com/repos/chef/chef-server/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/chef/chef-server/teams","hooks_url":"https://api.github.com/repos/chef/chef-server/hooks","issue_events_url":"https://api.github.com/repos/chef/chef-server/issues/events{/number}","events_url":"https://api.github.com/repos/chef/chef-server/events","assignees_url":"https://api.github.com/repos/chef/chef-server/assignees{/user}","branches_url":"https://api.github.com/repos/chef/chef-server/branches{/branch}","tags_url":"https://api.github.com/repos/chef/chef-server/tags","blobs_url":"https://api.github.com/repos/chef/chef-server/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/chef/chef-server/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/chef/chef-server/git/refs{/sha}","trees_url":"https://api.github.com/repos/chef/chef-server/git/trees{/sha}","statuses_url":"https://api.github.com/repos/chef/chef-server/statuses/{sha}","languages_url":"https://api.github.com/repos/chef/chef-server/languages","stargazers_url":"https://api.github.com/repos/chef/chef-server/stargazers","contributors_url":"https://api.github.com/repos/chef/chef-server/contributors","subscribers_url":"https://api.github.com/repos/chef/chef-server/subscribers","subscription_url":"https://api.github.com/repos/chef/chef-server/subscription","commits_url":"https://api.github.com/repos/chef/chef-server/commits{/sha}","git_commits_url":"https://api.github.com/repos/chef/chef-server/git/commits{/sha}","comments_url":"https://api.github.com/repos/chef/chef-server/comments{/number}","issue_comment_url":"https://api.github.com/repos/chef/chef-server/issues/comments{/number}","contents_url":"https://api.github.com/repos/chef/chef-server/contents/{+path}","compare_url":"https://api.github.com/repos/chef/chef-server/compare/{base}...{head}","merges_url":"https://api.github.com/repos/chef/chef-server/merges","archive_url":"https://api.github.com/repos/chef/chef-server/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/chef/chef-server/downloads","issues_url":"https://api.github.com/repos/chef/chef-server/issues{/number}","pulls_url":"https://api.github.com/repos/chef/chef-server/pulls{/number}","milestones_url":"https://api.github.com/repos/chef/chef-server/milestones{/number}","notifications_url":"https://api.github.com/repos/chef/chef-server/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/chef/chef-server/labels{/name}","releases_url":"https://api.github.com/repos/chef/chef-server/releases{/id}","created_at":"2014-10-02T19:04:18Z","updated_at":"2015-07-15T15:03:23Z","pushed_at":"2015-07-29T15:20:44Z","git_url":"git://github.com/chef/chef-server.git","ssh_url":"git@github.com:chef/chef-server.git","clone_url":"https://github.com/chef/chef-server.git","svn_url":"https://github.com/chef/chef-server","homepage":"https://www.chef.io/chef/","size":15371,"stargazers_count":32,"watchers_count":32,"language":"Erlang","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":22,"mirror_url":null,"open_issues_count":181,"forks":22,"open_issues":181,"watchers":32,"default_branch":"master"}},"base":{"label":"chef:master","ref":"master","sha":"c03e20406b8f16f5b1ac3bdd58b43c6e35ead59d","user":{"login":"chef","id":29740,"avatar_url":"https://avatars.githubusercontent.com/u/29740?v=3","gravatar_id":"","url":"https://api.github.com/users/chef","html_url":"https://github.com/chef","followers_url":"https://api.github.com/users/chef/followers","following_url":"https://api.github.com/users/chef/following{/other_user}","gists_url":"https://api.github.com/users/chef/gists{/gist_id}","starred_url":"https://api.github.com/users/chef/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chef/subscriptions","organizations_url":"https://api.github.com/users/chef/orgs","repos_url":"https://api.github.com/users/chef/repos","events_url":"https://api.github.com/users/chef/events{/privacy}","received_events_url":"https://api.github.com/users/chef/received_events","type":"Organization","site_admin":false},"repo":{"id":24732433,"name":"chef-server","full_name":"chef/chef-server","owner":{"login":"chef","id":29740,"avatar_url":"https://avatars.githubusercontent.com/u/29740?v=3","gravatar_id":"","url":"https://api.github.com/users/chef","html_url":"https://github.com/chef","followers_url":"https://api.github.com/users/chef/followers","following_url":"https://api.github.com/users/chef/following{/other_user}","gists_url":"https://api.github.com/users/chef/gists{/gist_id}","starred_url":"https://api.github.com/users/chef/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chef/subscriptions","organizations_url":"https://api.github.com/users/chef/orgs","repos_url":"https://api.github.com/users/chef/repos","events_url":"https://api.github.com/users/chef/events{/privacy}","received_events_url":"https://api.github.com/users/chef/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/chef/chef-server","description":"The Chef Server ","fork":false,"url":"https://api.github.com/repos/chef/chef-server","forks_url":"https://api.github.com/repos/chef/chef-server/forks","keys_url":"https://api.github.com/repos/chef/chef-server/keys{/key_id}","collaborators_url":"https://api.github.com/repos/chef/chef-server/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/chef/chef-server/teams","hooks_url":"https://api.github.com/repos/chef/chef-server/hooks","issue_events_url":"https://api.github.com/repos/chef/chef-server/issues/events{/number}","events_url":"https://api.github.com/repos/chef/chef-server/events","assignees_url":"https://api.github.com/repos/chef/chef-server/assignees{/user}","branches_url":"https://api.github.com/repos/chef/chef-server/branches{/branch}","tags_url":"https://api.github.com/repos/chef/chef-server/tags","blobs_url":"https://api.github.com/repos/chef/chef-server/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/chef/chef-server/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/chef/chef-server/git/refs{/sha}","trees_url":"https://api.github.com/repos/chef/chef-server/git/trees{/sha}","statuses_url":"https://api.github.com/repos/chef/chef-server/statuses/{sha}","languages_url":"https://api.github.com/repos/chef/chef-server/languages","stargazers_url":"https://api.github.com/repos/chef/chef-server/stargazers","contributors_url":"https://api.github.com/repos/chef/chef-server/contributors","subscribers_url":"https://api.github.com/repos/chef/chef-server/subscribers","subscription_url":"https://api.github.com/repos/chef/chef-server/subscription","commits_url":"https://api.github.com/repos/chef/chef-server/commits{/sha}","git_commits_url":"https://api.github.com/repos/chef/chef-server/git/commits{/sha}","comments_url":"https://api.github.com/repos/chef/chef-server/comments{/number}","issue_comment_url":"https://api.github.com/repos/chef/chef-server/issues/comments{/number}","contents_url":"https://api.github.com/repos/chef/chef-server/contents/{+path}","compare_url":"https://api.github.com/repos/chef/chef-server/compare/{base}...{head}","merges_url":"https://api.github.com/repos/chef/chef-server/merges","archive_url":"https://api.github.com/repos/chef/chef-server/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/chef/chef-server/downloads","issues_url":"https://api.github.com/repos/chef/chef-server/issues{/number}","pulls_url":"https://api.github.com/repos/chef/chef-server/pulls{/number}","milestones_url":"https://api.github.com/repos/chef/chef-server/milestones{/number}","notifications_url":"https://api.github.com/repos/chef/chef-server/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/chef/chef-server/labels{/name}","releases_url":"https://api.github.com/repos/chef/chef-server/releases{/id}","created_at":"2014-10-02T19:04:18Z","updated_at":"2015-07-15T15:03:23Z","pushed_at":"2015-07-29T15:20:44Z","git_url":"git://github.com/chef/chef-server.git","ssh_url":"git@github.com:chef/chef-server.git","clone_url":"https://github.com/chef/chef-server.git","svn_url":"https://github.com/chef/chef-server","homepage":"https://www.chef.io/chef/","size":15371,"stargazers_count":32,"watchers_count":32,"language":"Erlang","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":22,"mirror_url":null,"open_issues_count":181,"forks":22,"open_issues":181,"watchers":32,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/chef/chef-server/pulls/456"},"html":{"href":"https://github.com/chef/chef-server/pull/456"},"issue":{"href":"https://api.github.com/repos/chef/chef-server/issues/456"},"comments":{"href":"https://api.github.com/repos/chef/chef-server/issues/456/comments"},"review_comments":{"href":"https://api.github.com/repos/chef/chef-server/pulls/456/comments"},"review_comment":{"href":"https://api.github.com/repos/chef/chef-server/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/chef/chef-server/pulls/456/commits"},"statuses":{"href":"https://api.github.com/repos/chef/chef-server/statuses/5539252a5a1588a5adf986728736d6b86f11e536"}}}}