{"action":"created","comment":{"url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/comments/38133744","id":38133744,"diff_hunk":"@@ -48,8 +48,16 @@ export JSVC_HOME=/usr/lib/bigtop-utils\n # On secure datanodes, user to run the datanode as after dropping privileges\n export HADOOP_SECURE_DN_USER=hdfs\n \n+<% datanode_heap = [node[\"bcpc\"][\"hadoop\"][\"datanode\"][\"xmx\"][\"max_size\"], (node[\"memory\"][\"total\"].to_i * node[\"bcpc\"][\"hadoop\"][\"datanode\"][\"xmx\"][\"max_ratio\"]/1024).floor].min %>\n+<% datanode_newsize = [(0.125*datanode_heap).ceil, 3072].min %>\n+<% namenode_heap = [node[\"bcpc\"][\"hadoop\"][\"namenode\"][\"xmx\"][\"max_size\"], (node[\"memory\"][\"total\"].to_i * node[\"bcpc\"][\"hadoop\"][\"namenode\"][\"xmx\"][\"max_ratio\"]/1024).floor].min %>\n+<% namenode_newsize = [(0.125*namenode_heap).ceil, 3072].min %>\n+\n+export HADOOP_DATANODE_OPTS=\"$HADOOP_DATANODE_OPTS -Xms<%= datanode_heap %>m -Xmx<%= datanode_heap %>m -XX:NewSize=<%= datanode_newsize %>m -XX:MaxNewSize=<%= datanode_newsize %>m -XX:PermSize=128m -XX:MaxPermSize=256m <%= node[\"bcpc\"][\"hadoop\"][\"datanode\"][\"gc_opts\"] %>\"\n+export HADOOP_NAMENODE_OPTS=\"$HADOOP_NAMENODE_OPTS -Xms<%= namenode_heap %>m -Xmx<%= namenode_heap %>m -XX:NewSize=<%= namenode_newsize %>m -XX:MaxNewSize=<%= namenode_newsize %>m -XX:PermSize=128m -XX:MaxPermSize=256m <%= node[\"bcpc\"][\"hadoop\"][\"namenode\"][\"gc_opts\"] %>\"","path":"cookbooks/bcpc-hadoop/templates/default/hdp_hadoop-env.sh.erb","position":10,"original_position":10,"commit_id":"63f277e393cc7f475cb47704dd1b84ef938f7003","original_commit_id":"63f277e393cc7f475cb47704dd1b84ef938f7003","user":{"login":"bijugs","id":7581305,"avatar_url":"https://avatars.githubusercontent.com/u/7581305?v=3","gravatar_id":"","url":"https://api.github.com/users/bijugs","html_url":"https://github.com/bijugs","followers_url":"https://api.github.com/users/bijugs/followers","following_url":"https://api.github.com/users/bijugs/following{/other_user}","gists_url":"https://api.github.com/users/bijugs/gists{/gist_id}","starred_url":"https://api.github.com/users/bijugs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bijugs/subscriptions","organizations_url":"https://api.github.com/users/bijugs/orgs","repos_url":"https://api.github.com/users/bijugs/repos","events_url":"https://api.github.com/users/bijugs/events{/privacy}","received_events_url":"https://api.github.com/users/bijugs/received_events","type":"User","site_admin":false},"body":"Will allocating a higher value for ```PermSize`` negatively impact clusters with less memory like VM clusters. By default ``PermSize`` seem to default to 32 or 64mb.","created_at":"2015-08-27T19:02:50Z","updated_at":"2015-08-27T19:02:50Z","html_url":"https://github.com/bloomberg/chef-bach/pull/255#discussion_r38133744","pull_request_url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/255","_links":{"self":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/comments/38133744"},"html":{"href":"https://github.com/bloomberg/chef-bach/pull/255#discussion_r38133744"},"pull_request":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/255"}}},"pull_request":{"url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/255","id":43551688,"html_url":"https://github.com/bloomberg/chef-bach/pull/255","diff_url":"https://github.com/bloomberg/chef-bach/pull/255.diff","patch_url":"https://github.com/bloomberg/chef-bach/pull/255.patch","issue_url":"https://api.github.com/repos/bloomberg/chef-bach/issues/255","number":255,"state":"open","locked":false,"title":"Clean-up namenode and datanode memory parameters - provide new, perm …","user":{"login":"cbaenziger","id":4192761,"avatar_url":"https://avatars.githubusercontent.com/u/4192761?v=3","gravatar_id":"","url":"https://api.github.com/users/cbaenziger","html_url":"https://github.com/cbaenziger","followers_url":"https://api.github.com/users/cbaenziger/followers","following_url":"https://api.github.com/users/cbaenziger/following{/other_user}","gists_url":"https://api.github.com/users/cbaenziger/gists{/gist_id}","starred_url":"https://api.github.com/users/cbaenziger/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cbaenziger/subscriptions","organizations_url":"https://api.github.com/users/cbaenziger/orgs","repos_url":"https://api.github.com/users/cbaenziger/repos","events_url":"https://api.github.com/users/cbaenziger/events{/privacy}","received_events_url":"https://api.github.com/users/cbaenziger/received_events","type":"User","site_admin":false},"body":"This provides:\r\n* cleans up DN and NN Java opts to not be under the JMX conditional\r\n* provides GC options for DN as was available for the NN\r\n* sets new gen size to 1/8th the size of the heap up to 3GB","created_at":"2015-08-27T18:40:52Z","updated_at":"2015-08-27T19:02:50Z","closed_at":null,"merged_at":null,"merge_commit_sha":"cd08f961010b997da1806bf8d77d0461c7fa57ef","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/255/commits","review_comments_url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/255/comments","review_comment_url":"https://api.github.com/repos/bloomberg/chef-bach/pulls/comments{/number}","comments_url":"https://api.github.com/repos/bloomberg/chef-bach/issues/255/comments","statuses_url":"https://api.github.com/repos/bloomberg/chef-bach/statuses/63f277e393cc7f475cb47704dd1b84ef938f7003","head":{"label":"cbaenziger:dn_memory_sizing","ref":"dn_memory_sizing","sha":"63f277e393cc7f475cb47704dd1b84ef938f7003","user":{"login":"cbaenziger","id":4192761,"avatar_url":"https://avatars.githubusercontent.com/u/4192761?v=3","gravatar_id":"","url":"https://api.github.com/users/cbaenziger","html_url":"https://github.com/cbaenziger","followers_url":"https://api.github.com/users/cbaenziger/followers","following_url":"https://api.github.com/users/cbaenziger/following{/other_user}","gists_url":"https://api.github.com/users/cbaenziger/gists{/gist_id}","starred_url":"https://api.github.com/users/cbaenziger/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cbaenziger/subscriptions","organizations_url":"https://api.github.com/users/cbaenziger/orgs","repos_url":"https://api.github.com/users/cbaenziger/repos","events_url":"https://api.github.com/users/cbaenziger/events{/privacy}","received_events_url":"https://api.github.com/users/cbaenziger/received_events","type":"User","site_admin":false},"repo":{"id":25824446,"name":"chef-bach","full_name":"cbaenziger/chef-bach","owner":{"login":"cbaenziger","id":4192761,"avatar_url":"https://avatars.githubusercontent.com/u/4192761?v=3","gravatar_id":"","url":"https://api.github.com/users/cbaenziger","html_url":"https://github.com/cbaenziger","followers_url":"https://api.github.com/users/cbaenziger/followers","following_url":"https://api.github.com/users/cbaenziger/following{/other_user}","gists_url":"https://api.github.com/users/cbaenziger/gists{/gist_id}","starred_url":"https://api.github.com/users/cbaenziger/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cbaenziger/subscriptions","organizations_url":"https://api.github.com/users/cbaenziger/orgs","repos_url":"https://api.github.com/users/cbaenziger/repos","events_url":"https://api.github.com/users/cbaenziger/events{/privacy}","received_events_url":"https://api.github.com/users/cbaenziger/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/cbaenziger/chef-bach","description":"Chef recipes for Bloomberg's deployment of Hadoop and related components","fork":true,"url":"https://api.github.com/repos/cbaenziger/chef-bach","forks_url":"https://api.github.com/repos/cbaenziger/chef-bach/forks","keys_url":"https://api.github.com/repos/cbaenziger/chef-bach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cbaenziger/chef-bach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cbaenziger/chef-bach/teams","hooks_url":"https://api.github.com/repos/cbaenziger/chef-bach/hooks","issue_events_url":"https://api.github.com/repos/cbaenziger/chef-bach/issues/events{/number}","events_url":"https://api.github.com/repos/cbaenziger/chef-bach/events","assignees_url":"https://api.github.com/repos/cbaenziger/chef-bach/assignees{/user}","branches_url":"https://api.github.com/repos/cbaenziger/chef-bach/branches{/branch}","tags_url":"https://api.github.com/repos/cbaenziger/chef-bach/tags","blobs_url":"https://api.github.com/repos/cbaenziger/chef-bach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cbaenziger/chef-bach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cbaenziger/chef-bach/git/refs{/sha}","trees_url":"https://api.github.com/repos/cbaenziger/chef-bach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cbaenziger/chef-bach/statuses/{sha}","languages_url":"https://api.github.com/repos/cbaenziger/chef-bach/languages","stargazers_url":"https://api.github.com/repos/cbaenziger/chef-bach/stargazers","contributors_url":"https://api.github.com/repos/cbaenziger/chef-bach/contributors","subscribers_url":"https://api.github.com/repos/cbaenziger/chef-bach/subscribers","subscription_url":"https://api.github.com/repos/cbaenziger/chef-bach/subscription","commits_url":"https://api.github.com/repos/cbaenziger/chef-bach/commits{/sha}","git_commits_url":"https://api.github.com/repos/cbaenziger/chef-bach/git/commits{/sha}","comments_url":"https://api.github.com/repos/cbaenziger/chef-bach/comments{/number}","issue_comment_url":"https://api.github.com/repos/cbaenziger/chef-bach/issues/comments{/number}","contents_url":"https://api.github.com/repos/cbaenziger/chef-bach/contents/{+path}","compare_url":"https://api.github.com/repos/cbaenziger/chef-bach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cbaenziger/chef-bach/merges","archive_url":"https://api.github.com/repos/cbaenziger/chef-bach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cbaenziger/chef-bach/downloads","issues_url":"https://api.github.com/repos/cbaenziger/chef-bach/issues{/number}","pulls_url":"https://api.github.com/repos/cbaenziger/chef-bach/pulls{/number}","milestones_url":"https://api.github.com/repos/cbaenziger/chef-bach/milestones{/number}","notifications_url":"https://api.github.com/repos/cbaenziger/chef-bach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cbaenziger/chef-bach/labels{/name}","releases_url":"https://api.github.com/repos/cbaenziger/chef-bach/releases{/id}","created_at":"2014-10-27T15:02:22Z","updated_at":"2015-04-12T23:24:06Z","pushed_at":"2015-08-27T18:21:38Z","git_url":"git://github.com/cbaenziger/chef-bach.git","ssh_url":"git@github.com:cbaenziger/chef-bach.git","clone_url":"https://github.com/cbaenziger/chef-bach.git","svn_url":"https://github.com/cbaenziger/chef-bach","homepage":null,"size":2646,"stargazers_count":0,"watchers_count":0,"language":"HTML","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"bloomberg:master","ref":"master","sha":"2329cd511d8116e6c8c2f5a362a816914ed25f28","user":{"login":"bloomberg","id":1416818,"avatar_url":"https://avatars.githubusercontent.com/u/1416818?v=3","gravatar_id":"","url":"https://api.github.com/users/bloomberg","html_url":"https://github.com/bloomberg","followers_url":"https://api.github.com/users/bloomberg/followers","following_url":"https://api.github.com/users/bloomberg/following{/other_user}","gists_url":"https://api.github.com/users/bloomberg/gists{/gist_id}","starred_url":"https://api.github.com/users/bloomberg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bloomberg/subscriptions","organizations_url":"https://api.github.com/users/bloomberg/orgs","repos_url":"https://api.github.com/users/bloomberg/repos","events_url":"https://api.github.com/users/bloomberg/events{/privacy}","received_events_url":"https://api.github.com/users/bloomberg/received_events","type":"Organization","site_admin":false},"repo":{"id":24933148,"name":"chef-bach","full_name":"bloomberg/chef-bach","owner":{"login":"bloomberg","id":1416818,"avatar_url":"https://avatars.githubusercontent.com/u/1416818?v=3","gravatar_id":"","url":"https://api.github.com/users/bloomberg","html_url":"https://github.com/bloomberg","followers_url":"https://api.github.com/users/bloomberg/followers","following_url":"https://api.github.com/users/bloomberg/following{/other_user}","gists_url":"https://api.github.com/users/bloomberg/gists{/gist_id}","starred_url":"https://api.github.com/users/bloomberg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bloomberg/subscriptions","organizations_url":"https://api.github.com/users/bloomberg/orgs","repos_url":"https://api.github.com/users/bloomberg/repos","events_url":"https://api.github.com/users/bloomberg/events{/privacy}","received_events_url":"https://api.github.com/users/bloomberg/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/bloomberg/chef-bach","description":"Chef recipes for Bloomberg's deployment of Hadoop and related components","fork":false,"url":"https://api.github.com/repos/bloomberg/chef-bach","forks_url":"https://api.github.com/repos/bloomberg/chef-bach/forks","keys_url":"https://api.github.com/repos/bloomberg/chef-bach/keys{/key_id}","collaborators_url":"https://api.github.com/repos/bloomberg/chef-bach/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/bloomberg/chef-bach/teams","hooks_url":"https://api.github.com/repos/bloomberg/chef-bach/hooks","issue_events_url":"https://api.github.com/repos/bloomberg/chef-bach/issues/events{/number}","events_url":"https://api.github.com/repos/bloomberg/chef-bach/events","assignees_url":"https://api.github.com/repos/bloomberg/chef-bach/assignees{/user}","branches_url":"https://api.github.com/repos/bloomberg/chef-bach/branches{/branch}","tags_url":"https://api.github.com/repos/bloomberg/chef-bach/tags","blobs_url":"https://api.github.com/repos/bloomberg/chef-bach/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/bloomberg/chef-bach/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/bloomberg/chef-bach/git/refs{/sha}","trees_url":"https://api.github.com/repos/bloomberg/chef-bach/git/trees{/sha}","statuses_url":"https://api.github.com/repos/bloomberg/chef-bach/statuses/{sha}","languages_url":"https://api.github.com/repos/bloomberg/chef-bach/languages","stargazers_url":"https://api.github.com/repos/bloomberg/chef-bach/stargazers","contributors_url":"https://api.github.com/repos/bloomberg/chef-bach/contributors","subscribers_url":"https://api.github.com/repos/bloomberg/chef-bach/subscribers","subscription_url":"https://api.github.com/repos/bloomberg/chef-bach/subscription","commits_url":"https://api.github.com/repos/bloomberg/chef-bach/commits{/sha}","git_commits_url":"https://api.github.com/repos/bloomberg/chef-bach/git/commits{/sha}","comments_url":"https://api.github.com/repos/bloomberg/chef-bach/comments{/number}","issue_comment_url":"https://api.github.com/repos/bloomberg/chef-bach/issues/comments{/number}","contents_url":"https://api.github.com/repos/bloomberg/chef-bach/contents/{+path}","compare_url":"https://api.github.com/repos/bloomberg/chef-bach/compare/{base}...{head}","merges_url":"https://api.github.com/repos/bloomberg/chef-bach/merges","archive_url":"https://api.github.com/repos/bloomberg/chef-bach/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/bloomberg/chef-bach/downloads","issues_url":"https://api.github.com/repos/bloomberg/chef-bach/issues{/number}","pulls_url":"https://api.github.com/repos/bloomberg/chef-bach/pulls{/number}","milestones_url":"https://api.github.com/repos/bloomberg/chef-bach/milestones{/number}","notifications_url":"https://api.github.com/repos/bloomberg/chef-bach/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/bloomberg/chef-bach/labels{/name}","releases_url":"https://api.github.com/repos/bloomberg/chef-bach/releases{/id}","created_at":"2014-10-08T09:16:26Z","updated_at":"2015-07-29T01:21:18Z","pushed_at":"2015-08-27T18:40:52Z","git_url":"git://github.com/bloomberg/chef-bach.git","ssh_url":"git@github.com:bloomberg/chef-bach.git","clone_url":"https://github.com/bloomberg/chef-bach.git","svn_url":"https://github.com/bloomberg/chef-bach","homepage":null,"size":4802,"stargazers_count":17,"watchers_count":17,"language":"HTML","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":12,"mirror_url":null,"open_issues_count":73,"forks":12,"open_issues":73,"watchers":17,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/255"},"html":{"href":"https://github.com/bloomberg/chef-bach/pull/255"},"issue":{"href":"https://api.github.com/repos/bloomberg/chef-bach/issues/255"},"comments":{"href":"https://api.github.com/repos/bloomberg/chef-bach/issues/255/comments"},"review_comments":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/255/comments"},"review_comment":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/bloomberg/chef-bach/pulls/255/commits"},"statuses":{"href":"https://api.github.com/repos/bloomberg/chef-bach/statuses/63f277e393cc7f475cb47704dd1b84ef938f7003"}}}}