{"action":"created","comment":{"url":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/comments/46375063","id":46375063,"diff_hunk":"@@ -0,0 +1,167 @@\n+#\n+# Cookbook Name:: hadoop\n+# Attribute:: zzz_system_tuning\n+#\n+# Copyright © 2013-2015 Cask Data, Inc.\n+#\n+# Licensed under the Apache License, Version 2.0 (the \"License\");\n+# you may not use this file except in compliance with the License.\n+# You may obtain a copy of the License at\n+#\n+#    http://www.apache.org/licenses/LICENSE-2.0\n+#\n+# Unless required by applicable law or agreed to in writing, software\n+# distributed under the License is distributed on an \"AS IS\" BASIS,\n+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+# See the License for the specific language governing permissions and\n+# limitations under the License.\n+#\n+\n+# This attributes file sets up the port reservations for sysctl for COOK-79. Since these values must be compiled into\n+# a single value, everything is done in this one file, and is done at Chef compile time, versus in a recipe. This is\n+# to attempt to ensure that the cookbook only gets a single value for this setting on any one cluster.\n+#\n+# The source for these ports is http://private-repo-1.hortonworks.com.s3.amazonaws.com/HDPDocuments/HDP2/HDP-2.0.9.0/bk_using_Ambari_book/content/reference_chap2_2x.html\n+\n+# net.ipv4.ip_local_reserved_ports setting (COOK-79)\n+default['hadoop']['sysctl']['net.ipv4.ip_local_reserved_ports'] = []\n+ports = []\n+\n+###\n+# HDFS\n+###\n+\n+# fs.defaultFS\n+if node['hadoop']['core_site'].key?('fs.defaultFS')\n+  # Embedded in URI? hdfs://hostname:port\n+  if node['hadoop']['core_site']['fs.defaultFS'].split(':')[2]\n+    ports += [node['hadoop']['core_site']['fs.defaultFS'].split(':')[2].to_i]\n+  else\n+    # Default port for hdfs:// is 8020\n+    ports += [8020]\n+  end\n+else\n+  ports += [8020]\n+end\n+\n+# These are all address:port pairs with no protocol prefix specified\n+# eg. 0.0.0.0:50070\n+addr_ports = {\n+  'dfs.namenode.http-address' => 50_070,\n+  'dfs.namenode.https-address' => 50_470,\n+  'dfs.datanode.http.address' => 50_075,\n+  'dfs.datanode.https.address' => 50_475,\n+  'dfs.namenode.secondary.http-address' => 50_090,\n+  'dfs.namenode.secondary.https-address' => 50_091,\n+  'dfs.datanode.address' => 50_010,\n+  'dfs.datanode.ipc.address' => 50_020,\n+  'dfs.namenode.backup.address' => 50_100,\n+  'dfs.namenode.backup.http-address' => 50_105,\n+  'dfs.journalnode.rpc-address' => 8485,\n+  'dfs.journalnode.http-address' => 8480,\n+  'dfs.journalnode.https-address' => 8481\n+}\n+\n+addr_ports.each do |k,v|\n+  if node['hadoop']['hdfs_site'].key?(k)\n+    ports += [node['hadoop']['hdfs_site'][k].split(':')[1].to_i]\n+  else\n+    ports += [v]\n+  end\n+end\n+\n+###\n+# MapReduce\n+###\n+\n+if node['hadoop'].key?('mapred_site') && node['hadoop']['mapred_site'].key?('mapreduce.jobhistory.webapp.address')\n+  ports += [node['hadoop']['mapred_site']['mapreduce.jobhistory.webapp.address'].split(':')[1].to_i]\n+else\n+  ports += [19_888]\n+end\n+\n+###\n+# YARN\n+###\n+\n+addr_ports = {\n+  'yarn.resourcemanager.scheduler.address' => 8030,\n+  'yarn.resourcemanager.resource-tracker.address' => 8031,\n+  'yarn.resourcemanager.address' => 8032,\n+  'yarn.resourcemanager.admin.address' => 8033,\n+  'yarn.resourcemanager.webapp.address' => 8088,\n+  'yarn.resourcemanager.webapp.https.address' => 8090,\n+  'yarn.nodemanager.localizer.address' => 8040,\n+  'yarn.nodemanager.webapp.address' => 8042,\n+  'yarn.timeline-service.address' => 10_200,\n+  'yarn.timeline-service.webapp.address' => 8188,\n+  'yarn.timeline-service.webapp.https.address' => 8190\n+}\n+\n+addr_ports.each do |k,v|\n+  if node['hadoop']['yarn_site'].key?('k')\n+    ports += [node['hadoop']['yarn_site'][k].split(':')[1].to_i]\n+  else\n+    ports += [v]\n+  end\n+end\n+\n+###\n+# HBase\n+###\n+\n+addr_ports = {\n+  'hbase.master.port' => 60_000,\n+  'hbase.master.info.port' => 60_010,\n+  'hbase.regionserver.port' => 60_020,\n+  'hbase.regionserver.info.port' => 60_030,\n+  'hbase.rest.port' => 8080,\n+  'hbase.rest.info.port' => 8085,\n+  'hbase.thrift.port' => 9090,\n+  'hbase.thrift.info.port' => 9095\n+}\n+\n+addr_ports.each do |k,v|\n+  if node['hbase']['hbase_site'].key?(k)\n+    ports += [node['hbase']['hbase_site'][k].to_i]\n+  else\n+    ports += [v]\n+  end\n+end\n+\n+###\n+# Hive\n+###\n+\n+if node['hive']['hive_site'].key?('hive.metastore.uris')\n+  ms_arr = node['hive']['hive_site']['hive.metastore.uris'].split(',')\n+  ms_arr.each do |ms|\n+    # thrift://hostname:port\n+    ports += [ms.split(':')[2].to_i]\n+  end\n+end\n+\n+###\n+# ZooKeeper\n+###\n+\n+if node['zookeeper'].key?('zoocfg') && node['zookeeper']['zoocfg'].key?('clientPort')","path":"attributes/zzz_system_tuning.rb","position":148,"original_position":148,"commit_id":"1783a1c1311eac436d99368426f070e014cee7f0","original_commit_id":"1783a1c1311eac436d99368426f070e014cee7f0","user":{"login":"dereklwood","id":1816517,"avatar_url":"https://avatars.githubusercontent.com/u/1816517?v=3","gravatar_id":"","url":"https://api.github.com/users/dereklwood","html_url":"https://github.com/dereklwood","followers_url":"https://api.github.com/users/dereklwood/followers","following_url":"https://api.github.com/users/dereklwood/following{/other_user}","gists_url":"https://api.github.com/users/dereklwood/gists{/gist_id}","starred_url":"https://api.github.com/users/dereklwood/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dereklwood/subscriptions","organizations_url":"https://api.github.com/users/dereklwood/orgs","repos_url":"https://api.github.com/users/dereklwood/repos","events_url":"https://api.github.com/users/dereklwood/events{/privacy}","received_events_url":"https://api.github.com/users/dereklwood/received_events","type":"User","site_admin":false},"body":"we check if ``node['zookeeper'].key?('zoocfg')`` here, but not below on line 156. We should be consistent.  I think since there is a ``node['zookeeper']['zoocfg']`` default attribute, we can remove the check here.","created_at":"2015-12-02T03:59:04Z","updated_at":"2015-12-02T03:59:04Z","html_url":"https://github.com/caskdata/hadoop_cookbook/pull/233#discussion_r46375063","pull_request_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/233","_links":{"self":{"href":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/comments/46375063"},"html":{"href":"https://github.com/caskdata/hadoop_cookbook/pull/233#discussion_r46375063"},"pull_request":{"href":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/233"}}},"pull_request":{"url":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/233","id":52339085,"html_url":"https://github.com/caskdata/hadoop_cookbook/pull/233","diff_url":"https://github.com/caskdata/hadoop_cookbook/pull/233.diff","patch_url":"https://github.com/caskdata/hadoop_cookbook/pull/233.patch","issue_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/issues/233","number":233,"state":"open","locked":false,"title":"COOK-79 Reserve Hadoop ports from being used as local ports","user":{"login":"wolf31o2","id":380021,"avatar_url":"https://avatars.githubusercontent.com/u/380021?v=3","gravatar_id":"","url":"https://api.github.com/users/wolf31o2","html_url":"https://github.com/wolf31o2","followers_url":"https://api.github.com/users/wolf31o2/followers","following_url":"https://api.github.com/users/wolf31o2/following{/other_user}","gists_url":"https://api.github.com/users/wolf31o2/gists{/gist_id}","starred_url":"https://api.github.com/users/wolf31o2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wolf31o2/subscriptions","organizations_url":"https://api.github.com/users/wolf31o2/orgs","repos_url":"https://api.github.com/users/wolf31o2/repos","events_url":"https://api.github.com/users/wolf31o2/events{/privacy}","received_events_url":"https://api.github.com/users/wolf31o2/received_events","type":"User","site_admin":false},"body":"This takes the configured ports (or defaults) from several Hadoop\r\nservices (HDFS, MapReduce, YARN, ZooKeeper, HBase, and Hive) and\r\ninstructs the Linux kernel to skip those ports when automatically\r\nassigning local client ports for TCP connections. This prevents an\r\nissue where services fail to start due to a client connection open\r\nand using the configured daemon port.","created_at":"2015-12-02T00:49:11Z","updated_at":"2015-12-02T03:59:05Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4e4ab8af6a1c5da9e9e2ce7d5e7a7b5ddfde375b","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/233/commits","review_comments_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/233/comments","review_comment_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/comments{/number}","comments_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/issues/233/comments","statuses_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/statuses/1783a1c1311eac436d99368426f070e014cee7f0","head":{"label":"caskdata:feature/cook-79","ref":"feature/cook-79","sha":"1783a1c1311eac436d99368426f070e014cee7f0","user":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"repo":{"id":13817108,"name":"hadoop_cookbook","full_name":"caskdata/hadoop_cookbook","owner":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/caskdata/hadoop_cookbook","description":"Cookbook to install Hadoop 2.0+ using Chef","fork":false,"url":"https://api.github.com/repos/caskdata/hadoop_cookbook","forks_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/forks","keys_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/keys{/key_id}","collaborators_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/teams","hooks_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/hooks","issue_events_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/issues/events{/number}","events_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/events","assignees_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/assignees{/user}","branches_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/branches{/branch}","tags_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/tags","blobs_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/git/refs{/sha}","trees_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/git/trees{/sha}","statuses_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/statuses/{sha}","languages_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/languages","stargazers_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/stargazers","contributors_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/contributors","subscribers_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/subscribers","subscription_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/subscription","commits_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/commits{/sha}","git_commits_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/git/commits{/sha}","comments_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/comments{/number}","issue_comment_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/issues/comments{/number}","contents_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/contents/{+path}","compare_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/compare/{base}...{head}","merges_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/merges","archive_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/downloads","issues_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/issues{/number}","pulls_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls{/number}","milestones_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/milestones{/number}","notifications_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/labels{/name}","releases_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/releases{/id}","created_at":"2013-10-23T22:56:32Z","updated_at":"2015-11-25T21:07:53Z","pushed_at":"2015-12-02T00:49:11Z","git_url":"git://github.com/caskdata/hadoop_cookbook.git","ssh_url":"git@github.com:caskdata/hadoop_cookbook.git","clone_url":"https://github.com/caskdata/hadoop_cookbook.git","svn_url":"https://github.com/caskdata/hadoop_cookbook","homepage":"http://supermarket.chef.io/cookbooks/hadoop","size":1036,"stargazers_count":50,"watchers_count":50,"language":"Ruby","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":47,"mirror_url":null,"open_issues_count":5,"forks":47,"open_issues":5,"watchers":50,"default_branch":"master"}},"base":{"label":"caskdata:master","ref":"master","sha":"860a83bfba7dfe8d34718ff2cd1c61813b1aad00","user":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"repo":{"id":13817108,"name":"hadoop_cookbook","full_name":"caskdata/hadoop_cookbook","owner":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/caskdata/hadoop_cookbook","description":"Cookbook to install Hadoop 2.0+ using Chef","fork":false,"url":"https://api.github.com/repos/caskdata/hadoop_cookbook","forks_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/forks","keys_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/keys{/key_id}","collaborators_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/teams","hooks_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/hooks","issue_events_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/issues/events{/number}","events_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/events","assignees_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/assignees{/user}","branches_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/branches{/branch}","tags_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/tags","blobs_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/git/refs{/sha}","trees_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/git/trees{/sha}","statuses_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/statuses/{sha}","languages_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/languages","stargazers_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/stargazers","contributors_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/contributors","subscribers_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/subscribers","subscription_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/subscription","commits_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/commits{/sha}","git_commits_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/git/commits{/sha}","comments_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/comments{/number}","issue_comment_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/issues/comments{/number}","contents_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/contents/{+path}","compare_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/compare/{base}...{head}","merges_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/merges","archive_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/downloads","issues_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/issues{/number}","pulls_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls{/number}","milestones_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/milestones{/number}","notifications_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/labels{/name}","releases_url":"https://api.github.com/repos/caskdata/hadoop_cookbook/releases{/id}","created_at":"2013-10-23T22:56:32Z","updated_at":"2015-11-25T21:07:53Z","pushed_at":"2015-12-02T00:49:11Z","git_url":"git://github.com/caskdata/hadoop_cookbook.git","ssh_url":"git@github.com:caskdata/hadoop_cookbook.git","clone_url":"https://github.com/caskdata/hadoop_cookbook.git","svn_url":"https://github.com/caskdata/hadoop_cookbook","homepage":"http://supermarket.chef.io/cookbooks/hadoop","size":1036,"stargazers_count":50,"watchers_count":50,"language":"Ruby","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":47,"mirror_url":null,"open_issues_count":5,"forks":47,"open_issues":5,"watchers":50,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/233"},"html":{"href":"https://github.com/caskdata/hadoop_cookbook/pull/233"},"issue":{"href":"https://api.github.com/repos/caskdata/hadoop_cookbook/issues/233"},"comments":{"href":"https://api.github.com/repos/caskdata/hadoop_cookbook/issues/233/comments"},"review_comments":{"href":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/233/comments"},"review_comment":{"href":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/caskdata/hadoop_cookbook/pulls/233/commits"},"statuses":{"href":"https://api.github.com/repos/caskdata/hadoop_cookbook/statuses/1783a1c1311eac436d99368426f070e014cee7f0"}}}}