{"action":"created","comment":{"url":"https://api.github.com/repos/18F/foia/pulls/comments/24587558","id":24587558,"diff_hunk":"@@ -1,132 +1,168 @@\n-#!/usr/bin/env python\n-\n-import yaml\n-import xlrd\n import os\n-import json\n-from glob import glob\n import re\n-import logging\n-\n-\"\"\" Update YAML files with usa_id, description, and acronyms. \"\"\"\n-\n-ACRONYM_FINDER = re.compile('\\((.*?)\\)')\n-\n-\n-def float_to_int_str(number):\n-    \"\"\"Converts input to ints and then to strings to clean xls data.\"\"\"\n-    try:\n-        return str(int(float(number)))\n-    except:\n-        return number\n-\n-\n-def extract_acronym(usa_name):\n-    \"\"\"Extracts acronyms from a string if there is one.\"\"\"\n-    acronym_list = ACRONYM_FINDER.findall(usa_name)\n-    if len(acronym_list) == 1:\n-        return acronym_list[0]\n-    elif len(acronym_list) == 0:\n-        return \"\"\n-    else:\n-        return \"Massive Error\"\n-\n-\n-def load_all_usa_data():\n-    \"\"\"Loads data from all_usa_data.json file.\"\"\"\n-    with open('layering_data/all_usa_data.json', 'r') as f:\n-        all_usa_data = json.loads(f.read())\n-    data = {}\n-    for office in all_usa_data:\n-        if office.get('Language') == \"en\":\n-\n-            data[office['Name']] = {\n-                'description': office.get('Description', 'No Description'),\n-                'id': office.get('Id', 'No Id'),\n-                'acronym_usa_contacts': extract_acronym(office['Name'])}\n-    return data\n-\n+import yaml\n \n-def load_usacontacts():\n-    \"\"\"Loads data from usacontacts.xls \"\"\"\n-    data = {}\n-    xls_path = \"layering_data/foiaHub-usaContacts-matches.xlsx\"\n-    workbook = xlrd.open_workbook(xls_path)\n-    sheet = workbook.sheet_by_name(workbook.sheet_names()[0])\n-    header_names = [sheet.cell_value(0, i) for i in range(sheet.ncols)]\n-    for row_num in range(1, sheet.nrows):\n-        row = {\n-            header_names[i]: sheet.cell_value(row_num, i)\n-            for i in range(sheet.ncols)}\n-        row['usa_id'] = float_to_int_str(row['usa_id'])\n-\n-        if row.get('description', \"No Description\") == \"\":\n-            row['description'] = \"No Description\"\n-        data[row['fh_name']] = row\n+from glob import glob\n+from requests_cache.core import CachedSession\n+\n+\"\"\"\n+This script updates the yaml files with usa_id, description, and acronyms.\n+\"\"\"\n+\n+\n+USA_CONTACTS_API = 'http://www.usa.gov/api/USAGovAPI/contacts.json/contacts'\n+ACRONYM = re.compile('\\((.*?)\\)')\n+\n+# Subsitutions and replacements for name, must be in this order\n+REPLACEMENTS = (\n+    (\"Purchase from People Who Are Blind or Severely Disabled\",\n+        \"U.S. AbilityOne Commission\"),\n+    (\"Office of the Secretary and Joint Staff\", \"Joint Chiefs of Staff\"),\n+    (\"Department of the Army - Freedom of Information and Privacy Office\",\n+        \"U.S. Army\"),\n+    (\"Marine Corps - FOIA Program Office (ARSF)\", \"U.S. Marines\"),\n+    ('Federal Bureau of Prisons', 'Bureau of Prisons'),\n+    ('Office of Community Oriented Policing Services',\n+        'Community Oriented Policing Services'),\n+    ('AMTRAK', 'National Railroad Passenger Corporation'),\n+    ('Jobs Corps', 'Job Corps'),\n+    ('INTERPOL-United States National Central Bureau',\n+        'U.S. National Central Bureau - Interpol'),\n+    ('Center for', 'Centers for'),\n+    (' for the District of Columbia', ''),\n+    (' - FOIA Program Office (ARSF)', ''),\n+    (' - Main Office', ''),\n+    (' Activity', ''),\n+    (' - Headquarters Office', ''),\n+    (' - Headquarters', ''),\n+    ('U.S.', ''),\n+    ('United States', ''),\n+    (' & ', ' and '),\n+    ('Bureau', ''),\n+    ('Committee for ', ''),\n+    ('Office of the ', ''),\n+    ('/ICIO', ''),\n+    (' of the ', ' of '),\n+    ('Department of ', ''),\n+)\n+\n+\n+def clean_name(name):\n+    \"\"\" Cleans name to try to match it with names in yaml files \"\"\"\n+\n+    name = ACRONYM.sub('', name)\n+    for item, replacement in REPLACEMENTS:\n+        name = name.replace(item, replacement)\n+    return name.strip(' ')\n+\n+\n+def extract_abbreviation(name):\n+    \"\"\" Extract abbreviation from name string \"\"\"\n+\n+    match = ACRONYM.search(name)\n+    abbreviation = None\n+    if match:\n+        abbreviation = match.group(0).strip(\"() \")\n+    return abbreviation\n+\n+\n+def create_contact_dict(data):\n+    \"\"\"\n+    Generates a dictionary containing a usa id, description, and\n+    abbreviation when possible\n+    \"\"\"\n+\n+    new_dict = {'usa_id': data['Id']}\n+    abbreviation = extract_abbreviation(name=data['Name'])\n+    if abbreviation:\n+        new_dict.update({'abbreviation': abbreviation})","path":"contacts/layer_with_usa_contacts.py","position":140,"original_position":140,"commit_id":"ea7854183415164f0d0651964eb272010f559244","original_commit_id":"ea7854183415164f0d0651964eb272010f559244","user":{"login":"ramirezg","id":4596845,"avatar_url":"https://avatars.githubusercontent.com/u/4596845?v=3","gravatar_id":"","url":"https://api.github.com/users/ramirezg","html_url":"https://github.com/ramirezg","followers_url":"https://api.github.com/users/ramirezg/followers","following_url":"https://api.github.com/users/ramirezg/following{/other_user}","gists_url":"https://api.github.com/users/ramirezg/gists{/gist_id}","starred_url":"https://api.github.com/users/ramirezg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ramirezg/subscriptions","organizations_url":"https://api.github.com/users/ramirezg/orgs","repos_url":"https://api.github.com/users/ramirezg/repos","events_url":"https://api.github.com/users/ramirezg/events{/privacy}","received_events_url":"https://api.github.com/users/ramirezg/received_events","type":"User","site_admin":false},"body":"no idea why I used update. ","created_at":"2015-02-12T15:17:03Z","updated_at":"2015-02-12T15:17:03Z","html_url":"https://github.com/18F/foia/pull/103#discussion_r24587558","pull_request_url":"https://api.github.com/repos/18F/foia/pulls/103","_links":{"self":{"href":"https://api.github.com/repos/18F/foia/pulls/comments/24587558"},"html":{"href":"https://github.com/18F/foia/pull/103#discussion_r24587558"},"pull_request":{"href":"https://api.github.com/repos/18F/foia/pulls/103"}}},"pull_request":{"url":"https://api.github.com/repos/18F/foia/pulls/103","id":28590296,"html_url":"https://github.com/18F/foia/pull/103","diff_url":"https://github.com/18F/foia/pull/103.diff","patch_url":"https://github.com/18F/foia/pull/103.patch","issue_url":"https://api.github.com/repos/18F/foia/issues/103","number":103,"state":"open","locked":false,"title":"Adding additional descriptions","user":{"login":"ramirezg","id":4596845,"avatar_url":"https://avatars.githubusercontent.com/u/4596845?v=3","gravatar_id":"","url":"https://api.github.com/users/ramirezg","html_url":"https://github.com/ramirezg","followers_url":"https://api.github.com/users/ramirezg/followers","following_url":"https://api.github.com/users/ramirezg/following{/other_user}","gists_url":"https://api.github.com/users/ramirezg/gists{/gist_id}","starred_url":"https://api.github.com/users/ramirezg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ramirezg/subscriptions","organizations_url":"https://api.github.com/users/ramirezg/orgs","repos_url":"https://api.github.com/users/ramirezg/repos","events_url":"https://api.github.com/users/ramirezg/events{/privacy}","received_events_url":"https://api.github.com/users/ramirezg/received_events","type":"User","site_admin":false},"body":"8 of the 9 descriptions were added via the all_usa_contacts.json file. \r\n\r\nThe descriptions for these agencies were not included in the first iteration because originally they were not top-level agencies due to a [manual data mapping issue](https://github.com/18F/foia/pull/74).\r\n\r\nLooking back at this code, I realize that it might be possible to also add descriptions to many, many more non-top level agencies with some mapping work.  ","created_at":"2015-02-03T21:24:12Z","updated_at":"2015-02-12T15:17:03Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b6734065c531828f4f43bedc9333f7f53b57ddd0","assignee":{"login":"khandelwal","id":636752,"avatar_url":"https://avatars.githubusercontent.com/u/636752?v=3","gravatar_id":"","url":"https://api.github.com/users/khandelwal","html_url":"https://github.com/khandelwal","followers_url":"https://api.github.com/users/khandelwal/followers","following_url":"https://api.github.com/users/khandelwal/following{/other_user}","gists_url":"https://api.github.com/users/khandelwal/gists{/gist_id}","starred_url":"https://api.github.com/users/khandelwal/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/khandelwal/subscriptions","organizations_url":"https://api.github.com/users/khandelwal/orgs","repos_url":"https://api.github.com/users/khandelwal/repos","events_url":"https://api.github.com/users/khandelwal/events{/privacy}","received_events_url":"https://api.github.com/users/khandelwal/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/18F/foia/pulls/103/commits","review_comments_url":"https://api.github.com/repos/18F/foia/pulls/103/comments","review_comment_url":"https://api.github.com/repos/18F/foia/pulls/comments/{number}","comments_url":"https://api.github.com/repos/18F/foia/issues/103/comments","statuses_url":"https://api.github.com/repos/18F/foia/statuses/ea7854183415164f0d0651964eb272010f559244","head":{"label":"18F:missing_data","ref":"missing_data","sha":"ea7854183415164f0d0651964eb272010f559244","user":{"login":"18F","id":6233994,"avatar_url":"https://avatars.githubusercontent.com/u/6233994?v=3","gravatar_id":"","url":"https://api.github.com/users/18F","html_url":"https://github.com/18F","followers_url":"https://api.github.com/users/18F/followers","following_url":"https://api.github.com/users/18F/following{/other_user}","gists_url":"https://api.github.com/users/18F/gists{/gist_id}","starred_url":"https://api.github.com/users/18F/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/18F/subscriptions","organizations_url":"https://api.github.com/users/18F/orgs","repos_url":"https://api.github.com/users/18F/repos","events_url":"https://api.github.com/users/18F/events{/privacy}","received_events_url":"https://api.github.com/users/18F/received_events","type":"Organization","site_admin":false},"repo":{"id":20223769,"name":"foia","full_name":"18F/foia","owner":{"login":"18F","id":6233994,"avatar_url":"https://avatars.githubusercontent.com/u/6233994?v=3","gravatar_id":"","url":"https://api.github.com/users/18F","html_url":"https://github.com/18F","followers_url":"https://api.github.com/users/18F/followers","following_url":"https://api.github.com/users/18F/following{/other_user}","gists_url":"https://api.github.com/users/18F/gists{/gist_id}","starred_url":"https://api.github.com/users/18F/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/18F/subscriptions","organizations_url":"https://api.github.com/users/18F/orgs","repos_url":"https://api.github.com/users/18F/repos","events_url":"https://api.github.com/users/18F/events{/privacy}","received_events_url":"https://api.github.com/users/18F/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/18F/foia","description":"Project wide discussion, blogging, and scratch space for 18F's FOIA modernization team.","fork":false,"url":"https://api.github.com/repos/18F/foia","forks_url":"https://api.github.com/repos/18F/foia/forks","keys_url":"https://api.github.com/repos/18F/foia/keys{/key_id}","collaborators_url":"https://api.github.com/repos/18F/foia/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/18F/foia/teams","hooks_url":"https://api.github.com/repos/18F/foia/hooks","issue_events_url":"https://api.github.com/repos/18F/foia/issues/events{/number}","events_url":"https://api.github.com/repos/18F/foia/events","assignees_url":"https://api.github.com/repos/18F/foia/assignees{/user}","branches_url":"https://api.github.com/repos/18F/foia/branches{/branch}","tags_url":"https://api.github.com/repos/18F/foia/tags","blobs_url":"https://api.github.com/repos/18F/foia/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/18F/foia/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/18F/foia/git/refs{/sha}","trees_url":"https://api.github.com/repos/18F/foia/git/trees{/sha}","statuses_url":"https://api.github.com/repos/18F/foia/statuses/{sha}","languages_url":"https://api.github.com/repos/18F/foia/languages","stargazers_url":"https://api.github.com/repos/18F/foia/stargazers","contributors_url":"https://api.github.com/repos/18F/foia/contributors","subscribers_url":"https://api.github.com/repos/18F/foia/subscribers","subscription_url":"https://api.github.com/repos/18F/foia/subscription","commits_url":"https://api.github.com/repos/18F/foia/commits{/sha}","git_commits_url":"https://api.github.com/repos/18F/foia/git/commits{/sha}","comments_url":"https://api.github.com/repos/18F/foia/comments{/number}","issue_comment_url":"https://api.github.com/repos/18F/foia/issues/comments/{number}","contents_url":"https://api.github.com/repos/18F/foia/contents/{+path}","compare_url":"https://api.github.com/repos/18F/foia/compare/{base}...{head}","merges_url":"https://api.github.com/repos/18F/foia/merges","archive_url":"https://api.github.com/repos/18F/foia/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/18F/foia/downloads","issues_url":"https://api.github.com/repos/18F/foia/issues{/number}","pulls_url":"https://api.github.com/repos/18F/foia/pulls{/number}","milestones_url":"https://api.github.com/repos/18F/foia/milestones{/number}","notifications_url":"https://api.github.com/repos/18F/foia/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/18F/foia/labels{/name}","releases_url":"https://api.github.com/repos/18F/foia/releases{/id}","created_at":"2014-05-27T14:52:27Z","updated_at":"2015-02-03T07:51:07Z","pushed_at":"2015-02-06T15:01:37Z","git_url":"git://github.com/18F/foia.git","ssh_url":"git@github.com:18F/foia.git","clone_url":"https://github.com/18F/foia.git","svn_url":"https://github.com/18F/foia","homepage":"","size":12259,"stargazers_count":35,"watchers_count":35,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":7,"mirror_url":null,"open_issues_count":2,"forks":7,"open_issues":2,"watchers":35,"default_branch":"master"}},"base":{"label":"18F:master","ref":"master","sha":"1b7ae846eef2394dfd16af66f0b279f05d195a74","user":{"login":"18F","id":6233994,"avatar_url":"https://avatars.githubusercontent.com/u/6233994?v=3","gravatar_id":"","url":"https://api.github.com/users/18F","html_url":"https://github.com/18F","followers_url":"https://api.github.com/users/18F/followers","following_url":"https://api.github.com/users/18F/following{/other_user}","gists_url":"https://api.github.com/users/18F/gists{/gist_id}","starred_url":"https://api.github.com/users/18F/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/18F/subscriptions","organizations_url":"https://api.github.com/users/18F/orgs","repos_url":"https://api.github.com/users/18F/repos","events_url":"https://api.github.com/users/18F/events{/privacy}","received_events_url":"https://api.github.com/users/18F/received_events","type":"Organization","site_admin":false},"repo":{"id":20223769,"name":"foia","full_name":"18F/foia","owner":{"login":"18F","id":6233994,"avatar_url":"https://avatars.githubusercontent.com/u/6233994?v=3","gravatar_id":"","url":"https://api.github.com/users/18F","html_url":"https://github.com/18F","followers_url":"https://api.github.com/users/18F/followers","following_url":"https://api.github.com/users/18F/following{/other_user}","gists_url":"https://api.github.com/users/18F/gists{/gist_id}","starred_url":"https://api.github.com/users/18F/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/18F/subscriptions","organizations_url":"https://api.github.com/users/18F/orgs","repos_url":"https://api.github.com/users/18F/repos","events_url":"https://api.github.com/users/18F/events{/privacy}","received_events_url":"https://api.github.com/users/18F/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/18F/foia","description":"Project wide discussion, blogging, and scratch space for 18F's FOIA modernization team.","fork":false,"url":"https://api.github.com/repos/18F/foia","forks_url":"https://api.github.com/repos/18F/foia/forks","keys_url":"https://api.github.com/repos/18F/foia/keys{/key_id}","collaborators_url":"https://api.github.com/repos/18F/foia/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/18F/foia/teams","hooks_url":"https://api.github.com/repos/18F/foia/hooks","issue_events_url":"https://api.github.com/repos/18F/foia/issues/events{/number}","events_url":"https://api.github.com/repos/18F/foia/events","assignees_url":"https://api.github.com/repos/18F/foia/assignees{/user}","branches_url":"https://api.github.com/repos/18F/foia/branches{/branch}","tags_url":"https://api.github.com/repos/18F/foia/tags","blobs_url":"https://api.github.com/repos/18F/foia/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/18F/foia/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/18F/foia/git/refs{/sha}","trees_url":"https://api.github.com/repos/18F/foia/git/trees{/sha}","statuses_url":"https://api.github.com/repos/18F/foia/statuses/{sha}","languages_url":"https://api.github.com/repos/18F/foia/languages","stargazers_url":"https://api.github.com/repos/18F/foia/stargazers","contributors_url":"https://api.github.com/repos/18F/foia/contributors","subscribers_url":"https://api.github.com/repos/18F/foia/subscribers","subscription_url":"https://api.github.com/repos/18F/foia/subscription","commits_url":"https://api.github.com/repos/18F/foia/commits{/sha}","git_commits_url":"https://api.github.com/repos/18F/foia/git/commits{/sha}","comments_url":"https://api.github.com/repos/18F/foia/comments{/number}","issue_comment_url":"https://api.github.com/repos/18F/foia/issues/comments/{number}","contents_url":"https://api.github.com/repos/18F/foia/contents/{+path}","compare_url":"https://api.github.com/repos/18F/foia/compare/{base}...{head}","merges_url":"https://api.github.com/repos/18F/foia/merges","archive_url":"https://api.github.com/repos/18F/foia/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/18F/foia/downloads","issues_url":"https://api.github.com/repos/18F/foia/issues{/number}","pulls_url":"https://api.github.com/repos/18F/foia/pulls{/number}","milestones_url":"https://api.github.com/repos/18F/foia/milestones{/number}","notifications_url":"https://api.github.com/repos/18F/foia/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/18F/foia/labels{/name}","releases_url":"https://api.github.com/repos/18F/foia/releases{/id}","created_at":"2014-05-27T14:52:27Z","updated_at":"2015-02-03T07:51:07Z","pushed_at":"2015-02-06T15:01:37Z","git_url":"git://github.com/18F/foia.git","ssh_url":"git@github.com:18F/foia.git","clone_url":"https://github.com/18F/foia.git","svn_url":"https://github.com/18F/foia","homepage":"","size":12259,"stargazers_count":35,"watchers_count":35,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":7,"mirror_url":null,"open_issues_count":2,"forks":7,"open_issues":2,"watchers":35,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/18F/foia/pulls/103"},"html":{"href":"https://github.com/18F/foia/pull/103"},"issue":{"href":"https://api.github.com/repos/18F/foia/issues/103"},"comments":{"href":"https://api.github.com/repos/18F/foia/issues/103/comments"},"review_comments":{"href":"https://api.github.com/repos/18F/foia/pulls/103/comments"},"review_comment":{"href":"https://api.github.com/repos/18F/foia/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/18F/foia/pulls/103/commits"},"statuses":{"href":"https://api.github.com/repos/18F/foia/statuses/ea7854183415164f0d0651964eb272010f559244"}}}}