{"action":"created","comment":{"url":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/comments/42113427","id":42113427,"diff_hunk":"@@ -0,0 +1,274 @@\n+# -*- coding: utf-8 -*-\n+\n+from datetime import date\n+import dateutil.parser\n+import csv\n+from os.path import dirname, join\n+import re\n+import string\n+import urllib\n+import codecs\n+\n+import requests\n+from slumber.exceptions import HttpClientError\n+\n+from django.conf import settings\n+from django.contrib.auth.models import User\n+from django.core.files.storage import FileSystemStorage\n+from django.core.management.base import BaseCommand, CommandError\n+\n+from candidates.cache import get_post_cached\n+from candidates.election_specific import MAPIT_DATA, PARTY_DATA, AREA_POST_DATA\n+from candidates.models import PopItPerson\n+from candidates.popit import create_popit_api_object, get_search_url\n+from candidates.utils import strip_accents\n+from candidates.views.version_data import get_change_metadata\n+\n+UNKNOWN_PARTY_ID = 'unknown'\n+USER_AGENT = (\n+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '\n+    '(KHTML, like Gecko) Ubuntu Chromium/38.0.2125.111 '\n+    'Chrome/38.0.2125.111Safari/537.36'\n+)\n+\n+def get_post_data(api, election_id, province=None):\n+    ynr_election_data = settings.ELECTIONS[election_id]\n+    ynr_election_data['id'] = election_id\n+    mapit_key = ( ynr_election_data['mapit_types'][0], ynr_election_data['mapit_generation'])\n+    if province:\n+        mapit_areas_by_name = MAPIT_DATA.areas_by_name[mapit_key]\n+        if province != 'Burkina Faso':\n+            province = strip_accents(province).upper()\n+        mapit_area = mapit_areas_by_name[province]\n+        post_id = AREA_POST_DATA.get_post_id(\n+            election_id, mapit_area['type'], mapit_area['id']\n+        )\n+    else:\n+        # It must be the presidential election:\n+        post_id = ynr_election_data['for_post_role']\n+    post_data = get_post_cached(api, post_id)['result']\n+    return ynr_election_data, post_data\n+\n+\n+def get_existing_popit_person(vi_person_id):\n+    # See if this person already exists by searching for the\n+    # ID they were imported with:\n+    query_format = \\\n+        'identifiers.identifier:\"{id}\" AND ' + \\\n+        'identifiers.scheme:\"{scheme}\"'\n+    search_url = get_search_url(\n+        'persons',\n+        query_format.format(\n+            id=vi_person_id, scheme='import-id'\n+        ),\n+        embed='membership.organization'\n+    )\n+    results = requests.get(search_url).json()\n+    total = results['total']\n+    if total > 1:\n+        message = \"Multiple matches for CI ID {0}\"\n+        raise Exception(message.format(vi_person_id))\n+    if total == 0:\n+        return None\n+    # Otherwise there was exactly one result:\n+    return PopItPerson.create_from_dict(results['result'][0])\n+\n+def get_party_data(party_name):\n+    # See if this person already exists by searching for the\n+    # ID they were imported with:\n+    party_name = party_name.replace('/', '')\n+    party_name = party_name.decode('utf-8')\n+    query_format = \\\n+        'name:\"{name}\"'\n+    search_url = get_search_url(\n+        'organizations',\n+        query_format.format(\n+            name=party_name\n+        )\n+    )\n+    print party_name\n+    results = requests.get(search_url).json()\n+    print results\n+    total = results['total']\n+    if total > 1:\n+        message = \"Multiple matches for party {0}\"\n+        raise Exception(message.format(party_name))\n+    if total == 0:\n+        return None\n+    # Otherwise there was exactly one result:\n+    return results['result'][0]\n+\n+def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):","path":"elections/bf_elections_2015/management/commands/bf_elections_2015_import_candidate.py","position":101,"original_position":101,"commit_id":"b223dbf50fcec6eb0596b2c56bdc6724a671ccfc","original_commit_id":"b223dbf50fcec6eb0596b2c56bdc6724a671ccfc","user":{"login":"mhl","id":7907,"avatar_url":"https://avatars.githubusercontent.com/u/7907?v=3","gravatar_id":"","url":"https://api.github.com/users/mhl","html_url":"https://github.com/mhl","followers_url":"https://api.github.com/users/mhl/followers","following_url":"https://api.github.com/users/mhl/following{/other_user}","gists_url":"https://api.github.com/users/mhl/gists{/gist_id}","starred_url":"https://api.github.com/users/mhl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mhl/subscriptions","organizations_url":"https://api.github.com/users/mhl/orgs","repos_url":"https://api.github.com/users/mhl/repos","events_url":"https://api.github.com/users/mhl/events{/privacy}","received_events_url":"https://api.github.com/users/mhl/received_events","type":"User","site_admin":false},"body":"I know this example is straight from the docs, but it's really surprising / confusing, I think. `csv.reader` usually takes a file-like object, but this example suggests that all `csv.reader` actually needs it something that will produce a string per row when you iterate over it. Fair enough, I suppose, but it's an assumption about the implementation of `csv.reader`. Also, it's just weird that what happens here is: `str [Windows 1252] --(codecs.open)--> unicode --(encode)--> str [UTF-8]` so that you can passing UTF-8 `str`s to `csv.reader`, and the have to called `decode` on the yielded cell data to turn it back into `unicode`.\r\n\r\nSo, anyway - I think the only change I'd make here is to add a comment with a link to https://docs.python.org/2/library/csv.html#examples indicating that these two functions are straight from there.","created_at":"2015-10-15T11:41:35Z","updated_at":"2015-10-15T11:41:35Z","html_url":"https://github.com/mysociety/yournextrepresentative/pull/555#discussion_r42113427","pull_request_url":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/555","_links":{"self":{"href":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/comments/42113427"},"html":{"href":"https://github.com/mysociety/yournextrepresentative/pull/555#discussion_r42113427"},"pull_request":{"href":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/555"}}},"pull_request":{"url":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/555","id":47405348,"html_url":"https://github.com/mysociety/yournextrepresentative/pull/555","diff_url":"https://github.com/mysociety/yournextrepresentative/pull/555.diff","patch_url":"https://github.com/mysociety/yournextrepresentative/pull/555.patch","issue_url":"https://api.github.com/repos/mysociety/yournextrepresentative/issues/555","number":555,"state":"open","locked":false,"title":"Import existing bf candidates","user":{"login":"struan","id":5310,"avatar_url":"https://avatars.githubusercontent.com/u/5310?v=3","gravatar_id":"","url":"https://api.github.com/users/struan","html_url":"https://github.com/struan","followers_url":"https://api.github.com/users/struan/followers","following_url":"https://api.github.com/users/struan/following{/other_user}","gists_url":"https://api.github.com/users/struan/gists{/gist_id}","starred_url":"https://api.github.com/users/struan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/struan/subscriptions","organizations_url":"https://api.github.com/users/struan/orgs","repos_url":"https://api.github.com/users/struan/repos","events_url":"https://api.github.com/users/struan/events{/privacy}","received_events_url":"https://api.github.com/users/struan/received_events","type":"User","site_admin":false},"body":"This takes CSVs of existing candidates from the official Burkina Faso site and imports them into YNR. It only imports the candidates and not the 'backup' candidates in the data.\r\n\r\nFixes #547","created_at":"2015-10-12T10:37:12Z","updated_at":"2015-10-15T11:41:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"950857673cf8c6198c8e3475b7d1fbcfba3c6e40","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/555/commits","review_comments_url":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/555/comments","review_comment_url":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mysociety/yournextrepresentative/issues/555/comments","statuses_url":"https://api.github.com/repos/mysociety/yournextrepresentative/statuses/b223dbf50fcec6eb0596b2c56bdc6724a671ccfc","head":{"label":"mysociety:import_existing_bf_candidates","ref":"import_existing_bf_candidates","sha":"b223dbf50fcec6eb0596b2c56bdc6724a671ccfc","user":{"login":"mysociety","id":141338,"avatar_url":"https://avatars.githubusercontent.com/u/141338?v=3","gravatar_id":"","url":"https://api.github.com/users/mysociety","html_url":"https://github.com/mysociety","followers_url":"https://api.github.com/users/mysociety/followers","following_url":"https://api.github.com/users/mysociety/following{/other_user}","gists_url":"https://api.github.com/users/mysociety/gists{/gist_id}","starred_url":"https://api.github.com/users/mysociety/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mysociety/subscriptions","organizations_url":"https://api.github.com/users/mysociety/orgs","repos_url":"https://api.github.com/users/mysociety/repos","events_url":"https://api.github.com/users/mysociety/events{/privacy}","received_events_url":"https://api.github.com/users/mysociety/received_events","type":"Organization","site_admin":false},"repo":{"id":23865131,"name":"yournextrepresentative","full_name":"mysociety/yournextrepresentative","owner":{"login":"mysociety","id":141338,"avatar_url":"https://avatars.githubusercontent.com/u/141338?v=3","gravatar_id":"","url":"https://api.github.com/users/mysociety","html_url":"https://github.com/mysociety","followers_url":"https://api.github.com/users/mysociety/followers","following_url":"https://api.github.com/users/mysociety/following{/other_user}","gists_url":"https://api.github.com/users/mysociety/gists{/gist_id}","starred_url":"https://api.github.com/users/mysociety/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mysociety/subscriptions","organizations_url":"https://api.github.com/users/mysociety/orgs","repos_url":"https://api.github.com/users/mysociety/repos","events_url":"https://api.github.com/users/mysociety/events{/privacy}","received_events_url":"https://api.github.com/users/mysociety/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mysociety/yournextrepresentative","description":"An experimental front-end to PopIt for 2015 GE candidates","fork":false,"url":"https://api.github.com/repos/mysociety/yournextrepresentative","forks_url":"https://api.github.com/repos/mysociety/yournextrepresentative/forks","keys_url":"https://api.github.com/repos/mysociety/yournextrepresentative/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mysociety/yournextrepresentative/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mysociety/yournextrepresentative/teams","hooks_url":"https://api.github.com/repos/mysociety/yournextrepresentative/hooks","issue_events_url":"https://api.github.com/repos/mysociety/yournextrepresentative/issues/events{/number}","events_url":"https://api.github.com/repos/mysociety/yournextrepresentative/events","assignees_url":"https://api.github.com/repos/mysociety/yournextrepresentative/assignees{/user}","branches_url":"https://api.github.com/repos/mysociety/yournextrepresentative/branches{/branch}","tags_url":"https://api.github.com/repos/mysociety/yournextrepresentative/tags","blobs_url":"https://api.github.com/repos/mysociety/yournextrepresentative/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mysociety/yournextrepresentative/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mysociety/yournextrepresentative/git/refs{/sha}","trees_url":"https://api.github.com/repos/mysociety/yournextrepresentative/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mysociety/yournextrepresentative/statuses/{sha}","languages_url":"https://api.github.com/repos/mysociety/yournextrepresentative/languages","stargazers_url":"https://api.github.com/repos/mysociety/yournextrepresentative/stargazers","contributors_url":"https://api.github.com/repos/mysociety/yournextrepresentative/contributors","subscribers_url":"https://api.github.com/repos/mysociety/yournextrepresentative/subscribers","subscription_url":"https://api.github.com/repos/mysociety/yournextrepresentative/subscription","commits_url":"https://api.github.com/repos/mysociety/yournextrepresentative/commits{/sha}","git_commits_url":"https://api.github.com/repos/mysociety/yournextrepresentative/git/commits{/sha}","comments_url":"https://api.github.com/repos/mysociety/yournextrepresentative/comments{/number}","issue_comment_url":"https://api.github.com/repos/mysociety/yournextrepresentative/issues/comments{/number}","contents_url":"https://api.github.com/repos/mysociety/yournextrepresentative/contents/{+path}","compare_url":"https://api.github.com/repos/mysociety/yournextrepresentative/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mysociety/yournextrepresentative/merges","archive_url":"https://api.github.com/repos/mysociety/yournextrepresentative/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mysociety/yournextrepresentative/downloads","issues_url":"https://api.github.com/repos/mysociety/yournextrepresentative/issues{/number}","pulls_url":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls{/number}","milestones_url":"https://api.github.com/repos/mysociety/yournextrepresentative/milestones{/number}","notifications_url":"https://api.github.com/repos/mysociety/yournextrepresentative/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mysociety/yournextrepresentative/labels{/name}","releases_url":"https://api.github.com/repos/mysociety/yournextrepresentative/releases{/id}","created_at":"2014-09-10T07:32:13Z","updated_at":"2015-09-24T14:15:20Z","pushed_at":"2015-10-14T16:41:04Z","git_url":"git://github.com/mysociety/yournextrepresentative.git","ssh_url":"git@github.com:mysociety/yournextrepresentative.git","clone_url":"https://github.com/mysociety/yournextrepresentative.git","svn_url":"https://github.com/mysociety/yournextrepresentative","homepage":"http://yournextmp.mysociety.org","size":6941,"stargazers_count":32,"watchers_count":32,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":14,"mirror_url":null,"open_issues_count":153,"forks":14,"open_issues":153,"watchers":32,"default_branch":"master"}},"base":{"label":"mysociety:master","ref":"master","sha":"887419952362b018265fcf58f9218b5c16333140","user":{"login":"mysociety","id":141338,"avatar_url":"https://avatars.githubusercontent.com/u/141338?v=3","gravatar_id":"","url":"https://api.github.com/users/mysociety","html_url":"https://github.com/mysociety","followers_url":"https://api.github.com/users/mysociety/followers","following_url":"https://api.github.com/users/mysociety/following{/other_user}","gists_url":"https://api.github.com/users/mysociety/gists{/gist_id}","starred_url":"https://api.github.com/users/mysociety/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mysociety/subscriptions","organizations_url":"https://api.github.com/users/mysociety/orgs","repos_url":"https://api.github.com/users/mysociety/repos","events_url":"https://api.github.com/users/mysociety/events{/privacy}","received_events_url":"https://api.github.com/users/mysociety/received_events","type":"Organization","site_admin":false},"repo":{"id":23865131,"name":"yournextrepresentative","full_name":"mysociety/yournextrepresentative","owner":{"login":"mysociety","id":141338,"avatar_url":"https://avatars.githubusercontent.com/u/141338?v=3","gravatar_id":"","url":"https://api.github.com/users/mysociety","html_url":"https://github.com/mysociety","followers_url":"https://api.github.com/users/mysociety/followers","following_url":"https://api.github.com/users/mysociety/following{/other_user}","gists_url":"https://api.github.com/users/mysociety/gists{/gist_id}","starred_url":"https://api.github.com/users/mysociety/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mysociety/subscriptions","organizations_url":"https://api.github.com/users/mysociety/orgs","repos_url":"https://api.github.com/users/mysociety/repos","events_url":"https://api.github.com/users/mysociety/events{/privacy}","received_events_url":"https://api.github.com/users/mysociety/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mysociety/yournextrepresentative","description":"An experimental front-end to PopIt for 2015 GE candidates","fork":false,"url":"https://api.github.com/repos/mysociety/yournextrepresentative","forks_url":"https://api.github.com/repos/mysociety/yournextrepresentative/forks","keys_url":"https://api.github.com/repos/mysociety/yournextrepresentative/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mysociety/yournextrepresentative/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mysociety/yournextrepresentative/teams","hooks_url":"https://api.github.com/repos/mysociety/yournextrepresentative/hooks","issue_events_url":"https://api.github.com/repos/mysociety/yournextrepresentative/issues/events{/number}","events_url":"https://api.github.com/repos/mysociety/yournextrepresentative/events","assignees_url":"https://api.github.com/repos/mysociety/yournextrepresentative/assignees{/user}","branches_url":"https://api.github.com/repos/mysociety/yournextrepresentative/branches{/branch}","tags_url":"https://api.github.com/repos/mysociety/yournextrepresentative/tags","blobs_url":"https://api.github.com/repos/mysociety/yournextrepresentative/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mysociety/yournextrepresentative/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mysociety/yournextrepresentative/git/refs{/sha}","trees_url":"https://api.github.com/repos/mysociety/yournextrepresentative/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mysociety/yournextrepresentative/statuses/{sha}","languages_url":"https://api.github.com/repos/mysociety/yournextrepresentative/languages","stargazers_url":"https://api.github.com/repos/mysociety/yournextrepresentative/stargazers","contributors_url":"https://api.github.com/repos/mysociety/yournextrepresentative/contributors","subscribers_url":"https://api.github.com/repos/mysociety/yournextrepresentative/subscribers","subscription_url":"https://api.github.com/repos/mysociety/yournextrepresentative/subscription","commits_url":"https://api.github.com/repos/mysociety/yournextrepresentative/commits{/sha}","git_commits_url":"https://api.github.com/repos/mysociety/yournextrepresentative/git/commits{/sha}","comments_url":"https://api.github.com/repos/mysociety/yournextrepresentative/comments{/number}","issue_comment_url":"https://api.github.com/repos/mysociety/yournextrepresentative/issues/comments{/number}","contents_url":"https://api.github.com/repos/mysociety/yournextrepresentative/contents/{+path}","compare_url":"https://api.github.com/repos/mysociety/yournextrepresentative/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mysociety/yournextrepresentative/merges","archive_url":"https://api.github.com/repos/mysociety/yournextrepresentative/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mysociety/yournextrepresentative/downloads","issues_url":"https://api.github.com/repos/mysociety/yournextrepresentative/issues{/number}","pulls_url":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls{/number}","milestones_url":"https://api.github.com/repos/mysociety/yournextrepresentative/milestones{/number}","notifications_url":"https://api.github.com/repos/mysociety/yournextrepresentative/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mysociety/yournextrepresentative/labels{/name}","releases_url":"https://api.github.com/repos/mysociety/yournextrepresentative/releases{/id}","created_at":"2014-09-10T07:32:13Z","updated_at":"2015-09-24T14:15:20Z","pushed_at":"2015-10-14T16:41:04Z","git_url":"git://github.com/mysociety/yournextrepresentative.git","ssh_url":"git@github.com:mysociety/yournextrepresentative.git","clone_url":"https://github.com/mysociety/yournextrepresentative.git","svn_url":"https://github.com/mysociety/yournextrepresentative","homepage":"http://yournextmp.mysociety.org","size":6941,"stargazers_count":32,"watchers_count":32,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":14,"mirror_url":null,"open_issues_count":153,"forks":14,"open_issues":153,"watchers":32,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/555"},"html":{"href":"https://github.com/mysociety/yournextrepresentative/pull/555"},"issue":{"href":"https://api.github.com/repos/mysociety/yournextrepresentative/issues/555"},"comments":{"href":"https://api.github.com/repos/mysociety/yournextrepresentative/issues/555/comments"},"review_comments":{"href":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/555/comments"},"review_comment":{"href":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mysociety/yournextrepresentative/pulls/555/commits"},"statuses":{"href":"https://api.github.com/repos/mysociety/yournextrepresentative/statuses/b223dbf50fcec6eb0596b2c56bdc6724a671ccfc"}}}}