{"action":"created","comment":{"url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/comments/41761979","id":41761979,"diff_hunk":"@@ -82,215 +86,397 @@ class ImportProcess(Thread):\n     Loads/Updates data from xls spreadsheet\n     \"\"\"\n \n-    def __init__(self, path, should_prime_geocoder=True, clear_db=True, single_transaction=True):\n+    worksheet_names = (\n+        'LOCAL ADVICE ORG',\n+        'OFFICE LOCATION',\n+        'CAT OF LAW CRIME',\n+        'CAT OF LAW CIVIL',\n+        'OUTREACH SERVICE',\n+    )\n+\n+    def __init__(self, xlsx_file, record, **options):\n         super(ImportProcess, self).__init__()\n+        self.options = options\n         self.progress = {'task': 'initialising'}\n+        self.record = record\n         self._interrupt = Event()\n-        self.should_prime_geocoder = should_prime_geocoder\n-        self.clear_db = clear_db\n-        if single_transaction:\n-            self.run = transaction.atomic(self.run)\n-        workbook = xlrd.open_workbook(path)\n-        self.organisation_sheet = workbook.sheet_by_name('LOCAL ADVICE ORG')\n-        self.office_sheet = workbook.sheet_by_name('OFFICE LOCATION')\n-        self.category_criminal_sheet = workbook.sheet_by_name(\n-            'CAT OF LAW CRIME')\n-        self.category_civil_sheet = workbook.sheet_by_name('CAT OF LAW CIVIL')\n-        self.outreach_sheet = workbook.sheet_by_name('OUTREACH SERVICE')\n+        self.sheets = {}\n+        self.temp_dir = tempfile.mkdtemp()\n \n-    def interrupt(self):\n-        self._interrupt.set()\n+        # if options.pop('single_transaction', False):","path":"laalaa/apps/advisers/importer.py","position":119,"original_position":119,"commit_id":"55a53011665d71ea79585e8bff39e89e1849b22a","original_commit_id":"55a53011665d71ea79585e8bff39e89e1849b22a","user":{"login":"andyhd","id":35194,"avatar_url":"https://avatars.githubusercontent.com/u/35194?v=3","gravatar_id":"","url":"https://api.github.com/users/andyhd","html_url":"https://github.com/andyhd","followers_url":"https://api.github.com/users/andyhd/followers","following_url":"https://api.github.com/users/andyhd/following{/other_user}","gists_url":"https://api.github.com/users/andyhd/gists{/gist_id}","starred_url":"https://api.github.com/users/andyhd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andyhd/subscriptions","organizations_url":"https://api.github.com/users/andyhd/orgs","repos_url":"https://api.github.com/users/andyhd/repos","events_url":"https://api.github.com/users/andyhd/events{/privacy}","received_events_url":"https://api.github.com/users/andyhd/received_events","type":"User","site_admin":false},"body":"Ah, I forgot to put that back in","created_at":"2015-10-12T14:31:00Z","updated_at":"2015-10-12T14:31:00Z","html_url":"https://github.com/ministryofjustice/laa-legal-adviser-api/pull/32#discussion_r41761979","pull_request_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/32","_links":{"self":{"href":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/comments/41761979"},"html":{"href":"https://github.com/ministryofjustice/laa-legal-adviser-api/pull/32#discussion_r41761979"},"pull_request":{"href":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/32"}}},"pull_request":{"url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/32","id":47424682,"html_url":"https://github.com/ministryofjustice/laa-legal-adviser-api/pull/32","diff_url":"https://github.com/ministryofjustice/laa-legal-adviser-api/pull/32.diff","patch_url":"https://github.com/ministryofjustice/laa-legal-adviser-api/pull/32.patch","issue_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/issues/32","number":32,"state":"open","locked":false,"title":"Refactor advisers importer","user":{"login":"andyhd","id":35194,"avatar_url":"https://avatars.githubusercontent.com/u/35194?v=3","gravatar_id":"","url":"https://api.github.com/users/andyhd","html_url":"https://github.com/andyhd","followers_url":"https://api.github.com/users/andyhd/followers","following_url":"https://api.github.com/users/andyhd/following{/other_user}","gists_url":"https://api.github.com/users/andyhd/gists{/gist_id}","starred_url":"https://api.github.com/users/andyhd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andyhd/subscriptions","organizations_url":"https://api.github.com/users/andyhd/orgs","repos_url":"https://api.github.com/users/andyhd/repos","events_url":"https://api.github.com/users/andyhd/events{/privacy}","received_events_url":"https://api.github.com/users/andyhd/received_events","type":"User","site_admin":false},"body":"Refactored the importer to this process:\r\n\r\n1. Load the Excel spreadsheet into memory\r\n2. Save each sheet to a temporary CSV file\r\n3. Use COPY FROM to load the CSV file into the db\r\n4. Translate the data to Django model tables using SQL\r\n5. Geocode the postcodes and update the db\r\n\r\nThis is quite a lot faster than the previous method of parsing the Excel file\r\nand generating Django models and adding them using the ORM.","created_at":"2015-10-12T14:18:23Z","updated_at":"2015-10-12T14:31:00Z","closed_at":null,"merged_at":null,"merge_commit_sha":"641241899ecffadb1653fe84c0e19951b0805956","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/32/commits","review_comments_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/32/comments","review_comment_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/issues/32/comments","statuses_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/statuses/55a53011665d71ea79585e8bff39e89e1849b22a","head":{"label":"ministryofjustice:fix-upload","ref":"fix-upload","sha":"55a53011665d71ea79585e8bff39e89e1849b22a","user":{"login":"ministryofjustice","id":2203574,"avatar_url":"https://avatars.githubusercontent.com/u/2203574?v=3","gravatar_id":"","url":"https://api.github.com/users/ministryofjustice","html_url":"https://github.com/ministryofjustice","followers_url":"https://api.github.com/users/ministryofjustice/followers","following_url":"https://api.github.com/users/ministryofjustice/following{/other_user}","gists_url":"https://api.github.com/users/ministryofjustice/gists{/gist_id}","starred_url":"https://api.github.com/users/ministryofjustice/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ministryofjustice/subscriptions","organizations_url":"https://api.github.com/users/ministryofjustice/orgs","repos_url":"https://api.github.com/users/ministryofjustice/repos","events_url":"https://api.github.com/users/ministryofjustice/events{/privacy}","received_events_url":"https://api.github.com/users/ministryofjustice/received_events","type":"Organization","site_admin":false},"repo":{"id":31312435,"name":"laa-legal-adviser-api","full_name":"ministryofjustice/laa-legal-adviser-api","owner":{"login":"ministryofjustice","id":2203574,"avatar_url":"https://avatars.githubusercontent.com/u/2203574?v=3","gravatar_id":"","url":"https://api.github.com/users/ministryofjustice","html_url":"https://github.com/ministryofjustice","followers_url":"https://api.github.com/users/ministryofjustice/followers","following_url":"https://api.github.com/users/ministryofjustice/following{/other_user}","gists_url":"https://api.github.com/users/ministryofjustice/gists{/gist_id}","starred_url":"https://api.github.com/users/ministryofjustice/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ministryofjustice/subscriptions","organizations_url":"https://api.github.com/users/ministryofjustice/orgs","repos_url":"https://api.github.com/users/ministryofjustice/repos","events_url":"https://api.github.com/users/ministryofjustice/events{/privacy}","received_events_url":"https://api.github.com/users/ministryofjustice/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ministryofjustice/laa-legal-adviser-api","description":"LAA Legal Adviser API","fork":false,"url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api","forks_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/forks","keys_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/teams","hooks_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/hooks","issue_events_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/issues/events{/number}","events_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/events","assignees_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/assignees{/user}","branches_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/branches{/branch}","tags_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/tags","blobs_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/git/refs{/sha}","trees_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/statuses/{sha}","languages_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/languages","stargazers_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/stargazers","contributors_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/contributors","subscribers_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/subscribers","subscription_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/subscription","commits_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/commits{/sha}","git_commits_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/git/commits{/sha}","comments_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/comments{/number}","issue_comment_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/issues/comments{/number}","contents_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/contents/{+path}","compare_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/merges","archive_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/downloads","issues_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/issues{/number}","pulls_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls{/number}","milestones_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/milestones{/number}","notifications_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/labels{/name}","releases_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/releases{/id}","created_at":"2015-02-25T12:25:58Z","updated_at":"2015-05-05T16:19:18Z","pushed_at":"2015-10-12T14:18:24Z","git_url":"git://github.com/ministryofjustice/laa-legal-adviser-api.git","ssh_url":"git@github.com:ministryofjustice/laa-legal-adviser-api.git","clone_url":"https://github.com/ministryofjustice/laa-legal-adviser-api.git","svn_url":"https://github.com/ministryofjustice/laa-legal-adviser-api","homepage":null,"size":1020,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":5,"forks":0,"open_issues":5,"watchers":0,"default_branch":"master"}},"base":{"label":"ministryofjustice:master","ref":"master","sha":"428f3e85c340e2e8e64d4ad23d43419e44112263","user":{"login":"ministryofjustice","id":2203574,"avatar_url":"https://avatars.githubusercontent.com/u/2203574?v=3","gravatar_id":"","url":"https://api.github.com/users/ministryofjustice","html_url":"https://github.com/ministryofjustice","followers_url":"https://api.github.com/users/ministryofjustice/followers","following_url":"https://api.github.com/users/ministryofjustice/following{/other_user}","gists_url":"https://api.github.com/users/ministryofjustice/gists{/gist_id}","starred_url":"https://api.github.com/users/ministryofjustice/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ministryofjustice/subscriptions","organizations_url":"https://api.github.com/users/ministryofjustice/orgs","repos_url":"https://api.github.com/users/ministryofjustice/repos","events_url":"https://api.github.com/users/ministryofjustice/events{/privacy}","received_events_url":"https://api.github.com/users/ministryofjustice/received_events","type":"Organization","site_admin":false},"repo":{"id":31312435,"name":"laa-legal-adviser-api","full_name":"ministryofjustice/laa-legal-adviser-api","owner":{"login":"ministryofjustice","id":2203574,"avatar_url":"https://avatars.githubusercontent.com/u/2203574?v=3","gravatar_id":"","url":"https://api.github.com/users/ministryofjustice","html_url":"https://github.com/ministryofjustice","followers_url":"https://api.github.com/users/ministryofjustice/followers","following_url":"https://api.github.com/users/ministryofjustice/following{/other_user}","gists_url":"https://api.github.com/users/ministryofjustice/gists{/gist_id}","starred_url":"https://api.github.com/users/ministryofjustice/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ministryofjustice/subscriptions","organizations_url":"https://api.github.com/users/ministryofjustice/orgs","repos_url":"https://api.github.com/users/ministryofjustice/repos","events_url":"https://api.github.com/users/ministryofjustice/events{/privacy}","received_events_url":"https://api.github.com/users/ministryofjustice/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ministryofjustice/laa-legal-adviser-api","description":"LAA Legal Adviser API","fork":false,"url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api","forks_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/forks","keys_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/teams","hooks_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/hooks","issue_events_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/issues/events{/number}","events_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/events","assignees_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/assignees{/user}","branches_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/branches{/branch}","tags_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/tags","blobs_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/git/refs{/sha}","trees_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/statuses/{sha}","languages_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/languages","stargazers_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/stargazers","contributors_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/contributors","subscribers_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/subscribers","subscription_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/subscription","commits_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/commits{/sha}","git_commits_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/git/commits{/sha}","comments_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/comments{/number}","issue_comment_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/issues/comments{/number}","contents_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/contents/{+path}","compare_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/merges","archive_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/downloads","issues_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/issues{/number}","pulls_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls{/number}","milestones_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/milestones{/number}","notifications_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/labels{/name}","releases_url":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/releases{/id}","created_at":"2015-02-25T12:25:58Z","updated_at":"2015-05-05T16:19:18Z","pushed_at":"2015-10-12T14:18:24Z","git_url":"git://github.com/ministryofjustice/laa-legal-adviser-api.git","ssh_url":"git@github.com:ministryofjustice/laa-legal-adviser-api.git","clone_url":"https://github.com/ministryofjustice/laa-legal-adviser-api.git","svn_url":"https://github.com/ministryofjustice/laa-legal-adviser-api","homepage":null,"size":1020,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":5,"forks":0,"open_issues":5,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/32"},"html":{"href":"https://github.com/ministryofjustice/laa-legal-adviser-api/pull/32"},"issue":{"href":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/issues/32"},"comments":{"href":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/issues/32/comments"},"review_comments":{"href":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/32/comments"},"review_comment":{"href":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/pulls/32/commits"},"statuses":{"href":"https://api.github.com/repos/ministryofjustice/laa-legal-adviser-api/statuses/55a53011665d71ea79585e8bff39e89e1849b22a"}}}}