{"action":"created","comment":{"url":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/comments/47073507","id":47073507,"diff_hunk":"@@ -108,3 +113,45 @@ def test_build_dict(list_for_dict):\n     assert dict_from_list['John']['age'] == 20\n     assert dict_from_list['Mark']['age'] == 27\n     assert dict_from_list['Bruce']['age'] == 9\n+\n+\n+def test_split_fullname():\n+    \"\"\"Test author fullname splitting.\"\"\"\n+    author1 = 'Doe, John Magic'\n+    author2 = 'Doe Boe, John Magic'\n+    author3 = 'Doe Boe John Magic'\n+    author4 = 'John Magic Doe'\n+    author5 = 'John Magic Doe Boe'\n+    author6 = 'John Magic, Doe Boe'\n+    assert split_fullname(author1) == ('Doe', 'John Magic')\n+    assert split_fullname(author2) == ('Doe Boe', 'John Magic')\n+    assert split_fullname(author3) == ('Doe', 'Boe John Magic')\n+    assert split_fullname(author4, surname='last') == ('Doe', 'John Magic')\n+    assert split_fullname(author5, surname='last') == ('Boe', 'John Magic Doe')\n+    assert split_fullname(author6, surname='last') == ('Doe Boe', 'John Magic') \n+\n+\n+def test_parse_domain():\n+    \"\"\"Test domain parsing.\"\"\"\n+    url = 'http://www.example.com/0351/content'\n+    domain = 'http://www.example.com/'\n+    assert parse_domain(url) == domain\n+\n+\n+@responses.activate\n+def test_get_mime_type():\n+    \"\"\"Test MIME type getting.\n+\n+    Apparently the mocked response always has 'Content-Type'\n+    and it seems to default to 'text/html'?\n+    Because of this, right now is not testing if missing header\n+    information raises and exception.\n+    \"\"\"\n+    url = 'http://www.example.com/files/einstein_1905.pdf'\n+    responses.add(responses.HEAD, url, status=200, content_type='application/pdf')\n+    resp = requests.head(url)","path":"tests/test_utils.py","position":60,"original_position":60,"commit_id":"1187f7a2405eff72b26156259d5233ca364f4147","original_commit_id":"1187f7a2405eff72b26156259d5233ca364f4147","user":{"login":"jalavik","id":633680,"avatar_url":"https://avatars.githubusercontent.com/u/633680?v=3","gravatar_id":"","url":"https://api.github.com/users/jalavik","html_url":"https://github.com/jalavik","followers_url":"https://api.github.com/users/jalavik/followers","following_url":"https://api.github.com/users/jalavik/following{/other_user}","gists_url":"https://api.github.com/users/jalavik/gists{/gist_id}","starred_url":"https://api.github.com/users/jalavik/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jalavik/subscriptions","organizations_url":"https://api.github.com/users/jalavik/orgs","repos_url":"https://api.github.com/users/jalavik/repos","events_url":"https://api.github.com/users/jalavik/events{/privacy}","received_events_url":"https://api.github.com/users/jalavik/received_events","type":"User","site_admin":false},"body":"Actually, I think you can remove this line and the `if` check below and just have\r\n\r\n```python\r\nassert get_mime_type(url) == 'application/pdf'\r\nassert get_mime_type(None) == ''\r\nassert get_mime_type('') == ''\r\n```","created_at":"2015-12-09T10:26:46Z","updated_at":"2015-12-09T10:26:46Z","html_url":"https://github.com/inspirehep/hepcrawl/pull/2#discussion_r47073507","pull_request_url":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/2","_links":{"self":{"href":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/comments/47073507"},"html":{"href":"https://github.com/inspirehep/hepcrawl/pull/2#discussion_r47073507"},"pull_request":{"href":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/2"}}},"pull_request":{"url":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/2","id":51671912,"html_url":"https://github.com/inspirehep/hepcrawl/pull/2","diff_url":"https://github.com/inspirehep/hepcrawl/pull/2.diff","patch_url":"https://github.com/inspirehep/hepcrawl/pull/2.patch","issue_url":"https://api.github.com/repos/inspirehep/hepcrawl/issues/2","number":2,"state":"open","locked":false,"title":"Alphaharvest","user":{"login":"bittirousku","id":13130439,"avatar_url":"https://avatars.githubusercontent.com/u/13130439?v=3","gravatar_id":"","url":"https://api.github.com/users/bittirousku","html_url":"https://github.com/bittirousku","followers_url":"https://api.github.com/users/bittirousku/followers","following_url":"https://api.github.com/users/bittirousku/following{/other_user}","gists_url":"https://api.github.com/users/bittirousku/gists{/gist_id}","starred_url":"https://api.github.com/users/bittirousku/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bittirousku/subscriptions","organizations_url":"https://api.github.com/users/bittirousku/orgs","repos_url":"https://api.github.com/users/bittirousku/repos","events_url":"https://api.github.com/users/bittirousku/events{/privacy}","received_events_url":"https://api.github.com/users/bittirousku/received_events","type":"User","site_admin":false},"body":"Alphaharvest is working now, tests also.\r\nBaseharvest still has problems, old and new:\r\n* Testing doesn't work, because parse_node() is not returning items! Is this\r\n    the wrong way to go? How else can I iterate through all the records? Or should\r\n    the testing be done differently?\r\n* It's consistently outputting only 184 records when using a test file of 1000 records! (tests/responses/base/test_1000.xml)\r\n   * It works when using smaller test files.\r\n* Australian thesis server sometimes returns errors: `twisted.python.failure.Failure twisted.internet.error.ConnectionDone: Connection was closed cleanly`.\r\n* Other error messages encountered:\r\n   * `IOError: [Errno socket error] [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:590)`\r\n   * `IOError: [Errno socket error] EOF occurred in violation of protocol (_ssl.c:590)`\r\n   * `IOError: [Errno socket error] [Errno 104] Connection reset by peer`\r\n* CALtech thesis server was asking for a password\r\n* Namespaces are ignored with brute force. There are better ways?\r\n* JSON pipeline is not writing commas between records right now. The original solution to add \", \\n\" also added a comma after the last record (not cool).\r\n* Something else?\r\n\r\n    \r\n   \r\n   \r\n    ","created_at":"2015-11-24T15:14:07Z","updated_at":"2015-12-09T10:26:46Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e64de6bccebbba11da8c0b0e582efb12141766f0","assignee":{"login":"jalavik","id":633680,"avatar_url":"https://avatars.githubusercontent.com/u/633680?v=3","gravatar_id":"","url":"https://api.github.com/users/jalavik","html_url":"https://github.com/jalavik","followers_url":"https://api.github.com/users/jalavik/followers","following_url":"https://api.github.com/users/jalavik/following{/other_user}","gists_url":"https://api.github.com/users/jalavik/gists{/gist_id}","starred_url":"https://api.github.com/users/jalavik/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jalavik/subscriptions","organizations_url":"https://api.github.com/users/jalavik/orgs","repos_url":"https://api.github.com/users/jalavik/repos","events_url":"https://api.github.com/users/jalavik/events{/privacy}","received_events_url":"https://api.github.com/users/jalavik/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/2/commits","review_comments_url":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/2/comments","review_comment_url":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/comments{/number}","comments_url":"https://api.github.com/repos/inspirehep/hepcrawl/issues/2/comments","statuses_url":"https://api.github.com/repos/inspirehep/hepcrawl/statuses/1187f7a2405eff72b26156259d5233ca364f4147","head":{"label":"bittirousku:alphaharvest","ref":"alphaharvest","sha":"1187f7a2405eff72b26156259d5233ca364f4147","user":{"login":"bittirousku","id":13130439,"avatar_url":"https://avatars.githubusercontent.com/u/13130439?v=3","gravatar_id":"","url":"https://api.github.com/users/bittirousku","html_url":"https://github.com/bittirousku","followers_url":"https://api.github.com/users/bittirousku/followers","following_url":"https://api.github.com/users/bittirousku/following{/other_user}","gists_url":"https://api.github.com/users/bittirousku/gists{/gist_id}","starred_url":"https://api.github.com/users/bittirousku/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bittirousku/subscriptions","organizations_url":"https://api.github.com/users/bittirousku/orgs","repos_url":"https://api.github.com/users/bittirousku/repos","events_url":"https://api.github.com/users/bittirousku/events{/privacy}","received_events_url":"https://api.github.com/users/bittirousku/received_events","type":"User","site_admin":false},"repo":{"id":46273473,"name":"hepcrawl","full_name":"bittirousku/hepcrawl","owner":{"login":"bittirousku","id":13130439,"avatar_url":"https://avatars.githubusercontent.com/u/13130439?v=3","gravatar_id":"","url":"https://api.github.com/users/bittirousku","html_url":"https://github.com/bittirousku","followers_url":"https://api.github.com/users/bittirousku/followers","following_url":"https://api.github.com/users/bittirousku/following{/other_user}","gists_url":"https://api.github.com/users/bittirousku/gists{/gist_id}","starred_url":"https://api.github.com/users/bittirousku/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bittirousku/subscriptions","organizations_url":"https://api.github.com/users/bittirousku/orgs","repos_url":"https://api.github.com/users/bittirousku/repos","events_url":"https://api.github.com/users/bittirousku/events{/privacy}","received_events_url":"https://api.github.com/users/bittirousku/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/bittirousku/hepcrawl","description":"Scrapy project for feeds into INSPIRE-HEP","fork":true,"url":"https://api.github.com/repos/bittirousku/hepcrawl","forks_url":"https://api.github.com/repos/bittirousku/hepcrawl/forks","keys_url":"https://api.github.com/repos/bittirousku/hepcrawl/keys{/key_id}","collaborators_url":"https://api.github.com/repos/bittirousku/hepcrawl/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/bittirousku/hepcrawl/teams","hooks_url":"https://api.github.com/repos/bittirousku/hepcrawl/hooks","issue_events_url":"https://api.github.com/repos/bittirousku/hepcrawl/issues/events{/number}","events_url":"https://api.github.com/repos/bittirousku/hepcrawl/events","assignees_url":"https://api.github.com/repos/bittirousku/hepcrawl/assignees{/user}","branches_url":"https://api.github.com/repos/bittirousku/hepcrawl/branches{/branch}","tags_url":"https://api.github.com/repos/bittirousku/hepcrawl/tags","blobs_url":"https://api.github.com/repos/bittirousku/hepcrawl/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/bittirousku/hepcrawl/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/bittirousku/hepcrawl/git/refs{/sha}","trees_url":"https://api.github.com/repos/bittirousku/hepcrawl/git/trees{/sha}","statuses_url":"https://api.github.com/repos/bittirousku/hepcrawl/statuses/{sha}","languages_url":"https://api.github.com/repos/bittirousku/hepcrawl/languages","stargazers_url":"https://api.github.com/repos/bittirousku/hepcrawl/stargazers","contributors_url":"https://api.github.com/repos/bittirousku/hepcrawl/contributors","subscribers_url":"https://api.github.com/repos/bittirousku/hepcrawl/subscribers","subscription_url":"https://api.github.com/repos/bittirousku/hepcrawl/subscription","commits_url":"https://api.github.com/repos/bittirousku/hepcrawl/commits{/sha}","git_commits_url":"https://api.github.com/repos/bittirousku/hepcrawl/git/commits{/sha}","comments_url":"https://api.github.com/repos/bittirousku/hepcrawl/comments{/number}","issue_comment_url":"https://api.github.com/repos/bittirousku/hepcrawl/issues/comments{/number}","contents_url":"https://api.github.com/repos/bittirousku/hepcrawl/contents/{+path}","compare_url":"https://api.github.com/repos/bittirousku/hepcrawl/compare/{base}...{head}","merges_url":"https://api.github.com/repos/bittirousku/hepcrawl/merges","archive_url":"https://api.github.com/repos/bittirousku/hepcrawl/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/bittirousku/hepcrawl/downloads","issues_url":"https://api.github.com/repos/bittirousku/hepcrawl/issues{/number}","pulls_url":"https://api.github.com/repos/bittirousku/hepcrawl/pulls{/number}","milestones_url":"https://api.github.com/repos/bittirousku/hepcrawl/milestones{/number}","notifications_url":"https://api.github.com/repos/bittirousku/hepcrawl/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/bittirousku/hepcrawl/labels{/name}","releases_url":"https://api.github.com/repos/bittirousku/hepcrawl/releases{/id}","created_at":"2015-11-16T12:33:07Z","updated_at":"2015-11-16T12:33:08Z","pushed_at":"2015-12-08T11:44:26Z","git_url":"git://github.com/bittirousku/hepcrawl.git","ssh_url":"git@github.com:bittirousku/hepcrawl.git","clone_url":"https://github.com/bittirousku/hepcrawl.git","svn_url":"https://github.com/bittirousku/hepcrawl","homepage":"http://inspirehep.net","size":1268,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"inspirehep:master","ref":"master","sha":"cb26cc04a27cda3a281c683eddf9835554f05677","user":{"login":"inspirehep","id":356664,"avatar_url":"https://avatars.githubusercontent.com/u/356664?v=3","gravatar_id":"","url":"https://api.github.com/users/inspirehep","html_url":"https://github.com/inspirehep","followers_url":"https://api.github.com/users/inspirehep/followers","following_url":"https://api.github.com/users/inspirehep/following{/other_user}","gists_url":"https://api.github.com/users/inspirehep/gists{/gist_id}","starred_url":"https://api.github.com/users/inspirehep/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/inspirehep/subscriptions","organizations_url":"https://api.github.com/users/inspirehep/orgs","repos_url":"https://api.github.com/users/inspirehep/repos","events_url":"https://api.github.com/users/inspirehep/events{/privacy}","received_events_url":"https://api.github.com/users/inspirehep/received_events","type":"Organization","site_admin":false},"repo":{"id":44979576,"name":"hepcrawl","full_name":"inspirehep/hepcrawl","owner":{"login":"inspirehep","id":356664,"avatar_url":"https://avatars.githubusercontent.com/u/356664?v=3","gravatar_id":"","url":"https://api.github.com/users/inspirehep","html_url":"https://github.com/inspirehep","followers_url":"https://api.github.com/users/inspirehep/followers","following_url":"https://api.github.com/users/inspirehep/following{/other_user}","gists_url":"https://api.github.com/users/inspirehep/gists{/gist_id}","starred_url":"https://api.github.com/users/inspirehep/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/inspirehep/subscriptions","organizations_url":"https://api.github.com/users/inspirehep/orgs","repos_url":"https://api.github.com/users/inspirehep/repos","events_url":"https://api.github.com/users/inspirehep/events{/privacy}","received_events_url":"https://api.github.com/users/inspirehep/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/inspirehep/hepcrawl","description":"Scrapy project for feeds into INSPIRE-HEP","fork":false,"url":"https://api.github.com/repos/inspirehep/hepcrawl","forks_url":"https://api.github.com/repos/inspirehep/hepcrawl/forks","keys_url":"https://api.github.com/repos/inspirehep/hepcrawl/keys{/key_id}","collaborators_url":"https://api.github.com/repos/inspirehep/hepcrawl/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/inspirehep/hepcrawl/teams","hooks_url":"https://api.github.com/repos/inspirehep/hepcrawl/hooks","issue_events_url":"https://api.github.com/repos/inspirehep/hepcrawl/issues/events{/number}","events_url":"https://api.github.com/repos/inspirehep/hepcrawl/events","assignees_url":"https://api.github.com/repos/inspirehep/hepcrawl/assignees{/user}","branches_url":"https://api.github.com/repos/inspirehep/hepcrawl/branches{/branch}","tags_url":"https://api.github.com/repos/inspirehep/hepcrawl/tags","blobs_url":"https://api.github.com/repos/inspirehep/hepcrawl/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/inspirehep/hepcrawl/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/inspirehep/hepcrawl/git/refs{/sha}","trees_url":"https://api.github.com/repos/inspirehep/hepcrawl/git/trees{/sha}","statuses_url":"https://api.github.com/repos/inspirehep/hepcrawl/statuses/{sha}","languages_url":"https://api.github.com/repos/inspirehep/hepcrawl/languages","stargazers_url":"https://api.github.com/repos/inspirehep/hepcrawl/stargazers","contributors_url":"https://api.github.com/repos/inspirehep/hepcrawl/contributors","subscribers_url":"https://api.github.com/repos/inspirehep/hepcrawl/subscribers","subscription_url":"https://api.github.com/repos/inspirehep/hepcrawl/subscription","commits_url":"https://api.github.com/repos/inspirehep/hepcrawl/commits{/sha}","git_commits_url":"https://api.github.com/repos/inspirehep/hepcrawl/git/commits{/sha}","comments_url":"https://api.github.com/repos/inspirehep/hepcrawl/comments{/number}","issue_comment_url":"https://api.github.com/repos/inspirehep/hepcrawl/issues/comments{/number}","contents_url":"https://api.github.com/repos/inspirehep/hepcrawl/contents/{+path}","compare_url":"https://api.github.com/repos/inspirehep/hepcrawl/compare/{base}...{head}","merges_url":"https://api.github.com/repos/inspirehep/hepcrawl/merges","archive_url":"https://api.github.com/repos/inspirehep/hepcrawl/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/inspirehep/hepcrawl/downloads","issues_url":"https://api.github.com/repos/inspirehep/hepcrawl/issues{/number}","pulls_url":"https://api.github.com/repos/inspirehep/hepcrawl/pulls{/number}","milestones_url":"https://api.github.com/repos/inspirehep/hepcrawl/milestones{/number}","notifications_url":"https://api.github.com/repos/inspirehep/hepcrawl/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/inspirehep/hepcrawl/labels{/name}","releases_url":"https://api.github.com/repos/inspirehep/hepcrawl/releases{/id}","created_at":"2015-10-26T15:55:58Z","updated_at":"2015-10-27T13:56:12Z","pushed_at":"2015-12-08T11:44:26Z","git_url":"git://github.com/inspirehep/hepcrawl.git","ssh_url":"git@github.com:inspirehep/hepcrawl.git","clone_url":"https://github.com/inspirehep/hepcrawl.git","svn_url":"https://github.com/inspirehep/hepcrawl","homepage":"http://inspirehep.net","size":0,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":2,"forks":5,"open_issues":2,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/2"},"html":{"href":"https://github.com/inspirehep/hepcrawl/pull/2"},"issue":{"href":"https://api.github.com/repos/inspirehep/hepcrawl/issues/2"},"comments":{"href":"https://api.github.com/repos/inspirehep/hepcrawl/issues/2/comments"},"review_comments":{"href":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/2/comments"},"review_comment":{"href":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/inspirehep/hepcrawl/pulls/2/commits"},"statuses":{"href":"https://api.github.com/repos/inspirehep/hepcrawl/statuses/1187f7a2405eff72b26156259d5233ca364f4147"}}}}