{"action":"created","comment":{"url":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/comments/39979782","id":39979782,"diff_hunk":"@@ -0,0 +1,210 @@\n+\"\"\"\n+A Lake Winnipeg Basin Information Network (BIN) harvester for the SHARE project\n+\n+Example API request: http://130.179.67.140/api/3/action/package_search?q= (problematic)\n+http://130.179.67.140/api/3/action/current_package_list_with_resources (currently using)\n+It oddly returns 5 more datasets than all searchable ones on LWBIN data hub.\n+\n+Known issues:\n+1 -- Five datasets can be searched but cannot be accessed via LWBIN.\n+Clicking on the searching result would result in linking to a redirected page like this:\n+http://130.179.67.140/user/login?came_from=http://130.179.67.140/dataset/mpca-surface-water-data-access-interactive-map\n+Within each dataset there are resouces that contain urls to source pages. For future work considering using resources\n+urls as canonical urls.\n+2 -- Resouces properties contained in raw metadata of the datasets are not added to the normalized metadata at this\n+point.\n+3 -- Single name contributors can be used as filters or an invalid query will be returned. Has nothing to do with scrapi but the frontend.\n+\"\"\"\n+\n+from __future__ import unicode_literals\n+\n+import json\n+import logging\n+\n+from nameparser import HumanName\n+from dateutil.parser import parse\n+\n+from scrapi import requests\n+from scrapi.base import JSONHarvester\n+from scrapi.linter.document import RawDocument\n+from scrapi.base.helpers import build_properties, date_formatter\n+\n+\n+logger = logging.getLogger(__name__)\n+\n+ORGANIZATIONS = (\n+    \"organization\", \"Organization\", \"fund\", \"Fund\", \"canada\", \"Canada\", \"agriculture\", \"Agriculture\", \"commitee\",\n+    \"Commitee\", \"international\", \"International\", \"council\", \"Council\", \"office\", \"Office\", \"of\", \"Of\",\n+    \"observation\", \"Observation\", \"LWBIN\", \"CoCoRaHS\", \"USGS\", \"NSIDC\"\n+)\n+\n+\n+def is_organization(name):\n+    \"\"\"Return a boolean to indicate if the name passed to the function is an organization\n+    \"\"\"\n+    words = name.split(' ')\n+    return any(word.strip(\";\") in ORGANIZATIONS for word in words)\n+\n+\n+def process_contributors(authors, emails):\n+    \"\"\"Process authors and add author emails\n+    If multiple authors and one email, put email in a new author\n+    \"\"\"\n+\n+    authors = authors.strip().replace('<span class=\"author-names\">', '').replace('</span>', '')\n+    authors = authors.split(',')\n+    emails = emails.split(',')\n+\n+    for author in authors:\n+        if is_organization(author):\n+            break\n+    else:\n+        if ' and ' in authors[-1] or ' <em>et al.</em>' in authors[-1]:\n+            split_name = authors.pop(-1).replace(' <em>et al.</em>', '').split(' and ')\n+            authors.extend(split_name)\n+\n+    contributor_list = []\n+    append_emails = True\n+    if len(authors) != 1 or len(emails) != 1 or emails[0] == u'':\n+        append_emails = False  # append the email to the author only when 1 record is observed\n+\n+    for ind, author in enumerate(authors):","path":"scrapi/harvesters/lwbin.py","position":71,"original_position":71,"commit_id":"1ff1b9ea7697ea17edebb2f9c2676050f733d6f0","original_commit_id":"1ff1b9ea7697ea17edebb2f9c2676050f733d6f0","user":{"login":"fabianvf","id":2431974,"avatar_url":"https://avatars.githubusercontent.com/u/2431974?v=3","gravatar_id":"","url":"https://api.github.com/users/fabianvf","html_url":"https://github.com/fabianvf","followers_url":"https://api.github.com/users/fabianvf/followers","following_url":"https://api.github.com/users/fabianvf/following{/other_user}","gists_url":"https://api.github.com/users/fabianvf/gists{/gist_id}","starred_url":"https://api.github.com/users/fabianvf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabianvf/subscriptions","organizations_url":"https://api.github.com/users/fabianvf/orgs","repos_url":"https://api.github.com/users/fabianvf/repos","events_url":"https://api.github.com/users/fabianvf/events{/privacy}","received_events_url":"https://api.github.com/users/fabianvf/received_events","type":"User","site_admin":false},"body":"`i` is much more common as an indicator of an index variable than `ind`","created_at":"2015-09-21T14:53:31Z","updated_at":"2015-09-21T14:53:31Z","html_url":"https://github.com/CenterForOpenScience/scrapi/pull/379#discussion_r39979782","pull_request_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/379","_links":{"self":{"href":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/comments/39979782"},"html":{"href":"https://github.com/CenterForOpenScience/scrapi/pull/379#discussion_r39979782"},"pull_request":{"href":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/379"}}},"pull_request":{"url":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/379","id":45454511,"html_url":"https://github.com/CenterForOpenScience/scrapi/pull/379","diff_url":"https://github.com/CenterForOpenScience/scrapi/pull/379.diff","patch_url":"https://github.com/CenterForOpenScience/scrapi/pull/379.patch","issue_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/issues/379","number":379,"state":"open","locked":false,"title":"Add harvester for Lake Winnipeg Basin Information Network Data Hub","user":{"login":"jeffreyliu3230","id":5091997,"avatar_url":"https://avatars.githubusercontent.com/u/5091997?v=3","gravatar_id":"","url":"https://api.github.com/users/jeffreyliu3230","html_url":"https://github.com/jeffreyliu3230","followers_url":"https://api.github.com/users/jeffreyliu3230/followers","following_url":"https://api.github.com/users/jeffreyliu3230/following{/other_user}","gists_url":"https://api.github.com/users/jeffreyliu3230/gists{/gist_id}","starred_url":"https://api.github.com/users/jeffreyliu3230/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jeffreyliu3230/subscriptions","organizations_url":"https://api.github.com/users/jeffreyliu3230/orgs","repos_url":"https://api.github.com/users/jeffreyliu3230/repos","events_url":"https://api.github.com/users/jeffreyliu3230/events{/privacy}","received_events_url":"https://api.github.com/users/jeffreyliu3230/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-09-18T21:25:18Z","updated_at":"2015-09-21T14:53:31Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f346ec1961df7d3202700d501d274ff1b9e0ac7b","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/379/commits","review_comments_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/379/comments","review_comment_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/comments{/number}","comments_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/issues/379/comments","statuses_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/statuses/1ff1b9ea7697ea17edebb2f9c2676050f733d6f0","head":{"label":"jeffreyliu3230:feature/lwbin_harvester","ref":"feature/lwbin_harvester","sha":"1ff1b9ea7697ea17edebb2f9c2676050f733d6f0","user":{"login":"jeffreyliu3230","id":5091997,"avatar_url":"https://avatars.githubusercontent.com/u/5091997?v=3","gravatar_id":"","url":"https://api.github.com/users/jeffreyliu3230","html_url":"https://github.com/jeffreyliu3230","followers_url":"https://api.github.com/users/jeffreyliu3230/followers","following_url":"https://api.github.com/users/jeffreyliu3230/following{/other_user}","gists_url":"https://api.github.com/users/jeffreyliu3230/gists{/gist_id}","starred_url":"https://api.github.com/users/jeffreyliu3230/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jeffreyliu3230/subscriptions","organizations_url":"https://api.github.com/users/jeffreyliu3230/orgs","repos_url":"https://api.github.com/users/jeffreyliu3230/repos","events_url":"https://api.github.com/users/jeffreyliu3230/events{/privacy}","received_events_url":"https://api.github.com/users/jeffreyliu3230/received_events","type":"User","site_admin":false},"repo":{"id":38698336,"name":"scrapi","full_name":"jeffreyliu3230/scrapi","owner":{"login":"jeffreyliu3230","id":5091997,"avatar_url":"https://avatars.githubusercontent.com/u/5091997?v=3","gravatar_id":"","url":"https://api.github.com/users/jeffreyliu3230","html_url":"https://github.com/jeffreyliu3230","followers_url":"https://api.github.com/users/jeffreyliu3230/followers","following_url":"https://api.github.com/users/jeffreyliu3230/following{/other_user}","gists_url":"https://api.github.com/users/jeffreyliu3230/gists{/gist_id}","starred_url":"https://api.github.com/users/jeffreyliu3230/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jeffreyliu3230/subscriptions","organizations_url":"https://api.github.com/users/jeffreyliu3230/orgs","repos_url":"https://api.github.com/users/jeffreyliu3230/repos","events_url":"https://api.github.com/users/jeffreyliu3230/events{/privacy}","received_events_url":"https://api.github.com/users/jeffreyliu3230/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jeffreyliu3230/scrapi","description":"A data processing pipeline that schedules and runs content harvesters, normalizes their data, and outputs that normalized data to a variety of output streams. This is part of the SHARE project, and will be used to create a free and open dataset of research (meta)data. Data collected can be explored at https://osf.io/share/, and viewed at https://osf.io/api/v1/share/search/. Developer docs can be viewed at https://osf.io/wur56/wiki","fork":true,"url":"https://api.github.com/repos/jeffreyliu3230/scrapi","forks_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/forks","keys_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/teams","hooks_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/hooks","issue_events_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/issues/events{/number}","events_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/events","assignees_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/assignees{/user}","branches_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/branches{/branch}","tags_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/tags","blobs_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/git/refs{/sha}","trees_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/statuses/{sha}","languages_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/languages","stargazers_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/stargazers","contributors_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/contributors","subscribers_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/subscribers","subscription_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/subscription","commits_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/commits{/sha}","git_commits_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/git/commits{/sha}","comments_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/comments{/number}","issue_comment_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/issues/comments{/number}","contents_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/contents/{+path}","compare_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/merges","archive_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/downloads","issues_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/issues{/number}","pulls_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/pulls{/number}","milestones_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/milestones{/number}","notifications_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/labels{/name}","releases_url":"https://api.github.com/repos/jeffreyliu3230/scrapi/releases{/id}","created_at":"2015-07-07T15:54:12Z","updated_at":"2015-07-07T15:54:13Z","pushed_at":"2015-09-18T21:23:46Z","git_url":"git://github.com/jeffreyliu3230/scrapi.git","ssh_url":"git@github.com:jeffreyliu3230/scrapi.git","clone_url":"https://github.com/jeffreyliu3230/scrapi.git","svn_url":"https://github.com/jeffreyliu3230/scrapi","homepage":"","size":31475,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"develop"}},"base":{"label":"CenterForOpenScience:develop","ref":"develop","sha":"55d9ce4614e940e45efab3285458fb766245c47f","user":{"login":"CenterForOpenScience","id":3344584,"avatar_url":"https://avatars.githubusercontent.com/u/3344584?v=3","gravatar_id":"","url":"https://api.github.com/users/CenterForOpenScience","html_url":"https://github.com/CenterForOpenScience","followers_url":"https://api.github.com/users/CenterForOpenScience/followers","following_url":"https://api.github.com/users/CenterForOpenScience/following{/other_user}","gists_url":"https://api.github.com/users/CenterForOpenScience/gists{/gist_id}","starred_url":"https://api.github.com/users/CenterForOpenScience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CenterForOpenScience/subscriptions","organizations_url":"https://api.github.com/users/CenterForOpenScience/orgs","repos_url":"https://api.github.com/users/CenterForOpenScience/repos","events_url":"https://api.github.com/users/CenterForOpenScience/events{/privacy}","received_events_url":"https://api.github.com/users/CenterForOpenScience/received_events","type":"Organization","site_admin":false},"repo":{"id":21210102,"name":"scrapi","full_name":"CenterForOpenScience/scrapi","owner":{"login":"CenterForOpenScience","id":3344584,"avatar_url":"https://avatars.githubusercontent.com/u/3344584?v=3","gravatar_id":"","url":"https://api.github.com/users/CenterForOpenScience","html_url":"https://github.com/CenterForOpenScience","followers_url":"https://api.github.com/users/CenterForOpenScience/followers","following_url":"https://api.github.com/users/CenterForOpenScience/following{/other_user}","gists_url":"https://api.github.com/users/CenterForOpenScience/gists{/gist_id}","starred_url":"https://api.github.com/users/CenterForOpenScience/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CenterForOpenScience/subscriptions","organizations_url":"https://api.github.com/users/CenterForOpenScience/orgs","repos_url":"https://api.github.com/users/CenterForOpenScience/repos","events_url":"https://api.github.com/users/CenterForOpenScience/events{/privacy}","received_events_url":"https://api.github.com/users/CenterForOpenScience/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CenterForOpenScience/scrapi","description":"A data processing pipeline that schedules and runs content harvesters, normalizes their data, and outputs that normalized data to a variety of output streams. This is part of the SHARE project, and will be used to create a free and open dataset of research (meta)data. Data collected can be explored at https://osf.io/share/, and viewed at https://osf.io/api/v1/share/search/. Developer docs can be viewed at https://osf.io/wur56/wiki","fork":false,"url":"https://api.github.com/repos/CenterForOpenScience/scrapi","forks_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/forks","keys_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/teams","hooks_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/hooks","issue_events_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/issues/events{/number}","events_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/events","assignees_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/assignees{/user}","branches_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/branches{/branch}","tags_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/tags","blobs_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/git/refs{/sha}","trees_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/statuses/{sha}","languages_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/languages","stargazers_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/stargazers","contributors_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/contributors","subscribers_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/subscribers","subscription_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/subscription","commits_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/commits{/sha}","git_commits_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/git/commits{/sha}","comments_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/comments{/number}","issue_comment_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/issues/comments{/number}","contents_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/contents/{+path}","compare_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/merges","archive_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/downloads","issues_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/issues{/number}","pulls_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls{/number}","milestones_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/milestones{/number}","notifications_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/labels{/name}","releases_url":"https://api.github.com/repos/CenterForOpenScience/scrapi/releases{/id}","created_at":"2014-06-25T16:21:08Z","updated_at":"2015-09-03T18:52:44Z","pushed_at":"2015-09-18T21:25:19Z","git_url":"git://github.com/CenterForOpenScience/scrapi.git","ssh_url":"git@github.com:CenterForOpenScience/scrapi.git","clone_url":"https://github.com/CenterForOpenScience/scrapi.git","svn_url":"https://github.com/CenterForOpenScience/scrapi","homepage":"","size":36372,"stargazers_count":10,"watchers_count":10,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":30,"mirror_url":null,"open_issues_count":33,"forks":30,"open_issues":33,"watchers":10,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/379"},"html":{"href":"https://github.com/CenterForOpenScience/scrapi/pull/379"},"issue":{"href":"https://api.github.com/repos/CenterForOpenScience/scrapi/issues/379"},"comments":{"href":"https://api.github.com/repos/CenterForOpenScience/scrapi/issues/379/comments"},"review_comments":{"href":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/379/comments"},"review_comment":{"href":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/CenterForOpenScience/scrapi/pulls/379/commits"},"statuses":{"href":"https://api.github.com/repos/CenterForOpenScience/scrapi/statuses/1ff1b9ea7697ea17edebb2f9c2676050f733d6f0"}}}}