{"action":"created","comment":{"url":"https://api.github.com/repos/fabianvf/scrapi/pulls/comments/30045699","id":30045699,"diff_hunk":"@@ -55,20 +51,21 @@ def harvest(self, days_back=1):\n                 'doc': etree.tostring(record),\n                 'docID': record.xpath('dc:ostiId/node()', namespaces=self.namespaces)[0].decode('utf-8'),\n             })\n-            for record in self._fetch_records(days_back)\n+            for record in self._fetch_records(start_date, end_date)\n         ]\n \n-    def _fetch_records(self, days_back):\n+    def _fetch_records(self, start_date, end_date):\n         page = 0\n         morepages = True\n-        to_date = datetime.date.today().strftime('%m/%d/%Y')\n-        from_date = (datetime.date.today() - datetime.timedelta(days_back)).strftime('%m/%d/%Y')\n+\n+        start_date = start_date or date.today()\n+        end_date = end_date or date.today() - timedelta(1)\n \n         while morepages:\n             resp = requests.get(self.base_url, params={\n                 'page': page,\n-                'EntryDateTo': to_date,\n-                'EntryDateFrom': from_date,\n+                'EntryDateTo': end_date.strftime('%m/%d/%Y'),","path":"scrapi/harvesters/scitech.py","position":51,"original_position":51,"commit_id":"4d01d4a2a925508e27f9b11313a1784b141a70c3","original_commit_id":"4d01d4a2a925508e27f9b11313a1784b141a70c3","user":{"login":"fabianvf","id":2431974,"avatar_url":"https://avatars.githubusercontent.com/u/2431974?v=3","gravatar_id":"","url":"https://api.github.com/users/fabianvf","html_url":"https://github.com/fabianvf","followers_url":"https://api.github.com/users/fabianvf/followers","following_url":"https://api.github.com/users/fabianvf/following{/other_user}","gists_url":"https://api.github.com/users/fabianvf/gists{/gist_id}","starred_url":"https://api.github.com/users/fabianvf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabianvf/subscriptions","organizations_url":"https://api.github.com/users/fabianvf/orgs","repos_url":"https://api.github.com/users/fabianvf/repos","events_url":"https://api.github.com/users/fabianvf/events{/privacy}","received_events_url":"https://api.github.com/users/fabianvf/received_events","type":"User","site_admin":false},"body":"If this ```strftime('%m/%d/%Y')``` pops up a lot, you may want to stick it into a function for readability.","created_at":"2015-05-11T14:57:52Z","updated_at":"2015-05-11T14:57:52Z","html_url":"https://github.com/fabianvf/scrapi/pull/189#discussion_r30045699","pull_request_url":"https://api.github.com/repos/fabianvf/scrapi/pulls/189","_links":{"self":{"href":"https://api.github.com/repos/fabianvf/scrapi/pulls/comments/30045699"},"html":{"href":"https://github.com/fabianvf/scrapi/pull/189#discussion_r30045699"},"pull_request":{"href":"https://api.github.com/repos/fabianvf/scrapi/pulls/189"}}},"pull_request":{"url":"https://api.github.com/repos/fabianvf/scrapi/pulls/189","id":34664495,"html_url":"https://github.com/fabianvf/scrapi/pull/189","diff_url":"https://github.com/fabianvf/scrapi/pull/189.diff","patch_url":"https://github.com/fabianvf/scrapi/pull/189.patch","issue_url":"https://api.github.com/repos/fabianvf/scrapi/issues/189","number":189,"state":"open","locked":false,"title":"Add date range to all harvesters ","user":{"login":"erinspace","id":801594,"avatar_url":"https://avatars.githubusercontent.com/u/801594?v=3","gravatar_id":"","url":"https://api.github.com/users/erinspace","html_url":"https://github.com/erinspace","followers_url":"https://api.github.com/users/erinspace/followers","following_url":"https://api.github.com/users/erinspace/following{/other_user}","gists_url":"https://api.github.com/users/erinspace/gists{/gist_id}","starred_url":"https://api.github.com/users/erinspace/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erinspace/subscriptions","organizations_url":"https://api.github.com/users/erinspace/orgs","repos_url":"https://api.github.com/users/erinspace/repos","events_url":"https://api.github.com/users/erinspace/events{/privacy}","received_events_url":"https://api.github.com/users/erinspace/received_events","type":"User","site_admin":false},"body":"Remove the \"days_back\" argument from harvesters and instead use a date range. Defaults to harvesting from the day before to the current day.\r\n\r\n- [x] arxiv\r\n- [x] asu\r\n- [x] bhl\r\n- [x] biomedcentral\r\n- [x] calhoun\r\n- [x] calpoly\r\n- [x] caltech\r\n- [x] clinicaltrials\r\n- [x] cmu\r\n- [x] cogprints\r\n- [x] columbia\r\n- [x] crossref\r\n- [x] cuscholar\r\n- [x] dataone\r\n- [x] doepages\r\n- [x] figshare\r\n- [x] mit\r\n- [x] opensiuc\r\n- [x] osf\r\n- [x] plos\r\n- [x] pubmedcentral\r\n- [x] scholarsbank\r\n- [x] scitech\r\n- [x] shareok\r\n- [x] spdataverse\r\n- [x] stcloud\r\n- [x] stepic\r\n- [x] tdar\r\n- [x] texasstate\r\n- [x] trinity\r\n- [x] ucescholarship\r\n- [x] uiucideals\r\n- [x] upennsylvania\r\n- [x] utaustin\r\n- [x] uwashington\r\n- [x] valposcholar\r\n- [x] vtech\r\n- [x] waynestate\r\n- [x] zenodo\r\n","created_at":"2015-05-04T19:04:00Z","updated_at":"2015-05-11T14:57:52Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bc3d09dc06af09cbfc7f098a403479961688b399","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/fabianvf/scrapi/pulls/189/commits","review_comments_url":"https://api.github.com/repos/fabianvf/scrapi/pulls/189/comments","review_comment_url":"https://api.github.com/repos/fabianvf/scrapi/pulls/comments{/number}","comments_url":"https://api.github.com/repos/fabianvf/scrapi/issues/189/comments","statuses_url":"https://api.github.com/repos/fabianvf/scrapi/statuses/4d01d4a2a925508e27f9b11313a1784b141a70c3","head":{"label":"erinspace:feature/date_range","ref":"feature/date_range","sha":"4d01d4a2a925508e27f9b11313a1784b141a70c3","user":{"login":"erinspace","id":801594,"avatar_url":"https://avatars.githubusercontent.com/u/801594?v=3","gravatar_id":"","url":"https://api.github.com/users/erinspace","html_url":"https://github.com/erinspace","followers_url":"https://api.github.com/users/erinspace/followers","following_url":"https://api.github.com/users/erinspace/following{/other_user}","gists_url":"https://api.github.com/users/erinspace/gists{/gist_id}","starred_url":"https://api.github.com/users/erinspace/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erinspace/subscriptions","organizations_url":"https://api.github.com/users/erinspace/orgs","repos_url":"https://api.github.com/users/erinspace/repos","events_url":"https://api.github.com/users/erinspace/events{/privacy}","received_events_url":"https://api.github.com/users/erinspace/received_events","type":"User","site_admin":false},"repo":{"id":21392337,"name":"scrapi","full_name":"erinspace/scrapi","owner":{"login":"erinspace","id":801594,"avatar_url":"https://avatars.githubusercontent.com/u/801594?v=3","gravatar_id":"","url":"https://api.github.com/users/erinspace","html_url":"https://github.com/erinspace","followers_url":"https://api.github.com/users/erinspace/followers","following_url":"https://api.github.com/users/erinspace/following{/other_user}","gists_url":"https://api.github.com/users/erinspace/gists{/gist_id}","starred_url":"https://api.github.com/users/erinspace/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/erinspace/subscriptions","organizations_url":"https://api.github.com/users/erinspace/orgs","repos_url":"https://api.github.com/users/erinspace/repos","events_url":"https://api.github.com/users/erinspace/events{/privacy}","received_events_url":"https://api.github.com/users/erinspace/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/erinspace/scrapi","description":"Scrapers + api ","fork":true,"url":"https://api.github.com/repos/erinspace/scrapi","forks_url":"https://api.github.com/repos/erinspace/scrapi/forks","keys_url":"https://api.github.com/repos/erinspace/scrapi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/erinspace/scrapi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/erinspace/scrapi/teams","hooks_url":"https://api.github.com/repos/erinspace/scrapi/hooks","issue_events_url":"https://api.github.com/repos/erinspace/scrapi/issues/events{/number}","events_url":"https://api.github.com/repos/erinspace/scrapi/events","assignees_url":"https://api.github.com/repos/erinspace/scrapi/assignees{/user}","branches_url":"https://api.github.com/repos/erinspace/scrapi/branches{/branch}","tags_url":"https://api.github.com/repos/erinspace/scrapi/tags","blobs_url":"https://api.github.com/repos/erinspace/scrapi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/erinspace/scrapi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/erinspace/scrapi/git/refs{/sha}","trees_url":"https://api.github.com/repos/erinspace/scrapi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/erinspace/scrapi/statuses/{sha}","languages_url":"https://api.github.com/repos/erinspace/scrapi/languages","stargazers_url":"https://api.github.com/repos/erinspace/scrapi/stargazers","contributors_url":"https://api.github.com/repos/erinspace/scrapi/contributors","subscribers_url":"https://api.github.com/repos/erinspace/scrapi/subscribers","subscription_url":"https://api.github.com/repos/erinspace/scrapi/subscription","commits_url":"https://api.github.com/repos/erinspace/scrapi/commits{/sha}","git_commits_url":"https://api.github.com/repos/erinspace/scrapi/git/commits{/sha}","comments_url":"https://api.github.com/repos/erinspace/scrapi/comments{/number}","issue_comment_url":"https://api.github.com/repos/erinspace/scrapi/issues/comments{/number}","contents_url":"https://api.github.com/repos/erinspace/scrapi/contents/{+path}","compare_url":"https://api.github.com/repos/erinspace/scrapi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/erinspace/scrapi/merges","archive_url":"https://api.github.com/repos/erinspace/scrapi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/erinspace/scrapi/downloads","issues_url":"https://api.github.com/repos/erinspace/scrapi/issues{/number}","pulls_url":"https://api.github.com/repos/erinspace/scrapi/pulls{/number}","milestones_url":"https://api.github.com/repos/erinspace/scrapi/milestones{/number}","notifications_url":"https://api.github.com/repos/erinspace/scrapi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/erinspace/scrapi/labels{/name}","releases_url":"https://api.github.com/repos/erinspace/scrapi/releases{/id}","created_at":"2014-07-01T15:12:14Z","updated_at":"2015-05-08T18:35:53Z","pushed_at":"2015-05-08T18:35:53Z","git_url":"git://github.com/erinspace/scrapi.git","ssh_url":"git@github.com:erinspace/scrapi.git","clone_url":"https://github.com/erinspace/scrapi.git","svn_url":"https://github.com/erinspace/scrapi","homepage":null,"size":27900,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"develop"}},"base":{"label":"fabianvf:develop","ref":"develop","sha":"99136fdcf1981ab091981f664a228f49f94c3d4d","user":{"login":"fabianvf","id":2431974,"avatar_url":"https://avatars.githubusercontent.com/u/2431974?v=3","gravatar_id":"","url":"https://api.github.com/users/fabianvf","html_url":"https://github.com/fabianvf","followers_url":"https://api.github.com/users/fabianvf/followers","following_url":"https://api.github.com/users/fabianvf/following{/other_user}","gists_url":"https://api.github.com/users/fabianvf/gists{/gist_id}","starred_url":"https://api.github.com/users/fabianvf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabianvf/subscriptions","organizations_url":"https://api.github.com/users/fabianvf/orgs","repos_url":"https://api.github.com/users/fabianvf/repos","events_url":"https://api.github.com/users/fabianvf/events{/privacy}","received_events_url":"https://api.github.com/users/fabianvf/received_events","type":"User","site_admin":false},"repo":{"id":21210102,"name":"scrapi","full_name":"fabianvf/scrapi","owner":{"login":"fabianvf","id":2431974,"avatar_url":"https://avatars.githubusercontent.com/u/2431974?v=3","gravatar_id":"","url":"https://api.github.com/users/fabianvf","html_url":"https://github.com/fabianvf","followers_url":"https://api.github.com/users/fabianvf/followers","following_url":"https://api.github.com/users/fabianvf/following{/other_user}","gists_url":"https://api.github.com/users/fabianvf/gists{/gist_id}","starred_url":"https://api.github.com/users/fabianvf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabianvf/subscriptions","organizations_url":"https://api.github.com/users/fabianvf/orgs","repos_url":"https://api.github.com/users/fabianvf/repos","events_url":"https://api.github.com/users/fabianvf/events{/privacy}","received_events_url":"https://api.github.com/users/fabianvf/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/fabianvf/scrapi","description":"A data processing pipeline that schedules and runs content harvesters, normalizes their data, and outputs that normalized data to a variety of output streams. Data collected can be explored at https://osf.io/share/, and viewed at https://osf.io/api/v1/share/search/. Developer docs can be viewed at https://osf.io/wur56/wiki","fork":false,"url":"https://api.github.com/repos/fabianvf/scrapi","forks_url":"https://api.github.com/repos/fabianvf/scrapi/forks","keys_url":"https://api.github.com/repos/fabianvf/scrapi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fabianvf/scrapi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fabianvf/scrapi/teams","hooks_url":"https://api.github.com/repos/fabianvf/scrapi/hooks","issue_events_url":"https://api.github.com/repos/fabianvf/scrapi/issues/events{/number}","events_url":"https://api.github.com/repos/fabianvf/scrapi/events","assignees_url":"https://api.github.com/repos/fabianvf/scrapi/assignees{/user}","branches_url":"https://api.github.com/repos/fabianvf/scrapi/branches{/branch}","tags_url":"https://api.github.com/repos/fabianvf/scrapi/tags","blobs_url":"https://api.github.com/repos/fabianvf/scrapi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fabianvf/scrapi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fabianvf/scrapi/git/refs{/sha}","trees_url":"https://api.github.com/repos/fabianvf/scrapi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fabianvf/scrapi/statuses/{sha}","languages_url":"https://api.github.com/repos/fabianvf/scrapi/languages","stargazers_url":"https://api.github.com/repos/fabianvf/scrapi/stargazers","contributors_url":"https://api.github.com/repos/fabianvf/scrapi/contributors","subscribers_url":"https://api.github.com/repos/fabianvf/scrapi/subscribers","subscription_url":"https://api.github.com/repos/fabianvf/scrapi/subscription","commits_url":"https://api.github.com/repos/fabianvf/scrapi/commits{/sha}","git_commits_url":"https://api.github.com/repos/fabianvf/scrapi/git/commits{/sha}","comments_url":"https://api.github.com/repos/fabianvf/scrapi/comments{/number}","issue_comment_url":"https://api.github.com/repos/fabianvf/scrapi/issues/comments{/number}","contents_url":"https://api.github.com/repos/fabianvf/scrapi/contents/{+path}","compare_url":"https://api.github.com/repos/fabianvf/scrapi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fabianvf/scrapi/merges","archive_url":"https://api.github.com/repos/fabianvf/scrapi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fabianvf/scrapi/downloads","issues_url":"https://api.github.com/repos/fabianvf/scrapi/issues{/number}","pulls_url":"https://api.github.com/repos/fabianvf/scrapi/pulls{/number}","milestones_url":"https://api.github.com/repos/fabianvf/scrapi/milestones{/number}","notifications_url":"https://api.github.com/repos/fabianvf/scrapi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fabianvf/scrapi/labels{/name}","releases_url":"https://api.github.com/repos/fabianvf/scrapi/releases{/id}","created_at":"2014-06-25T16:21:08Z","updated_at":"2015-05-07T15:36:27Z","pushed_at":"2015-05-08T18:45:56Z","git_url":"git://github.com/fabianvf/scrapi.git","ssh_url":"git@github.com:fabianvf/scrapi.git","clone_url":"https://github.com/fabianvf/scrapi.git","svn_url":"https://github.com/fabianvf/scrapi","homepage":"","size":30620,"stargazers_count":6,"watchers_count":6,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":19,"mirror_url":null,"open_issues_count":18,"forks":19,"open_issues":18,"watchers":6,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/fabianvf/scrapi/pulls/189"},"html":{"href":"https://github.com/fabianvf/scrapi/pull/189"},"issue":{"href":"https://api.github.com/repos/fabianvf/scrapi/issues/189"},"comments":{"href":"https://api.github.com/repos/fabianvf/scrapi/issues/189/comments"},"review_comments":{"href":"https://api.github.com/repos/fabianvf/scrapi/pulls/189/comments"},"review_comment":{"href":"https://api.github.com/repos/fabianvf/scrapi/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/fabianvf/scrapi/pulls/189/commits"},"statuses":{"href":"https://api.github.com/repos/fabianvf/scrapi/statuses/4d01d4a2a925508e27f9b11313a1784b141a70c3"}}}}