{"action":"created","comment":{"url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/comments/48239827","id":48239827,"diff_hunk":"@@ -0,0 +1,93 @@\n+import logging\n+import os\n+from urlparse import urlunsplit\n+from httplib2 import Http\n+import transaction\n+\n+from thinkhazard_common.models import DBSession\n+\n+from . import settings\n+from .models import Layer\n+\n+\n+logger = logging.getLogger(__name__)\n+\n+ch = logging.StreamHandler()\n+ch.setLevel(logging.DEBUG)\n+formatter = logging.Formatter('%(levelname)s - %(message)s')\n+ch.setFormatter(formatter)\n+logger.addHandler(ch)\n+\n+logger.setLevel(logging.DEBUG)\n+\n+\n+geonode = settings['geonode']\n+\n+\n+def clearall():\n+    logger.info('Reset all layer to not downloaded state.')\n+    DBSession.query(Layer).update({\n+        Layer.downloaded: False\n+    })\n+    DBSession.flush()\n+\n+\n+def download(title=None, force=False, dry_run=False):\n+    if force:\n+        try:\n+            clearall()\n+            if dry_run:\n+                transaction.abort()\n+            else:\n+                transaction.commit()\n+        except:\n+            transaction.abort()\n+            raise\n+\n+    geonode_ids = DBSession.query(Layer.geonode_id)\n+\n+    if not force:\n+        geonode_ids = geonode_ids.filter(Layer.downloaded.is_(False))\n+\n+    if title is not None:\n+        geonode_ids = geonode_ids.filter(Layer.title == title)\n+\n+    for geonode_id in geonode_ids:\n+        try:\n+            download_layer(geonode_id)\n+            if dry_run:\n+                transaction.abort()\n+            else:\n+                transaction.commit()\n+        except Exception as e:\n+            transaction.abort()\n+            logger.error('{} raise an exception :\\n{}'\n+                         .format(geonode_id, e.message))\n+\n+\n+def download_layer(geonode_id):\n+    layer = DBSession.query(Layer).get(geonode_id)\n+    if layer is None:\n+        raise Exception('Layer {} does not exist.'.format(geonode_id))\n+\n+    logger.info('Downloading layer {}'.format(layer.name()))\n+\n+    dir_path = os.path.dirname(layer.path())\n+    if not os.path.exists(dir_path):\n+        os.makedirs(dir_path)\n+\n+    if not os.path.isfile(layer.path()):\n+        h = Http()\n+        url = urlunsplit((geonode['scheme'],\n+                          geonode['netloc'],\n+                          layer.download_url,\n+                          '',\n+                          ''))\n+        logger.info('Retrieving {}'.format(url))\n+        response, content = h.request(url)\n+\n+        with open(layer.path(), 'wb') as f:\n+            f.write(content)\n+\n+    layer.downloaded = True","path":"thinkhazard_processing/downloading.py","position":92,"original_position":92,"commit_id":"b4a135a80948ba866d783fc65ddf03a7ed76e876","original_commit_id":"b4a135a80948ba866d783fc65ddf03a7ed76e876","user":{"login":"fvanderbiest","id":265319,"avatar_url":"https://avatars.githubusercontent.com/u/265319?v=3","gravatar_id":"","url":"https://api.github.com/users/fvanderbiest","html_url":"https://github.com/fvanderbiest","followers_url":"https://api.github.com/users/fvanderbiest/followers","following_url":"https://api.github.com/users/fvanderbiest/following{/other_user}","gists_url":"https://api.github.com/users/fvanderbiest/gists{/gist_id}","starred_url":"https://api.github.com/users/fvanderbiest/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fvanderbiest/subscriptions","organizations_url":"https://api.github.com/users/fvanderbiest/orgs","repos_url":"https://api.github.com/users/fvanderbiest/repos","events_url":"https://api.github.com/users/fvanderbiest/events{/privacy}","received_events_url":"https://api.github.com/users/fvanderbiest/received_events","type":"User","site_admin":false},"body":"layer should be marked as downloaded on file write success","created_at":"2015-12-22T10:17:11Z","updated_at":"2015-12-22T10:17:11Z","html_url":"https://github.com/GFDRR/thinkhazard_processing/pull/23#discussion_r48239827","pull_request_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/23","_links":{"self":{"href":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/comments/48239827"},"html":{"href":"https://github.com/GFDRR/thinkhazard_processing/pull/23#discussion_r48239827"},"pull_request":{"href":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/23"}}},"pull_request":{"url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/23","id":53953740,"html_url":"https://github.com/GFDRR/thinkhazard_processing/pull/23","diff_url":"https://github.com/GFDRR/thinkhazard_processing/pull/23.diff","patch_url":"https://github.com/GFDRR/thinkhazard_processing/pull/23.patch","issue_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/issues/23","number":23,"state":"open","locked":false,"title":"[WIP] Amorvan harvesting based on #22","user":{"login":"arnaud-morvan","id":6095494,"avatar_url":"https://avatars.githubusercontent.com/u/6095494?v=3","gravatar_id":"","url":"https://api.github.com/users/arnaud-morvan","html_url":"https://github.com/arnaud-morvan","followers_url":"https://api.github.com/users/arnaud-morvan/followers","following_url":"https://api.github.com/users/arnaud-morvan/following{/other_user}","gists_url":"https://api.github.com/users/arnaud-morvan/gists{/gist_id}","starred_url":"https://api.github.com/users/arnaud-morvan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/arnaud-morvan/subscriptions","organizations_url":"https://api.github.com/users/arnaud-morvan/orgs","repos_url":"https://api.github.com/users/arnaud-morvan/repos","events_url":"https://api.github.com/users/arnaud-morvan/events{/privacy}","received_events_url":"https://api.github.com/users/arnaud-morvan/received_events","type":"User","site_admin":false},"body":"This PR adds:\r\n* harvesting from geonode\r\n* downloading raster data\r\n* completing hazardsets\r\n\r\nDoes not yet react to medata changes, but comments are welcome.","created_at":"2015-12-17T09:42:27Z","updated_at":"2015-12-22T10:17:12Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2d94d06e6ca410bb6e89c1deb9503943403b751e","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/23/commits","review_comments_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/23/comments","review_comment_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/comments{/number}","comments_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/issues/23/comments","statuses_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/statuses/b4a135a80948ba866d783fc65ddf03a7ed76e876","head":{"label":"GFDRR:amorvan_harvesting","ref":"amorvan_harvesting","sha":"b4a135a80948ba866d783fc65ddf03a7ed76e876","user":{"login":"GFDRR","id":708300,"avatar_url":"https://avatars.githubusercontent.com/u/708300?v=3","gravatar_id":"","url":"https://api.github.com/users/GFDRR","html_url":"https://github.com/GFDRR","followers_url":"https://api.github.com/users/GFDRR/followers","following_url":"https://api.github.com/users/GFDRR/following{/other_user}","gists_url":"https://api.github.com/users/GFDRR/gists{/gist_id}","starred_url":"https://api.github.com/users/GFDRR/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GFDRR/subscriptions","organizations_url":"https://api.github.com/users/GFDRR/orgs","repos_url":"https://api.github.com/users/GFDRR/repos","events_url":"https://api.github.com/users/GFDRR/events{/privacy}","received_events_url":"https://api.github.com/users/GFDRR/received_events","type":"Organization","site_admin":false},"repo":{"id":45904735,"name":"thinkhazard_processing","full_name":"GFDRR/thinkhazard_processing","owner":{"login":"GFDRR","id":708300,"avatar_url":"https://avatars.githubusercontent.com/u/708300?v=3","gravatar_id":"","url":"https://api.github.com/users/GFDRR","html_url":"https://github.com/GFDRR","followers_url":"https://api.github.com/users/GFDRR/followers","following_url":"https://api.github.com/users/GFDRR/following{/other_user}","gists_url":"https://api.github.com/users/GFDRR/gists{/gist_id}","starred_url":"https://api.github.com/users/GFDRR/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GFDRR/subscriptions","organizations_url":"https://api.github.com/users/GFDRR/orgs","repos_url":"https://api.github.com/users/GFDRR/repos","events_url":"https://api.github.com/users/GFDRR/events{/privacy}","received_events_url":"https://api.github.com/users/GFDRR/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/GFDRR/thinkhazard_processing","description":"Think Hazard: Overcome Risk - Processing module","fork":false,"url":"https://api.github.com/repos/GFDRR/thinkhazard_processing","forks_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/forks","keys_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/keys{/key_id}","collaborators_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/teams","hooks_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/hooks","issue_events_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/issues/events{/number}","events_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/events","assignees_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/assignees{/user}","branches_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/branches{/branch}","tags_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/tags","blobs_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/git/refs{/sha}","trees_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/git/trees{/sha}","statuses_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/statuses/{sha}","languages_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/languages","stargazers_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/stargazers","contributors_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/contributors","subscribers_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/subscribers","subscription_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/subscription","commits_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/commits{/sha}","git_commits_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/git/commits{/sha}","comments_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/comments{/number}","issue_comment_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/issues/comments{/number}","contents_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/contents/{+path}","compare_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/compare/{base}...{head}","merges_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/merges","archive_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/downloads","issues_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/issues{/number}","pulls_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls{/number}","milestones_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/milestones{/number}","notifications_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/labels{/name}","releases_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/releases{/id}","created_at":"2015-11-10T10:28:07Z","updated_at":"2015-11-19T16:26:19Z","pushed_at":"2015-12-22T09:06:21Z","git_url":"git://github.com/GFDRR/thinkhazard_processing.git","ssh_url":"git@github.com:GFDRR/thinkhazard_processing.git","clone_url":"https://github.com/GFDRR/thinkhazard_processing.git","svn_url":"https://github.com/GFDRR/thinkhazard_processing","homepage":null,"size":83,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":10,"forks":1,"open_issues":10,"watchers":0,"default_branch":"master"}},"base":{"label":"GFDRR:master","ref":"master","sha":"160d54aeb4943c36ee40a75fd3658243cf3729c0","user":{"login":"GFDRR","id":708300,"avatar_url":"https://avatars.githubusercontent.com/u/708300?v=3","gravatar_id":"","url":"https://api.github.com/users/GFDRR","html_url":"https://github.com/GFDRR","followers_url":"https://api.github.com/users/GFDRR/followers","following_url":"https://api.github.com/users/GFDRR/following{/other_user}","gists_url":"https://api.github.com/users/GFDRR/gists{/gist_id}","starred_url":"https://api.github.com/users/GFDRR/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GFDRR/subscriptions","organizations_url":"https://api.github.com/users/GFDRR/orgs","repos_url":"https://api.github.com/users/GFDRR/repos","events_url":"https://api.github.com/users/GFDRR/events{/privacy}","received_events_url":"https://api.github.com/users/GFDRR/received_events","type":"Organization","site_admin":false},"repo":{"id":45904735,"name":"thinkhazard_processing","full_name":"GFDRR/thinkhazard_processing","owner":{"login":"GFDRR","id":708300,"avatar_url":"https://avatars.githubusercontent.com/u/708300?v=3","gravatar_id":"","url":"https://api.github.com/users/GFDRR","html_url":"https://github.com/GFDRR","followers_url":"https://api.github.com/users/GFDRR/followers","following_url":"https://api.github.com/users/GFDRR/following{/other_user}","gists_url":"https://api.github.com/users/GFDRR/gists{/gist_id}","starred_url":"https://api.github.com/users/GFDRR/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GFDRR/subscriptions","organizations_url":"https://api.github.com/users/GFDRR/orgs","repos_url":"https://api.github.com/users/GFDRR/repos","events_url":"https://api.github.com/users/GFDRR/events{/privacy}","received_events_url":"https://api.github.com/users/GFDRR/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/GFDRR/thinkhazard_processing","description":"Think Hazard: Overcome Risk - Processing module","fork":false,"url":"https://api.github.com/repos/GFDRR/thinkhazard_processing","forks_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/forks","keys_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/keys{/key_id}","collaborators_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/teams","hooks_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/hooks","issue_events_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/issues/events{/number}","events_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/events","assignees_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/assignees{/user}","branches_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/branches{/branch}","tags_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/tags","blobs_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/git/refs{/sha}","trees_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/git/trees{/sha}","statuses_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/statuses/{sha}","languages_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/languages","stargazers_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/stargazers","contributors_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/contributors","subscribers_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/subscribers","subscription_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/subscription","commits_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/commits{/sha}","git_commits_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/git/commits{/sha}","comments_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/comments{/number}","issue_comment_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/issues/comments{/number}","contents_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/contents/{+path}","compare_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/compare/{base}...{head}","merges_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/merges","archive_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/downloads","issues_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/issues{/number}","pulls_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls{/number}","milestones_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/milestones{/number}","notifications_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/labels{/name}","releases_url":"https://api.github.com/repos/GFDRR/thinkhazard_processing/releases{/id}","created_at":"2015-11-10T10:28:07Z","updated_at":"2015-11-19T16:26:19Z","pushed_at":"2015-12-22T09:06:21Z","git_url":"git://github.com/GFDRR/thinkhazard_processing.git","ssh_url":"git@github.com:GFDRR/thinkhazard_processing.git","clone_url":"https://github.com/GFDRR/thinkhazard_processing.git","svn_url":"https://github.com/GFDRR/thinkhazard_processing","homepage":null,"size":83,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":10,"forks":1,"open_issues":10,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/23"},"html":{"href":"https://github.com/GFDRR/thinkhazard_processing/pull/23"},"issue":{"href":"https://api.github.com/repos/GFDRR/thinkhazard_processing/issues/23"},"comments":{"href":"https://api.github.com/repos/GFDRR/thinkhazard_processing/issues/23/comments"},"review_comments":{"href":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/23/comments"},"review_comment":{"href":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/GFDRR/thinkhazard_processing/pulls/23/commits"},"statuses":{"href":"https://api.github.com/repos/GFDRR/thinkhazard_processing/statuses/b4a135a80948ba866d783fc65ddf03a7ed76e876"}}}}