{"action":"created","comment":{"url":"https://api.github.com/repos/mozilla/zamboni/pulls/comments/32987614","id":32987614,"diff_hunk":"@@ -244,3 +246,114 @@ def test_status_filtering(self):\n         eq_(res.status_code, 200)\n         objs = res.json['objects']\n         eq_(len(objs), 1)\n+\n+\n+class TestWebsiteScrape(RestOAuth, TestCase):\n+    def setUp(self):\n+        self.url = reverse('api-v2:website-scrape')\n+        super(TestWebsiteScrape, self).setUp()\n+\n+    def go(self, url=None):\n+        qs = {}\n+        if url:\n+            qs = {'url': url}\n+        response = self.anon.get(self.url, qs)\n+        return response, json.loads(response.content)\n+\n+    def test_no_url(self):\n+        response, content = self.go()\n+        eq_(response.status_code, 400)\n+        eq_(content, WebsiteMetadataScraperView.errors['no_url'])\n+\n+    @responses.activate\n+    def test_site_404(self):\n+        URL = 'https://marketplace.firefox.com/'\n+        responses.add(responses.GET, URL, status=404)\n+        response, content = self.go(url=URL)\n+        eq_(response.status_code, 400)\n+        eq_(content, WebsiteMetadataScraperView.errors['network'])\n+\n+    @responses.activate\n+    def test_site_500(self):\n+        URL = 'https://marketplace.firefox.com/'\n+        responses.add(responses.GET, URL, status=500)\n+        response, content = self.go(url=URL)\n+        eq_(response.status_code, 400)\n+        eq_(content, WebsiteMetadataScraperView.errors['network'])\n+\n+    @responses.activate\n+    def test_empty_body(self):\n+        URL = 'https://marketplace.firefox.com/'\n+        responses.add(responses.GET, URL, '', status=200)\n+        response, content = self.go(url=URL)\n+        eq_(response.status_code, 400)\n+        eq_(content, WebsiteMetadataScraperView.errors['malformed_data'])\n+\n+    @responses.activate\n+    def test_valid(self):\n+        URL = 'https://marketplace.firefox.com/'\n+        responses.add(responses.GET, URL, '<html />', status=200)\n+        response, content = self.go(url=URL)\n+        eq_(response.status_code, 200)\n+\n+    def test_verbs(self):\n+        self._allowed_verbs(self.url, ['get'])\n+\n+    def test_has_cors(self):\n+        self.assertCORS(self.client.get(self.url), 'get')\n+\n+\n+class TestWebsiteScrapeName(TestCase):\n+    \"\"\"\n+    WebsiteMetadataScraperView.get_name will attempt to fetch an appropriate\n+    name for a passed PyQuery document. Here, we're unit testing that logic.\n+\n+    get_name searches for an appropriate website name using a preference order\n+    that reverses the order of self.fragments. We test that it works by:\n+\n+    1) Constructing an HTML document with no children, and ensuring that\n+       get_name returns None.\n+    2) Constructing a document with only <meta name=\"application-name\" />, and\n+       ensuring that get_name returns the value of that tag's `content`\n+       attribute.\n+    3) Constructing a document with both <meta name=\"application-name\" /> and\n+       <meta name=\"apple-mobile-web-app-title\" />, ensuring that get_name\n+       returns the value of the latter's `content` attribute.\n+    4) And so on, adding an item from self.fragments for each subsequent\n+       iteration.\n+    \"\"\"\n+    fragments = [\n+        '<meta name=\"application-name\" content=\"%s\" />',\n+        '<meta name=\"apple-mobile-web-app-title\" content=\"%s\" />',\n+        '<meta property=\"og:title\" content=\"%s\" />',\n+        '<title>%s</title>'\n+    ]\n+    value = 'Pied Piper %s'\n+\n+    def setUp(self):\n+        self.view = WebsiteMetadataScraperView()\n+        super(TestWebsiteScrapeName, self).setUp()\n+\n+    def test_no_values(self):\n+        \"\"\"\n+        Test that None is returned when we can't detect any metadata.\n+        \"\"\"\n+        doc = pq('<html></html>')\n+        eq_(None, self.view.get_name(doc))\n+\n+    def get_value(self, iteration):\n+        return self.value % str(iteration)\n+\n+    def get_fragment(self, iteration):\n+        return self.fragments[iteration - 1] % self.get_value(iteration)\n+\n+    def get_document(self, iteration):\n+        doc = '<html>%s</html>'\n+        inner = [self.get_fragment(i + 1) for i in xrange(0, iteration)]","path":"mkt/websites/tests/test_views.py","position":null,"original_position":126,"commit_id":"c2c85127031ccd33ca6b5a49194dd79a8134de9d","original_commit_id":"13225d9b0792868d02a3edc7d2fa83603f868237","user":{"login":"chuckharmston","id":23885,"avatar_url":"https://avatars.githubusercontent.com/u/23885?v=3","gravatar_id":"","url":"https://api.github.com/users/chuckharmston","html_url":"https://github.com/chuckharmston","followers_url":"https://api.github.com/users/chuckharmston/followers","following_url":"https://api.github.com/users/chuckharmston/following{/other_user}","gists_url":"https://api.github.com/users/chuckharmston/gists{/gist_id}","starred_url":"https://api.github.com/users/chuckharmston/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chuckharmston/subscriptions","organizations_url":"https://api.github.com/users/chuckharmston/orgs","repos_url":"https://api.github.com/users/chuckharmston/repos","events_url":"https://api.github.com/users/chuckharmston/events{/privacy}","received_events_url":"https://api.github.com/users/chuckharmston/received_events","type":"User","site_admin":false},"body":"Also shuffled the array, good call.","created_at":"2015-06-22T21:46:22Z","updated_at":"2015-06-22T21:46:22Z","html_url":"https://github.com/mozilla/zamboni/pull/3148#discussion_r32987614","pull_request_url":"https://api.github.com/repos/mozilla/zamboni/pulls/3148","_links":{"self":{"href":"https://api.github.com/repos/mozilla/zamboni/pulls/comments/32987614"},"html":{"href":"https://github.com/mozilla/zamboni/pull/3148#discussion_r32987614"},"pull_request":{"href":"https://api.github.com/repos/mozilla/zamboni/pulls/3148"}}},"pull_request":{"url":"https://api.github.com/repos/mozilla/zamboni/pulls/3148","id":38161177,"html_url":"https://github.com/mozilla/zamboni/pull/3148","diff_url":"https://github.com/mozilla/zamboni/pull/3148.diff","patch_url":"https://github.com/mozilla/zamboni/pull/3148.patch","issue_url":"https://api.github.com/repos/mozilla/zamboni/issues/3148","number":3148,"state":"open","locked":false,"title":"WIP: add an API endpoint to scrape a URL and return metadata about the site.","user":{"login":"chuckharmston","id":23885,"avatar_url":"https://avatars.githubusercontent.com/u/23885?v=3","gravatar_id":"","url":"https://api.github.com/users/chuckharmston","html_url":"https://github.com/chuckharmston","followers_url":"https://api.github.com/users/chuckharmston/followers","following_url":"https://api.github.com/users/chuckharmston/following{/other_user}","gists_url":"https://api.github.com/users/chuckharmston/gists{/gist_id}","starred_url":"https://api.github.com/users/chuckharmston/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chuckharmston/subscriptions","organizations_url":"https://api.github.com/users/chuckharmston/orgs","repos_url":"https://api.github.com/users/chuckharmston/repos","events_url":"https://api.github.com/users/chuckharmston/events{/privacy}","received_events_url":"https://api.github.com/users/chuckharmston/received_events","type":"User","site_admin":false},"body":"WIP, please don't merge.\r\n\r\nWould love some feedback from smart Python folks on a few items I'll specifically comment on. I wouldn't turn down any other feedback, as well.\r\n\r\nr? @robhudson @ngokevin @diox ","created_at":"2015-06-20T02:21:52Z","updated_at":"2015-06-22T21:46:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a3487a40813c1d93495a3e0284e8451a40bb0c8b","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/mozilla/zamboni/pulls/3148/commits","review_comments_url":"https://api.github.com/repos/mozilla/zamboni/pulls/3148/comments","review_comment_url":"https://api.github.com/repos/mozilla/zamboni/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mozilla/zamboni/issues/3148/comments","statuses_url":"https://api.github.com/repos/mozilla/zamboni/statuses/c2c85127031ccd33ca6b5a49194dd79a8134de9d","head":{"label":"chuckharmston:scraper","ref":"scraper","sha":"c2c85127031ccd33ca6b5a49194dd79a8134de9d","user":{"login":"chuckharmston","id":23885,"avatar_url":"https://avatars.githubusercontent.com/u/23885?v=3","gravatar_id":"","url":"https://api.github.com/users/chuckharmston","html_url":"https://github.com/chuckharmston","followers_url":"https://api.github.com/users/chuckharmston/followers","following_url":"https://api.github.com/users/chuckharmston/following{/other_user}","gists_url":"https://api.github.com/users/chuckharmston/gists{/gist_id}","starred_url":"https://api.github.com/users/chuckharmston/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chuckharmston/subscriptions","organizations_url":"https://api.github.com/users/chuckharmston/orgs","repos_url":"https://api.github.com/users/chuckharmston/repos","events_url":"https://api.github.com/users/chuckharmston/events{/privacy}","received_events_url":"https://api.github.com/users/chuckharmston/received_events","type":"User","site_admin":false},"repo":{"id":17600281,"name":"zamboni","full_name":"chuckharmston/zamboni","owner":{"login":"chuckharmston","id":23885,"avatar_url":"https://avatars.githubusercontent.com/u/23885?v=3","gravatar_id":"","url":"https://api.github.com/users/chuckharmston","html_url":"https://github.com/chuckharmston","followers_url":"https://api.github.com/users/chuckharmston/followers","following_url":"https://api.github.com/users/chuckharmston/following{/other_user}","gists_url":"https://api.github.com/users/chuckharmston/gists{/gist_id}","starred_url":"https://api.github.com/users/chuckharmston/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chuckharmston/subscriptions","organizations_url":"https://api.github.com/users/chuckharmston/orgs","repos_url":"https://api.github.com/users/chuckharmston/repos","events_url":"https://api.github.com/users/chuckharmston/events{/privacy}","received_events_url":"https://api.github.com/users/chuckharmston/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/chuckharmston/zamboni","description":"The Firefox Marketplace and addons.mozilla.org","fork":true,"url":"https://api.github.com/repos/chuckharmston/zamboni","forks_url":"https://api.github.com/repos/chuckharmston/zamboni/forks","keys_url":"https://api.github.com/repos/chuckharmston/zamboni/keys{/key_id}","collaborators_url":"https://api.github.com/repos/chuckharmston/zamboni/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/chuckharmston/zamboni/teams","hooks_url":"https://api.github.com/repos/chuckharmston/zamboni/hooks","issue_events_url":"https://api.github.com/repos/chuckharmston/zamboni/issues/events{/number}","events_url":"https://api.github.com/repos/chuckharmston/zamboni/events","assignees_url":"https://api.github.com/repos/chuckharmston/zamboni/assignees{/user}","branches_url":"https://api.github.com/repos/chuckharmston/zamboni/branches{/branch}","tags_url":"https://api.github.com/repos/chuckharmston/zamboni/tags","blobs_url":"https://api.github.com/repos/chuckharmston/zamboni/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/chuckharmston/zamboni/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/chuckharmston/zamboni/git/refs{/sha}","trees_url":"https://api.github.com/repos/chuckharmston/zamboni/git/trees{/sha}","statuses_url":"https://api.github.com/repos/chuckharmston/zamboni/statuses/{sha}","languages_url":"https://api.github.com/repos/chuckharmston/zamboni/languages","stargazers_url":"https://api.github.com/repos/chuckharmston/zamboni/stargazers","contributors_url":"https://api.github.com/repos/chuckharmston/zamboni/contributors","subscribers_url":"https://api.github.com/repos/chuckharmston/zamboni/subscribers","subscription_url":"https://api.github.com/repos/chuckharmston/zamboni/subscription","commits_url":"https://api.github.com/repos/chuckharmston/zamboni/commits{/sha}","git_commits_url":"https://api.github.com/repos/chuckharmston/zamboni/git/commits{/sha}","comments_url":"https://api.github.com/repos/chuckharmston/zamboni/comments{/number}","issue_comment_url":"https://api.github.com/repos/chuckharmston/zamboni/issues/comments{/number}","contents_url":"https://api.github.com/repos/chuckharmston/zamboni/contents/{+path}","compare_url":"https://api.github.com/repos/chuckharmston/zamboni/compare/{base}...{head}","merges_url":"https://api.github.com/repos/chuckharmston/zamboni/merges","archive_url":"https://api.github.com/repos/chuckharmston/zamboni/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/chuckharmston/zamboni/downloads","issues_url":"https://api.github.com/repos/chuckharmston/zamboni/issues{/number}","pulls_url":"https://api.github.com/repos/chuckharmston/zamboni/pulls{/number}","milestones_url":"https://api.github.com/repos/chuckharmston/zamboni/milestones{/number}","notifications_url":"https://api.github.com/repos/chuckharmston/zamboni/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/chuckharmston/zamboni/labels{/name}","releases_url":"https://api.github.com/repos/chuckharmston/zamboni/releases{/id}","created_at":"2014-03-10T16:49:52Z","updated_at":"2015-03-21T19:46:50Z","pushed_at":"2015-06-22T21:44:39Z","git_url":"git://github.com/chuckharmston/zamboni.git","ssh_url":"git@github.com:chuckharmston/zamboni.git","clone_url":"https://github.com/chuckharmston/zamboni.git","svn_url":"https://github.com/chuckharmston/zamboni","homepage":"http://zamboni.readthedocs.org","size":362613,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":false,"has_wiki":false,"has_pages":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"mozilla:master","ref":"master","sha":"32f95899492d51e78caaf9a08caa28fcfcb2c172","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":347655,"name":"zamboni","full_name":"mozilla/zamboni","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/zamboni","description":"Backend for the Firefox Marketplace","fork":false,"url":"https://api.github.com/repos/mozilla/zamboni","forks_url":"https://api.github.com/repos/mozilla/zamboni/forks","keys_url":"https://api.github.com/repos/mozilla/zamboni/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/zamboni/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/zamboni/teams","hooks_url":"https://api.github.com/repos/mozilla/zamboni/hooks","issue_events_url":"https://api.github.com/repos/mozilla/zamboni/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/zamboni/events","assignees_url":"https://api.github.com/repos/mozilla/zamboni/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/zamboni/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/zamboni/tags","blobs_url":"https://api.github.com/repos/mozilla/zamboni/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/zamboni/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/zamboni/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/zamboni/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/zamboni/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/zamboni/languages","stargazers_url":"https://api.github.com/repos/mozilla/zamboni/stargazers","contributors_url":"https://api.github.com/repos/mozilla/zamboni/contributors","subscribers_url":"https://api.github.com/repos/mozilla/zamboni/subscribers","subscription_url":"https://api.github.com/repos/mozilla/zamboni/subscription","commits_url":"https://api.github.com/repos/mozilla/zamboni/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/zamboni/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/zamboni/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/zamboni/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/zamboni/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/zamboni/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/zamboni/merges","archive_url":"https://api.github.com/repos/mozilla/zamboni/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/zamboni/downloads","issues_url":"https://api.github.com/repos/mozilla/zamboni/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/zamboni/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/zamboni/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/zamboni/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/zamboni/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/zamboni/releases{/id}","created_at":"2009-10-23T19:35:34Z","updated_at":"2015-06-20T20:16:00Z","pushed_at":"2015-06-22T21:44:41Z","git_url":"git://github.com/mozilla/zamboni.git","ssh_url":"git@github.com:mozilla/zamboni.git","clone_url":"https://github.com/mozilla/zamboni.git","svn_url":"https://github.com/mozilla/zamboni","homepage":"http://marketplace.readthedocs.org/en/latest/topics/backend.html","size":433132,"stargazers_count":464,"watchers_count":464,"language":"Python","has_issues":false,"has_downloads":false,"has_wiki":false,"has_pages":false,"forks_count":244,"mirror_url":null,"open_issues_count":17,"forks":244,"open_issues":17,"watchers":464,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mozilla/zamboni/pulls/3148"},"html":{"href":"https://github.com/mozilla/zamboni/pull/3148"},"issue":{"href":"https://api.github.com/repos/mozilla/zamboni/issues/3148"},"comments":{"href":"https://api.github.com/repos/mozilla/zamboni/issues/3148/comments"},"review_comments":{"href":"https://api.github.com/repos/mozilla/zamboni/pulls/3148/comments"},"review_comment":{"href":"https://api.github.com/repos/mozilla/zamboni/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mozilla/zamboni/pulls/3148/commits"},"statuses":{"href":"https://api.github.com/repos/mozilla/zamboni/statuses/c2c85127031ccd33ca6b5a49194dd79a8134de9d"}}}}