{"action":"created","comment":{"url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/comments/44708109","id":44708109,"diff_hunk":"@@ -0,0 +1,336 @@\n+import json\n+import time\n+from collections import defaultdict\n+from datetime import datetime, timedelta\n+\n+from dateutil.parser import parse as parse_date\n+from pylons import app_globals as g\n+from sqlalchemy.orm import scoped_session, sessionmaker\n+\n+from r2.lib import (\n+    amqp,\n+    promote,\n+)\n+from r2.models import (\n+    Link,\n+    PromoCampaign,\n+)\n+from r2.models.traffic import (\n+    engine,\n+    AdserverClickthroughsByCodename,\n+    AdserverImpressionsByCodename,\n+    AdserverSpentByCodename,\n+    AdserverTargetedClickthroughsByCodename,\n+    AdserverTargetedImpressionsByCodename,\n+    AdserverTargetedSpentByCodename,\n+)\n+\n+from reddit_adzerk import (\n+    adzerk_api,\n+    report,\n+)\n+\n+def _ctr(clicks, impressions):\n+    if not impressions:\n+        return 0\n+\n+    return clicks / (impressions * 1.0) * 100\n+\n+\n+def queue_promo_reports():\n+    promos = promote.get_served_promos_between(-1, 0)\n+    already_processed_links = set()\n+\n+    for campaign, link in promos:\n+        if link._id36 not in already_processed_links:\n+            _queue_link_report(link)\n+            already_processed_links.add(link._id36)\n+\n+        _queue_promo_report(campaign)\n+\n+\n+def _queue_link_report(link):\n+    amqp.add_item(\"adzerk_reporting_q\", json.dumps({\n+        \"action\": \"queue_daily_link_report\",\n+        \"link_id\": link._id,\n+    }))\n+\n+\n+def _queue_promo_report(campaign):\n+    amqp.add_item(\"adzerk_reporting_q\", json.dumps({\n+        \"action\": \"queue_lifetime_campaign_report\",\n+        \"campaign_id\": campaign._id,\n+    }))\n+\n+\n+def process_report_q():\n+    def _handle_lifetime_campaign_report(campaign_id, report_id, report_queued):\n+        campaign = PromoCampaign._byID(campaign_id, data=True)\n+\n+        try:\n+            report_result = report.fetch_report(report_id)\n+        except report.ReportPendingException as e:\n+            time.sleep(1)\n+            raise\n+        except report.ReportFailedException as e:\n+            g.log.error(e)\n+\n+            # retry if report failed\n+            _queue_promo_report(campaign)\n+            return\n+\n+        impressions = report_result.get(\"TotalImpressions\", 0)\n+        clicks = report_result.get(\"TotalClicks\", 0) - report_result.get(\"TotalFraudulentClicks\", 0)\n+\n+        # adzerk processes clicks faster than impressions\n+        if clicks > impressions:\n+            return\n+\n+        campaign.adserver_spent = report_result.get(\"TotalTrueRevenue\", 0)\n+        campaign.adserver_impressions = impressions\n+        campaign.adserver_clicks = clicks\n+        campaign.last_lifetime_report = report_id\n+        campaign.last_lifetime_report_run = report_queued\n+\n+        campaign._commit()\n+\n+    def _handle_daily_link_report(link_id, report_id, report_queued):\n+        link = Link._byID(link_id, data=True)\n+\n+        try:\n+            report_result = report.fetch_report(report_id)\n+        except report.ReportPendingException as e:\n+            time.sleep(1)\n+            raise\n+        except report.ReportFailedException as e:\n+            g.log.error(e)\n+\n+            # retry if report failed\n+            _queue_link_report(link)\n+            return\n+\n+        g.log.debug(report_data)\n+\n+        campaigns_by_fullname = {campaign._fullname: campaign for campaign\n+            in PromoCampaign._by_link(link._id)}\n+\n+        session = scoped_session(sessionmaker(bind=engine))\n+\n+        for record in report_result.get(\"Records\", []):\n+            impressions = record.get(\"Impressions\", 0)\n+            clicks = record.get(\"Clicks\", 0) - record.get(\"FraudulentClicks\", 0)\n+\n+            # adzerk processes clicks faster than impressions\n+            if clicks > impressions:\n+                continue","path":"reddit_adzerk/adzerkreporting.py","position":null,"original_position":125,"commit_id":"5b0c097e9bcf239101d1f17360203f8c98659456","original_commit_id":"fa7cbcddbbf3a419e2540b58d6e83954cdb819fe","user":{"login":"bsimpson63","id":833317,"avatar_url":"https://avatars.githubusercontent.com/u/833317?v=3","gravatar_id":"","url":"https://api.github.com/users/bsimpson63","html_url":"https://github.com/bsimpson63","followers_url":"https://api.github.com/users/bsimpson63/followers","following_url":"https://api.github.com/users/bsimpson63/following{/other_user}","gists_url":"https://api.github.com/users/bsimpson63/gists{/gist_id}","starred_url":"https://api.github.com/users/bsimpson63/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bsimpson63/subscriptions","organizations_url":"https://api.github.com/users/bsimpson63/orgs","repos_url":"https://api.github.com/users/bsimpson63/repos","events_url":"https://api.github.com/users/bsimpson63/events{/privacy}","received_events_url":"https://api.github.com/users/bsimpson63/received_events","type":"User","site_admin":false},"body":"That's putting a lot of faith in adzerk and that this will only happen when adzerk is still processing. What if this condition remained for a while and we didn't update clicks and impressions for this ad? Instead, could you write out all the data and filter it at display time so that clicks don't exceed impressions?","created_at":"2015-11-12T20:11:49Z","updated_at":"2015-11-12T20:11:49Z","html_url":"https://github.com/reddit/reddit-plugin-adzerk/pull/108#discussion_r44708109","pull_request_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/108","_links":{"self":{"href":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/comments/44708109"},"html":{"href":"https://github.com/reddit/reddit-plugin-adzerk/pull/108#discussion_r44708109"},"pull_request":{"href":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/108"}}},"pull_request":{"url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/108","id":50086972,"html_url":"https://github.com/reddit/reddit-plugin-adzerk/pull/108","diff_url":"https://github.com/reddit/reddit-plugin-adzerk/pull/108.diff","patch_url":"https://github.com/reddit/reddit-plugin-adzerk/pull/108.patch","issue_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/issues/108","number":108,"state":"open","locked":false,"title":"Ads reporting: Add jobs to run/process link and campaign reports.","user":{"login":"dwick","id":823747,"avatar_url":"https://avatars.githubusercontent.com/u/823747?v=3","gravatar_id":"","url":"https://api.github.com/users/dwick","html_url":"https://github.com/dwick","followers_url":"https://api.github.com/users/dwick/followers","following_url":"https://api.github.com/users/dwick/following{/other_user}","gists_url":"https://api.github.com/users/dwick/gists{/gist_id}","starred_url":"https://api.github.com/users/dwick/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwick/subscriptions","organizations_url":"https://api.github.com/users/dwick/orgs","repos_url":"https://api.github.com/users/dwick/repos","events_url":"https://api.github.com/users/dwick/events{/privacy}","received_events_url":"https://api.github.com/users/dwick/received_events","type":"User","site_admin":false},"body":" - Campaign reports are lifetime totals.\r\n - Link reports get a daily breakdown by campaign.\r\n\r\nHourly job: reddit-job-queue_promo_reports.conf\r\nQueue processor: reddit-consumer-adzerk_reporting_q.conf\r\n\r\n:eyeglasses: @bsimpson63 @zeantsoi \r\n\r\npuppet stuff coming soon","created_at":"2015-11-09T05:44:47Z","updated_at":"2015-11-12T20:11:49Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2d71eb78047d491ba2c39e1153a2481790d2c474","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/108/commits","review_comments_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/108/comments","review_comment_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/comments{/number}","comments_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/issues/108/comments","statuses_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/statuses/5b0c097e9bcf239101d1f17360203f8c98659456","head":{"label":"dwick:reporting","ref":"reporting","sha":"5b0c097e9bcf239101d1f17360203f8c98659456","user":{"login":"dwick","id":823747,"avatar_url":"https://avatars.githubusercontent.com/u/823747?v=3","gravatar_id":"","url":"https://api.github.com/users/dwick","html_url":"https://github.com/dwick","followers_url":"https://api.github.com/users/dwick/followers","following_url":"https://api.github.com/users/dwick/following{/other_user}","gists_url":"https://api.github.com/users/dwick/gists{/gist_id}","starred_url":"https://api.github.com/users/dwick/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwick/subscriptions","organizations_url":"https://api.github.com/users/dwick/orgs","repos_url":"https://api.github.com/users/dwick/repos","events_url":"https://api.github.com/users/dwick/events{/privacy}","received_events_url":"https://api.github.com/users/dwick/received_events","type":"User","site_admin":false},"repo":{"id":30618338,"name":"reddit-plugin-adzerk","full_name":"dwick/reddit-plugin-adzerk","owner":{"login":"dwick","id":823747,"avatar_url":"https://avatars.githubusercontent.com/u/823747?v=3","gravatar_id":"","url":"https://api.github.com/users/dwick","html_url":"https://github.com/dwick","followers_url":"https://api.github.com/users/dwick/followers","following_url":"https://api.github.com/users/dwick/following{/other_user}","gists_url":"https://api.github.com/users/dwick/gists{/gist_id}","starred_url":"https://api.github.com/users/dwick/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwick/subscriptions","organizations_url":"https://api.github.com/users/dwick/orgs","repos_url":"https://api.github.com/users/dwick/repos","events_url":"https://api.github.com/users/dwick/events{/privacy}","received_events_url":"https://api.github.com/users/dwick/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/dwick/reddit-plugin-adzerk","description":"reddit plugin for adzerk-hosted adverts","fork":true,"url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk","forks_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/forks","keys_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/teams","hooks_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/hooks","issue_events_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/issues/events{/number}","events_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/events","assignees_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/assignees{/user}","branches_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/branches{/branch}","tags_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/tags","blobs_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/git/refs{/sha}","trees_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/statuses/{sha}","languages_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/languages","stargazers_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/stargazers","contributors_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/contributors","subscribers_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/subscribers","subscription_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/subscription","commits_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/commits{/sha}","git_commits_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/git/commits{/sha}","comments_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/comments{/number}","issue_comment_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/issues/comments{/number}","contents_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/contents/{+path}","compare_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/merges","archive_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/downloads","issues_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/issues{/number}","pulls_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/pulls{/number}","milestones_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/milestones{/number}","notifications_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/labels{/name}","releases_url":"https://api.github.com/repos/dwick/reddit-plugin-adzerk/releases{/id}","created_at":"2015-02-10T22:37:08Z","updated_at":"2015-02-10T22:37:09Z","pushed_at":"2015-11-12T19:11:07Z","git_url":"git://github.com/dwick/reddit-plugin-adzerk.git","ssh_url":"git@github.com:dwick/reddit-plugin-adzerk.git","clone_url":"https://github.com/dwick/reddit-plugin-adzerk.git","svn_url":"https://github.com/dwick/reddit-plugin-adzerk","homepage":"","size":559,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"reddit:master","ref":"master","sha":"db00057736819edc80f5596ab0b92bafd2265cc0","user":{"login":"reddit","id":14248,"avatar_url":"https://avatars.githubusercontent.com/u/14248?v=3","gravatar_id":"","url":"https://api.github.com/users/reddit","html_url":"https://github.com/reddit","followers_url":"https://api.github.com/users/reddit/followers","following_url":"https://api.github.com/users/reddit/following{/other_user}","gists_url":"https://api.github.com/users/reddit/gists{/gist_id}","starred_url":"https://api.github.com/users/reddit/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/reddit/subscriptions","organizations_url":"https://api.github.com/users/reddit/orgs","repos_url":"https://api.github.com/users/reddit/repos","events_url":"https://api.github.com/users/reddit/events{/privacy}","received_events_url":"https://api.github.com/users/reddit/received_events","type":"Organization","site_admin":false},"repo":{"id":7122209,"name":"reddit-plugin-adzerk","full_name":"reddit/reddit-plugin-adzerk","owner":{"login":"reddit","id":14248,"avatar_url":"https://avatars.githubusercontent.com/u/14248?v=3","gravatar_id":"","url":"https://api.github.com/users/reddit","html_url":"https://github.com/reddit","followers_url":"https://api.github.com/users/reddit/followers","following_url":"https://api.github.com/users/reddit/following{/other_user}","gists_url":"https://api.github.com/users/reddit/gists{/gist_id}","starred_url":"https://api.github.com/users/reddit/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/reddit/subscriptions","organizations_url":"https://api.github.com/users/reddit/orgs","repos_url":"https://api.github.com/users/reddit/repos","events_url":"https://api.github.com/users/reddit/events{/privacy}","received_events_url":"https://api.github.com/users/reddit/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/reddit/reddit-plugin-adzerk","description":"reddit plugin for adzerk-hosted adverts","fork":false,"url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk","forks_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/forks","keys_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/teams","hooks_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/hooks","issue_events_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/issues/events{/number}","events_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/events","assignees_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/assignees{/user}","branches_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/branches{/branch}","tags_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/tags","blobs_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/git/refs{/sha}","trees_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/statuses/{sha}","languages_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/languages","stargazers_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/stargazers","contributors_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/contributors","subscribers_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/subscribers","subscription_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/subscription","commits_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/commits{/sha}","git_commits_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/git/commits{/sha}","comments_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/comments{/number}","issue_comment_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/issues/comments{/number}","contents_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/contents/{+path}","compare_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/merges","archive_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/downloads","issues_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/issues{/number}","pulls_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls{/number}","milestones_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/milestones{/number}","notifications_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/labels{/name}","releases_url":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/releases{/id}","created_at":"2012-12-12T00:59:20Z","updated_at":"2015-04-30T21:19:46Z","pushed_at":"2015-11-12T19:11:08Z","git_url":"git://github.com/reddit/reddit-plugin-adzerk.git","ssh_url":"git@github.com:reddit/reddit-plugin-adzerk.git","clone_url":"https://github.com/reddit/reddit-plugin-adzerk.git","svn_url":"https://github.com/reddit/reddit-plugin-adzerk","homepage":"","size":1851,"stargazers_count":4,"watchers_count":4,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":14,"mirror_url":null,"open_issues_count":7,"forks":14,"open_issues":7,"watchers":4,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/108"},"html":{"href":"https://github.com/reddit/reddit-plugin-adzerk/pull/108"},"issue":{"href":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/issues/108"},"comments":{"href":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/issues/108/comments"},"review_comments":{"href":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/108/comments"},"review_comment":{"href":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/pulls/108/commits"},"statuses":{"href":"https://api.github.com/repos/reddit/reddit-plugin-adzerk/statuses/5b0c097e9bcf239101d1f17360203f8c98659456"}}}}