{"action":"created","comment":{"url":"https://api.github.com/repos/anyTV/cli-applications/pulls/comments/33129029","id":33129029,"diff_hunk":"@@ -0,0 +1,285 @@\n+'use strict';\n+\n+var mysql = require('anytv-node-mysql'),\n+    cudl = require('cuddle'),\n+    moment = require('moment'),\n+    ProgressBar = require('progress'),\n+    app = {};\n+\n+module.exports = function (cb) {\n+\n+    var app = this,\n+        limit = 20,\n+        offset = 0,\n+        query = '',\n+        success = 0,\n+        failed = 0,\n+        p_count = 0,\n+        channels = {},\n+        needs_retry = [],\n+        network = {},\n+\n+        start = function () {\n+            app.log.info('Fetching ...');\n+\n+            query = \"SELECT distinct(channel_id)\" +\n+                \" FROM jobs \";\n+\n+            if (app.argv.limit && app.argv.limit > 0) {\n+                limit = app.argv.limit;\n+            }\n+\n+            query += 'LIMIT ' + limit;\n+\n+            if (app.argv.offset && app.argv.offset > 0) {\n+                offset = app.argv.offset;\n+                query += ' OFFSET ' + offset;\n+            }\n+\n+            mysql.open(app.config.get('database').heartbeat)\n+                .query(query, authenticate)\n+        },\n+\n+        authenticate = function (err, rows) {\n+            if (err) {\n+                app.log.error(err);\n+                p_count = 1;\n+\n+                return done();\n+            }\n+\n+            if (!rows.length) {\n+                app.log.info('No channels to update');\n+\n+                process.exit();\n+            }\n+\n+            channels = rows;\n+\n+            app.log.info('Authenticating ...');\n+            cudl.post\n+                .to('accounts.google.com', 443, '/o/oauth2/token')\n+                .secured()\n+                .send({\n+                    client_id : app.config.get('google_api')[0].client_id,\n+                    client_secret : app.config.get('google_api')[0].client_secret,\n+                    refresh_token : app.config.get('google_api')[0].refresh_token,\n+                    grant_type : 'refresh_token'\n+                })\n+                .then(get_dates);\n+        },\n+\n+        get_dates = function (err, result) {\n+            var query = '';\n+\n+            if (err) {\n+                app.log.error('Failed to authenticate');\n+                app.log.error(err);\n+                p_count = 1;\n+\n+                return done();\n+            }\n+\n+            network.access_token = result.access_token;\n+            p_count = channels.length;\n+\n+            app.log.info('Channels to update: ' + p_count);\n+\n+            query = \"SELECT channel_id, dates, network \" +\n+                \"FROM jobs \";\n+\n+            mysql.open(app.config.get('database').heartbeat_2);\n+\n+            channels.forEach(function (channel) {\n+                channel.date_list = [];\n+\n+                mysql.query(query +\n+                    \"WHERE channel_id = '\" + channel.channel_id +\n+                    \"' ORDER BY dates\",\n+                    function (err, rows) {\n+                        if (err) {\n+                            app.log.error(err);\n+\n+                            return done();\n+                        }\n+\n+                        channel.start_date = rows[0].dates;\n+                        channel.end_date = rows[rows.length - 1].dates;\n+                        channel.network = rows[0].network;\n+\n+                        rows.forEach(function (row) {\n+                            channel.date_list.push(row.dates);\n+                        });\n+\n+                        return get_reports(channel);\n+                    });\n+            });\n+        },\n+\n+\n+        get_reports = function (channel) {\n+            app.log.info('Retrieving reports for ' + channel.channel_id + ' network: ' + channel.network);\n+\n+            cudl.get\n+                .to('www.googleapis.com', 443, '/youtube/analytics/v1/reports')\n+                .set_max_retry(10)\n+                .secured()\n+                .add_header('Authorization', 'Bearer ' + network.access_token)\n+                .send({\n+                    'ids' : 'contentOwner==' + channel.network,\n+                    'start-date' : moment(channel.start_date).format('YYYY-MM-DD'),\n+                    'end-date' : moment(channel.end_date).format('YYYY-MM-DD'),\n+                    'metrics' : 'views,subscribersGained,subscribersLost,comments',\n+                    'dimensions' : 'day',\n+                    'fields' : 'rows',\n+                    'filters' : 'channel==' + channel.channel_id,\n+                    'sort' : 'day'\n+                })\n+                .args(channel)\n+                .then(compute_rows);\n+        },\n+\n+        compute_rows = function (err, result, request, args) {\n+            var channel = args[0],\n+                days = [],\n+                days_length = 0;\n+\n+            if (err) {\n+\n+                if (err.status_code >= 500) {\n+                    return request.retry();\n+                }\n+\n+                if (err.status_code == undefined) {\n+                    needs_retry.push(channel.channel_id)\n+                    console.log(err);\n+                }\n+\n+                channel.bar = new ProgressBar(channel.channel_id + ' - ' + err.status_code + ' :  [:bar]', {\n+                    complete: '\\u001b[41m \\u001b[0m',\n+                    incomplete: ' ',\n+                    width:80,\n+                    total: 1\n+                });\n+\n+                failed++;\n+\n+                channel.bar.tick();\n+\n+                return done();\n+            }\n+\n+            if (result && result.rows && result.rows[0]) {\n+                channel.records = result.rows;\n+                days_length = channel.records.length;\n+                success++;\n+\n+                if (!channel.records.length) {\n+                    return done();\n+                }\n+\n+                //app.log.info(channel.channel_id + ' : ' + channel.record_length);\n+\n+                while (days_length --) {\n+                    days.push(channel.records[days_length][0]);\n+                }\n+\n+                channel.date_list.forEach(function (d) {\n+                    if (days.indexOf(d) === -1) {\n+                        channel.records.push([moment(d).format('YYYY-MM-DD'), 0, 0, 0, 0]);\n+                    }\n+                });\n+\n+                channel.record_length = channel.records.length;\n+\n+                channel.bar = new ProgressBar(' ' + channel.channel_id +' [:bar] :percent :current/:total', {\n+                    complete: '\\u001b[42m \\u001b[0m',\n+                    incomplete: '\\u001b[41m \\u001b[0m',\n+                    width:40,\n+                    total: channel.record_length\n+                });\n+\n+                channel.records.forEach(function (row) {\n+\n+                    mysql.query('REPLACE INTO channel_growth(channel_id, date, views, ' +\n+                            'subscribers_gained, subscribers_lost, comments, insert_date) ' +\n+                            'VALUES (?, ?, ?, ?, ?, ?, ?);',\n+                            [\n+                                channel.channel_id,\n+                                row[0],\n+                                row[1],\n+                                row[2],\n+                                row[3],\n+                                row[4],\n+                                moment().format('YYYY-MM-DD')\n+                            ],\n+                            function (err) {\n+                                channel.record_length --;\n+\n+                                if (err) {\n+                                    app.log.error(channel.channel_id + ' : ' + row[0] + ' : ' +err);\n+                                }\n+\n+                                channel.bar.tick();\n+\n+                                if (!channel.record_length) {\n+                                    return done();\n+                                }\n+                            }\n+                        );\n+                });\n+            }\n+            else {\n+                failed++;\n+\n+                return done();\n+            }\n+        },\n+\n+        done = function () {\n+            p_count --;\n+\n+            if (!p_count) {\n+                //mysql.end();\n+                app.log.info('Script finished');\n+                app.log.info('Failed: ' + failed.toString());\n+                app.log.info('Total no. of processed channels: ' + success + '/' + channels.length);\n+\n+                app.log.info('Cleaning jobs');\n+\n+                clean_job();\n+            }\n+\n+            return;\n+        },\n+\n+        clean_job = function () {\n+            var ids = [],\n+                length = 0;\n+\n+            length = channels.length;\n+\n+            while(length --) {\n+                ids.push(channels[length].channel_id);\n+            }\n+\n+            needs_retry.forEach(function (c_id) {\n+                ids.splice(ids.indexOf(c_id), 1);\n+            });\n+\n+            console.log(ids.length);","path":"commands/channel_growth.js","position":270,"original_position":270,"commit_id":"7774fadacbcd9a5c0fe66a582bd10788fa469925","original_commit_id":"7774fadacbcd9a5c0fe66a582bd10788fa469925","user":{"login":"adin234","id":3828145,"avatar_url":"https://avatars.githubusercontent.com/u/3828145?v=3","gravatar_id":"","url":"https://api.github.com/users/adin234","html_url":"https://github.com/adin234","followers_url":"https://api.github.com/users/adin234/followers","following_url":"https://api.github.com/users/adin234/following{/other_user}","gists_url":"https://api.github.com/users/adin234/gists{/gist_id}","starred_url":"https://api.github.com/users/adin234/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/adin234/subscriptions","organizations_url":"https://api.github.com/users/adin234/orgs","repos_url":"https://api.github.com/users/adin234/repos","events_url":"https://api.github.com/users/adin234/events{/privacy}","received_events_url":"https://api.github.com/users/adin234/received_events","type":"User","site_admin":false},"body":"be consistent with `app.log` and `console.log`","created_at":"2015-06-24T08:56:58Z","updated_at":"2015-06-24T08:56:58Z","html_url":"https://github.com/anyTV/cli-applications/pull/7#discussion_r33129029","pull_request_url":"https://api.github.com/repos/anyTV/cli-applications/pulls/7","_links":{"self":{"href":"https://api.github.com/repos/anyTV/cli-applications/pulls/comments/33129029"},"html":{"href":"https://github.com/anyTV/cli-applications/pull/7#discussion_r33129029"},"pull_request":{"href":"https://api.github.com/repos/anyTV/cli-applications/pulls/7"}}},"pull_request":{"url":"https://api.github.com/repos/anyTV/cli-applications/pulls/7","id":38418167,"html_url":"https://github.com/anyTV/cli-applications/pull/7","diff_url":"https://github.com/anyTV/cli-applications/pull/7.diff","patch_url":"https://github.com/anyTV/cli-applications/pull/7.patch","issue_url":"https://api.github.com/repos/anyTV/cli-applications/issues/7","number":7,"state":"open","locked":false,"title":"Feature/channel growth per day","user":{"login":"eshleebien","id":4904268,"avatar_url":"https://avatars.githubusercontent.com/u/4904268?v=3","gravatar_id":"","url":"https://api.github.com/users/eshleebien","html_url":"https://github.com/eshleebien","followers_url":"https://api.github.com/users/eshleebien/followers","following_url":"https://api.github.com/users/eshleebien/following{/other_user}","gists_url":"https://api.github.com/users/eshleebien/gists{/gist_id}","starred_url":"https://api.github.com/users/eshleebien/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/eshleebien/subscriptions","organizations_url":"https://api.github.com/users/eshleebien/orgs","repos_url":"https://api.github.com/users/eshleebien/repos","events_url":"https://api.github.com/users/eshleebien/events{/privacy}","received_events_url":"https://api.github.com/users/eshleebien/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-06-24T08:25:47Z","updated_at":"2015-06-24T08:56:58Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e3130c03463acdf5e690129468b4c1cad6aa384e","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/anyTV/cli-applications/pulls/7/commits","review_comments_url":"https://api.github.com/repos/anyTV/cli-applications/pulls/7/comments","review_comment_url":"https://api.github.com/repos/anyTV/cli-applications/pulls/comments{/number}","comments_url":"https://api.github.com/repos/anyTV/cli-applications/issues/7/comments","statuses_url":"https://api.github.com/repos/anyTV/cli-applications/statuses/7774fadacbcd9a5c0fe66a582bd10788fa469925","head":{"label":"anyTV:feature/growth_report","ref":"feature/growth_report","sha":"7774fadacbcd9a5c0fe66a582bd10788fa469925","user":{"login":"anyTV","id":3164673,"avatar_url":"https://avatars.githubusercontent.com/u/3164673?v=3","gravatar_id":"","url":"https://api.github.com/users/anyTV","html_url":"https://github.com/anyTV","followers_url":"https://api.github.com/users/anyTV/followers","following_url":"https://api.github.com/users/anyTV/following{/other_user}","gists_url":"https://api.github.com/users/anyTV/gists{/gist_id}","starred_url":"https://api.github.com/users/anyTV/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anyTV/subscriptions","organizations_url":"https://api.github.com/users/anyTV/orgs","repos_url":"https://api.github.com/users/anyTV/repos","events_url":"https://api.github.com/users/anyTV/events{/privacy}","received_events_url":"https://api.github.com/users/anyTV/received_events","type":"Organization","site_admin":false},"repo":{"id":33480849,"name":"cli-applications","full_name":"anyTV/cli-applications","owner":{"login":"anyTV","id":3164673,"avatar_url":"https://avatars.githubusercontent.com/u/3164673?v=3","gravatar_id":"","url":"https://api.github.com/users/anyTV","html_url":"https://github.com/anyTV","followers_url":"https://api.github.com/users/anyTV/followers","following_url":"https://api.github.com/users/anyTV/following{/other_user}","gists_url":"https://api.github.com/users/anyTV/gists{/gist_id}","starred_url":"https://api.github.com/users/anyTV/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anyTV/subscriptions","organizations_url":"https://api.github.com/users/anyTV/orgs","repos_url":"https://api.github.com/users/anyTV/repos","events_url":"https://api.github.com/users/anyTV/events{/privacy}","received_events_url":"https://api.github.com/users/anyTV/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/anyTV/cli-applications","description":"Sync channels table etc","fork":false,"url":"https://api.github.com/repos/anyTV/cli-applications","forks_url":"https://api.github.com/repos/anyTV/cli-applications/forks","keys_url":"https://api.github.com/repos/anyTV/cli-applications/keys{/key_id}","collaborators_url":"https://api.github.com/repos/anyTV/cli-applications/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/anyTV/cli-applications/teams","hooks_url":"https://api.github.com/repos/anyTV/cli-applications/hooks","issue_events_url":"https://api.github.com/repos/anyTV/cli-applications/issues/events{/number}","events_url":"https://api.github.com/repos/anyTV/cli-applications/events","assignees_url":"https://api.github.com/repos/anyTV/cli-applications/assignees{/user}","branches_url":"https://api.github.com/repos/anyTV/cli-applications/branches{/branch}","tags_url":"https://api.github.com/repos/anyTV/cli-applications/tags","blobs_url":"https://api.github.com/repos/anyTV/cli-applications/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/anyTV/cli-applications/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/anyTV/cli-applications/git/refs{/sha}","trees_url":"https://api.github.com/repos/anyTV/cli-applications/git/trees{/sha}","statuses_url":"https://api.github.com/repos/anyTV/cli-applications/statuses/{sha}","languages_url":"https://api.github.com/repos/anyTV/cli-applications/languages","stargazers_url":"https://api.github.com/repos/anyTV/cli-applications/stargazers","contributors_url":"https://api.github.com/repos/anyTV/cli-applications/contributors","subscribers_url":"https://api.github.com/repos/anyTV/cli-applications/subscribers","subscription_url":"https://api.github.com/repos/anyTV/cli-applications/subscription","commits_url":"https://api.github.com/repos/anyTV/cli-applications/commits{/sha}","git_commits_url":"https://api.github.com/repos/anyTV/cli-applications/git/commits{/sha}","comments_url":"https://api.github.com/repos/anyTV/cli-applications/comments{/number}","issue_comment_url":"https://api.github.com/repos/anyTV/cli-applications/issues/comments{/number}","contents_url":"https://api.github.com/repos/anyTV/cli-applications/contents/{+path}","compare_url":"https://api.github.com/repos/anyTV/cli-applications/compare/{base}...{head}","merges_url":"https://api.github.com/repos/anyTV/cli-applications/merges","archive_url":"https://api.github.com/repos/anyTV/cli-applications/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/anyTV/cli-applications/downloads","issues_url":"https://api.github.com/repos/anyTV/cli-applications/issues{/number}","pulls_url":"https://api.github.com/repos/anyTV/cli-applications/pulls{/number}","milestones_url":"https://api.github.com/repos/anyTV/cli-applications/milestones{/number}","notifications_url":"https://api.github.com/repos/anyTV/cli-applications/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/anyTV/cli-applications/labels{/name}","releases_url":"https://api.github.com/repos/anyTV/cli-applications/releases{/id}","created_at":"2015-04-06T12:28:07Z","updated_at":"2015-05-14T03:02:48Z","pushed_at":"2015-06-24T08:25:47Z","git_url":"git://github.com/anyTV/cli-applications.git","ssh_url":"git@github.com:anyTV/cli-applications.git","clone_url":"https://github.com/anyTV/cli-applications.git","svn_url":"https://github.com/anyTV/cli-applications","homepage":null,"size":237,"stargazers_count":0,"watchers_count":0,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"anyTV:master","ref":"master","sha":"e8c7fa1cc29de105e3146cfc088525f867ac240f","user":{"login":"anyTV","id":3164673,"avatar_url":"https://avatars.githubusercontent.com/u/3164673?v=3","gravatar_id":"","url":"https://api.github.com/users/anyTV","html_url":"https://github.com/anyTV","followers_url":"https://api.github.com/users/anyTV/followers","following_url":"https://api.github.com/users/anyTV/following{/other_user}","gists_url":"https://api.github.com/users/anyTV/gists{/gist_id}","starred_url":"https://api.github.com/users/anyTV/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anyTV/subscriptions","organizations_url":"https://api.github.com/users/anyTV/orgs","repos_url":"https://api.github.com/users/anyTV/repos","events_url":"https://api.github.com/users/anyTV/events{/privacy}","received_events_url":"https://api.github.com/users/anyTV/received_events","type":"Organization","site_admin":false},"repo":{"id":33480849,"name":"cli-applications","full_name":"anyTV/cli-applications","owner":{"login":"anyTV","id":3164673,"avatar_url":"https://avatars.githubusercontent.com/u/3164673?v=3","gravatar_id":"","url":"https://api.github.com/users/anyTV","html_url":"https://github.com/anyTV","followers_url":"https://api.github.com/users/anyTV/followers","following_url":"https://api.github.com/users/anyTV/following{/other_user}","gists_url":"https://api.github.com/users/anyTV/gists{/gist_id}","starred_url":"https://api.github.com/users/anyTV/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anyTV/subscriptions","organizations_url":"https://api.github.com/users/anyTV/orgs","repos_url":"https://api.github.com/users/anyTV/repos","events_url":"https://api.github.com/users/anyTV/events{/privacy}","received_events_url":"https://api.github.com/users/anyTV/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/anyTV/cli-applications","description":"Sync channels table etc","fork":false,"url":"https://api.github.com/repos/anyTV/cli-applications","forks_url":"https://api.github.com/repos/anyTV/cli-applications/forks","keys_url":"https://api.github.com/repos/anyTV/cli-applications/keys{/key_id}","collaborators_url":"https://api.github.com/repos/anyTV/cli-applications/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/anyTV/cli-applications/teams","hooks_url":"https://api.github.com/repos/anyTV/cli-applications/hooks","issue_events_url":"https://api.github.com/repos/anyTV/cli-applications/issues/events{/number}","events_url":"https://api.github.com/repos/anyTV/cli-applications/events","assignees_url":"https://api.github.com/repos/anyTV/cli-applications/assignees{/user}","branches_url":"https://api.github.com/repos/anyTV/cli-applications/branches{/branch}","tags_url":"https://api.github.com/repos/anyTV/cli-applications/tags","blobs_url":"https://api.github.com/repos/anyTV/cli-applications/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/anyTV/cli-applications/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/anyTV/cli-applications/git/refs{/sha}","trees_url":"https://api.github.com/repos/anyTV/cli-applications/git/trees{/sha}","statuses_url":"https://api.github.com/repos/anyTV/cli-applications/statuses/{sha}","languages_url":"https://api.github.com/repos/anyTV/cli-applications/languages","stargazers_url":"https://api.github.com/repos/anyTV/cli-applications/stargazers","contributors_url":"https://api.github.com/repos/anyTV/cli-applications/contributors","subscribers_url":"https://api.github.com/repos/anyTV/cli-applications/subscribers","subscription_url":"https://api.github.com/repos/anyTV/cli-applications/subscription","commits_url":"https://api.github.com/repos/anyTV/cli-applications/commits{/sha}","git_commits_url":"https://api.github.com/repos/anyTV/cli-applications/git/commits{/sha}","comments_url":"https://api.github.com/repos/anyTV/cli-applications/comments{/number}","issue_comment_url":"https://api.github.com/repos/anyTV/cli-applications/issues/comments{/number}","contents_url":"https://api.github.com/repos/anyTV/cli-applications/contents/{+path}","compare_url":"https://api.github.com/repos/anyTV/cli-applications/compare/{base}...{head}","merges_url":"https://api.github.com/repos/anyTV/cli-applications/merges","archive_url":"https://api.github.com/repos/anyTV/cli-applications/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/anyTV/cli-applications/downloads","issues_url":"https://api.github.com/repos/anyTV/cli-applications/issues{/number}","pulls_url":"https://api.github.com/repos/anyTV/cli-applications/pulls{/number}","milestones_url":"https://api.github.com/repos/anyTV/cli-applications/milestones{/number}","notifications_url":"https://api.github.com/repos/anyTV/cli-applications/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/anyTV/cli-applications/labels{/name}","releases_url":"https://api.github.com/repos/anyTV/cli-applications/releases{/id}","created_at":"2015-04-06T12:28:07Z","updated_at":"2015-05-14T03:02:48Z","pushed_at":"2015-06-24T08:25:47Z","git_url":"git://github.com/anyTV/cli-applications.git","ssh_url":"git@github.com:anyTV/cli-applications.git","clone_url":"https://github.com/anyTV/cli-applications.git","svn_url":"https://github.com/anyTV/cli-applications","homepage":null,"size":237,"stargazers_count":0,"watchers_count":0,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/anyTV/cli-applications/pulls/7"},"html":{"href":"https://github.com/anyTV/cli-applications/pull/7"},"issue":{"href":"https://api.github.com/repos/anyTV/cli-applications/issues/7"},"comments":{"href":"https://api.github.com/repos/anyTV/cli-applications/issues/7/comments"},"review_comments":{"href":"https://api.github.com/repos/anyTV/cli-applications/pulls/7/comments"},"review_comment":{"href":"https://api.github.com/repos/anyTV/cli-applications/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/anyTV/cli-applications/pulls/7/commits"},"statuses":{"href":"https://api.github.com/repos/anyTV/cli-applications/statuses/7774fadacbcd9a5c0fe66a582bd10788fa469925"}}}}