{"action":"created","comment":{"url":"https://api.github.com/repos/oaeproject/Hilary/pulls/comments/41029286","id":41029286,"diff_hunk":"@@ -0,0 +1,510 @@\n+/*!\n+ * Copyright 2015 Apereo Foundation (AF) Licensed under the\n+ * Educational Community License, Version 2.0 (the \"License\"); you may\n+ * not use this file except in compliance with the License. You may\n+ * obtain a copy of the License at\n+ *\n+ *     http://opensource.org/licenses/ECL-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an \"AS IS\"\n+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n+ * or implied. See the License for the specific language governing\n+ * permissions and limitations under the License.\n+ */\n+\n+/**\n+ * An analysis script designed to determine how active user accounts are for a tenant.\n+ */\n+\n+var _ = require('underscore');\n+var bunyan = require('bunyan');\n+var csv = require('csv');\n+var fs = require('fs');\n+var optimist = require('optimist');\n+var path = require('path');\n+var util = require('util');\n+\n+var ActivityDAO = require('oae-activity/lib/internal/dao');\n+var AuthzUtil = require('oae-authz/lib/util');\n+var Cassandra = require('oae-util/lib/cassandra');\n+var log = require('oae-logger').logger('oae-script-main');\n+var OAE = require('oae-util/lib/oae');\n+var PrincipalsDAO = require('oae-principals/lib/internal/dao');\n+var Validator = require('oae-util/lib/validator').Validator;\n+\n+var argv = optimist.usage('$0 [-t cam] [--config <path/to/config.js>]')\n+    .alias('t', 'tenant')\n+    .describe('t', 'Specify the tenant alias of the tenant whose users activity to analyze. By default, analyze all tenants')\n+\n+    .alias('c', 'config')\n+    .describe('c', 'Specify an alternate config file')\n+    .default('c', 'config.js')\n+\n+    .alias('h', 'help')\n+    .describe('h', 'Show usage information')\n+    .argv;\n+\n+if (argv.help) {\n+    optimist.showHelp();\n+    return process.exit(1);\n+}\n+\n+// Get the config\n+var configPath = path.resolve(process.cwd(), argv.config);\n+var config = require(configPath).config;\n+var tenantAlias = argv.tenant;\n+\n+// Ensure that this application server does NOT start processing any preview images\n+config.previews.enabled = false;\n+\n+// Ensure that we're logging to standard out/err\n+config.log = {\n+    'streams': [\n+        {\n+            'level': 'info',\n+            'stream': process.stdout\n+        }\n+    ],\n+    'serializers': {\n+        'err': bunyan.stdSerializers.err,\n+        'req': bunyan.stdSerializers.req,\n+        'res': bunyan.stdSerializers.res\n+    }\n+};\n+\n+var start = Date.now();\n+var streamInfo = _createCsvStream();\n+var csvStream = streamInfo.csv;\n+var fileStream = streamInfo.file;\n+var filePath = streamInfo.filePath;\n+\n+// Spin up the application container. This will allow us to re-use existing APIs\n+OAE.init(config, function(err) {\n+    if (err) {\n+        log().error({'err': err}, 'Unable to spin up the application server');\n+        return process.exit(err.code);\n+    }\n+\n+    _groupUsersByEmail(tenantAlias, function(usersByEmail) {\n+        var batches = _.chain(usersByEmail)\n+            // Only keep profiles that have duplicate emails\n+            .filter(function(userHashes) { return (userHashes.length > 1); })\n+            .flatten()\n+            .tap(function(userHashes) {\n+                log().info('Preparing to write a total of %s users who have duplicate emails', userHashes.length);\n+            })\n+\n+            // Re-group things into sane concurrent batches\n+            .groupBy(function(userHash, i) { return Math.floor(i / 5); })\n+            .values()\n+            .value();\n+\n+        log().info('Begin writing %s batches of user activity to CSV', batches.length);\n+\n+        return _writeUserRows(batches);\n+    });\n+});\n+\n+/**\n+ * Build and write all user rows concurrently for the given batches of user hashes\n+ *\n+ * @param  {Object[][]}     userHashBatches     A collection of batches for which to generate user rows and write them to the csv file\n+ * @api private\n+ */\n+function _writeUserRows(userHashBatches) {\n+    if (_.isEmpty(userHashBatches)) {\n+        log().info('Processing complete');\n+        return _exit(0);\n+    }\n+\n+    log().info('There are %d batches remaining to process', userHashBatches.length);\n+\n+    // Process the next batch\n+    var userHashBatch = userHashBatches.shift();\n+    var _done = _.after(userHashBatch.length, function() {\n+        log().info('Successfully completed batch of %d users', userHashBatch.length);\n+        return _writeUserRows(userHashBatches);\n+    });\n+\n+    _.each(userHashBatch, function(userHash) {\n+        _createUserRow(userHash, function(err, row) {\n+            if (err) {\n+                log().warn({'err': err, 'user': userHash}, 'Failed to create row from a user hash');\n+                return _done();\n+            }\n+\n+            return csvStream.write(row, _done);\n+        });\n+    });\n+}\n+\n+/**\n+ * Page through all the users in the system, filtering by tenant and those that have email\n+ * addresses, and group them by their email address\n+ *\n+ * @param  {String}     tenantAlias             The tenant alias to filter by\n+ * @param  {Function}   callback                Invoked when users have been grouped by email address\n+ * @param  {Object}     usersByEmail            An object keyed by email, whose value is an array of user hashes\n+ * @api private\n+ */\n+function _groupUsersByEmail(tenantAlias, callback) {\n+    var userHashes = [];\n+\n+    PrincipalsDAO.iterateAll(['principalId', 'tenantAlias', 'displayName', 'email', 'visibility', 'lastModified', 'notificationsLastRead'], 100, _aggregateUsers, function(err) {\n+        if (err) {\n+            log().error({'err': err}, 'Failed to iterate all users');\n+            return process.exit(1);\n+        }\n+\n+        log().info('Finished indexing %s users by email', userHashes.length);\n+\n+        return callback(_.groupBy(userHashes, 'email'));\n+    });\n+\n+    /*!\n+     * Given a paged result set of principal hashes, filter them down to those with email addresses\n+     * and those that are part of the specified tenant. Then add them to the shared `userHashes`\n+     * array\n+     *\n+     * @param  {Object[]}   principalHashes     The principals to filter and aggregate\n+     * @param  {Function}   callback            Will be invoked when the principals are aggregated\n+     */\n+    function _aggregateUsers(principalHashes, callback) {","path":"etc/scripts/analysis/user-activity.js","position":173,"original_position":173,"commit_id":"e54a4f5361b303e95f3534af07872f18e9ba923b","original_commit_id":"e54a4f5361b303e95f3534af07872f18e9ba923b","user":{"login":"nicolaasmatthijs","id":109850,"avatar_url":"https://avatars.githubusercontent.com/u/109850?v=3","gravatar_id":"","url":"https://api.github.com/users/nicolaasmatthijs","html_url":"https://github.com/nicolaasmatthijs","followers_url":"https://api.github.com/users/nicolaasmatthijs/followers","following_url":"https://api.github.com/users/nicolaasmatthijs/following{/other_user}","gists_url":"https://api.github.com/users/nicolaasmatthijs/gists{/gist_id}","starred_url":"https://api.github.com/users/nicolaasmatthijs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nicolaasmatthijs/subscriptions","organizations_url":"https://api.github.com/users/nicolaasmatthijs/orgs","repos_url":"https://api.github.com/users/nicolaasmatthijs/repos","events_url":"https://api.github.com/users/nicolaasmatthijs/events{/privacy}","received_events_url":"https://api.github.com/users/nicolaasmatthijs/received_events","type":"User","site_admin":false},"body":"That makes sense. I'm fine with it too.","created_at":"2015-10-02T14:49:57Z","updated_at":"2015-10-02T14:49:57Z","html_url":"https://github.com/oaeproject/Hilary/pull/1239#discussion_r41029286","pull_request_url":"https://api.github.com/repos/oaeproject/Hilary/pulls/1239","_links":{"self":{"href":"https://api.github.com/repos/oaeproject/Hilary/pulls/comments/41029286"},"html":{"href":"https://github.com/oaeproject/Hilary/pull/1239#discussion_r41029286"},"pull_request":{"href":"https://api.github.com/repos/oaeproject/Hilary/pulls/1239"}}},"pull_request":{"url":"https://api.github.com/repos/oaeproject/Hilary/pulls/1239","id":46152849,"html_url":"https://github.com/oaeproject/Hilary/pull/1239","diff_url":"https://github.com/oaeproject/Hilary/pull/1239.diff","patch_url":"https://github.com/oaeproject/Hilary/pull/1239.patch","issue_url":"https://api.github.com/repos/oaeproject/Hilary/issues/1239","number":1239,"state":"open","locked":false,"title":"Create script to identify accounts with duplicate emails and their activity","user":{"login":"mrvisser","id":102265,"avatar_url":"https://avatars.githubusercontent.com/u/102265?v=3","gravatar_id":"","url":"https://api.github.com/users/mrvisser","html_url":"https://github.com/mrvisser","followers_url":"https://api.github.com/users/mrvisser/followers","following_url":"https://api.github.com/users/mrvisser/following{/other_user}","gists_url":"https://api.github.com/users/mrvisser/gists{/gist_id}","starred_url":"https://api.github.com/users/mrvisser/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrvisser/subscriptions","organizations_url":"https://api.github.com/users/mrvisser/orgs","repos_url":"https://api.github.com/users/mrvisser/repos","events_url":"https://api.github.com/users/mrvisser/events{/privacy}","received_events_url":"https://api.github.com/users/mrvisser/received_events","type":"User","site_admin":false},"body":null,"created_at":"2015-09-28T12:53:49Z","updated_at":"2015-10-02T14:49:57Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0397ac19571eea6c972004aca508bd1a0172ca3d","assignee":{"login":"simong","id":92180,"avatar_url":"https://avatars.githubusercontent.com/u/92180?v=3","gravatar_id":"","url":"https://api.github.com/users/simong","html_url":"https://github.com/simong","followers_url":"https://api.github.com/users/simong/followers","following_url":"https://api.github.com/users/simong/following{/other_user}","gists_url":"https://api.github.com/users/simong/gists{/gist_id}","starred_url":"https://api.github.com/users/simong/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/simong/subscriptions","organizations_url":"https://api.github.com/users/simong/orgs","repos_url":"https://api.github.com/users/simong/repos","events_url":"https://api.github.com/users/simong/events{/privacy}","received_events_url":"https://api.github.com/users/simong/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/oaeproject/Hilary/pulls/1239/commits","review_comments_url":"https://api.github.com/repos/oaeproject/Hilary/pulls/1239/comments","review_comment_url":"https://api.github.com/repos/oaeproject/Hilary/pulls/comments{/number}","comments_url":"https://api.github.com/repos/oaeproject/Hilary/issues/1239/comments","statuses_url":"https://api.github.com/repos/oaeproject/Hilary/statuses/e54a4f5361b303e95f3534af07872f18e9ba923b","head":{"label":"mrvisser:active-accounts-2","ref":"active-accounts-2","sha":"e54a4f5361b303e95f3534af07872f18e9ba923b","user":{"login":"mrvisser","id":102265,"avatar_url":"https://avatars.githubusercontent.com/u/102265?v=3","gravatar_id":"","url":"https://api.github.com/users/mrvisser","html_url":"https://github.com/mrvisser","followers_url":"https://api.github.com/users/mrvisser/followers","following_url":"https://api.github.com/users/mrvisser/following{/other_user}","gists_url":"https://api.github.com/users/mrvisser/gists{/gist_id}","starred_url":"https://api.github.com/users/mrvisser/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrvisser/subscriptions","organizations_url":"https://api.github.com/users/mrvisser/orgs","repos_url":"https://api.github.com/users/mrvisser/repos","events_url":"https://api.github.com/users/mrvisser/events{/privacy}","received_events_url":"https://api.github.com/users/mrvisser/received_events","type":"User","site_admin":false},"repo":{"id":5658997,"name":"Hilary","full_name":"mrvisser/Hilary","owner":{"login":"mrvisser","id":102265,"avatar_url":"https://avatars.githubusercontent.com/u/102265?v=3","gravatar_id":"","url":"https://api.github.com/users/mrvisser","html_url":"https://github.com/mrvisser","followers_url":"https://api.github.com/users/mrvisser/followers","following_url":"https://api.github.com/users/mrvisser/following{/other_user}","gists_url":"https://api.github.com/users/mrvisser/gists{/gist_id}","starred_url":"https://api.github.com/users/mrvisser/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrvisser/subscriptions","organizations_url":"https://api.github.com/users/mrvisser/orgs","repos_url":"https://api.github.com/users/mrvisser/repos","events_url":"https://api.github.com/users/mrvisser/events{/privacy}","received_events_url":"https://api.github.com/users/mrvisser/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/mrvisser/Hilary","description":"","fork":true,"url":"https://api.github.com/repos/mrvisser/Hilary","forks_url":"https://api.github.com/repos/mrvisser/Hilary/forks","keys_url":"https://api.github.com/repos/mrvisser/Hilary/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mrvisser/Hilary/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mrvisser/Hilary/teams","hooks_url":"https://api.github.com/repos/mrvisser/Hilary/hooks","issue_events_url":"https://api.github.com/repos/mrvisser/Hilary/issues/events{/number}","events_url":"https://api.github.com/repos/mrvisser/Hilary/events","assignees_url":"https://api.github.com/repos/mrvisser/Hilary/assignees{/user}","branches_url":"https://api.github.com/repos/mrvisser/Hilary/branches{/branch}","tags_url":"https://api.github.com/repos/mrvisser/Hilary/tags","blobs_url":"https://api.github.com/repos/mrvisser/Hilary/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mrvisser/Hilary/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mrvisser/Hilary/git/refs{/sha}","trees_url":"https://api.github.com/repos/mrvisser/Hilary/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mrvisser/Hilary/statuses/{sha}","languages_url":"https://api.github.com/repos/mrvisser/Hilary/languages","stargazers_url":"https://api.github.com/repos/mrvisser/Hilary/stargazers","contributors_url":"https://api.github.com/repos/mrvisser/Hilary/contributors","subscribers_url":"https://api.github.com/repos/mrvisser/Hilary/subscribers","subscription_url":"https://api.github.com/repos/mrvisser/Hilary/subscription","commits_url":"https://api.github.com/repos/mrvisser/Hilary/commits{/sha}","git_commits_url":"https://api.github.com/repos/mrvisser/Hilary/git/commits{/sha}","comments_url":"https://api.github.com/repos/mrvisser/Hilary/comments{/number}","issue_comment_url":"https://api.github.com/repos/mrvisser/Hilary/issues/comments{/number}","contents_url":"https://api.github.com/repos/mrvisser/Hilary/contents/{+path}","compare_url":"https://api.github.com/repos/mrvisser/Hilary/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mrvisser/Hilary/merges","archive_url":"https://api.github.com/repos/mrvisser/Hilary/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mrvisser/Hilary/downloads","issues_url":"https://api.github.com/repos/mrvisser/Hilary/issues{/number}","pulls_url":"https://api.github.com/repos/mrvisser/Hilary/pulls{/number}","milestones_url":"https://api.github.com/repos/mrvisser/Hilary/milestones{/number}","notifications_url":"https://api.github.com/repos/mrvisser/Hilary/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mrvisser/Hilary/labels{/name}","releases_url":"https://api.github.com/repos/mrvisser/Hilary/releases{/id}","created_at":"2012-09-03T13:08:54Z","updated_at":"2015-07-28T23:04:27Z","pushed_at":"2015-09-29T16:53:14Z","git_url":"git://github.com/mrvisser/Hilary.git","ssh_url":"git@github.com:mrvisser/Hilary.git","clone_url":"https://github.com/mrvisser/Hilary.git","svn_url":"https://github.com/mrvisser/Hilary","homepage":null,"size":28418,"stargazers_count":0,"watchers_count":0,"language":"JavaScript","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"oaeproject:master","ref":"master","sha":"3f928ae12e17fd3842cd70c0f41ae076ed4425ed","user":{"login":"oaeproject","id":4659912,"avatar_url":"https://avatars.githubusercontent.com/u/4659912?v=3","gravatar_id":"","url":"https://api.github.com/users/oaeproject","html_url":"https://github.com/oaeproject","followers_url":"https://api.github.com/users/oaeproject/followers","following_url":"https://api.github.com/users/oaeproject/following{/other_user}","gists_url":"https://api.github.com/users/oaeproject/gists{/gist_id}","starred_url":"https://api.github.com/users/oaeproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/oaeproject/subscriptions","organizations_url":"https://api.github.com/users/oaeproject/orgs","repos_url":"https://api.github.com/users/oaeproject/repos","events_url":"https://api.github.com/users/oaeproject/events{/privacy}","received_events_url":"https://api.github.com/users/oaeproject/received_events","type":"Organization","site_admin":false},"repo":{"id":5638565,"name":"Hilary","full_name":"oaeproject/Hilary","owner":{"login":"oaeproject","id":4659912,"avatar_url":"https://avatars.githubusercontent.com/u/4659912?v=3","gravatar_id":"","url":"https://api.github.com/users/oaeproject","html_url":"https://github.com/oaeproject","followers_url":"https://api.github.com/users/oaeproject/followers","following_url":"https://api.github.com/users/oaeproject/following{/other_user}","gists_url":"https://api.github.com/users/oaeproject/gists{/gist_id}","starred_url":"https://api.github.com/users/oaeproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/oaeproject/subscriptions","organizations_url":"https://api.github.com/users/oaeproject/orgs","repos_url":"https://api.github.com/users/oaeproject/repos","events_url":"https://api.github.com/users/oaeproject/events{/privacy}","received_events_url":"https://api.github.com/users/oaeproject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/oaeproject/Hilary","description":"Open Academic Environment (OAE) Back-End","fork":false,"url":"https://api.github.com/repos/oaeproject/Hilary","forks_url":"https://api.github.com/repos/oaeproject/Hilary/forks","keys_url":"https://api.github.com/repos/oaeproject/Hilary/keys{/key_id}","collaborators_url":"https://api.github.com/repos/oaeproject/Hilary/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/oaeproject/Hilary/teams","hooks_url":"https://api.github.com/repos/oaeproject/Hilary/hooks","issue_events_url":"https://api.github.com/repos/oaeproject/Hilary/issues/events{/number}","events_url":"https://api.github.com/repos/oaeproject/Hilary/events","assignees_url":"https://api.github.com/repos/oaeproject/Hilary/assignees{/user}","branches_url":"https://api.github.com/repos/oaeproject/Hilary/branches{/branch}","tags_url":"https://api.github.com/repos/oaeproject/Hilary/tags","blobs_url":"https://api.github.com/repos/oaeproject/Hilary/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/oaeproject/Hilary/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/oaeproject/Hilary/git/refs{/sha}","trees_url":"https://api.github.com/repos/oaeproject/Hilary/git/trees{/sha}","statuses_url":"https://api.github.com/repos/oaeproject/Hilary/statuses/{sha}","languages_url":"https://api.github.com/repos/oaeproject/Hilary/languages","stargazers_url":"https://api.github.com/repos/oaeproject/Hilary/stargazers","contributors_url":"https://api.github.com/repos/oaeproject/Hilary/contributors","subscribers_url":"https://api.github.com/repos/oaeproject/Hilary/subscribers","subscription_url":"https://api.github.com/repos/oaeproject/Hilary/subscription","commits_url":"https://api.github.com/repos/oaeproject/Hilary/commits{/sha}","git_commits_url":"https://api.github.com/repos/oaeproject/Hilary/git/commits{/sha}","comments_url":"https://api.github.com/repos/oaeproject/Hilary/comments{/number}","issue_comment_url":"https://api.github.com/repos/oaeproject/Hilary/issues/comments{/number}","contents_url":"https://api.github.com/repos/oaeproject/Hilary/contents/{+path}","compare_url":"https://api.github.com/repos/oaeproject/Hilary/compare/{base}...{head}","merges_url":"https://api.github.com/repos/oaeproject/Hilary/merges","archive_url":"https://api.github.com/repos/oaeproject/Hilary/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/oaeproject/Hilary/downloads","issues_url":"https://api.github.com/repos/oaeproject/Hilary/issues{/number}","pulls_url":"https://api.github.com/repos/oaeproject/Hilary/pulls{/number}","milestones_url":"https://api.github.com/repos/oaeproject/Hilary/milestones{/number}","notifications_url":"https://api.github.com/repos/oaeproject/Hilary/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/oaeproject/Hilary/labels{/name}","releases_url":"https://api.github.com/repos/oaeproject/Hilary/releases{/id}","created_at":"2012-09-01T10:05:13Z","updated_at":"2015-09-09T11:50:05Z","pushed_at":"2015-09-29T17:22:32Z","git_url":"git://github.com/oaeproject/Hilary.git","ssh_url":"git@github.com:oaeproject/Hilary.git","clone_url":"https://github.com/oaeproject/Hilary.git","svn_url":"https://github.com/oaeproject/Hilary","homepage":"http://www.oaeproject.org","size":31390,"stargazers_count":35,"watchers_count":35,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":65,"mirror_url":null,"open_issues_count":36,"forks":65,"open_issues":36,"watchers":35,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/oaeproject/Hilary/pulls/1239"},"html":{"href":"https://github.com/oaeproject/Hilary/pull/1239"},"issue":{"href":"https://api.github.com/repos/oaeproject/Hilary/issues/1239"},"comments":{"href":"https://api.github.com/repos/oaeproject/Hilary/issues/1239/comments"},"review_comments":{"href":"https://api.github.com/repos/oaeproject/Hilary/pulls/1239/comments"},"review_comment":{"href":"https://api.github.com/repos/oaeproject/Hilary/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/oaeproject/Hilary/pulls/1239/commits"},"statuses":{"href":"https://api.github.com/repos/oaeproject/Hilary/statuses/e54a4f5361b303e95f3534af07872f18e9ba923b"}}}}