{"action":"created","comment":{"url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/comments/36790718","id":36790718,"diff_hunk":"@@ -0,0 +1,782 @@\n+#!/usr/bin/env python\n+'''****************************************************************************\n+FILE: mapreduce_logfile.py\n+\n+PURPOSE: Uses the map / reduce architecture to parse information from a logfile\n+    and then produce data and reports.\n+\n+PROJECT: Land Satellites Data System Science Research and Development (LSRD)\n+    at the USGS EROS\n+\n+LICENSE TYPE: NASA Open Source Agreement Version 1.3\n+\n+AUTHOR: ngenetzky@usgs.gov\n+\n+Notes:\n+    Expected Apache log format (single line replace (\\n with space):\n+    '$remote_addr - $remote_user [$time_local] \"$request\" $status $bytes_sent\n+    \"$http_referer\" \"$http_user_agent\"\n+\n+    The script assumes the url portion of $request does not have spaces, and\n+    will report BAD_PARSE if one is found. There are numerous other causes\n+    for a field to be labeled BAD_PARSE.\n+****************************************************************************'''\n+import datetime\n+import collections\n+import inspect\n+import urllib\n+'''\n+        Debug Functions\n+'''\n+log_badparse_in_reports = False\n+lines_that_failed_to_parse = []\n+\n+\n+def log_parses_that_failed(outfile):\n+    '''Will write lines that have failed to parse to a logfile\n+\n+    Precondition:\n+        Functions have been called that attempted to parse lines in logfile.\n+    Postcondition:\n+        Logfile will contain entries that are separated by '\\n'.\n+        Each entry will describe what couldn't be parsed and the line that was\n+             unable to be parsed.\n+    '''\n+    if(log_badparse_in_reports):\n+        global lines_that_failed_to_parse\n+        with open(outfile, 'w+') as fp:\n+            failed_parses = set(lines_that_failed_to_parse)\n+            fp.write('# lines that failed to parse: {0}'\n+                     .format(len(failed_parses)))\n+            for line in failed_parses:\n+                fp.write(str(line))\n+\n+\n+def fail_to_parse(value, line):\n+    '''Will add entry into list of lines that couldn't be parsed.\n+\n+    Precondition:\n+        value contains a descriptive name of what value was attempting to be\n+             extracted.\n+        line contains the line that was attempted to be parsed.\n+    Postcondition\n+        returns 'BAD_PARSE'\n+    '''\n+    global lines_that_failed_to_parse\n+    lines_that_failed_to_parse.append('Failed to parse for {0} in <\\n{1}>'\n+                                      .format(value, line))\n+    return 'BAD_PARSE'\n+\n+\n+def get_log_name():\n+    return 'bad_parse_log/{0}.txt'.format(inspect.stack()[1][3])\n+\n+\n+'''\n+        Helper Functions\n+'''\n+\n+\n+def substring_between(s, start, finish):\n+    '''Find string between two substrings'''\n+    end_of_start = s.index(start) + len(start)\n+    start_of_finish = s.index(finish, end_of_start)\n+    return s[end_of_start:start_of_finish]\n+\n+'''\n+        Extract Data from line in logfile\n+'''\n+\n+\n+def get_rtcode(line):\n+    '''Obtain a return_code from a line of text\n+\n+    Precondition: line is a ' ' separated list of data.\n+                Return code is the first int in the items from 6 to 11.\n+    Postcondition: return return_code\n+    '''\n+    data = line.split()\n+    if(data[8].isdigit()):\n+        return data[8]\n+    else:\n+        return fail_to_parse('rtcode', line)\n+\n+\n+def get_bytes(line):\n+    '''Obtain a return_code from a line of text\n+\n+    Precondition: line is a ' ' separated list of data.\n+                Bytes downloaded is the second int in the items from 6 to 12.\n+    Postcondition: return bytes_downloaded\n+    '''\n+    data = line.split()\n+    if(data[9].isdigit()):\n+        return data[9]\n+    else:\n+        return fail_to_parse('bytes', line)\n+\n+\n+def get_datetime(line):\n+    time_local = substring_between(line, '[', '] \"')\n+    try:\n+        return datetime.datetime.strptime(time_local,\n+                                          '%d/%b/%Y:%H:%M:%S -0500')\n+    except ValueError:\n+        return fail_to_parse('datetime', line)\n+\n+\n+def get_date(line):\n+    try:\n+        return get_datetime(line).date()\n+    except ValueError:\n+        return fail_to_parse('date', line)\n+\n+\n+def get_user_email(line):\n+    request = substring_between(line, '] \"', '\" ')\n+    request = urllib.unquote(request)\n+    try:\n+        return substring_between(request, 'orders/', '-')\n+    except ValueError:\n+        return fail_to_parse('user_email', line)\n+\n+\n+def get_scene_id(line):\n+    try:\n+        response_after_orderid = substring_between(line, 'orders/', '\" ')\n+        return substring_between(response_after_orderid, '/', '.tar.gz')\n+    except ValueError:\n+        try:\n+            response_after_orderid = substring_between(line, 'orders/', '\" ')\n+            return substring_between(response_after_orderid, '/', '.cksum')\n+        except ValueError:\n+            return fail_to_parse('sceneid', line)\n+\n+\n+def get_order_id(line):\n+    request = substring_between(line, '] \"', '\" ')\n+    try:\n+        return substring_between(request, 'orders/', '/')\n+    except ValueError:\n+        return fail_to_parse('orderid', line)\n+\n+'''\n+        Filter helper functions\n+'''\n+\n+\n+def n_filters(filters, iterable):","path":"reporting/mapreduce_logfile.py","position":168,"original_position":168,"commit_id":"72790961f9591e99532492aee3e719071692c75d","original_commit_id":"72790961f9591e99532492aee3e719071692c75d","user":{"login":"dhill-usgs","id":12154564,"avatar_url":"https://avatars.githubusercontent.com/u/12154564?v=3","gravatar_id":"","url":"https://api.github.com/users/dhill-usgs","html_url":"https://github.com/dhill-usgs","followers_url":"https://api.github.com/users/dhill-usgs/followers","following_url":"https://api.github.com/users/dhill-usgs/following{/other_user}","gists_url":"https://api.github.com/users/dhill-usgs/gists{/gist_id}","starred_url":"https://api.github.com/users/dhill-usgs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dhill-usgs/subscriptions","organizations_url":"https://api.github.com/users/dhill-usgs/orgs","repos_url":"https://api.github.com/users/dhill-usgs/repos","events_url":"https://api.github.com/users/dhill-usgs/events{/privacy}","received_events_url":"https://api.github.com/users/dhill-usgs/received_events","type":"User","site_admin":false},"body":"comments please.  no idea what you're trying to accomplish","created_at":"2015-08-11T19:55:47Z","updated_at":"2015-08-11T19:55:47Z","html_url":"https://github.com/USGS-EROS/espa-maintenance/pull/1#discussion_r36790718","pull_request_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/1","_links":{"self":{"href":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/comments/36790718"},"html":{"href":"https://github.com/USGS-EROS/espa-maintenance/pull/1#discussion_r36790718"},"pull_request":{"href":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/1","id":42149975,"html_url":"https://github.com/USGS-EROS/espa-maintenance/pull/1","diff_url":"https://github.com/USGS-EROS/espa-maintenance/pull/1.diff","patch_url":"https://github.com/USGS-EROS/espa-maintenance/pull/1.patch","issue_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/issues/1","number":1,"state":"open","locked":false,"title":"Created report scripts that use logic in mapreduce_logfile.py. No date support in argparse yet.","user":{"login":"NGenetzky","id":12733783,"avatar_url":"https://avatars.githubusercontent.com/u/12733783?v=3","gravatar_id":"","url":"https://api.github.com/users/NGenetzky","html_url":"https://github.com/NGenetzky","followers_url":"https://api.github.com/users/NGenetzky/followers","following_url":"https://api.github.com/users/NGenetzky/following{/other_user}","gists_url":"https://api.github.com/users/NGenetzky/gists{/gist_id}","starred_url":"https://api.github.com/users/NGenetzky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NGenetzky/subscriptions","organizations_url":"https://api.github.com/users/NGenetzky/orgs","repos_url":"https://api.github.com/users/NGenetzky/repos","events_url":"https://api.github.com/users/NGenetzky/events{/privacy}","received_events_url":"https://api.github.com/users/NGenetzky/received_events","type":"User","site_admin":false},"body":"Created scripts that will analyze Apache log files to determine useful information. Have implemented date range filtering in the logic but need a user interface to use it. A rough one is in bytes_downloaded_report.py but it needs to be replaced.","created_at":"2015-08-11T16:36:50Z","updated_at":"2015-08-11T19:55:47Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e324984929e97bdf43b2d9f9abeaddb0cef0d994","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/1/commits","review_comments_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/1/comments","review_comment_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/comments{/number}","comments_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/issues/1/comments","statuses_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/statuses/72790961f9591e99532492aee3e719071692c75d","head":{"label":"NGenetzky:mapreduced_stats","ref":"mapreduced_stats","sha":"72790961f9591e99532492aee3e719071692c75d","user":{"login":"NGenetzky","id":12733783,"avatar_url":"https://avatars.githubusercontent.com/u/12733783?v=3","gravatar_id":"","url":"https://api.github.com/users/NGenetzky","html_url":"https://github.com/NGenetzky","followers_url":"https://api.github.com/users/NGenetzky/followers","following_url":"https://api.github.com/users/NGenetzky/following{/other_user}","gists_url":"https://api.github.com/users/NGenetzky/gists{/gist_id}","starred_url":"https://api.github.com/users/NGenetzky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NGenetzky/subscriptions","organizations_url":"https://api.github.com/users/NGenetzky/orgs","repos_url":"https://api.github.com/users/NGenetzky/repos","events_url":"https://api.github.com/users/NGenetzky/events{/privacy}","received_events_url":"https://api.github.com/users/NGenetzky/received_events","type":"User","site_admin":false},"repo":{"id":40551231,"name":"espa-maintenance","full_name":"NGenetzky/espa-maintenance","owner":{"login":"NGenetzky","id":12733783,"avatar_url":"https://avatars.githubusercontent.com/u/12733783?v=3","gravatar_id":"","url":"https://api.github.com/users/NGenetzky","html_url":"https://github.com/NGenetzky","followers_url":"https://api.github.com/users/NGenetzky/followers","following_url":"https://api.github.com/users/NGenetzky/following{/other_user}","gists_url":"https://api.github.com/users/NGenetzky/gists{/gist_id}","starred_url":"https://api.github.com/users/NGenetzky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NGenetzky/subscriptions","organizations_url":"https://api.github.com/users/NGenetzky/orgs","repos_url":"https://api.github.com/users/NGenetzky/repos","events_url":"https://api.github.com/users/NGenetzky/events{/privacy}","received_events_url":"https://api.github.com/users/NGenetzky/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/NGenetzky/espa-maintenance","description":"","fork":true,"url":"https://api.github.com/repos/NGenetzky/espa-maintenance","forks_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/forks","keys_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/keys{/key_id}","collaborators_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/teams","hooks_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/hooks","issue_events_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/issues/events{/number}","events_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/events","assignees_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/assignees{/user}","branches_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/branches{/branch}","tags_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/tags","blobs_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/git/refs{/sha}","trees_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/git/trees{/sha}","statuses_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/statuses/{sha}","languages_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/languages","stargazers_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/stargazers","contributors_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/contributors","subscribers_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/subscribers","subscription_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/subscription","commits_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/commits{/sha}","git_commits_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/git/commits{/sha}","comments_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/comments{/number}","issue_comment_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/issues/comments{/number}","contents_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/contents/{+path}","compare_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/compare/{base}...{head}","merges_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/merges","archive_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/downloads","issues_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/issues{/number}","pulls_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/pulls{/number}","milestones_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/milestones{/number}","notifications_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/labels{/name}","releases_url":"https://api.github.com/repos/NGenetzky/espa-maintenance/releases{/id}","created_at":"2015-08-11T15:56:24Z","updated_at":"2015-08-11T15:56:25Z","pushed_at":"2015-08-11T16:26:46Z","git_url":"git://github.com/NGenetzky/espa-maintenance.git","ssh_url":"git@github.com:NGenetzky/espa-maintenance.git","clone_url":"https://github.com/NGenetzky/espa-maintenance.git","svn_url":"https://github.com/NGenetzky/espa-maintenance","homepage":null,"size":14564,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"USGS-EROS:mapreduced_stats","ref":"mapreduced_stats","sha":"a37c64986de31fae5f1cea3827ff638c7a8331f7","user":{"login":"USGS-EROS","id":12549192,"avatar_url":"https://avatars.githubusercontent.com/u/12549192?v=3","gravatar_id":"","url":"https://api.github.com/users/USGS-EROS","html_url":"https://github.com/USGS-EROS","followers_url":"https://api.github.com/users/USGS-EROS/followers","following_url":"https://api.github.com/users/USGS-EROS/following{/other_user}","gists_url":"https://api.github.com/users/USGS-EROS/gists{/gist_id}","starred_url":"https://api.github.com/users/USGS-EROS/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/USGS-EROS/subscriptions","organizations_url":"https://api.github.com/users/USGS-EROS/orgs","repos_url":"https://api.github.com/users/USGS-EROS/repos","events_url":"https://api.github.com/users/USGS-EROS/events{/privacy}","received_events_url":"https://api.github.com/users/USGS-EROS/received_events","type":"Organization","site_admin":false},"repo":{"id":39914505,"name":"espa-maintenance","full_name":"USGS-EROS/espa-maintenance","owner":{"login":"USGS-EROS","id":12549192,"avatar_url":"https://avatars.githubusercontent.com/u/12549192?v=3","gravatar_id":"","url":"https://api.github.com/users/USGS-EROS","html_url":"https://github.com/USGS-EROS","followers_url":"https://api.github.com/users/USGS-EROS/followers","following_url":"https://api.github.com/users/USGS-EROS/following{/other_user}","gists_url":"https://api.github.com/users/USGS-EROS/gists{/gist_id}","starred_url":"https://api.github.com/users/USGS-EROS/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/USGS-EROS/subscriptions","organizations_url":"https://api.github.com/users/USGS-EROS/orgs","repos_url":"https://api.github.com/users/USGS-EROS/repos","events_url":"https://api.github.com/users/USGS-EROS/events{/privacy}","received_events_url":"https://api.github.com/users/USGS-EROS/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/USGS-EROS/espa-maintenance","description":"","fork":false,"url":"https://api.github.com/repos/USGS-EROS/espa-maintenance","forks_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/forks","keys_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/keys{/key_id}","collaborators_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/teams","hooks_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/hooks","issue_events_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/issues/events{/number}","events_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/events","assignees_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/assignees{/user}","branches_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/branches{/branch}","tags_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/tags","blobs_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/git/refs{/sha}","trees_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/git/trees{/sha}","statuses_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/statuses/{sha}","languages_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/languages","stargazers_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/stargazers","contributors_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/contributors","subscribers_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/subscribers","subscription_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/subscription","commits_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/commits{/sha}","git_commits_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/git/commits{/sha}","comments_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/comments{/number}","issue_comment_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/issues/comments{/number}","contents_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/contents/{+path}","compare_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/compare/{base}...{head}","merges_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/merges","archive_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/downloads","issues_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/issues{/number}","pulls_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls{/number}","milestones_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/milestones{/number}","notifications_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/labels{/name}","releases_url":"https://api.github.com/repos/USGS-EROS/espa-maintenance/releases{/id}","created_at":"2015-07-29T20:24:42Z","updated_at":"2015-08-10T18:33:05Z","pushed_at":"2015-08-11T16:36:51Z","git_url":"git://github.com/USGS-EROS/espa-maintenance.git","ssh_url":"git@github.com:USGS-EROS/espa-maintenance.git","clone_url":"https://github.com/USGS-EROS/espa-maintenance.git","svn_url":"https://github.com/USGS-EROS/espa-maintenance","homepage":null,"size":14564,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":1,"forks":1,"open_issues":1,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/1"},"html":{"href":"https://github.com/USGS-EROS/espa-maintenance/pull/1"},"issue":{"href":"https://api.github.com/repos/USGS-EROS/espa-maintenance/issues/1"},"comments":{"href":"https://api.github.com/repos/USGS-EROS/espa-maintenance/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/USGS-EROS/espa-maintenance/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/USGS-EROS/espa-maintenance/statuses/72790961f9591e99532492aee3e719071692c75d"}}}}