{"action":"created","comment":{"url":"https://api.github.com/repos/nltk/nltk/pulls/comments/33488629","id":33488629,"diff_hunk":"@@ -0,0 +1,362 @@\n+# -*- coding: utf-8 -*-\n+# Natural Language Toolkit: Twitter client\n+#\n+# Copyright (C) 2001-2015 NLTK Project\n+# Author: Ewan Klein <ewan@inf.ed.ac.uk>\n+#         Lorenzo Rubio <lrnzcig@gmail.com>\n+# URL: <http://nltk.org/>\n+# For license information, see LICENSE.TXT\n+\n+\"\"\"\n+Utility functions to accompany :module:`twitterclient`.\n+\"\"\"\n+from __future__ import print_function\n+import csv\n+import json\n+import os\n+import pprint\n+import nltk.compat as compat\n+\n+from twython import Twython\n+\n+\n+def extract_fields(tweet, fields):\n+    \"\"\"\n+    Extract field values from a full tweet and return them as a list\n+\n+    :param json tweet: The tweet in JSON format\n+    :param list fields: The fields to be extracted from the tweet\n+    :rtype: list(str)\n+    \"\"\"\n+    out = []\n+    for field in fields:\n+        try:\n+            _add_field_to_out(tweet, field, out)\n+        except TypeError:\n+            raise RuntimeError('Fatal error when extracting fields. Cannot find field ', field)\n+    return out\n+\n+\n+def _add_field_to_out(json, field, out):\n+    if isinstance(field, dict):\n+        for key, value in field.items():\n+            _add_field_to_out(json[key], value, out)\n+    else:\n+        if isinstance(field, compat.string_types):\n+            out += [json[field]]\n+        else:\n+            out += [json[value] for value in field]\n+\n+def _get_entity_recursive(json, entity):\n+    if not json:\n+        return None\n+    elif isinstance(json, dict):\n+        for key, value in json.items():\n+            if key == entity:\n+                return value\n+            '''\n+            'entities' and 'extended_entities' are wrappers in twitter json\n+            structure that contain other twitter objects. See:\n+            https://dev.twitter.com/overview/api/entities-in-twitter-objects\n+            '''\n+            if key == 'entities' or key == 'extended_entities':\n+                candidate = _get_entity_recursive(value, entity)\n+                if candidate != None:\n+                    return candidate\n+        return None\n+    elif isinstance(json, list):\n+        for item in json:\n+            candidate = _get_entity_recursive(item, entity)\n+            if candidate != None:\n+                return candidate\n+        return None\n+    else:\n+        return None\n+\n+def json2csv(fp, outfile, fields, encoding='utf8', errors='replace'):\n+    \"\"\"\n+    Extract selected fields from a file of line-separated JSON tweets and\n+    write to a file in CSV format.\n+\n+    This utility function allows a file of full tweets to be easily converted\n+    to a CSV file for easier processing. For example, just tweetIDs or\n+    just the text content of the tweets can be extracted.\n+\n+    Additionally, the function allows combinations of fields of other Twitter\n+    objects (mainly the users, see below).\n+\n+    For Twitter entities (e.g. hashtags of a tweet), and for geolocation, see\n+    `json2csv_entities`\n+\n+    :param str infile: The name of the file containing full tweets\n+\n+    :param str outfile: The name of the text file where results should be\\\n+    written\n+\n+    :param list fields: The list of fields to be extracted. Useful examples\\\n+    are 'id_str' for the tweetID and 'text' for the text of the tweet. See\\\n+    <https://dev.twitter.com/overview/api/tweets> for a full list of fields.\n+    e. g.: ['id_str'], ['id', 'text', 'favorite_count', 'retweet_count']\n+    Addionally, it allows IDs from other Twitter objects, e. g.,\\\n+    ['id', 'text', {'user' : ['id', 'followers_count', 'friends_count']}]\n+\n+\n+    :param error: Behaviour for encoding errors, see\\\n+    https://docs.python.org/3/library/codecs.html#codec-base-classes\n+    \"\"\"\n+    (writer, outf) = outf_writer_compat(outfile, encoding, errors)\n+    for line in fp:\n+        tweet = json.loads(line)\n+        row = extract_fields(tweet, fields)\n+        writer.writerow(row)\n+    outf.close()\n+\n+def outf_writer_compat(outfile, encoding, errors):\n+    \"\"\"\n+    Identify appropriate CSV writer given the Python version\n+    \"\"\"\n+    if compat.PY3 == True:\n+        outf = open(outfile, 'w', encoding=encoding, errors=errors)\n+        writer = csv.writer(outf)\n+    else:\n+        outf = open(outfile, 'wb')\n+        writer = compat.UnicodeWriter(outf, encoding=encoding, errors=errors)\n+    return (writer, outf)\n+\n+\n+\n+\n+def json2csv_entities(fp, outfile, main_fields, entity_name, entity_fields,\n+                      encoding='utf8', errors='replace'):\n+    \"\"\"\n+    Extract selected fields from a file of line-separated JSON tweets and\n+    write to a file in CSV format.\n+\n+    This utility function allows a file of full tweets to be easily converted\n+    to a CSV file for easier processing of Twitter entities. For example, the\n+    hashtags or media elements of a tweet can be extracted.","path":"nltk/twitter/util.py","position":137,"original_position":137,"commit_id":"f0659dca174434242b611a11dcdd16e57a7a3f57","original_commit_id":"f0659dca174434242b611a11dcdd16e57a7a3f57","user":{"login":"lrnzcig","id":10001669,"avatar_url":"https://avatars.githubusercontent.com/u/10001669?v=3","gravatar_id":"","url":"https://api.github.com/users/lrnzcig","html_url":"https://github.com/lrnzcig","followers_url":"https://api.github.com/users/lrnzcig/followers","following_url":"https://api.github.com/users/lrnzcig/following{/other_user}","gists_url":"https://api.github.com/users/lrnzcig/gists{/gist_id}","starred_url":"https://api.github.com/users/lrnzcig/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lrnzcig/subscriptions","organizations_url":"https://api.github.com/users/lrnzcig/orgs","repos_url":"https://api.github.com/users/lrnzcig/repos","events_url":"https://api.github.com/users/lrnzcig/events{/privacy}","received_events_url":"https://api.github.com/users/lrnzcig/received_events","type":"User","site_admin":false},"body":"@ewan-klein done in #1028 ","created_at":"2015-06-29T17:25:13Z","updated_at":"2015-06-29T17:25:13Z","html_url":"https://github.com/nltk/nltk/pull/1021#discussion_r33488629","pull_request_url":"https://api.github.com/repos/nltk/nltk/pulls/1021","_links":{"self":{"href":"https://api.github.com/repos/nltk/nltk/pulls/comments/33488629"},"html":{"href":"https://github.com/nltk/nltk/pull/1021#discussion_r33488629"},"pull_request":{"href":"https://api.github.com/repos/nltk/nltk/pulls/1021"}}},"pull_request":{"url":"https://api.github.com/repos/nltk/nltk/pulls/1021","id":37669719,"html_url":"https://github.com/nltk/nltk/pull/1021","diff_url":"https://github.com/nltk/nltk/pull/1021.diff","patch_url":"https://github.com/nltk/nltk/pull/1021.patch","issue_url":"https://api.github.com/repos/nltk/nltk/issues/1021","number":1021,"state":"open","locked":false,"title":"Support for Twitter processing","user":{"login":"ewan-klein","id":1103150,"avatar_url":"https://avatars.githubusercontent.com/u/1103150?v=3","gravatar_id":"","url":"https://api.github.com/users/ewan-klein","html_url":"https://github.com/ewan-klein","followers_url":"https://api.github.com/users/ewan-klein/followers","following_url":"https://api.github.com/users/ewan-klein/following{/other_user}","gists_url":"https://api.github.com/users/ewan-klein/gists{/gist_id}","starred_url":"https://api.github.com/users/ewan-klein/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ewan-klein/subscriptions","organizations_url":"https://api.github.com/users/ewan-klein/orgs","repos_url":"https://api.github.com/users/ewan-klein/repos","events_url":"https://api.github.com/users/ewan-klein/events{/privacy}","received_events_url":"https://api.github.com/users/ewan-klein/received_events","type":"User","site_admin":false},"body":"This PR provides a basis on which Twitter processing can be built, and consists primarily of:\r\n\r\n* a client module, plus demo file etc, in the `twitter `package; the client is built on top of Twython, \r\n  which is a new external dependency;\r\n* a 'casual text' tokeniser by Chris Potts;\r\n* a corpus reader.\r\n\r\nAt present, there is no support for Tweet normalisation, POS tagging, or any other downstream linguistic analysis.\r\n\r\nIn general, it's not possible to carry out automatic regression tests against either the live Twitter stream or past Tweets. However the `nltk/test/unit` directory contains tests for extracting fields from a fixed corpus and tests for the code that reads authentication information from a user file.\r\n\r\nThe demos will only run if you have set up Twitter authentication for yourself following the instructions in `nltk/test/twitter.ipynb`, which should be run via `ipython notebook` from the command line. The notebook was created under IPython 3.x. More work still needs to be done on fleshing out the examples in this notebook.","created_at":"2015-06-15T08:48:50Z","updated_at":"2015-06-29T17:25:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"66b98437de129f9ed79a8a3f2ac7ce3141a9f577","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/nltk/nltk/pulls/1021/commits","review_comments_url":"https://api.github.com/repos/nltk/nltk/pulls/1021/comments","review_comment_url":"https://api.github.com/repos/nltk/nltk/pulls/comments{/number}","comments_url":"https://api.github.com/repos/nltk/nltk/issues/1021/comments","statuses_url":"https://api.github.com/repos/nltk/nltk/statuses/f0659dca174434242b611a11dcdd16e57a7a3f57","head":{"label":"nltk:twitter","ref":"twitter","sha":"f0659dca174434242b611a11dcdd16e57a7a3f57","user":{"login":"nltk","id":124114,"avatar_url":"https://avatars.githubusercontent.com/u/124114?v=3","gravatar_id":"","url":"https://api.github.com/users/nltk","html_url":"https://github.com/nltk","followers_url":"https://api.github.com/users/nltk/followers","following_url":"https://api.github.com/users/nltk/following{/other_user}","gists_url":"https://api.github.com/users/nltk/gists{/gist_id}","starred_url":"https://api.github.com/users/nltk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nltk/subscriptions","organizations_url":"https://api.github.com/users/nltk/orgs","repos_url":"https://api.github.com/users/nltk/repos","events_url":"https://api.github.com/users/nltk/events{/privacy}","received_events_url":"https://api.github.com/users/nltk/received_events","type":"Organization","site_admin":false},"repo":{"id":299862,"name":"nltk","full_name":"nltk/nltk","owner":{"login":"nltk","id":124114,"avatar_url":"https://avatars.githubusercontent.com/u/124114?v=3","gravatar_id":"","url":"https://api.github.com/users/nltk","html_url":"https://github.com/nltk","followers_url":"https://api.github.com/users/nltk/followers","following_url":"https://api.github.com/users/nltk/following{/other_user}","gists_url":"https://api.github.com/users/nltk/gists{/gist_id}","starred_url":"https://api.github.com/users/nltk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nltk/subscriptions","organizations_url":"https://api.github.com/users/nltk/orgs","repos_url":"https://api.github.com/users/nltk/repos","events_url":"https://api.github.com/users/nltk/events{/privacy}","received_events_url":"https://api.github.com/users/nltk/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/nltk/nltk","description":"NLTK Source","fork":false,"url":"https://api.github.com/repos/nltk/nltk","forks_url":"https://api.github.com/repos/nltk/nltk/forks","keys_url":"https://api.github.com/repos/nltk/nltk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nltk/nltk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nltk/nltk/teams","hooks_url":"https://api.github.com/repos/nltk/nltk/hooks","issue_events_url":"https://api.github.com/repos/nltk/nltk/issues/events{/number}","events_url":"https://api.github.com/repos/nltk/nltk/events","assignees_url":"https://api.github.com/repos/nltk/nltk/assignees{/user}","branches_url":"https://api.github.com/repos/nltk/nltk/branches{/branch}","tags_url":"https://api.github.com/repos/nltk/nltk/tags","blobs_url":"https://api.github.com/repos/nltk/nltk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nltk/nltk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nltk/nltk/git/refs{/sha}","trees_url":"https://api.github.com/repos/nltk/nltk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nltk/nltk/statuses/{sha}","languages_url":"https://api.github.com/repos/nltk/nltk/languages","stargazers_url":"https://api.github.com/repos/nltk/nltk/stargazers","contributors_url":"https://api.github.com/repos/nltk/nltk/contributors","subscribers_url":"https://api.github.com/repos/nltk/nltk/subscribers","subscription_url":"https://api.github.com/repos/nltk/nltk/subscription","commits_url":"https://api.github.com/repos/nltk/nltk/commits{/sha}","git_commits_url":"https://api.github.com/repos/nltk/nltk/git/commits{/sha}","comments_url":"https://api.github.com/repos/nltk/nltk/comments{/number}","issue_comment_url":"https://api.github.com/repos/nltk/nltk/issues/comments{/number}","contents_url":"https://api.github.com/repos/nltk/nltk/contents/{+path}","compare_url":"https://api.github.com/repos/nltk/nltk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nltk/nltk/merges","archive_url":"https://api.github.com/repos/nltk/nltk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nltk/nltk/downloads","issues_url":"https://api.github.com/repos/nltk/nltk/issues{/number}","pulls_url":"https://api.github.com/repos/nltk/nltk/pulls{/number}","milestones_url":"https://api.github.com/repos/nltk/nltk/milestones{/number}","notifications_url":"https://api.github.com/repos/nltk/nltk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nltk/nltk/labels{/name}","releases_url":"https://api.github.com/repos/nltk/nltk/releases{/id}","created_at":"2009-09-07T10:53:58Z","updated_at":"2015-06-29T06:36:07Z","pushed_at":"2015-06-29T17:20:50Z","git_url":"git://github.com/nltk/nltk.git","ssh_url":"git@github.com:nltk/nltk.git","clone_url":"https://github.com/nltk/nltk.git","svn_url":"https://github.com/nltk/nltk","homepage":"http://nltk.github.com/","size":378632,"stargazers_count":2355,"watchers_count":2355,"language":"Python","has_issues":true,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":698,"mirror_url":null,"open_issues_count":241,"forks":698,"open_issues":241,"watchers":2355,"default_branch":"develop"}},"base":{"label":"nltk:develop","ref":"develop","sha":"ae4049842ffc8a8bb67bc7c21a93a6c521b907d5","user":{"login":"nltk","id":124114,"avatar_url":"https://avatars.githubusercontent.com/u/124114?v=3","gravatar_id":"","url":"https://api.github.com/users/nltk","html_url":"https://github.com/nltk","followers_url":"https://api.github.com/users/nltk/followers","following_url":"https://api.github.com/users/nltk/following{/other_user}","gists_url":"https://api.github.com/users/nltk/gists{/gist_id}","starred_url":"https://api.github.com/users/nltk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nltk/subscriptions","organizations_url":"https://api.github.com/users/nltk/orgs","repos_url":"https://api.github.com/users/nltk/repos","events_url":"https://api.github.com/users/nltk/events{/privacy}","received_events_url":"https://api.github.com/users/nltk/received_events","type":"Organization","site_admin":false},"repo":{"id":299862,"name":"nltk","full_name":"nltk/nltk","owner":{"login":"nltk","id":124114,"avatar_url":"https://avatars.githubusercontent.com/u/124114?v=3","gravatar_id":"","url":"https://api.github.com/users/nltk","html_url":"https://github.com/nltk","followers_url":"https://api.github.com/users/nltk/followers","following_url":"https://api.github.com/users/nltk/following{/other_user}","gists_url":"https://api.github.com/users/nltk/gists{/gist_id}","starred_url":"https://api.github.com/users/nltk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nltk/subscriptions","organizations_url":"https://api.github.com/users/nltk/orgs","repos_url":"https://api.github.com/users/nltk/repos","events_url":"https://api.github.com/users/nltk/events{/privacy}","received_events_url":"https://api.github.com/users/nltk/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/nltk/nltk","description":"NLTK Source","fork":false,"url":"https://api.github.com/repos/nltk/nltk","forks_url":"https://api.github.com/repos/nltk/nltk/forks","keys_url":"https://api.github.com/repos/nltk/nltk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nltk/nltk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nltk/nltk/teams","hooks_url":"https://api.github.com/repos/nltk/nltk/hooks","issue_events_url":"https://api.github.com/repos/nltk/nltk/issues/events{/number}","events_url":"https://api.github.com/repos/nltk/nltk/events","assignees_url":"https://api.github.com/repos/nltk/nltk/assignees{/user}","branches_url":"https://api.github.com/repos/nltk/nltk/branches{/branch}","tags_url":"https://api.github.com/repos/nltk/nltk/tags","blobs_url":"https://api.github.com/repos/nltk/nltk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nltk/nltk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nltk/nltk/git/refs{/sha}","trees_url":"https://api.github.com/repos/nltk/nltk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nltk/nltk/statuses/{sha}","languages_url":"https://api.github.com/repos/nltk/nltk/languages","stargazers_url":"https://api.github.com/repos/nltk/nltk/stargazers","contributors_url":"https://api.github.com/repos/nltk/nltk/contributors","subscribers_url":"https://api.github.com/repos/nltk/nltk/subscribers","subscription_url":"https://api.github.com/repos/nltk/nltk/subscription","commits_url":"https://api.github.com/repos/nltk/nltk/commits{/sha}","git_commits_url":"https://api.github.com/repos/nltk/nltk/git/commits{/sha}","comments_url":"https://api.github.com/repos/nltk/nltk/comments{/number}","issue_comment_url":"https://api.github.com/repos/nltk/nltk/issues/comments{/number}","contents_url":"https://api.github.com/repos/nltk/nltk/contents/{+path}","compare_url":"https://api.github.com/repos/nltk/nltk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nltk/nltk/merges","archive_url":"https://api.github.com/repos/nltk/nltk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nltk/nltk/downloads","issues_url":"https://api.github.com/repos/nltk/nltk/issues{/number}","pulls_url":"https://api.github.com/repos/nltk/nltk/pulls{/number}","milestones_url":"https://api.github.com/repos/nltk/nltk/milestones{/number}","notifications_url":"https://api.github.com/repos/nltk/nltk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nltk/nltk/labels{/name}","releases_url":"https://api.github.com/repos/nltk/nltk/releases{/id}","created_at":"2009-09-07T10:53:58Z","updated_at":"2015-06-29T06:36:07Z","pushed_at":"2015-06-29T17:20:50Z","git_url":"git://github.com/nltk/nltk.git","ssh_url":"git@github.com:nltk/nltk.git","clone_url":"https://github.com/nltk/nltk.git","svn_url":"https://github.com/nltk/nltk","homepage":"http://nltk.github.com/","size":378632,"stargazers_count":2355,"watchers_count":2355,"language":"Python","has_issues":true,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":698,"mirror_url":null,"open_issues_count":241,"forks":698,"open_issues":241,"watchers":2355,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/nltk/nltk/pulls/1021"},"html":{"href":"https://github.com/nltk/nltk/pull/1021"},"issue":{"href":"https://api.github.com/repos/nltk/nltk/issues/1021"},"comments":{"href":"https://api.github.com/repos/nltk/nltk/issues/1021/comments"},"review_comments":{"href":"https://api.github.com/repos/nltk/nltk/pulls/1021/comments"},"review_comment":{"href":"https://api.github.com/repos/nltk/nltk/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/nltk/nltk/pulls/1021/commits"},"statuses":{"href":"https://api.github.com/repos/nltk/nltk/statuses/f0659dca174434242b611a11dcdd16e57a7a3f57"}}}}