{"action":"created","comment":{"url":"https://api.github.com/repos/nltk/nltk/pulls/comments/25760471","id":25760471,"diff_hunk":"@@ -0,0 +1,162 @@\n+# -*- coding: utf-8 -*-\n+# Natural Language Toolkit: Interface to the CRFSuite Tagger\n+#\n+# Copyright (C) 2001-2015 NLTK Project\n+# Author: Long Duong <longdt219@gmail.com>\n+# URL: <http://nltk.org/>\n+# For license information, see LICENSE.TXT\n+\n+\"\"\"\n+A module for interfacing with the CRFSuite taggers.\n+\"\"\"\n+from __future__ import absolute_import\n+from __future__ import unicode_literals\n+import unicodedata\n+import re \n+from nltk.tag.api import TaggerI\n+import pycrfsuite\n+\n+_py_crfsuite_url = 'https://pypi.python.org/pypi/python-crfsuite'\n+\n+\n+class CRFTagger(TaggerI):\n+    \"\"\"\n+    An interface to Python CRFSuite taggers. https://pypi.python.org/pypi/python-crfsuite\n+    \n+    >>> from nltk.tag.crfsuite import CRFTagger\n+    >>> ct = CRFTagger()\n+ \n+    >>> train_data = [[('University','Noun'), ('is','Verb'), ('a','Det'), ('good','Adj'), ('place','Noun')],\n+    ... [('dog','Noun'),('eat','Verb'),('meat','Noun')]]\n+    \n+    >>> ct.train(train_data,'model.crf.tagger')\n+    >>> ct.tag_sents([['dog','is','good'], ['Cat','eat','meat']])\n+    [[('dog', 'Noun'), ('is', 'Verb'), ('good', 'Adj')], [('Cat', 'Noun'), ('eat', 'Verb'), ('meat', 'Noun')]]\n+    \n+    >>> gold_sentences = [[('dog','Noun'),('is','Verb'),('good','Adj')] , [('Cat','Noun'),('eat','Verb'), ('meat','Noun')]] \n+    >>> ct.evaluate(gold_sentences) \n+    1.0\n+    \n+    Setting learned model file  \n+    >>> ct = CRFTagger() \n+    >>> ct.set_model_file('model.crf.tagger')\n+    >>> ct.evaluate(gold_sentences)\n+    1.0\n+    \n+    \"\"\"\n+    \n+    def __init__(self, file_path=''):\n+               \n+        self._model_file = ''\n+    \n+\n+    def set_model_file(self, model_file):\n+        self._model_file = model_file\n+            \n+    def _get_features(self,data):\n+        \"\"\"\n+        Extract basic features about this word including \n+             - Current Word \n+             - Is Capitalized ?\n+             - Has Punctuation ?\n+             - Has Number ?\n+             - Suffixes up to length 3 \n+        :return : a string which contains the features   \n+        \n+        \"\"\" \n+        feature_list = []  \n+        # Capitalization \n+        if data[0].isupper():\n+            feature_list.append('CAPITALIZATION')\n+        \n+        # Number \n+        pattern = re.compile('\\\\d')\n+        if re.search(pattern, data) is not None:\n+            feature_list.append('HAS_NUM') \n+        \n+        # Punctuation\n+        punc_cat = set([\"Pc\", \"Pd\", \"Ps\", \"Pe\", \"Pi\", \"Pf\", \"Po\"])\n+        if all (unicodedata.category(x) in punc_cat for x in data):\n+            feature_list.append('PUNCTUATION')\n+        \n+        # Suffix up to length 3\n+        if len(data) > 1:\n+            feature_list.append('SUF_' + data[-1:]) \n+        if len(data) > 2: \n+            feature_list.append('SUF_' + data[-2:])    \n+        if len(data) > 3: \n+            feature_list.append('SUF_' + data[-3:])\n+            \n+        feature_list.append('WORD_' + data )\n+        \n+        return feature_list\n+        \n+    def tag_sents(self, sents):\n+        '''\n+        Tag a list of sentences. NB before using this function, user should specify the mode_file either by \n+                       - Train a new model using ``train'' function \n+                       - Use the pre-trained model which is set via ``set_model_file'' function  \n+        :params sentences : list of sentences needed to tag. \n+        :type sentences : list(list(str))\n+        :return : list of tagged sentences. \n+        :rtype : list (list (tuple(str,str))) \n+        '''\n+        tagger = pycrfsuite.Tagger()\n+        tagger.open(self._model_file)","path":"nltk/tag/crfsuite.py","position":105,"original_position":105,"commit_id":"447a4a5e552e3fbea2a33a167156b1160495aefa","original_commit_id":"447a4a5e552e3fbea2a33a167156b1160495aefa","user":{"login":"stevenbird","id":55406,"avatar_url":"https://avatars.githubusercontent.com/u/55406?v=3","gravatar_id":"","url":"https://api.github.com/users/stevenbird","html_url":"https://github.com/stevenbird","followers_url":"https://api.github.com/users/stevenbird/followers","following_url":"https://api.github.com/users/stevenbird/following{/other_user}","gists_url":"https://api.github.com/users/stevenbird/gists{/gist_id}","starred_url":"https://api.github.com/users/stevenbird/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stevenbird/subscriptions","organizations_url":"https://api.github.com/users/stevenbird/orgs","repos_url":"https://api.github.com/users/stevenbird/repos","events_url":"https://api.github.com/users/stevenbird/events{/privacy}","received_events_url":"https://api.github.com/users/stevenbird/received_events","type":"User","site_admin":false},"body":"Good point, and we have plenty of examples of this already in nltk","created_at":"2015-03-04T09:10:02Z","updated_at":"2015-03-04T09:10:02Z","html_url":"https://github.com/nltk/nltk/pull/896#discussion_r25760471","pull_request_url":"https://api.github.com/repos/nltk/nltk/pulls/896","_links":{"self":{"href":"https://api.github.com/repos/nltk/nltk/pulls/comments/25760471"},"html":{"href":"https://github.com/nltk/nltk/pull/896#discussion_r25760471"},"pull_request":{"href":"https://api.github.com/repos/nltk/nltk/pulls/896"}}},"pull_request":{"url":"https://api.github.com/repos/nltk/nltk/pulls/896","id":29784839,"html_url":"https://github.com/nltk/nltk/pull/896","diff_url":"https://github.com/nltk/nltk/pull/896.diff","patch_url":"https://github.com/nltk/nltk/pull/896.patch","issue_url":"https://api.github.com/repos/nltk/nltk/issues/896","number":896,"state":"open","locked":false,"title":"Upgrade mallet #104","user":{"login":"longdt219","id":7554260,"avatar_url":"https://avatars.githubusercontent.com/u/7554260?v=3","gravatar_id":"","url":"https://api.github.com/users/longdt219","html_url":"https://github.com/longdt219","followers_url":"https://api.github.com/users/longdt219/followers","following_url":"https://api.github.com/users/longdt219/following{/other_user}","gists_url":"https://api.github.com/users/longdt219/gists{/gist_id}","starred_url":"https://api.github.com/users/longdt219/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/longdt219/subscriptions","organizations_url":"https://api.github.com/users/longdt219/orgs","repos_url":"https://api.github.com/users/longdt219/repos","events_url":"https://api.github.com/users/longdt219/events{/privacy}","received_events_url":"https://api.github.com/users/longdt219/received_events","type":"User","site_admin":false},"body":"Hi @stevenbird, \r\n\r\nHere is the code for using CRFSuite for tagging to resolve the #104. \r\nI added the script to automatically download and build the necessary file. I also modified the wiki for using CRFSuite. \r\n\r\nRunning on part of Spanish treebank (ConLL dataset) when training on 3000 sentences and testing on the ConLL Spanish test data achieved the accuracy of 93.6 %\r\n\r\n","created_at":"2015-02-23T03:48:32Z","updated_at":"2015-03-04T09:10:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"67d6ec29758cc48c75dd4a5088517e3f4f076b44","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/nltk/nltk/pulls/896/commits","review_comments_url":"https://api.github.com/repos/nltk/nltk/pulls/896/comments","review_comment_url":"https://api.github.com/repos/nltk/nltk/pulls/comments{/number}","comments_url":"https://api.github.com/repos/nltk/nltk/issues/896/comments","statuses_url":"https://api.github.com/repos/nltk/nltk/statuses/447a4a5e552e3fbea2a33a167156b1160495aefa","head":{"label":"longdt219:UpgradeMallet_104","ref":"UpgradeMallet_104","sha":"447a4a5e552e3fbea2a33a167156b1160495aefa","user":{"login":"longdt219","id":7554260,"avatar_url":"https://avatars.githubusercontent.com/u/7554260?v=3","gravatar_id":"","url":"https://api.github.com/users/longdt219","html_url":"https://github.com/longdt219","followers_url":"https://api.github.com/users/longdt219/followers","following_url":"https://api.github.com/users/longdt219/following{/other_user}","gists_url":"https://api.github.com/users/longdt219/gists{/gist_id}","starred_url":"https://api.github.com/users/longdt219/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/longdt219/subscriptions","organizations_url":"https://api.github.com/users/longdt219/orgs","repos_url":"https://api.github.com/users/longdt219/repos","events_url":"https://api.github.com/users/longdt219/events{/privacy}","received_events_url":"https://api.github.com/users/longdt219/received_events","type":"User","site_admin":false},"repo":{"id":27750502,"name":"nltk","full_name":"longdt219/nltk","owner":{"login":"longdt219","id":7554260,"avatar_url":"https://avatars.githubusercontent.com/u/7554260?v=3","gravatar_id":"","url":"https://api.github.com/users/longdt219","html_url":"https://github.com/longdt219","followers_url":"https://api.github.com/users/longdt219/followers","following_url":"https://api.github.com/users/longdt219/following{/other_user}","gists_url":"https://api.github.com/users/longdt219/gists{/gist_id}","starred_url":"https://api.github.com/users/longdt219/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/longdt219/subscriptions","organizations_url":"https://api.github.com/users/longdt219/orgs","repos_url":"https://api.github.com/users/longdt219/repos","events_url":"https://api.github.com/users/longdt219/events{/privacy}","received_events_url":"https://api.github.com/users/longdt219/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/longdt219/nltk","description":"NLTK Source","fork":true,"url":"https://api.github.com/repos/longdt219/nltk","forks_url":"https://api.github.com/repos/longdt219/nltk/forks","keys_url":"https://api.github.com/repos/longdt219/nltk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/longdt219/nltk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/longdt219/nltk/teams","hooks_url":"https://api.github.com/repos/longdt219/nltk/hooks","issue_events_url":"https://api.github.com/repos/longdt219/nltk/issues/events{/number}","events_url":"https://api.github.com/repos/longdt219/nltk/events","assignees_url":"https://api.github.com/repos/longdt219/nltk/assignees{/user}","branches_url":"https://api.github.com/repos/longdt219/nltk/branches{/branch}","tags_url":"https://api.github.com/repos/longdt219/nltk/tags","blobs_url":"https://api.github.com/repos/longdt219/nltk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/longdt219/nltk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/longdt219/nltk/git/refs{/sha}","trees_url":"https://api.github.com/repos/longdt219/nltk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/longdt219/nltk/statuses/{sha}","languages_url":"https://api.github.com/repos/longdt219/nltk/languages","stargazers_url":"https://api.github.com/repos/longdt219/nltk/stargazers","contributors_url":"https://api.github.com/repos/longdt219/nltk/contributors","subscribers_url":"https://api.github.com/repos/longdt219/nltk/subscribers","subscription_url":"https://api.github.com/repos/longdt219/nltk/subscription","commits_url":"https://api.github.com/repos/longdt219/nltk/commits{/sha}","git_commits_url":"https://api.github.com/repos/longdt219/nltk/git/commits{/sha}","comments_url":"https://api.github.com/repos/longdt219/nltk/comments{/number}","issue_comment_url":"https://api.github.com/repos/longdt219/nltk/issues/comments{/number}","contents_url":"https://api.github.com/repos/longdt219/nltk/contents/{+path}","compare_url":"https://api.github.com/repos/longdt219/nltk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/longdt219/nltk/merges","archive_url":"https://api.github.com/repos/longdt219/nltk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/longdt219/nltk/downloads","issues_url":"https://api.github.com/repos/longdt219/nltk/issues{/number}","pulls_url":"https://api.github.com/repos/longdt219/nltk/pulls{/number}","milestones_url":"https://api.github.com/repos/longdt219/nltk/milestones{/number}","notifications_url":"https://api.github.com/repos/longdt219/nltk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/longdt219/nltk/labels{/name}","releases_url":"https://api.github.com/repos/longdt219/nltk/releases{/id}","created_at":"2014-12-09T05:07:25Z","updated_at":"2015-03-03T10:45:10Z","pushed_at":"2015-03-04T05:37:23Z","git_url":"git://github.com/longdt219/nltk.git","ssh_url":"git@github.com:longdt219/nltk.git","clone_url":"https://github.com/longdt219/nltk.git","svn_url":"https://github.com/longdt219/nltk","homepage":"http://nltk.github.com/","size":69588,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"develop"}},"base":{"label":"nltk:develop","ref":"develop","sha":"bfbbcaab796989219a0db7b2207e99123ff86c6e","user":{"login":"nltk","id":124114,"avatar_url":"https://avatars.githubusercontent.com/u/124114?v=3","gravatar_id":"","url":"https://api.github.com/users/nltk","html_url":"https://github.com/nltk","followers_url":"https://api.github.com/users/nltk/followers","following_url":"https://api.github.com/users/nltk/following{/other_user}","gists_url":"https://api.github.com/users/nltk/gists{/gist_id}","starred_url":"https://api.github.com/users/nltk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nltk/subscriptions","organizations_url":"https://api.github.com/users/nltk/orgs","repos_url":"https://api.github.com/users/nltk/repos","events_url":"https://api.github.com/users/nltk/events{/privacy}","received_events_url":"https://api.github.com/users/nltk/received_events","type":"Organization","site_admin":false},"repo":{"id":299862,"name":"nltk","full_name":"nltk/nltk","owner":{"login":"nltk","id":124114,"avatar_url":"https://avatars.githubusercontent.com/u/124114?v=3","gravatar_id":"","url":"https://api.github.com/users/nltk","html_url":"https://github.com/nltk","followers_url":"https://api.github.com/users/nltk/followers","following_url":"https://api.github.com/users/nltk/following{/other_user}","gists_url":"https://api.github.com/users/nltk/gists{/gist_id}","starred_url":"https://api.github.com/users/nltk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nltk/subscriptions","organizations_url":"https://api.github.com/users/nltk/orgs","repos_url":"https://api.github.com/users/nltk/repos","events_url":"https://api.github.com/users/nltk/events{/privacy}","received_events_url":"https://api.github.com/users/nltk/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/nltk/nltk","description":"NLTK Source","fork":false,"url":"https://api.github.com/repos/nltk/nltk","forks_url":"https://api.github.com/repos/nltk/nltk/forks","keys_url":"https://api.github.com/repos/nltk/nltk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nltk/nltk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nltk/nltk/teams","hooks_url":"https://api.github.com/repos/nltk/nltk/hooks","issue_events_url":"https://api.github.com/repos/nltk/nltk/issues/events{/number}","events_url":"https://api.github.com/repos/nltk/nltk/events","assignees_url":"https://api.github.com/repos/nltk/nltk/assignees{/user}","branches_url":"https://api.github.com/repos/nltk/nltk/branches{/branch}","tags_url":"https://api.github.com/repos/nltk/nltk/tags","blobs_url":"https://api.github.com/repos/nltk/nltk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nltk/nltk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nltk/nltk/git/refs{/sha}","trees_url":"https://api.github.com/repos/nltk/nltk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nltk/nltk/statuses/{sha}","languages_url":"https://api.github.com/repos/nltk/nltk/languages","stargazers_url":"https://api.github.com/repos/nltk/nltk/stargazers","contributors_url":"https://api.github.com/repos/nltk/nltk/contributors","subscribers_url":"https://api.github.com/repos/nltk/nltk/subscribers","subscription_url":"https://api.github.com/repos/nltk/nltk/subscription","commits_url":"https://api.github.com/repos/nltk/nltk/commits{/sha}","git_commits_url":"https://api.github.com/repos/nltk/nltk/git/commits{/sha}","comments_url":"https://api.github.com/repos/nltk/nltk/comments{/number}","issue_comment_url":"https://api.github.com/repos/nltk/nltk/issues/comments{/number}","contents_url":"https://api.github.com/repos/nltk/nltk/contents/{+path}","compare_url":"https://api.github.com/repos/nltk/nltk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nltk/nltk/merges","archive_url":"https://api.github.com/repos/nltk/nltk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nltk/nltk/downloads","issues_url":"https://api.github.com/repos/nltk/nltk/issues{/number}","pulls_url":"https://api.github.com/repos/nltk/nltk/pulls{/number}","milestones_url":"https://api.github.com/repos/nltk/nltk/milestones{/number}","notifications_url":"https://api.github.com/repos/nltk/nltk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nltk/nltk/labels{/name}","releases_url":"https://api.github.com/repos/nltk/nltk/releases{/id}","created_at":"2009-09-07T10:53:58Z","updated_at":"2015-03-04T00:29:10Z","pushed_at":"2015-02-26T11:26:14Z","git_url":"git://github.com/nltk/nltk.git","ssh_url":"git@github.com:nltk/nltk.git","clone_url":"https://github.com/nltk/nltk.git","svn_url":"https://github.com/nltk/nltk","homepage":"http://nltk.github.com/","size":329015,"stargazers_count":2063,"watchers_count":2063,"language":"Python","has_issues":true,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":598,"mirror_url":null,"open_issues_count":216,"forks":598,"open_issues":216,"watchers":2063,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/nltk/nltk/pulls/896"},"html":{"href":"https://github.com/nltk/nltk/pull/896"},"issue":{"href":"https://api.github.com/repos/nltk/nltk/issues/896"},"comments":{"href":"https://api.github.com/repos/nltk/nltk/issues/896/comments"},"review_comments":{"href":"https://api.github.com/repos/nltk/nltk/pulls/896/comments"},"review_comment":{"href":"https://api.github.com/repos/nltk/nltk/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/nltk/nltk/pulls/896/commits"},"statuses":{"href":"https://api.github.com/repos/nltk/nltk/statuses/447a4a5e552e3fbea2a33a167156b1160495aefa"}}}}