{"action":"created","comment":{"url":"https://api.github.com/repos/nltk/nltk/pulls/comments/22856218","id":22856218,"diff_hunk":"@@ -153,70 +162,113 @@ def right_children(self, node_index):\n         Returns the number of right children under the node specified\n         by the given address.\n         \"\"\"\n-        children = self.nodelist[node_index]['deps']\n-        index = self.nodelist[node_index]['address']\n+        children = self.nodes[node_index]['deps']\n+        index = self.nodes[node_index]['address']\n         return sum(1 for c in children if c > index)\n \n     def add_node(self, node):\n         if not self.contains_address(node['address']):\n-            self.nodelist.append(node)\n-\n-    def _parse(self, input, zero_based):\n-        lines = [DependencyGraph._normalize(line) for line in input.split('\\n') if line.strip()]\n-        temp = []\n-        for index, line in enumerate(lines):\n-#           print line\n-            try:\n-                cells = line.split('\\t')\n-                nrCells = len(cells)\n-                if nrCells == 3:\n-                    word, tag, head = cells\n-                    lemma, ctag, feats, rel = word, tag, '', ''\n-                elif nrCells == 4:\n-                    word, tag, head, rel = cells\n-                    lemma, ctag, feats = word, tag, ''\n-                elif nrCells == 10:\n-                    _, word, lemma, ctag, tag, feats, head, rel, _, _ = cells\n-                else:\n-                    raise ValueError('Number of tab-delimited fields (%d) not supported by CoNLL(10) or Malt-Tab(4) format' % (nrCells))\n+            self.nodes[node['address']].update(node)\n \n-                head = int(head)\n-                if zero_based:\n-                    head += 1\n-                self.nodelist.append({'address': index+1, 'word': word, 'lemma': lemma, 'ctag': ctag, 'tag': tag, 'feats': feats, 'head': head, 'rel': rel,\n-                                      'deps': [d for (d,h) in temp if h == index+1]})\n+    def _parse(self, input_, cell_extractor=None, zero_based=False, cell_separator=None):\n+        \"\"\"Parse a sentence.\n \n-                try:\n-                    self.nodelist[head]['deps'].append(index+1)\n-                except IndexError:\n-                    temp.append((index+1, head))\n+        :param extractor: a function that given a tuple of cells returns a\n+        7-tuple, where the values are ``word, lemma, ctag, tag, feats, head,\n+        rel``.\n+\n+        :param str cell_separator: the cell separator. If not provided, cells\n+        are split by whitespace.\n+\n+        \"\"\"\n+\n+        def extract_3_cells(cells):\n+            word, tag, head = cells\n+            return word, word, tag, tag, '', head, ''\n+\n+        def extract_4_cells(cells):\n+            word, tag, head, rel = cells\n+            return word, word, tag, tag, '', head, rel\n+\n+        def extract_10_cells(cells):\n+            _, word, lemma, ctag, tag, feats, head, rel, _, _ = cells\n+            return word, lemma, ctag, tag, feats, head, rel\n \n-            except ValueError:\n-                break\n+        extractors = {\n+            3: extract_3_cells,\n+            4: extract_4_cells,\n+            10: extract_10_cells,\n+        }\n \n-        root_address = self.nodelist[0]['deps'][0]\n-        self.root = self.nodelist[root_address]\n+        if isinstance(input_, string_types):\n+            input_ = (line for line in input_.split('\\n'))\n+\n+        lines = (l.rstrip() for l in input_)\n+        lines = (l for l in lines if l)\n+\n+        for index, line in enumerate(lines, start=1):\n+            cells = line.split(cell_separator)\n+            nrCells = len(cells)","path":"nltk/parse/dependencygraph.py","position":281,"original_position":281,"commit_id":"e0f0630155eec7e0df5b907ee523ad7ed8a04f5b","original_commit_id":"e0f0630155eec7e0df5b907ee523ad7ed8a04f5b","user":{"login":"kmike","id":107893,"avatar_url":"https://avatars.githubusercontent.com/u/107893?v=3","gravatar_id":"","url":"https://api.github.com/users/kmike","html_url":"https://github.com/kmike","followers_url":"https://api.github.com/users/kmike/followers","following_url":"https://api.github.com/users/kmike/following{/other_user}","gists_url":"https://api.github.com/users/kmike/gists{/gist_id}","starred_url":"https://api.github.com/users/kmike/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kmike/subscriptions","organizations_url":"https://api.github.com/users/kmike/orgs","repos_url":"https://api.github.com/users/kmike/repos","events_url":"https://api.github.com/users/kmike/events{/privacy}","received_events_url":"https://api.github.com/users/kmike/received_events","type":"User","site_admin":false},"body":"maybe rename it to conform to PEP8?","created_at":"2015-01-13T11:25:54Z","updated_at":"2015-01-13T11:25:54Z","html_url":"https://github.com/nltk/nltk/pull/822#discussion_r22856218","pull_request_url":"https://api.github.com/repos/nltk/nltk/pulls/822","_links":{"self":{"href":"https://api.github.com/repos/nltk/nltk/pulls/comments/22856218"},"html":{"href":"https://github.com/nltk/nltk/pull/822#discussion_r22856218"},"pull_request":{"href":"https://api.github.com/repos/nltk/nltk/pulls/822"}}},"pull_request":{"url":"https://api.github.com/repos/nltk/nltk/pulls/822","id":26628518,"html_url":"https://github.com/nltk/nltk/pull/822","diff_url":"https://github.com/nltk/nltk/pull/822.diff","patch_url":"https://github.com/nltk/nltk/pull/822.patch","issue_url":"https://api.github.com/repos/nltk/nltk/issues/822","number":822,"state":"closed","locked":false,"title":"Refactor DependencyGraph","user":{"login":"dimazest","id":885751,"avatar_url":"https://avatars.githubusercontent.com/u/885751?v=3","gravatar_id":"","url":"https://api.github.com/users/dimazest","html_url":"https://github.com/dimazest","followers_url":"https://api.github.com/users/dimazest/followers","following_url":"https://api.github.com/users/dimazest/following{/other_user}","gists_url":"https://api.github.com/users/dimazest/gists{/gist_id}","starred_url":"https://api.github.com/users/dimazest/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimazest/subscriptions","organizations_url":"https://api.github.com/users/dimazest/orgs","repos_url":"https://api.github.com/users/dimazest/repos","events_url":"https://api.github.com/users/dimazest/events{/privacy}","received_events_url":"https://api.github.com/users/dimazest/received_events","type":"User","site_admin":false},"body":"Note that this is still work in progress.\r\n\r\n- [x] DependencyGraph stores nodes in a list, where position of nodes is it's address (or index). In a situation when a node depends on an unseen node this is problematic (so variable `temp` was created to deal with this problem).\r\n- [x] `input_` for `_parse` can be either a string (the old behavior) or an iterable of lines (so it's possible to use DependencyGraph to parse a large corpus, for example ukwac).\r\n- [x] Tests don't cover all the functionality, I'll try to cover it as much as possible.\r\n- [x] It's possible to specify cell separator.\r\n- [ ] `DependencyGraph.nodes` should be ordered by address.\r\n- [ ] `DependencyGraph.nodes[...]['deps']` should be an ordered dict as well, where keys are relations. This will make it easy to retrieve all the dependents of a node. Btw, are relations unique?\r\n\r\nSome other points\r\n\r\n- I'm not sure whether `remove_by_address`, `get_by_address` and `contains_address` are needed. They duplicate dictionary API.\r\n- `right_children()` should return the list of children, computing the length of it is trivial.\r\n- Maybe, it makes sense make `triples()` to return node, not just a word, its head and a ctag.\r\n\r\nIn general, the PR is not ready yet, I might change it a bit: restructure the code, write more tests and documentation.","created_at":"2014-12-28T22:38:57Z","updated_at":"2015-01-13T11:25:54Z","closed_at":"2015-01-12T05:53:27Z","merged_at":"2015-01-12T05:53:27Z","merge_commit_sha":"19b351744ce3e4862ba3d64d3603fb0b9e331586","assignee":{"login":"stevenbird","id":55406,"avatar_url":"https://avatars.githubusercontent.com/u/55406?v=3","gravatar_id":"","url":"https://api.github.com/users/stevenbird","html_url":"https://github.com/stevenbird","followers_url":"https://api.github.com/users/stevenbird/followers","following_url":"https://api.github.com/users/stevenbird/following{/other_user}","gists_url":"https://api.github.com/users/stevenbird/gists{/gist_id}","starred_url":"https://api.github.com/users/stevenbird/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stevenbird/subscriptions","organizations_url":"https://api.github.com/users/stevenbird/orgs","repos_url":"https://api.github.com/users/stevenbird/repos","events_url":"https://api.github.com/users/stevenbird/events{/privacy}","received_events_url":"https://api.github.com/users/stevenbird/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/nltk/nltk/pulls/822/commits","review_comments_url":"https://api.github.com/repos/nltk/nltk/pulls/822/comments","review_comment_url":"https://api.github.com/repos/nltk/nltk/pulls/comments/{number}","comments_url":"https://api.github.com/repos/nltk/nltk/issues/822/comments","statuses_url":"https://api.github.com/repos/nltk/nltk/statuses/e0f0630155eec7e0df5b907ee523ad7ed8a04f5b","head":{"label":"dimazest:dependencygraph","ref":"dependencygraph","sha":"e0f0630155eec7e0df5b907ee523ad7ed8a04f5b","user":{"login":"dimazest","id":885751,"avatar_url":"https://avatars.githubusercontent.com/u/885751?v=3","gravatar_id":"","url":"https://api.github.com/users/dimazest","html_url":"https://github.com/dimazest","followers_url":"https://api.github.com/users/dimazest/followers","following_url":"https://api.github.com/users/dimazest/following{/other_user}","gists_url":"https://api.github.com/users/dimazest/gists{/gist_id}","starred_url":"https://api.github.com/users/dimazest/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimazest/subscriptions","organizations_url":"https://api.github.com/users/dimazest/orgs","repos_url":"https://api.github.com/users/dimazest/repos","events_url":"https://api.github.com/users/dimazest/events{/privacy}","received_events_url":"https://api.github.com/users/dimazest/received_events","type":"User","site_admin":false},"repo":{"id":16585836,"name":"nltk","full_name":"dimazest/nltk","owner":{"login":"dimazest","id":885751,"avatar_url":"https://avatars.githubusercontent.com/u/885751?v=3","gravatar_id":"","url":"https://api.github.com/users/dimazest","html_url":"https://github.com/dimazest","followers_url":"https://api.github.com/users/dimazest/followers","following_url":"https://api.github.com/users/dimazest/following{/other_user}","gists_url":"https://api.github.com/users/dimazest/gists{/gist_id}","starred_url":"https://api.github.com/users/dimazest/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimazest/subscriptions","organizations_url":"https://api.github.com/users/dimazest/orgs","repos_url":"https://api.github.com/users/dimazest/repos","events_url":"https://api.github.com/users/dimazest/events{/privacy}","received_events_url":"https://api.github.com/users/dimazest/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/dimazest/nltk","description":"NLTK Source","fork":true,"url":"https://api.github.com/repos/dimazest/nltk","forks_url":"https://api.github.com/repos/dimazest/nltk/forks","keys_url":"https://api.github.com/repos/dimazest/nltk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dimazest/nltk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dimazest/nltk/teams","hooks_url":"https://api.github.com/repos/dimazest/nltk/hooks","issue_events_url":"https://api.github.com/repos/dimazest/nltk/issues/events{/number}","events_url":"https://api.github.com/repos/dimazest/nltk/events","assignees_url":"https://api.github.com/repos/dimazest/nltk/assignees{/user}","branches_url":"https://api.github.com/repos/dimazest/nltk/branches{/branch}","tags_url":"https://api.github.com/repos/dimazest/nltk/tags","blobs_url":"https://api.github.com/repos/dimazest/nltk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dimazest/nltk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dimazest/nltk/git/refs{/sha}","trees_url":"https://api.github.com/repos/dimazest/nltk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dimazest/nltk/statuses/{sha}","languages_url":"https://api.github.com/repos/dimazest/nltk/languages","stargazers_url":"https://api.github.com/repos/dimazest/nltk/stargazers","contributors_url":"https://api.github.com/repos/dimazest/nltk/contributors","subscribers_url":"https://api.github.com/repos/dimazest/nltk/subscribers","subscription_url":"https://api.github.com/repos/dimazest/nltk/subscription","commits_url":"https://api.github.com/repos/dimazest/nltk/commits{/sha}","git_commits_url":"https://api.github.com/repos/dimazest/nltk/git/commits{/sha}","comments_url":"https://api.github.com/repos/dimazest/nltk/comments{/number}","issue_comment_url":"https://api.github.com/repos/dimazest/nltk/issues/comments/{number}","contents_url":"https://api.github.com/repos/dimazest/nltk/contents/{+path}","compare_url":"https://api.github.com/repos/dimazest/nltk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dimazest/nltk/merges","archive_url":"https://api.github.com/repos/dimazest/nltk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dimazest/nltk/downloads","issues_url":"https://api.github.com/repos/dimazest/nltk/issues{/number}","pulls_url":"https://api.github.com/repos/dimazest/nltk/pulls{/number}","milestones_url":"https://api.github.com/repos/dimazest/nltk/milestones{/number}","notifications_url":"https://api.github.com/repos/dimazest/nltk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dimazest/nltk/labels{/name}","releases_url":"https://api.github.com/repos/dimazest/nltk/releases{/id}","created_at":"2014-02-06T16:41:48Z","updated_at":"2014-12-20T23:20:25Z","pushed_at":"2015-01-10T00:01:30Z","git_url":"git://github.com/dimazest/nltk.git","ssh_url":"git@github.com:dimazest/nltk.git","clone_url":"https://github.com/dimazest/nltk.git","svn_url":"https://github.com/dimazest/nltk","homepage":"http://nltk.github.com/","size":69155,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"develop"}},"base":{"label":"nltk:develop","ref":"develop","sha":"e5064cfd61b6b238b552eca705f35296a776a39c","user":{"login":"nltk","id":124114,"avatar_url":"https://avatars.githubusercontent.com/u/124114?v=3","gravatar_id":"","url":"https://api.github.com/users/nltk","html_url":"https://github.com/nltk","followers_url":"https://api.github.com/users/nltk/followers","following_url":"https://api.github.com/users/nltk/following{/other_user}","gists_url":"https://api.github.com/users/nltk/gists{/gist_id}","starred_url":"https://api.github.com/users/nltk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nltk/subscriptions","organizations_url":"https://api.github.com/users/nltk/orgs","repos_url":"https://api.github.com/users/nltk/repos","events_url":"https://api.github.com/users/nltk/events{/privacy}","received_events_url":"https://api.github.com/users/nltk/received_events","type":"Organization","site_admin":false},"repo":{"id":299862,"name":"nltk","full_name":"nltk/nltk","owner":{"login":"nltk","id":124114,"avatar_url":"https://avatars.githubusercontent.com/u/124114?v=3","gravatar_id":"","url":"https://api.github.com/users/nltk","html_url":"https://github.com/nltk","followers_url":"https://api.github.com/users/nltk/followers","following_url":"https://api.github.com/users/nltk/following{/other_user}","gists_url":"https://api.github.com/users/nltk/gists{/gist_id}","starred_url":"https://api.github.com/users/nltk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nltk/subscriptions","organizations_url":"https://api.github.com/users/nltk/orgs","repos_url":"https://api.github.com/users/nltk/repos","events_url":"https://api.github.com/users/nltk/events{/privacy}","received_events_url":"https://api.github.com/users/nltk/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/nltk/nltk","description":"NLTK Source","fork":false,"url":"https://api.github.com/repos/nltk/nltk","forks_url":"https://api.github.com/repos/nltk/nltk/forks","keys_url":"https://api.github.com/repos/nltk/nltk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nltk/nltk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nltk/nltk/teams","hooks_url":"https://api.github.com/repos/nltk/nltk/hooks","issue_events_url":"https://api.github.com/repos/nltk/nltk/issues/events{/number}","events_url":"https://api.github.com/repos/nltk/nltk/events","assignees_url":"https://api.github.com/repos/nltk/nltk/assignees{/user}","branches_url":"https://api.github.com/repos/nltk/nltk/branches{/branch}","tags_url":"https://api.github.com/repos/nltk/nltk/tags","blobs_url":"https://api.github.com/repos/nltk/nltk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nltk/nltk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nltk/nltk/git/refs{/sha}","trees_url":"https://api.github.com/repos/nltk/nltk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nltk/nltk/statuses/{sha}","languages_url":"https://api.github.com/repos/nltk/nltk/languages","stargazers_url":"https://api.github.com/repos/nltk/nltk/stargazers","contributors_url":"https://api.github.com/repos/nltk/nltk/contributors","subscribers_url":"https://api.github.com/repos/nltk/nltk/subscribers","subscription_url":"https://api.github.com/repos/nltk/nltk/subscription","commits_url":"https://api.github.com/repos/nltk/nltk/commits{/sha}","git_commits_url":"https://api.github.com/repos/nltk/nltk/git/commits{/sha}","comments_url":"https://api.github.com/repos/nltk/nltk/comments{/number}","issue_comment_url":"https://api.github.com/repos/nltk/nltk/issues/comments/{number}","contents_url":"https://api.github.com/repos/nltk/nltk/contents/{+path}","compare_url":"https://api.github.com/repos/nltk/nltk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nltk/nltk/merges","archive_url":"https://api.github.com/repos/nltk/nltk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nltk/nltk/downloads","issues_url":"https://api.github.com/repos/nltk/nltk/issues{/number}","pulls_url":"https://api.github.com/repos/nltk/nltk/pulls{/number}","milestones_url":"https://api.github.com/repos/nltk/nltk/milestones{/number}","notifications_url":"https://api.github.com/repos/nltk/nltk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nltk/nltk/labels{/name}","releases_url":"https://api.github.com/repos/nltk/nltk/releases{/id}","created_at":"2009-09-07T10:53:58Z","updated_at":"2015-01-13T08:22:03Z","pushed_at":"2015-01-12T22:48:21Z","git_url":"git://github.com/nltk/nltk.git","ssh_url":"git@github.com:nltk/nltk.git","clone_url":"https://github.com/nltk/nltk.git","svn_url":"https://github.com/nltk/nltk","homepage":"http://nltk.github.com/","size":327276,"stargazers_count":1946,"watchers_count":1946,"language":"Python","has_issues":true,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":571,"mirror_url":null,"open_issues_count":215,"forks":571,"open_issues":215,"watchers":1946,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/nltk/nltk/pulls/822"},"html":{"href":"https://github.com/nltk/nltk/pull/822"},"issue":{"href":"https://api.github.com/repos/nltk/nltk/issues/822"},"comments":{"href":"https://api.github.com/repos/nltk/nltk/issues/822/comments"},"review_comments":{"href":"https://api.github.com/repos/nltk/nltk/pulls/822/comments"},"review_comment":{"href":"https://api.github.com/repos/nltk/nltk/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/nltk/nltk/pulls/822/commits"},"statuses":{"href":"https://api.github.com/repos/nltk/nltk/statuses/e0f0630155eec7e0df5b907ee523ad7ed8a04f5b"}}}}