{"action":"created","comment":{"url":"https://api.github.com/repos/lsst/metaserv/pulls/comments/26266875","id":26266875,"diff_hunk":"@@ -0,0 +1,271 @@\n+#!/usr/bin/env python\n+#\n+# LSST Data Management System\n+# Copyright 2008-2015 LSST Corporation.\n+#\n+# This product includes software developed by the\n+# LSST Project (http://www.lsst.org/).\n+#\n+# This program is free software: you can redistribute it and/or modify\n+# it under the terms of the GNU General Public License as published by\n+# the Free Software Foundation, either version 3 of the License, or\n+# (at your option) any later version.\n+#\n+# This program is distributed in the hope that it will be useful,\n+# but WITHOUT ANY WARRANTY; without even the implied warranty of\n+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+# GNU General Public License for more details.\n+#\n+# You should have received a copy of the LSST License Statement and\n+# the GNU General Public License along with this program.  If not,\n+# see <http://www.lsstcorp.org/LegalNotices/>.\n+\n+\n+import commands\n+import optparse\n+import os\n+import pprint\n+import re\n+import sys\n+\n+class SchemaToMeta(object):\n+    \"\"\"\n+    SchemaToMeta class parses mysql schema file that can optionally contain\n+    extra tokens in comments. Extracts information for each table:\n+    * name\n+    * engine\n+    * per column:\n+      - type\n+      - notnull\n+      - defaultValue\n+      - <descr>...</descr>\n+      - <unit>...</unit>\n+      - <ucd>...</ucd>\n+    and saves it in an array.\n+\n+    This code was originally written for schema browser\n+    (in cat/bin/schema_to_metadata.py).\n+    \"\"\"\n+\n+    def __init__(self, inputFileName):\n+        \"\"\"\n+        Constructor, prepares for execution.\n+\n+        @param inputFileName:   ASCII file containing the schem to be parsed\n+        \"\"\"\n+        if not os.path.isfile(inputFileName):\n+            sys.stderr.write(\"File '%s' does not exist\\n\" % inputFileName)\n+            sys.exit(1)\n+        self._inFName = inputFileName\n+\n+\n+    def parse(self):\n+        \"\"\"Do actual parsing. Returns the retrieved structure as a table.\"\"\"\n+        in_table = None\n+        in_col = None\n+        in_colDescr = None\n+        table = {}\n+\n+        tableStart = re.compile(r'CREATE TABLE (\\w+)*')\n+        tableEnd = re.compile(r\"\\)\")\n+        engineLine = re.compile(r'\\) (ENGINE|TYPE)=(\\w+)*;')\n+        columnLine = re.compile(r'[\\s]+(\\w+) ([\\w\\(\\)]+)')\n+        descrStart = re.compile(r'<descr>')\n+        descrEnd = re.compile(r'</descr>')\n+        unitStart = re.compile(r'<unit>')\n+        unitEnd = re.compile(r'</unit>')\n+\n+        colNum = 1\n+\n+        iF = open(self._inFName, mode='r')\n+        for line in iF:\n+            # print \"processing \", line\n+            m = tableStart.search(line)\n+            if m is not None:\n+                tableName = m.group(1)\n+                table[tableName] = {}\n+                colNum = 1\n+                in_table = table[tableName]\n+                in_col = None\n+                #print \"Found table \", in_table\n+            elif tableEnd.match(line):\n+                m = engineLine.match(line)\n+                if m is not None:\n+                    engineName = m.group(2)\n+                    in_table[\"engine\"] = engineName\n+                # print \"end of the table\"\n+                # print in_table\n+                in_table = None\n+            elif in_table is not None: # process columns for given table\n+                m = columnLine.match(line)\n+                if m is not None:\n+                    firstWord = m.group(1)\n+                    if self._isIndexDefinition(firstWord):\n+                        t = \"-\"\n+                        if firstWord == \"PRIMARY\":\n+                            t = \"PRIMARY KEY\"\n+                        elif firstWord == \"UNIQUE\":\n+                            t = \"UNIQUE\"\n+                        idxInfo = {\"type\" : t,\n+                                   \"columns\" : self._retrColumns(line)\n+                               }\n+                        if \"indexes\" not in in_table:\n+                            in_table[\"indexes\"] = []\n+                        in_table[\"indexes\"].append(idxInfo)\n+                    else:\n+                        in_col = {\"name\" : firstWord,\n+                                  \"displayOrder\" : str(colNum),\n+                                  \"type\" : self._retrType(line),\n+                                  \"notNull\" : self._retrIsNotNull(line),\n+                              }\n+                        dv = self._retrDefaultValue(line)\n+                        if dv is not None:\n+                            in_col[\"defaultValue\"] = dv\n+                        colNum += 1\n+                        if \"columns\" not in in_table:\n+                            in_table[\"columns\"] = []\n+                        in_table[\"columns\"].append(in_col)\n+                    # print \"found col: \", in_col\n+                elif self._isCommentLine(line): # handle comments\n+                    if in_col is None:    # table comment\n+\n+                        if self._containsDescrTagStart(line):\n+                            if self._containsDescrTagEnd(line):\n+                                in_table[\"description\"] = self._retrDescr(line)\n+                            else:\n+                                in_table[\"description\"] = self._retrDescrStart(line)\n+                        elif \"description\" in in_table:\n+                            if self._containsDescrTagEnd(line):\n+                                in_table[\"description\"] += self._retrDescrEnd(line)\n+                            else:\n+                                in_table[\"description\"] += self._retrDescrMid(line)\n+                    else:\n+                                          # column comment\n+                        if self._containsDescrTagStart(line):\n+                            if self._containsDescrTagEnd(line):\n+                                in_col[\"description\"] = self._retrDescr(line)\n+                            else:\n+                                in_col[\"description\"] = self._retrDescrStart(line)\n+                                in_colDescr = 1\n+                        elif in_colDescr:\n+                            if self._containsDescrTagEnd(line):\n+                                in_col[\"description\"] += self._retrDescrEnd(line)\n+                                in_colDescr = None\n+                            else:\n+                                in_col[\"description\"] += self._retrDescrMid(line)\n+\n+                                          # units\n+                        if self._isUnitLine(line):\n+                            in_col[\"unit\"] = self._retrUnit(line)\n+\n+                                          # ucds\n+                        if self._isUcdLine(line):\n+                            in_col[\"ucd\"] = self._retrUcd(line)\n+\n+        iF.close()\n+        return table\n+\n+    ###########################################################################\n+    # Helper functions\n+    ###########################################################################\n+\n+    def _isIndexDefinition(self, c):\n+        return c in [\"PRIMARY\", \"KEY\", \"INDEX\", \"UNIQUE\"]\n+\n+    def _isCommentLine(self, str):\n+        return re.match('[\\s]*--', str) is not None\n+\n+    def _isUnitLine(self, str):\n+        return re.search(r'<unit>(.+)</unit>', str) is not None\n+\n+    def _isUcdLine(self, str):\n+        return re.search(r'<ucd>(.+)</ucd>', str) is not None\n+\n+    def _retrUnit(self, str):\n+        xx = re.compile(r'<unit>(.+)</unit>')","path":"python/lsst/metaserv/schemaToMeta.py","position":185,"original_position":185,"commit_id":"1749f7ad75d44378ea0b501c976988088e899cea","original_commit_id":"1749f7ad75d44378ea0b501c976988088e899cea","user":{"login":"andy-slac","id":4266950,"avatar_url":"https://avatars.githubusercontent.com/u/4266950?v=3","gravatar_id":"","url":"https://api.github.com/users/andy-slac","html_url":"https://github.com/andy-slac","followers_url":"https://api.github.com/users/andy-slac/followers","following_url":"https://api.github.com/users/andy-slac/following{/other_user}","gists_url":"https://api.github.com/users/andy-slac/gists{/gist_id}","starred_url":"https://api.github.com/users/andy-slac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andy-slac/subscriptions","organizations_url":"https://api.github.com/users/andy-slac/orgs","repos_url":"https://api.github.com/users/andy-slac/repos","events_url":"https://api.github.com/users/andy-slac/events{/privacy}","received_events_url":"https://api.github.com/users/andy-slac/received_events","type":"User","site_admin":false},"body":"Here and in other places where you do `re.search` and `re.compile` regular expression is compiled on every call to the method, which is likely a significant overhead for this call.","created_at":"2015-03-11T23:36:16Z","updated_at":"2015-03-11T23:36:16Z","html_url":"https://github.com/lsst/metaserv/pull/2#discussion_r26266875","pull_request_url":"https://api.github.com/repos/lsst/metaserv/pulls/2","_links":{"self":{"href":"https://api.github.com/repos/lsst/metaserv/pulls/comments/26266875"},"html":{"href":"https://github.com/lsst/metaserv/pull/2#discussion_r26266875"},"pull_request":{"href":"https://api.github.com/repos/lsst/metaserv/pulls/2"}}},"pull_request":{"url":"https://api.github.com/repos/lsst/metaserv/pulls/2","id":30970065,"html_url":"https://github.com/lsst/metaserv/pull/2","diff_url":"https://github.com/lsst/metaserv/pull/2.diff","patch_url":"https://github.com/lsst/metaserv/pull/2.patch","issue_url":"https://api.github.com/repos/lsst/metaserv/issues/2","number":2,"state":"open","locked":false,"title":"tickets/DM-2139","user":{"login":"andy-slac","id":4266950,"avatar_url":"https://avatars.githubusercontent.com/u/4266950?v=3","gravatar_id":"","url":"https://api.github.com/users/andy-slac","html_url":"https://github.com/andy-slac","followers_url":"https://api.github.com/users/andy-slac/followers","following_url":"https://api.github.com/users/andy-slac/following{/other_user}","gists_url":"https://api.github.com/users/andy-slac/gists{/gist_id}","starred_url":"https://api.github.com/users/andy-slac/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andy-slac/subscriptions","organizations_url":"https://api.github.com/users/andy-slac/orgs","repos_url":"https://api.github.com/users/andy-slac/repos","events_url":"https://api.github.com/users/andy-slac/events{/privacy}","received_events_url":"https://api.github.com/users/andy-slac/received_events","type":"User","site_admin":false},"body":"Reviewing DM-2139","created_at":"2015-03-11T18:13:26Z","updated_at":"2015-03-11T23:36:16Z","closed_at":null,"merged_at":null,"merge_commit_sha":"fcca700bfaa6200c79dff8d21207c04276163adc","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/lsst/metaserv/pulls/2/commits","review_comments_url":"https://api.github.com/repos/lsst/metaserv/pulls/2/comments","review_comment_url":"https://api.github.com/repos/lsst/metaserv/pulls/comments{/number}","comments_url":"https://api.github.com/repos/lsst/metaserv/issues/2/comments","statuses_url":"https://api.github.com/repos/lsst/metaserv/statuses/1749f7ad75d44378ea0b501c976988088e899cea","head":{"label":"lsst:tickets/DM-2139","ref":"tickets/DM-2139","sha":"1749f7ad75d44378ea0b501c976988088e899cea","user":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"repo":{"id":29326134,"name":"metaserv","full_name":"lsst/metaserv","owner":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/lsst/metaserv","description":"Web Interface for LSST Metadata Services","fork":false,"url":"https://api.github.com/repos/lsst/metaserv","forks_url":"https://api.github.com/repos/lsst/metaserv/forks","keys_url":"https://api.github.com/repos/lsst/metaserv/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lsst/metaserv/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lsst/metaserv/teams","hooks_url":"https://api.github.com/repos/lsst/metaserv/hooks","issue_events_url":"https://api.github.com/repos/lsst/metaserv/issues/events{/number}","events_url":"https://api.github.com/repos/lsst/metaserv/events","assignees_url":"https://api.github.com/repos/lsst/metaserv/assignees{/user}","branches_url":"https://api.github.com/repos/lsst/metaserv/branches{/branch}","tags_url":"https://api.github.com/repos/lsst/metaserv/tags","blobs_url":"https://api.github.com/repos/lsst/metaserv/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lsst/metaserv/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lsst/metaserv/git/refs{/sha}","trees_url":"https://api.github.com/repos/lsst/metaserv/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lsst/metaserv/statuses/{sha}","languages_url":"https://api.github.com/repos/lsst/metaserv/languages","stargazers_url":"https://api.github.com/repos/lsst/metaserv/stargazers","contributors_url":"https://api.github.com/repos/lsst/metaserv/contributors","subscribers_url":"https://api.github.com/repos/lsst/metaserv/subscribers","subscription_url":"https://api.github.com/repos/lsst/metaserv/subscription","commits_url":"https://api.github.com/repos/lsst/metaserv/commits{/sha}","git_commits_url":"https://api.github.com/repos/lsst/metaserv/git/commits{/sha}","comments_url":"https://api.github.com/repos/lsst/metaserv/comments{/number}","issue_comment_url":"https://api.github.com/repos/lsst/metaserv/issues/comments{/number}","contents_url":"https://api.github.com/repos/lsst/metaserv/contents/{+path}","compare_url":"https://api.github.com/repos/lsst/metaserv/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lsst/metaserv/merges","archive_url":"https://api.github.com/repos/lsst/metaserv/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lsst/metaserv/downloads","issues_url":"https://api.github.com/repos/lsst/metaserv/issues{/number}","pulls_url":"https://api.github.com/repos/lsst/metaserv/pulls{/number}","milestones_url":"https://api.github.com/repos/lsst/metaserv/milestones{/number}","notifications_url":"https://api.github.com/repos/lsst/metaserv/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lsst/metaserv/labels{/name}","releases_url":"https://api.github.com/repos/lsst/metaserv/releases{/id}","created_at":"2015-01-16T01:04:07Z","updated_at":"2015-03-04T17:11:41Z","pushed_at":"2015-03-11T04:07:52Z","git_url":"git://github.com/lsst/metaserv.git","ssh_url":"git@github.com:lsst/metaserv.git","clone_url":"https://github.com/lsst/metaserv.git","svn_url":"https://github.com/lsst/metaserv","homepage":"http://dm.lsst.org/","size":404,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"lsst:master","ref":"master","sha":"8c95041527df2ff7fa42e61759d6340a964cd4bb","user":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"repo":{"id":29326134,"name":"metaserv","full_name":"lsst/metaserv","owner":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/lsst/metaserv","description":"Web Interface for LSST Metadata Services","fork":false,"url":"https://api.github.com/repos/lsst/metaserv","forks_url":"https://api.github.com/repos/lsst/metaserv/forks","keys_url":"https://api.github.com/repos/lsst/metaserv/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lsst/metaserv/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lsst/metaserv/teams","hooks_url":"https://api.github.com/repos/lsst/metaserv/hooks","issue_events_url":"https://api.github.com/repos/lsst/metaserv/issues/events{/number}","events_url":"https://api.github.com/repos/lsst/metaserv/events","assignees_url":"https://api.github.com/repos/lsst/metaserv/assignees{/user}","branches_url":"https://api.github.com/repos/lsst/metaserv/branches{/branch}","tags_url":"https://api.github.com/repos/lsst/metaserv/tags","blobs_url":"https://api.github.com/repos/lsst/metaserv/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lsst/metaserv/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lsst/metaserv/git/refs{/sha}","trees_url":"https://api.github.com/repos/lsst/metaserv/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lsst/metaserv/statuses/{sha}","languages_url":"https://api.github.com/repos/lsst/metaserv/languages","stargazers_url":"https://api.github.com/repos/lsst/metaserv/stargazers","contributors_url":"https://api.github.com/repos/lsst/metaserv/contributors","subscribers_url":"https://api.github.com/repos/lsst/metaserv/subscribers","subscription_url":"https://api.github.com/repos/lsst/metaserv/subscription","commits_url":"https://api.github.com/repos/lsst/metaserv/commits{/sha}","git_commits_url":"https://api.github.com/repos/lsst/metaserv/git/commits{/sha}","comments_url":"https://api.github.com/repos/lsst/metaserv/comments{/number}","issue_comment_url":"https://api.github.com/repos/lsst/metaserv/issues/comments{/number}","contents_url":"https://api.github.com/repos/lsst/metaserv/contents/{+path}","compare_url":"https://api.github.com/repos/lsst/metaserv/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lsst/metaserv/merges","archive_url":"https://api.github.com/repos/lsst/metaserv/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lsst/metaserv/downloads","issues_url":"https://api.github.com/repos/lsst/metaserv/issues{/number}","pulls_url":"https://api.github.com/repos/lsst/metaserv/pulls{/number}","milestones_url":"https://api.github.com/repos/lsst/metaserv/milestones{/number}","notifications_url":"https://api.github.com/repos/lsst/metaserv/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lsst/metaserv/labels{/name}","releases_url":"https://api.github.com/repos/lsst/metaserv/releases{/id}","created_at":"2015-01-16T01:04:07Z","updated_at":"2015-03-04T17:11:41Z","pushed_at":"2015-03-11T04:07:52Z","git_url":"git://github.com/lsst/metaserv.git","ssh_url":"git@github.com:lsst/metaserv.git","clone_url":"https://github.com/lsst/metaserv.git","svn_url":"https://github.com/lsst/metaserv","homepage":"http://dm.lsst.org/","size":404,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/lsst/metaserv/pulls/2"},"html":{"href":"https://github.com/lsst/metaserv/pull/2"},"issue":{"href":"https://api.github.com/repos/lsst/metaserv/issues/2"},"comments":{"href":"https://api.github.com/repos/lsst/metaserv/issues/2/comments"},"review_comments":{"href":"https://api.github.com/repos/lsst/metaserv/pulls/2/comments"},"review_comment":{"href":"https://api.github.com/repos/lsst/metaserv/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/lsst/metaserv/pulls/2/commits"},"statuses":{"href":"https://api.github.com/repos/lsst/metaserv/statuses/1749f7ad75d44378ea0b501c976988088e899cea"}}}}