{"action":"created","comment":{"url":"https://api.github.com/repos/lsst/daf_ingest/pulls/comments/36364162","id":36364162,"diff_hunk":"@@ -0,0 +1,580 @@\n+#\n+# LSST Data Management System\n+#\n+# Copyright 2008-2015  AURA/LSST.\n+#\n+# This product includes software developed by the\n+# LSST Project (http://www.lsst.org/).\n+#\n+# This program is free software: you can redistribute it and/or modify\n+# it under the terms of the GNU General Public License as published by\n+# the Free Software Foundation, either version 3 of the License, or\n+# (at your option) any later version.\n+#\n+# This program is distributed in the hope that it will be useful,\n+# but WITHOUT ANY WARRANTY; without even the implied warranty of\n+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+# GNU General Public License for more details.\n+#\n+# You should have received a copy of the LSST License Statement and\n+# the GNU General Public License along with this program.  If not,\n+# see <https://www.lsstcorp.org/LegalNotices/>.\n+#\n+\n+from contextlib import closing\n+import MySQLdb\n+import math\n+import re\n+import struct\n+\n+import lsst.afw.table as afwTable\n+from lsst.daf.persistence import DbAuth\n+import lsst.pex.config as pexConfig\n+import lsst.pipe.base as pipeBase\n+\n+## \\addtogroup LSST_task_documentation\n+## \\{\n+## \\page daf_ingest_IngestCatalogTask\n+## \\ref IngestCatalogTask_ \"IngestCatalogTask\"\n+##      \\copybrief IngestCatalogTask\n+## \\}\n+\n+class FieldFormatter(object):\n+    \"\"\"A class for formatting fields in an afw BaseCatalog.\n+\n+    This class is a container for a function that maps a Field to a MySQL type,\n+    and a function that maps a field value to a literal suitable for use in a\n+    MySQL INSERT/REPLACE statement.\n+    \"\"\"\n+\n+    def __init__(self, sqlTypeCallable, formatValueCallable):\n+        \"\"\"Store the field formatting information.\n+        \"\"\"\n+        self.sqlTypeCallable = sqlTypeCallable\n+        self.formatValueCallable = formatValueCallable\n+\n+    def sqlType(self, field):\n+        \"\"\"Return the SQL type for the given field's values.\n+        \"\"\"\n+        return self.sqlTypeCallable(field)\n+\n+    def formatValue(self, value):\n+        \"\"\"Return a string representation of value suitable for use in an\n+        INSERT/REPLACE statement (not a CSV file).  Nones are always\n+        transformed to SQL NULLs.\n+        \"\"\"\n+        if value is None:\n+            return \"NULL\"\n+        return self.formatValueCallable(value)\n+\n+\n+def _formatNumber(formatString, number):\n+    \"\"\"Helper function for formatting floating point numbers. NaNs and\n+    infinities are converted to NULL, because MySQL does not seem to support\n+    NaN/Inf in FLOAT or DOUBLE columns.\n+    \"\"\"\n+    if math.isnan(number) or math.isinf(number):\n+        return \"NULL\"\n+    return formatString.format(number)\n+\n+def _formatString(string):\n+    \"\"\"Helper function for formatting a string. Embedded backslashes and single\n+    quotes are backslash-escaped.\n+    \"\"\"\n+    return \"'\" + string.replace(\"\\\\\", \"\\\\\\\\\").replace(\"'\", \"\\\\'\") + \"'\"\n+\n+def _formatArray(formatChar, array):\n+    \"\"\"Auxiliary function for formatting an array. The array elements are packed\n+    into a sequence of bytes, with bytes comprising individual elements arranged\n+    in little endian order. This sequence is then transformed into a MySQL\n+    hexadecimal literal and returned.\n+    \"\"\"\n+    bytestring = struct.pack(\"<\" + str(len(array)) + formatChar, *array)\n+    return \"x'\" + bytestring.encode('hex_codec') + \"'\"\n+\n+\n+def _sqlTypeForString(field):\n+    \"\"\"Helper function for computing the SQL column type of a string valued field.\n+    \"\"\"\n+    sz = field.getSize()\n+    if sz > 65535:\n+        # If necessary, longer strings could be ingested as TEXT.\n+        raise RuntimeError(\"String field is too large for ingestion\")\n+    elif sz == 0:\n+        raise RuntimeError(\"String field has zero size\")\n+    # A string containing trailing spaces cannot round-trip to a CHAR\n+    # column and back. Therefore, use VARCHAR. Also, assume strings are\n+    # ASCII for now.\n+    return \"VARCHAR({}) CHARACTER SET ascii COLLATE ascii_bin NOT NULL\".format(sz)\n+\n+def _sqlTypeForArray(formatChar, field):\n+    \"\"\"Helper function for computing the SQL column type of an array valued field.\n+    See https://docs.python.org/2/library/struct.html#format-characters for possible\n+    format characters. The formatChar passed in must correspond to the array element\n+    type.\n+    \"\"\"\n+    sz = field.getSize()\n+    if sz == 0:\n+        return \"BLOB NOT NULL\"\n+    sz *= struct.calcsize(\"<\" + formatChar)\n+    if sz > 65535:\n+        raise RuntimeError(\"Array field is too large for ingestion\")\n+    return \"BINARY({}) NOT NULL\".format(sz)\n+\n+\n+\"\"\"A dictionary mapping afw field type strings to corresponding formatters.\n+\n+This dictionary is used by IngestCatalogTask to determine how to format each\n+type of field in the input catalog.  If new field types are added to the afw\n+table library, they should also be added here.\n+\"\"\"\n+fieldFormatters = dict(\n+    U = FieldFormatter(lambda f: \"SMALLINT UNSIGNED NOT NULL\", lambda v: str(v)),\n+    I = FieldFormatter(lambda f: \"INT NOT NULL\", lambda v: str(v)),\n+    L = FieldFormatter(lambda f: \"BIGINT NOT NULL\", lambda v: str(v)),\n+    F = FieldFormatter(lambda f: \"FLOAT\", lambda v: _formatNumber(\"{:.9g}\", v)),\n+    D = FieldFormatter(lambda f: \"DOUBLE\", lambda v: _formatNumber(\"{:.17g}\", v)),\n+    Flag = FieldFormatter(lambda f: \"BIT NOT NULL\", lambda v: \"1\" if v else \"0\"),\n+    Angle = FieldFormatter(lambda f: \"DOUBLE\", lambda v: _formatNumber(\"{:.17g}\", v.asDegrees())),\n+    String = FieldFormatter(_sqlTypeForString, _formatString),\n+    ArrayU = FieldFormatter(lambda f: _sqlTypeForArray(\"H\", f), lambda v: _formatArray(\"H\", v)),\n+    ArrayI = FieldFormatter(lambda f: _sqlTypeForArray(\"i\", f), lambda v: _formatArray(\"i\", v)),\n+    ArrayF = FieldFormatter(lambda f: _sqlTypeForArray(\"f\", f), lambda v: _formatArray(\"f\", v)),\n+    ArrayD = FieldFormatter(lambda f: _sqlTypeForArray(\"d\", f), lambda v: _formatArray(\"d\", v)),\n+)\n+\n+\n+def canonicalizeFieldName(fieldName):\n+    \"\"\"Return a MySQL-compatible version of the schema field name, for now by\n+    changing any non-word characters to underscores.\n+    \"\"\"\n+    return re.sub(r\"[^\\w]\", \"_\", fieldName)\n+\n+def quoteIdentifier(identifier):\n+    \"\"\"Return a MySQL compatible version of the given identifier by quoting\n+    it with back-ticks (and escaping embedded back-ticks).\n+    \"\"\"\n+    return \"`\" + identifier.replace(\"`\", \"``\") + \"`\"\n+\n+def aliasesFor(name, mappings):\n+    \"\"\"Compute and return the set of possible aliases for the given field name.\n+\n+    The afw table library processes a given name F by replacing the longest\n+    prefix of F that can be found in a schema's alias map with the corresponding\n+    alias target. Replacement is repeated until either no such prefix is found,\n+    or N replacements have been made (where N is the size of the alias map).\n+\n+    The mappings argument must be a sorted list of substitutions, where a single\n+    substitution is a 2-tuple of strings (source, target).\n+    \"\"\"\n+    n = 0\n+    aliases, names = set(), set()\n+    names.add(name)\n+    while n < len(mappings) and len(names) > 0:\n+        # Perform one round of reverse alias substitution\n+        n += 1\n+        newNames = set()\n+        for name in names:\n+            for i, (source, target) in enumerate(mappings):\n+                if name.startswith(target):\n+                    alias = source + name[len(target):]\n+                    # alias is only valid if source is its longest prefix in mappings.\n+                    # A prefix strictly longer than source must occur after it in the\n+                    # sorted list of mappings.\n+                    valid = True\n+                    for s, _ in mappings[i + 1:]:\n+                        if not s.startswith(source):\n+                            break;\n+                        if alias.startswith(s):\n+                            valid = False\n+                            break\n+                    if valid:\n+                        aliases.add(alias)\n+                        newNames.add(alias)\n+        names = newNames\n+    return aliases\n+\n+\n+class IngestCatalogConfig(pexConfig.Config):\n+    \"\"\"Configuration for the IngestCatalogTask.\n+    \"\"\"\n+\n+    allowReplace = pexConfig.Field(\n+        \"Allow replacement of existing rows with the same unique IDs\",\n+        bool, default=False\n+    )\n+\n+    maxQueryLen = pexConfig.Field(\n+        \"Maximum length of a query string. None means use a non-standard, \"\n+        \"database-specific way to get the maximum.\",\n+        int, optional=True, default=None\n+    )\n+\n+    maxColumnLen = pexConfig.Field(\n+        \"Maximum length of a database column name or alias. Fields \"\n+        \"that map to longer column names will not be ingested.\",\n+        int, default=64\n+    )\n+\n+    idFieldName = pexConfig.Field(\n+        \"Name of the unique ID field\",\n+        str, optional=True, default=\"id\"\n+    )\n+\n+    remap = pexConfig.DictField(\n+        \"A mapping from afw table field names to desired SQL column names. \"\n+        \"The user must quote desired names containing special characters.\",\n+        keytype=str, itemtype=str, optional=True, default={}\n+    )\n+\n+    extraColumns = pexConfig.Field(\n+        \"Extra column definitions, comma-separated, to put into the \"\n+        \"CREATE TABLE statement if the table is being created\",\n+        str, optional=True, default=\"\"\n+    )\n+\n+\n+class IngestCatalogRunner(pipeBase.TaskRunner):\n+    @staticmethod\n+    def getTargetList(parsedCmd):\n+        \"\"\"Override the target list to add additional run() method parameters.\n+        \"\"\"\n+        return pipeBase.TaskRunner.getTargetList(\n+            parsedCmd,\n+            dstype=parsedCmd.dstype,\n+            tableName=parsedCmd.tableName,\n+            viewName=parsedCmd.viewName,\n+            host=parsedCmd.host,\n+            db=parsedCmd.db,\n+            port=parsedCmd.port,\n+            user=parsedCmd.user\n+        )\n+\n+    def precall(self, parsedCmd):\n+        \"\"\"Override the precall to not write schemas, not require writing of\n+        configs, and set the task's name appropriately.\n+        \"\"\"\n+        self.TaskClass._DefaultName += \"_\" + parsedCmd.dstype\n+        task = self.TaskClass(config=self.config, log=self.log)\n+        try:\n+            task.writeConfig(parsedCmd.butler, clobber=self.clobberConfig)\n+        except Exception, e:\n+            # Often no mapping for config, but in any case just skip\n+            task.log.warn(\"Could not persist config: %s\" % (e,))\n+        return True\n+\n+\n+class IngestCatalogTask(pipeBase.CmdLineTask):\n+    \"\"\"!\n+    \\anchor IngestCatalogTask_\n+    \\brief Ingest an afw catalog into a database table.\n+\n+    \\section daf_ingest_IngestCatalogTask_Contents Contents\n+\n+    - \\ref daf_ingest_IngestCatalogTask_Purpose\n+    - \\ref daf_ingest_IngestCatalogTask_Config\n+    - \\ref daf_ingest_IngestCatalogTask_Example\n+\n+    \\section daf_ingest_IngestCatalogTask_Purpose Description\n+\n+    This task ingests an afw catalog (of any catalog subclass and with an\n+    arbitrary schema) into a database table.\n+\n+    It contacts a MySQL server using connection information given through\n+    command line arguments or \\ref IngestCatalogTask.run \"run\" parameters.\n+    It attempts to use a my.cnf file if present (by omitting a password from\n+    the connection parameters) and falls back to using credentials obtained via\n+    the DbAuth interface if not.\n+\n+    If run from the command line, it will ingest each catalog specified by a\n+    data id and dataset type. As usual for tasks, multiple --id options may be\n+    specified, or ranges and lists of values can be specified for data id keys.\n+\n+    There are also two methods (\\ref IngestCatalogTask.ingest \"ingest\" and\n+    \\ref IngestCatalogTask.runFile \"runFile\") that can be manually called to\n+    ingest catalogs, either by passing the catalog explicitly or by passing\n+    the name of a FITS file containing the catalog.  Both, like\n+    \\ref IngestCatalogTask.run \"run\", require database connection information.\n+\n+    The ingestion process creates the destination table in the database if it\n+    doesn't exist.  The schema is translated from the input catalog's schema,\n+    and may contain a (configurable) unique identifier field.  The only index\n+    provided is a unique one on this field.  (Additional ones can be created\n+    later, of course.)  Additionally, a database view that provides the field\n+    aliases of the input catalog's schema can be created.\n+\n+    Rows are inserted into the database via INSERT statements.  As many rows\n+    as possible are packed into each INSERT to maximize throughput.  The limit\n+    on INSERT statement length is either set by configuration or determined by\n+    querying the database (in a MySQL-specific way).  This may not be as\n+    efficient in its use of the database as converting to CSV and doing a bulk\n+    load, but it eliminates the use of (often shared) disk resources.  The use\n+    of INSERTs (committed once at the end) may not be fully parallelizable\n+    (particularly if a unique id index exists), but tests seem to indicate that\n+    it is at least not much slower to execute many such INSERTs in parallel\n+    compared with executing them all sequentially.  This remains an area for\n+    future optimization.\n+\n+    \\section daf_ingest_IngestCatalogTask_Config Configuration Parameters\n+\n+    The optional idFieldName configuration parameter identifies a unique\n+    identifier field in the source table. If it is specified and the\n+    field exists, a unique index is created for the corresponding column.\n+\n+    Fields and field aliases with database names longer than the maxColumnLen\n+    configuration parameter are automatically dropped.\n+\n+    If a \\ref canonicalizeFieldName \"canonicalized\" afw table field name is\n+    problematic (say because it is too long, or because it matches a SQL\n+    keyword), then it can be changed by providing a more suitable name via the\n+    remap configuration parameter.\n+\n+    Extra columns (e.g. ones to be filled in later by spatial indexing code)\n+    can be added to the database table via the extraColumns configuration\n+    parameter.\n+\n+    \\section daf_ingest_IngestCatalogTask_Example Example\n+\n+    A sample invocation of IngestCatalogTask for ingesting some of the LSST\n+    DM stack demo output is:\n+\n+    \\code\n+        $DAF_INGEST_DIR/bin/ingestCatalog.py \\\n+                $LSST_DM_STACK_DEMO_DIR/output \\\n+                --host lsst-db.ncsa.illinois.edu \\\n+                --database $USER_test \\\n+                --table Source \\\n+                --dstype src \\\n+                --id filter=g\n+    \\endcode\n+    \"\"\"\n+\n+    ConfigClass = IngestCatalogConfig\n+    _DefaultName = \"ingestCatalog\"\n+    RunnerClass = IngestCatalogRunner\n+\n+    @classmethod\n+    def _makeArgumentParser(cls):\n+        \"\"\"Extend the default argument parser with database-specific\n+        arguments and the dataset type for the catalogs to be read.\n+        \"\"\"\n+        parser = pipeBase.ArgumentParser(name=cls._DefaultName)\n+        parser.add_argument(\n+            \"--host\", dest=\"host\", required=True,\n+            help=\"Database hostname\")\n+        parser.add_argument(\n+            \"--database\", dest=\"db\", required=True,\n+            help=\"Database name\")\n+        parser.add_argument(\n+            \"--user\", dest=\"user\",\n+            help=\"Database username (optional)\", default=None)\n+        parser.add_argument(\n+            \"--port\", dest=\"port\",\n+            help=\"Database port number (optional)\", default=3306)\n+        parser.add_argument(\n+            \"--table\", dest=\"tableName\", required=True,\n+            help=\"Table to ingest into\")\n+        parser.add_argument(\n+            \"--view\", dest=\"viewName\",\n+            help=\"View to create containing column aliases\")\n+        # Use DatasetArgument to require dataset type be specified on\n+        # the command line\n+        parser.add_id_argument(\n+            \"--id\", pipeBase.DatasetArgument(\"dstype\"),\n+            help=\"Dataset data id to ingest\")\n+        return parser\n+\n+    def runFile(self, fileName, tableName, host, db, port=3306, user=None, viewName=None):\n+        \"\"\"Ingest a BaseCatalog specified by a filename.\n+        \"\"\"\n+        cat = afwTable.BaseCatalog.readFits(fileName)\n+        self.ingest(cat, tableName, host, db, port, user, viewName)\n+\n+    def run(self, dataRef, dstype, tableName, host, db, port=3306, user=None, viewName=None):\n+        \"\"\"Ingest a BaseCatalog specified by a dataref and dataset type.\n+        \"\"\"\n+        self.ingest(dataRef.get(dstype), tableName, host, db, port, user, viewName)\n+\n+    @pipeBase.timeMethod\n+    def ingest(self, cat, tableName, host, db, port=3306, user=None, viewName=None):\n+        \"\"\"Ingest a BaseCatalog passed as an object.\n+\n+        @param cat (BaseCatalog or derivative) Catalog to ingest.\n+        @param tableName (str)   Name of the database table to create.\n+        @param host (str)        Name of the database host machine.\n+        @param db (str)          Name of the database to ingest into.\n+        @param port (int)        Port number on the database host.\n+        @param user (str)        User name to use for the database.\n+        @param viewName (str)    Name of the database view to create.\n+        \"\"\"\n+        tableName = quoteIdentifier(tableName)\n+        viewName = quoteIdentifier(viewName) if viewName else None\n+        with closing(self.connect(host, port, db, user)) as conn:\n+            # Determine the maximum query length (MySQL-specific) if not\n+            # configured.\n+            if self.config.maxQueryLen is None:\n+                with closing(conn.cursor()) as cursor:\n+                    cursor.execute(\n+                        \"\"\"SELECT variable_value\n+                        FROM information_schema.session_variables\n+                        WHERE variable_name = 'max_allowed_packet'\n+                        \"\"\"\n+                    )\n+                    maxQueryLen = int(cursor.fetchone()[0])\n+            else:\n+                maxQueryLen = self.config.maxQueryLen\n+            self.log.logdebug(\"maxQueryLen: {}\".format(maxQueryLen))\n+            self._createTable(conn, tableName, cat.schema)\n+            if viewName is not None:\n+                self._createView(conn, tableName, viewName, cat.schema)\n+            self._ingest(conn, cat, tableName, maxQueryLen)\n+\n+    @staticmethod\n+    def connect(host, port, db, user=None):\n+        kwargs = dict(host=host, port=port, db=db)\n+        if user is not None:\n+            kwargs[\"user\"] = user\n+        try:\n+            # See if we can connect without a password (e.g. via my.cnf)\n+            return MySQLdb.connect(**kwargs)\n+        except:\n+            # Fallback to DbAuth\n+            kwargs[\"user\"] = DbAuth.username(host, str(port))\n+            kwargs[\"password\"] = DbAuth.password(host, str(port))\n+            return MySQLdb.connect(**kwargs)\n+\n+    def _executeSql(self, conn, sql):\n+        \"\"\"Execute a SQL query with no expectation of a result.\n+        \"\"\"\n+        self.log.logdebug(sql)\n+        conn.query(sql)\n+\n+    def _schemaItems(self, schema):\n+        \"\"\"A generator over ingestible schema items.\n+        \"\"\"\n+        for item in schema:\n+            field = item.field\n+            if field.getTypeString() not in fieldFormatters:\n+                self.log.warn(\n+                    \"Skipping field {}: type {} not supported\".format(\n+                        field.getName(), field.getTypeString()))\n+            else:\n+                column = self._columnName(field.getName())\n+                if len(column) > self.config.maxColumnLen:\n+                    self.log.warn(\n+                        \"Skipping field {}: column name {} too long\".format(\n+                            field.getName(), column))\n+                else:\n+                    yield item\n+\n+    def _ingest(self, conn, cat, tableName, maxQueryLen):\n+        \"\"\"Ingest a BaseCatalog by converting it to one or more (large) INSERT or\n+        REPLACE statements, executing those statements, and committing the result.\n+        \"\"\"\n+        sqlPrefix = \"REPLACE\" if self.config.allowReplace else \"INSERT\"\n+        sqlPrefix += \" INTO {} (\".format(tableName)\n+        keys = []\n+        columnNames = []\n+        for item in self._schemaItems(cat.schema):\n+            keys.append((item.key, fieldFormatters[item.field.getTypeString()]))\n+            columnNames.append(self._columnName(item.field.getName()))\n+        sqlPrefix += \",\".join(columnNames)\n+        sqlPrefix += \") VALUES \"\n+        pos = 0\n+        while pos < len(cat):\n+            sql = sqlPrefix\n+            initialPos = pos\n+            maxValueLen = maxQueryLen - len(sql)\n+            while pos < len(cat):\n+                row = cat[pos]\n+                value = \"(\"\n+                value += \",\".join([f.formatValue(row.get(k)) for (k, f) in keys])\n+                value += \"),\"\n+                maxValueLen -= len(value)\n+                if maxValueLen < 0:\n+                    break\n+                else:\n+                    sql += value\n+                    pos += 1\n+            if pos == initialPos:\n+                # Have not made progress\n+                raise RuntimeError(\"Single row is too large to insert\")\n+            self._executeSql(conn, sql[:-1])\n+        conn.commit()\n+\n+    def _columnName(self, fieldName):\n+        \"\"\"Return the SQL column name for a given afw table field.\n+        \"\"\"\n+        if fieldName in self.config.remap:\n+            return self.config.remap[fieldName]\n+        return canonicalizeFieldName(fieldName)\n+\n+    def _columnDef(self, field):\n+        \"\"\"Return the SQL column definition for a given afw table field.\n+        \"\"\"\n+        sqlType = fieldFormatters[field.getTypeString()].sqlType(field)\n+        return self._columnName(field.getName()) + \" \" + sqlType\n+\n+    def _createTable(self, conn, tableName, schema):\n+        \"\"\"Create a table using column definitions based on the provided afw\n+        table schema, adding in any extra columns specified in the config.  If\n+        one exists, the unique id column is given a key.\n+        \"\"\"\n+        fields = [item.field for item in self._schemaItems(schema)]\n+        names = [f.getName() for f in fields]\n+        equivalenceClasses = {}\n+        for name in names:\n+            equivalenceClasses.setdefault(name.lower(), []).append(name)\n+        clashes = ',\\n'.join('\\t{' + ', '.join(c) + '}'\n+                             for c in equivalenceClasses.itervalues() if len(c) > 1)\n+        if clashes:\n+            raise RuntimeError(\n+                \"Schema contains columns that differ only by non-word characters \"\n+                \"and/or case:\\n{}\\nIn the database, these cannot be distinguished \"\n+                \"and hence result in column name duplicates. Use the remap \"\n+                \"configuration parameter to resolve this ambiguity.\".format(clashes))\n+        sql = \"CREATE TABLE IF NOT EXISTS {} (\\n\\t\".format(tableName)\n+        sql += \",\\n\\t\".join(self._columnDef(field) for field in fields)\n+        if self.config.extraColumns:\n+            sql += \",\\n\\t\" + self.config.extraColumns\n+        if self.config.idFieldName:\n+            if self.config.idFieldName in names:\n+                sql += \",\\n\\tUNIQUE({})\".format(self._columnName(self.config.idFieldName))\n+            else:\n+                self.log.warn(\n+                    \"No field matches the configured unique ID field name \"\n+                    \"({})\".format(self.config.idFieldName))\n+        sql += \"\\n)\"\n+        self._executeSql(conn, sql)\n+\n+    def _createView(self, conn, tableName, viewName, schema):\n+        \"\"\"Create a view that allows table columns to be referred to by their\n+        afw table field aliases.\n+        \"\"\"\n+        sql = (\"CREATE OR REPLACE \"\n+               \"ALGORITHM = MERGE SQL SECURITY INVOKER \"\n+               \"VIEW {} AS SELECT\\n\\t\").format(viewName)\n+        with closing(conn.cursor()) as cursor:\n+            cursor.execute(\"SHOW COLUMNS FROM \" + tableName)\n+            columnNames = [row[0] for row in cursor.fetchall()]\n+        sql += \",\\n\\t\".join(columnNames)\n+        # Technically, this isn't quite right. In afw, it appears to be legal\n+        # for an alias to shadow an actual field name. So for full rigor,\n+        # shadowed field names would have to be removed from the columnNames\n+        # list.\n+        #\n+        # For now, construct an invalid view and fail in this case.","path":"python/lsst/daf/ingest/ingestCatalogTask.py","position":566,"original_position":566,"commit_id":"6e83f660b22168c713490548578112f118a1d5aa","original_commit_id":"6e83f660b22168c713490548578112f118a1d5aa","user":{"login":"smonkewitz","id":620114,"avatar_url":"https://avatars.githubusercontent.com/u/620114?v=3","gravatar_id":"","url":"https://api.github.com/users/smonkewitz","html_url":"https://github.com/smonkewitz","followers_url":"https://api.github.com/users/smonkewitz/followers","following_url":"https://api.github.com/users/smonkewitz/following{/other_user}","gists_url":"https://api.github.com/users/smonkewitz/gists{/gist_id}","starred_url":"https://api.github.com/users/smonkewitz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/smonkewitz/subscriptions","organizations_url":"https://api.github.com/users/smonkewitz/orgs","repos_url":"https://api.github.com/users/smonkewitz/repos","events_url":"https://api.github.com/users/smonkewitz/events{/privacy}","received_events_url":"https://api.github.com/users/smonkewitz/received_events","type":"User","site_admin":false},"body":"I should probably just implement this, but I'm not sure it's really a good idea. Having an alias that shadows a field name just seems like it should be an error. @TallJimbo - any thoughts? If you agree that it should be an error, should I add a note to DM-3401 or make a separate ticket?","created_at":"2015-08-05T22:50:03Z","updated_at":"2015-08-05T22:50:03Z","html_url":"https://github.com/lsst/daf_ingest/pull/1#discussion_r36364162","pull_request_url":"https://api.github.com/repos/lsst/daf_ingest/pulls/1","_links":{"self":{"href":"https://api.github.com/repos/lsst/daf_ingest/pulls/comments/36364162"},"html":{"href":"https://github.com/lsst/daf_ingest/pull/1#discussion_r36364162"},"pull_request":{"href":"https://api.github.com/repos/lsst/daf_ingest/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/lsst/daf_ingest/pulls/1","id":41066678,"html_url":"https://github.com/lsst/daf_ingest/pull/1","diff_url":"https://github.com/lsst/daf_ingest/pull/1.diff","patch_url":"https://github.com/lsst/daf_ingest/pull/1.patch","issue_url":"https://api.github.com/repos/lsst/daf_ingest/issues/1","number":1,"state":"open","locked":false,"title":"tickets/DM-1674: Data ingest scripts cleanup","user":{"login":"smonkewitz","id":620114,"avatar_url":"https://avatars.githubusercontent.com/u/620114?v=3","gravatar_id":"","url":"https://api.github.com/users/smonkewitz","html_url":"https://github.com/smonkewitz","followers_url":"https://api.github.com/users/smonkewitz/followers","following_url":"https://api.github.com/users/smonkewitz/following{/other_user}","gists_url":"https://api.github.com/users/smonkewitz/gists{/gist_id}","starred_url":"https://api.github.com/users/smonkewitz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/smonkewitz/subscriptions","organizations_url":"https://api.github.com/users/smonkewitz/orgs","repos_url":"https://api.github.com/users/smonkewitz/repos","events_url":"https://api.github.com/users/smonkewitz/events{/privacy}","received_events_url":"https://api.github.com/users/smonkewitz/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-07-28T22:07:37Z","updated_at":"2015-08-05T22:50:03Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a456656aadfa116b1c91cec68ac6833fa4d4ba6b","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/lsst/daf_ingest/pulls/1/commits","review_comments_url":"https://api.github.com/repos/lsst/daf_ingest/pulls/1/comments","review_comment_url":"https://api.github.com/repos/lsst/daf_ingest/pulls/comments{/number}","comments_url":"https://api.github.com/repos/lsst/daf_ingest/issues/1/comments","statuses_url":"https://api.github.com/repos/lsst/daf_ingest/statuses/6e83f660b22168c713490548578112f118a1d5aa","head":{"label":"lsst:tickets/DM-1674","ref":"tickets/DM-1674","sha":"6e83f660b22168c713490548578112f118a1d5aa","user":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"repo":{"id":23013273,"name":"daf_ingest","full_name":"lsst/daf_ingest","owner":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/lsst/daf_ingest","description":null,"fork":false,"url":"https://api.github.com/repos/lsst/daf_ingest","forks_url":"https://api.github.com/repos/lsst/daf_ingest/forks","keys_url":"https://api.github.com/repos/lsst/daf_ingest/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lsst/daf_ingest/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lsst/daf_ingest/teams","hooks_url":"https://api.github.com/repos/lsst/daf_ingest/hooks","issue_events_url":"https://api.github.com/repos/lsst/daf_ingest/issues/events{/number}","events_url":"https://api.github.com/repos/lsst/daf_ingest/events","assignees_url":"https://api.github.com/repos/lsst/daf_ingest/assignees{/user}","branches_url":"https://api.github.com/repos/lsst/daf_ingest/branches{/branch}","tags_url":"https://api.github.com/repos/lsst/daf_ingest/tags","blobs_url":"https://api.github.com/repos/lsst/daf_ingest/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lsst/daf_ingest/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lsst/daf_ingest/git/refs{/sha}","trees_url":"https://api.github.com/repos/lsst/daf_ingest/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lsst/daf_ingest/statuses/{sha}","languages_url":"https://api.github.com/repos/lsst/daf_ingest/languages","stargazers_url":"https://api.github.com/repos/lsst/daf_ingest/stargazers","contributors_url":"https://api.github.com/repos/lsst/daf_ingest/contributors","subscribers_url":"https://api.github.com/repos/lsst/daf_ingest/subscribers","subscription_url":"https://api.github.com/repos/lsst/daf_ingest/subscription","commits_url":"https://api.github.com/repos/lsst/daf_ingest/commits{/sha}","git_commits_url":"https://api.github.com/repos/lsst/daf_ingest/git/commits{/sha}","comments_url":"https://api.github.com/repos/lsst/daf_ingest/comments{/number}","issue_comment_url":"https://api.github.com/repos/lsst/daf_ingest/issues/comments{/number}","contents_url":"https://api.github.com/repos/lsst/daf_ingest/contents/{+path}","compare_url":"https://api.github.com/repos/lsst/daf_ingest/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lsst/daf_ingest/merges","archive_url":"https://api.github.com/repos/lsst/daf_ingest/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lsst/daf_ingest/downloads","issues_url":"https://api.github.com/repos/lsst/daf_ingest/issues{/number}","pulls_url":"https://api.github.com/repos/lsst/daf_ingest/pulls{/number}","milestones_url":"https://api.github.com/repos/lsst/daf_ingest/milestones{/number}","notifications_url":"https://api.github.com/repos/lsst/daf_ingest/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lsst/daf_ingest/labels{/name}","releases_url":"https://api.github.com/repos/lsst/daf_ingest/releases{/id}","created_at":"2014-08-16T07:41:42Z","updated_at":"2014-08-16T07:41:42Z","pushed_at":"2015-08-04T22:56:13Z","git_url":"git://github.com/lsst/daf_ingest.git","ssh_url":"git@github.com:lsst/daf_ingest.git","clone_url":"https://github.com/lsst/daf_ingest.git","svn_url":"https://github.com/lsst/daf_ingest","homepage":null,"size":4353,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":1,"forks":1,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"lsst:master","ref":"master","sha":"df9bbbb32296990969842741c91c51dbd1ddd3cb","user":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"repo":{"id":23013273,"name":"daf_ingest","full_name":"lsst/daf_ingest","owner":{"login":"lsst","id":1134504,"avatar_url":"https://avatars.githubusercontent.com/u/1134504?v=3","gravatar_id":"","url":"https://api.github.com/users/lsst","html_url":"https://github.com/lsst","followers_url":"https://api.github.com/users/lsst/followers","following_url":"https://api.github.com/users/lsst/following{/other_user}","gists_url":"https://api.github.com/users/lsst/gists{/gist_id}","starred_url":"https://api.github.com/users/lsst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lsst/subscriptions","organizations_url":"https://api.github.com/users/lsst/orgs","repos_url":"https://api.github.com/users/lsst/repos","events_url":"https://api.github.com/users/lsst/events{/privacy}","received_events_url":"https://api.github.com/users/lsst/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/lsst/daf_ingest","description":null,"fork":false,"url":"https://api.github.com/repos/lsst/daf_ingest","forks_url":"https://api.github.com/repos/lsst/daf_ingest/forks","keys_url":"https://api.github.com/repos/lsst/daf_ingest/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lsst/daf_ingest/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lsst/daf_ingest/teams","hooks_url":"https://api.github.com/repos/lsst/daf_ingest/hooks","issue_events_url":"https://api.github.com/repos/lsst/daf_ingest/issues/events{/number}","events_url":"https://api.github.com/repos/lsst/daf_ingest/events","assignees_url":"https://api.github.com/repos/lsst/daf_ingest/assignees{/user}","branches_url":"https://api.github.com/repos/lsst/daf_ingest/branches{/branch}","tags_url":"https://api.github.com/repos/lsst/daf_ingest/tags","blobs_url":"https://api.github.com/repos/lsst/daf_ingest/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lsst/daf_ingest/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lsst/daf_ingest/git/refs{/sha}","trees_url":"https://api.github.com/repos/lsst/daf_ingest/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lsst/daf_ingest/statuses/{sha}","languages_url":"https://api.github.com/repos/lsst/daf_ingest/languages","stargazers_url":"https://api.github.com/repos/lsst/daf_ingest/stargazers","contributors_url":"https://api.github.com/repos/lsst/daf_ingest/contributors","subscribers_url":"https://api.github.com/repos/lsst/daf_ingest/subscribers","subscription_url":"https://api.github.com/repos/lsst/daf_ingest/subscription","commits_url":"https://api.github.com/repos/lsst/daf_ingest/commits{/sha}","git_commits_url":"https://api.github.com/repos/lsst/daf_ingest/git/commits{/sha}","comments_url":"https://api.github.com/repos/lsst/daf_ingest/comments{/number}","issue_comment_url":"https://api.github.com/repos/lsst/daf_ingest/issues/comments{/number}","contents_url":"https://api.github.com/repos/lsst/daf_ingest/contents/{+path}","compare_url":"https://api.github.com/repos/lsst/daf_ingest/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lsst/daf_ingest/merges","archive_url":"https://api.github.com/repos/lsst/daf_ingest/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lsst/daf_ingest/downloads","issues_url":"https://api.github.com/repos/lsst/daf_ingest/issues{/number}","pulls_url":"https://api.github.com/repos/lsst/daf_ingest/pulls{/number}","milestones_url":"https://api.github.com/repos/lsst/daf_ingest/milestones{/number}","notifications_url":"https://api.github.com/repos/lsst/daf_ingest/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lsst/daf_ingest/labels{/name}","releases_url":"https://api.github.com/repos/lsst/daf_ingest/releases{/id}","created_at":"2014-08-16T07:41:42Z","updated_at":"2014-08-16T07:41:42Z","pushed_at":"2015-08-04T22:56:13Z","git_url":"git://github.com/lsst/daf_ingest.git","ssh_url":"git@github.com:lsst/daf_ingest.git","clone_url":"https://github.com/lsst/daf_ingest.git","svn_url":"https://github.com/lsst/daf_ingest","homepage":null,"size":4353,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":1,"forks":1,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/lsst/daf_ingest/pulls/1"},"html":{"href":"https://github.com/lsst/daf_ingest/pull/1"},"issue":{"href":"https://api.github.com/repos/lsst/daf_ingest/issues/1"},"comments":{"href":"https://api.github.com/repos/lsst/daf_ingest/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/lsst/daf_ingest/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/lsst/daf_ingest/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/lsst/daf_ingest/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/lsst/daf_ingest/statuses/6e83f660b22168c713490548578112f118a1d5aa"}}}}