{"action":"created","comment":{"url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/comments/40264091","id":40264091,"diff_hunk":"@@ -0,0 +1,906 @@\n+#! /usr/bin/env python\n+'''\n+    PURPOSE: Retieves archived NARR files from the NCEP for the dates\n+             requested.  Extracts the variables LST requires (HGT, TMP, SPFH)\n+             and repackages them into our internal location and filenames.\n+\n+    PROJECT: Land Satellites Data Systems Science Research and Development\n+             (LSRD) at the USGS EROS\n+\n+    LICENSE: NASA Open Source Agreement 1.3\n+\n+    NOTES:\n+\n+          NCEP     - National Centers for Environmental Prediction\n+                     http://www.ncep.noaa.gov\n+\n+          NARR     - NCEP North American Regional Reanalysis\n+\n+          CISL RDA - Computational & Information Systems Lab\n+                     Research Data Archive http://rda.ucar.edu\n+\n+          NCAR     - National Center for Atmospheric Research\n+                     http://ncar.ucar.edu\n+\n+          UCAR     - University Corporation for Atmospheric Research\n+                     http://www2.ucar.edu\n+'''\n+import os\n+import sys\n+import shutil\n+import logging\n+import errno\n+import commands\n+import requests\n+import calendar\n+import itertools\n+import multiprocessing as mp\n+from cStringIO import StringIO\n+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter\n+from osgeo import gdal, osr\n+from time import sleep\n+from datetime import datetime, timedelta, date\n+from contextlib import closing\n+import collections\n+import json\n+\n+\n+def debug(tag, message):\n+    logger = logging.getLogger(tag)\n+    logger.debug(message)\n+\n+\n+def substring_between(s, start, finish):\n+    '''Find string between two substrings'''\n+    end_of_start = s.index(start) + len(start)\n+    start_of_finish = s.index(finish, end_of_start)\n+    return s[end_of_start:start_of_finish]\n+\n+\n+# ============================================================================\n+class Web(object):\n+    '''\n+    Description:\n+        Provides methods for interfacing with web resources.\n+    '''\n+\n+    # ------------------------------------------------------------------------\n+    class Session(object):\n+        '''\n+        Description:\n+            Manages an http(s) session.\n+        '''\n+\n+        # --------------------------------------------------------------------\n+        def __init__(self, max_retries=3, block_size=None, timeout=300.0):\n+            super(Web.Session, self).__init__()\n+\n+            self.logger = logging.getLogger(__name__)\n+\n+            self.session = requests.Session()\n+\n+            self.timeout = timeout\n+            self.max_retries = max_retries\n+\n+            # Determine if we are streaming or not based on block_size\n+            self.block_size = block_size\n+            self.stream = False\n+            if self.block_size is not None:\n+                self.stream = True\n+\n+            adapter = requests.adapters.HTTPAdapter(max_retries=max_retries)\n+            self.session.mount('http://', adapter)\n+            self.session.mount('https://', adapter)\n+\n+        # --------------------------------------------------------------------\n+        def login(self, login_url, login_data):\n+            '''\n+            Description:\n+                Provides for establishing a logged in session.\n+            '''\n+\n+            # Login to the site\n+            self.session.post(url=login_url, data=login_data)\n+\n+        # --------------------------------------------------------------------\n+        def _get_file(self, download_url, destination_file, headers=None):\n+            '''\n+            Notes: Downloading this way will place the whole source file into\n+                   memory before dumping to the local file.\n+            '''\n+\n+            req = self.session.get(url=download_url, timeout=self.timeout,\n+                                   headers=headers)\n+\n+            if not req.ok:\n+                self.logger.error('HTTP - Transfer of [{0}] - FAILED'\n+                                  .format(download_url))\n+                # The raise_for_status gets caught by this try's except block\n+                req.raise_for_status()\n+\n+            # Write the downloaded data to the destination file\n+            with open(destination_file, 'wb') as local_fd:\n+                local_fd.write(req.content)\n+\n+                return True\n+\n+        # --------------------------------------------------------------------\n+        def _stream_file(self, download_url, destination_file, headers=None):\n+            '''\n+            Notes: Downloading this way streams 'block_size' of data at a\n+                   time.\n+            '''\n+\n+            retrieved_bytes = 0\n+            with closing(self.session.get(url=download_url,\n+                                          timeout=self.timeout,\n+                                          stream=True,\n+                                          headers=headers)) as req:\n+                file_size = int(req.headers['content-length'])\n+\n+                # Set block size based on streaming\n+                if self.stream:\n+                    block_size = self.block_size\n+                else:\n+                    block_size = file_size\n+\n+                # Write the downloaded data to the destination file\n+                with open(destination_file, 'wb') as local_fd:\n+                    for data_chunk in req.iter_content(block_size):\n+                        local_fd.write(data_chunk)\n+                        retrieved_bytes += len(data_chunk)\n+\n+                if retrieved_bytes != file_size:\n+                    raise Exception('Transfer Failed - HTTP -'\n+                                    ' Retrieved {0} out of {1} bytes'\n+                                    .format(retrieved_bytes, file_size))\n+                else:\n+                    return True\n+\n+        # --------------------------------------------------------------------\n+        def http_transfer_file(self, download_url, destination_file):\n+            '''\n+            Description:\n+                Use http to transfer a file from a source location to a\n+                destination file on the localhost.\n+            Returns:\n+                status_code - One of the following\n+                            - 200, requests.codes['ok']\n+                            - 404, requests.codes['not_found']:\n+                            - 503, requests.codes['service_unavailable']:\n+            Notes:\n+                If a 503 is returned, the logged exception should be reviewed\n+                to determine the real cause of the error.\n+            '''\n+\n+            self.logger.info(download_url)\n+\n+            retry_attempt = 0\n+            done = False\n+            while not done:\n+                status_code = requests.codes['ok']\n+                try:\n+\n+                    done = self._stream_file(download_url,\n+                                             destination_file)\n+\n+                    if done:\n+                        self.logger.info(\"Transfer Complete - HTTP\")\n+\n+                except Exception:\n+                    self.logger.exception('HTTP - Transfer Issue')\n+\n+                    if status_code != requests.codes['not_found']:\n+                        if retry_attempt > self.max_retries:\n+                            self.logger.info('HTTP - Transfer Failed'\n+                                             ' - exceeded retry limit')\n+                            done = True\n+                        else:\n+                            retry_attempt += 1\n+                            sleep(int(1.5 * retry_attempt))\n+                    else:\n+                        # Not Found - So break the looping because we are done\n+                        done = True\n+\n+            return status_code\n+\n+        # --------------------------------------------------------------------\n+        def get_last_modified(self, download_url, destination_file, headers=None):\n+            '''\n+            Notes: Downloading this way streams 'block_size' of data at a\n+                   time.\n+            '''\n+            with closing(self.session.get(url=download_url,\n+                                          timeout=self.timeout,\n+                                          stream=self.stream,\n+                                          headers=headers)) as req:\n+                last_modified = req.headers['last-modified']\n+            return last_modified\n+\n+        # --------------------------------------------------------------------\n+        def get_lines_from_url(self, download_url):\n+            data = []\n+            with closing(self.session.get(url=download_url,\n+                                          timeout=self.timeout,\n+                                          stream=self.stream)) as req:\n+                for line in req.iter_lines():\n+                    data.append(line)\n+\n+            return data\n+\n+\n+# ============================================================================\n+class System(object):\n+    '''\n+    Description:\n+        Provides methods for interfacing with the host server.\n+    '''\n+\n+    # ------------------------------------------------------------------------\n+    @staticmethod\n+    def execute_cmd(cmd):\n+        '''\n+        Description:\n+            Execute a command line and return the terminal output or raise an\n+            exception\n+\n+        Returns:\n+            output - The stdout and/or stderr from the executed command.\n+        '''\n+\n+        logger = logging.getLogger(__name__)\n+\n+        output = ''\n+\n+        logger.info('Executing [{0}]'.format(cmd))\n+        (status, output) = commands.getstatusoutput(cmd)\n+\n+        if status < 0:\n+            message = 'Application terminated by signal [{0}]'.format(cmd)\n+            if len(output) > 0:\n+                message = ' Stdout/Stderr is: '.join([message, output])\n+            raise Exception(message)\n+\n+        if status != 0:\n+            message = 'Application failed to execute [{0}]'.format(cmd)\n+            if len(output) > 0:\n+                message = ' Stdout/Stderr is: '.join([message, output])\n+            raise Exception(message)\n+\n+        if os.WEXITSTATUS(status) != 0:\n+            message = ('Application [{0}] returned error code [{1}]'\n+                       .format(cmd, os.WEXITSTATUS(status)))\n+            if len(output) > 0:\n+                message = ' Stdout/Stderr is: '.join([message, output])\n+            raise Exception(message)\n+\n+        return output\n+\n+    # ------------------------------------------------------------------------\n+    @staticmethod\n+    def create_directory(directory):\n+        '''\n+        Description:\n+            Create the specified directory with some error checking.\n+        '''\n+\n+        # Create/Make sure the directory exists\n+        try:\n+            os.makedirs(directory, mode=0755)\n+        except OSError as ose:\n+            if ose.errno == errno.EEXIST and os.path.isdir(directory):\n+                pass\n+            else:\n+                raise\n+\n+\n+class Config(object):\n+    '''Access to configurable attributes of script\n+\n+    First it will try to read json data from the config file.\n+    If the file does not exist then all config will be from default values.\n+    If Key/value pair doesn't exist in file then default values will be used.\n+    '''\n+    config = None  # Stores result of reading json object from file.\n+    default_config = {'ncep_url_format': 'http://ftp.cpc.ncep.noaa.gov/wd51we/NARR_archive/{0}',\n+                      'remote_name_format': 'rcdas.{0:03}{1:02}{2:02}{3:02}.awip32.merged',\n+                      'archive_directory_format': '{0}/{1:0>4}/{2:0>2}/{3:0>2}',\n+                      'archive_name_format': 'NARR_3D.{0}.{1:04}{2:02}{3:02}.{4:04}.{5}'\n+                     }\n+    @classmethod\n+    def read_config(cls, cfg_file='lst_auxillary.config.example'):\n+        # Load the file\n+        with open(cfg_file, 'r') as fd:\n+            lines = list()\n+            for line in fd:\n+                # Skip rudimentary comments\n+                if line.strip().startswith('#'):\n+                    continue\n+\n+                lines.append(line)\n+\n+            cls.config = json.loads(' '.join(lines))\n+\n+        if cls.config is None:\n+            raise Exception('Failed loading configuration')\n+        return cls.config\n+\n+    @classmethod\n+    def get(cls, attr):\n+        try:\n+            if(cls.config is None):\n+                cls.config = cls.read_config()\n+            if attr in cls.config:\n+                return cls.config[attr]\n+            else:\n+                return cls.default_config[attr]\n+        except IOError:\n+            # If no config is read then use defaults\n+            return cls.default_config[attr]\n+            pass\n+\n+\n+class Ncep(object):\n+    '''Interface for interacting with Ncep website\n+\n+    NarrData depends on the following functionality of this class:\n+        Provide means of getting a particular grib file into current working\n+            directory via get_grib_file()\n+        Provides a dict of all files available via get_dict_of_date_modified()\n+            keys are filenames, values are time of last modification\n+        Provides format of the grib filename via get_filename()\n+    '''\n+    mtime_by_name = None\n+    session = None\n+\n+    # NOAA_FMT.format(year, month, day, hour)\n+    @staticmethod\n+    def get_url(filename):\n+        return Config.get('ncep_url_format').format(filename)\n+\n+    @staticmethod\n+    def get_filename(year, month, day, hour):\n+        fmt = Config.get('remote_name_format')\n+        return fmt.format(year, month, day, hour)\n+\n+    def get_datetime_from_filename(self, filename):\n+        '''Extracts tuple (year, month, day, hour) from filename\n+\n+        Precondition:\n+            File is of format \"rcdas.2015081321.awip32.merged\" for the\n+                measurement at 21:00 (9pm) on August 13th and similar for all\n+                other measurements\n+        Postcondition:\n+            Returns datetime object of the time when the measurement took place\n+        '''\n+        return datetime.strptime(filename.split('.')[1],\n+                                 '%Y%m%d%H')\n+\n+    @classmethod\n+    def get_grib_file(cls, filename):\n+        logger = logging.getLogger(__name__)\n+        last_modified = (cls.get_session()\n+                            .get_last_modified(cls.get_url(filename),\n+                                               filename))\n+        logger.info('{0} last modified at {1} or at {2}'\n+                    .format(filename, last_modified,\n+                            cls.get_dict_of_date_modified()[filename]))\n+\n+        if os.path.isfile(filename):\n+            logger.info('{0} already exists. Skipping download.'\n+                        .format(filename))\n+        else:\n+            logger.info('Retrieving {0}'.format(filename))\n+            cls.get_session().http_transfer_file(cls.get_url(filename),\n+                                                 filename)\n+\n+    @staticmethod\n+    def get_list_of_external_data2():\n+        '''Retrieves list of available data from website's directory listing\n+\n+        Sample line from url reqest the list of files (single line):\n+        '<tr><td><a href=\"rcdas.2015010300.awip32.merged.b\">\n+            rcdas.2015010300.awip32.merged.b</a></td>\n+        <td align=\"right\">08-Jan-2015 10:12  </td>\n+        <td align=\"right\">1.3M</td></tr>\\n'\n+        '''\n+        logger = logging.getLogger(__name__)\n+        ArchiveData = collections.namedtuple('ArchiveData',\n+                                             ['name', 'mtime', 'size'])\n+        lines_thrown = 0\n+        data_list = []\n+        try:\n+            response = requests.get()\n+        except requests.ConnectionError as ce:\n+            logger.error('ConnectionError for request='+str(Ncep.get_url('')))\n+            raise\n+\n+        for line in response.iter_lines():\n+            if('awip' not in line):\n+                lines_thrown = lines_thrown + 1\n+                continue  # go to next line\n+            (garbage, partial_line) = line.split('\">', 1)\n+            (name, partial_line) = partial_line.split('</a>', 1)\n+            (garbage, partial_line) = partial_line.split('\">', 1)\n+            (mtime, partial_line) = partial_line.split('</td>', 1)\n+            (garbage, partial_line) = partial_line.split('\">', 1)\n+            (size, partial_line) = partial_line.split('</td>', 1)\n+\n+            mtime = mtime.strip()  # Remove extra space\n+            data_list.append(ArchiveData(name=name, mtime=mtime, size=size))\n+        return data_list\n+\n+    @classmethod\n+    def get_list_of_external_data(cls):\n+        '''Retrieves list of available data from website's directory listing\n+\n+        Sample line from url reqest the list of files (single line):\n+        '<tr><td><a href=\"rcdas.2015010300.awip32.merged.b\">\n+            rcdas.2015010300.awip32.merged.b</a></td>\n+        <td align=\"right\">08-Jan-2015 10:12  </td>\n+        <td align=\"right\">1.3M</td></tr>\\n'\n+        '''\n+        logger = logging.getLogger(__name__)\n+        ArchiveData = collections.namedtuple('ArchiveData',\n+                                             ['name', 'mtime', 'size'])\n+\n+        lines_thrown = 0\n+        data_list = []\n+\n+        custom_session = cls.get_session()\n+        data = custom_session.get_lines_from_url(Ncep.get_url(''))\n+\n+        for line in data:\n+            if('awip' not in line):\n+                lines_thrown = lines_thrown + 1\n+                continue  # go to next line\n+            (garbage, partial_line) = line.split('\">', 1)\n+            (name, partial_line) = partial_line.split('</a>', 1)\n+            (garbage, partial_line) = partial_line.split('\">', 1)\n+            (mtime, partial_line) = partial_line.split('</td>', 1)\n+            (garbage, partial_line) = partial_line.split('\">', 1)\n+            (size, partial_line) = partial_line.split('</td>', 1)\n+\n+            mtime = mtime.strip()  # Remove extra space\n+            data_list.append(ArchiveData(name=name, mtime=mtime, size=size))\n+        return data_list\n+\n+    @classmethod\n+    def get_session(cls):\n+        '''Obtains and then retains session used for downloading'''\n+        if(cls.session is None):\n+            # Establish a logged in session\n+            cls.session = Web.Session()\n+        return cls.session\n+\n+    @classmethod\n+    def get_dict_of_date_modified(cls):\n+        '''Returns a dictionary of mtime for ext. files with filename as key\n+\n+        Precondition:\n+            Requires that get_list_of_external_data() returns list containing\n+                NamedTuples with tuple.mtime and tuple.name defined.\n+        Postcondition: Returns a dictionary\n+            filename as key, external last modified time as value\n+            '''\n+        if(cls.mtime_by_name is None):\n+            data_list = Ncep.get_list_of_external_data()\n+            cls.mtime_by_name = {}\n+            for item in data_list:\n+                date_modified = datetime.strptime(item.mtime,\n+                                                  '%d-%b-%Y %H:%M')\n+                cls.mtime_by_name[item.name] = date_modified\n+        return cls.mtime_by_name\n+\n+\n+class NarrData(object):\n+    variables = ['HGT', 'TMP', 'SPFH']  # Variables that will be extracted\n+\n+    class FileMissing(Exception):\n+        '''Exception raised when file is missing internally or on website'''\n+        pass\n+\n+    def __init__(self, year, month, day, hour=00):\n+        hour = hour/3*3  # Ensures it is a multiple of 3\n+        self.dt = datetime(year, month, day, hour=hour)\n+\n+    @staticmethod\n+    def from_external_name(external_name):\n+        '''Creates NarrData object from name of external file'''\n+        date_measured = Ncep.get_datetime_from_filename(external_name)\n+        return NarrData(year=date_measured.year, month=date_measured.month,\n+                        day=date_measured.day, hour=date_measured.hour)\n+\n+    @staticmethod\n+    def get_next_narr_data_gen(s_date, e_date, interval=timedelta(hours=3)):\n+        '''Generator to iterate through NarrData objects in time interval\n+\n+        Note: time of datetime objects is ignored.\n+            start of day is used for s_date and end of day for e_date.\n+        Precondition:\n+            s_date and e_date are of type datetime.datetime\n+            s_date and e_date can also be of type datetime.date\n+            interval is of type datetime.timedelta\n+        Postcondition: Returns NarrData instance\n+            s_date < datetime of NarrData < e_date\n+            Consider narr[i-1], narr[i], narr[i+1] as previous, current and\n+                next values of iterator. The following statement will be true:\n+                narr[i-1].dt + interval == narr[i].dt == narr[i].dt - interval\n+        '''\n+        try:  # Handles if datetime objects are passed in.\n+            start_time = datetime.combine(s_date.date(), datetime.min.time())\n+            end_time = datetime.combine(e_date.date(), datetime.max.time())\n+        except AttributeError:  # Handles if date objects are passed in.\n+            start_time = datetime.combine(s_date, datetime.min.time())\n+            end_time = datetime.combine(e_date, datetime.max.time())\n+\n+        logger = logging.getLogger(__name__)\n+        logger.info('Generating list of data from {0} to {1}'\n+                    .format(start_time.isoformat(), end_time.isoformat()))\n+\n+        current = NarrData(year=start_time.year, month=start_time.month,\n+                           day=start_time.day, hour=start_time.hour)\n+        while current.dt <= end_time:\n+            yield(current)\n+            current = current.get_next(interval)\n+\n+    def get_internal_drectory(self):\n+        return NarrArchive.get_arch_dir(self.dt.year, self.dt.month,\n+                                        self.dt.day)\n+\n+    def get_internal_filename(self, variable, ext):\n+        return NarrArchive.get_arch_filename(variable, self.dt.year,\n+                                             self.dt.month, self.dt.day,\n+                                             self.dt.hour, ext)\n+\n+    def get_internal_filepath(self, variable, ext):\n+        return os.path.join(self.get_internal_drectory(),\n+                            self.get_internal_filename(variable, ext))\n+\n+    def get_internal_last_modified(self, variable='HGT', ext='hdr'):\n+        '''Stat internal file for mtime. Default to HGT's hdr file.\n+\n+        Precondition:\n+            File must exist at path given by get_internal_filepath\n+        Postcondition:\n+            returns time of last modification of internal file\n+            raises NarrData.FileMissing if precondition is violated\n+        '''\n+        try:\n+            filename = self.get_internal_filepath(variable, ext)\n+            ts_epoch = os.stat(filename).st_mtime\n+            return datetime.fromtimestamp(ts_epoch)\n+        except OSError:  # Expecting 'No such file or directory'\n+            raise NarrData.FileMissing\n+\n+    def get_external_filename(self):\n+        '''Returns the name of the grib file as choosen by data source'''\n+        return Ncep.get_filename(self.dt.year, self.dt.month, self.dt.day,\n+                                 self.dt.hour)\n+\n+    def get_external_last_modified(self):\n+        '''Returns last_modified time from dictionary stored in Ncep\n+\n+        Precondition:\n+            Ncep class must be able to obtain list of data from website.\n+            filename must exist as key in dict of date modified\n+        Postcondition:\n+            returns date of last modification of the entry with filename as key\n+            Raises NarrData.FileMissing if either precondition is violated.\n+        '''\n+        filename = self.get_external_filename()\n+        try:\n+            return Ncep.get_dict_of_date_modified()[filename]\n+        except KeyError:\n+            raise NarrData.FileMissing\n+\n+    def need_to_update(self):\n+        '''Returns boolean of whether file neads to be donwloaded\n+\n+        Precondition:\n+            get_internal_last_modified and get_external_last_modified return\n+                value of type datetime.\n+        Postcondition:\n+            returns True if either of these conditions are true:\n+                (1) Internal copy of file is older than remote copy\n+                (2) Internal file is missing\n+            returns False if either of these conditions are true:\n+                (1) Internal modified time is more recent then external\n+                    modified time\n+                (2) External file is missing\n+        '''\n+        logger = logging.getLogger(__name__)\n+        try:\n+            ext_mtime = self.get_external_last_modified()\n+        except NarrData.FileMissing:  # Expecting 'No such file or directory'\n+            logger.debug('{0} is missing from list of external files'\n+                         .format(self.get_external_filename()))\n+            return False  # File is not available to download\n+\n+        try:\n+            # Check if existing data is stale\n+            return (self.get_internal_last_modified() < ext_mtime)\n+        except NarrData.FileMissing:  # Expecting 'No such file or directory'\n+            return True  # The file does not exist internally.\n+\n+    def get_grib_file(self):\n+        Ncep.get_grib_file(self.get_external_filename())\n+\n+    def extract_vars_from_grib(self):\n+        '''process_grib_for_variable for each var in NarrData.variables'''\n+        for var in NarrData.variables:\n+            self.process_grib_for_variable(var)\n+\n+    def move_files_to_archive(self):\n+        '''move_to_archive for each var in NarrData.variables'''\n+        for var in NarrData.variables:\n+            self.move_to_archive(var)\n+\n+    def remove_grib_file(self):\n+        logger = logging.getLogger(__name__)\n+        logger.debug('ExternalFile(Exists:{0}, Name:{1})'\n+                     .format(os.path.exists(self.get_external_filename()),\n+                            self.get_external_filename()))\n+        if os.path.exists(self.get_external_filename()):\n+            os.unlink(self.get_external_filename())\n+\n+    def get_next(self, time_increment=timedelta(hours=3)):\n+        dt = self.dt + time_increment\n+        return NarrData(year=dt.year, month=dt.month,\n+                        day=dt.day, hour=dt.hour)\n+\n+    def process_grib_for_variable(self, variable, verbose=False):\n+        '''Extract the specified variable from the grib file and archive it.\n+\n+        Precondition:\n+            A grib file, with the name get_external_filename(), exists in\n+                current working directory.\n+            wgrib must be installed on the system\n+        Postcondition:\n+            A grib and header for variable will exist in current working\n+                directory with the name given by get_internal_filename()\n+        '''\n+        logger = logging.getLogger(__name__)\n+\n+        grib_file = self.get_external_filename()\n+        hdr_name = self.get_internal_filename(variable, 'hdr')\n+        grb_name = self.get_internal_filename(variable, 'grb')\n+\n+        if(os.path.isfile(grb_name) and os.path.isfile(hdr_name)):\n+            logger.warning('{0} and {1} already exist. Skipping extraction.'\n+                           .format(hdr_name, grb_name))\n+            return\n+        logger.info(\"Processing [{0}]\".format(grib_file))\n+\n+        # Create inventory/header file to extract the variable data\n+        cmd_create_temp_header = ['wgrib', grib_file, '|', 'grep', variable,\n+                                  '>', hdr_name]\n+        # Create grib files for each variable\n+        cmd_create_final_grib = ['cat', hdr_name, '|',\n+                                 'wgrib', grib_file, '-i', '-grib',\n+                                 '-o', grb_name]\n+        # Create new inventory/header file for the variable\n+        cmd_create_final_header = ['wgrib', grb_name, '|', 'grep', variable,\n+                                   '>', hdr_name]\n+\n+        cmds = [cmd_create_temp_header, cmd_create_final_grib,\n+                cmd_create_final_header]\n+\n+        for cmd_list in cmds:\n+            cmd = ' '.join(cmd_list)\n+            output = ''\n+            logger.info('Executing [{0}]'.format(cmd))\n+            output = System.execute_cmd(cmd)\n+            if(verbose):\n+                if len(output) > 0:\n+                    logger.info(output)\n+\n+    def move_to_archive(self, variable):\n+        '''Moves grb and hdr files to archive location.\n+\n+        Precondition:\n+            Header and Grib files for variable exist in current directory\n+        Postcondition:\n+            Header and Grib files for variable exist in archive directory\n+            Header and Grib files for variable don't exist in current directory\n+        '''\n+        logger = logging.getLogger(__name__)\n+\n+        dest_path = self.get_internal_drectory()  # Determine the directory\n+        hdr_name = self.get_internal_filename(variable, 'hdr')\n+        grb_name = self.get_internal_filename(variable, 'grb')\n+\n+        System.create_directory(dest_path)  # create it if it does not exist\n+\n+        # Archive the files\n+        logger.info('Archiving into [{0}]'.format(dest_path))\n+        # GRIB\n+        dest_file = os.path.join(dest_path, grb_name)\n+        shutil.copyfile(grb_name, dest_file)\n+        # HEADER\n+        dest_file = os.path.join(dest_path, hdr_name)\n+        shutil.copyfile(hdr_name, dest_file)\n+\n+        # Cleanup the working directory\n+        if os.path.exists(grb_name):\n+            os.unlink(grb_name)\n+        if os.path.exists(hdr_name):\n+            os.unlink(hdr_name)\n+\n+\n+class NarrArchive(object):\n+    _base_aux_dir = None\n+\n+    @classmethod\n+    def get_arch_filename(cls, variable, year, month, day, hour, ext):\n+        return (Config.get('archive_name_format')\n+                      .format(variable, year, month, day, hour*100, ext))\n+\n+    @classmethod\n+    def get_arch_dir(cls, year, month, day):\n+        return (Config.get('archive_directory_format')\n+                      .format(cls.get_base_aux_dir(), year, month, day))\n+\n+    @classmethod\n+    def get_base_aux_dir(cls):\n+        if cls._base_aux_dir is None:  # Check if its not already stored\n+            logger = logging.getLogger(__name__)\n+            cls._base_aux_dir = os.environ.get('LST_AUX_DIR')\n+            # print(\"$LST_AUX_DIR=\"+str(base_aux_dir))\n+            if cls._base_aux_dir is None:\n+                logger.info('Missing environment variable LST_AUX_DIR')\n+                sys.exit(1)\n+            if not os.path.isdir(cls._base_aux_dir):\n+                logger.info('LST_AUX_DIR directory does not exist')\n+                sys.exit(1)\n+        return cls._base_aux_dir\n+\n+\n+def update(data_to_be_updated):\n+    '''Downloads, extracts vars, and cleans temp files for data passed in\n+\n+    Precondition:\n+        data_to_be_updated is a list of NarrData objects\n+        External files exist for every data item\n+    Postcondition:\n+        Header and Grib files for each variable for each item in\n+            data_to_be_updated exist in archive directory\n+        No temporary files exist in the working directory\n+    '''\n+    for data in data_to_be_updated:\n+        data.get_grib_file()\n+\n+    for data in data_to_be_updated:\n+        data.extract_vars_from_grib()\n+\n+    for data in data_to_be_updated:\n+        data.move_files_to_archive()\n+\n+    for data in data_to_be_updated:\n+        data.remove_grib_file()\n+\n+\n+def report(data_to_report):\n+    '''Provides measured time, internal mtime and external mtime of data passed in\n+\n+    Note:\n+        Reports number of files to be downloaded\n+        Includes header to describe data being output.\n+        Reports [measured time, internal mtime, external mtime] as csv\n+    '''\n+    # Statements helpful for debugging\n+    # print('\\n'.join(Ncep.get_list_of_external_data()))\n+    # print(Ncep.get_dict_of_date_modified())\n+\n+    report = []\n+    report.append('Measured, UpdatedLocally, UpdatedOnline')\n+    for data in data_to_report:\n+        line = []\n+        line.append(data.dt.isoformat())\n+        try:\n+            line.append(data.get_internal_last_modified().isoformat())\n+        except NarrData.FileMissing:\n+            line.append('-')\n+        try:\n+            line.append(data.get_external_last_modified().isoformat())\n+        except NarrData.FileMissing:\n+            line.append('-')","path":"lst_auxillary_data/update_narr_aux_data.py","position":807,"original_position":807,"commit_id":"e5dfd6c1c448493b8fbbd4fd7fbe3830722f5e85","original_commit_id":"e5dfd6c1c448493b8fbbd4fd7fbe3830722f5e85","user":{"login":"NGenetzky","id":12733783,"avatar_url":"https://avatars.githubusercontent.com/u/12733783?v=3","gravatar_id":"","url":"https://api.github.com/users/NGenetzky","html_url":"https://github.com/NGenetzky","followers_url":"https://api.github.com/users/NGenetzky/followers","following_url":"https://api.github.com/users/NGenetzky/following{/other_user}","gists_url":"https://api.github.com/users/NGenetzky/gists{/gist_id}","starred_url":"https://api.github.com/users/NGenetzky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NGenetzky/subscriptions","organizations_url":"https://api.github.com/users/NGenetzky/orgs","repos_url":"https://api.github.com/users/NGenetzky/repos","events_url":"https://api.github.com/users/NGenetzky/events{/privacy}","received_events_url":"https://api.github.com/users/NGenetzky/received_events","type":"User","site_admin":false},"body":"Fixed:\r\n```\r\n    report = []\r\n    report.append('Measured, UpdatedLocally, UpdatedOnline')  # Header\r\n\r\n    for data in data_to_report:\r\n        line = []\r\n        line.append(data.dt.isoformat())  # Measured datetime\r\n\r\n        try:\r\n            line.append(data.get_internal_last_modified().isoformat())\r\n        except NarrData.FileMissing:\r\n            line.append('-')\r\n\r\n        try:\r\n            line.append(data.get_external_last_modified().isoformat())\r\n        except NarrData.FileMissing:\r\n            line.append('-')\r\n\r\n        report.append(', '.join(line))\r\n\r\n    return '\\n'.join(report)\r\n```","created_at":"2015-09-23T22:05:33Z","updated_at":"2015-09-23T22:05:33Z","html_url":"https://github.com/USGS-EROS/espa-land-surface-temperature/pull/3#discussion_r40264091","pull_request_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/3","_links":{"self":{"href":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/comments/40264091"},"html":{"href":"https://github.com/USGS-EROS/espa-land-surface-temperature/pull/3#discussion_r40264091"},"pull_request":{"href":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/3"}}},"pull_request":{"url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/3","id":45219203,"html_url":"https://github.com/USGS-EROS/espa-land-surface-temperature/pull/3","diff_url":"https://github.com/USGS-EROS/espa-land-surface-temperature/pull/3.diff","patch_url":"https://github.com/USGS-EROS/espa-land-surface-temperature/pull/3.patch","issue_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/issues/3","number":3,"state":"open","locked":false,"title":"Finished 'update_narr_aux_data.py'","user":{"login":"NGenetzky","id":12733783,"avatar_url":"https://avatars.githubusercontent.com/u/12733783?v=3","gravatar_id":"","url":"https://api.github.com/users/NGenetzky","html_url":"https://github.com/NGenetzky","followers_url":"https://api.github.com/users/NGenetzky/followers","following_url":"https://api.github.com/users/NGenetzky/following{/other_user}","gists_url":"https://api.github.com/users/NGenetzky/gists{/gist_id}","starred_url":"https://api.github.com/users/NGenetzky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NGenetzky/subscriptions","organizations_url":"https://api.github.com/users/NGenetzky/orgs","repos_url":"https://api.github.com/users/NGenetzky/repos","events_url":"https://api.github.com/users/NGenetzky/events{/privacy}","received_events_url":"https://api.github.com/users/NGenetzky/received_events","type":"User","site_admin":false},"body":"The script is used to download the narr aux data from the NCEP data source.","created_at":"2015-09-16T20:21:02Z","updated_at":"2015-09-23T22:05:33Z","closed_at":null,"merged_at":null,"merge_commit_sha":"65115cb39c896303bb39ccdeaaa050f1e9230f96","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/3/commits","review_comments_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/3/comments","review_comment_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/comments{/number}","comments_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/issues/3/comments","statuses_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/statuses/e5dfd6c1c448493b8fbbd4fd7fbe3830722f5e85","head":{"label":"NGenetzky:dev_v0.0.1-a","ref":"dev_v0.0.1-a","sha":"e5dfd6c1c448493b8fbbd4fd7fbe3830722f5e85","user":{"login":"NGenetzky","id":12733783,"avatar_url":"https://avatars.githubusercontent.com/u/12733783?v=3","gravatar_id":"","url":"https://api.github.com/users/NGenetzky","html_url":"https://github.com/NGenetzky","followers_url":"https://api.github.com/users/NGenetzky/followers","following_url":"https://api.github.com/users/NGenetzky/following{/other_user}","gists_url":"https://api.github.com/users/NGenetzky/gists{/gist_id}","starred_url":"https://api.github.com/users/NGenetzky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NGenetzky/subscriptions","organizations_url":"https://api.github.com/users/NGenetzky/orgs","repos_url":"https://api.github.com/users/NGenetzky/repos","events_url":"https://api.github.com/users/NGenetzky/events{/privacy}","received_events_url":"https://api.github.com/users/NGenetzky/received_events","type":"User","site_admin":false},"repo":{"id":40251689,"name":"espa-land-surface-temperature","full_name":"NGenetzky/espa-land-surface-temperature","owner":{"login":"NGenetzky","id":12733783,"avatar_url":"https://avatars.githubusercontent.com/u/12733783?v=3","gravatar_id":"","url":"https://api.github.com/users/NGenetzky","html_url":"https://github.com/NGenetzky","followers_url":"https://api.github.com/users/NGenetzky/followers","following_url":"https://api.github.com/users/NGenetzky/following{/other_user}","gists_url":"https://api.github.com/users/NGenetzky/gists{/gist_id}","starred_url":"https://api.github.com/users/NGenetzky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NGenetzky/subscriptions","organizations_url":"https://api.github.com/users/NGenetzky/orgs","repos_url":"https://api.github.com/users/NGenetzky/repos","events_url":"https://api.github.com/users/NGenetzky/events{/privacy}","received_events_url":"https://api.github.com/users/NGenetzky/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/NGenetzky/espa-land-surface-temperature","description":"ESPA modules for determining land surface temperature","fork":true,"url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature","forks_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/forks","keys_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/keys{/key_id}","collaborators_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/teams","hooks_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/hooks","issue_events_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/issues/events{/number}","events_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/events","assignees_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/assignees{/user}","branches_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/branches{/branch}","tags_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/tags","blobs_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/git/refs{/sha}","trees_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/git/trees{/sha}","statuses_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/statuses/{sha}","languages_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/languages","stargazers_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/stargazers","contributors_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/contributors","subscribers_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/subscribers","subscription_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/subscription","commits_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/commits{/sha}","git_commits_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/git/commits{/sha}","comments_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/comments{/number}","issue_comment_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/issues/comments{/number}","contents_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/contents/{+path}","compare_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/compare/{base}...{head}","merges_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/merges","archive_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/downloads","issues_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/issues{/number}","pulls_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/pulls{/number}","milestones_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/milestones{/number}","notifications_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/labels{/name}","releases_url":"https://api.github.com/repos/NGenetzky/espa-land-surface-temperature/releases{/id}","created_at":"2015-08-05T14:59:28Z","updated_at":"2015-08-05T14:59:30Z","pushed_at":"2015-09-16T20:16:24Z","git_url":"git://github.com/NGenetzky/espa-land-surface-temperature.git","ssh_url":"git@github.com:NGenetzky/espa-land-surface-temperature.git","clone_url":"https://github.com/NGenetzky/espa-land-surface-temperature.git","svn_url":"https://github.com/NGenetzky/espa-land-surface-temperature","homepage":null,"size":13300,"stargazers_count":0,"watchers_count":0,"language":"C","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"USGS-EROS:dev_v0.0.1","ref":"dev_v0.0.1","sha":"1c29d9557fa83c60c7f7abca2038edb22ab8f6ef","user":{"login":"USGS-EROS","id":12549192,"avatar_url":"https://avatars.githubusercontent.com/u/12549192?v=3","gravatar_id":"","url":"https://api.github.com/users/USGS-EROS","html_url":"https://github.com/USGS-EROS","followers_url":"https://api.github.com/users/USGS-EROS/followers","following_url":"https://api.github.com/users/USGS-EROS/following{/other_user}","gists_url":"https://api.github.com/users/USGS-EROS/gists{/gist_id}","starred_url":"https://api.github.com/users/USGS-EROS/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/USGS-EROS/subscriptions","organizations_url":"https://api.github.com/users/USGS-EROS/orgs","repos_url":"https://api.github.com/users/USGS-EROS/repos","events_url":"https://api.github.com/users/USGS-EROS/events{/privacy}","received_events_url":"https://api.github.com/users/USGS-EROS/received_events","type":"Organization","site_admin":false},"repo":{"id":36674207,"name":"espa-land-surface-temperature","full_name":"USGS-EROS/espa-land-surface-temperature","owner":{"login":"USGS-EROS","id":12549192,"avatar_url":"https://avatars.githubusercontent.com/u/12549192?v=3","gravatar_id":"","url":"https://api.github.com/users/USGS-EROS","html_url":"https://github.com/USGS-EROS","followers_url":"https://api.github.com/users/USGS-EROS/followers","following_url":"https://api.github.com/users/USGS-EROS/following{/other_user}","gists_url":"https://api.github.com/users/USGS-EROS/gists{/gist_id}","starred_url":"https://api.github.com/users/USGS-EROS/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/USGS-EROS/subscriptions","organizations_url":"https://api.github.com/users/USGS-EROS/orgs","repos_url":"https://api.github.com/users/USGS-EROS/repos","events_url":"https://api.github.com/users/USGS-EROS/events{/privacy}","received_events_url":"https://api.github.com/users/USGS-EROS/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/USGS-EROS/espa-land-surface-temperature","description":"ESPA modules for determining land surface temperature","fork":false,"url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature","forks_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/forks","keys_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/keys{/key_id}","collaborators_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/teams","hooks_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/hooks","issue_events_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/issues/events{/number}","events_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/events","assignees_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/assignees{/user}","branches_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/branches{/branch}","tags_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/tags","blobs_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/git/refs{/sha}","trees_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/git/trees{/sha}","statuses_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/statuses/{sha}","languages_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/languages","stargazers_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/stargazers","contributors_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/contributors","subscribers_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/subscribers","subscription_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/subscription","commits_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/commits{/sha}","git_commits_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/git/commits{/sha}","comments_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/comments{/number}","issue_comment_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/issues/comments{/number}","contents_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/contents/{+path}","compare_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/compare/{base}...{head}","merges_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/merges","archive_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/downloads","issues_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/issues{/number}","pulls_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls{/number}","milestones_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/milestones{/number}","notifications_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/labels{/name}","releases_url":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/releases{/id}","created_at":"2015-06-01T16:40:39Z","updated_at":"2015-09-02T18:42:41Z","pushed_at":"2015-09-16T20:21:02Z","git_url":"git://github.com/USGS-EROS/espa-land-surface-temperature.git","ssh_url":"git@github.com:USGS-EROS/espa-land-surface-temperature.git","clone_url":"https://github.com/USGS-EROS/espa-land-surface-temperature.git","svn_url":"https://github.com/USGS-EROS/espa-land-surface-temperature","homepage":null,"size":13622,"stargazers_count":3,"watchers_count":3,"language":"C","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":1,"forks":1,"open_issues":1,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/3"},"html":{"href":"https://github.com/USGS-EROS/espa-land-surface-temperature/pull/3"},"issue":{"href":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/issues/3"},"comments":{"href":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/issues/3/comments"},"review_comments":{"href":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/3/comments"},"review_comment":{"href":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/pulls/3/commits"},"statuses":{"href":"https://api.github.com/repos/USGS-EROS/espa-land-surface-temperature/statuses/e5dfd6c1c448493b8fbbd4fd7fbe3830722f5e85"}}}}