{"action":"created","comment":{"url":"https://api.github.com/repos/obspy/obspy/pulls/comments/41471836","id":41471836,"diff_hunk":"@@ -0,0 +1,1140 @@\n+#!/usr/bin/env python\n+# -*- coding: utf-8 -*-\n+\"\"\"\n+Helpers for the mass downloader.\n+\n+Intended to simplify and stabilize the logic of the mass downloader and make\n+it understandable in the first place.\n+\n+:copyright:\n+    Lion Krischer (krischer@geophysik.uni-muenchen.de), 2014-2015\n+:license:\n+    GNU Lesser General Public License, Version 3\n+    (http://www.gnu.org/copyleft/lesser.html)\n+\"\"\"\n+from __future__ import (absolute_import, division, print_function,\n+                        unicode_literals)\n+from future.builtins import *  # NOQA\n+from future import standard_library\n+with standard_library.hooks():\n+    import itertools\n+\n+import collections\n+import copy\n+from multiprocessing.pool import ThreadPool\n+import numpy as np\n+import os\n+import time\n+import timeit\n+\n+import obspy\n+from obspy.core.util import Enum\n+\n+from . import utils\n+\n+# The current status of an entity.\n+STATUS = Enum([\"none\", \"needs_downloading\", \"downloaded\", \"ignore\", \"exists\",\n+               \"download_failed\", \"download_rejected\",\n+               \"download_partially_failed\"])\n+\n+\n+class Station(object):\n+    \"\"\"\n+    Object representing a seismic station within the download helper classes.\n+\n+    It knows the coordinates of the station to perform the filtering,\n+    its channels and the filename and status of the StationXML files.\n+\n+    :param network: The network code.\n+    :type network: str\n+    :param station: The station code.\n+    :type station: str\n+    :param latitude: The latitude of the station.\n+    :type latitude: float\n+    :param longitude: The longitude of the station.\n+    :type longitude: float\n+    :param channels: The channels of the station.\n+    :type channels: list of :class:`~.Channel` objects\n+    :param stationxml_filename: The filename of the StationXML file.\n+    :type stationxml_filename: str\n+    :param stationxml_status: The current status of the station.\n+    :type stationxml_filename:\n+        :class:`~.STATUS`\n+    \"\"\"\n+    __slots__ = [\"network\", \"station\", \"latitude\", \"longitude\", \"channels\",\n+                 \"_stationxml_filename\", \"want_station_information\",\n+                 \"miss_station_information\", \"have_station_information\",\n+                 \"stationxml_status\"]\n+\n+    def __init__(self, network, station, latitude, longitude, channels,\n+                 stationxml_filename=None, stationxml_status=None):\n+        # Station attributes.\n+        self.network = network\n+        self.station = station\n+        self.latitude = latitude\n+        self.longitude = longitude\n+        self.channels = channels\n+        # Station information settings.\n+        self.stationxml_filename = stationxml_filename\n+        self.stationxml_status = stationxml_status and STATUS.NONE\n+\n+        # Internally keep track of which channels and time interval want\n+        # station information, which miss station information and which\n+        # already have some. want_station_information should always be the\n+        # union of miss and have.\n+        self.want_station_information = {}\n+        self.miss_station_information = {}\n+        self.have_station_information = {}\n+\n+    @property\n+    def has_existing_or_downloaded_time_intervals(self):\n+        \"\"\"\n+        Returns true if any of the station's time intervals have status\n+        \"DOWNLOADED\" or \"EXISTS\". Otherwise it returns False meaning it does\n+        not have to be considered anymore.\n+        \"\"\"\n+        status = []\n+        for chan in self.channels:\n+            for ti in chan.intervals:\n+                status.append(ti.status)\n+        status = list(set(status))\n+        if STATUS.EXISTS in status or STATUS.DOWNLOADED in status:\n+            return True\n+        return False\n+\n+    @property\n+    def has_existing_time_intervals(self):\n+        \"\"\"\n+        Returns True if any of the station's time intervals already exist.\n+        \"\"\"\n+        for chan in self.channels:\n+            for ti in chan.intervals:\n+                if ti.status == STATUS.EXISTS:\n+                    return True\n+        return False\n+\n+    def remove_files(self, logger, reason):\n+        \"\"\"\n+        Delete all files under it. Only delete stuff that actually has been\n+        downloaded!\n+        \"\"\"\n+        for chan in self.channels:\n+            for ti in chan.intervals:\n+                if ti.status != STATUS.DOWNLOADED or not ti.filename:\n+                    continue\n+                if os.path.exists(ti.filename):\n+                    logger.info(\"Deleting MiniSEED file '%s'. Reason: %s\" % (\n+                        ti.filename, reason))\n+                    utils.safe_delete(ti.filename)\n+\n+        if self.stationxml_status == STATUS.DOWNLOADED and \\\n+                self.stationxml_filename and \\\n+                os.path.exists(self.stationxml_filename):\n+            logger.info(\"Deleting StationXMl file '%s'. Reason: %s\" %\n+                        (self.stationxml_filename, reason))\n+            utils.safe_delete(self.stationxml_filename)\n+\n+    @property\n+    def stationxml_filename(self):\n+        return self._stationxml_filename\n+\n+    @stationxml_filename.setter\n+    def stationxml_filename(self, value):\n+        \"\"\"\n+        Setter creating the directory for the file if does not already exist.\n+        \"\"\"\n+        self._stationxml_filename = value\n+        if not value:\n+            return\n+        dirname = os.path.dirname(value)\n+        if not os.path.exists(dirname):\n+            os.makedirs(dirname)\n+\n+    @property\n+    def temporal_bounds(self):\n+        \"\"\"\n+        Return the temporal bounds for the station.\n+        \"\"\"\n+        starttimes = []\n+        endtimes = []\n+        for channel in self.channels:\n+            s, e = channel.temporal_bounds\n+            starttimes.append(s)\n+            endtimes.append(e)\n+        return min(starttimes), max(endtimes)\n+\n+    def __str__(self):\n+        channels = \"\\n\".join(str(i) for i in self.channels)\n+        channels = \"\\n\\t\".join(channels.splitlines())\n+        return (\n+            \"Station '{network}.{station}' [Lat: {lat:.2f}, Lng: {lng:.2f}]\\n\"\n+            \"\\t-> Filename: {filename} ({status})\\n\"\n+            \"\\t-> Wants station information for channels:  {want}\\n\"\n+            \"\\t-> Has station information for channels:    {has}\\n\"\n+            \"\\t-> Misses station information for channels: {miss}\\n\"\n+            \"\\t{channels}\"\n+        ).format(\n+            network=self.network,\n+            station=self.station,\n+            lat=self.latitude,\n+            lng=self.longitude,\n+            filename=self.stationxml_filename,\n+            status=\"exist\" if (self.stationxml_filename and os.path.exists(\n+                self.stationxml_filename)) else \"does not yet exist\",\n+            want=\", \".join([\"%s.%s\" % (_i[0], _i[1]) for _i in\n+                            self.want_station_information.keys()]),\n+            has=\", \".join([\"%s.%s\" % (_i[0], _i[1]) for _i in\n+                          self.have_station_information.keys()]),\n+            miss=\", \".join([\"%s.%s\" % (_i[0], _i[1]) for _i in\n+                           self.miss_station_information.keys()]),\n+            channels=channels)\n+\n+    def prepare_stationxml_download(self, stationxml_storage, logger):\n+        \"\"\"\n+        Figure out what to download.\n+\n+        :param stationxml_storage:\n+        \"\"\"\n+        # Determine what channels actually want to have station information.\n+        # This will be a tuple of location code, channel code, starttime,\n+        # and endtime.\n+        self.want_station_information = {}\n+        for channel in self.channels:\n+            if channel.needs_station_file is False:\n+                continue\n+            self.want_station_information[\n+                (channel.location, channel.channel)] = channel.temporal_bounds\n+\n+        # No channel has any data, thus nothing will happen.\n+        if not self.want_station_information:\n+            self.stationxml_status = STATUS.NONE\n+            return\n+\n+        # Only those channels that now actually want station information\n+        # will be treated in the following.\n+        s, e = self.temporal_bounds\n+        storage = utils.get_stationxml_filename(\n+            stationxml_storage, self.network, self.station,\n+            list(self.want_station_information.keys()),\n+            starttime=s, endtime=e)\n+\n+        # The simplest case. The function returns a string. Now two things\n+        # can happen.\n+        if isinstance(storage, (str, bytes)):\n+            filename = storage\n+            self.stationxml_filename = filename\n+            # 1. The file does not yet exist. Thus all channels must be\n+            # downloaded.\n+            if not os.path.exists(filename):\n+                self.miss_station_information = \\\n+                    copy.deepcopy(self.want_station_information)\n+                self.have_station_information = {}\n+                self.stationxml_status = STATUS.NEEDS_DOWNLOADING\n+                return\n+            # 2. The file does exist. It will be parsed. If it contains ALL\n+            # necessary information, nothing will happen. Otherwise it will\n+            # be overwritten.\n+            else:\n+                info = utils.get_stationxml_contents(filename)\n+                for c_id, times in self.want_station_information.items():\n+                    # Get the temporal range of information in the file.\n+                    c_info = [_i for _i in info if\n+                              _i.network == self.network and\n+                              _i.station == self.station and\n+                              _i.location == c_id[0] and\n+                              _i.channel == c_id[1]]\n+                    if not c_info:\n+                        break\n+                    starttime = min([_i.starttime for _i in c_info])\n+                    endtime = max([_i.endtime for _i in c_info])\n+                    if starttime > times[0] or endtime < times[1]:\n+                        break\n+                # All good if no break is called.\n+                else:\n+                    self.have_station_information = \\\n+                        copy.deepcopy(self.want_station_information)\n+                    self.miss_station_information = {}\n+                    self.stationxml_status = STATUS.EXISTS\n+                    return\n+                # Otherwise everything will be downloaded.\n+                self.miss_station_information = \\\n+                    copy.deepcopy(self.want_station_information)\n+                self.have_station_information = {}\n+                self.stationxml_status = STATUS.NEEDS_DOWNLOADING\n+                return\n+        # The other possibility is that a dictionary is returned.\n+        else:\n+            # The types are already checked by the get_stationxml_filename()\n+            # function.\n+            missing_channels = storage[\"missing_channels\"]\n+            available_channels = storage[\"available_channels\"]\n+\n+            # Get the channels wanting station information and filter them.\n+            channels_wanting_station_information = copy.deepcopy(\n+                self.want_station_information\n+            )\n+\n+            # Figure out what channels are missing and will be downloaded.\n+            self.miss_station_information = {}\n+            for channel in missing_channels:\n+                if channel not in channels_wanting_station_information:\n+                    continue\n+                self.miss_station_information[channel] = \\\n+                    channels_wanting_station_information[channel]\n+\n+            # Same thing but with the already available channels.\n+            self.have_station_information = {}\n+            for channel in available_channels:\n+                if channel not in channels_wanting_station_information:\n+                    continue\n+                self.have_station_information[channel] = \\\n+                    channels_wanting_station_information[channel]\n+\n+            self.stationxml_filename = storage[\"filename\"]\n+\n+            # Raise a warning if something is missing, but do not raise an\n+            # exception or halt the programm at this point.\n+            have_channels = set(self.have_station_information.keys())\n+            miss_channels = set(self.miss_station_information.keys())\n+            want_channels = set(self.want_station_information.keys())\n+            if have_channels.union(miss_channels) != want_channels:\n+                logger.warning(\n+                    \"The custom `stationxml_storage` did not return \"\n+                    \"information about channels %s\" %\n+                    str(want_channels.difference(have_channels.union(\n+                        miss_channels))))\n+\n+            if self.miss_station_information:\n+                self.stationxml_status = STATUS.NEEDS_DOWNLOADING\n+            elif not self.miss_station_information and \\\n+                    self.have_station_information:\n+                self.stationxml_status = STATUS.EXISTS\n+            else:\n+                self.stationxml_status = STATUS.IGNORE\n+\n+    def prepare_mseed_download(self, mseed_storage):\n+        \"\"\"\n+        Loop through all channels of the station and distribute filenames\n+        and the current status of the channel.\n+\n+        A MiniSEED interval will be ignored, if the `mseed_storage` function\n+        returns `True`.\n+        Possible statuses after this the method execution are IGNORE, EXISTS,\n+        and NEEDS_DOWNLOADING.\n+\n+        :param mseed_storage:\n+        \"\"\"\n+        for channel in self.channels:\n+            for interval in channel.intervals:\n+                interval.filename = utils.get_mseed_filename(\n+                    mseed_storage, self.network, self.station,\n+                    channel.location, channel.channel, interval.start,\n+                    interval.end)\n+                if interval.filename is True:\n+                    interval.status = STATUS.IGNORE\n+                elif os.path.exists(interval.filename):\n+                    interval.status = STATUS.EXISTS\n+                else:\n+                    if not os.path.exists(os.path.dirname(interval.filename)):\n+                        os.makedirs(os.path.dirname(interval.filename))\n+                    interval.status = STATUS.NEEDS_DOWNLOADING\n+\n+    def sanitize_downloads(self, logger):\n+        \"\"\"\n+        Should be run after the MiniSEED and StationXML downloads finished.\n+        It will make sure that every MiniSEED file also has a corresponding\n+        StationXML file.\n+\n+        It will delete MiniSEED files but never a StationXML file. The logic\n+        of the download helpers does not allow for a StationXML file with no\n+        data.\n+        \"\"\"\n+        # All or nothing for each channel.\n+        for id in self.miss_station_information.keys():\n+            logger.warning(\"No station information could be downloaded for \"\n+                           \"%s.%s.%s.%s. All downloaded MiniSEED files \"\n+                           \"will be deleted!\" % (\n+                               self.network, self.station, id[0], id[1]))\n+            channel = [_i for _i in self.channels if\n+                       (_i.location, _i.channel) == id][0]\n+            for time_interval in channel.intervals:\n+                # Only delete downloaded things!\n+                if time_interval.status == STATUS.DOWNLOADED:\n+                    utils.safe_delete(time_interval.filename)\n+                    time_interval.status = STATUS.DOWNLOAD_REJECTED\n+\n+\n+class Channel(object):\n+    \"\"\"\n+    Object representing a Channel. Each time interval should end up in one\n+    MiniSEED file.\n+    \"\"\"\n+    __slots__ = [\"location\", \"channel\", \"intervals\"]\n+\n+    def __init__(self, location, channel, intervals):\n+        self.location = location\n+        self.channel = channel\n+        self.intervals = intervals\n+\n+    @property\n+    def needs_station_file(self):\n+        \"\"\"\n+        Determine if the channel requires any station information.\n+\n+        As soon as at the status of at least one interval is either\n+        ``DOWNLOADED`` or ``EXISTS`` the whole channel will be thought of as\n+        requiring station information. This does not yet mean that station\n+        information will be downloaded. That is decided at a later stage.\n+        \"\"\"\n+        status = set([_i.status for _i in self.intervals])\n+        if STATUS.DOWNLOADED in status or STATUS.EXISTS in status:\n+            return True\n+        return False\n+\n+    @property\n+    def temporal_bounds(self):\n+        \"\"\"\n+        Returns a tuple of the minimum start time and the maximum end time.\n+        \"\"\"\n+        return (min([_i.start for _i in self.intervals]),\n+                max([_i.end for _i in self.intervals]))\n+\n+    def __str__(self):\n+        return \"Channel '{location}.{channel}':\\n\\t{intervals}\".format(\n+            location=self.location, channel=self.channel,\n+            intervals=\"\\n\\t\".join([str(i) for i in self.intervals]))\n+\n+\n+class TimeInterval(object):\n+    \"\"\"\n+    Simple object representing a time interval of a channel.\n+\n+    It knows the temporal bounds of the interval, the (desired) filename,\n+    and the current status of the interval.\n+\n+    :param start: The start of the interval.\n+    :type start: :class:`~obspy.core.utcdatetime.UTCDateTime`\n+    :param end: The end of the interval.\n+    :type end: :class:`~obspy.core.utcdatetime.UTCDateTime`\n+    :param filename: The filename of the interval.\n+    :type filename: str\n+    :param status: The status of the time interval.\n+    :param status: :class:`~.STATUS`\n+    \"\"\"\n+    __slots__ = [\"start\", \"end\", \"filename\", \"status\"]\n+\n+    def __init__(self, start, end, filename=None, status=None):\n+        self.start = start\n+        self.end = end\n+        self.filename = filename\n+        self.status = status if status is not None else STATUS.NONE\n+\n+    def __repr__(self):\n+        return \"TimeInterval(start={start}, end={end}, filename={filename}, \" \\\n+               \"status='{status}')\".format(\n+                   start=repr(self.start),\n+                   end=repr(self.end),\n+                   filename=\"'%s'\" % self.filename\n+                   if self.filename is not None else \"None\",\n+                   status=str(self.status))\n+\n+\n+class ClientDownloadHelper(object):\n+    \"\"\"\n+    :type client: :class:`obspy.fdsn.client.Client`\n+    :param client: An initialized FDSN client.\n+    :type client_name: str\n+    :param client_name: The name of the client. Only used for logging.\n+    :type restrictions: :class:`~.restrictions.Restrictions`\n+    :param restrictions: The non-domain related restrictions for the query.\n+    :type domain: :class:`~.domain.Domain` subclass\n+    :param domain: The domain definition.\n+    :param mseed_storage: The MiniSEED storage settings.\n+    :param stationxml_storage: THe StationXML storage settings.\n+    :param logger: An active logger instance.\n+    \"\"\"\n+    def __init__(self, client, client_name, restrictions, domain,\n+                 mseed_storage, stationxml_storage, logger):\n+        self.client = client\n+        self.client_name = client_name\n+        self.restrictions = restrictions\n+        self.domain = domain\n+        self.mseed_storage = mseed_storage\n+        self.stationxml_storage = stationxml_storage\n+        self.logger = logger\n+        self.stations = {}\n+        self.is_availability_reliable = None\n+\n+    def __bool__(self):\n+        return bool(len(self))\n+\n+    def __str__(self):\n+        avail_map = {\n+            None: \"Unknown reliability of availability information\",\n+            True: \"Reliable availability information\",\n+            False: \"Non-reliable availability information\"\n+\n+        }\n+        reliability = avail_map[self.is_availability_reliable]\n+        return (\n+            \"ClientDownloadHelper object for client '{client}' ({url})\\n\"\n+            \"-> {reliability}\\n\"\n+            \"-> Manages {station_count} stations.\\n{stations}\").format(\n+                client=self.client_name,\n+                url=self.client.base_url,\n+                reliability=reliability,\n+                station_count=len(self),\n+                stations=\"\\n\".join([str(_i) for _i in self.stations.values()]))\n+\n+    def __len__(self):\n+        return len(self.stations)\n+\n+    def prepare_mseed_download(self):\n+        \"\"\"\n+        Prepare each Station for the MiniSEED downloading stage.\n+\n+        This will distribute filenames and identify files that require\n+        downloading.\n+        \"\"\"\n+        for station in self.stations.values():\n+            station.prepare_mseed_download(mseed_storage=self.mseed_storage)\n+\n+    def filter_stations_based_on_minimum_distance(\n+            self, existing_client_dl_helpers):\n+        \"\"\"\n+        Removes stations until all stations have a certain minimum distance to\n+        each other.\n+\n+        Returns the rejected stations which is mainly useful for testing.\n+\n+        :param existing_client_dl_helpers: Instances of already existing\n+            client download helpers.\n+        :type existing_client_dl_helpers: list of\n+            :class:`~.ClientDownloadHelper`\n+        \"\"\"\n+        if not self.restrictions.minimum_interstation_distance_in_m:\n+            # No rejected stations.\n+            return []\n+\n+        # Create a sorted copy that will be used in the following. Make it\n+        # more deterministic by sorting the stations based on the id.\n+        stations = copy.copy(list(self.stations.values()))\n+        stations = sorted(stations, key=lambda x: (x.network, x.station))\n+\n+        existing_stations = []\n+        for dlh in existing_client_dl_helpers:\n+            existing_stations.extend(list(dlh.stations.values()))\n+\n+        remaining_stations = []\n+        rejected_stations = []\n+\n+        # There are essentially two possibilities. If no station exists yet,\n+        # it will choose the largest subset of stations satisfying the\n+        # minimum inter-station distance constraint.\n+        if not existing_stations and \\\n+                not any([_i.has_existing_time_intervals for _i in stations]):\n+            # Build k-d-tree and query for the neighbours of each point within\n+            # the minimum distance.\n+            kd_tree = utils.SphericalNearestNeighbour(stations)\n+            nns = kd_tree.query_pairs(\n+                self.restrictions.minimum_interstation_distance_in_m)\n+\n+            indexes_to_remove = []\n+            # Keep removing the station with the most pairs until no pairs are\n+            # left.\n+            while nns:\n+                most_common = collections.Counter(\n+                    itertools.chain.from_iterable(nns)).most_common()[0][0]\n+                indexes_to_remove.append(most_common)\n+                nns = list(itertools.filterfalse(\n+                    lambda x: most_common in x, nns))\n+\n+            # Remove these indices this results in a set of stations we wish to\n+            # keep.\n+            remaining_stations.extend(\n+                set([_i[1] for _i in itertools.filterfalse(\n+                    lambda x: x[0] in indexes_to_remove,\n+                    enumerate(stations))]))\n+            rejected_stations.extend(\n+                set([_i[1] for _i in filter(\n+                    lambda x: x[0] in indexes_to_remove,\n+                    enumerate(stations))]))\n+        # Otherwise it will add new stations approximating a Poisson disk\n+        # distribution.\n+        else:\n+            while stations:\n+                # kd-tree with all existing_stations\n+                existing_kd_tree = utils.SphericalNearestNeighbour(\n+                    existing_stations)\n+                # Now we have to get the distance to the closest existing\n+                # station for all new stations.\n+                distances = np.ma.array(existing_kd_tree.query(stations)[0][0])\n+                if np.isinf(distances[0]):\n+                    break\n+                distances.mask = False\n+\n+                # Step one is to get rid of all stations that are closer\n+                # then the minimum distance to any existing station.\n+                remove = np.where(\n+                    distances <\n+                    self.restrictions.minimum_interstation_distance_in_m)[0]\n+                rejected_stations.extend([stations[_i] for _i in remove])\n+\n+                keep = np.where(\n+                    distances >=\n+                    self.restrictions.minimum_interstation_distance_in_m)[0]\n+                distances.mask[remove] = True\n+\n+                if len(keep):\n+                    # Station with the largest distance to next closer station.\n+                    largest = np.argmax(distances)\n+                    remaining_stations.append(stations[largest])\n+                    existing_stations.append(stations[largest])\n+\n+                    # Add all rejected stations here.\n+                    stations = [stations[_i] for _i in keep if _i != largest]\n+                else:\n+                    stations = []\n+\n+        # Now actually delete the files and everything of the rejected\n+        # stations.\n+        for station in rejected_stations:\n+            station.remove_files(logger=self.logger,\n+                                 reason=\"Minimum distance filtering.\")\n+        self.stations = {}\n+        for station in remaining_stations:\n+            self.stations[(station.network, station.station)] = station\n+\n+        # Return the rejected stations.\n+        return {(_i.network, _i.station): _i for _i in rejected_stations}\n+\n+    def prepare_stationxml_download(self):\n+        \"\"\"\n+        Prepare each Station for the StationXML downloading stage.\n+\n+        This will distribute filenames and identify files that require\n+        downloading.\n+        \"\"\"\n+        for station in self.stations.values():\n+            station.prepare_stationxml_download(\n+                stationxml_storage=self.stationxml_storage,\n+                logger=self.logger)\n+\n+    def download_stationxml(self, threads=3):\n+        \"\"\"\n+        Actually download the StationXML files.\n+\n+        :param threads: Limits the maximum number of threads for the client.\n+        \"\"\"\n+\n+        def star_download_station(args):\n+            \"\"\"\n+            Maps arguments to the utils.download_stationxml() function.\n+\n+            :param args: The to-be mapped arguments.\n+            \"\"\"\n+            try:\n+                ret_val = utils.download_stationxml(*args, logger=self.logger)\n+            except utils.ERRORS as e:\n+                self.logger.error(str(e))\n+                return None\n+            return ret_val\n+\n+        # Build up everything we want to download.\n+        arguments = []\n+        for station in self.stations.values():\n+            if not station.miss_station_information:\n+                continue\n+            s, e = station.temporal_bounds\n+            if self.restrictions.station_starttime:\n+                s = self.restrictions.station_starttime\n+            if self.restrictions.station_endtime:\n+                e = self.restrictions.station_endtime\n+            bulk = [(station.network, station.station, channel.location,\n+                     channel.channel, s, e) for channel in station.channels]\n+            arguments.append((self.client, self.client_name, bulk,\n+                              station.stationxml_filename))\n+\n+        if not arguments:\n+            self.logger.info(\"Client '%s' - No station information to \"\n+                             \"download.\" % self.client_name)\n+            return\n+\n+        # Download it.\n+        s_time = timeit.default_timer()\n+        pool = ThreadPool(min(threads, len(arguments)))\n+        results = pool.map(star_download_station, arguments)\n+        pool.close()\n+        e_time = timeit.default_timer()\n+\n+        results = [_i for _i in results if _i is not None]\n+\n+        # Check it.\n+        filecount = 0\n+        download_size = 0\n+\n+        # Update the station structures. Loop over each returned file.\n+        for s_id, filename in results:\n+            filecount += 1\n+            station = self.stations[s_id]\n+            size = os.path.getsize(filename)\n+            download_size += size\n+\n+            # Extract information about that file.\n+            info = utils.get_stationxml_contents(filename)\n+\n+            still_missing = {}\n+            # Make sure all missing information has been downloaded by\n+            # looping over each channel of the station that originally\n+            # requested to be downloaded.\n+            for c_id, times in station.miss_station_information.items():\n+                # Get the temporal range of information in the file.\n+                c_info = [_i for _i in info if\n+                          _i.network == station.network and\n+                          _i.station == station.station and\n+                          _i.location == c_id[0] and\n+                          _i.channel == c_id[1]]\n+                if not c_info:\n+                    continue\n+                starttime = min([_i.starttime for _i in c_info])\n+                endtime = max([_i.endtime for _i in c_info])\n+                if starttime > times[0] or endtime < times[1]:\n+                    still_missing[c_id] = times\n+                    continue\n+                station.have_station_information[c_id] = times\n+\n+            station.miss_station_information = still_missing\n+            if still_missing:\n+                station.stationxml_status = STATUS.DOWNLOAD_PARTIALLY_FAILED\n+            else:\n+                station.stationxml_status = STATUS.DOWNLOADED\n+\n+        # Now loop over all stations and that the status of the ones that\n+        # still need downloading to download failed.\n+        for station in self.stations.values():\n+            if station.stationxml_status == STATUS.NEEDS_DOWNLOADING:\n+                station.stationxml_status = STATUS.DOWNLOAD_FAILED\n+\n+        self.logger.info(\"Client '%s' - Downloaded %i station files [%.1f MB] \"\n+                         \"in %.1f seconds [%.2f KB/sec].\" % (\n+                             self.client_name, filecount,\n+                             download_size / 1024.0 ** 2,\n+                             e_time - s_time,\n+                             (download_size / 1024.0) / (e_time - s_time)))\n+\n+    def download_mseed(self, chunk_size_in_mb=25, threads_per_client=3):\n+        \"\"\"\n+        Actually download MiniSEED data.\n+\n+        :param chunk_size_in_mb:\n+        :param threads_per_client:\n+        :return:\n+        \"\"\"\n+        # Estimate the download size to have equally sized chunks.\n+        channel_sampling_rate = {\n+            \"F\": 5000, \"G\": 5000, \"D\": 1000, \"C\": 1000, \"E\": 250, \"S\": 80,\n+            \"H\": 250, \"B\": 80, \"M\": 10, \"L\": 1, \"V\": 0.1, \"U\": 0.01,\n+            \"R\": 0.001, \"P\": 0.0001, \"T\": 0.00001, \"Q\": 0.000001, \"A\": 5000,\n+            \"O\": 5000}\n+\n+        # Split into chunks of about equal size in terms of filesize.\n+        chunks = []\n+        chunks_curr = []\n+        curr_chunks_mb = 0\n+\n+        # Don't request more than 50 chunks at once to not choke the servers.\n+        MAX_CHUNK_LENGTH = 50\n+\n+        counter = collections.Counter()\n+\n+        # Keep track of attempted downloads.\n+        for sta in self.stations.values():\n+            for cha in sta.channels:\n+                # The band code is used to estimate the sampling rate of the\n+                # data to be downloaded.\n+                band_code = cha.channel[0].upper()\n+                try:\n+                    sr = channel_sampling_rate[band_code]\n+                except KeyError:\n+                    # Generic sampling rate for exotic band codes.\n+                    sr = 1.0\n+\n+                for interval in cha.intervals:\n+                    counter[interval.status] += 1\n+                    # Only take those time intervals that actually require\n+                    # some downloading.\n+                    if interval.status != STATUS.NEEDS_DOWNLOADING:\n+                        continue\n+                    chunks_curr.append((\n+                        sta.network, sta.station, cha.location, cha.channel,\n+                        interval.start, interval.end, interval.filename))\n+                    # Assume that each sample needs 4 byte, STEIM\n+                    # compression reduces size to about a third.\n+                    # chunk size is in MB\n+                    duration = interval.end - interval.start\n+                    curr_chunks_mb += \\\n+                        sr * duration * 4.0 / 3.0 / 1024.0 / 1024.0\n+                    if curr_chunks_mb >= chunk_size_in_mb or \\\n+                            len(chunks_curr) >= MAX_CHUNK_LENGTH:\n+                        chunks.append(chunks_curr)\n+                        chunks_curr = []\n+                        curr_chunks_mb = 0\n+        if chunks_curr:\n+            chunks.append(chunks_curr)\n+\n+        keys = sorted(counter.keys())\n+        for key in keys:\n+            self.logger.info(\n+                \"Client '%s' - Status for %i time intervals/channels before \"\n+                \"downloading: %s\" % (self.client_name, counter[key],\n+                                     key.upper()))\n+\n+        if not chunks:\n+            return\n+\n+        def star_download_mseed(args):\n+            \"\"\"\n+            Star maps the arguments to the\n+            utils.download_and_split_mseed_bulk() function.\n+\n+            :param args: The arguments to be passed.\n+            \"\"\"\n+            try:\n+                ret_val = utils.download_and_split_mseed_bulk(\n+                    *args, logger=self.logger)\n+            except utils.ERRORS as e:\n+                msg = (\"Client '%s' - \" % args[1]) + str(e)\n+                if \"no data available\" in msg.lower():\n+                    self.logger.info(msg.split(\"Detailed response\")[0].strip())\n+                else:\n+                    self.logger.error(msg)\n+                return []\n+            return ret_val\n+\n+        pool = ThreadPool(min(threads_per_client, len(chunks)))\n+\n+        d_start = timeit.default_timer()\n+        pool.map(\n+            star_download_mseed,\n+            [(self.client, self.client_name, chunk) for chunk in chunks])\n+        pool.close()\n+        d_end = timeit.default_timer()\n+\n+        self.logger.info(\"Client '%s' - Launching basic QC checks...\" %\n+                         self.client_name)\n+        downloaded_bytes, discarded_bytes = self._check_downloaded_data()\n+        total_bytes = downloaded_bytes + discarded_bytes\n+\n+        self.logger.info(\"Client '%s' - Downloaded %.1f MB [%.2f KB/sec] of \"\n+                         \"data, %.1f MB of which were discarded afterwards.\" %\n+                         (self.client_name, total_bytes / 1024.0 ** 2,\n+                          total_bytes / 1024.0 / (d_end - d_start),\n+                          discarded_bytes / 1024.0 ** 2))\n+\n+        # Recount everything to be able to emit some nice statistics.\n+        counter = collections.Counter()\n+        for sta in self.stations.values():\n+            for chan in sta.channels:\n+                for interval in chan.intervals:\n+                    counter[interval.status] += 1\n+        keys = sorted(counter.keys())\n+        for key in keys:\n+            self.logger.info(\n+                \"Client '%s' - Status for %i time intervals/channels after \"\n+                \"downloading: %s\" % (\n+                    self.client_name, counter[key], key.upper()))\n+\n+        self._remove_failed_and_ignored_stations()\n+\n+    def _remove_failed_and_ignored_stations(self):\n+        \"\"\"\n+        Removes all stations that have no time interval with either exists\n+        or downloaded status.\n+        \"\"\"\n+        to_be_removed_keys = []\n+        for key, station in self.stations.items():\n+            if station.has_existing_or_downloaded_time_intervals is True:\n+                continue\n+            to_be_removed_keys.append(key)\n+        for key in to_be_removed_keys:\n+            del self.stations[key]\n+\n+    def sanitize_downloads(self):\n+        \"\"\"\n+        Should be run after the MiniSEED and StationXML downloads finished.\n+        It will make sure that every MiniSEED file also has a corresponding\n+        StationXML file.\n+        \"\"\"\n+        for station in self.stations.values():\n+            station.sanitize_downloads(logger=self.logger)\n+\n+    def _check_downloaded_data(self):\n+        \"\"\"\n+        Read the downloaded data, set the proper status flags and a remove\n+        data that does not meet the QC criteria. It just checks the\n+        downloaded data for minimum length and gaps/overlaps.\n+\n+        Returns the downloaded_bytes and the discarded_bytes.\n+        \"\"\"\n+        downloaded_bytes = 0\n+        discarded_bytes = 0\n+        for sta in self.stations.values():\n+            for cha in sta.channels:\n+                for interval in cha.intervals:\n+                    # The status of the interval should not have changed if\n+                    # it did not require doenloading in the first place.\n+                    if interval.status != STATUS.NEEDS_DOWNLOADING:\n+                        continue\n+\n+                    # If the file does not exist, mark the time interval as\n+                    # download failed.\n+                    if not os.path.exists(interval.filename):\n+                        interval.status = STATUS.DOWNLOAD_FAILED\n+                        continue\n+\n+                    size = os.path.getsize(interval.filename)\n+                    if size == 0:\n+                        self.logger.warning(\"Zero byte file '%s'. Will be \"\n+                                            \"deleted.\" % interval.filename)\n+                        utils.safe_delete(interval.filename)\n+                        interval.status = STATUS.DOWNLOAD_FAILED\n+                        continue\n+\n+                    # Guard against faulty files.\n+                    try:\n+                        st = obspy.read(interval.filename, headonly=True)\n+                    except Exception as e:\n+                        self.logger.warning(\n+                            \"Could not read file '%s' due to: %s\\n\"\n+                            \"Will be discarded.\" % (interval.filename, str(e)))\n+                        utils.safe_delete(interval.filename)\n+                        discarded_bytes += size\n+                        interval.status = STATUS.DOWNLOAD_FAILED\n+                        continue\n+\n+                    # Valid files with no data.\n+                    if len(st) == 0:\n+                        self.logger.warning(\n+                            \"Empty file '%s'. Will be deleted.\" %\n+                            interval.filename)\n+                        utils.safe_delete(interval.filename)\n+                        discarded_bytes += size\n+                        interval.status = STATUS.DOWNLOAD_FAILED\n+                        continue\n+\n+                    # If user did not want gappy files, remove them.\n+                    if self.restrictions.reject_channels_with_gaps is True and\\\n+                            len(st) > 1:\n+                        self.logger.info(\"File '%s' contains %i traces. Will \"\n+                                         \"be deleted.\" % (interval.filename,\n+                                                          len(st)))\n+                        utils.safe_delete(interval.filename)\n+                        discarded_bytes += size\n+                        interval.status = STATUS.DOWNLOAD_REJECTED\n+                        continue\n+\n+                    if self.restrictions.minimum_length:\n+                        duration = sum([tr.stats.endtime - tr.stats.starttime\n+                                        for tr in st])\n+                        expected_min_duration = \\\n+                            self.restrictions.minimum_length * \\\n+                            (interval.end - interval.start)\n+                        if duration < expected_min_duration:\n+                            self.logger.info(\n+                                \"File '%s' has only %.2f seconds of data. \"\n+                                \"%.2f are required. File will be deleted.\" %\n+                                (interval.filename, duration,\n+                                 expected_min_duration))\n+                            utils.safe_delete(interval.filename)\n+                            discarded_bytes += size\n+                            interval.status = STATUS.DOWNLOAD_REJECTED\n+                            continue\n+\n+                    downloaded_bytes += size\n+                    interval.status = STATUS.DOWNLOADED\n+        return downloaded_bytes, discarded_bytes\n+\n+    def _parse_miniseed_filenames(self, filenames, restrictions):\n+        time_range = restrictions.minimum_length * (restrictions.endtime -\n+                                                    restrictions.starttime)\n+        channel_availability = []\n+        for filename in filenames:\n+            st = obspy.read(filename, format=\"MSEED\", headonly=True)\n+            if restrictions.reject_channels_with_gaps and len(st) > 1:\n+                self.logger.warning(\"Channel %s has gap or overlap. Will be \"\n+                                    \"removed.\" % st[0].id)\n+                try:\n+                    os.remove(filename)\n+                except OSError:\n+                    pass\n+                continue\n+            elif len(st) == 0:\n+                self.logger.error(\"MiniSEED file with no data detected. \"\n+                                  \"Should not happen!\")\n+                continue\n+            tr = st[0]\n+            duration = tr.stats.endtime - tr.stats.starttime\n+            if restrictions.minimum_length and duration < time_range:\n+                self.logger.warning(\"Channel %s does not satisfy the minimum \"\n+                                    \"length requirement. %.2f seconds instead \"\n+                                    \"of the required %.2f seconds.\" % (\n+                                        tr.id, duration, time_range))\n+                try:\n+                    os.remove(filename)\n+                except OSError:\n+                    pass\n+                continue\n+            channel_availability.append(utils.ChannelAvailability(\n+                tr.stats.network, tr.stats.station, tr.stats.location,\n+                tr.stats.channel, tr.stats.starttime, tr.stats.endtime,\n+                filename))\n+        return channel_availability\n+\n+    def discard_stations(self, existing_client_dl_helpers):\n+        \"\"\"\n+        Discard all stations part of any of the already existing client\n+        download helper instances. The station discarding happens purely\n+        based on station ids.\n+\n+        :param existing_client_dl_helpers: Instances of already existing\n+            client download helpers. All stations part of this will not be\n+            downloaded anymore.\n+        :type existing_client_dl_helpers: list of\n+            :class:`~.ClientDownloadHelper`\n+        \"\"\"\n+        station_ids = []\n+        for helper in existing_client_dl_helpers:\n+            station_ids.extend(helper.stations.keys())\n+\n+        for station_id in station_ids:\n+            try:\n+                del self.stations[station_id]\n+            except KeyError:\n+                pass\n+\n+    def get_availability(self):\n+        \"\"\"\n+        Queries the current client for information of what stations are\n+        available given the spatial and temporal restrictions.\n+        \"\"\"\n+        # Check if stations needs to be filtered after downloading or if the\n+        # restrictions one can impose with the FDSN webservices queries are\n+        # enough. This depends on the domain definition.\n+        try:\n+            self.domain.is_in_domain(0, 0)\n+            needs_filtering = True\n+        except NotImplementedError:\n+            needs_filtering = False\n+\n+        arguments = {\n+            \"network\": self.restrictions.network,\n+            \"station\": self.restrictions.station,\n+            \"location\": self.restrictions.location,\n+            \"channel\": self.restrictions.channel,\n+            \"starttime\": self.restrictions.starttime,\n+            \"endtime\": self.restrictions.endtime,\n+            # Request at the channel level.\n+            \"level\": \"channel\"\n+        }\n+        # Add the domain specific query parameters.\n+        arguments.update(self.domain.get_query_parameters())\n+\n+        # Check the capabilities of the service and see what is the most\n+        # appropriate way of acquiring availability information. Some services\n+        # right now require manual overwriting of what they claim to be\n+        # capable of.\n+        if \"matchtimeseries\" in self.client.services[\"station\"]:\n+            arguments[\"matchtimeseries\"] = True\n+            if \"format\" in self.client.services[\"station\"]:\n+                arguments[\"format\"] = \"text\"\n+            self.is_availability_reliable = True\n+        else:\n+            if \"format\" in self.client.services[\"station\"]:\n+                arguments[\"format\"] = \"text\"\n+            self.is_availability_reliable = False\n+\n+        if self.is_availability_reliable:\n+            self.logger.info(\"Client '%s' - Requesting reliable \"\n+                             \"availability.\" % self.client_name)\n+        else:\n+            self.logger.info(\n+                \"Client '%s' - Requesting unreliable availability.\" %\n+                self.client_name)\n+\n+        try:\n+            start = time.time()\n+            inv = self.client.get_stations(**arguments)\n+            end = time.time()\n+        except utils.ERRORS as e:\n+            if \"no data available\" in str(e).lower():\n+                self.logger.info(\n+                    \"Client '%s' - No data available for request.\" %\n+                    self.client_name)\n+                return\n+            self.logger.error(\n+                \"Client '{0}' - Failed getting availability: %s\".format(\n+                    self.client_name), str(e))\n+            return\n+        self.logger.info(\"Client '%s' - Successfully requested availability \"\n+                         \"(%.2f seconds)\" % (self.client_name, end - start))\n+\n+        # Get the time intervals from the restrictions.\n+        intervals = [TimeInterval(start=_i[0], end=_i[1])\n+                     for _i in self.restrictions]\n+\n+        for network in inv:\n+            for station in network:\n+                # Skip the station if it is not in the desired domain.\n+                if needs_filtering is True and \\","path":"obspy/clients/fdsn/mass_downloader/download_helpers.py","position":1088,"original_position":1088,"commit_id":"3cbc27334ee86710ec3fdb52ff02e298ef326f16","original_commit_id":"3cbc27334ee86710ec3fdb52ff02e298ef326f16","user":{"login":"QuLogic","id":302469,"avatar_url":"https://avatars.githubusercontent.com/u/302469?v=3","gravatar_id":"","url":"https://api.github.com/users/QuLogic","html_url":"https://github.com/QuLogic","followers_url":"https://api.github.com/users/QuLogic/followers","following_url":"https://api.github.com/users/QuLogic/following{/other_user}","gists_url":"https://api.github.com/users/QuLogic/gists{/gist_id}","starred_url":"https://api.github.com/users/QuLogic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/QuLogic/subscriptions","organizations_url":"https://api.github.com/users/QuLogic/orgs","repos_url":"https://api.github.com/users/QuLogic/repos","events_url":"https://api.github.com/users/QuLogic/events{/privacy}","received_events_url":"https://api.github.com/users/QuLogic/received_events","type":"User","site_admin":false},"body":"This `is True` is not needed.","created_at":"2015-10-08T02:58:01Z","updated_at":"2015-10-08T02:58:01Z","html_url":"https://github.com/obspy/obspy/pull/1131#discussion_r41471836","pull_request_url":"https://api.github.com/repos/obspy/obspy/pulls/1131","_links":{"self":{"href":"https://api.github.com/repos/obspy/obspy/pulls/comments/41471836"},"html":{"href":"https://github.com/obspy/obspy/pull/1131#discussion_r41471836"},"pull_request":{"href":"https://api.github.com/repos/obspy/obspy/pulls/1131"}}},"pull_request":{"url":"https://api.github.com/repos/obspy/obspy/pulls/1131","id":46141826,"html_url":"https://github.com/obspy/obspy/pull/1131","diff_url":"https://github.com/obspy/obspy/pull/1131.diff","patch_url":"https://github.com/obspy/obspy/pull/1131.patch","issue_url":"https://api.github.com/repos/obspy/obspy/issues/1131","number":1131,"state":"open","locked":false,"title":"Mass Downloader for the FDSN Web Services","user":{"login":"krischer","id":800487,"avatar_url":"https://avatars.githubusercontent.com/u/800487?v=3","gravatar_id":"","url":"https://api.github.com/users/krischer","html_url":"https://github.com/krischer","followers_url":"https://api.github.com/users/krischer/followers","following_url":"https://api.github.com/users/krischer/following{/other_user}","gists_url":"https://api.github.com/users/krischer/gists{/gist_id}","starred_url":"https://api.github.com/users/krischer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/krischer/subscriptions","organizations_url":"https://api.github.com/users/krischer/orgs","repos_url":"https://api.github.com/users/krischer/repos","events_url":"https://api.github.com/users/krischer/events{/privacy}","received_events_url":"https://api.github.com/users/krischer/received_events","type":"User","site_admin":false},"body":"Hi guys,\r\n\r\nfirst of all I want to apologize that this is so big but I have been working on (an using) this for quite a while and it kind of grew and it would be a real pain to split it up in smaller pull requests.\r\n\r\nThis PR adds a couple of things:\r\n\r\n* Support for requesting gzipped data for XML and text requests from the FDSN web services. This results in much smaller and thus faster requests. If a service supports it it will be used.\r\n* Support for reading the `text` output of the FDSN `station` web services. It contains a limited amount of information compared to the full StationXML but once again is much faster.\r\n* A number of stability improvements to `obspy.clients.fdsn`.\r\n* Helper functionality to download large amount of data across a number of FDSN web services. This is the bulk of the PR and I'll write a bit more about it here.\r\n\r\nIt is pretty well documented and test coverage is also quite alright considering what is does (lots of mock tests and other tricks).\r\n\r\n\r\nRendered Documentation + Motivation (please read it for more information - I don't want to repeat everything here):\r\n\r\n* https://www.geophysik.uni-muenchen.de/~krischer/_build/packages/autogen/obspy.clients.fdsn.mass_downloader.html#module-obspy.clients.fdsn.mass_downloader\r\n* https://www.geophysik.uni-muenchen.de/~krischer/_build/packages/autogen/obspy.clients.fdsn.mass_downloader.domain.Domain.html#obspy.clients.fdsn.mass_downloader.domain.Domain\r\n* https://www.geophysik.uni-muenchen.de/~krischer/_build/packages/autogen/obspy.clients.fdsn.mass_downloader.restrictions.Restrictions.html#obspy.clients.fdsn.mass_downloader.restrictions.Restrictions\r\n\r\nAn important point is that the whole thing is usable as a library so it can be embedded into larger projects quite easily. Thus I think it has a place in ObsPy.\r\n\r\nThe mass downloader solves lots of real world issue when attempting to download data across FDSN data centers. I realize there are plans from data centers and organizations to make these things cleaner and easier in the future but this can be used right now independent of data center politics and what not.\r\n\r\nIt tries to make things as easy as possible for users but also data centers. It downloads in parallel, merges, batches, and splits requests and data, has some QC, ...\r\n\r\nExample usage:\r\n\r\n```python\r\nimport obspy\r\nfrom obspy.clients.fdsn.mass_downloader import CircularDomain, \\\r\n    Restrictions, MassDownloader\r\n\r\norigin_time = obspy.UTCDateTime(2011, 3, 11, 5, 47, 32)\r\n\r\n# Circular domain around the epicenter. This will download all data between\r\n# 70 and 90 degrees distance from the epicenter. This module also offers\r\n# rectangular and global domains. More complex domains can be defined by\r\n# inheriting from the Domain class.\r\ndomain = CircularDomain(latitude=37.52, longitude=143.04,\r\n                        minradius=70.0, maxradius=90.0)\r\n\r\nrestrictions = Restrictions(\r\n    # Get data from 5 minutes before the event to one hours after the\r\n    # event. This defines the temporal bounds of the waveform data.\r\n    starttime=origin_time - 5 * 60,\r\n    endtime=origin_time + 3600,\r\n    # You might not want to deal with gaps in the data. If this settings is\r\n    # True, any trace with a gap/overlap will be discarded.\r\n    reject_channels_with_gaps=True,\r\n    # And you might only want waveforms that have data for at least 95 % of\r\n    # the requested time span. Any trace that is shorter than 95 % of the\r\n    # desired total duration will be discarded.\r\n    minimum_length=0.95,\r\n    # No two stations should be closer than 10 km to each other. This is\r\n    # useful to for example filter out stations that are part of different\r\n    # networks but at the same physical station. Settings this to zero or\r\n    # None will disable that filtering.\r\n    minimum_interstation_distance_in_m=10E3,\r\n    # Only HH or BH channels. If a station has HH channels, those will be\r\n    # downloaded, otherwise the BH. Nothing will be downloaded if it has\r\n    # neither. You can add more/less patterns if you like.\r\n    channel_priorities=(\"HH[Z,N,E]\", \"BH[Z,N,E]\"),\r\n    # Locations codes are arbitrary and there is no rule which location is\r\n    # best. Same logic as for the previous setting.\r\n    location_priorities=(\"\", \"00\", \"10\"))\r\n\r\n# No specified providers will result in all known ones being queried.\r\nmdl = MassDownloader()\r\n# The data will be downloaded to ``./waveforms/`` ans ``./stations`` with\r\n# autmatically chosen names.\r\nmdl.download(domain, restrictions, mseed_storage=\"waveforms\",\r\n             stationxml_storage=\"stations\")\r\n```","created_at":"2015-09-28T10:36:04Z","updated_at":"2015-10-08T02:58:01Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a536d350cf47949348976c005077ca5db3984256","assignee":null,"milestone":{"url":"https://api.github.com/repos/obspy/obspy/milestones/8","html_url":"https://github.com/obspy/obspy/milestones/0.11.0","labels_url":"https://api.github.com/repos/obspy/obspy/milestones/8/labels","id":725009,"number":8,"title":"0.11.0","description":"","creator":{"login":"barsch","id":602450,"avatar_url":"https://avatars.githubusercontent.com/u/602450?v=3","gravatar_id":"","url":"https://api.github.com/users/barsch","html_url":"https://github.com/barsch","followers_url":"https://api.github.com/users/barsch/followers","following_url":"https://api.github.com/users/barsch/following{/other_user}","gists_url":"https://api.github.com/users/barsch/gists{/gist_id}","starred_url":"https://api.github.com/users/barsch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/barsch/subscriptions","organizations_url":"https://api.github.com/users/barsch/orgs","repos_url":"https://api.github.com/users/barsch/repos","events_url":"https://api.github.com/users/barsch/events{/privacy}","received_events_url":"https://api.github.com/users/barsch/received_events","type":"User","site_admin":false},"open_issues":34,"closed_issues":32,"state":"open","created_at":"2014-07-18T08:42:03Z","updated_at":"2015-09-28T13:14:26Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/obspy/obspy/pulls/1131/commits","review_comments_url":"https://api.github.com/repos/obspy/obspy/pulls/1131/comments","review_comment_url":"https://api.github.com/repos/obspy/obspy/pulls/comments{/number}","comments_url":"https://api.github.com/repos/obspy/obspy/issues/1131/comments","statuses_url":"https://api.github.com/repos/obspy/obspy/statuses/3cbc27334ee86710ec3fdb52ff02e298ef326f16","head":{"label":"krischer:mass_downloader","ref":"mass_downloader","sha":"3cbc27334ee86710ec3fdb52ff02e298ef326f16","user":{"login":"krischer","id":800487,"avatar_url":"https://avatars.githubusercontent.com/u/800487?v=3","gravatar_id":"","url":"https://api.github.com/users/krischer","html_url":"https://github.com/krischer","followers_url":"https://api.github.com/users/krischer/followers","following_url":"https://api.github.com/users/krischer/following{/other_user}","gists_url":"https://api.github.com/users/krischer/gists{/gist_id}","starred_url":"https://api.github.com/users/krischer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/krischer/subscriptions","organizations_url":"https://api.github.com/users/krischer/orgs","repos_url":"https://api.github.com/users/krischer/repos","events_url":"https://api.github.com/users/krischer/events{/privacy}","received_events_url":"https://api.github.com/users/krischer/received_events","type":"User","site_admin":false},"repo":{"id":5830460,"name":"obspy","full_name":"krischer/obspy","owner":{"login":"krischer","id":800487,"avatar_url":"https://avatars.githubusercontent.com/u/800487?v=3","gravatar_id":"","url":"https://api.github.com/users/krischer","html_url":"https://github.com/krischer","followers_url":"https://api.github.com/users/krischer/followers","following_url":"https://api.github.com/users/krischer/following{/other_user}","gists_url":"https://api.github.com/users/krischer/gists{/gist_id}","starred_url":"https://api.github.com/users/krischer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/krischer/subscriptions","organizations_url":"https://api.github.com/users/krischer/orgs","repos_url":"https://api.github.com/users/krischer/repos","events_url":"https://api.github.com/users/krischer/events{/privacy}","received_events_url":"https://api.github.com/users/krischer/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/krischer/obspy","description":"ObsPy: A Python Toolbox for seismology/seismological observatories.","fork":true,"url":"https://api.github.com/repos/krischer/obspy","forks_url":"https://api.github.com/repos/krischer/obspy/forks","keys_url":"https://api.github.com/repos/krischer/obspy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/krischer/obspy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/krischer/obspy/teams","hooks_url":"https://api.github.com/repos/krischer/obspy/hooks","issue_events_url":"https://api.github.com/repos/krischer/obspy/issues/events{/number}","events_url":"https://api.github.com/repos/krischer/obspy/events","assignees_url":"https://api.github.com/repos/krischer/obspy/assignees{/user}","branches_url":"https://api.github.com/repos/krischer/obspy/branches{/branch}","tags_url":"https://api.github.com/repos/krischer/obspy/tags","blobs_url":"https://api.github.com/repos/krischer/obspy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/krischer/obspy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/krischer/obspy/git/refs{/sha}","trees_url":"https://api.github.com/repos/krischer/obspy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/krischer/obspy/statuses/{sha}","languages_url":"https://api.github.com/repos/krischer/obspy/languages","stargazers_url":"https://api.github.com/repos/krischer/obspy/stargazers","contributors_url":"https://api.github.com/repos/krischer/obspy/contributors","subscribers_url":"https://api.github.com/repos/krischer/obspy/subscribers","subscription_url":"https://api.github.com/repos/krischer/obspy/subscription","commits_url":"https://api.github.com/repos/krischer/obspy/commits{/sha}","git_commits_url":"https://api.github.com/repos/krischer/obspy/git/commits{/sha}","comments_url":"https://api.github.com/repos/krischer/obspy/comments{/number}","issue_comment_url":"https://api.github.com/repos/krischer/obspy/issues/comments{/number}","contents_url":"https://api.github.com/repos/krischer/obspy/contents/{+path}","compare_url":"https://api.github.com/repos/krischer/obspy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/krischer/obspy/merges","archive_url":"https://api.github.com/repos/krischer/obspy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/krischer/obspy/downloads","issues_url":"https://api.github.com/repos/krischer/obspy/issues{/number}","pulls_url":"https://api.github.com/repos/krischer/obspy/pulls{/number}","milestones_url":"https://api.github.com/repos/krischer/obspy/milestones{/number}","notifications_url":"https://api.github.com/repos/krischer/obspy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/krischer/obspy/labels{/name}","releases_url":"https://api.github.com/repos/krischer/obspy/releases{/id}","created_at":"2012-09-16T15:34:42Z","updated_at":"2014-11-20T12:15:18Z","pushed_at":"2015-10-02T08:09:11Z","git_url":"git://github.com/krischer/obspy.git","ssh_url":"git@github.com:krischer/obspy.git","clone_url":"https://github.com/krischer/obspy.git","svn_url":"https://github.com/krischer/obspy","homepage":"https://github.com/obspy/obspy/wiki","size":66739,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":0,"forks":2,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"obspy:master","ref":"master","sha":"1804da1156038af5c68f2392bfbe394defc2d530","user":{"login":"obspy","id":1701221,"avatar_url":"https://avatars.githubusercontent.com/u/1701221?v=3","gravatar_id":"","url":"https://api.github.com/users/obspy","html_url":"https://github.com/obspy","followers_url":"https://api.github.com/users/obspy/followers","following_url":"https://api.github.com/users/obspy/following{/other_user}","gists_url":"https://api.github.com/users/obspy/gists{/gist_id}","starred_url":"https://api.github.com/users/obspy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/obspy/subscriptions","organizations_url":"https://api.github.com/users/obspy/orgs","repos_url":"https://api.github.com/users/obspy/repos","events_url":"https://api.github.com/users/obspy/events{/privacy}","received_events_url":"https://api.github.com/users/obspy/received_events","type":"Organization","site_admin":false},"repo":{"id":5731348,"name":"obspy","full_name":"obspy/obspy","owner":{"login":"obspy","id":1701221,"avatar_url":"https://avatars.githubusercontent.com/u/1701221?v=3","gravatar_id":"","url":"https://api.github.com/users/obspy","html_url":"https://github.com/obspy","followers_url":"https://api.github.com/users/obspy/followers","following_url":"https://api.github.com/users/obspy/following{/other_user}","gists_url":"https://api.github.com/users/obspy/gists{/gist_id}","starred_url":"https://api.github.com/users/obspy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/obspy/subscriptions","organizations_url":"https://api.github.com/users/obspy/orgs","repos_url":"https://api.github.com/users/obspy/repos","events_url":"https://api.github.com/users/obspy/events{/privacy}","received_events_url":"https://api.github.com/users/obspy/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/obspy/obspy","description":"ObsPy: A Python Toolbox for seismology/seismological observatories.","fork":false,"url":"https://api.github.com/repos/obspy/obspy","forks_url":"https://api.github.com/repos/obspy/obspy/forks","keys_url":"https://api.github.com/repos/obspy/obspy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/obspy/obspy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/obspy/obspy/teams","hooks_url":"https://api.github.com/repos/obspy/obspy/hooks","issue_events_url":"https://api.github.com/repos/obspy/obspy/issues/events{/number}","events_url":"https://api.github.com/repos/obspy/obspy/events","assignees_url":"https://api.github.com/repos/obspy/obspy/assignees{/user}","branches_url":"https://api.github.com/repos/obspy/obspy/branches{/branch}","tags_url":"https://api.github.com/repos/obspy/obspy/tags","blobs_url":"https://api.github.com/repos/obspy/obspy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/obspy/obspy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/obspy/obspy/git/refs{/sha}","trees_url":"https://api.github.com/repos/obspy/obspy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/obspy/obspy/statuses/{sha}","languages_url":"https://api.github.com/repos/obspy/obspy/languages","stargazers_url":"https://api.github.com/repos/obspy/obspy/stargazers","contributors_url":"https://api.github.com/repos/obspy/obspy/contributors","subscribers_url":"https://api.github.com/repos/obspy/obspy/subscribers","subscription_url":"https://api.github.com/repos/obspy/obspy/subscription","commits_url":"https://api.github.com/repos/obspy/obspy/commits{/sha}","git_commits_url":"https://api.github.com/repos/obspy/obspy/git/commits{/sha}","comments_url":"https://api.github.com/repos/obspy/obspy/comments{/number}","issue_comment_url":"https://api.github.com/repos/obspy/obspy/issues/comments{/number}","contents_url":"https://api.github.com/repos/obspy/obspy/contents/{+path}","compare_url":"https://api.github.com/repos/obspy/obspy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/obspy/obspy/merges","archive_url":"https://api.github.com/repos/obspy/obspy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/obspy/obspy/downloads","issues_url":"https://api.github.com/repos/obspy/obspy/issues{/number}","pulls_url":"https://api.github.com/repos/obspy/obspy/pulls{/number}","milestones_url":"https://api.github.com/repos/obspy/obspy/milestones{/number}","notifications_url":"https://api.github.com/repos/obspy/obspy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/obspy/obspy/labels{/name}","releases_url":"https://api.github.com/repos/obspy/obspy/releases{/id}","created_at":"2012-09-08T19:17:46Z","updated_at":"2015-10-07T15:46:19Z","pushed_at":"2015-10-07T12:51:55Z","git_url":"git://github.com/obspy/obspy.git","ssh_url":"git@github.com:obspy/obspy.git","clone_url":"https://github.com/obspy/obspy.git","svn_url":"https://github.com/obspy/obspy","homepage":"http://www.obspy.org","size":86795,"stargazers_count":152,"watchers_count":152,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":108,"mirror_url":null,"open_issues_count":99,"forks":108,"open_issues":99,"watchers":152,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/obspy/obspy/pulls/1131"},"html":{"href":"https://github.com/obspy/obspy/pull/1131"},"issue":{"href":"https://api.github.com/repos/obspy/obspy/issues/1131"},"comments":{"href":"https://api.github.com/repos/obspy/obspy/issues/1131/comments"},"review_comments":{"href":"https://api.github.com/repos/obspy/obspy/pulls/1131/comments"},"review_comment":{"href":"https://api.github.com/repos/obspy/obspy/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/obspy/obspy/pulls/1131/commits"},"statuses":{"href":"https://api.github.com/repos/obspy/obspy/statuses/3cbc27334ee86710ec3fdb52ff02e298ef326f16"}}}}