Browse Source

fdsn, iris: overhaul, improved docs

release
Sebastian Heimann 8 months ago
parent
commit
ff802c5f18
7 changed files with 433 additions and 94 deletions
  1. +2
    -15
      doc/source/library/reference/client/catalog.rst
  2. +3
    -4
      doc/source/library/reference/client/waveform.rst
  3. +0
    -1
      doc/source/library/reference/io/quakeml.rst
  4. +5
    -6
      src/client/README.md
  5. +380
    -28
      src/client/fdsn.py
  6. +17
    -40
      src/client/iris.py
  7. +26
    -0
      test/base/test_fdsn.py

+ 2
- 15
doc/source/library/reference/client/catalog.rst View File

@ -1,9 +1,8 @@
Earthquake Catalogs (``client.catalog``)
========================================
Online earthquake catalogs (``client.catalog``)
===============================================
.. automodule :: pyrocko.client.catalog
:members:
:undoc-members:
``client.catalog.Geofon``
@ -12,7 +11,6 @@ Earthquake Catalogs (``client.catalog``)
.. autoclass :: pyrocko.client.catalog.Geofon
:show-inheritance:
:members:
:undoc-members:
``client.catalog.GlobalCMT``
----------------------------
@ -20,7 +18,6 @@ Earthquake Catalogs (``client.catalog``)
.. autoclass :: pyrocko.client.catalog.GlobalCMT
:show-inheritance:
:members:
:undoc-members:
``client.catalog.USGS``
-----------------------
@ -28,15 +25,6 @@ Earthquake Catalogs (``client.catalog``)
.. autoclass :: pyrocko.client.catalog.USGS
:show-inheritance:
:members:
:undoc-members:
``client.catalog.Kinherd``
--------------------------
.. autoclass :: pyrocko.client.catalog.Kinherd
:show-inheritance:
:members:
:undoc-members:
``client.catalog.Saxony``
-------------------------
@ -44,4 +32,3 @@ Earthquake Catalogs (``client.catalog``)
.. autoclass :: pyrocko.client.catalog.Saxony
:show-inheritance:
:members:
:undoc-members:

+ 3
- 4
doc/source/library/reference/client/waveform.rst View File

@ -1,12 +1,12 @@
Online Waveform Archives (``client``)
=====================================
Online waveform and station metadata archives (``client``)
==========================================================
``client.fdsn``
---------------
.. automodule :: pyrocko.client.fdsn
:members:
:undoc-members:
:show-inheritance:
``client.iris``
@ -14,4 +14,3 @@ Online Waveform Archives (``client``)
.. automodule:: pyrocko.client.iris
:members:
:undoc-members:

+ 0
- 1
doc/source/library/reference/io/quakeml.rst View File

@ -3,5 +3,4 @@
.. automodule:: pyrocko.io.quakeml
:members:
:undoc-members:

+ 5
- 6
src/client/README.md View File

@ -1,18 +1,17 @@
# Pyrocko Client Submodule
# Pyrocko client subpackage
The pyrocko software package comes with several clients for requesting online earthquake catalogs and waveform archives.
The Pyrocko software package comes with several clients for requesting online earthquake catalogs and waveform archives.
## Waveform Access
## Waveform access
* `pyrocko.client.iris` Gives access to waveform data from the IRIS archive (http://service.iris.edu/).
* `pyrocko.client.fdsn` Is a client for FDSN web services (http://www.fdsn.org/).
* `pyrocko.client.iris` Gives access to waveform data from the IRIS archive (http://service.iris.edu/).
## Earthquake Catalogs
## Earthquake catalogs
* `pyrocko.client.catalog.Geofon` Access the Geofon earthquake catalog (http://geofon.gfz-potsdam.de/)
* `pyrocko.client.catalog.GlobalCMT` Query the USGS earthquake catalog (https://earthquake.usgs.gov/)
* `pyrocko.client.catalog.USGS` Get earhquake events from the Global CMT catalog (http://www.globalcmt.org/)
* `pyrocko.client.catalog.Kinherd` Kinherd's earthquake catalog at (http://kinherd.org/quakes/)
* `pyrocko.client.catalog.Saxony` Regional Catalog of Saxony, Germany from the University of Leipzig (http://home.uni-leipzig.de/collm/auswertung_temp.html)
(Also accessible through `pyrocko.client.catalog`)

+ 380
- 28
src/client/fdsn.py View File

@ -2,6 +2,29 @@
#
# The Pyrocko Developers, 21st Century
# ---|P------/S----------~Lg----------
'''
Low-level FDSN web service client.
This module provides basic functionality to download station metadata, time
series data and event information from FDSN web services. Password and token
authentication are supported. Query responses are returned as open file-like
objects or can be parsed into Pyrocko's native data structures where
appropriate.
.. _registered-site-names:
Registered site names
.....................
A list of known FDSN site names is maintained within the module for quick
selection by the user. This list currently contains the following sites:
%s
Any other site can be specified by providing its full URL.
'''
from __future__ import absolute_import
import re
@ -76,6 +99,30 @@ g_default_query_args = {
'nodata', 'mergegaps', 'show'}}
def doc_escape_slist(l):
return ', '.join("``'%s'``" % s for s in l)
def doc_table_dict(d, khead, vhead, indent=''):
keys, vals = zip(*sorted(d.items()))
lk = max(max(len(k) for k in keys), len(khead))
lv = max(max(len(v) for v in vals), len(vhead))
hr = '=' * lk + ' ' + '=' * lv
lines = [
hr,
'%s %s' % (khead.ljust(lk), vhead.ljust(lv)),
hr]
for k, v in zip(keys, vals):
lines.append('%s %s' % (k.ljust(lk), v.ljust(lv)))
lines.append(hr)
return '\n'.join(indent + line for line in lines)
def strip_html(s):
s = s.decode('utf-8')
s = re.sub(r'<[^>]+>', '', s)
@ -104,10 +151,16 @@ re_realm_from_auth_header = re.compile(r'(realm)\s*[:=]\s*"([^"]*)"?')
class CannotGetRealmFromAuthHeader(DownloadError):
'''
Raised when failing to parse server response during authentication.
'''
pass
class CannotGetCredentialsFromAuthRequest(DownloadError):
'''
Raised when failing to parse server response during token authentication.
'''
pass
@ -126,6 +179,9 @@ def sdatetime(t):
class EmptyResult(DownloadError):
'''
Raised when an empty server response is retrieved.
'''
def __init__(self, url):
DownloadError.__init__(self)
self._url = url
@ -135,6 +191,9 @@ class EmptyResult(DownloadError):
class RequestEntityTooLarge(DownloadError):
'''
Raised when the server indicates that too much data was requested.
'''
def __init__(self, url):
DownloadError.__init__(self)
self._url = url
@ -144,10 +203,16 @@ class RequestEntityTooLarge(DownloadError):
class InvalidRequest(DownloadError):
'''
Raised when an invalid request would be sent / has been sent.
'''
pass
class Timeout(DownloadError):
'''
Raised when the server does not respond within the allowed timeout period.
'''
pass
@ -304,12 +369,57 @@ def station(
site=g_default_site,
url=g_url,
majorversion=1,
parsed=True,
selection=None,
timeout=g_timeout,
check=True,
selection=None,
parsed=True,
**kwargs):
'''
Query FDSN web service for station metadata.
:param site:
:ref:`Registered site name <registered-site-names>` or full base URL of
the service (e.g. ``'https://geofon.gfz-potsdam.de'``).
:type site: str, optional
:param url:
URL template (default should work in 99% of cases).
:type url: str, optional
:param majorversion:
Major version of the service to query (always ``1`` at the time of
writing).
:type majorversion: int, optional
:param timeout:
Network timeout in [s]. Global default timeout can be configured in
Pyrocko's configuration file under ``fdsn_timeout``.
:type timeout: float, optional
:param check:
If ``True`` arguments are checked against self-description (WADL) of
the queried web service if available or FDSN specification.
:type check: bool, optional
:param selection:
If given, selection to be queried as a list of tuples
``(network, station, location, channel, tmin, tmax)``. Useful for
detailed queries.
:type selection: list of tuples, optional
:param parsed:
If ``True`` parse received content into
:py:class:`~pyrocko.io.stationxml.FDSNStationXML`
object, otherwise return open file handle to raw data stream.
:type parsed: bool, optional
:param \\*\\*kwargs:
Parameters passed to the server (see `FDSN web services specification
<https://www.fdsn.org/webservices>`_).
:returns:
See description of ``parsed`` argument above.
:raises:
On failure, :py:exc:`~pyrocko.util.DownloadError` or one of its
sub-types defined in the :py:mod:`~pyrocko.client.fdsn` module is
raised.
'''
service = 'station'
if check:
@ -345,7 +455,6 @@ def station(
'level="channel" is required')
elif format == 'xml':
assert kwargs.get('format', 'xml') == 'xml'
return stationxml.load_xml(
stream=_request(url, timeout=timeout, **params))
else:
@ -372,14 +481,55 @@ def dataselect(
site=g_default_site,
url=g_url,
majorversion=1,
selection=None,
timeout=g_timeout,
check=True,
user=None,
passwd=None,
token=None,
timeout=g_timeout,
check=True,
selection=None,
**kwargs):
'''
Query FDSN web service for time series data in miniSEED format.
:param site:
:ref:`Registered site name <registered-site-names>` or full base URL of
the service (e.g. ``'https://geofon.gfz-potsdam.de'``).
:type site: str, optional
:param url:
URL template (default should work in 99% of cases).
:type url: str, optional
:param majorversion:
Major version of the service to query (always ``1`` at the time of
writing).
:type majorversion: int, optional
:param timeout:
Network timeout in [s]. Global default timeout can be configured in
Pyrocko's configuration file under ``fdsn_timeout``.
:type timeout: float, optional
:param check:
If ``True`` arguments are checked against self-description (WADL) of
the queried web service if available or FDSN specification.
:type check: bool, optional
:param user: User name for user/password authentication.
:type user: str, optional
:param passwd: Password for user/password authentication.
:type passwd: str, optional
:param token: Token for `token authentication
<https://geofon.gfz-potsdam.de/waveform/archive/auth/auth-overview.php>`_.
:type token: str, optional
:param selection:
If given, selection to be queried as a list of tuples
``(network, station, location, channel, tmin, tmax)``.
:type selection: list of tuples, optional
:param \\*\\*kwargs:
Parameters passed to the server (see `FDSN web services specification
<https://www.fdsn.org/webservices>`_).
:returns:
Open file-like object providing raw miniSEED data.
'''
service = 'dataselect'
if user or token:
@ -422,19 +572,58 @@ def event(
site=g_default_site,
url=g_url,
majorversion=1,
timeout=g_timeout,
check=True,
user=None,
passwd=None,
token=None,
timeout=g_timeout,
check=True,
parsed=False,
**kwargs):
'''Query FDSN web service for events.
'''
Query FDSN web service for events.
:param site:
:ref:`Registered site name <registered-site-names>` or full base URL of
the service (e.g. ``'https://geofon.gfz-potsdam.de'``).
:type site: str, optional
:param url:
URL template (default should work in 99% of cases).
:type url: str, optional
:param majorversion:
Major version of the service to query (always ``1`` at the time of
writing).
:type majorversion: int, optional
:param timeout:
Network timeout in [s]. Global default timeout can be configured in
Pyrocko's configuration file under ``fdsn_timeout``.
:type timeout: float, optional
:param check:
If ``True`` arguments are checked against self-description (WADL) of
the queried web service if available or FDSN specification.
:type check: bool, optional
:param user: User name for user/password authentication.
:type user: str, optional
:param passwd: Password for user/password authentication.
:type passwd: str, optional
:param token: Token for `token authentication
<https://geofon.gfz-potsdam.de/waveform/archive/auth/auth-overview.php>`_.
:type token: str, optional
:param parsed:
If ``True`` parse received content into
:py:class:`~pyrocko.io.quakeml.QuakeML`
object, otherwise return open file handle to raw data stream. Note:
by default unparsed data is retrieved, differently from the default
behaviour of :py:func:`station` (for backward compatibility).
:type parsed: bool, optional
:param \\*\\*kwargs:
Parameters passed to the server (see `FDSN web services specification
<https://www.fdsn.org/webservices>`_).
:returns:
See description of ``parsed`` argument above.
On success, will return a list of events in QuakeML format.
Check the documentation of FDSN for allowed arguments:
https://www.fdsn.org/webservices
'''
service = 'event'
@ -454,7 +643,19 @@ def event(
params = fix_params(kwargs)
url = fillurl(service, site, url, majorversion, method)
return _request(url, user=user, passwd=passwd, timeout=timeout, **params)
fh = _request(url, user=user, passwd=passwd, timeout=timeout, **params)
if parsed:
from pyrocko.io import quakeml
format = kwargs.get('format', 'xml')
if format != 'xml':
raise InvalidRequest(
'If parsed=True is selected, format="xml" must be selected.')
return quakeml.load_xml(stream=fh)
else:
return fh
def availability(
@ -462,13 +663,56 @@ def availability(
site=g_default_site,
url=g_url,
majorversion=1,
timeout=g_timeout,
check=True,
user=None,
passwd=None,
token=None,
timeout=g_timeout,
check=True,
selection=None,
**kwargs):
'''
Query FDSN web service for time series data availablity.
:param method: Availablility method to call: ``'query'``, or ``'extent'``.
:param site:
:ref:`Registered site name <registered-site-names>` or full base URL of
the service (e.g. ``'https://geofon.gfz-potsdam.de'``).
:type site: str, optional
:param url:
URL template (default should work in 99% of cases).
:type url: str, optional
:param majorversion:
Major version of the service to query (always ``1`` at the time of
writing).
:type majorversion: int, optional
:param timeout:
Network timeout in [s]. Global default timeout can be configured in
Pyrocko's configuration file under ``fdsn_timeout``.
:type timeout: float, optional
:param check:
If ``True`` arguments are checked against self-description (WADL) of
the queried web service if available or FDSN specification.
:type check: bool, optional
:param user: User name for user/password authentication.
:type user: str, optional
:param passwd: Password for user/password authentication.
:type passwd: str, optional
:param token: Token for `token authentication
<https://geofon.gfz-potsdam.de/waveform/archive/auth/auth-overview.php>`_.
:type token: str, optional
:param selection:
If given, selection to be queried as a list of tuples
``(network, station, location, channel, tmin, tmax)``.
:type selection: list of tuples, optional
:param \\*\\*kwargs:
Parameters passed to the server (see `FDSN web services specification
<https://www.fdsn.org/webservices>`_).
:returns:
Open file-like object providing raw response.
'''
service = 'availability'
assert method in ('query', 'extent')
@ -486,7 +730,25 @@ def availability(
params = fix_params(kwargs)
url = fillurl(service, site, url, majorversion, method)
return _request(url, user=user, passwd=passwd, timeout=timeout, **params)
if selection:
lst = []
for k, v in params.items():
lst.append('%s=%s' % (k, v))
for (network, station, location, channel, tmin, tmax) in selection:
if location == '':
location = '--'
lst.append(' '.join((network, station, location, channel,
sdatetime(tmin), sdatetime(tmax))))
post = '\n'.join(lst)
return _request(
url, user=user, passwd=passwd, post=post.encode(), timeout=timeout)
else:
return _request(
url, user=user, passwd=passwd, timeout=timeout, **params)
def check_params(
@ -498,6 +760,37 @@ def check_params(
method='query',
**kwargs):
'''
Check query parameters against self-description of web service.
Downloads WADL description of the given service and site and checks
parameters if they are available. Queried WADLs are cached in memory.
:param service: ``'station'``, ``'dataselect'``, ``'event'`` or
``'availability'``.
:param site:
:ref:`Registered site name <registered-site-names>` or full base URL of
the service (e.g. ``'https://geofon.gfz-potsdam.de'``).
:type site: str, optional
:param url:
URL template (default should work in 99% of cases).
:type url: str, optional
:param majorversion:
Major version of the service to query (always ``1`` at the time of
writing).
:type majorversion: int, optional
:param timeout:
Network timeout in [s]. Global default timeout can be configured in
Pyrocko's configuration file under ``fdsn_timeout``.
:type timeout: float, optional
:param \\*\\*kwargs:
Parameters that would be passed to the server (see `FDSN web services
specification <https://www.fdsn.org/webservices>`_).
:raises: :py:exc:`ValueError` is raised if unsupported parameters are
encountered.
'''
avail = supported_params_wadl(
service, site, url, majorversion, timeout, method)
@ -519,6 +812,34 @@ def supported_params_wadl(
timeout=g_timeout,
method='query'):
'''
Get query parameter names supported by a given FDSN site and service.
If no WADL is provided by the queried service, default parameter sets from
the FDSN standard are returned. Queried WADLs are cached in memory.
:param service: ``'station'``, ``'dataselect'``, ``'event'`` or
``'availability'``.
:param site:
:ref:`Registered site name <registered-site-names>` or full base URL of
the service (e.g. ``'https://geofon.gfz-potsdam.de'``).
:type site: str, optional
:param url:
URL template (default should work in 99% of cases).
:type url: str, optional
:param majorversion:
Major version of the service to query (always ``1`` at the time of
writing).
:type majorversion: int, optional
:param timeout:
Network timeout in [s]. Global default timeout can be configured in
Pyrocko's configuration file under ``fdsn_timeout``.
:type timeout: float, optional
:returns: Supported parameter names.
:rtype: set of str
'''
wadl = cached_wadl(service, site, url, majorversion, timeout)
if wadl:
@ -528,6 +849,42 @@ def supported_params_wadl(
return g_default_query_args[service]
def wadl(
service,
site=g_default_site,
url=g_url,
majorversion=1,
timeout=g_timeout):
'''
Retrieve self-description of a specific FDSN service.
:param service: ``'station'``, ``'dataselect'``, ``'event'`` or
``'availability'``.
:param site:
:ref:`Registered site name <registered-site-names>` or full base URL of
the service (e.g. ``'https://geofon.gfz-potsdam.de'``).
:type site: str, optional
:param url:
URL template (default should work in 99% of cases).
:type url: str, optional
:param majorversion:
Major version of the service to query (always ``1`` at the time of
writing).
:type majorversion: int, optional
:param timeout:
Network timeout in [s]. Global default timeout can be configured in
Pyrocko's configuration file under ``fdsn_timeout``.
:type timeout: float, optional
'''
from pyrocko.client.wadl import load_xml
url = fillurl(service, site, url, majorversion, 'application.wadl')
return load_xml(stream=_request(url, timeout=timeout))
g_wadls = {}
@ -538,6 +895,12 @@ def cached_wadl(
majorversion=1,
timeout=g_timeout):
'''
Get self-description of a specific FDSN service.
Same as :py:func:`wadl`, but results are cached in memory.
'''
k = (service, site, url, majorversion)
if k not in g_wadls:
try:
@ -556,18 +919,7 @@ def cached_wadl(
return g_wadls[k]
def wadl(
service,
site=g_default_site,
url=g_url,
majorversion=1,
timeout=g_timeout):
from pyrocko.client.wadl import load_xml
url = fillurl(service, site, url, majorversion, 'application.wadl')
return load_xml(stream=_request(url, timeout=timeout))
__doc__ %= doc_table_dict(g_site_abbr, 'Site name', 'URL', ' ')
if __name__ == '__main__':


+ 17
- 40
src/client/iris.py View File

@ -2,6 +2,11 @@
#
# The Pyrocko Developers, 21st Century
# ---|P------/S----------~Lg----------
'''
Access to some IRIS (non-FDSN) web services.
'''
from __future__ import absolute_import
@ -114,6 +119,9 @@ def xmlzip(source, watch, bufsize=10000):
class NotFound(Exception):
'''
Raised when the server sends an 404 response.
'''
def __init__(self, url):
Exception.__init__(self)
self._url = url
@ -143,21 +151,10 @@ def ws_request(url, post=False, **kwargs):
raise e
def ws_station(**kwargs):
for k in 'startbefore', 'startafter', 'endbefore', 'endafter':
if k in kwargs:
kwargs[k] = sdate(kwargs[k])
if 'timewindow' in kwargs:
tmin, tmax = kwargs.pop('timewindow')
kwargs['startbefore'] = sdate(tmin)
kwargs['endafter'] = sdate(tmax)
return ws_request(base_url + '/station/1/query', **kwargs)
def ws_virtualnetwork(**kwargs):
'''
Query IRIS virtualnetwork web service.
'''
for k in 'starttime', 'endtime':
if k in kwargs:
@ -171,32 +168,6 @@ def ws_virtualnetwork(**kwargs):
return ws_request(base_url + '/virtualnetwork/1/query', **kwargs)
def ws_bulkdataselect(
selection,
quality=None,
minimumlength=None,
longestonly=False):
sel = []
if quality is not None:
sel.append('quality %s' % quality)
if minimumlength is not None:
sel.append('minimumlength %s' % minimumlength)
if longestonly:
sel.append('longestonly')
for (network, station, location, channel, tmin, tmax) in selection:
if location == '':
location = '--'
sel.append(' '.join((
network, station, location, channel,
sdatetime(tmin), sdatetime(tmax))))
return ws_request(base_url + '/bulkdataselect/1/query',
post='\n'.join(sel))
def ws_sacpz(
network=None,
station=None,
@ -205,6 +176,9 @@ def ws_sacpz(
time=None,
tmin=None,
tmax=None):
'''
Query IRIS sacpz web service.
'''
d = {}
if network:
@ -235,6 +209,9 @@ def ws_resp(
time=None,
tmin=None,
tmax=None):
'''
Query IRIS resp web service.
'''
d = {}
if network:


+ 26
- 0
test/base/test_fdsn.py View File

@ -95,6 +95,32 @@ class FDSNTestCase(unittest.TestCase):
fdsn.dataselect(site='geofon', selection=selection)
fdsn.station(site='geofon', selection=selection, level='response')
@common.require_internet
@common.skip_on_download_error
def test_availability(self):
tmin = stt('2010-01-15 10:00:00')
tmax = stt('2010-01-15 10:01:00')
selection = [
('GE', 'EIL', '*', 'SHZ', tmin, tmax),
]
xx = fdsn.availability('query', site='iris', selection=selection)
assert xx.read().decode('utf8').splitlines()[1].startswith('GE')
xx = fdsn.availability('extent', site='iris', selection=selection)
assert xx.read().decode('utf8').splitlines()[1].startswith('GE')
@common.require_internet
@common.skip_on_download_error
def test_check_params(self):
for site in ['geofon']:
with self.assertRaises(ValueError):
fdsn.station(site=site,
network='GE',
station='EIL',
level='channel',
no_such_parameter=True)
def test_read_big(self):
for site in ['iris']:
fpath = common.test_data_file('%s_1014-01-01_all.xml' % site)


Loading…
Cancel
Save