Browse Source

squirrel: initial implementation

conda
Sebastian Heimann 8 years ago
parent
commit
307b5c4269
  1. 1
      doc/source/library/reference/index.rst
  2. 6
      doc/source/library/reference/squirrel/base.rst
  3. 6
      doc/source/library/reference/squirrel/client/index.rst
  4. 6
      doc/source/library/reference/squirrel/error.rst
  5. 15
      doc/source/library/reference/squirrel/index.rst
  6. 6
      doc/source/library/reference/squirrel/io/index.rst
  7. 6
      doc/source/library/reference/squirrel/model.rst
  8. 6
      doc/source/library/reference/squirrel/pile.rst
  9. 7
      setup.cfg
  10. 11
      setup.py
  11. 5
      src/__init__.py
  12. 11
      src/apps/squirrel.py
  13. 107
      src/get_terminal_size.py
  14. 185
      src/gui/pile_viewer.py
  15. 22
      src/guts.py
  16. 19
      src/io/io_common.py
  17. 2
      src/model/__init__.py
  18. 62
      src/model/content.py
  19. 33
      src/model/event.py
  20. 344
      src/progress.py
  21. 20
      src/squirrel/__init__.py
  22. 2823
      src/squirrel/base.py
  23. 84
      src/squirrel/cache.py
  24. 14
      src/squirrel/client/__init__.py
  25. 80
      src/squirrel/client/base.py
  26. 291
      src/squirrel/client/catalog.py
  27. 557
      src/squirrel/client/fdsn.py
  28. 90
      src/squirrel/environment.py
  29. 19
      src/squirrel/error.py
  30. 11
      src/squirrel/io/__init__.py
  31. 6
      src/squirrel/io/backends/__init__.py
  32. 51
      src/squirrel/io/backends/datacube.py
  33. 51
      src/squirrel/io/backends/mseed.py
  34. 143
      src/squirrel/io/backends/sac.py
  35. 108
      src/squirrel/io/backends/stationxml.py
  36. 207
      src/squirrel/io/backends/textfiles.py
  37. 73
      src/squirrel/io/backends/virtual.py
  38. 52
      src/squirrel/io/backends/yaml.py
  39. 328
      src/squirrel/io/base.py
  40. 21
      src/squirrel/lock.py
  41. 772
      src/squirrel/model.py
  42. 354
      src/squirrel/pile.py
  43. 11
      src/squirrel/tool/__init__.py
  44. 81
      src/squirrel/tool/cli.py
  45. 13
      src/squirrel/tool/commands/__init__.py
  46. 34
      src/squirrel/tool/commands/files.py
  47. 24
      src/squirrel/tool/commands/info.py
  48. 22
      src/squirrel/tool/commands/init.py
  49. 25
      src/squirrel/tool/commands/nuts.py
  50. 42
      src/squirrel/tool/commands/scan.py
  51. 22
      src/squirrel/tool/commands/snuffler.py
  52. 155
      src/squirrel/tool/common.py
  53. 55
      src/trace.py
  54. 155
      src/util.py
  55. 3
      test/base/test_io.py
  56. 17
      test/base/test_pile.py
  57. 82
      test/base/test_progress.py
  58. 809
      test/base/test_squirrel.py
  59. 11
      test/base/test_util.py
  60. 56
      test/common.py

1
doc/source/library/reference/index.rst

@ -22,6 +22,7 @@ Reference
plot
pz
scenario
squirrel <squirrel/index>
streaming
trace
util

6
doc/source/library/reference/squirrel/base.rst

@ -0,0 +1,6 @@
``squirrel.base``
=================
.. automodule :: pyrocko.squirrel.base
:members:
:show-inheritance:

6
doc/source/library/reference/squirrel/client/index.rst

@ -0,0 +1,6 @@
``squirrel.client``
===================
.. automodule :: pyrocko.squirrel.client
:members:
:show-inheritance:

6
doc/source/library/reference/squirrel/error.rst

@ -0,0 +1,6 @@
``squirrel.error``
==================
.. automodule :: pyrocko.squirrel.error
:members:
:show-inheritance:

15
doc/source/library/reference/squirrel/index.rst

@ -0,0 +1,15 @@
``squirrel``
============
Prompt seismological data access with a fluffy tail.
.. toctree::
:maxdepth: 2
:caption: Contents
base
model
pile
error
io <io/index>
client <client/index>

6
doc/source/library/reference/squirrel/io/index.rst

@ -0,0 +1,6 @@
``squirrel.io``
==================
.. automodule :: pyrocko.squirrel.io
:members:
:show-inheritance:

6
doc/source/library/reference/squirrel/model.rst

@ -0,0 +1,6 @@
``squirrel.model``
==================
.. automodule :: pyrocko.squirrel.model
:members:
:show-inheritance:

6
doc/source/library/reference/squirrel/pile.rst

@ -0,0 +1,6 @@
``squirrel.pile``
==================
.. automodule :: pyrocko.squirrel.pile
:members:
:show-inheritance:

7
setup.cfg

@ -9,7 +9,6 @@ inplace=0
[nosetests]
verbosity=2
detailed-errors=1
cover-erase=1
cover-package=pyrocko
@ -25,6 +24,12 @@ exclude_lines =
logger.error
if pyrocko.grumpy:
[coverage:paths]
source =
src/
/usr/lib/*/pyrocko/
/usr/local/lib/*/pyrocko/
[flake8]
exclude =
src/gui/qt_compat.py

11
setup.py

@ -559,6 +559,12 @@ subpacknames = [
'pyrocko.scenario.targets',
'pyrocko.scenario.sources',
'pyrocko.obspy_compat',
'pyrocko.squirrel',
'pyrocko.squirrel.io',
'pyrocko.squirrel.io.backends',
'pyrocko.squirrel.client',
'pyrocko.squirrel.tool',
'pyrocko.squirrel.tool.commands',
]
cmdclass = {
@ -751,9 +757,10 @@ setup(
'automap = pyrocko.apps.automap:main',
'hamster = pyrocko.apps.hamster:main',
'jackseis = pyrocko.apps.jackseis:main',
'colosseo = pyrocko.apps.colosseo:main'],
'colosseo = pyrocko.apps.colosseo:main',
'squirrel = pyrocko.apps.squirrel:main'],
'gui_scripts':
['snuffler = pyrocko.apps.snuffler:main']
['snuffler = pyrocko.apps.snuffler:main'],
},
package_data={

5
src/__init__.py

@ -12,3 +12,8 @@ grumpy = 0 # noqa
class ExternalProgramMissing(Exception):
pass
def make_squirrel(*args, **kwargs):
from pyrocko.squirrel import Squirrel
return Squirrel(*args, **kwargs)

11
src/apps/squirrel.py

@ -0,0 +1,11 @@
#!/usr/bin/env python
# http://pyrocko.org - GPLv3
#
# The Pyrocko Developers, 21st Century
# ---|P------/S----------~Lg----------
from pyrocko.squirrel import main
if __name__ == '__main__':
import sys
main(sys.argv)

107
src/get_terminal_size.py

@ -0,0 +1,107 @@
from __future__ import absolute_import, print_function
#
# This file is NOT part of Pyrocko.
#
# Code is considered public domain, based on
# https://gist.github.com/jtriley/1108174
#
import os
import struct
import platform
import subprocess
def get_terminal_size():
'''
Get terminal size.
Works on linux,os x,windows,cygwin(windows)
originally retrieved from:
http://stackoverflow.com/questions/566746/how-to-get-console-window-width-in-python
'''
current_os = platform.system()
tuple_xy = None
if current_os == 'Windows':
tuple_xy = _get_terminal_size_windows()
if tuple_xy is None:
tuple_xy = _get_terminal_size_tput()
# needed for window's python in cygwin's xterm!
if current_os in ['Linux', 'Darwin'] or current_os.startswith('CYGWIN'):
tuple_xy = _get_terminal_size_linux()
if tuple_xy is None:
tuple_xy = (80, 25) # default value
return tuple_xy
def _get_terminal_size_windows():
try:
from ctypes import windll, create_string_buffer
# stdin handle is -10
# stdout handle is -11
# stderr handle is -12
h = windll.kernel32.GetStdHandle(-12)
csbi = create_string_buffer(22)
res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
if res:
(bufx, bufy, curx, cury, wattr,
left, top, right, bottom,
maxx, maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw)
sizex = right - left + 1
sizey = bottom - top + 1
return sizex, sizey
except Exception:
pass
def _get_terminal_size_tput():
# get terminal width
# src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window # noqa
try:
cols = int(subprocess.check_call(['tput' 'cols']))
rows = int(subprocess.check_call(['tput', 'lines']))
return (cols, rows)
except Exception:
pass
def _get_terminal_size_linux():
def ioctl_GWINSZ(fd):
try:
import fcntl
import termios
cr = struct.unpack(
'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
return cr
except Exception:
pass
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
if not cr:
try:
fd = os.open(os.ctermid(), os.O_RDONLY)
cr = ioctl_GWINSZ(fd)
os.close(fd)
except Exception:
pass
if not cr:
try:
cr = (os.environ['LINES'], os.environ['COLUMNS'])
except Exception:
return None
return int(cr[1]), int(cr[0])
if __name__ == "__main__":
sizex, sizey = get_terminal_size()
print(sizex, sizey)

185
src/gui/pile_viewer.py

@ -20,7 +20,6 @@ from itertools import groupby
import numpy as num
import pyrocko.model
import pyrocko.pile
import pyrocko.shadow_pile
import pyrocko.trace
import pyrocko.util
import pyrocko.plot
@ -191,16 +190,6 @@ class Timer(object):
return tuple([a[i] - b[i] for i in range(5)])
class Integrator(pyrocko.shadow_pile.ShadowPile):
def process(self, iblock, tmin, tmax, traces):
for trace in traces:
trace.ydata = trace.ydata - trace.ydata.mean()
trace.ydata = num.cumsum(trace.ydata)
return traces
class ObjectStyle(object):
def __init__(self, frame_pen, fill_brush):
self.frame_pen = frame_pen
@ -218,6 +207,25 @@ for color in 'orange skyblue butter chameleon chocolate plum ' \
*(pyrocko.plot.tango_colors[color+'1'] + (box_alpha,)))),
))
box_styles_coverage = [
ObjectStyle(
qg.QPen(
qg.QColor(*pyrocko.plot.tango_colors['aluminium3']),
1, qc.Qt.DashLine),
qg.QBrush(qg.QColor(
*(pyrocko.plot.tango_colors['aluminium1'] + (50,)))),
),
ObjectStyle(
qg.QPen(qg.QColor(*pyrocko.plot.tango_colors['aluminium4'])),
qg.QBrush(qg.QColor(
*(pyrocko.plot.tango_colors['aluminium2'] + (50,)))),
),
ObjectStyle(
qg.QPen(qg.QColor(*pyrocko.plot.tango_colors['plum3'])),
qg.QBrush(qg.QColor(
*(pyrocko.plot.tango_colors['plum1'] + (50,)))),
)]
sday = 60*60*24. # \
smonth = 60*60*24*30. # | only used as approx. intervals...
syear = 60*60*24*365. # /
@ -962,38 +970,25 @@ def MakePileViewerMainClass(base):
menudef = [
('Subsort by Network, Station, Location, Channel',
(lambda tr: self.ssort(tr) + tr.nslc_id, # gathering
lambda a: a, # sorting
((0, 1, 2, 3), # gathering
lambda tr: tr.location)), # coloring
('Subsort by Network, Station, Channel, Location',
(lambda tr: self.ssort(tr) + (
tr.network, tr.station, tr.channel, tr.location),
lambda a: a,
((0, 1, 3, 2),
lambda tr: tr.channel)),
('Subsort by Station, Network, Channel, Location',
(lambda tr: self.ssort(tr) + (
tr.station, tr.network, tr.channel, tr.location),
lambda a: a,
((1, 0, 3, 2),
lambda tr: tr.channel)),
('Subsort by Location, Network, Station, Channel',
(lambda tr: self.ssort(tr) + (
tr.location, tr.network, tr.station, tr.channel),
lambda a: a,
((2, 0, 1, 3),
lambda tr: tr.channel)),
('Subsort by Channel, Network, Station, Location',
(lambda tr: self.ssort(tr) + (
tr.channel, tr.network, tr.station, tr.location),
lambda a: a,
((3, 0, 1, 2),
lambda tr: (tr.network, tr.station, tr.location))),
('Subsort by Network, Station, Channel (Grouped by Location)',
(lambda tr: self.ssort(tr) + (
tr.network, tr.station, tr.channel),
lambda a: a,
((0, 1, 3),
lambda tr: tr.location)),
('Subsort by Station, Network, Channel (Grouped by Location)',
(lambda tr: self.ssort(tr) + (
tr.station, tr.network, tr.channel),
lambda a: a,
((1, 0, 3),
lambda tr: tr.location)),
]
@ -1416,22 +1411,6 @@ def MakePileViewerMainClass(base):
ignore = self.menuitem_distances_3d.isChecked()
self.set_event_marker_as_origin(ignore)
def toggletest(self, checked):
if checked:
sp = Integrator()
self.add_shadow_pile(sp)
else:
self.remove_shadow_piles()
def add_shadow_pile(self, shadow_pile):
shadow_pile.set_basepile(self.pile)
shadow_pile.add_listener(self)
self.pile = shadow_pile
def remove_shadow_piles(self):
self.pile = self.pile.get_basepile()
def iter_snuffling_modules(self):
pjoin = os.path.join
for path in self.snuffling_paths:
@ -1671,22 +1650,26 @@ def MakePileViewerMainClass(base):
if self.automatic_updates:
self.update()
def set_gathering(self, gather=None, order=None, color=None):
def set_gathering(self, gather=None, color=None):
if gather is None:
def gather(tr):
def gather_func(tr):
return tr.nslc_id
if order is None:
def order(a):
return a
gather = (0, 1, 2, 3)
else:
def gather_func(tr):
return (
self.ssort(tr) + tuple(tr.nslc_id[i] for i in gather))
if color is None:
def color(tr):
return tr.location
self.gather = gather
keys = self.pile.gather_keys(gather, self.trace_filter)
self.gather = gather_func
keys = self.pile.gather_keys(gather_func, self.trace_filter)
self.color_gather = color
self.color_keys = self.pile.gather_keys(color)
previous_ntracks = self.ntracks
@ -1705,7 +1688,17 @@ def MakePileViewerMainClass(base):
key_at_top = self.track_keys[low]
n = high-low
self.track_keys = sorted(keys, key=order)
self.track_keys = sorted(keys)
track_patterns = []
for k in self.track_keys:
pat = ['*', '*', '*', '*']
for i, j in enumerate(gather):
pat[j] = k[-len(gather)+i]
track_patterns.append(pat)
self.track_patterns = track_patterns
if key_at_top is not None:
try:
@ -2955,6 +2948,63 @@ def MakePileViewerMainClass(base):
p, self.time_projection, vcenter_projection,
with_label=True)
def get_squirrel(self):
try:
return self.pile._squirrel
except AttributeError:
return None
def draw_coverage(self, p, time_projection, track_projections):
sq = self.get_squirrel()
if sq is None:
return
def drawbox(itrack, tmin, tmax, style):
v_projection = track_projections[itrack]
dvmin = v_projection(0.)
dvmax = v_projection(1.)
dtmin = time_projection.clipped(tmin)
dtmax = time_projection.clipped(tmax)
rect = qc.QRectF(dtmin, dvmin, float(dtmax-dtmin), dvmax-dvmin)
p.fillRect(rect, style.fill_brush)
p.setPen(style.frame_pen)
p.drawRect(rect)
pattern_list = []
pattern_to_itrack = {}
for key in self.track_keys:
itrack = self.key_to_row[key]
if itrack not in track_projections:
continue
pattern = self.track_patterns[itrack]
pattern_to_itrack[tuple(pattern)] = itrack
pattern_list.append(pattern)
vmin, vmax = self.get_time_range()
for entry in sq.get_coverage(
'waveform', vmin, vmax, pattern_list, limit=500):
pattern, codes, deltat, tmin, tmax, cover_data = entry
itrack = pattern_to_itrack[tuple(pattern)]
if cover_data is None:
drawbox(itrack, tmin, tmax, box_styles_coverage[0])
else:
t = None
pcount = 0
for tb, count in cover_data:
if t is not None and tb > t:
if pcount > 0:
drawbox(
itrack, t, tb,
box_styles_coverage[
min(len(box_styles_coverage)-1,
pcount)])
t = tb
pcount = count
def drawit(self, p, printmode=False, w=None, h=None):
'''
This performs the actual drawing.
@ -2962,6 +3012,7 @@ def MakePileViewerMainClass(base):
self.timer_draw.start()
show_boxes = self.menuitem_showboxes.isChecked()
sq = self.get_squirrel()
if self.gather is None:
self.set_gathering()
@ -2971,7 +3022,7 @@ def MakePileViewerMainClass(base):
if not self.sortingmode_change_delayed():
self.sortingmode_change()
if show_boxes:
if show_boxes and sq is None:
self.determine_box_styles()
self.pile_has_changed = False
@ -3037,13 +3088,17 @@ def MakePileViewerMainClass(base):
demean=self.menuitem_demean.isChecked())
if not printmode and show_boxes:
if self.view_mode is ViewMode.Wiggle:
self.draw_trace_boxes(
p, self.time_projection, track_projections)
elif self.view_mode is ViewMode.Waterfall \
and not processed_traces:
self.draw_trace_boxes(
p, self.time_projection, track_projections)
if (self.view_mode is ViewMode.Wiggle) \
or (self.view_mode is ViewMode.Waterfall
and not processed_traces):
if sq is None:
self.draw_trace_boxes(
p, self.time_projection, track_projections)
else:
self.draw_coverage(
p, self.time_projection, track_projections)
p.setFont(font)
label_bg = qg.QBrush(qg.QColor(255, 255, 255, 100))
@ -3738,9 +3793,9 @@ def MakePileViewerMainClass(base):
self.sortingmode_change()
def sortingmode_change(self, ignore=None):
for menuitem, (gather, order, color) in self.menuitems_sorting:
for menuitem, (gather, color) in self.menuitems_sorting:
if menuitem.isChecked():
self.set_gathering(gather, order, color)
self.set_gathering(gather, color)
self.sortingmode_change_time = time.time()

22
src/guts.py

@ -1279,6 +1279,28 @@ class Tuple(Object):
strip_module=strip_module))
unit_factors = dict(
s=1.0,
m=60.0,
h=3600.0,
d=24*3600.0,
y=365*24*3600.0)
class Duration(Object):
dummy_for = float
class __T(TBase):
def regularize_extra(self, val):
if isinstance(val, (str, newstr)):
unit = val[-1]
if unit in unit_factors:
return float(val[:-1]) * unit_factors[unit]
else:
return float(val)
re_tz = re.compile(r'(Z|([+-][0-2][0-9])(:?([0-5][0-9]))?)$')

19
src/io/io_common.py

@ -3,6 +3,8 @@
# The Pyrocko Developers, 21st Century
# ---|P------/S----------~Lg----------
import os
class FileError(Exception):
def __init__(self, *args, **kwargs):
@ -34,3 +36,20 @@ class FileSaveError(FileError):
'''
Raised when a problem occurred while saving of a file.
'''
def get_stats(path):
try:
s = os.stat(path)
return float(s.st_mtime), s.st_size
except OSError as e:
raise FileLoadError(e)
def touch(path):
try:
with open(path, 'a'):
os.utime(path)
except OSError as e:
raise FileLoadError(e)

2
src/model/__init__.py

@ -6,6 +6,8 @@
'''
from __future__ import absolute_import
from .content import * # noqa
from .location import * # noqa
from .station import * # noqa
from .event import * # noqa

62
src/model/content.py

@ -0,0 +1,62 @@
# http://pyrocko.org - GPLv3
#
# The Pyrocko Developers, 21st Century
# ---|P------/S----------~Lg----------
import math
from pyrocko.guts import Object
from pyrocko import util
g_tmin, g_tmax = util.get_working_system_time_range()[:2]
def time_or_none_to_str(x, format):
if x is None:
return '...'.ljust(17)
else:
return util.time_to_str(x, format=format)
class Content(Object):
'''
Base class for Pyrocko content objects.
'''
@property
def str_codes(self):
return '.'.join(self.codes)
@property
def str_time_span(self):
tmin, tmax = self.time_span
deltat = getattr(self, 'deltat', 0)
if deltat > 0:
fmt = min(9, max(0, -int(math.floor(math.log10(self.deltat)))))
else:
fmt = 6
if tmin == tmax:
return '%s' % time_or_none_to_str(tmin, fmt)
else:
return '%s - %s' % (
time_or_none_to_str(tmin, fmt), time_or_none_to_str(tmax, fmt))
@property
def summary(self):
return '%s %-16s %s' % (
self.__class__.__name__, self.str_codes, self.str_time_span)
def __lt__(self, other):
return self.__key__() < other.__key__()
def __key__(self):
return self.codes, self.time_span_g_clipped
@property
def time_span_g_clipped(self):
tmin, tmax = self.time_span
return (
tmin if tmin is not None else g_tmin,
tmax if tmax is not None else g_tmax)

33
src/model/event.py

@ -309,20 +309,17 @@ class Event(Location):
@staticmethod
def load_catalog(filename):
file = open(filename, 'r')
try:
while True:
try:
ev = Event(loadf=file)
yield ev
except EmptyEvent:
pass
except EOF:
pass
with open(filename, 'r') as file:
try:
while True:
try:
ev = Event(loadf=file)
yield ev
except EmptyEvent:
pass
file.close()
except EOF:
pass
def get_hash(self):
e = self
@ -369,6 +366,16 @@ class Event(Location):
return '\n'.join(s)
@property
def summary(self):
return '%s: %s, %s, %s, %s' % (
self.__class__.__name__,
self.name,
util.time_to_str(self.time),
'%-3s %3.1f' % (self.magnitude_type or ' ', self.magnitude)
if self.magnitude else 'M ---',
self.region)
def detect_format(filename):
with open(filename, 'r') as f:

344
src/progress.py

@ -0,0 +1,344 @@
from __future__ import absolute_import, print_function
import sys
import time
from .get_terminal_size import get_terminal_size
spinner = u'\u25dc\u25dd\u25de\u25df'
skull = u'\u2620'
check = u'\u2714'
bar = u'[- ]'
blocks = u'\u2588\u2589\u258a\u258b\u258c\u258d\u258e\u258f '
ansi_up = u'\033[%iA'
ansi_down = u'\033[%iB'
ansi_left = u'\033[%iC'
ansi_right = u'\033[%iD'
ansi_next_line = u'\033E'
ansi_erase_display = u'\033[2J'
ansi_window = u'\033[%i;%ir'
ansi_move_to = u'\033[%i;%iH'
ansi_clear_down = u'\033[0J'
ansi_clear_up = u'\033[1J'
ansi_clear = u'\033[2J'
ansi_clear_right = u'\033[0K'
ansi_scroll_up = u'\033D'
ansi_scroll_down = u'\033M'
ansi_reset = u'\033c'
g_force_viewer_off = False
class TerminalStatusWindow(object):
def __init__(self, parent=None):
self._terminal_size = get_terminal_size()
self._height = 0
self._state = 0
self._parent = parent
def __enter__(self):
return self
def __exit__(self, *_):
self.stop()
def print(self, s):
print(s, end='', file=sys.stderr)
def flush(self):
print('', end='', flush=True, file=sys.stderr)
def start(self):
sx, sy = self._terminal_size
self._state = 1
def stop(self):
if self._state == 1:
sx, sy = self._terminal_size
self._resize(0)
self.print(ansi_move_to % (sy-self._height, 1))
self.flush()
self._state = 2
if self._parent:
self._parent.hide()
def _start_show(self):
sx, sy = self._terminal_size
self.print(ansi_move_to % (sy-self._height+1, 1))
def _end_show(self):
sx, sy = self._terminal_size
self.print(ansi_move_to % (sy-self._height, 1))
self.print(ansi_clear_right)
def _resize(self, height):
sx, sy = self._terminal_size
k = height - self._height
if k > 0:
self.print(ansi_scroll_up * k)
self.print(ansi_window % (1, sy-height))
if k < 0:
self.print(ansi_window % (1, sy-height))
self.print(ansi_scroll_down * abs(k))
self._height = height
def draw(self, lines):
if self._state == 0:
self.start()
if self._state != 1:
return
self._terminal_size = get_terminal_size()
sx, sy = self._terminal_size
nlines = len(lines)
self._resize(nlines)
self._start_show()
for iline, line in enumerate(lines):
if len(line) > sx - 1:
line = line[:sx-1]
self.print(ansi_clear_right + line)
if iline != nlines - 1:
self.print(ansi_next_line)
self._end_show()
self.flush()
class DummyStatusWindow(object):
def __init__(self, parent=None):
self._parent = parent
def __enter__(self):
return self
def __exit__(self, *_):
self.stop()
def stop(self):
if self._parent:
self._parent.hide()
def draw(self, lines):
pass
class Task(object):
def __init__(
self, progress, id, name, n, state='working', logger=None,
group=None):
self._id = id
self._name = name
self._condition = ''
self._ispin = 0
self._i = None
self._n = n
self._done = False
assert state in ('waiting', 'working')
self._state = state
self._progress = progress
self._logger = logger
self._tcreate = time.time()
self._group = group
def __call__(self, it):
err = False
try:
n = 0
for obj in it:
self.update(n)
yield obj
n += 1
self.update(n)
except Exception:
err = True
raise
finally:
if err:
self.fail()
else:
self.done()
def log(self, s):
if self._logger is not None:
self._logger.info(s)
def get_group_time_start(self):
if self._group:
return self._group.get_group_time_start()
else:
return self._tcreate
def task(self, *args, **kwargs):
kwargs['group'] = self
return self._progress.task(*args, **kwargs)
def update(self, i=None, condition=''):
self._state = 'working'
self._condition = condition
if i is not None:
if self._n is not None:
i = min(i, self._n)
self._i = i
self._progress._update()
def done(self, condition=''):
self.duration = time.time() - self._tcreate
if self._state in ('done', 'failed'):
return
self._condition = condition
self._state = 'done'
self._progress._end(self)
def fail(self, condition=''):
self.duration = time.time() - self._tcreate
self._condition = condition
self._state = 'failed'
self._progress._end(self)
def _str_state(self):
s = self._state
if s == 'waiting':
return ' '
elif s == 'working':
self._ispin += 1
return spinner[self._ispin % len(spinner)] + ' '
elif s == 'done':
return '' # check
elif s == 'failed':
return skull + ' '
else:
return '? '
def _str_progress(self):
if self._i is None:
return self._state
elif self._n is None:
return '%i' % self._i
else:
nw = len(str(self._n))
return ('%' + str(nw) + 'i / %i%s') % (
self._i, self._n,
' %3.0f%%' % ((100. * self._i) / self._n)
if self._state == 'working' else '')
def _str_condition(self):
if self._condition:
return '%s' % self._condition
else:
return ''
def _str_bar(self):
if self._state == 'working' and self._n and self._i is not None:
nb = 20
fb = nb * float(self._i) / self._n
ib = int(fb)
ip = int((fb - ib) * (len(blocks)-1))
s = blocks[0] * ib
if ib < nb:
s += blocks[-1-ip] + (nb - ib - 1) * blocks[-1] + blocks[-2]
# s = ' ' + bar[0] + bar[1] * ib + bar[2] * (nb - ib) + bar[3]
return s
else:
return ''
def __str__(self):
return '%s%s: %s' % (
self._str_state(),
self._name,
' '.join([
self._str_progress(),
self._str_bar(),
self._str_condition(),
]))
class Progress(object):
def __init__(self):
self._current_id = 0
self._current_group_id = 0
self._tasks = {}
self._tasks_done = []
self._last_update = 0.0
self._term = None
def view(self, viewer='terminal'):
if g_force_viewer_off:
viewer = 'off'
if viewer == 'terminal':
self._term = TerminalStatusWindow(self)
elif viewer == 'off':
self._term = DummyStatusWindow(self)
else:
raise ValueError('Invalid viewer choice: %s' % viewer)
return self._term
def hide(self):
self._update(force=True)
self._term = None
def task(self, name, n=None, logger=None, group=None):
self._current_id += 1
task = Task(
self, self._current_id, name, n, logger=logger, group=group)
self._tasks[task._id] = task
self._update(force=True)
return task
def _end(self, task):
del self._tasks[task._id]
self._tasks_done.append(task)
self._update(force=True)
def _update(self, force=False):
now = time.time()
if self._last_update + 0.1 < now or force:
tasks_done = self._tasks_done
self._tasks_done = []
if self._term:
for task in tasks_done:
task.log(str(task))
lines = self._lines()
if self._term:
self._term.draw(lines)
self._last_update = now
def _lines(self):
task_ids = sorted(self._tasks)
lines = []
for task_id in task_ids:
task = self._tasks[task_id]
lines.extend(str(task).splitlines())
return lines
progress = Progress()

20
src/squirrel/__init__.py

@ -0,0 +1,20 @@
# http://pyrocko.org - GPLv3
#
# The Pyrocko Developers, 21st Century
# ---|P------/S----------~Lg----------
from __future__ import absolute_import, print_function
from . import base, model, io, client, tool, error, environment
from .base import * # noqa
from .model import * # noqa
from .io import * # noqa
from .client import * # noqa
from .tool import * # noqa
from .error import * # noqa
from .environment import * # noqa
__all__ = base.__all__ + model.__all__ + io.__all__ + client.__all__ \
+ tool.__all__ + error.__all__ + environment.__all__

2823
src/squirrel/base.py

File diff suppressed because it is too large Load Diff

84
src/squirrel/cache.py

@ -0,0 +1,84 @@
# http://pyrocko.org - GPLv3
#
# The Pyrocko Developers, 21st Century
# ---|P------/S----------~Lg----------
class ContentCache(object):
def __init__(self):
self._entries = {}
self._accessor_ticks = {}
def advance_accessor(self, accessor):
if accessor not in self._accessor_ticks:
self._accessor_ticks[accessor] = 0
ta = self._accessor_ticks[accessor]
delete = []
for path_segment, entry in self._entries.items():
t = entry[2].get(accessor, ta)
if t < ta:
del entry[2][accessor]
if not entry[2]:
delete.append(path_segment)
for path_segment in delete:
del self._entries[path_segment]
self._accessor_ticks[accessor] += 1
def clear_accessor(self, accessor):
delete = []
for path_segment, entry in self._entries.items():
del entry[2][accessor]
if not entry[2]:
delete.append(path_segment)
for path_segment in delete:
del self._entries[path_segment]
del self._accessor_ticks[accessor]
def clear(self):
self._entries = {}
self._accessor_ticks = {}
def has(self, nut):
path, segment, element, nut_mtime = nut.key
try:
cache_mtime = self._entries[path, segment][0]
except KeyError:
return False
return cache_mtime == nut_mtime
def get(self, nut, accessor='default'):
path, segment, element, mtime = nut.key
entry = self._entries[path, segment]
if accessor not in self._accessor_ticks:
self._accessor_ticks[accessor] = 0
entry[2][accessor] = self._accessor_ticks[accessor]
return entry[1][element]
def _prune_outdated(self, path, segment, nut_mtime):
try:
cache_mtime = self._entries[path, segment][0]
except KeyError:
return
if cache_mtime != nut_mtime:
del self._entries[path, segment]
def put(self, nut):
path, segment, element, mtime = nut.key
self._prune_outdated(path, segment, nut.file_mtime)
if (path, segment) not in self._entries:
self._entries[path, segment] = nut.file_mtime, {}, {}
self._entries[path, segment][1][element] = nut.content

14
src/squirrel/client/__init__.py

@ -0,0 +1,14 @@
# http://pyrocko.org - GPLv3
#
# The Pyrocko Developers, 21st Century
# ---|P------/S----------~Lg----------
from __future__ import absolute_import, print_function
from . import base, fdsn, catalog
from .base import * # noqa
from .fdsn import * # noqa
from .catalog import * # noqa
__all__ = base.__all__ + fdsn.__all__ + catalog.__all__

80
src/squirrel/client/base.py

@ -0,0 +1,80 @@
# http://pyrocko.org - GPLv3
#
# The Pyrocko Developers, 21st Century
# ---|P------/S----------~Lg----------
from __future__ import absolute_import, print_function
from pyrocko.guts import Object, Timestamp
class Constraint(Object):
tmin = Timestamp.T(optional=True)
tmax = Timestamp.T(optional=True)
def contains(self, constraint):
'''
Check if the constraint completely includes a more restrictive one.
'''
if self.tmin is not None and constraint.tmin is not None:
b1 = self.tmin <= constraint.tmin
elif self.tmin is None:
b1 = True
else:
b1 = False
if self.tmax is not None and constraint.tmax is not None:
b2 = constraint.tmax <= self.tmax
elif self.tmax is None:
b2 = True
else:
b2 = False
return b1 and b2
def expand(self, constraint):
'''
Widen constraint to include another given constraint.
'''
if constraint.tmin is None or self.tmin is None:
self.tmin = None
else:
self.tmin = min(constraint.tmin, self.tmin)
if constraint.tmax is None or self.tmax is None:
self.tmax = None
else:
self.tmax = max(constraint.tmax, self.tmax)
class Source(Object):
def update_channel_inventory(self, squirrel, constraint):
'''
Let local inventory be up-to-date with remote for a given constraint.
'''
pass
def update_event_inventory(self, squirrel, constraint):
'''
Let local inventory be up-to-date with remote for a given constraint.
'''
pass
def update_waveform_promises(self, squirrel, constraint):
'''
Let local inventory be up-to-date with remote for a given constraint.
'''
pass
__all__ = [
'Source',
'Constraint',
]

291
src/squirrel/client/catalog.py

@ -0,0 +1,291 @@
# http://pyrocko.org - GPLv3
#
# The Pyrocko Developers, 21st Century
# ---|P------/S----------~Lg----------
from __future__ import absolute_import, print_function
import os.path as op
import logging
import time
try:
import cPickle as pickle
except ImportError:
import pickle
from pyrocko import util
from pyrocko.guts import String, Dict, Duration, dump_all
from .base import Source
from ..model import ehash
from ..lock import LockDir
logger = logging.getLogger('pyrocko.squirrel.client.catalog')
class Link(object):
def __init__(self, tmin, tmax, tmodified, nevents=-1, content_id=None):
self.tmin = tmin
self.tmax = tmax
self.tmodified = tmodified
self.nevents = nevents
self.content_id = content_id
def __str__(self):
return 'span %s - %s, access %s, nevents %i' % (
util.tts(self.tmin),
util.tts(self.tmax),
util.tts(self.tmodified),
self.nevents)