squirrel: initial implementation
parent
8168edb952
commit
307b5c4269
@ -0,0 +1,6 @@
|
||||
``squirrel.base``
|
||||
=================
|
||||
|
||||
.. automodule :: pyrocko.squirrel.base
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@ -0,0 +1,6 @@
|
||||
``squirrel.client``
|
||||
===================
|
||||
|
||||
.. automodule :: pyrocko.squirrel.client
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@ -0,0 +1,6 @@
|
||||
``squirrel.error``
|
||||
==================
|
||||
|
||||
.. automodule :: pyrocko.squirrel.error
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@ -0,0 +1,15 @@
|
||||
``squirrel``
|
||||
============
|
||||
|
||||
Prompt seismological data access with a fluffy tail.
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents
|
||||
|
||||
base
|
||||
model
|
||||
pile
|
||||
error
|
||||
io <io/index>
|
||||
client <client/index>
|
||||
@ -0,0 +1,6 @@
|
||||
``squirrel.io``
|
||||
==================
|
||||
|
||||
.. automodule :: pyrocko.squirrel.io
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@ -0,0 +1,6 @@
|
||||
``squirrel.model``
|
||||
==================
|
||||
|
||||
.. automodule :: pyrocko.squirrel.model
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@ -0,0 +1,6 @@
|
||||
``squirrel.pile``
|
||||
==================
|
||||
|
||||
.. automodule :: pyrocko.squirrel.pile
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env python
|
||||
# http://pyrocko.org - GPLv3
|
||||
#
|
||||
# The Pyrocko Developers, 21st Century
|
||||
# ---|P------/S----------~Lg----------
|
||||
|
||||
from pyrocko.squirrel import main
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
main(sys.argv)
|
||||
@ -0,0 +1,107 @@
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
#
|
||||
# This file is NOT part of Pyrocko.
|
||||
#
|
||||
# Code is considered public domain, based on
|
||||
# https://gist.github.com/jtriley/1108174
|
||||
#
|
||||
|
||||
import os
|
||||
import struct
|
||||
import platform
|
||||
import subprocess
|
||||
|
||||
|
||||
def get_terminal_size():
|
||||
'''
|
||||
Get terminal size.
|
||||
|
||||
Works on linux,os x,windows,cygwin(windows)
|
||||
|
||||
originally retrieved from:
|
||||
http://stackoverflow.com/questions/566746/how-to-get-console-window-width-in-python
|
||||
'''
|
||||
|
||||
current_os = platform.system()
|
||||
tuple_xy = None
|
||||
if current_os == 'Windows':
|
||||
tuple_xy = _get_terminal_size_windows()
|
||||
if tuple_xy is None:
|
||||
tuple_xy = _get_terminal_size_tput()
|
||||
# needed for window's python in cygwin's xterm!
|
||||
|
||||
if current_os in ['Linux', 'Darwin'] or current_os.startswith('CYGWIN'):
|
||||
tuple_xy = _get_terminal_size_linux()
|
||||
|
||||
if tuple_xy is None:
|
||||
tuple_xy = (80, 25) # default value
|
||||
|
||||
return tuple_xy
|
||||
|
||||
|
||||
def _get_terminal_size_windows():
|
||||
try:
|
||||
from ctypes import windll, create_string_buffer
|
||||
# stdin handle is -10
|
||||
# stdout handle is -11
|
||||
# stderr handle is -12
|
||||
h = windll.kernel32.GetStdHandle(-12)
|
||||
csbi = create_string_buffer(22)
|
||||
res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
|
||||
if res:
|
||||
(bufx, bufy, curx, cury, wattr,
|
||||
left, top, right, bottom,
|
||||
maxx, maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw)
|
||||
sizex = right - left + 1
|
||||
sizey = bottom - top + 1
|
||||
return sizex, sizey
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _get_terminal_size_tput():
|
||||
# get terminal width
|
||||
# src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window # noqa
|
||||
try:
|
||||
cols = int(subprocess.check_call(['tput' 'cols']))
|
||||
rows = int(subprocess.check_call(['tput', 'lines']))
|
||||
return (cols, rows)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _get_terminal_size_linux():
|
||||
|
||||
def ioctl_GWINSZ(fd):
|
||||
try:
|
||||
import fcntl
|
||||
import termios
|
||||
cr = struct.unpack(
|
||||
'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
|
||||
return cr
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
|
||||
|
||||
if not cr:
|
||||
try:
|
||||
fd = os.open(os.ctermid(), os.O_RDONLY)
|
||||
cr = ioctl_GWINSZ(fd)
|
||||
os.close(fd)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not cr:
|
||||
try:
|
||||
cr = (os.environ['LINES'], os.environ['COLUMNS'])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
return int(cr[1]), int(cr[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sizex, sizey = get_terminal_size()
|
||||
print(sizex, sizey)
|
||||
@ -0,0 +1,62 @@
|
||||
# http://pyrocko.org - GPLv3
|
||||
#
|
||||
# The Pyrocko Developers, 21st Century
|
||||
# ---|P------/S----------~Lg----------
|
||||
|
||||
import math
|
||||
|
||||
from pyrocko.guts import Object
|
||||
from pyrocko import util
|
||||
|
||||
|
||||
g_tmin, g_tmax = util.get_working_system_time_range()[:2]
|
||||
|
||||
|
||||
def time_or_none_to_str(x, format):
|
||||
if x is None:
|
||||
return '...'.ljust(17)
|
||||
else:
|
||||
return util.time_to_str(x, format=format)
|
||||
|
||||
|
||||
class Content(Object):
|
||||
'''
|
||||
Base class for Pyrocko content objects.
|
||||
'''
|
||||
|
||||
@property
|
||||
def str_codes(self):
|
||||
return '.'.join(self.codes)
|
||||
|
||||
@property
|
||||
def str_time_span(self):
|
||||
tmin, tmax = self.time_span
|
||||
deltat = getattr(self, 'deltat', 0)
|
||||
if deltat > 0:
|
||||
fmt = min(9, max(0, -int(math.floor(math.log10(self.deltat)))))
|
||||
else:
|
||||
fmt = 6
|
||||
|
||||
if tmin == tmax:
|
||||
return '%s' % time_or_none_to_str(tmin, fmt)
|
||||
else:
|
||||
return '%s - %s' % (
|
||||
time_or_none_to_str(tmin, fmt), time_or_none_to_str(tmax, fmt))
|
||||
|
||||
@property
|
||||
def summary(self):
|
||||
return '%s %-16s %s' % (
|
||||
self.__class__.__name__, self.str_codes, self.str_time_span)
|
||||
|
||||
def __lt__(self, other):
|
||||
return self.__key__() < other.__key__()
|
||||
|
||||
def __key__(self):
|
||||
return self.codes, self.time_span_g_clipped
|
||||
|
||||
@property
|
||||
def time_span_g_clipped(self):
|
||||
tmin, tmax = self.time_span
|
||||
return (
|
||||
tmin if tmin is not None else g_tmin,
|
||||
tmax if tmax is not None else g_tmax)
|
||||
@ -0,0 +1,344 @@
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
from .get_terminal_size import get_terminal_size
|
||||
|
||||
|
||||
spinner = u'\u25dc\u25dd\u25de\u25df'
|
||||
skull = u'\u2620'
|
||||
check = u'\u2714'
|
||||
bar = u'[- ]'
|
||||
blocks = u'\u2588\u2589\u258a\u258b\u258c\u258d\u258e\u258f '
|
||||
|
||||
ansi_up = u'\033[%iA'
|
||||
ansi_down = u'\033[%iB'
|
||||
ansi_left = u'\033[%iC'
|
||||
ansi_right = u'\033[%iD'
|
||||
ansi_next_line = u'\033E'
|
||||
|
||||
ansi_erase_display = u'\033[2J'
|
||||
ansi_window = u'\033[%i;%ir'
|
||||
ansi_move_to = u'\033[%i;%iH'
|
||||
|
||||
ansi_clear_down = u'\033[0J'
|
||||
ansi_clear_up = u'\033[1J'
|
||||
ansi_clear = u'\033[2J'
|
||||
|
||||
ansi_clear_right = u'\033[0K'
|
||||
|
||||
ansi_scroll_up = u'\033D'
|
||||
ansi_scroll_down = u'\033M'
|
||||
|
||||
ansi_reset = u'\033c'
|
||||
|
||||
|
||||
g_force_viewer_off = False
|
||||
|
||||
|
||||
class TerminalStatusWindow(object):
|
||||
def __init__(self, parent=None):
|
||||
self._terminal_size = get_terminal_size()
|
||||
self._height = 0
|
||||
self._state = 0
|
||||
self._parent = parent
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
self.stop()
|
||||
|
||||
def print(self, s):
|
||||
print(s, end='', file=sys.stderr)
|
||||
|
||||
def flush(self):
|
||||
print('', end='', flush=True, file=sys.stderr)
|
||||
|
||||
def start(self):
|
||||
sx, sy = self._terminal_size
|
||||
self._state = 1
|
||||
|
||||
def stop(self):
|
||||
if self._state == 1:
|
||||
sx, sy = self._terminal_size
|
||||
self._resize(0)
|
||||
self.print(ansi_move_to % (sy-self._height, 1))
|
||||
self.flush()
|
||||
|
||||
self._state = 2
|
||||
if self._parent:
|
||||
self._parent.hide()
|
||||
|
||||
def _start_show(self):
|
||||
sx, sy = self._terminal_size
|
||||
self.print(ansi_move_to % (sy-self._height+1, 1))
|
||||
|
||||
def _end_show(self):
|
||||
sx, sy = self._terminal_size
|
||||
self.print(ansi_move_to % (sy-self._height, 1))
|
||||
self.print(ansi_clear_right)
|
||||
|
||||
def _resize(self, height):
|
||||
sx, sy = self._terminal_size
|
||||
k = height - self._height
|
||||
if k > 0:
|
||||
self.print(ansi_scroll_up * k)
|
||||
self.print(ansi_window % (1, sy-height))
|
||||
if k < 0:
|
||||
self.print(ansi_window % (1, sy-height))
|
||||
self.print(ansi_scroll_down * abs(k))
|
||||
|
||||
self._height = height
|
||||
|
||||
def draw(self, lines):
|
||||
if self._state == 0:
|
||||
self.start()
|
||||
|
||||
if self._state != 1:
|
||||
return
|
||||
|
||||
self._terminal_size = get_terminal_size()
|
||||
sx, sy = self._terminal_size
|
||||
nlines = len(lines)
|
||||
self._resize(nlines)
|
||||
self._start_show()
|
||||
|
||||
for iline, line in enumerate(lines):
|
||||
if len(line) > sx - 1:
|
||||
line = line[:sx-1]
|
||||
|
||||
self.print(ansi_clear_right + line)
|
||||
if iline != nlines - 1:
|
||||
self.print(ansi_next_line)
|
||||
|
||||
self._end_show()
|
||||
self.flush()
|
||||
|
||||
|
||||
class DummyStatusWindow(object):
|
||||
|
||||
def __init__(self, parent=None):
|
||||
self._parent = parent
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
self.stop()
|
||||
|
||||
def stop(self):
|
||||
if self._parent:
|
||||
self._parent.hide()
|
||||
|
||||
def draw(self, lines):
|
||||
pass
|
||||
|
||||
|
||||
class Task(object):
|
||||
def __init__(
|
||||
self, progress, id, name, n, state='working', logger=None,
|
||||
group=None):
|
||||
|
||||
self._id = id
|
||||
self._name = name
|
||||
self._condition = ''
|
||||
self._ispin = 0
|
||||
self._i = None
|
||||
self._n = n
|
||||
self._done = False
|
||||
assert state in ('waiting', 'working')
|
||||
self._state = state
|
||||
self._progress = progress
|
||||
self._logger = logger
|
||||
self._tcreate = time.time()
|
||||
self._group = group
|
||||
|
||||
def __call__(self, it):
|
||||
err = False
|
||||
try:
|
||||
n = 0
|
||||
for obj in it:
|
||||
self.update(n)
|
||||
yield obj
|
||||
n += 1
|
||||
|
||||
self.update(n)
|
||||
|
||||
except Exception:
|
||||
err = True
|
||||
raise
|
||||
|
||||
finally:
|
||||
if err:
|
||||
self.fail()
|
||||
else:
|
||||
self.done()
|
||||
|
||||
def log(self, s):
|
||||
if self._logger is not None:
|
||||
self._logger.info(s)
|
||||
|
||||
def get_group_time_start(self):
|
||||
if self._group:
|
||||
return self._group.get_group_time_start()
|
||||
else:
|
||||
return self._tcreate
|
||||
|
||||
def task(self, *args, **kwargs):
|
||||
kwargs['group'] = self
|
||||
return self._progress.task(*args, **kwargs)
|
||||
|
||||
def update(self, i=None, condition=''):
|
||||
self._state = 'working'
|
||||
self._condition = condition
|
||||
if i is not None:
|
||||
if self._n is not None:
|
||||
i = min(i, self._n)
|
||||
|
||||
self._i = i
|
||||
|
||||
self._progress._update()
|
||||
|
||||
def done(self, condition=''):
|
||||
self.duration = time.time() - self._tcreate
|
||||
|
||||
if self._state in ('done', 'failed'):
|
||||
return
|
||||
|
||||
self._condition = condition
|
||||
self._state = 'done'
|
||||
self._progress._end(self)
|
||||
|
||||
def fail(self, condition=''):
|
||||
self.duration = time.time() - self._tcreate
|
||||
|
||||
self._condition = condition
|
||||
self._state = 'failed'
|
||||
self._progress._end(self)
|
||||
|
||||
def _str_state(self):
|
||||
s = self._state
|
||||
if s == 'waiting':
|
||||
return ' '
|
||||
elif s == 'working':
|
||||
self._ispin += 1
|
||||
return spinner[self._ispin % len(spinner)] + ' '
|
||||
elif s == 'done':
|
||||
return '' # check
|
||||
elif s == 'failed':
|
||||
return skull + ' '
|
||||
else:
|
||||
return '? '
|
||||
|
||||
def _str_progress(self):
|
||||
if self._i is None:
|
||||
return self._state
|
||||
elif self._n is None:
|
||||
return '%i' % self._i
|
||||
else:
|
||||
nw = len(str(self._n))
|
||||
return ('%' + str(nw) + 'i / %i%s') % (
|
||||
self._i, self._n,
|
||||
' %3.0f%%' % ((100. * self._i) / self._n)
|
||||
if self._state == 'working' else '')
|
||||
|
||||
def _str_condition(self):
|
||||
if self._condition:
|
||||
return '%s' % self._condition
|
||||
else:
|
||||
return ''
|
||||
|
||||
def _str_bar(self):
|
||||
if self._state == 'working' and self._n and self._i is not None:
|
||||
nb = 20
|
||||
fb = nb * float(self._i) / self._n
|
||||
ib = int(fb)
|
||||
ip = int((fb - ib) * (len(blocks)-1))
|
||||
s = blocks[0] * ib
|
||||
if ib < nb:
|
||||
s += blocks[-1-ip] + (nb - ib - 1) * blocks[-1] + blocks[-2]
|
||||
|
||||
# s = ' ' + bar[0] + bar[1] * ib + bar[2] * (nb - ib) + bar[3]
|
||||
return s
|
||||
else:
|
||||
return ''
|
||||
|
||||
def __str__(self):
|
||||
return '%s%s: %s' % (
|
||||
self._str_state(),
|
||||
self._name,
|
||||
' '.join([
|
||||
self._str_progress(),
|
||||
self._str_bar(),
|
||||
self._str_condition(),
|
||||
]))
|
||||
|
||||
|
||||
class Progress(object):
|
||||
|
||||
def __init__(self):
|
||||
self._current_id = 0
|
||||
self._current_group_id = 0
|
||||
self._tasks = {}
|
||||
self._tasks_done = []
|
||||
self._last_update = 0.0
|
||||
self._term = None
|
||||
|
||||
def view(self, viewer='terminal'):
|
||||
if g_force_viewer_off:
|
||||
viewer = 'off'
|
||||
|
||||
if viewer == 'terminal':
|
||||
self._term = TerminalStatusWindow(self)
|
||||
elif viewer == 'off':
|
||||
self._term = DummyStatusWindow(self)
|
||||
else:
|
||||
raise ValueError('Invalid viewer choice: %s' % viewer)
|
||||
|
||||
return self._term
|
||||
|
||||
def hide(self):
|
||||
self._update(force=True)
|
||||
self._term = None
|
||||
|
||||
def task(self, name, n=None, logger=None, group=None):
|
||||
self._current_id += 1
|
||||
task = Task(
|
||||
self, self._current_id, name, n, logger=logger, group=group)
|
||||
self._tasks[task._id] = task
|
||||
self._update(force=True)
|
||||
return task
|
||||
|
||||
def _end(self, task):
|
||||
del self._tasks[task._id]
|
||||
self._tasks_done.append(task)
|
||||
self._update(force=True)
|
||||
|
||||
def _update(self, force=False):
|
||||
now = time.time()
|
||||
if self._last_update + 0.1 < now or force:
|
||||
tasks_done = self._tasks_done
|
||||
self._tasks_done = []
|
||||
if self._term:
|
||||
for task in tasks_done:
|
||||
task.log(str(task))
|
||||
|
||||
lines = self._lines()
|
||||
if self._term:
|
||||
self._term.draw(lines)
|
||||
|
||||
self._last_update = now
|
||||
|
||||
def _lines(self):
|
||||
task_ids = sorted(self._tasks)
|
||||
lines = []
|
||||
for task_id in task_ids:
|
||||
task = self._tasks[task_id]
|
||||
lines.extend(str(task).splitlines())
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
progress = Progress()
|
||||
@ -0,0 +1,20 @@
|
||||
# http://pyrocko.org - GPLv3
|
||||
#
|
||||
# The Pyrocko Developers, 21st Century
|
||||
# ---|P------/S----------~Lg----------
|
||||
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
|
||||
from . import base, model, io, client, tool, error, environment
|
||||
|
||||
from .base import * # noqa
|
||||
from .model import * # noqa
|
||||
from .io import * # noqa
|
||||
from .client import * # noqa
|
||||
from .tool import * # noqa
|
||||
from .error import * # noqa
|
||||
from .environment import * # noqa
|
||||
|
||||
__all__ = base.__all__ + model.__all__ + io.__all__ + client.__all__ \
|
||||
+ tool.__all__ + error.__all__ + environment.__all__
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,84 @@
|
||||
# http://pyrocko.org - GPLv3
|
||||
#
|
||||
# The Pyrocko Developers, 21st Century
|
||||
# ---|P------/S----------~Lg----------
|
||||
|
||||
class ContentCache(object):
|
||||
|
||||
def __init__(self):
|
||||
self._entries = {}
|
||||
self._accessor_ticks = {}
|
||||
|
||||
def advance_accessor(self, accessor):
|
||||
if accessor not in self._accessor_ticks:
|
||||
self._accessor_ticks[accessor] = 0
|
||||
|
||||
ta = self._accessor_ticks[accessor]
|
||||
|
||||
delete = []
|
||||
for path_segment, entry in self._entries.items():
|
||||
t = entry[2].get(accessor, ta)
|
||||
if t < ta:
|
||||
del entry[2][accessor]
|
||||
if not entry[2]:
|
||||
delete.append(path_segment)
|
||||
|
||||
for path_segment in delete:
|
||||
del self._entries[path_segment]
|
||||
|
||||
self._accessor_ticks[accessor] += 1
|
||||
|
||||
def clear_accessor(self, accessor):
|
||||
delete = []
|
||||
for path_segment, entry in self._entries.items():
|
||||
del entry[2][accessor]
|
||||
if not entry[2]:
|
||||
delete.append(path_segment)
|
||||
|
||||
for path_segment in delete:
|
||||
del self._entries[path_segment]
|
||||
|
||||
del self._accessor_ticks[accessor]
|
||||
|
||||
def clear(self):
|
||||
self._entries = {}
|
||||
self._accessor_ticks = {}
|
||||
|
||||
def has(self, nut):
|
||||
path, segment, element, nut_mtime = nut.key
|
||||
|
||||
try:
|
||||
cache_mtime = self._entries[path, segment][0]
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
return cache_mtime == nut_mtime
|
||||
|
||||
def get(self, nut, accessor='default'):
|
||||
path, segment, element, mtime = nut.key
|
||||
entry = self._entries[path, segment]
|
||||
|
||||
if accessor not in self._accessor_ticks:
|
||||
self._accessor_ticks[accessor] = 0
|
||||
|
||||
entry[2][accessor] = self._accessor_ticks[accessor]
|
||||
|
||||
return entry[1][element]
|
||||
|
||||
def _prune_outdated(self, path, segment, nut_mtime):
|
||||
try:
|
||||
cache_mtime = self._entries[path, segment][0]
|
||||
except KeyError:
|
||||
return
|
||||
|
||||
if cache_mtime != nut_mtime:
|
||||
del self._entries[path, segment]
|
||||
|
||||
def put(self, nut):
|
||||
path, segment, element, mtime = nut.key
|
||||
self._prune_outdated(path, segment, nut.file_mtime)
|
||||
|
||||
if (path, segment) not in self._entries:
|
||||
self._entries[path, segment] = nut.file_mtime, {}, {}
|
||||
|
||||
self._entries[path, segment][1][element] = nut.content
|
||||
@ -0,0 +1,14 @@
|
||||
# http://pyrocko.org - GPLv3
|
||||
#
|
||||
# The Pyrocko Developers, 21st Century
|
||||
# ---|P------/S----------~Lg----------
|
||||
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
from . import base, fdsn, catalog
|
||||
|
||||
from .base import * # noqa
|
||||
from .fdsn import * # noqa
|
||||
from .catalog import * # noqa
|
||||
|
||||
__all__ = base.__all__ + fdsn.__all__ + catalog.__all__
|
||||
@ -0,0 +1,80 @@
|
||||
# http://pyrocko.org - GPLv3
|
||||
#
|
||||
# The Pyrocko Developers, 21st Century
|
||||
# ---|P------/S----------~Lg----------
|
||||
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
from pyrocko.guts import Object, Timestamp
|
||||
|
||||
|
||||
class Constraint(Object):
|
||||
|
||||
tmin = Timestamp.T(optional=True)
|
||||
tmax = Timestamp.T(optional=True)
|
||||
|
||||
def contains(self, constraint):
|
||||
'''
|
||||
Check if the constraint completely includes a more restrictive one.
|
||||
'''
|
||||
|
||||
if self.tmin is not None and constraint.tmin is not None:
|
||||
b1 = self.tmin <= constraint.tmin
|
||||
elif self.tmin is None:
|
||||
b1 = True
|
||||
else:
|
||||
b1 = False
|
||||
|
||||
if self.tmax is not None and constraint.tmax is not None:
|
||||
b2 = constraint.tmax <= self.tmax
|
||||
elif self.tmax is None:
|
||||
b2 = True
|
||||
else:
|
||||
b2 = False
|
||||
|
||||
return b1 and b2
|
||||
|
||||
def expand(self, constraint):
|
||||
'''
|
||||
Widen constraint to include another given constraint.
|
||||
'''
|
||||
|
||||
if constraint.tmin is None or self.tmin is None:
|
||||
self.tmin = None
|
||||
else:
|
||||
self.tmin = min(constraint.tmin, self.tmin)
|
||||
|
||||
if constraint.tmax is None or self.tmax is None:
|
||||
self.tmax = None
|
||||
else:
|
||||
self.tmax = max(constraint.tmax, self.tmax)
|
||||
|
||||
|
||||
class Source(Object):
|
||||
|
||||
def update_channel_inventory(self, squirrel, constraint):
|
||||
'''
|
||||
Let local inventory be up-to-date with remote for a given constraint.
|
||||
'''
|
||||
|
||||
pass
|
||||
|
||||
def update_event_inventory(self, squirrel, constraint):
|
||||
'''
|
||||
Let local inventory be up-to-date with remote for a given constraint.
|
||||
'''
|
||||
|
||||
pass
|
||||
|
||||
def update_waveform_promises(self, squirrel, constraint):
|
||||
'''
|
||||
Let local inventory be up-to-date with remote for a given constraint.
|
||||
'''
|
||||
|
||||
pass
|
||||
|
||||
|
||||
__all__ = [
|
||||
'Source',
|
||||
'Constraint',
|
||||
]
|
||||
@ -0,0 +1,291 @@
|
||||
# http://pyrocko.org - GPLv3
|
||||
#
|
||||
# The Pyrocko Developers, 21st Century
|
||||
# ---|P------/S----------~Lg----------
|
||||
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
import os.path as op
|
||||
import logging
|
||||
import time
|
||||
try:
|
||||
import cPickle as pickle
|
||||
except ImportError:
|
||||
import pickle
|
||||
|
||||
from pyrocko import util
|
||||
from pyrocko.guts import String, Dict, Duration, dump_all
|
||||
|
||||
from .base import Source
|
||||
from ..model import ehash
|
||||
from ..lock import LockDir
|
||||
|
||||
logger = logging.getLogger('pyrocko.squirrel.client.catalog')
|
||||
|
||||
|
||||
class Link(object):
|
||||
def __init__(self, tmin, tmax, tmodified, nevents=-1, content_id=None):
|
||||
self.tmin = tmin
|
||||
self.tmax = tmax
|
||||
self.tmodified = tmodified
|
||||
self.nevents = nevents
|
||||
self.content_id = content_id
|
||||
|
||||
def __str__(self):
|
||||
return 'span %s - %s, access %s, nevents %i' % (
|
||||
util.tts(self.tmin),
|
||||
util.tts(self.tmax),
|
||||
util.tts(self.tmodified),
|
||||
self.nevents)
|
||||
|
||||
|
||||
class NoSuchCatalog(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_catalog(name):
|
||||
if name == 'geofon':
|
||||
from pyrocko.client.geofon import Geofon
|
||||
return Geofon()
|
||||
elif name == 'gcmt':
|
||||
from pyrocko.client.globalcmt import GlobalCMT
|
||||
return GlobalCMT()
|
||||
else:
|
||||
raise NoSuchCatalog(name)
|
||||
|
||||
|
||||
class CatalogSource(Source):
|
||||
|
||||
catalog = String.T()
|
||||
query_args = Dict.T(String.T(), String.T(), optional=True)
|
||||
expires = Duration.T(optional=True)
|
||||
anxious = Duration.T(optional=True)
|
||||
cache_path = String.T(optional=True)
|
||||
|
||||
def __init__(self, catalog, query_args=None, **kwargs):
|
||||
Source.__init__(self, catalog=catalog, query_args=query_args, **kwargs)
|
||||
|
||||
self._hash = self.make_hash()
|
||||
self._nevents_query_hint = 1000
|
||||
self._nevents_chunk_hint = 5000
|
||||
self._tquery = 3600.*24.
|
||||
self._tquery_limits = (3600., 3600.*24.*365.)
|
||||
|
||||
def setup(self, squirrel):
|
||||
self._force_query_age_max = self.anxious
|
||||
self._catalog = get_catalog(self.catalog)
|
||||
|
||||
self._cache_path = op.join(
|
||||
self.cache_path or squirrel._cache_path,
|
||||
'catalog',
|
||||
self.get_hash())
|
||||
|
||||
util.ensuredir(self._cache_path)
|
||||
|
||||
def make_hash(self):
|
||||
s = self.catalog
|
||||
if self.query_args is not None:
|
||||
s += ','.join(
|
||||
'%s:%s' % (k, self.query_args[k])
|
||||
for k in sorted(self.query_args.keys()))
|
||||
else:
|
||||
s += 'noqueryargs'
|
||||
|
||||
return ehash(s)
|
||||
|
||||
def get_hash(self):
|
||||
return self._hash
|
||||
|
||||
def update_event_inventory(self, squirrel, constraint=None):
|
||||
|
||||
with LockDir(self._cache_path):
|
||||
self._load_chain()
|
||||
|
||||
assert constraint is not None
|
||||
if constraint is not None:
|
||||
tmin, tmax = constraint.tmin, constraint.tmax
|
||||
|
||||
tmin_sq, tmax_sq = squirrel.get_time_span()
|
||||
|
||||
if tmin is None:
|
||||
tmin = tmin_sq
|
||||
|
||||
if tmax is None:
|
||||
tmax = tmax_sq
|
||||
|
||||
if tmin is None or tmax is None:
|
||||
logger.warn(
|
||||
'Cannot query catalog source "%s" without time '
|
||||
'constraint. Could not determine appropriate time '
|
||||
'constraint from current data holdings (no data?).'
|
||||
% self.catalog)
|
||||
|
||||
return
|
||||
|
||||
if tmin >= tmax:
|
||||
return
|
||||
|
||||
tnow = time.time()
|
||||
modified = False
|
||||
|
||||
if not self._chain:
|
||||
self._chain = [Link(tmin, tmax, tnow)]
|
||||
modified = True
|
||||
else:
|
||||
if tmin < self._chain[0].tmin:
|
||||
self._chain[0:0] = [Link(tmin, self._chain[0].tmin, tnow)]
|
||||
modified = True
|
||||
if self._chain[-1].tmax < tmax:
|
||||
self._chain.append(Link(self._chain[-1].tmax, tmax, tnow))
|
||||
modified = True
|
||||
|
||||
chain = []
|
||||
remove = []
|
||||
for link in self._chain:
|
||||
if tmin < link.tmax and link.tmin < tmax \
|
||||
and self._outdated(link, tnow):
|
||||
|
||||
if link.content_id:
|
||||
remove.append(
|
||||
self._get_events_file_path(link.content_id))
|
||||
|
||||