Browse Source

mole: setup of general running procedure

- modules contain specific tasks
- modules are scheduled
- offline mode is enabled using an eventid
pull/5/head
mmetz 1 year ago
parent
commit
621c0443ce
  1. BIN
      doc/source/static/mole_flow_chart.odg
  2. 12
      src/apps/mole.py
  3. 4
      src/gm/config.py
  4. 4
      src/gm/module.py
  5. 77
      src/io/grond.py
  6. 4
      src/ls/config.py
  7. 4
      src/ls/module.py
  8. 38
      src/module.py
  9. 4
      src/si/config.py
  10. 15
      src/si/module.py
  11. 56
      src/store.py

BIN
doc/source/static/mole_flow_chart.odg

Binary file not shown.

12
src/apps/mole.py

@ -150,16 +150,22 @@ def command_mine(args):
'default is %(default)s')
parser.add_argument(
'--offline', dest='offline', action='store_true',
help='run in offline not realtime mode (use downloaded data). '
'--event', dest='event_id', action='store',
help='If given, evaluate local data in ./data/<event_id>'
'default is %(default)s')
# parser.add_argument(
# '--offline', dest='offline', action='store_true',
# help='run in offline not realtime mode (use downloaded data). '
# 'default is %(default)s')
parser, args = cl_parse('mine', args, setup=setup)
storage = store.ResultStorage(store_dir='.')
storage.open()
storage.run(offline=args.offline)
storage.run(event_id=args.event_id)
# storage.run(offline=args.offline)
# TODO capture logs, errors etc

4
src/gm/config.py

@ -9,8 +9,8 @@ from ewrica.module import ModuleConfig
logger = logging.getLogger('ewrica.gm.config')
def get_config():
return GroundMotionConfig()
def get_config(*args, **kwargs):
return GroundMotionConfig(*args, **kwargs)
class GroundMotionConfig(ModuleConfig):

4
src/gm/module.py

@ -8,8 +8,8 @@ from ..module import BaseModule
logger = logging.getLogger('ewrica.gm.module')
def get_module():
return GroundMotionModule
def get_module(*args, **kwargs):
return GroundMotionModule(*args, **kwargs)
class GroundMotionModule(BaseModule):

77
src/io/grond.py

@ -0,0 +1,77 @@
# GPLv3
#
# The Developers, 21st Century
import logging
import numpy as num
import grond
from grond.environment import Environment
from grond.problems.base import ProblemDataNotAvailable
from pyrocko import guts # noqa
from pyrocko import moment_tensor as pmt
from pyrocko import orthodrome as pod
logger = logging.getLogger('ewrica.io.grond')
def history_to_sources(problem, history):
'''Convert grond history into ewrica.sources of appropiate type
'''
from ewrica import sources as ewsources
xs = history.get_sorted_primary_models()
bootstrap_misfits = history.get_sorted_primary_misfits()
base_source = problem.base_source
if isinstance(problem, grond.problems.cmt.problem.CMTProblem):
source = ewsources.MTSource
stf = base_source.stf.T.cls
moments = pmt.magnitude_to_moment(xs[:, 4])
xs[:, 5:11] *= moments[:, num.newaxis]
lats, lons = pod.ne_to_latlon(
base_source.lat, base_source.lon, xs[:, 1], xs[:, 2])
xs[:, 1] = lats
xs[:, 2] = lons
return [source(
time=base_source.time + i[0],
lat=i[1],
lon=i[2],
depth=i[3],
m6=i[5:11],
stf=stf(duration=i[11]),
stf_mode=base_source.stf_mode,
misfit=mf)
for i, mf in zip(xs, bootstrap_misfits)]
else:
# TODO
raise TypeError('Not implemented yet for the current problem type.')
def grond_to_sources(rundir, subset='harvest'):
'''Extract models from the grond run directory
'''
env = Environment(rundir)
problem = env.get_problem()
try:
history = env.get_history(subset=subset)
except ProblemDataNotAvailable:
# TODO Check behaviour
grond.harvest(
rundir, problem=None, nbest=10, force=False, weed=0,
export_fits=[])
history = env.get_history(subset=subset)
return history_to_sources(problem, history)

4
src/ls/config.py

@ -9,8 +9,8 @@ from ewrica.module import ModuleConfig
logger = logging.getLogger('ewrica.ls.config')
def get_config():
return LandSlideConfig()
def get_config(*args, **kwargs):
return LandSlideConfig(*args, **kwargs)
class LandSlideConfig(ModuleConfig):

4
src/ls/module.py

@ -9,8 +9,8 @@ from ..module import BaseModule
logger = logging.getLogger('ewrica.ls.module')
def get_module():
return LandSlideModule
def get_module(*args, **kwargs):
return LandSlideModule(*args, **kwargs)
class LandSlideModule(BaseModule):

38
src/module.py

@ -10,12 +10,15 @@ logger = logging.getLogger('ewrica.module')
class ModuleConfig(Object):
'''Configuration of the module run
'''
run = Bool.T(
default=True,
help='Set to True, if the module shall be runned, else False')
def dump_module_config(conf, filename=None, include_help=False):
# TODO check, if help can be dumped as well
dump(conf, filename=filename, header=True)
@ -23,7 +26,40 @@ def load_module_config(filename):
return load(filename=filename)
class ModuleLog(Object):
'''Log of the module run
'''
pass
def dump_module_log(log, filename=None):
dump(log, filename=filename, header=True)
def load_module_log(filename):
return load(filename=filename)
class BaseModule(Object):
# TODO generate basic functionalities as start function, evaluate, etc?
pass
config = ModuleConfig.T(default=ModuleConfig())
log = ModuleLog.T(default=ModuleLog())
def run(self):
if not self.config.run:
return
pass
def dump_config(self, *args, **kwargs):
dump_module_config(self.config, *args, **kwargs)
def load_config(self, *args, **kwargs):
self.config = load_module_config(*args, **kwargs)
def dump_log(self, *args, **kwargs):
dump_module_log(self.log, *args, **kwargs)
def load_log(self, *args, **kwargs):
self.log = load_module_log(*args, **kwargs)

4
src/si/config.py

@ -10,8 +10,8 @@ from ewrica.module import ModuleConfig
logger = logging.getLogger('ewrica.si.config')
def get_config():
return SourceInversionConfig()
def get_config(*args, **kwargs):
return SourceInversionConfig(*args, **kwargs)
class SourceInversionConfig(ModuleConfig):

15
src/si/module.py

@ -8,9 +8,18 @@ from ..module import BaseModule
logger = logging.getLogger('ewrica.si.module')
def get_module():
return SourceInversionModule
def get_module(*args, **kwargs):
return SourceInversionModule(*args, **kwargs)
class SourceInversionModule(BaseModule):
pass
def run(self):
# TODO call grond
# TODO use grond results (raw, so misfit and models) for statistics
# TODO convert grond results to own source models
if not self.config.run:
return
logger.info('running Source Inversion')

56
src/store.py

@ -20,16 +20,31 @@ from .si import config as si_conf, module as si_mod
logger = logging.getLogger('ewrica.store')
module_configs = dict(
available_module_configs = dict(
gm=gm_conf,
ls=ls_conf,
si=si_conf)
modules = dict(
gm=gm_mod,
ls=ls_mod,
si=si_mod)
schedule = {
si_conf.get_config().T.classname: 0,
gm_conf.get_config().T.classname: 1,
ls_conf.get_config().T.classname: 2}
config_to_module = {
si_conf.get_config().T.classname: si_mod,
gm_conf.get_config().T.classname: gm_mod,
ls_conf.get_config().T.classname: ls_mod}
def schedule_configs(module_config):
return schedule[module_config.T.classname]
def get_module_for_config(module_config):
return config_to_module[module_config.T.classname].get_module(
config=module_config)
def get_module_path(store_dir, module):
@ -83,7 +98,8 @@ class ResultStorage(Object):
for m in modules:
fn = get_module_path(store_dir, m)
remove_if_exists(fn, force=force)
dump_module_config(module_configs[m].get_config(), filename=fn)
dump_module_config(
available_module_configs[m].get_config(), filename=fn)
fns.append(fn)
@ -121,21 +137,29 @@ class ResultStorage(Object):
for fn in conf_fns:
self.configs.append(load_module_config(filename=fn))
def start_modules(self):
# TODO
self.configs.sort(key=schedule_configs)
def start_modules_offline(self, event_id):
# TODO Check for event folder, event file, meta data and data
modules = [get_module_for_config(c) for c in self.configs]
for module in modules:
module.run()
def start_modules_online(self):
# TODO invoke squirrel
# TODO downsample broad band data from seed link
# TODO Flow: Data download etc, messaging listener, start_modules
# TODO Parallel processes as new data comes in?
pass
def run(self, offline=False):
# TODO run schedule (1st si, 2nd gm, 3rd ls)
def run(self, event_id=None):
# TODO run all modules with config and config.run == True
if offline:
self.start_modules()
if event_id:
self.start_modules_offline(event_id=event_id)
else:
# TODO invoke squirrel
# TODO downsample broad band data from seed link
# TODO Flow: Data download etc, messaging listener, start_modules
pass
self.start_modules_online()
__all__ = [

Loading…
Cancel
Save