mole: setup of general running procedure

- modules contain specific tasks
- modules are scheduled
- offline mode is enabled using an eventid

src/apps/mole.py

@@ -150,16 +150,22 @@ def command_mine(args):
                  'default is %(default)s')
 
         parser.add_argument(
-            '--offline', dest='offline', action='store_true',
-            help='run in offline not realtime mode (use downloaded data). '
+            '--event', dest='event_id', action='store',
+            help='If given, evaluate local data in ./data/<event_id>'
                  'default is %(default)s')
 
+        # parser.add_argument(
+        #     '--offline', dest='offline', action='store_true',
+        #     help='run in offline not realtime mode (use downloaded data). '
+        #          'default is %(default)s')
+
     parser, args = cl_parse('mine', args, setup=setup)
 
     storage = store.ResultStorage(store_dir='.')
     storage.open()
 
-    storage.run(offline=args.offline)
+    storage.run(event_id=args.event_id)
+    # storage.run(offline=args.offline)
 
     # TODO capture logs, errors etc
 

src/gm/config.py

@@ -9,8 +9,8 @@ from ewrica.module import ModuleConfig
 logger = logging.getLogger('ewrica.gm.config')
 
 
-def get_config():
-    return GroundMotionConfig()
+def get_config(*args, **kwargs):
+    return GroundMotionConfig(*args, **kwargs)
 
 
 class GroundMotionConfig(ModuleConfig):

src/gm/module.py

@@ -8,8 +8,8 @@ from ..module import BaseModule
 logger = logging.getLogger('ewrica.gm.module')
 
 
-def get_module():
-    return GroundMotionModule
+def get_module(*args, **kwargs):
+    return GroundMotionModule(*args, **kwargs)
 
 
 class GroundMotionModule(BaseModule):

src/io/grond.py

@@ -0,0 +1,77 @@
+# GPLv3
+#
+# The Developers, 21st Century
+import logging
+
+import numpy as num
+
+import grond
+from grond.environment import Environment
+from grond.problems.base import ProblemDataNotAvailable
+
+from pyrocko import guts  # noqa
+from pyrocko import moment_tensor as pmt
+from pyrocko import orthodrome as pod
+
+logger = logging.getLogger('ewrica.io.grond')
+
+
+def history_to_sources(problem, history):
+    '''Convert grond history into ewrica.sources of appropiate type
+
+    '''
+    from ewrica import sources as ewsources
+
+    xs = history.get_sorted_primary_models()
+    bootstrap_misfits = history.get_sorted_primary_misfits()
+
+    base_source = problem.base_source
+
+    if isinstance(problem, grond.problems.cmt.problem.CMTProblem):
+        source = ewsources.MTSource
+
+        stf = base_source.stf.T.cls
+
+        moments = pmt.magnitude_to_moment(xs[:, 4])
+        xs[:, 5:11] *= moments[:, num.newaxis]
+
+        lats, lons = pod.ne_to_latlon(
+            base_source.lat, base_source.lon, xs[:, 1], xs[:, 2])
+        xs[:, 1] = lats
+        xs[:, 2] = lons
+
+        return [source(
+            time=base_source.time + i[0],
+            lat=i[1],
+            lon=i[2],
+            depth=i[3],
+            m6=i[5:11],
+            stf=stf(duration=i[11]),
+            stf_mode=base_source.stf_mode,
+            misfit=mf)
+            for i, mf in zip(xs, bootstrap_misfits)]
+
+    else:
+        # TODO
+        raise TypeError('Not implemented yet for the current problem type.')
+
+
+def grond_to_sources(rundir, subset='harvest'):
+    '''Extract models from the grond run directory
+
+    '''
+    env = Environment(rundir)
+
+    problem = env.get_problem()
+
+    try:
+        history = env.get_history(subset=subset)
+    except ProblemDataNotAvailable:
+        # TODO Check behaviour
+        grond.harvest(
+            rundir, problem=None, nbest=10, force=False, weed=0,
+            export_fits=[])
+
+        history = env.get_history(subset=subset)
+
+    return history_to_sources(problem, history)

src/ls/config.py

@@ -9,8 +9,8 @@ from ewrica.module import ModuleConfig
 logger = logging.getLogger('ewrica.ls.config')
 
 
-def get_config():
-    return LandSlideConfig()
+def get_config(*args, **kwargs):
+    return LandSlideConfig(*args, **kwargs)
 
 
 class LandSlideConfig(ModuleConfig):

src/ls/module.py

@@ -9,8 +9,8 @@ from ..module import BaseModule
 logger = logging.getLogger('ewrica.ls.module')
 
 
-def get_module():
-    return LandSlideModule
+def get_module(*args, **kwargs):
+    return LandSlideModule(*args, **kwargs)
 
 
 class LandSlideModule(BaseModule):

src/module.py

@@ -10,12 +10,15 @@ logger = logging.getLogger('ewrica.module')
 
 
 class ModuleConfig(Object):
+    '''Configuration of the module run
+    '''
     run = Bool.T(
         default=True,
         help='Set to True, if the module shall be runned, else False')
 
 
 def dump_module_config(conf, filename=None, include_help=False):
+    # TODO check, if help can be dumped as well
     dump(conf, filename=filename, header=True)
 
 
@@ -23,7 +26,40 @@ def load_module_config(filename):
     return load(filename=filename)
 
 
+class ModuleLog(Object):
+    '''Log of the module run
+    '''
+    pass
+
+
+def dump_module_log(log, filename=None):
+    dump(log, filename=filename, header=True)
+
+
+def load_module_log(filename):
+    return load(filename=filename)
+
+
 class BaseModule(Object):
     # TODO generate basic functionalities as start function, evaluate, etc?
 
-    pass
+    config = ModuleConfig.T(default=ModuleConfig())
+    log = ModuleLog.T(default=ModuleLog())
+
+    def run(self):
+        if not self.config.run:
+            return
+
+        pass
+
+    def dump_config(self, *args, **kwargs):
+        dump_module_config(self.config, *args, **kwargs)
+
+    def load_config(self, *args, **kwargs):
+        self.config = load_module_config(*args, **kwargs)
+
+    def dump_log(self, *args, **kwargs):
+        dump_module_log(self.log, *args, **kwargs)
+
+    def load_log(self, *args, **kwargs):
+        self.log = load_module_log(*args, **kwargs)

src/si/config.py

@@ -10,8 +10,8 @@ from ewrica.module import ModuleConfig
 logger = logging.getLogger('ewrica.si.config')
 
 
-def get_config():
-    return SourceInversionConfig()
+def get_config(*args, **kwargs):
+    return SourceInversionConfig(*args, **kwargs)
 
 
 class SourceInversionConfig(ModuleConfig):

src/si/module.py

@@ -8,9 +8,18 @@ from ..module import BaseModule
 logger = logging.getLogger('ewrica.si.module')
 
 
-def get_module():
-    return SourceInversionModule
+def get_module(*args, **kwargs):
+    return SourceInversionModule(*args, **kwargs)
 
 
 class SourceInversionModule(BaseModule):
-    pass
+
+    def run(self):
+        # TODO call grond
+        # TODO use grond results (raw, so misfit and models) for statistics
+        # TODO convert grond results to own source models
+
+        if not self.config.run:
+            return
+
+        logger.info('running Source Inversion')

src/store.py

@@ -20,16 +20,31 @@ from .si import config as si_conf, module as si_mod
 logger = logging.getLogger('ewrica.store')
 
 
-module_configs = dict(
+available_module_configs = dict(
     gm=gm_conf,
     ls=ls_conf,
     si=si_conf)
 
 
-modules = dict(
-    gm=gm_mod,
-    ls=ls_mod,
-    si=si_mod)
+schedule = {
+    si_conf.get_config().T.classname: 0,
+    gm_conf.get_config().T.classname: 1,
+    ls_conf.get_config().T.classname: 2}
+
+
+config_to_module = {
+    si_conf.get_config().T.classname: si_mod,
+    gm_conf.get_config().T.classname: gm_mod,
+    ls_conf.get_config().T.classname: ls_mod}
+
+
+def schedule_configs(module_config):
+    return schedule[module_config.T.classname]
+
+
+def get_module_for_config(module_config):
+    return config_to_module[module_config.T.classname].get_module(
+        config=module_config)
 
 
 def get_module_path(store_dir, module):
@@ -83,7 +98,8 @@ class ResultStorage(Object):
             for m in modules:
                 fn = get_module_path(store_dir, m)
                 remove_if_exists(fn, force=force)
-                dump_module_config(module_configs[m].get_config(), filename=fn)
+                dump_module_config(
+                    available_module_configs[m].get_config(), filename=fn)
 
                 fns.append(fn)
 
@@ -121,21 +137,29 @@ class ResultStorage(Object):
         for fn in conf_fns:
             self.configs.append(load_module_config(filename=fn))
 
-    def start_modules(self):
-        # TODO
+        self.configs.sort(key=schedule_configs)
+
+    def start_modules_offline(self, event_id):
+        # TODO Check for event folder, event file, meta data and data
+        modules = [get_module_for_config(c) for c in self.configs]
+
+        for module in modules:
+            module.run()
+
+    def start_modules_online(self):
+        # TODO invoke squirrel
+        # TODO downsample broad band data from seed link
+        # TODO Flow: Data download etc, messaging listener, start_modules
+        # TODO Parallel processes as new data comes in?
         pass
 
-    def run(self, offline=False):
-        # TODO run schedule (1st si, 2nd gm, 3rd ls)
+    def run(self, event_id=None):
         # TODO run all modules with config and config.run == True
 
-        if offline:
-            self.start_modules()
+        if event_id:
+            self.start_modules_offline(event_id=event_id)
         else:
-            # TODO invoke squirrel
-            # TODO downsample broad band data from seed link
-            # TODO Flow: Data download etc, messaging listener, start_modules
-            pass
+            self.start_modules_online()
 
 
 __all__ = [