Browse Source

mole, module work, basic architecture

pull/5/head
mmetz 1 year ago
parent
commit
a448a9c1e9
  1. 1
      README.md
  2. 1
      setup.py
  3. 15
      src/apps/mole.py
  4. 47
      src/config.py
  5. 0
      src/data/examples
  6. 13
      src/gm/config.py
  7. 16
      src/gm/module.py
  8. 13
      src/ls/config.py
  9. 17
      src/ls/module.py
  10. 29
      src/module.py
  11. 13
      src/si/config.py
  12. 16
      src/si/module.py
  13. 74
      src/store.py

1
README.md

@ -0,0 +1 @@
Here could be your Readme

1
setup.py

@ -88,5 +88,6 @@ setup(
'Topic :: Scientific/Engineering'],
package_data={
packname: ['data/tsumaps_all_params.*',
'data/examples/*'
] + get_readme_paths()}
)

15
src/apps/mole.py

@ -137,7 +137,7 @@ def command_init(args):
def command_mine(args):
# ToDo create caller for infinite running with multistep inversion etc
# TODO create caller for infinite running with multistep inversion etc
# Data check on regular base (or when xml is changed) (squirrel)
# Synchronize via seed link with GNSS data etc. -> Store in cache
# When trigger, start multistep inversion based on config
@ -156,17 +156,12 @@ def command_mine(args):
parser, args = cl_parse('mine', args, setup=setup)
store.ResultStorage.is_store(store_dir='.')
storage = store.ResultStorage(store_dir='.')
storage.open()
# TODO define modules used based on general config
# or check available configs?
if args.offline:
pass
else:
# TODO invoke squirrel
pass
storage.run(offline=args.offline)
# TODO capture logs, errors etc
def main(args=None):

47
src/config.py

@ -3,12 +3,11 @@
# The Developers, 21st Century
import os
import os.path as op
import yaml
from pyrocko.config import (
PathWithPlaceholders, ConfigBase, BadConfig, expand, processed,
mtime)
from pyrocko.guts import load, Object, GutsSafeDumper, expand_stream_args
from pyrocko.guts import load, dump
from pyrocko.util import ensuredirs
try:
@ -82,47 +81,3 @@ def write_config(conf, config_name='config'):
conf_path = expand(make_conf_path_tmpl(config_name))
ensuredirs(conf_path)
dump(conf, filename=conf_path)
class StoreConfig(Object):
pass
def dump_store_config(conf, filename=None, include_help=False):
dump(conf, filename=filename, header=True)
@expand_stream_args('w')
def dump(*args, **kwargs):
return _dump_with_help(*args, **kwargs)
def load_store_config(filename):
return load(filename=filename)
def _dump_with_help(
object, stream,
header=False,
Dumper=GutsSafeDumper,
_dump_function=yaml.dump):
if not getattr(stream, 'encoding', None):
enc = encode_utf8
else:
enc = no_encode
if header:
stream.write(enc(u'%YAML 1.1\n'))
if isinstance(header, (str, newstr)):
banner = u'\n'.join('# ' + x for x in header.splitlines()) + '\n'
stream.write(enc(banner))
# TODO add help as string into object between lines
# print(object.__dir__())
_dump_function(
object,
stream=stream,
encoding='utf-8',
explicit_start=True,
Dumper=Dumper)

0
src/data/examples

13
src/gm/config.py

@ -1,8 +1,17 @@
# GPLv3
#
# The Developers, 21st Century
from ewrica.config import StoreConfig
import logging
from ewrica.module import ModuleConfig
class Config(StoreConfig):
logger = logging.getLogger('ewrica.gm.config')
def get_config():
return GroundMotionConfig()
class GroundMotionConfig(ModuleConfig):
pass

16
src/gm/module.py

@ -0,0 +1,16 @@
# GPLv3
#
# The Developers, 21st Century
import logging
from ..module import BaseModule
logger = logging.getLogger('ewrica.gm.module')
def get_module():
return GroundMotionModule
class GroundMotionModule(BaseModule):
pass

13
src/ls/config.py

@ -1,8 +1,17 @@
# GPLv3
#
# The Developers, 21st Century
from ewrica.config import StoreConfig
import logging
from ewrica.module import ModuleConfig
class Config(StoreConfig):
logger = logging.getLogger('ewrica.ls.config')
def get_config():
return LandSlideConfig()
class LandSlideConfig(ModuleConfig):
pass

17
src/ls/module.py

@ -0,0 +1,17 @@
# GPLv3
#
# The Developers, 21st Century
import logging
from ..module import BaseModule
logger = logging.getLogger('ewrica.ls.module')
def get_module():
return LandSlideModule
class LandSlideModule(BaseModule):
pass

29
src/module.py

@ -0,0 +1,29 @@
# GPLv3
#
# The Developers, 21st Century
import logging
from pyrocko.guts import load, dump, Bool, Object
logger = logging.getLogger('ewrica.module')
class ModuleConfig(Object):
run = Bool.T(
default=True,
help='Set to True, if the module shall be runned, else False')
def dump_module_config(conf, filename=None, include_help=False):
dump(conf, filename=filename, header=True)
def load_module_config(filename):
return load(filename=filename)
class BaseModule(Object):
# TODO generate basic functionalities as start function, evaluate, etc?
pass

13
src/si/config.py

@ -1,11 +1,20 @@
# GPLv3
#
# The Developers, 21st Century
import logging
from pyrocko.guts import Int
from ewrica.config import StoreConfig
from ewrica.module import ModuleConfig
logger = logging.getLogger('ewrica.si.config')
def get_config():
return SourceInversionConfig()
class Config(StoreConfig):
class SourceInversionConfig(ModuleConfig):
no_runs = Int.T(
default=1000.,
help='Number of runs of grond ToDo')

16
src/si/module.py

@ -0,0 +1,16 @@
# GPLv3
#
# The Developers, 21st Century
import logging
from ..module import BaseModule
logger = logging.getLogger('ewrica.si.module')
def get_module():
return SourceInversionModule
class SourceInversionModule(BaseModule):
pass

74
src/store.py

@ -10,20 +10,26 @@ from pyrocko import util as putil
from pyrocko.guts import Object
from ewrica import util
from .config import dump_store_config
from .module import dump_module_config, load_module_config
from .gm import config as gmconf
from .ls import config as lsconf
from .si import config as siconf
from .gm import config as gm_conf, module as gm_mod
from .ls import config as ls_conf, module as ls_mod
from .si import config as si_conf, module as si_mod
logger = logging.getLogger('ewrica.store')
module_configs = dict(
gm=gmconf,
ls=lsconf,
si=siconf)
gm=gm_conf,
ls=ls_conf,
si=si_conf)
modules = dict(
gm=gm_mod,
ls=ls_mod,
si=si_mod)
def get_module_path(store_dir, module):
@ -48,6 +54,10 @@ def remove_if_exists(fn, force=False):
raise CannotCreate('file %s already exists' % fn)
def store_subdirs(store_dir):
return [op.join(store_dir, d) for d in ['run', 'data', 'config']]
class CannotCreate(Exception):
pass
@ -56,9 +66,6 @@ class StoreError(Exception):
pass
store_subdirs = ['run', 'data', 'config']
class ResultStorage(Object):
@staticmethod
@ -68,16 +75,15 @@ class ResultStorage(Object):
except Exception:
raise CannotCreate('cannot create directory %s' % store_dir)
for sub_dir in store_subdirs:
dpath = op.join(store_dir, sub_dir)
remake_dir(dpath, force)
for sub_dir in store_subdirs(store_dir):
remake_dir(sub_dir, force)
fns = []
if modules:
for m in modules:
fn = get_module_path(store_dir, m)
remove_if_exists(fn, force=force)
dump_store_config(module_configs[m].Config(), filename=fn)
dump_module_config(module_configs[m].get_config(), filename=fn)
fns.append(fn)
@ -87,9 +93,9 @@ class ResultStorage(Object):
def is_store(store_dir):
fail_dir = []
for sub_dir in store_subdirs:
if not util.is_dir(op.join(store_dir, sub_dir)):
fail_dir.append(sub_dir)
for sub_dir in store_subdirs(store_dir):
if not util.is_dir(sub_dir):
fail_dir.append(op.basename(sub_dir))
if fail_dir:
raise StoreError(
@ -97,6 +103,40 @@ class ResultStorage(Object):
'The following directories are missing:\n - %s' % (
store_dir, store_dir, '\n - '.join(fail_dir)))
def __init__(self, store_dir):
try:
ResultStorage.is_store(store_dir)
self.store_dir = store_dir
except StoreError as e:
raise e
self.configs = []
def open(self):
conf_dir = op.join(self.store_dir, 'config')
conf_fns = [
op.join(conf_dir, fn) for fn in os.listdir(conf_dir)
if fn.endswith('.conf')]
for fn in conf_fns:
self.configs.append(load_module_config(filename=fn))
def start_modules(self):
# TODO
pass
def run(self, offline=False):
# TODO run schedule (1st si, 2nd gm, 3rd ls)
# TODO run all modules with config and config.run == True
if offline:
self.start_modules()
else:
# TODO invoke squirrel
# TODO downsample broad band data from seed link
# TODO Flow: Data download etc, messaging listener, start_modules
pass
__all__ = [
'ResultStorage']

Loading…
Cancel
Save