Browse Source

ewrica: WIP

- consistent config handling for different modules
- mole init refurbishments to generate config files (wip help strings)
pull/5/head
mmetz 1 year ago
parent
commit
5670c1e7cb
  1. 5
      examples/create_ensemble_files.py
  2. 7
      setup.py
  3. 20
      src/apps/mole.py
  4. 63
      src/config.py
  5. 5
      src/dataset/tsumaps.py
  6. 8
      src/gm/config.py
  7. 1940
      src/io/ids.py
  8. 8
      src/ls/config.py
  9. 11
      src/si/config.py
  10. 10
      src/sources.py
  11. 61
      src/store.py

5
examples/create_ensemble_files.py

@ -12,14 +12,13 @@ store_id = 'crust2_m2'
# would like to use, you can skip this step and point the *store_superdirs* in
# the next step to that directory.
# if not os.path.exists(store_id):
# ws.download_gf_store(site='kinherd', store_id=store_id)
if not os.path.exists(store_id):
ws.download_gf_store(site='kinherd', store_id=store_id)
# We need a pyrocko.gf.Engine object which provides us with the ground
# parameter. In this case we are going to use a local
# engine since we are going to query a local store.
engine = LocalEngine(store_superdirs=['.'])
engine = LocalEngine(store_superdirs=['/home/malde/src/gf_stores'])
store = engine.get_store(store_id)
# Create list of sources (should be of same type)

7
setup.py

@ -43,7 +43,12 @@ class CustomInstallCommand(install):
packages = [
'%s' % packname,
'%s.dataset' % packname,
'%s.apps' % packname]
'%s.apps' % packname,
'%s.gm' % packname,
'%s.ls' % packname,
'%s.si' % packname,
'%s.io' % packname,
'%s.message' % packname]
entry_points = {

20
src/apps/mole.py

@ -16,6 +16,7 @@ logger = logging.getLogger('ewrica.apps.mole')
def d2u(d):
return dict((k.replace('-', '_'), v) for (k, v) in d.items())
subcommand_descriptions = {
'init': 'create needed folder structure',
'mine': 'run source inversion and shake map estimation'
@ -97,6 +98,7 @@ def die(message, err=''):
def command_init(args):
modules = ('gm', 'ls', 'si')
def setup(parser):
parser.add_argument('storedir', help='main storage directory')
@ -109,7 +111,7 @@ def command_init(args):
'--modules',
action='store',
dest='modules',
choices=('si', 'gm', 'ls', 'all'),
choices=modules + ('all',),
default='all',
metavar='',
help='initialize only with certain module supports: '
@ -123,15 +125,18 @@ def command_init(args):
dir_dest = args.storedir
force = args.force
modules = args.modules
print(force)
store.ResultStorage.create_editeables(store_dir=dir_dest, force=force)
if 'all' in args.modules:
modules = modules
else:
modules = args.modules
# logger.info('(1) configure settings infiles:\n %s'
# % '\n '.join(config_filenames))
config_fns = store.ResultStorage.create_editeables(
store_dir=dir_dest, force=force, modules=modules)
# Create Mole config
logger.info('(1) configure settings in files:\n %s'
% '\n '.join(config_fns))
logger.info('(2) run "mole mine" in directory: "%s"' % dir_dest)
def command_mine(args):
@ -155,7 +160,6 @@ def command_mine(args):
parser, args = cl_parse('mine', args, setup=setup)
# Todo Check if in runing directory
pass
def main(args=None):

63
src/config.py

@ -3,13 +3,28 @@
# The Developers, 21st Century
import os
import os.path as op
import yaml
from pyrocko.config import (
PathWithPlaceholders, ConfigBase, BadConfig, expand, rec_expand, processed,
PathWithPlaceholders, ConfigBase, BadConfig, expand, processed,
mtime)
from pyrocko.guts import load, dump
from pyrocko.guts import load, Object, GutsSafeDumper, expand_stream_args
from pyrocko.util import ensuredirs
try:
newstr = unicode
range = xrange
except NameError:
newstr = str
def encode_utf8(s):
return s.encode('utf-8')
def no_encode(s):
return s
ewrica_dir_tmpl = os.environ.get('EWRICA_DIR', op.join('~', '.ewrica'))
@ -67,3 +82,47 @@ def write_config(conf, config_name='config'):
conf_path = expand(make_conf_path_tmpl(config_name))
ensuredirs(conf_path)
dump(conf, filename=conf_path)
class StoreConfig(Object):
pass
def dump_store_config(conf, filename=None, include_help=False):
dump(conf, filename=filename, header=True)
@expand_stream_args('w')
def dump(*args, **kwargs):
return _dump_with_help(*args, **kwargs)
def load_store_config(filename):
return load(filename=filename)
def _dump_with_help(
object, stream,
header=False,
Dumper=GutsSafeDumper,
_dump_function=yaml.dump):
if not getattr(stream, 'encoding', None):
enc = encode_utf8
else:
enc = no_encode
if header:
stream.write(enc(u'%YAML 1.1\n'))
if isinstance(header, (str, newstr)):
banner = u'\n'.join('# ' + x for x in header.splitlines()) + '\n'
stream.write(enc(banner))
# TODO add help as string into object between lines
print(object.__dir__())
_dump_function(
object,
stream=stream,
encoding='utf-8',
explicit_start=True,
Dumper=Dumper)

5
src/dataset/tsumaps.py

@ -1,3 +1,6 @@
# GPLv3
#
# The Developers, 21st Century
import os.path as op
import numpy as num
@ -20,6 +23,7 @@ def tsumaps_file():
return fn
def load():
'''Load tsumaps mat file into dictionary
@ -37,6 +41,7 @@ def load():
return sio.loadmat(tsumaps_file())
def get_probs(
lat, lon, interpolation='nearest_neighbor'):
'''Get tsumap focal mechanism probabilites at a given location

8
src/gm/config.py

@ -0,0 +1,8 @@
# GPLv3
#
# The Developers, 21st Century
from ewrica.config import StoreConfig
class Config(StoreConfig):
pass

1940
src/io/ids.py

File diff suppressed because it is too large

8
src/ls/config.py

@ -0,0 +1,8 @@
# GPLv3
#
# The Developers, 21st Century
from ewrica.config import StoreConfig
class Config(StoreConfig):
pass

11
src/si/config.py

@ -0,0 +1,11 @@
# GPLv3
#
# The Developers, 21st Century
from pyrocko.guts import Int
from ewrica.config import StoreConfig
class Config(StoreConfig):
no_runs = Int.T(
default=1000.,
help='Number of runs of grond ToDo')

10
src/sources.py

@ -5,7 +5,7 @@ import numpy as num
from pyrocko import gf, moment_tensor as pmt
from pyrocko.guts import (
Object, Float, List, String, StringChoice, Tuple, Timestamp,
Object, Float, List, String, StringChoice,
load_all, load_all_xml, dump_all, dump_all_xml, clone)
from pyrocko.guts_array import Array
@ -450,7 +450,7 @@ class SourceHeader(Object):
def set_help_string(self, source_class):
''' Gather the property helps for a source class in self.help_string
:param source_class: source class object which help functions are
gathered in the help string
:type source_class:
@ -536,7 +536,7 @@ def dump_sources(sources, dump_header=True, header=None, **kwargs):
:param header: If given, this header is used instead of an automatically
generated one.
:type header: optional, :py:class:`ewrica.sources.SourceHeader`
keyword arguments for :py:func:`pyrocko.guts.dump_all` as filename
'''
@ -563,7 +563,7 @@ def load_sources_xml(load_header=False, **kwargs):
:py:class:`ewrica.sources.EwricaIDSSource`
'''
#ToDo still buggy! Do not use
# ToDo still buggy! Do not use
return _source_and_header_list_from_load(
load_all_xml(**kwargs), load_header=load_header)
@ -586,7 +586,7 @@ def dump_sources_xml(sources, dump_header=True, header=None, **kwargs):
:param header: If given, this header is used instead of an automatically
generated one.
:type header: optional, :py:class:`ewrica.sources.SourceHeader`
keyword arguments for :py:func:`pyrocko.guts.dump_all` as filename
'''

61
src/store.py

@ -1,13 +1,51 @@
# GPLv3
#
# The Developers, 21st Century
import logging
import os
import os.path as op
import shutil
from pyrocko import util
from pyrocko.gf.store import CannotCreate, remake_dir
from pyrocko import util as putil
from pyrocko.guts import Object
from .config import dump_store_config
from .gm import config as gmconf
from .ls import config as lsconf
from .si import config as siconf
logger = logging.getLogger('ewrica.store')
module_configs = dict(
gm=gmconf,
ls=lsconf,
si=siconf)
def get_module_path(store_dir, module):
return op.join(store_dir, 'config', '%s.conf' % module)
def remake_dir(dpath, force):
if op.exists(dpath):
if force:
shutil.rmtree(dpath)
else:
raise CannotCreate('Directory "%s" already exists.' % dpath)
os.mkdir(dpath)
def remove_if_exists(fn, force=False):
if op.exists(fn):
if force:
os.remove(fn)
else:
raise CannotCreate('file %s already exists' % fn)
class CannotCreate(Exception):
pass
@ -16,12 +54,27 @@ class CannotCreate(Exception):
class ResultStorage(Object):
@staticmethod
def create_editeables(store_dir, force=False):
def create_editeables(store_dir, force=False, modules=None):
try:
util.ensuredir(store_dir)
putil.ensuredir(store_dir)
except Exception:
raise CannotCreate('cannot create directory %s' % store_dir)
for sub_dir in ['run', 'data', 'config']:
dpath = op.join(store_dir, sub_dir)
remake_dir(dpath, force)
fns = []
if modules:
for m in modules:
fn = get_module_path(store_dir, m)
remove_if_exists(fn, force=force)
dump_store_config(module_configs[m].Config(), filename=fn)
fns.append(fn)
return fns
__all__ = [
'ResultStorage']

Loading…
Cancel
Save