Tools, scripts, apps and functions to run fast source and ground shaking estimation
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

406 lines
12 KiB

# GPLv3
#
# The Developers, 21st Century
import logging
import tempfile
import shutil
import signal
from subprocess import Popen, PIPE
import os
import os.path as op
from pyrocko.guts import Object, Bool
from pyrocko.model.gnss import GNSSCampaign
from ewrica.sources import dump_sources, EwricaIDSSource, load_one_source
from ewrica.io.ids import (
load_ids_source, load_obssyn, load_obssyn_gnss, WaveformResultList)
from .config import IDSConfigFull
from .dataset import Dataset
from .targets import WaveformTargetGroup, GNSSTargetGroup, InSARTargetGroup
logger = logging.getLogger('ewrica.si.ids.core')
km = 1e3
program_bins = {
'ids.2020': 'ids2020'
}
class IDSError(Exception):
pass
class IDSRunner(Object):
config = IDSConfigFull.T(
default=IDSConfigFull())
source = EwricaIDSSource.T(optional=True)
keep_tmp = Bool.T(default=False)
def __init__(self, *args, **kwargs):
Object.__init__(self, *args, **kwargs)
self._tempdir = ''
self._dataset = None
@property
def tempdir(self):
if not self._tempdir:
self._tempdir = tempfile.mkdtemp('', 'idsrunner-tmp-')
return self._tempdir
@property
def inputdir(self):
return op.join(self.tempdir, self.config.inputdir)
@property
def waveformdir(self):
return op.join(self.tempdir, self.config.waveformdir)
@property
def gnssdir(self):
return op.join(self.tempdir, self.config.gnssdir)
@property
def gnssfile(self):
return op.join(self.tempdir, self.config.gnssfile)
@property
def insardir(self):
return op.join(self.tempdir, self.config.insardir)
@property
def faultdir(self):
return op.join(self.tempdir, self.config.faultdir)
@property
def faultfile(self):
return op.join(self.tempdir, self.config.faultfile)
@property
def rundir(self):
return op.join(self.tempdir, self.config.rundir)
@property
def resultdir(self):
return op.join(self.tempdir, 'results')
@property
def reportdir(self):
return 'report'
@property
def tracefile(self):
return op.join(self.resultdir, 'waveform_fits.yaml')
# return op.join(self.resultdir, 'traces.mseed')
@property
def gnssobs_file(self):
return op.join(self.resultdir, 'gnss_obs.yaml')
@property
def gnsssyn_file(self):
return op.join(self.resultdir, 'gnss_syn.yaml')
@property
def insarfile(self):
return op.join(self.resultdir, 'insar.yaml')
@property
def sourcefile(self):
return op.join(self.resultdir, 'idssource.yaml')
@property
def configfile(self):
return op.join(self.tempdir, self.config.inputdir, 'ids_config.yaml')
@property
def _input_fn(self):
return op.join(self.tempdir, self.config.inputfn)
@classmethod
def from_rundir(cls, rundir):
if not op.exists(rundir):
raise IDSError('Running directory {} not found'.format(rundir))
runner = cls()
runner._tempdir = rundir
runner.config = IDSConfigFull()
runner.config = IDSConfigFull.load(filename=runner.configfile)
if not op.exists(op.join(runner.inputdir)):
raise IDSError(
'Input directory {} not found'.format(runner.inputdir))
if not op.exists(op.join(runner.rundir)):
raise IDSError(
'Running directory {} not found'.format(runner.rundir))
if not op.exists(op.join(runner.resultdir)):
raise IDSError(
'Result directory {} not found'.format(runner.resultdir))
return runner
def get_dataset(self):
if self._dataset is None:
wf_targets = []
gnss_target = None
insar_target = None
for c in self.config.waveform_config:
wf_targets.append(WaveformTargetGroup(config=c))
if self.config.gnss_config:
gnss_target = GNSSTargetGroup(config=self.config.gnss_config)
if self.config.insar_config:
insar_target = InSARTargetGroup(
config=self.config.insar_config)
self._dataset = Dataset(
config=self.config.dataset_config,
waveform_targetgroups=wf_targets,
gnss_targetgroup=gnss_target,
insar_targetgroup=insar_target)
return self._dataset
@classmethod
def init(cls, rundir, force=False):
if op.exists(rundir) and not force:
raise FileExistsError(
'Given rundir {} already exists. Consider the '
'"force" option'.format(rundir))
elif op.exists(rundir):
shutil.rmtree(rundir)
os.makedirs(rundir)
os.makedirs(op.join(rundir, 'config'))
runner = cls()
runner.config.dump(
filename=op.join(rundir, 'config', 'idsconfig_example.conf'),
header=True)
def _prepare_ids_dir(self):
dirs = \
[self.waveformdir, self.gnssdir, self.insardir, self.faultdir] + \
[op.join(self.rundir, d) for d in ('O2S', 'EBC')]
for d in dirs:
os.makedirs(d)
def prepare(self, *args, **kwargs):
self._prepare_ids_dir(*args, **kwargs)
ds = self.get_dataset()
ds.set_waveform_data(self.waveformdir)
ds.set_gnss_data(self.gnssdir)
ds.set_insar_data(self.insardir)
if ds.config.subfault_path is not None:
ds.set_fault_data(self.faultfile)
self.config.dump(
filename=op.join(
self.tempdir, self.config.inputdir, 'ids_config.yaml'),
header=True)
def run(self):
config = self.config
input_fn = self._input_fn
with open(input_fn, 'wb') as f:
input_str = config.string_for_config(self.get_dataset())
logger.debug('===== begin ids input =====\n'
'%s===== end ids input =====' % input_str.decode())
f.write(input_str)
program = program_bins['ids.%s' % config.ids_version]
symlink = None
if config.exe_path is not None:
symlink = op.join(self.inputdir, program)
os.symlink(config.exe_path, symlink)
logger.info('Created symbolic link in {} to {}'.format(
self.inputdir, config.exe_path))
program = './' + program
old_wd = os.getcwd()
os.chdir(self.inputdir)
interrupted = []
def signal_handler(signum, frame):
os.kill(proc.pid, signal.SIGTERM)
interrupted.append(True)
logger.info('Start {} computation'.format(program))
original = signal.signal(signal.SIGINT, signal_handler)
try:
try:
proc = Popen(program, stdin=PIPE, stdout=PIPE, stderr=PIPE)
except FileNotFoundError:
proc = Popen(
'./' + program, stdin=PIPE, stdout=PIPE, stderr=PIPE)
except OSError:
os.chdir(old_wd)
raise IDSError(
'''could not start ids executable: "%s"
Available ids backends and download links to the modelling codes are listed
on
https://git.pyrocko.org/wanderelch/ewrica
''' % program)
(output_str, error_str) = proc.communicate(
'{}\n'.format(input_fn).encode())
finally:
signal.signal(signal.SIGINT, original)
if interrupted:
raise KeyboardInterrupt()
logger.info('===== begin ids output =====\n'
'%s===== end ids output =====' % output_str.decode())
errmess = []
if proc.returncode != 0:
errmess.append(
'ids had a non-zero exit state: %i' % proc.returncode)
if error_str:
if error_str.lower().find(b'error') != -1:
errmess.append("the string 'error' appeared in ids error")
else:
logger.warn(
'ids emitted something via stderr:\n\n%s'
% error_str.decode())
if output_str.lower().find(b'error') != -1:
errmess.append("the string 'error' appeared in ids output")
if errmess:
self.keep_tmp = True
os.chdir(old_wd)
raise IDSError('''
===== begin ids input =====
%s===== end ids input =====
===== begin ids output =====
%s===== end ids output =====
===== begin ids error =====
%s===== end ids error =====
%s
ids has been invoked as "%s"
in the directory %s'''.lstrip() % (
input_str.decode(), output_str.decode(), error_str.decode(),
'\n'.join(errmess), program, self.tempdir))
self.ids_output = output_str
self.ids_error = error_str
if symlink is not None:
os.remove(symlink)
os.chdir(old_wd)
def harvest(self, force=False):
# TODO return directory back, if error occurs
source = load_ids_source(self._input_fn)
source.name = self.config.dataset_config.event_name
outpath_tmpl = self.config.run_config.outpath_template
if outpath_tmpl is not None:
logger.info('Move run dir to {} ...'.format(outpath_tmpl))
if op.exists(outpath_tmpl) and not force:
raise IDSError('Configured outpath exists already. Consider '
'using the "force" argument')
if self.tempdir == outpath_tmpl:
self._tempdir = ''
shutil.copytree(outpath_tmpl, self.tempdir, dirs_exist_ok=True)
result_dir = self.resultdir
if op.exists(result_dir):
shutil.rmtree(result_dir)
os.makedirs(result_dir)
# Dump source
dump_sources(
[source],
filename=self.sourcefile)
# Dump observed/modelled traces
config_fn = op.join(self.tempdir, self.config.inputfn)
waveform_results = load_obssyn(config_fn=config_fn)
waveform_results.dump(
filename=self.tracefile,
header=True)
# Dump observed/modelled GNSS data (if available)
try:
gnss_obs, gnss_syn = load_obssyn_gnss(config_fn=config_fn)
gnss_obs.dump(filename=self.gnssobs_file, header=True)
gnss_syn.dump(filename=self.gnsssyn_file, header=True)
except FileNotFoundError:
pass
if op.exists(outpath_tmpl):
shutil.rmtree(outpath_tmpl)
shutil.copytree(
self.tempdir,
outpath_tmpl)
if not self.keep_tmp:
shutil.rmtree(self.tempdir)
return source
def report(self, cwd='.'):
from ewrica.si.ids.report import create_report
source = load_one_source(filename=self.sourcefile)
waveform_result = WaveformResultList.load(filename=self.tracefile)
gnss_obs, gnss_syn = None, None
if op.exists(self.gnssobs_file):
gnss_obs = GNSSCampaign.load(filename=self.gnssobs_file)
if op.exists(self.gnsssyn_file):
gnss_syn = GNSSCampaign.load(filename=self.gnsssyn_file)
reportdir = op.join(cwd, self.reportdir)
run = self.config.run_config.run
create_report(
reportdir,
run,
source,
waveform_result,
gnss_obs=gnss_obs,
gnss_syn=gnss_syn)