Browse Source

read in speedup and non combined bootstrap fix

obspy
asteinbe 2 years ago
parent
commit
ff65b7a7f8
  1. 105
      src/common/Logfile.py
  2. 2
      src/process/ProcessProgs.py
  3. 27
      src/process/array_crosscorrelation_v4.py
  4. 24
      src/process/main.py
  5. 5
      src/process/sembCalc.py
  6. 2
      src/process/semp.py

105
src/common/Logfile.py

@ -2,32 +2,27 @@ import os
import getpass
import datetime
import sys
import traceback
from palantiri.common import Basic
from palantiri.common import Globals
MSG_TOKEN = '##'
MAX_LINES = 200000 # Max number of lines in one program run
MAX_LINES = 200000
ABORT_TOKEN = MSG_TOKEN + ' ABORT '
# Global variables
g_ProtLineNr = 1
g_ProtLineNr= 1 # current linenumber in runtime log
g_RuntimeLog= None # Filename
g_RuntimeLog = None
g_UseRunTimeLog = True
g_ErrorLog = None
g_UseErrorLog = False
g_IsVisible = True # Output to terminal
g_ErrorLog = None
g_UseErrorLog = False
g_IsVisible = True
def baseLogFileName(postfix):
s = sys.argv [0]
s = sys.argv[0]
return Basic.baseFileName(s) + postfix + '.log'
@ -45,11 +40,6 @@ def setRuntimeLog(onOff):
global g_UseRunTimeLog
g_UseRunTimeLog = onOff
def onlyErrorLog(onOff):
if onOff: setVisible(False); setRuntimeLog(False); setErrorLog(True)
else: setVisible(True); setRuntimeLog(True); setErrorLog(False)
# -------------------------------------------------------------------------------------------------
def init(runTimeLog=None, errorLog=None, startMsg=None):
@ -64,11 +54,12 @@ def init(runTimeLog=None, errorLog=None, startMsg=None):
else:
g_RuntimeLog = initFile(runTimeLog)
if errorLog == None: g_ErrorLog = initFile()
else: g_ErrorLog = initFile(errorLog)
if errorLog == None:
g_ErrorLog = initFile()
else:
g_ErrorLog = initFile(errorLog)
# Remove information of older versions from errorlog
#
if startMsg:
if g_ErrorLog and os.path.isfile(g_ErrorLog):
lines = Basic.readTextFile(g_ErrorLog); lines2 = []; found = False
@ -81,14 +72,11 @@ def init(runTimeLog=None, errorLog=None, startMsg=None):
if len(lines2) > 0:
os.remove(g_ErrorLog)
Basic.writeTextFile(g_ErrorLog, lines2)
#endif
# Set start information
#
if startMsg != None: return setStartMsg(startMsg)
else: return True
# -------------------------------------------------------------------------------------------------
def initFile(fileName=None, postfix=''):
@ -121,11 +109,11 @@ def initFile(fileName=None, postfix=''):
newLines = lines [n-MAX_LINES:]
Basic.writeTextFile(log1, newLines)
#endif
return log1
# -------------------------------------------------------------------------------------------------
def appendToFile(fileName, lines):
if fileName == None: return
@ -140,7 +128,7 @@ def appendToFile(fileName, lines):
fp.close()
# -------------------------------------------------------------------------------------------------
def addLines(lines):
global g_ProtLineNr
@ -178,21 +166,23 @@ def addLines(lines):
print(MSG_TOKEN + ' Exception in Logfile.add() ')
sys.exit(1)
# -------------------------------------------------------------------------------------------------
def add(text, text2 = None, text3 = None):
lines = [text, text2, text3]; addLines(lines)
lines = [text, text2, text3]
addLines(lines)
def add2(prefix, text, text2, text3):
lines = [text, text2, text3]
lines = [text, text2, text3]
lines2 = []
for s in lines:
if s != None: lines2.append(prefix + s)
if s is not None:
lines2.append(prefix + s)
addLines(lines2)
# -------------------------------------------------------------------------------------------------
def showLabel(msg):
@ -201,16 +191,15 @@ def showLabel(msg):
add('*********** ' + msg)
add('********************************************************')
def red(line):
add(line)
# -------------------------------------------------------------------------------------------------
DELIM = '----------------------------------------------------------------'
def addDelim(): add(DELIM)
def error(text, text2 = None, text3 = None):
def error(text, text2=None, text3=None):
setErrorLog(True)
add2(MSG_TOKEN + ' Error : ', text, text2, text3)
@ -228,7 +217,7 @@ def warning(text, text2 = None, text3 = None):
return True
def debug(text, text2 = None, text3 = None):
def debug(text, text2=None, text3=None):
if Globals.isDebug:
setErrorLog(True)
@ -241,49 +230,13 @@ def fileOpenError(fileName):
def abort(text = None):
if text != None: add(text)
if text != None:
add(text)
add(ABORT_TOKEN + ' Abort program')
#assert False
sys.exit(1)
def exception(proc, text = None, abortProg = False):
if text != None: s = proc + ' - ' + text
else: s = proc
setErrorLog(True)
add(MSG_TOKEN + ' Exception: ' + s)
traceback.print_exc(file=sys.stdout)
setErrorLog(False)
if abortProg: abort()
return False
# -------------------------------------------------------------------------------------------------------------------------------
def addFile(dir, fileName = None):
try:
if fileName != None: name = dir
else: name = os.path.join(dir, fileName)
fp = openTextFile(name, 'r')
if fp == None: return
lines = fp.readlines(1000)
fp.close()
for s in lines: add(s[:-1])
except:
exception('addFile()', abortProg = True)
# -------------------------------------------------------------------------------------------------------------------------------
localTest = False
@ -322,7 +275,7 @@ def saveLogfile(errorLog, runtimeLog):
remove1(fileName_2)
return ret
# -------------------------------------------------------------------------------------------------------------------------------
def setStartMsg(version_string):
s = []

2
src/process/ProcessProgs.py

@ -49,7 +49,7 @@ def start(config):
at = os.path.join(os.getcwd(), 'Process', 'main.py')
workDir = [path, 'tmp2', 'process']
workDir = ['tmpProcess']
cmd = "palantiri_process"+ ' -f ' + path + ' -e ' + path_emp
cmd = "palantiri_process" + ' -f ' + path + ' -e ' + path_emp
except Exception:
at = os.path.join(os.getcwd(), 'Process', 'main.py')
workDir = [path, 'tmp2', 'process']

27
src/process/array_crosscorrelation_v4.py

@ -196,19 +196,14 @@ class Xcorr(object):
return stream, snr
def readWaveformsCross_pyrocko(self, station, tw, ttime):
def readWaveformsCross_pyrocko(self, station, tw, ttime, traces):
obspy_compat.plant()
cfg = ConfigObj(dict=self.Config)
t2 = UTCDateTime(self.Origin.time)
if cfg.quantity() == 'displacement':
try:
traces = io.load(self.EventPath+'/data/traces_rotated.mseed')
except Exception:
traces = io.load(self.EventPath+'/data/traces_restituted.mseed')
else:
traces = io.load(self.EventPath+'/data/traces_velocity.mseed')
print('cant find')
found = False
while found is False:
for tr in traces:
@ -246,7 +241,7 @@ class Xcorr(object):
stream.trim(tw['xcorrstart'], tw['xcorrend'])
found = True
return stream, snr
print('found')
if found is False:
print('Waveform missing!', tr_name, str(station))
@ -297,7 +292,7 @@ class Xcorr(object):
else:
pass
def traveltimes(self, phase):
def traveltimes(self, phase, traces):
Logfile.red('Enter AUTOMATIC CROSSCORRELATION ')
Logfile.red('\n\n+++++++++++++++++++++++++++++++++++++++++++++++\n ')
@ -306,6 +301,7 @@ class Xcorr(object):
SNR = OrderedDict()
Config = self.Config
cfg = ConfigObj(dict=Config)
for i in self.StationMeta:
Logfile.red('read in %s ' % (i))
de = loc2degrees(self.Origin, i)
@ -339,7 +335,8 @@ class Xcorr(object):
else:
tw = self.calculateTimeWindows(ptime)
if cfg.pyrocko_download() is True:
w, snr = self.readWaveformsCross_pyrocko(i, tw, ptime)
w, snr = self.readWaveformsCross_pyrocko(i, tw, ptime,
traces)
elif cfg.colesseo_input() is True:
w, snr = self.readWaveformsCross_colesseo(i, tw, ptime)
else:
@ -640,8 +637,8 @@ class Xcorr(object):
return fCD
def doXcorr(self, phase):
StreamDict, SNRDict = self.traveltimes(phase)
def doXcorr(self, phase, traces):
StreamDict, SNRDict = self.traveltimes(phase, traces)
t = self.f6(SNRDict)
Logfile.add('doXcorr: REFERENCE: ' + t)
@ -698,9 +695,9 @@ class Xcorr(object):
return corrDict, StreamDict[t], StreamDict
def runXcorr(self, phase):
def runXcorr(self, phase, traces):
CD, ref, WD = self.doXcorr(phase)
CD, ref, WD = self.doXcorr(phase, traces)
onset = 0
tdiff, triggerobject = self.refTrigger(ref, phase)

24
src/process/main.py

@ -15,7 +15,7 @@ from palantiri.tools import config
from palantiri.tools.config import Event
from palantiri.process import ttt, sembCalc, waveform, times, deserializer
from palantiri.process.array_crosscorrelation_v4 import Xcorr, cmpFilterMetavsXCORR
from pyrocko import util
from pyrocko import util, io
import numpy as num
if sys.version_info.major >= 3:
@ -160,7 +160,14 @@ def processLoop(traces=None, stations=None, cluster=None):
rpe = os.path.join(Folder['semb'], fobjreferenceshiftnameemp)
fobjpickleshiftname = newFreq + '_' + filtername + '.xcorrpkl'
ps = os.path.join(Folder['semb'], fobjpickleshiftname)
if cfg.Bool('synthetic_test') is False:
if cfg.quantity() == 'displacement':
try:
traces = io.load(evpath+'/data/traces_rotated.mseed')
except Exception:
traces = io.load(evpath+'/data/traces_restituted.mseed')
else:
traces = io.load(evpath+'/data/traces_velocity.mseed')
if(os.path.isfile(rp) and os.path.getsize(rp) != 0
and os.path.isfile(ps) and os.path.getsize(ps) != 0):
Logfile.add('xcorr/reference shift file exits : ' + rp)
@ -180,6 +187,14 @@ def processLoop(traces=None, stations=None, cluster=None):
for i in xcorrnetworks:
SL[i] = len(Config[i].split('|'))
else:
if cfg.Bool('synthetic_test') is False:
if cfg.quantity() == 'displacement':
try:
traces = io.load(evpath+'/data/traces_rotated.mseed')
except Exception:
traces = io.load(evpath+'/data/traces_restituted.mseed')
else:
traces = io.load(evpath+'/data/traces_velocity.mseed')
SL = {}
for i in xcorrnetworks:
W = {}
@ -195,7 +210,7 @@ def processLoop(traces=None, stations=None, cluster=None):
arrayfolder)
print("run Xcorr")
phase = phases[0]
W, triggerobject = A.runXcorr(phase)
W, triggerobject = A.runXcorr(phase, traces)
XDict[i] = W
RefDict[i] = triggerobject.tdiff
SL[i] = len(network)
@ -721,7 +736,6 @@ def processLoop(traces=None, stations=None, cluster=None):
rpe+str(arrayname), flag_rpe)
else:
if cfg.Bool('correct_shifts_empirical') is True:
if cfg.Bool('correct_shifts_empirical_run') is True:
winlen_emp = cfg.winlen_emp()
step_emp = cfg.step_emp()
@ -941,7 +955,7 @@ def processLoop(traces=None, stations=None, cluster=None):
syn_in, ASL, sembmax, evpath, XDict,
RefDict, workdepth, filterindex, Wdfs)
if ASL and cfg.Bool('bootstrap_array_weights') is False:
if ASL:
Logfile.red('collect semblance matrices from all arrays')
sembmax, tmp = sembCalc.collectSemb(ASL, Config, Origin,
Folder,

5
src/process/sembCalc.py

@ -480,8 +480,6 @@ def collectSemb(SembList, Config, Origin, Folder, ntimes, arrays, switch,
if temp_comb is not None:
nboot = 0
else:
nboot = 1
folder = Folder['semb']
sembmaxvaluev = num.ndarray(ntimes, dtype=float)
@ -494,8 +492,9 @@ def collectSemb(SembList, Config, Origin, Folder, ntimes, arrays, switch,
d = rc.timestamp
usedarrays = arrays
for boot in range(nboot):
for boot in range(0, nboot):
c = 0
print('booting', boot, 'of', nboot)
tmp = 1
for a in SembList:

2
src/process/semp.py

@ -441,7 +441,7 @@ def semblance_py(ncpus, nostat, nsamp, ntimes, nstep, dimX, dimY, mint,
cfx = num.fft.fft(data)
sums_schimmel = sums_schimmel + (cfx/(abs(cfx)) * num.exp(1j*2*num.pi*cfx))
try:
if do_bs_weights is True:
if do_bs_weights is True and combine is True:
sums += data*bs_weights[k]
else:
sums = sums+data

Loading…
Cancel
Save