Browse Source

pep8, cleanup

obspy
braunfuss 2 years ago
parent
commit
cfacfc4a50
  1. 370
      src/common/Basic.py
  2. 7
      src/process/deserializer.py
  3. 41
      src/tools/create.py
  4. 1
      src/tools/eventsearch.py

370
src/common/Basic.py

@ -88,7 +88,7 @@ def formatStrings(strings, format1):
try:
for i in range(len(strings)):
result.append(format1 %(strings[i]))
result.append(format1 % (strings[i]))
except Exception:
Logfile.exception('formatStrings', 'Illegal format', abortProg=True)
@ -101,13 +101,12 @@ def selectStrings(strings, mask):
result = []
for i in range(len(mask)):
if mask[i]:
result.append(strings[i])
if mask[i]:
result.append(strings[i])
return result
def _stringsEndsWith(strings, postfixList):
assert len(postfixList) > 0
@ -119,75 +118,88 @@ def _stringsEndsWith(strings, postfixList):
for postfix in postfixList:
if s.endswith(postfix):
b = True
break
b = True
break
mask.append(b)
assert len(mask) == len(strings)
return mask
def stringsEndsWith(strings, postfixList):
if type(postfixList) is str: list1 = list(postfixList)
else: list1 = postfixList
if type(postfixList) is str:
list1 = list(postfixList)
else:
list1 = postfixList
return _stringsEndsWith(strings, list1)
def toStringList(arg0, arg1=None,arg2=None,arg3=None,arg4=None):
def toStringList(arg0, arg1=None, arg2=None, arg3=None, arg4=None):
s = []
s.append(arg0)
if arg1 != None: s.append(arg1)
if arg2 != None: s.append(arg2)
if arg3 != None: s.append(arg3)
if arg4 != None: s.append(arg4)
if arg1 is not None:
s.append(arg1)
if arg2 is not None:
s.append(arg2)
if arg3 is not None:
s.append(arg3)
if arg4 is not None:
s.append(arg4)
return s
def Not(mask):
result = []
for i in range(len(mask)):
if mask[i]: result.append(False)
else: result.append(True)
if mask[i]:
result.append(False)
else:
result.append(True)
return result
def And(mask):
for i in range(len(mask)):
if not mask[i]: return False
if not mask[i]:
return False
return True
def baseFileName(fullName):
basename = os.path.basename(fullName)
filename = os.path.splitext(basename)
return filename[0]
def isNumber(s):
try : float(s)
except: return False
try:
float(s)
except:
return False
return True
def isInt(s):
if not isNumber(s): return False
try: int(s)
except: return False
if not isNumber(s):
return False
try:
int(s)
except:
return False
return True
@ -199,16 +211,21 @@ def checkIsNumber(string, minVal=None, maxVal=None):
msg = None
s1 = 'Value ' + string + ' '
if not isNumber(string): msg = s1 + 'is not a number'
if not isNumber(string):
msg = s1 + 'is not a number'
else:
val = float(string)
if minVal == None and maxVal == None: msg = None
elif minVal == None and val > maxVal : msg = s1 + '> ' + str(maxVal)
elif maxVal == None and val < minVal : msg = s1 + '< ' + str(minVal)
elif val < minVal or val > maxVal :
msg = s1 + 'outside range [' + str(minVal) + ',' + str(maxVal) + ']'
val = float(string)
if minVal is None and maxVal is None:
msg = None
elif minVal is None and val > maxVal:
msg = s1 + '> ' + str(maxVal)
elif maxVal is None and val < minVal:
msg = s1 + '< ' + str(minVal)
elif val < minVal or val > maxVal:
msg = s1 + 'outside range [' + str(minVal) + ',' +\
str(maxVal) + ']'
return msg
@ -217,12 +234,15 @@ def checkGreaterZero(string):
msg = None
s1 = 'Value ' + string + ' '
if not isNumber(string): msg = s1 + 'is not a number'
if not isNumber(string):
msg = s1 + 'is not a number'
else:
val = float(string)
val = float(string)
if val == 0.0: msg = s1 + 'is zero'
elif val < 0.0: msg = s1 + '< 0.0'
if val is 0.0:
msg = s1 + 'is zero'
elif val < 0.0:
msg = s1 + '< 0.0'
return msg
@ -231,108 +251,120 @@ def checkNotNegative(string):
msg = None
s1 = 'Value ' + string + ' '
if not isNumber(string) : msg = s1 + 'is not a number'
elif float(string) < 0.0: msg = s1 + '< 0.0'
if not isNumber(string):
msg = s1 + 'is not a number'
elif float(string) < 0.0:
msg = s1 + '< 0.0'
return msg
# Check if keys exists in a dictionary
def checkExistsKeys(dict, keyList, isAbort=False):
isOk = True
for key in keyList:
if not key in dict:
isOk = Logfile.error('Key <' + str(key) + '> missing in config file')
isOk = Logfile.error('Key <' + str(key) + '> missing in config file')
if isOk : return True
elif isAbort: Logfile.abort()
if isOk:
return True
elif isAbort:
Logfile.abort()
return False
def checkExistsDir(dirName, isAbort=False):
if os.path.isdir(dirName): return True
if os.path.isdir(dirName):
return True
Logfile.error('Cannot find directory', dirName)
if isAbort: Logfile.abort()
if isAbort:
Logfile.abort()
return False
def createDirectory(dirName, optional = False):
def createDirectory(dirName, optional=False):
if os.path.isdir(dirName): return True
if os.path.isdir(dirName):
return True
os.makedirs(dirName)
if os.path.isdir(dirName): return True
else: Logfile.error('Cannot open directory', dirName)
if os.path.isdir(dirName):
return True
else:
Logfile.error('Cannot open directory', dirName)
if not optional: Logfile.abort()
if not optional:
Logfile.abort()
return False
def changeDirectory(dirPath):
path = []
if not type(dirPath) is list: path.append(dirPath)
else: path = dirPath
if not type(dirPath) is list:
path.append(dirPath)
else:
path = dirPath
for dir in path:
createDirectory(dir)
os.chdir(dir)
return os.getcwd() # return current directory
createDirectory(dir)
os.chdir(dir)
return os.getcwd()
def checkFileExists(fileName, isAbort=False):
if os.path.isfile(fileName): return True
if os.path.isfile(fileName):
return True
Logfile.fileOpenError(fileName)
if isAbort: Logfile.abort('')
else: return False
if isAbort:
Logfile.abort('')
else:
return False
def openTextFile(fileName, mode):
if mode == 'r' and not checkFileExists(fileName):
return None
return None
return open(fileName, mode)
def readTextFile(fileName, maxLines = -1):
def readTextFile(fileName, maxLines=-1):
lines = []
fp= openTextFile(fileName, 'r')
fp = openTextFile(fileName, 'r')
if fp == None: return lines
if fp is None:
return lines
if maxLines == -1:
lines = fp.readlines()
if maxLines is -1:
lines = fp.readlines()
else:
lines = fp.readlines(maxLines)
lines = fp.readlines(maxLines)
fp.close()
return lines
def writeTextFile(fileName, lines):
fp = openTextFile(fileName, 'w')
if fp == None: return
if fp is None:
return
for s in lines:
fp.write(s)
@ -343,16 +375,19 @@ def writeTextFile(fileName, lines):
def appendToFile(fileName, lines):
fp = openTextFile(fileName, 'a')
if fp == None: return
if fp is None:
return
for s in lines: fp.write(s)
for s in lines:
fp.write(s)
fp.close()
def copyFile(srcFile, destFile, isAbort=False):
if not checkFileExists(srcFile, isAbort): return False
if not checkFileExists(srcFile, isAbort):
return False
shutil.copy(srcFile, destFile)
return True
@ -368,17 +403,16 @@ def dumpToFile(fileName, anyData):
output.close()
def loadDump(fileName):
if not os.path.isfile(fileName):
Logfile.fileOpenError(fileName)
return None
Logfile.fileOpenError(fileName)
return None
if sys.version_info.major >= 3:
f= open(fileName, 'rb')
f = open(fileName, 'rb')
else:
f= open(fileName)
f = open(fileName)
data = pickle.load(f)
f.close()
os.remove(fileName)
@ -386,143 +420,156 @@ def loadDump(fileName):
return data
def removeFile(files):
if not files is list: files2 = [files]
else: files2 = files
if not files is list:
files2 = [files]
else:
files2 = files
for f in files2:
if os.path.isfile(f) : os.remove(f)
if os.path.isfile(f):
os.remove(f)
def removeFiles(dir, prefix = None):
def removeFiles(dir, prefix=None):
names = os.listdir(dir)
names2 = []
if prefix == None: names2 = names
if prefix is None:
names2 = names
else:
for file in names:
if file.startswith(prefix): names2.append(file)
for file in names:
if file.startswith(prefix):
names2.append(file)
for file in names2:
if dir != '.': fullName = os.path.join(dir, file)
else: fullName = file
if dir != '.':
fullName = os.path.join(dir, file)
else:
fullName = file
if os.path.isfile(fullName):
if dir != '.': file1 = dir + '/' + file
else: file1 = file
if dir != '.':
file1 = dir + '/' + file
else:
file1 = file
os.remove(file1)
os.remove(file1)
def wait(seconds, prompt = True):
def wait(seconds, prompt=True):
for i in range(seconds):
if prompt: print(str(i))
time.sleep(1)
if prompt:
print(str(i))
time.sleep(1)
def systemCmd(cmd):
if type(cmd) is list: cmd2 = ' '.join(cmd)
else: cmd2 = cmd
if type(cmd) is list:
cmd2 = ' '.join(cmd)
else:
cmd2 = cmd
pipe = subprocess.Popen(cmd2, shell=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout
pipe = subprocess.Popen(cmd2, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT).stdout
lines = pipe.readlines()
lines2 = []
for s in lines:
if s.endswith('\r\n'): lines2.append(s[:-2])
else: lines2.append(s)
if s.endswith('\r\n'):
lines2.append(s[:-2])
else:
lines2.append(s)
return lines2
def killByPID(pidList):
ownPID = os.getpid()
for pid in pidList:
if pid == ownPID: continue
os.system('kill ' + str(pid))
#endfor
if pid == ownPID:
continue
os.system('kill ' + str(pid))
def removeTempFiles():
dir = '/tmp'
dir = '/tmp'
names = os.listdir(dir)
user = getpass.getuser()
cnt = 0
cnt = 0
for s in names:
if s .startswith('obspy-'):
os.remove(dir + '/' + s)
cnt += 1
#end for
if s .startswith('obspy-'):
os.remove(dir + '/' + s)
cnt += 1
if cnt != 0: Logfile.add('Remove ' + str(cnt) + ' temp files obspy-*.*')
if cnt != 0:
Logfile.add('Remove ' + str(cnt) + ' temp files obspy-*.*')
return cnt
def existsHTML_Page(url, text = None, withComment = False, isAbort=False):
def existsHTML_Page(url, text=None, withComment=False, isAbort=False):
h = httplib2.Http()
try:
resp = h.request(url, 'HEAD')
status = int(resp[0]['status'])
if status < 400: return True # page exists
resp = h.request(url, 'HEAD')
status = int(resp[0]['status'])
if status < 400:
return True # page exists
except:
status = 104 # error: [Errno 104] Connection reset by peer ???
status = 104 # error: [Errno 104] Connection reset by peer ???
if withComment :
s = 'HTML page not found, error = ' + str(status)
if withComment:
s = 'HTML page not found, error = ' + str(status)
if text == None: Logfile.error(s, url)
else: Logfile.error(text, s, url)
#endif
if text is None:
Logfile.error(s, url)
else:
Logfile.error(text, s, url)
if isAbort: Logfile.abort()
if isAbort:
Logfile.abort()
return False
def readURL(url, tmpFile=None):
lines = []
try:
datasource = urlopen(url)
datasource = urlopen(url)
while True:
line = datasource.readline()
while True:
line = datasource.readline()
if line == '': break
lines.append(line)
#endwhile
if line == '':
break
lines.append(line)
if len(lines) == 0:
Logfile.error('Cannot read from URL:', url)
lines = []
if len(lines) == 0:
Logfile.error('Cannot read from URL:', url)
lines = []
Logfile.add(str(len(lines)) + ' lines read from URL:', url)
Logfile.add(str(len(lines)) + ' lines read from URL:', url)
if tmpFile: writeTextFile(tmpFile, lines)
if tmpFile:
writeTextFile(tmpFile, lines)
except: Logfile.exception('readURL')
except:
Logfile.exception('readURL')
return lines
@ -532,17 +579,20 @@ def readUrl2(pythonScript, pythonScript_2):
assert pythonScript.endswith('.py')
try:
lines = []
url = 'http://www.staedtke-berlin.kilu.de/' + pythonScript
page = urlopen(url)
lines = []
url = 'http://www.staedtke-berlin.kilu.de/' + pythonScript
page = urlopen(url)
for line in page:
lines.append(line[:-1])
for line in page:
lines.append(line[:-1])
except: return False
except:
return False
try: writeTextFile(pythonScript_2, lines)
except: return False
try:
writeTextFile(pythonScript_2, lines)
except:
return False
return True
@ -556,22 +606,28 @@ def question(text):
print('c=<' + c + '>')
if c == 'y': return True
elif c == 'n': return False
else: continue
if c == 'y':
return True
elif c == 'n':
return False
else:
continue
else:
c = raw_input(text + ' <y/n>:')
c = c.lower()
print('c=<' + c + '>')
if c == 'y': return True
elif c == 'n': return False
else: continue
if c == 'y':
return True
elif c == 'n':
return False
else:
continue
def getUserName():
s = os.environ ['USERDOMAIN']
s = os.environ['USERDOMAIN']
return s

7
src/process/deserializer.py

@ -1,15 +1,12 @@
import os
import sys
from palantiri.common import Basic
from palantiri.common import Globals
from palantiri.common import Logfile
from collections import OrderedDict
from palantiri.process.ttt import MinTMaxT
'''
modul for deserializing pickle data from different processes
'''
def deserializeTTT(numproc):
L = []
@ -32,6 +29,7 @@ def deserializeTTT(numproc):
return TTTGridMap
def deserializeTTT_cube(numproc):
L = []
@ -54,6 +52,7 @@ def deserializeTTT_cube(numproc):
return TTTGridMap
def deserializeMinTMaxT(numproc):
L = []

41
src/tools/create.py

@ -8,7 +8,8 @@ import dateutil.parser
logger = logging.getLogger('ARRAY-MP')
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
formatter = logging.Formatter("%(asctime)s - %(name)s -\
%(levelname)s - %(message)s")
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
@ -38,7 +39,8 @@ def parseEvent(eventID):
eventID = eventID[1].replace('_', '')
url = 'http://service.iris.edu/fdsnws/event/1/query?eventid=%s&format=text' % (eventID)
url = 'http://service.iris.edu/fdsnws/event\
/1/query?eventid=%s&format=text' % (eventID)
data = urlopen(url).read()
data = data.decode('utf_8')
data = data.split('\n')
@ -47,7 +49,7 @@ def parseEvent(eventID):
i = i.split('|')
time = i[1].replace(':', '-').strip()
name = i[12].replace(' ', '-').strip()
eventname = ('%s_%s') % (name,time)
eventname = ('%s_%s') % (name, time)
return eventname
@ -83,7 +85,8 @@ def writeOriginFile(path, ev_id):
eventID = ev_id[1].replace('_', '')
url = 'http://service.iris.edu/fdsnws/event/1/query?eventid=%s&format=text' % (eventID)
url = 'http://service.iris.edu/fdsnws/event\
/1/query?eventid=%s&format=text' % (eventID)
data = urlopen(url).read()
data = data.decode('utf_8')
data = data.split('\n')
@ -107,7 +110,8 @@ def writeOriginFile(path, ev_id):
def writeSynFile(path, ev_id):
'''
method to write synthetic input(event) file in the event directory to be processed
method to write synthetic input(event) file in the event directory to be
processed
'''
fname = os.path.basename(path)+'.syn'
@ -116,7 +120,8 @@ def writeSynFile(path, ev_id):
eventID = ev_id[1].replace('_', '')
url = 'http://service.iris.edu/fdsnws/event/1/query?eventid=%s&format=text' % (eventID)
url = 'http://service.iris.edu/fdsnws/\
event/1/query?eventid=%s&format=text' % (eventID)
data = urlopen(url).read()
data = data.decode('utf_8')
data = data.split('\n')
@ -149,22 +154,10 @@ def writeSynFile(path, ev_id):
return time
def startAcquisition(sttime, sdsfolder, Dconfig):
'''
method to download waveform data from stations in global.conf specified keyfolder for the event
'''
sttime = str(sttime)
keyfolder = Dconfig['keyfilefolder']
tool = os.path.join(os.getcwd(), 'ev_wave_mt4.py')
sds = os.path.join(sdsfolder, 'data')
cmd = sys.executable + ' ' + tool + ' -t ' + sttime + ' -d ' + Dconfig['duration'] +' -m ' + sds + ' -k '+keyfolder+ ' -s '+Dconfig['keyfiles']
def copyConfigSkeleton(evfolder):
'''
method to copy the example config from skeleton directory to event directory
method to copy the example config from skeleton directory to the
event directory
'''
logger.info('\033[31m Copy example.config to %s \033[0m \n' % (evfolder))
@ -173,12 +166,16 @@ def copyConfigSkeleton(evfolder):
path = palantiri.__path__
src = os.path.join(path[0], 'skeleton', 'example.config')
dst = os.path.join(evfolder, dstfile)
logger.info('\033[31m Created event directory %s \033[0m \n' % (evfolder.split('/')[-1]))
logger.info('\033[31m Created event directory \
%s \033[0m \n' % (evfolder.split('/')[-1]))
shutil.copy(src, dst)
event = evfolder.split('/')[-1]
logger.info('\033[31mNEXT PROCESSING STEP: \n\n palantiri_down {evdirectory} "time" 10352.323104588522 0.001 10 --radius-min=1110 "name" \n\n\033[0m'.format(evdirectory=str(event.strip('[]'))))
logger.info('\033[31mNEXT PROCESSING STEP: \n\n \
palantiri_down {evdirectory} "time" 10352.323104588522\
0.001 10 --radius-min=1110 "name"\
\n\n\033[0m'.format(evdirectory=str(event.strip('[]'))))
def main():

1
src/tools/eventsearch.py

@ -1,5 +1,4 @@
from configparser import SafeConfigParser
import urllib
from urllib.parse import urlencode
from urllib.request import urlopen

Loading…
Cancel
Save