Browse Source

fomosto: better cleanup after build

prereq
Sebastian Heimann 8 months ago
committed by Pyrocko Test Debugger
parent
commit
738ac61b63
9 changed files with 43 additions and 8 deletions
  1. +3
    -0
      src/fomosto/ahfullgreen.py
  2. +3
    -0
      src/fomosto/dummy.py
  3. +3
    -0
      src/fomosto/poel.py
  4. +4
    -0
      src/fomosto/psgrn_pscmp.py
  5. +3
    -0
      src/fomosto/qseis.py
  6. +3
    -0
      src/fomosto/qseis2d.py
  7. +3
    -0
      src/fomosto/qssp.py
  8. +5
    -0
      src/gf/builder.py
  9. +16
    -8
      src/parimap.py

+ 3
- 0
src/fomosto/ahfullgreen.py View File

@ -101,6 +101,9 @@ class AhfullGFBuilder(gf.builder.Builder):
gf.builder.Builder.__init__(
self, self.store.config, step, block_size=block_size, force=force)
def cleanup(self):
self.store.close()
def work_block(self, index):
if len(self.store.config.ns) == 2:
(sz, firstx), (sz, lastx), (ns, nx) = \


+ 3
- 0
src/fomosto/dummy.py View File

@ -28,6 +28,9 @@ class DummyGFBuilder(gf.builder.Builder):
gf.builder.Builder.__init__(
self, self.store.config, step, block_size=(1, 51), force=force)
def cleanup(self):
self.store.close()
def work_block(self, index):
(sz, firstx), (sz, lastx), (ns, nx) = \
self.get_block_extents(index)


+ 3
- 0
src/fomosto/poel.py View File

@ -498,6 +498,9 @@ class PoelGFBuilder(gf.builder.Builder):
if self.tmp is not None:
util.ensuredir(self.tmp)
def cleanup(self):
self.store.close()
def work_block(self, index):
logger.info('Starting block %i / %i' % (index+1, self.nblocks))


+ 4
- 0
src/fomosto/psgrn_pscmp.py View File

@ -1320,6 +1320,7 @@ class PsGrnCmpGFBuilder(gf.builder.Builder):
def __init__(self, store_dir, step, shared, block_size=None, tmp=None,
force=False):
self.store = gf.store.Store(store_dir, 'w')
storeconf = self.store.config
@ -1367,6 +1368,9 @@ class PsGrnCmpGFBuilder(gf.builder.Builder):
self.cg = cg
self.cc = cc
def cleanup(self):
self.store.close()
def work_block(self, iblock):
if len(self.store.config.ns) == 2:


+ 3
- 0
src/fomosto/qseis.py View File

@ -793,6 +793,9 @@ class QSeisGFBuilder(gf.builder.Builder):
if self.tmp is not None:
util.ensuredir(self.tmp)
def cleanup(self):
self.store.close()
def work_block(self, index):
if len(self.store.config.ns) == 2:
(sz, firstx), (sz, lastx), (ns, nx) = \


+ 3
- 0
src/fomosto/qseis2d.py View File

@ -878,6 +878,9 @@ class QSeis2dGFBuilder(gf.builder.Builder):
util.ensuredir(baseconf.gf_directory)
def cleanup(self):
self.store.close()
def work_block(self, iblock):
if len(self.store.config.ns) == 2:
(sz, firstx), (sz, lastx), (ns, nx) = \


+ 3
- 0
src/fomosto/qssp.py View File

@ -847,6 +847,9 @@ class QSSPGFBuilder(gf.builder.Builder):
self.qssp_config = conf
def cleanup(self):
self.store.close()
def work_block(self, iblock):
if len(self.store.config.ns) == 2:
(sz, firstx), (sz, lastx), (ns, nx) = \


+ 5
- 0
src/gf/builder.py View File

@ -52,6 +52,7 @@ def work_block(args):
def cleanup():
for k in list(g_builders):
g_builders[k].cleanup()
del g_builders[k]
@ -73,6 +74,9 @@ class Builder(object):
self.warnings = defaultdict(int)
self._block_size = int_arr(*block_size)
def cleanup(self):
pass
@property
def nblocks(self):
return num.prod(self.block_dims)
@ -168,6 +172,7 @@ class Builder(object):
return
builder.cleanup()
del builder
original = signal.signal(signal.SIGINT, signal.SIG_IGN)


+ 16
- 8
src/parimap.py View File

@ -66,15 +66,23 @@ def parimap(function, *iterables, **kwargs):
if pshared is not None:
kwargs['pshared'] = pshared
while True:
args = []
for it in iterables:
try:
args.append(next(it))
except StopIteration:
return
if startup is not None:
startup(*startup_args)
yield function(*args, **kwargs)
try:
while True:
args = []
for it in iterables:
try:
args.append(next(it))
except StopIteration:
return
yield function(*args, **kwargs)
finally:
if cleanup is not None:
cleanup()
return


Loading…
Cancel
Save