Browse Source

fomosto: better cleanup after build

prereq
Sebastian Heimann 1 year ago
committed by Pyrocko Test Debugger
parent
commit
738ac61b63
  1. 3
      src/fomosto/ahfullgreen.py
  2. 3
      src/fomosto/dummy.py
  3. 3
      src/fomosto/poel.py
  4. 4
      src/fomosto/psgrn_pscmp.py
  5. 3
      src/fomosto/qseis.py
  6. 3
      src/fomosto/qseis2d.py
  7. 3
      src/fomosto/qssp.py
  8. 5
      src/gf/builder.py
  9. 24
      src/parimap.py

3
src/fomosto/ahfullgreen.py

@ -101,6 +101,9 @@ class AhfullGFBuilder(gf.builder.Builder):
gf.builder.Builder.__init__(
self, self.store.config, step, block_size=block_size, force=force)
def cleanup(self):
self.store.close()
def work_block(self, index):
if len(self.store.config.ns) == 2:
(sz, firstx), (sz, lastx), (ns, nx) = \

3
src/fomosto/dummy.py

@ -28,6 +28,9 @@ class DummyGFBuilder(gf.builder.Builder):
gf.builder.Builder.__init__(
self, self.store.config, step, block_size=(1, 51), force=force)
def cleanup(self):
self.store.close()
def work_block(self, index):
(sz, firstx), (sz, lastx), (ns, nx) = \
self.get_block_extents(index)

3
src/fomosto/poel.py

@ -498,6 +498,9 @@ class PoelGFBuilder(gf.builder.Builder):
if self.tmp is not None:
util.ensuredir(self.tmp)
def cleanup(self):
self.store.close()
def work_block(self, index):
logger.info('Starting block %i / %i' % (index+1, self.nblocks))

4
src/fomosto/psgrn_pscmp.py

@ -1320,6 +1320,7 @@ class PsGrnCmpGFBuilder(gf.builder.Builder):
def __init__(self, store_dir, step, shared, block_size=None, tmp=None,
force=False):
self.store = gf.store.Store(store_dir, 'w')
storeconf = self.store.config
@ -1367,6 +1368,9 @@ class PsGrnCmpGFBuilder(gf.builder.Builder):
self.cg = cg
self.cc = cc
def cleanup(self):
self.store.close()
def work_block(self, iblock):
if len(self.store.config.ns) == 2:

3
src/fomosto/qseis.py

@ -793,6 +793,9 @@ class QSeisGFBuilder(gf.builder.Builder):
if self.tmp is not None:
util.ensuredir(self.tmp)
def cleanup(self):
self.store.close()
def work_block(self, index):
if len(self.store.config.ns) == 2:
(sz, firstx), (sz, lastx), (ns, nx) = \

3
src/fomosto/qseis2d.py

@ -878,6 +878,9 @@ class QSeis2dGFBuilder(gf.builder.Builder):
util.ensuredir(baseconf.gf_directory)
def cleanup(self):
self.store.close()
def work_block(self, iblock):
if len(self.store.config.ns) == 2:
(sz, firstx), (sz, lastx), (ns, nx) = \

3
src/fomosto/qssp.py

@ -847,6 +847,9 @@ class QSSPGFBuilder(gf.builder.Builder):
self.qssp_config = conf
def cleanup(self):
self.store.close()
def work_block(self, iblock):
if len(self.store.config.ns) == 2:
(sz, firstx), (sz, lastx), (ns, nx) = \

5
src/gf/builder.py

@ -52,6 +52,7 @@ def work_block(args):
def cleanup():
for k in list(g_builders):
g_builders[k].cleanup()
del g_builders[k]
@ -73,6 +74,9 @@ class Builder(object):
self.warnings = defaultdict(int)
self._block_size = int_arr(*block_size)
def cleanup(self):
pass
@property
def nblocks(self):
return num.prod(self.block_dims)
@ -168,6 +172,7 @@ class Builder(object):
return
builder.cleanup()
del builder
original = signal.signal(signal.SIGINT, signal.SIG_IGN)

24
src/parimap.py

@ -66,15 +66,23 @@ def parimap(function, *iterables, **kwargs):
if pshared is not None:
kwargs['pshared'] = pshared
while True:
args = []
for it in iterables:
try:
args.append(next(it))
except StopIteration:
return
if startup is not None:
startup(*startup_args)
yield function(*args, **kwargs)
try:
while True:
args = []
for it in iterables:
try:
args.append(next(it))
except StopIteration:
return
yield function(*args, **kwargs)
finally:
if cleanup is not None:
cleanup()
return

Loading…
Cancel
Save