Browse Source

guts: adding !include constructor

master
miili 4 months ago
committed by Sebastian Heimann
parent
commit
d2e3727a47
5 changed files with 167 additions and 11 deletions
  1. +3
    -2
      CHANGELOG.md
  2. +11
    -2
      doc/source/library/examples/guts.rst
  3. +9
    -0
      examples/guts_example.yaml
  4. +76
    -7
      src/guts.py
  5. +68
    -0
      test/base/test_guts.py

+ 3
- 2
CHANGELOG.md View File

@ -14,8 +14,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
directivity effects for synthetics.
- Snuffler: load StationXML via menu.
- `io.mseed`: Adding option for STEIM2 compression.
- Jackseis: Adding `--output-steim` option to control compression.
Default compression changed to STEIM2.
- Jackseis: Adding `--output-steim` option to control compression. Default
compression changed to STEIM2.
- YAML files can now include other YAML files, when loaded through guts.
### Fixed
- Fix plotting issues in cake.


+ 11
- 2
doc/source/library/examples/guts.rst View File

@ -2,7 +2,7 @@ Working with the ``guts`` package
=================================
Introduction to ``guts``
-------------------------------
------------------------
Guts is a `Lightweight declarative YAML and XML data binding for Python
<https://github.com/emolch/guts>`_ (the link provides a basic introduction
@ -49,7 +49,7 @@ parameter set to ``True``. This will allow them to be exported with when the
attribute has been assigned a value.
Usage examples
-----------------
--------------
Now that a class has been defined (and imported) we can see how to use it.
@ -60,3 +60,12 @@ Download :download:`guts_usage.py </../../examples/guts_usage.py>`
:language: python
Example guts YAML File:
-----------------------
This is a minimal example of a guts YAML file.
Download :download:`guts_usage.py </../../examples/guts_example.yaml>`
.. literalinclude :: /../../examples/guts_example.yaml
:language: yaml

+ 9
- 0
examples/guts_example.yaml View File

@ -0,0 +1,9 @@
--- !gft.SensorArray
depth: 0.0
codes: !include codes.yml # We can include other yaml files
elevation: 0.0
interpolation: nearest_neighbor
distance_min: 1000.0
distance_max: 100000.0
strike: 0.0
sensor_count: 50

+ 76
- 7
src/guts.py View File

@ -12,6 +12,7 @@ import re
import sys
import types
import copy
import os.path as op
from collections import defaultdict
from io import BytesIO
@ -36,6 +37,9 @@ except NameError:
newstr = str
ALLOW_INCLUDE = False
class GutsSafeDumper(SafeDumper):
pass
@ -231,7 +235,9 @@ def expand_stream_args(mode):
def g(*args, **kwargs):
stream = kwargs.pop('stream', None)
filename = kwargs.pop('filename', None)
filename = kwargs.get('filename', None)
if mode != 'r':
filename = kwargs.pop('filename', None)
string = kwargs.pop('string', None)
assert sum(x is not None for x in (stream, filename, string)) <= 1
@ -1504,16 +1510,36 @@ def _dump_all(object, stream, header=True, Dumper=GutsSafeDumper):
_dump(object, stream=stream, header=header, _dump_function=yaml.dump_all)
def _load(stream, Loader=GutsSafeLoader):
return yaml.load(stream=stream, Loader=Loader)
def _load(stream,
Loader=GutsSafeLoader, allow_include=None, filename=None,
included_files=None):
class _Loader(Loader):
_filename = filename
_allow_include = allow_include
_included_files = included_files or []
return yaml.load(stream=stream, Loader=_Loader)
def _load_all(stream,
Loader=GutsSafeLoader, allow_include=None, filename=None):
def _load_all(stream, Loader=GutsSafeLoader):
return list(yaml.load_all(stream=stream, Loader=Loader))
class _Loader(Loader):
_filename = filename
_allow_include = allow_include
return list(yaml.load_all(stream=stream, Loader=_Loader))
def _iload_all(stream, Loader=GutsSafeLoader):
return yaml.load_all(stream=stream, Loader=Loader)
def _iload_all(stream,
Loader=GutsSafeLoader, allow_include=None, filename=None):
class _Loader(Loader):
_filename = filename
_allow_include = allow_include
return yaml.load_all(stream=stream, Loader=_Loader)
def multi_representer(dumper, data):
@ -1542,12 +1568,52 @@ def multi_constructor(loader, tag_suffix, node):
return o
def include_constructor(loader, node):
allow_include = loader._allow_include \
if loader._allow_include is not None \
else ALLOW_INCLUDE
if not allow_include:
raise EnvironmentError(
'Not allowed to include YAML. Load with allow_include=True')
if isinstance(node, yaml.nodes.ScalarNode):
inc_file = loader.construct_scalar(node)
else:
raise TypeError('Unsupported YAML node %s' % repr(node))
if loader._filename is not None and not op.isabs(inc_file):
inc_file = op.join(op.dirname(loader._filename), inc_file)
if not op.isfile(inc_file):
raise FileNotFoundError(inc_file)
included_files = list(loader._included_files)
if loader._filename is not None:
included_files.append(op.abspath(loader._filename))
for included_file in loader._included_files:
if op.samefile(inc_file, included_file):
raise ImportError(
'Circular import of file "%s". Include path: %s' % (
op.abspath(inc_file),
' -> '.join('"%s"' % s for s in included_files)))
with open(inc_file) as f:
return _load(
f,
Loader=loader.__class__, filename=inc_file,
allow_include=True,
included_files=included_files)
def dict_noflow_representer(dumper, data):
return dumper.represent_mapping(
'tag:yaml.org,2002:map', data, flow_style=False)
yaml.add_multi_representer(Object, multi_representer, Dumper=GutsSafeDumper)
yaml.add_constructor('!include', include_constructor, Loader=GutsSafeLoader)
yaml.add_multi_constructor('!', multi_constructor, Loader=GutsSafeLoader)
yaml.add_representer(dict, dict_noflow_representer, Dumper=GutsSafeDumper)
@ -2083,6 +2149,7 @@ def dump_xml(*args, **kwargs):
@expand_stream_args('r')
def load_xml(*args, **kwargs):
kwargs.pop('filename', None)
return _load_xml(*args, **kwargs)
@ -2097,11 +2164,13 @@ def dump_all_xml(*args, **kwargs):
@expand_stream_args('r')
def load_all_xml(*args, **kwargs):
kwargs.pop('filename', None)
return _load_all_xml(*args, **kwargs)
@expand_stream_args('r')
def iload_all_xml(*args, **kwargs):
kwargs.pop('filename', None)
return _iload_all_xml(*args, **kwargs)


+ 68
- 0
test/base/test_guts.py View File

@ -6,6 +6,7 @@ import math
import re
import sys
import datetime
import os.path as op
from contextlib import contextmanager
@ -17,6 +18,8 @@ from pyrocko.guts import StringPattern, Object, Bool, Int, Float, String, \
make_typed_list_class, walk, zip_walk, path_to_str, clone, set_elements, \
get_elements, YPathError
import pyrocko.guts
try:
unicode
@ -1264,6 +1267,71 @@ l_flow: ['a', 'b', 'c']
except ImportError:
pass
def testYAMLinclude(self):
from tempfile import NamedTemporaryFile as NFT
pyrocko.guts.ALLOW_INCLUDE = True
class A(Object):
f = Float.T()
i = Int.T()
s = String.T()
def assert_equal(obj):
assert obj.f == a.f
assert obj.i == a.i
assert obj.s == a.s
a = A(f=123.2, i=1024, s='hello')
with NFT('w') as f1:
a.dump(filename=f1.name)
with NFT('w') as f2:
f2.write('''---
a: !include {fname}
b: !include {fname}
c: [!include {fname}]
'''.format(fname=f1.name))
f2.flush()
o_abs = load(filename=f2.name)
assert_equal(o_abs['a'])
assert_equal(o_abs['b'])
assert_equal(o_abs['c'][0])
# Relative import
pyrocko.guts.ALLOW_INCLUDE = False
with NFT('w') as f2:
f2.write('''---
a: !include ./{fname}
b: !include ./{fname}
c: [!include ./{fname}]
'''.format(fname=op.basename(f1.name)))
f2.flush()
o_rel = load(filename=f2.name, allow_include=True)
assert_equal(o_rel['a'])
assert_equal(o_rel['b'])
assert_equal(o_rel['c'][0])
with self.assertRaises(ImportError):
with NFT('w') as f:
f.write('''
!include {fname}
'''.format(fname=f.name))
f.flush()
load(filename=f.name, allow_include=True)
with self.assertRaises(FileNotFoundError):
with NFT('w') as f:
f.write('!include /tmp/does_not_exist.yaml')
f.flush()
load(filename=f.name, allow_include=True)
with self.assertRaises(FileNotFoundError):
with NFT('w') as f:
f.write('!include ./does_not_exist.yaml')
f.flush()
load(filename=f.name, allow_include=True)
def makeBasicTypeTest(Type, sample, sample_in=None, xml=False):


Loading…
Cancel
Save