Browse Source

CI and pip deployment

deploy-docs
Sebastian Heimann 11 months ago
parent
commit
47c4f224c7
  1. 2
      .coveragerc
  2. 57
      .drone.yml
  3. 3
      MANIFEST.in
  4. 3
      docs/source/conf.py
  5. 32
      maintenance/deploy-docs.sh
  6. 18
      maintenance/docker/build_docker_images.sh
  7. 9
      maintenance/docker/docs/Dockerfile
  8. 7
      maintenance/docker/fat-aux/Dockerfile
  9. 4
      maintenance/docker/fat-nest/Dockerfile
  10. 6
      maintenance/docker/nest/Dockerfile
  11. 0
      maintenance/drone/deploy-docs.sh
  12. 0
      maintenance/drone/matterpost.py
  13. 0
      maintenance/drone/rsync.sh
  14. 77
      maintenance/pip/readme.rst
  15. 2
      maintenance/pip/upload.sh
  16. 4
      requirements.txt
  17. 1
      setup.cfg
  18. 8
      setup.py
  19. 2
      src/apps/grond.py
  20. 1
      src/environment.py
  21. 8
      src/meta.py
  22. 4
      src/problems/plot.py
  23. 2
      src/targets/gnss_campaign/plot.py
  24. 16
      src/targets/satellite/plot.py
  25. 3
      src/targets/waveform_phase_ratio/target.py
  26. 4
      test/__init__.py
  27. 4
      test/common.py

2
.coveragerc

@ -0,0 +1,2 @@
[run]
source = grond

57
.drone.yml

@ -1,3 +1,4 @@
kind: pipeline
type: docker
name: flake8
@ -27,7 +28,7 @@ steps:
image: pyrocko-aux
pull: never
commands:
- maintenance/drone-rsync.sh docs/build/html/ ${DRONE_COMMIT}/docs/
- maintenance/drone/rsync.sh docs/build/html/ ${DRONE_COMMIT}/docs/
environment:
RSYNC_HOST:
from_secret: rsync-host
@ -46,7 +47,7 @@ steps:
- deploy-docs
commands:
- maintenance/drone-rsync.sh docs/build/html/ ${DRONE_COMMIT}/docs/
- maintenance/drone/rsync.sh docs/build/html/ ${DRONE_COMMIT}/docs/
environment:
RSYNC_HOST:
from_secret: www-host
@ -65,7 +66,7 @@ steps:
- deploy-docs
commands:
- maintenance/drone-deploy-docs.sh
- maintenance/drone/deploy-docs.sh
environment:
WWW_HOST:
from_secret: www-host
@ -88,42 +89,51 @@ trigger:
steps:
- name: sdist-pip
- name: sdist
image: grond-nest
pull: never
commands:
- python setup.py sdist
- python3 setup.py sdist
- rm -f test/data
- ln -s /grond-test-data test/data
- ln -s /grond-test-playground test_playground
- name: test-pure-pip
image: pyrocko-aux
- name: test-pure
image: grond-fat-aux
pull: never
when:
branch:
exclude:
- release
commands:
- mkdir /grond-test-playground
- pip3 install pyrocko nose
- pip3 install utm
- pip3 install git+https://github.com/pyrocko/kite
- pip3 install -f dist --no-index --no-deps grond
- python3 -m nose test
depends_on:
- sdist
- name: test-mixed-pip-manylinux1_x86_64
image: grond-nest
- name: test-mixed
image: grond-fat-nest
pull: never
when:
branch:
exclude:
- release
commands:
- mkdir /grond-test-playground
- pip3 install -f dist --no-index --no-deps grond
- python3 -m nose test
depends_on:
- sdist
- name: stage
image: pyrocko-aux
pull: never
commands:
- maintenance/drone-rsync.sh dist/ ${DRONE_COMMIT}/dist/
- maintenance/drone/rsync.sh dist/ ${DRONE_COMMIT}/dist/
environment:
RSYNC_HOST:
from_secret: rsync-host
@ -131,35 +141,42 @@ steps:
from_secret: rsync-user
RSYNC_KEY:
from_secret: rsync-key
depends_on:
- test-pure
- test-mixed
- name: upload-testing
image: pyrocko-aux
image: grond-nest
pull: never
when:
branch:
- candidate
- pip
commands:
- maintenance/pip/upload_wheels.sh testing
- maintenance/pip/upload.sh testing
environment:
PYPI_USERNAME:
from_secret: pypi-username
PYPI_PASSWORD:
from_secret: pypi-password
depends_on:
- stage
- name: upload
image: pyrocko-aux
image: grond-nest
pull: never
when:
branch:
- release
commands:
- maintenance/pip/upload_wheels.sh live
- maintenance/pip/upload.sh live
environment:
PYPI_USERNAME:
from_secret: pypi-username
PYPI_PASSWORD:
from_secret: pypi-password
depends_on:
- stage
---
@ -169,9 +186,10 @@ name: tests-base
steps:
- name: tests-base
image: grond-nest
image: grond-fat-nest
pull: never
commands:
- ln -s /grond-test-data test/data
- python3 setup.py install
- python3 -m coverage run --parallel-mode -m nose test
- for x in .coverage.* ; do mv $x $${x#.} ; done
@ -180,7 +198,7 @@ steps:
image: pyrocko-aux
pull: never
commands:
- maintenance/drone-rsync.sh coverage.* ${DRONE_COMMIT}/coverage/data/
- maintenance/drone/rsync.sh coverage.* ${DRONE_COMMIT}/coverage/data/
environment:
RSYNC_HOST:
from_secret: rsync-host
@ -197,9 +215,6 @@ name: coverage
depends_on:
- tests-base
- tests-gf
- tests-examples
- tests-gui
steps:
- name: build
@ -215,7 +230,7 @@ steps:
image: pyrocko-aux
pull: never
commands:
- maintenance/drone-rsync.sh htmlcov/ ${DRONE_COMMIT}/coverage/
- maintenance/drone/rsync.sh htmlcov/ ${DRONE_COMMIT}/coverage/
environment:
RSYNC_HOST:
from_secret: rsync-host
@ -246,7 +261,7 @@ steps:
image: pyrocko-aux
pull: never
commands:
- maintenance/drone-matterpost.py
- maintenance/drone/matterpost.py
environment:
WEBHOOK:
from_secret: hive-webhook

3
MANIFEST.in

@ -0,0 +1,3 @@
recursive-include maintenance *.rst
include *.md

3
docs/source/conf.py

@ -30,6 +30,7 @@
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
from datetime import datetime as dt
import grond
import sphinx_sleekcat_theme
@ -58,7 +59,7 @@ master_doc = 'index'
# General information about the project.
project = u'Grond'
copyright = u'2017, The Grond Developers'
copyright = u'%d, The Grond Developers' % dt.now().year
author = u'The Grond Developers'
# The version info for the project you're documenting, acts as replacement for

32
maintenance/deploy-docs.sh

@ -1,32 +0,0 @@
#!/bin/bash
set -e
VERSION=v`python3 -c "import grond; print(grond.__version__);"`
if [ ! -f maintenance/deploy-docs.sh ] ; then
echo "must be run from grond's toplevel directory"
exit 1
fi
cd docs
rm -rf build/$VERSION
make clean; make html $1
cp -r build/html build/$VERSION
read -r -p "Are your sure to update live docs at http://pyrocko.org/grond/docs/$VERSION/ [y/N]?" resp
case $resp in
[yY][eE][sS]|[yY] )
rsync -av build/$VERSION/ pyrocko@hive:/var/www/pyrocko.org/grond/docs/$VERSION/;
;;
* ) ;;
esac
read -r -p "Do you want to link 'current' to the just uploaded version $VERSION [y/N]?" resp
case $resp in
[yY][eE][sS]|[yY] )
echo "Linking grond/docs/$VERSION to grond/docs/current";
ssh pyrocko@hive "rm -f /var/www/pyrocko.org/grond/docs/current; ln -s /var/www/pyrocko.org/grond/docs/$VERSION /var/www/pyrocko.org/grond/docs/current";
;;
* ) ;;
esac
cd ..

18
maintenance/docker/build_docker_images.sh

@ -0,0 +1,18 @@
#!/bin/bash
docker build nest -t grond-nest
docker build docs -t grond-docs
if [ ! -d "fat-nest/grond-test-data" ] ; then
if [ ! -d "../test/data" ] ; then
echo "Make sure complete test data is in "../test/data", by running the tests."
exit 1
fi
rsync -av "../test/data/" "fat-nest/grond-test-data/"
rsync -av "../test/data/" "fat-aux/grond-test-data/"
fi
docker build fat-nest -t grond-fat-nest
docker build fat-aux -t grond-fat-aux

9
maintenance/docker/docs/Dockerfile

@ -0,0 +1,9 @@
FROM grond-nest
# docs requirements
RUN apt-get install -y python3-sphinx \
texlive-fonts-recommended texlive-latex-extra \
texlive-latex-recommended texlive-generic-extra \
python3-sphinxcontrib.programoutput
RUN pip3 install git+https://git.pyrocko.org/pyrocko/sphinx-sleekcat-theme.git

7
maintenance/docker/fat-aux/Dockerfile

@ -0,0 +1,7 @@
FROM pyrocko-aux
# additional runtime requirements for gmt
RUN apt-get install -y \
gmt gmt-gshhg poppler-utils imagemagick
COPY grond-test-data /grond-test-data

4
maintenance/docker/fat-nest/Dockerfile

@ -0,0 +1,4 @@
FROM grond-nest
COPY grond-test-data /grond-test-data

6
maintenance/docker/nest/Dockerfile

@ -0,0 +1,6 @@
FROM pyrocko
WORKDIR /src
RUN pip3 install utm
RUN git clone https://github.com/pyrocko/kite.git && cd kite \
&& python3 setup.py install

0
maintenance/drone-deploy-docs.sh → maintenance/drone/deploy-docs.sh

0
maintenance/drone-matterpost.py → maintenance/drone/matterpost.py

0
maintenance/drone-rsync.sh → maintenance/drone/rsync.sh

77
maintenance/pip/readme.rst

@ -0,0 +1,77 @@
`Grond <https://pyrocko.org/grond/>`_ is an open source software tool for
robust characterization of earthquake sources. Moment tensors and finite fault
rupture models can be estimated from a combination of seismic waveforms,
waveform attributes and geodetic observations like InSAR and GNSS. It helps you
to investigate diverse magmatic, tectonic, and other geophysical processes at
all scales.
It delivers meaningful model uncertainties through a `Bayesian bootstrap-based
probabilistic joint inversion scheme
<https://pyrocko.org/grond/docs/current/method/>`_. The optimisation explores
the full model space and maps model parameter trade-offs with a flexible design
of objective functions.
Rapid forward modelling is enabled by using pre-computed `Green’s function
databases <https://greens-mill.pyrocko.org/>`_, handled through the `Pyrocko
<https://pyrocko.org/docs>`_ software library. They serve synthetic near-field
surface displacements and synthetic seismic waveforms for arbitrary earthquake
source models and geometries.
Installation with pip
---------------------
*See also:* `Grond Manual: Installation
<https://pyrocko.org/grond/docs/current/install>`_
Grond and all its dependencies can be installed by running
.. code-block:: bash
pip install grond
**but**, we recommend to make a conscious decision about how its main
dependency `Pyrocko <https://pyrocko.org/docs>`_ and especially Pyrocko's own
dependencies are installed. The `Pyrocko Installation Manual
<https://pyrocko.org/docs/current/install/>`_ describes different installation
schemes.
As a general advice, we recommend to exclusively use either, (1) the system's
native package manager, (2) Anaconda, or (3) pip only. In (1) and (2), only
resort to use pip for those few packages which are not available as native
packages. It is easy to prove that using multiple package managers on one
system leads to non-deterministic behaviour!
To prevent pip from automatically resolving dependencies run
.. code-block:: bash
pip install --no-deps grond
This assumes that `Pyrocko <https://pyrocko.org/docs>`_ and `Kite
<https://pyrocko.org/kite/>`_ have been installed beforehand.
Documentation
--------------
Documentation and examples can be found in the `Grond Manual
<https://pyrocko.org/grond/>`_.
Community
---------
Meet people from all over the world doing awesome research with Grond in our
community chat: use the *Grond* channel in the friendly `Pyrocko Hive
<https://hive.pyrocko.org>`_. This is the best place to talk about new features,
special techniques or to get help on setting up your first inversion with
Grond.
Development
-----------
Grond is open source.
Join us at our `Git repository <https://git.pyrocko.org/pyrocko/grond/>`_ and
read the `Contribution guide
<https://git.pyrocko.org/pyrocko/grond/src/branch/master/CONTRIBUTING.md>`_.
-- The Grond Developers

2
maintenance/pip/upload_wheels.sh → maintenance/pip/upload.sh

@ -3,7 +3,7 @@
destination=$1
if [ -z "$destination" ] ; then
echo "usage: upload_wheels.sh (testing|live)"
echo "usage: upload.sh (testing|live)"
fi
if [ "$destination" == 'live' ] ; then

4
requirements.txt

@ -1,5 +1 @@
utm
mpld3
numpy
scipy
pyrocko

1
setup.cfg

@ -1,5 +1,6 @@
[metadata]
description-file=README.md
long_description= file: maintenance/pip/readme.rst
license-file=LICENSE
[bdist_wheel]

8
setup.py

@ -71,6 +71,7 @@ installed_date = %s
with open(outfile, 'w') as f:
f.write(module_code)
class CustomBuildPyCommand(build_py):
def run(self):
@ -99,6 +100,7 @@ class CustomDevelopCommand(develop):
make_info_module('grond', version, outfile)
super().run()
setup(
cmdclass={
'build_py': CustomBuildPyCommand,
@ -111,6 +113,12 @@ setup(
description='A probabilistic earthquake source inversion framework. '
'Designed and crafted in Mordor.',
long_description=open(
'maintenance/pip/readme.rst', 'rb').read().decode('utf8'),
long_description_content_type='text/x-rst',
url='https://pyrocko.org/grond/',
version=version,
author='The Grond Developers',

2
src/apps/grond.py

@ -551,7 +551,7 @@ def command_init(args):
sys.stdout.write(config+'\n\n')
logger.info('Hint: To create a project, use: grond init <example> '
'<projectdir>'.format(c=Color, example=args[0]))
'<projectdir>')
elif op.exists(op.abspath(args[1])) and not options.force:
help_and_die(

1
src/environment.py

@ -2,7 +2,6 @@ import time
import logging
import shutil
import os
from contextlib import contextmanager
from grond.config import read_config, write_config
from grond import meta, run_info

8
src/meta.py

@ -61,9 +61,9 @@ def nextpow2(i):
return 2**int(math.ceil(math.log(i) / math.log(2.)))
def gather(l, key, sort=None, filter=None):
def gather(seq, key, sort=None, filter=None):
d = {}
for x in l:
for x in seq:
if filter is not None and not filter(x):
continue
@ -316,8 +316,8 @@ def nslc_to_pattern(s):
raise GrondError('Invalid net.sta.loc.cha pattern: %s' % s)
def nslcs_to_patterns(l):
return [nslc_to_pattern(s) for s in l]
def nslcs_to_patterns(seq):
return [nslc_to_pattern(s) for s in seq]
class SelectionError(GrondError):

4
src/problems/plot.py

@ -709,8 +709,8 @@ class MTLocationPlot(SectionPlotConfig):
default='dc')
normalisation_gamma = Float.T(
default=3.,
help='Normalisation of colors and alpha as :math:`x^\gamma`.'
'A linear colormap/alpha with :math:`\gamma=1`.')
help='Normalisation of colors and alpha as :math:`x^\\gamma`.'
'A linear colormap/alpha with :math:`\\gamma=1`.')
def make(self, environ):
environ.setup_modelling()

2
src/targets/gnss_campaign/plot.py

@ -8,7 +8,7 @@ from pyrocko import orthodrome as od
from grond.plot.config import PlotConfig
from grond.plot.collection import PlotItem
from grond.problems import CMTProblem, RectangularProblem, \
VolumePointProblem, VLVDProblem
VLVDProblem
from ..plot import StationDistributionPlot

16
src/targets/satellite/plot.py

@ -234,8 +234,8 @@ edge marking the upper fault edge. Complete data extent is shown.
arr = num.full_like(scene.displacement, fill_value=num.nan)
qt = scene.quadtree
for syn_v, l in zip(displacements, qt.leaves):
arr[l._slice_rows, l._slice_cols] = syn_v
for syn_v, leaf in zip(displacements, qt.leaves):
arr[leaf._slice_rows, leaf._slice_cols] = syn_v
arr[scene.displacement_mask] = num.nan
@ -313,11 +313,11 @@ edge marking the upper fault edge. Complete data extent is shown.
off_e = source.effective_lon - target.scene.frame.llLon
turE, turN, tllE, tllN = zip(
*[(l.gridE.max()-off_e,
l.gridN.max()-off_n,
l.gridE.min()-off_e,
l.gridN.min()-off_n)
for l in target.scene.quadtree.leaves])
*[(leaf.gridE.max()-off_e,
leaf.gridN.max()-off_n,
leaf.gridE.min()-off_e,
leaf.gridN.min()-off_n)
for leaf in target.scene.quadtree.leaves])
turE, turN = map(max, (turE, turN))
tllE, tllN = map(min, (tllE, tllN))
@ -345,7 +345,7 @@ edge marking the upper fault edge. Complete data extent is shown.
Surface displacements derived from satellite data.
(Left) the input data, (center) the modelled
data and (right) the model residual.
'''.format(meta=scene.meta))
''')
stat_obs = result.statics_obs
stat_syn = result.statics_syn['displacement.los']

3
src/targets/waveform_phase_ratio/target.py

@ -11,8 +11,7 @@ from . import measure as fm
from grond import dataset
from grond.meta import has_get_plot_classes
from ..waveform.target import StoreIDSelector, StoreIDSelectorError,\
Crust2StoreIDSelector, StationDictStoreIDSelector
from ..waveform.target import StoreIDSelector
guts_prefix = 'grond'
logger = logging.getLogger('grond.targets.waveform_phase_ratio.target')

4
test/__init__.py

@ -1,8 +1,8 @@
import matplotlib
matplotlib.use('Agg')
from pyrocko import util
from pyrocko import util # noqa
util.force_dummy_progressbar = True
util.setup_logging('grondtest', 'info')
import warnings
import warnings # noqa
warnings.simplefilter(action='ignore', category=FutureWarning)

4
test/common.py

@ -80,8 +80,8 @@ class Capture(object):
self.orig_stdout.write(data)
def writelines(self, lines):
for l in lines:
self.write(l)
for line in lines:
self.write(line)
def flush(self):
self.file.flush()

Loading…
Cancel
Save