CI and pip deployment

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+source = grond

.drone.yml

@@ -1,3 +1,4 @@
+
 kind: pipeline
 type: docker
 name: flake8
@@ -27,7 +28,7 @@ steps:
   image: pyrocko-aux
   pull: never
   commands:
-  - maintenance/drone-rsync.sh docs/build/html/ ${DRONE_COMMIT}/docs/
+  - maintenance/drone/rsync.sh docs/build/html/ ${DRONE_COMMIT}/docs/
   environment:
     RSYNC_HOST:
       from_secret: rsync-host
@@ -46,7 +47,7 @@ steps:
     - deploy-docs
 
   commands:
-  - maintenance/drone-rsync.sh docs/build/html/ ${DRONE_COMMIT}/docs/
+  - maintenance/drone/rsync.sh docs/build/html/ ${DRONE_COMMIT}/docs/
   environment:
     RSYNC_HOST:
       from_secret: www-host
@@ -65,7 +66,7 @@ steps:
     - deploy-docs
 
   commands:
-  - maintenance/drone-deploy-docs.sh
+  - maintenance/drone/deploy-docs.sh
   environment:
     WWW_HOST:
       from_secret: www-host
@@ -88,42 +89,51 @@ trigger:
 
 steps:
 
-- name: sdist-pip
+- name: sdist
   image: grond-nest
   pull: never
   commands:
-  - python setup.py sdist
+  - python3 setup.py sdist
+  - rm -f test/data
+  - ln -s /grond-test-data test/data
+  - ln -s /grond-test-playground test_playground
 
-- name: test-pure-pip
-  image: pyrocko-aux
+- name: test-pure
+  image: grond-fat-aux
   pull: never
   when:
     branch:
       exclude:
       - release
   commands:
+  - mkdir /grond-test-playground
   - pip3 install pyrocko nose
   - pip3 install utm
   - pip3 install git+https://github.com/pyrocko/kite
   - pip3 install -f dist --no-index --no-deps grond
   - python3 -m nose test
+  depends_on:
+  - sdist
 
-- name: test-mixed-pip-manylinux1_x86_64
-  image: grond-nest
+- name: test-mixed
+  image: grond-fat-nest
   pull: never
   when:
     branch:
       exclude:
       - release
   commands:
+  - mkdir /grond-test-playground
   - pip3 install -f dist --no-index --no-deps grond
   - python3 -m nose test
+  depends_on:
+  - sdist
 
 - name: stage
   image: pyrocko-aux
   pull: never
   commands:
-  - maintenance/drone-rsync.sh dist/ ${DRONE_COMMIT}/dist/
+  - maintenance/drone/rsync.sh dist/ ${DRONE_COMMIT}/dist/
   environment:
     RSYNC_HOST:
       from_secret: rsync-host
@@ -131,35 +141,42 @@ steps:
       from_secret: rsync-user
     RSYNC_KEY:
       from_secret: rsync-key
+  depends_on:
+  - test-pure
+  - test-mixed
 
 - name: upload-testing
-  image: pyrocko-aux
+  image: grond-nest
   pull: never
   when:
     branch:
     - candidate
     - pip
   commands:
-  - maintenance/pip/upload_wheels.sh testing
+  - maintenance/pip/upload.sh testing
   environment:
     PYPI_USERNAME:
       from_secret: pypi-username
     PYPI_PASSWORD:
       from_secret: pypi-password
+  depends_on:
+  - stage
 
 - name: upload
-  image: pyrocko-aux
+  image: grond-nest
   pull: never
   when:
     branch:
     - release
   commands:
-  - maintenance/pip/upload_wheels.sh live
+  - maintenance/pip/upload.sh live
   environment:
     PYPI_USERNAME:
       from_secret: pypi-username
     PYPI_PASSWORD:
       from_secret: pypi-password
+  depends_on:
+  - stage
 
 ---
 
@@ -169,9 +186,10 @@ name: tests-base
 
 steps:
 - name: tests-base
-  image: grond-nest
+  image: grond-fat-nest
   pull: never
   commands:
+  - ln -s /grond-test-data test/data
   - python3 setup.py install
   - python3 -m coverage run --parallel-mode -m nose test
   - for x in .coverage.* ; do mv $x $${x#.} ; done
@@ -180,7 +198,7 @@ steps:
   image: pyrocko-aux
   pull: never
   commands:
-  - maintenance/drone-rsync.sh coverage.*  ${DRONE_COMMIT}/coverage/data/
+  - maintenance/drone/rsync.sh coverage.*  ${DRONE_COMMIT}/coverage/data/
   environment:
     RSYNC_HOST:
       from_secret: rsync-host
@@ -197,9 +215,6 @@ name: coverage
 
 depends_on:
 - tests-base
-- tests-gf
-- tests-examples
-- tests-gui
 
 steps:
 - name: build
@@ -215,7 +230,7 @@ steps:
   image: pyrocko-aux
   pull: never
   commands:
-  - maintenance/drone-rsync.sh htmlcov/ ${DRONE_COMMIT}/coverage/
+  - maintenance/drone/rsync.sh htmlcov/ ${DRONE_COMMIT}/coverage/
   environment:
     RSYNC_HOST:
       from_secret: rsync-host
@@ -246,7 +261,7 @@ steps:
   image: pyrocko-aux
   pull: never
   commands:
-  - maintenance/drone-matterpost.py
+  - maintenance/drone/matterpost.py
   environment:
     WEBHOOK:
       from_secret: hive-webhook

MANIFEST.in

@@ -0,0 +1,3 @@
+recursive-include maintenance *.rst
+include *.md
+

docs/source/conf.py

@@ -30,6 +30,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
+from datetime import datetime as dt
 import grond
 import sphinx_sleekcat_theme
 
@@ -58,7 +59,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Grond'
-copyright = u'2017, The Grond Developers'
+copyright = u'%d, The Grond Developers' % dt.now().year
 author = u'The Grond Developers'
 
 # The version info for the project you're documenting, acts as replacement for

maintenance/deploy-docs.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-set -e
-VERSION=v`python3 -c "import grond; print(grond.__version__);"`
-
-if [ ! -f maintenance/deploy-docs.sh ] ; then
-    echo "must be run from grond's toplevel directory"
-    exit 1
-fi
-
-cd docs
-rm -rf build/$VERSION
-make clean; make html $1
-cp -r build/html build/$VERSION
-
-read -r -p "Are your sure to update live docs at http://pyrocko.org/grond/docs/$VERSION/ [y/N]?" resp
-case $resp in
-    [yY][eE][sS]|[yY] )
-        rsync -av build/$VERSION/ pyrocko@hive:/var/www/pyrocko.org/grond/docs/$VERSION/;
-        ;;
-    * ) ;;
-esac
-
-read -r -p "Do you want to link 'current' to the just uploaded version $VERSION [y/N]?" resp
-case $resp in
-    [yY][eE][sS]|[yY] )
-        echo "Linking grond/docs/$VERSION to grond/docs/current";
-        ssh pyrocko@hive "rm -f /var/www/pyrocko.org/grond/docs/current; ln -s /var/www/pyrocko.org/grond/docs/$VERSION /var/www/pyrocko.org/grond/docs/current";
-        ;;
-    * ) ;;
-esac
-
-cd ..

maintenance/docker/build_docker_images.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+docker build nest -t grond-nest
+docker build docs -t grond-docs
+
+
+if [ ! -d "fat-nest/grond-test-data" ] ; then
+    if [ ! -d "../test/data" ] ; then
+        echo "Make sure complete test data is in "../test/data", by running the tests."
+        exit 1
+    fi
+
+    rsync -av "../test/data/" "fat-nest/grond-test-data/"
+    rsync -av "../test/data/" "fat-aux/grond-test-data/"
+
+fi
+
+docker build fat-nest -t grond-fat-nest
+docker build fat-aux -t grond-fat-aux

maintenance/docker/docs/Dockerfile

@@ -0,0 +1,9 @@
+FROM grond-nest
+
+# docs requirements
+RUN apt-get install -y python3-sphinx \
+    texlive-fonts-recommended texlive-latex-extra \
+    texlive-latex-recommended texlive-generic-extra \
+    python3-sphinxcontrib.programoutput
+
+RUN pip3 install git+https://git.pyrocko.org/pyrocko/sphinx-sleekcat-theme.git

maintenance/docker/fat-aux/Dockerfile

@@ -0,0 +1,7 @@
+FROM pyrocko-aux
+
+# additional runtime requirements for gmt
+RUN apt-get install -y \
+        gmt gmt-gshhg poppler-utils imagemagick
+
+COPY grond-test-data /grond-test-data

maintenance/docker/fat-nest/Dockerfile

@@ -0,0 +1,4 @@
+FROM grond-nest
+
+
+COPY grond-test-data /grond-test-data

maintenance/docker/nest/Dockerfile

@@ -0,0 +1,6 @@
+FROM pyrocko
+
+WORKDIR /src
+RUN pip3 install utm
+RUN git clone https://github.com/pyrocko/kite.git && cd kite \
+    && python3 setup.py install

maintenance/pip/readme.rst

@@ -0,0 +1,77 @@
+`Grond <https://pyrocko.org/grond/>`_ is an open source software tool for
+robust characterization of earthquake sources. Moment tensors and finite fault
+rupture models can be estimated from a combination of seismic waveforms,
+waveform attributes and geodetic observations like InSAR and GNSS. It helps you
+to investigate diverse magmatic, tectonic, and other geophysical processes at
+all scales.
+
+It delivers meaningful model uncertainties through a `Bayesian bootstrap-based
+probabilistic joint inversion scheme
+<https://pyrocko.org/grond/docs/current/method/>`_. The optimisation explores
+the full model space and maps model parameter trade-offs with a flexible design
+of objective functions.
+
+Rapid forward modelling is enabled by using pre-computed `Green’s function
+databases <https://greens-mill.pyrocko.org/>`_, handled through the `Pyrocko
+<https://pyrocko.org/docs>`_ software library. They serve synthetic near-field
+surface displacements and synthetic seismic waveforms for arbitrary earthquake
+source models and geometries.
+
+Installation with pip
+---------------------
+
+*See also:* `Grond Manual: Installation
+<https://pyrocko.org/grond/docs/current/install>`_
+
+Grond and all its dependencies can be installed by running 
+
+.. code-block:: bash
+
+   pip install grond
+
+**but**, we recommend to make a conscious decision about how its main
+dependency `Pyrocko <https://pyrocko.org/docs>`_ and especially Pyrocko's own
+dependencies are installed. The `Pyrocko Installation Manual
+<https://pyrocko.org/docs/current/install/>`_ describes different installation
+schemes.
+
+As a general advice, we recommend to exclusively use either, (1) the system's
+native package manager, (2) Anaconda, or (3) pip only. In (1) and (2), only
+resort to use pip for those few packages which are not available as native
+packages. It is easy to prove that using multiple package managers on one
+system leads to non-deterministic behaviour!
+
+To prevent pip from automatically resolving dependencies run
+
+.. code-block:: bash
+
+   pip install --no-deps grond
+
+This assumes that `Pyrocko <https://pyrocko.org/docs>`_ and `Kite
+<https://pyrocko.org/kite/>`_ have been installed beforehand.
+
+Documentation
+--------------
+
+Documentation and examples can be found in the `Grond Manual
+<https://pyrocko.org/grond/>`_.
+
+Community
+---------
+
+Meet people from all over the world doing awesome research with Grond in our
+community chat: use the *Grond* channel in the friendly `Pyrocko Hive
+<https://hive.pyrocko.org>`_. This is the best place to talk about new features,
+special techniques or to get help on setting up your first inversion with
+Grond.
+
+Development
+-----------
+
+Grond is open source.
+
+Join us at our `Git repository <https://git.pyrocko.org/pyrocko/grond/>`_ and
+read the `Contribution guide
+<https://git.pyrocko.org/pyrocko/grond/src/branch/master/CONTRIBUTING.md>`_.
+
+-- The Grond Developers

maintenance/pip/upload.sh

@@ -3,7 +3,7 @@
 destination=$1
 
 if [ -z "$destination" ] ; then
-    echo "usage: upload_wheels.sh (testing|live)"
+    echo "usage: upload.sh (testing|live)"
 fi
 
 if [ "$destination" == 'live' ] ; then

requirements.txt

@@ -1,5 +1 @@
-utm
-mpld3
-numpy
-scipy
 pyrocko

setup.cfg

@@ -1,5 +1,6 @@
 [metadata]
 description-file=README.md
+long_description= file: maintenance/pip/readme.rst
 license-file=LICENSE
 
 [bdist_wheel]

setup.py

@@ -71,6 +71,7 @@ installed_date = %s
     with open(outfile, 'w') as f:
         f.write(module_code)
 
+
 class CustomBuildPyCommand(build_py):
 
     def run(self):
@@ -99,6 +100,7 @@ class CustomDevelopCommand(develop):
         make_info_module('grond', version, outfile)
         super().run()
 
+
 setup(
     cmdclass={
         'build_py': CustomBuildPyCommand,
@@ -111,6 +113,12 @@ setup(
     description='A probabilistic earthquake source inversion framework. '
                 'Designed and crafted in Mordor.',
 
+    long_description=open(
+        'maintenance/pip/readme.rst', 'rb').read().decode('utf8'),
+    long_description_content_type='text/x-rst',
+
+    url='https://pyrocko.org/grond/',
+
     version=version,
 
     author='The Grond Developers',

src/apps/grond.py

@@ -551,7 +551,7 @@ def command_init(args):
             sys.stdout.write(config+'\n\n')
 
             logger.info('Hint: To create a project, use: grond init <example> '
-                        '<projectdir>'.format(c=Color, example=args[0]))
+                        '<projectdir>')
 
         elif op.exists(op.abspath(args[1])) and not options.force:
             help_and_die(

src/environment.py

@@ -2,7 +2,6 @@ import time
 import logging
 import shutil
 import os
-from contextlib import contextmanager
 
 from grond.config import read_config, write_config
 from grond import meta, run_info

src/meta.py

@@ -61,9 +61,9 @@ def nextpow2(i):
     return 2**int(math.ceil(math.log(i) / math.log(2.)))
 
 
-def gather(l, key, sort=None, filter=None):
+def gather(seq, key, sort=None, filter=None):
     d = {}
-    for x in l:
+    for x in seq:
         if filter is not None and not filter(x):
             continue
 
@@ -316,8 +316,8 @@ def nslc_to_pattern(s):
         raise GrondError('Invalid net.sta.loc.cha pattern: %s' % s)
 
 
-def nslcs_to_patterns(l):
-    return [nslc_to_pattern(s) for s in l]
+def nslcs_to_patterns(seq):
+    return [nslc_to_pattern(s) for s in seq]
 
 
 class SelectionError(GrondError):

src/problems/plot.py

@@ -709,8 +709,8 @@ class MTLocationPlot(SectionPlotConfig):
         default='dc')
     normalisation_gamma = Float.T(
         default=3.,
-        help='Normalisation of colors and alpha as :math:`x^\gamma`.'
-             'A linear colormap/alpha with :math:`\gamma=1`.')
+        help='Normalisation of colors and alpha as :math:`x^\\gamma`.'
+             'A linear colormap/alpha with :math:`\\gamma=1`.')
 
     def make(self, environ):
         environ.setup_modelling()

src/targets/gnss_campaign/plot.py

@@ -8,7 +8,7 @@ from pyrocko import orthodrome as od
 from grond.plot.config import PlotConfig
 from grond.plot.collection import PlotItem
 from grond.problems import CMTProblem, RectangularProblem, \
-    VolumePointProblem, VLVDProblem
+    VLVDProblem
 
 from ..plot import StationDistributionPlot
 

src/targets/satellite/plot.py

@@ -234,8 +234,8 @@ edge marking the upper fault edge. Complete data extent is shown.
             arr = num.full_like(scene.displacement, fill_value=num.nan)
             qt = scene.quadtree
 
-            for syn_v, l in zip(displacements, qt.leaves):
-                arr[l._slice_rows, l._slice_cols] = syn_v
+            for syn_v, leaf in zip(displacements, qt.leaves):
+                arr[leaf._slice_rows, leaf._slice_cols] = syn_v
 
             arr[scene.displacement_mask] = num.nan
 
@@ -313,11 +313,11 @@ edge marking the upper fault edge. Complete data extent is shown.
                 off_e = source.effective_lon - target.scene.frame.llLon
 
             turE, turN, tllE, tllN = zip(
-                *[(l.gridE.max()-off_e,
-                   l.gridN.max()-off_n,
-                   l.gridE.min()-off_e,
-                   l.gridN.min()-off_n)
-                  for l in target.scene.quadtree.leaves])
+                *[(leaf.gridE.max()-off_e,
+                   leaf.gridN.max()-off_n,
+                   leaf.gridE.min()-off_e,
+                   leaf.gridN.min()-off_n)
+                  for leaf in target.scene.quadtree.leaves])
 
             turE, turN = map(max, (turE, turN))
             tllE, tllN = map(min, (tllE, tllN))
@@ -345,7 +345,7 @@ edge marking the upper fault edge. Complete data extent is shown.
 Surface displacements derived from satellite data.
 (Left) the input data, (center) the modelled
 data and (right) the model residual.
-'''.format(meta=scene.meta))
+''')
 
             stat_obs = result.statics_obs
             stat_syn = result.statics_syn['displacement.los']

src/targets/waveform_phase_ratio/target.py

@@ -11,8 +11,7 @@ from . import measure as fm
 from grond import dataset
 from grond.meta import has_get_plot_classes
 
-from ..waveform.target import StoreIDSelector, StoreIDSelectorError,\
-        Crust2StoreIDSelector, StationDictStoreIDSelector
+from ..waveform.target import StoreIDSelector
 
 guts_prefix = 'grond'
 logger = logging.getLogger('grond.targets.waveform_phase_ratio.target')

test/__init__.py

@@ -1,8 +1,8 @@
 import matplotlib
 matplotlib.use('Agg')
-from pyrocko import util
+from pyrocko import util  # noqa
 util.force_dummy_progressbar = True
 util.setup_logging('grondtest', 'info')
 
-import warnings
+import warnings  # noqa
 warnings.simplefilter(action='ignore', category=FutureWarning)

test/common.py

@@ -80,8 +80,8 @@ class Capture(object):
             self.orig_stdout.write(data)
 
     def writelines(self, lines):
-        for l in lines:
-            self.write(l)
+        for line in lines:
+            self.write(line)
 
     def flush(self):
         self.file.flush()