Browse Source

drone testing

forward-show
Sebastian Heimann 1 year ago
parent
commit
96b5fdbc97
  1. 252
      .drone.yml
  2. 21
      CONTRIBUTING.md
  3. 18
      maintenance/drone-deploy-docs.sh
  4. 115
      maintenance/drone-matterpost.py
  5. 22
      maintenance/drone-rsync.sh
  6. 17
      maintenance/pip/upload_wheels.sh

252
.drone.yml

@ -0,0 +1,252 @@
kind: pipeline
type: docker
name: flake8
steps:
- name: flake8
image: grond-nest
pull: never
commands:
- flake8 src test setup.py
---
kind: pipeline
type: docker
name: docs
steps:
- name: build
image: grond-docs
pull: never
commands:
- python3 setup.py install && cd docs && make html
- grond version --short > build/html/version.txt
- name: stage
image: pyrocko-aux
pull: never
commands:
- maintenance/drone-rsync.sh docs/build/html/ ${DRONE_COMMIT}/docs/
environment:
RSYNC_HOST:
from_secret: rsync-host
RSYNC_USER:
from_secret: rsync-user
RSYNC_KEY:
from_secret: rsync-key
- name: stage-www
image: pyrocko-aux
pull: never
when:
branch:
- release
- deploy-docs
commands:
- maintenance/drone-rsync.sh docs/build/html/ ${DRONE_COMMIT}/docs/
environment:
RSYNC_HOST:
from_secret: www-host
RSYNC_USER:
from_secret: www-user
RSYNC_KEY:
from_secret: rsync-key
- name: deploy-www
image: pyrocko-aux
pull: never
when:
branch:
- release
- deploy-docs
commands:
- maintenance/drone-deploy-docs.sh
environment:
WWW_HOST:
from_secret: www-host
WWW_USER:
from_secret: www-user
WWW_KEY:
from_secret: www-key
---
kind: pipeline
type: docker
name: pip
trigger:
branch:
- release
- candidate
- pip
steps:
- name: sdist-pip
image: grond-nest
pull: never
commands:
- python setup.py sdist
- name: test-pure-pip
image: pyrocko-aux
pull: never
when:
branch:
exclude:
- release
commands:
- pip3 install pyrocko nose
- pip3 install utm
- pip3 install git+https://github.com/pyrocko/kite
- pip3 install -f dist --no-index --no-deps grond
- python3 -m nose test
- name: test-mixed-pip-manylinux1_x86_64
image: grond-nest
pull: never
when:
branch:
exclude:
- release
commands:
- pip3 install -f dist --no-index --no-deps grond
- python3 -m nose test
- name: stage
image: pyrocko-aux
pull: never
commands:
- maintenance/drone-rsync.sh dist/ ${DRONE_COMMIT}/dist/
environment:
RSYNC_HOST:
from_secret: rsync-host
RSYNC_USER:
from_secret: rsync-user
RSYNC_KEY:
from_secret: rsync-key
- name: upload-testing
image: pyrocko-aux
pull: never
when:
branch:
- candidate
- pip
commands:
- maintenance/pip/upload_wheels.sh testing
environment:
PYPI_USERNAME:
from_secret: pypi-username
PYPI_PASSWORD:
from_secret: pypi-password
- name: upload
image: pyrocko-aux
pull: never
when:
branch:
- release
commands:
- maintenance/pip/upload_wheels.sh live
environment:
PYPI_USERNAME:
from_secret: pypi-username
PYPI_PASSWORD:
from_secret: pypi-password
---
kind: pipeline
type: docker
name: tests-base
steps:
- name: tests-base
image: grond-nest
pull: never
commands:
- python3 setup.py install
- python3 -m coverage run --parallel-mode -m nose test
- for x in .coverage.* ; do mv $x $${x#.} ; done
- name: stage
image: pyrocko-aux
pull: never
commands:
- maintenance/drone-rsync.sh coverage.* ${DRONE_COMMIT}/coverage/data/
environment:
RSYNC_HOST:
from_secret: rsync-host
RSYNC_USER:
from_secret: rsync-user
RSYNC_KEY:
from_secret: rsync-key
---
kind: pipeline
type: docker
name: coverage
depends_on:
- tests-base
- tests-gf
- tests-examples
- tests-gui
steps:
- name: build
image: grond-nest
pull: never
commands:
- python3 setup.py install
- wget -r -nH --cut-dirs=2 --no-parent --reject="index.html*" https://data.pyrocko.org/builds/${DRONE_COMMIT}/coverage/data/
- python3 -m coverage combine coverage/data/coverage.*
- python3 -m coverage html
- name: stage
image: pyrocko-aux
pull: never
commands:
- maintenance/drone-rsync.sh htmlcov/ ${DRONE_COMMIT}/coverage/
environment:
RSYNC_HOST:
from_secret: rsync-host
RSYNC_USER:
from_secret: rsync-user
RSYNC_KEY:
from_secret: rsync-key
---
kind: pipeline
type: docker
name: notify-hive
depends_on:
- coverage
- flake8
- docs
- pip
trigger:
status:
- success
- failure
steps:
- name: post
image: pyrocko-aux
pull: never
commands:
- maintenance/drone-matterpost.py
environment:
WEBHOOK:
from_secret: hive-webhook

21
CONTRIBUTING.md

@ -9,6 +9,27 @@ Grond is written in the Python programming language (versions ==2.7 and >=3.4).
Grond uses Setuptools for its installation script. See `setup.py` and
`setup.cfg` in the project root directory.
## Testing
Nosetests is used for testing. To run all tests, run
```sh
python -m nose test
```
## CI
Drone CI tests are run on any commits pushed to the repository. By default,
flake8 is run, tests are run and test coverage is measured and docs are built.
Success is reported to the Pyrocko Hive. Pushing to specific branches triggers
extra pipelines:
* pip: pip sdist is built, tested and uploaded to PyPi-testing
* deploy-docs: docs are published
* candidate: same as pip
* release: same as pip but without pip testing and upload to PyPi-live +
deploy-docs
## Versioning and releases
Git is used for version control. Use development branches for new features.

18
maintenance/drone-deploy-docs.sh

@ -0,0 +1,18 @@
#!/bin/bash
set -e
[ ! -z "$DRONE" ] || exit 1
keypath=/tmp/deploy-www-key-$$
umask 0077
echo "$WWW_KEY" > $keypath
umask 0022
ssh -o StrictHostKeyChecking=no -i $keypath "${WWW_USER}@${WWW_HOST}" \
"${DRONE_COMMIT}" "grond"
rm $keypath

115
maintenance/drone-matterpost.py

@ -0,0 +1,115 @@
#!/usr/bin/env python3
import os
import random
import requests
def quoteliteral(s):
s = '\n'.join(' '+x for x in s.splitlines())
return '''```
%s
```
''' % s
def to_message():
message = {
'username': 'Grond Build Factory',
# 'icon_url': 'https://pyrocko.org/_static/pyrocko.svg',
'attachments': [],
}
text = '\n'.join(
'%s: %s' % (k, os.environ[k]) for k in sorted(os.environ.keys()))
env = os.environ
keys = '''
commit commit_message
commit_author commit_author_name commit_author_email commit_link
build_number build_link build_status
'''.split()
env = dict(
(k, os.environ.get(
'DRONE_%s' % k.upper(), 'DRONE_' + k.upper() + ' undefined'))
for k in keys)
emos_success = [':%s:' % s for s in '''
sunny star hatched_chick hamster dog butterfly sunglasses smile
heart_eyes stuck_out_tongue smile_cat muscle +1 ok_hand clap rainbow
beer champagne clinking_glasses medal_sports medal_military
man_cartwheeling woman_cartwheeling fireworks'''.split()]
emos_failure = [':%s:' % s for s in '''
frowning_face weary skull skull_and_crossbones cold_sweat -1
middle_finger scream_cat hankey tornado cloud_with_rain fire 8ball
boxing_glove hocho bomb rage1'''.split()]
env['emo'] = random.choice(
emos_success if env['build_status'] == 'success' else emos_failure)
artifacts = {}
for name, link in [
('docs', 'https://data.pyrocko.org/builds/%s/docs/'),
('coverage', 'https://data.pyrocko.org/builds/%s/coverage/index.html'), # noqa
('wheels', 'https://data.pyrocko.org/builds/%s/wheels/')]:
link = link % env['commit']
r = requests.get(link)
if r.status_code == 200:
artifacts[name] = link
env['artifacts'] = ' '.join(
'[%s](%s)' % (name, link) for (name, link) in artifacts.items())
env['commit_message'] = '\n'.join(env['commit_message'].splitlines()[:1])
total_coverage = None
if 'coverage' in artifacts:
r = requests.get(os.path.join(
os.path.dirname(artifacts['coverage']), 'status.json'))
coverage = r.json()
statements = 0
missing = 0
excluded = 0
for file in coverage['files'].values():
statements += file['index']['nums'][1]
excluded += file['index']['nums'][1]
missing += file['index']['nums'][3]
if statements > 0:
total_coverage = (statements - missing) / statements
text = '''{emo} **Build [{build_number}]({build_link}): {build_status}**
[{commit:.7s}]({commit_link}): {commit_message} - [{commit_author_name}](mailto:{commit_author_email})
Artifacts: {artifacts}
'''.format(**env) # noqa
if total_coverage:
text += 'Total coverage: %.1f%%\n' % (total_coverage * 100.)
attachment = {
'fallback': 'test',
'color': '#33CC33' if env['build_status'] == 'success' else '#CC3333',
'text': text,
}
message['attachments'].append(attachment)
return message
def mattermost_post(webhook, message):
requests.post(webhook, json=message)
if __name__ == '__main__':
webhook = os.environ.get('WEBHOOK', None)
message = to_message()
if webhook:
mattermost_post(webhook, message)
else:
print(message)

22
maintenance/drone-rsync.sh

@ -0,0 +1,22 @@
#!/bin/bash
set -e
[ ! -z "$DRONE" ] || exit 1
keypath=/tmp/rsync-key-$$
umask 0077
echo "$RSYNC_KEY" > $keypath
umask 0022
temppath=/tmp/rsync-temp-$$
mkdir -p $temppath/$2
rsync -av $1 $temppath/$2
rsync -rv -e "ssh -o StrictHostKeyChecking=no -i $keypath" $temppath/ ${RSYNC_USER}@${RSYNC_HOST}:
rm $keypath

17
maintenance/pip/upload_wheels.sh

@ -0,0 +1,17 @@
#!/bin/bash
destination=$1
if [ -z "$destination" ] ; then
echo "usage: upload_wheels.sh (testing|live)"
fi
if [ "$destination" == 'live' ] ; then
twine upload dist/grond-* \
--username="$PYPI_USERNAME" --password="$PYPI_PASSWORD" \
--skip-existing --disable-progress-bar --comment='*grunz-grunz*'
else
twine upload --repository-url https://test.pypi.org/legacy/ dist/grond-* \
--username="$PYPI_USERNAME" --password="$PYPI_PASSWORD" \
--skip-existing --disable-progress-bar --comment='*grunz-grunz*'
fi
Loading…
Cancel
Save