From 5c883a734b973f4e17770374da15ba7c4b28bff7 Mon Sep 17 00:00:00 2001 From: Manu <3916435+m3nu@users.noreply.github.com> Date: Tue, 18 Feb 2020 00:21:21 +0800 Subject: [PATCH] Add testing with Molecule/Ansible-lint, undo #9 (#26) * Add testing with Molecule/Ansible-lint and fix linting errors * Undo PR #9 for now (sorry) --- .travis.yml | 11 +++++++ .yamllint | 33 +++++++++++++++++++ README.md | 14 +++++++- defaults/main.yml | 1 + molecule/default/Dockerfile.j2 | 22 +++++++++++++ molecule/default/INSTALL.rst | 22 +++++++++++++ molecule/default/molecule.yml | 26 +++++++++++++++ molecule/default/playbook.yml | 22 +++++++++++++ molecule/default/tests/test_default.py | 33 +++++++++++++++++++ requirements-dev.txt | 3 ++ tasks/main.yml | 45 ++++++++++++++------------ tasks/yum.yml | 2 +- templates/config.yaml.j2 | 11 ++++--- tests/inventory | 2 -- tests/playbook.yml | 12 ------- vars/apt.yml | 3 ++ vars/yum.yml | 3 ++ 17 files changed, 224 insertions(+), 41 deletions(-) create mode 100644 .travis.yml create mode 100644 .yamllint create mode 100644 molecule/default/Dockerfile.j2 create mode 100644 molecule/default/INSTALL.rst create mode 100644 molecule/default/molecule.yml create mode 100644 molecule/default/playbook.yml create mode 100644 molecule/default/tests/test_default.py create mode 100644 requirements-dev.txt delete mode 100644 tests/inventory delete mode 100644 tests/playbook.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..7d2aca6 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +--- +sudo: required +language: python +services: + - docker +before_install: + - sudo apt-get -qq update +install: + - pip install molecule[lint,docker] +script: + - molecule test \ No newline at end of file diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..8827676 --- /dev/null +++ b/.yamllint @@ -0,0 +1,33 @@ +--- +# Based on ansible-lint config +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + colons: + max-spaces-after: -1 + level: error + commas: + max-spaces-after: -1 + level: error + comments: disable + comments-indentation: disable + document-start: disable + empty-lines: + max: 3 + level: error + hyphens: + level: error + indentation: disable + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable + truthy: disable diff --git a/README.md b/README.md index 478bc31..1c398e3 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ An Ansible Role that sets up automated remote backups on the target machine. Use ### Optional Arguments - `borg_encryption_passphrase`: Password to use for repokey or keyfile. Empty if repo is unencrypted. +- `borgmatic_config_name`: Name to use for the borgmatic config file. Defaults to `config.yml` - `borgmatic_large_repo`: Does repo-checking on a weekly basis instead of daily. Good for repos with 100GB+ size. - `borgmatic_failure_command`: Run this command when an error occurs. E.g. `curl -s -F "token=xxx" -F "user=xxx" -F "message=Error during backup" https://api.pushover.net/1/messages.json` - `borgmatic_before_backup_command`: Run this command before the backup. E.g. `dump-a-database /to/file.sql` @@ -58,10 +59,21 @@ An Ansible Role that sets up automated remote backups on the target machine. Use ``` ## Planned features -- [ ] Testing via vagrant + +- [x] Testing - [ ] Multiple repos in one role-call instead of callng this role multiple times. - [ ] Support more OSs, like Red Hat/Fedora/CentOS, SuSE, Gentoo, Slackware, Arch, BSD + +## Contributing + +Pull requests (PR) are welcome, as long as they add features that are relevant for a meaningful number of users. All PRs are tested for style and functionality. To run tests locally (needs Docker): + +``` +$ pip install -r requirements-dev.txt +$ molecule test +``` + ## License MIT/BSD diff --git a/defaults/main.yml b/defaults/main.yml index c13fb28..e64c680 100755 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,7 @@ --- borg_encryption_passphrase: '' borg_exclude_patterns: [] +borgmatic_config_name: config.yaml borgmatic_large_repo: false borgmatic_failure_command: - echo "`date` - Error while creating a backup." diff --git a/molecule/default/Dockerfile.j2 b/molecule/default/Dockerfile.j2 new file mode 100644 index 0000000..0de39e6 --- /dev/null +++ b/molecule/default/Dockerfile.j2 @@ -0,0 +1,22 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +{% if item.env is defined %} +{% for var, value in item.env.items() %} +{% if value %} +ENV {{ var }} {{ value }} +{% endif %} +{% endfor %} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates iproute2 && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash iproute && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash iproute && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi diff --git a/molecule/default/INSTALL.rst b/molecule/default/INSTALL.rst new file mode 100644 index 0000000..6a44bde --- /dev/null +++ b/molecule/default/INSTALL.rst @@ -0,0 +1,22 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* Docker Engine + +Install +======= + +Please refer to the `Virtual environment`_ documentation for installation best +practices. If not using a virtual environment, please consider passing the +widely recommended `'--user' flag`_ when invoking ``pip``. + +.. _Virtual environment: https://virtualenv.pypa.io/en/latest/ +.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site + +.. code-block:: bash + + $ pip install 'molecule[docker]' diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml new file mode 100644 index 0000000..5012ed2 --- /dev/null +++ b/molecule/default/molecule.yml @@ -0,0 +1,26 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: centos-7 + image: centos:7 + # - name: centos-latest + # image: centos:latest + - name: debian-oldstable + image: debian:oldstable + - name: debian-stable + image: debian:stable + # - name: ubuntu-latest + # image: ubuntu:latest +provisioner: + name: ansible + lint: + name: ansible-lint +verifier: + name: testinfra + lint: + name: flake8 diff --git a/molecule/default/playbook.yml b/molecule/default/playbook.yml new file mode 100644 index 0000000..525a2e2 --- /dev/null +++ b/molecule/default/playbook.yml @@ -0,0 +1,22 @@ +--- +- name: Converge + hosts: all + pre_tasks: + - name: Install openssh + package: + name: openssh-server + state: present + roles: + - role: ansible-role-borgbackup + borg_encryption_passphrase: CHANGEME + borg_repository: m5vz9gp4@m5vz9gp4.repo.borgbase.com:repo + borg_source_directories: + - /srv/www + - /var/lib/automysqlbackup + borg_exclude_patterns: + - /srv/www/old-sites + borg_retention_policy: + keep_hourly: 3 + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 6 diff --git a/molecule/default/tests/test_default.py b/molecule/default/tests/test_default.py new file mode 100644 index 0000000..7e077e6 --- /dev/null +++ b/molecule/default/tests/test_default.py @@ -0,0 +1,33 @@ +""" +Validate host properties using Pytest after Ansible is finished. Uses Testinfra + +Possible tests: +- https://testinfra.readthedocs.io/en/latest/modules.html#host +""" + +import os +# import pytest +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE'] +).get_hosts('all') + + +def test_borgmatic_config(host): + f = host.file('/etc/borgmatic/config.yaml') + + assert f.exists + assert f.user == 'root' + assert f.group == 'root' + + +# @pytest.mark.parametrize('file, content', [ +# ("/etc/firewalld/zones/public.xml", ""), +# ("/var/www/html/index.html", "Managed by Ansible") +# ]) +# def test_files(host, file, content): +# file = host.file(file) + +# assert file.exists +# assert file.contains(content) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..74ecc22 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +molecule[lint,docker] +ansible-lint +testinfra \ No newline at end of file diff --git a/tasks/main.yml b/tasks/main.yml index 559b545..5fe512d 100755 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,6 +1,9 @@ --- -- include_vars: '{{ ansible_pkg_mgr }}.yml' -- include: "{{ ansible_pkg_mgr }}.yml" +- name: Include OS-specific variables + include_vars: '{{ ansible_pkg_mgr }}.yml' + +- name: Run OS-specific setup + include: "{{ ansible_pkg_mgr }}.yml" - name: Install required System Packages package: @@ -9,6 +12,7 @@ with_items: "{{ borg_packages }}" - name: Update setuptools if needed + tags: skip_ansible_lint pip: name: setuptools state: latest @@ -16,9 +20,8 @@ - name: Install required Python Packages pip: - name: "{{ item }}" + name: "{{ borg_python_packages }}" executable: "{{ pip_bin }}" - with_items: "{{ borg_python_packages }}" - name: Ensure root has SSH key. user: @@ -30,11 +33,11 @@ - debug: var: root_user['ssh_public_key'] - + - name: Create new repository for server borgbase: repository_name: "{{ bb_repo_name }}" - token: "{{ bb_token}}" + token: "{{ bb_token }}" new_ssh_key: "{{ bb_new_sshkey }}" ssh_key: "{{ bb_sshkey }}" append_only: "{{ bb_append }}" @@ -46,15 +49,15 @@ become: no register: repo_creation when: create_repo - + - name: Set Repository Fact set_fact: borg_repository: "{{ repo_creation['data']['repoPath'] }}" when: create_repo -- name: Ensures /etc/borgmatic.d exists +- name: Ensures /etc/borgmatic exists file: - path: /etc/borgmatic.d + path: /etc/borgmatic state: directory mode: 0700 owner: root @@ -62,42 +65,44 @@ - name: Add Borgmatic Configuration template: src: config.yaml.j2 - dest: "/etc/borgmatic.d/{{ item | regex_replace('\\/$', '') | basename }}.yaml" + dest: "/etc/borgmatic/{{ borgmatic_config_name }}" mode: 0600 - with_items: "{{ borg_source_directories }}" - -- debug: msg="/etc/borgmatic.d/{{ item | regex_replace('\\/$', '') | basename }}.yaml" - with_items: "{{ borg_source_directories }}" - name: Add cron-job for borgmatic (large repo, create and check separate) + tags: molecule-idempotence-notest block: - - cron: + - name: Add cron job for regular create and prune + cron: name: "borgmatic" hour: "{{ 6 |random }}" minute: "{{ 59 |random }}" user: "root" cron_file: borgmatic - job: "/usr/local/bin/borgmatic --create --prune" - - cron: + job: "/usr/local/bin/borgmatic -c /etc/borgmatic/{{ borgmatic_config_name }} --create --prune" + - name: Add cron job for infrequent checks + cron: name: "borgmatic-check" day: "{{ 28 | random }}" hour: "{{ range(7, 24) | random }}" minute: "{{ 59 | random }}" user: "root" cron_file: borgmatic - job: "/usr/local/bin/borgmatic --check" + job: "/usr/local/bin/borgmatic -c /etc/borgmatic/{{ borgmatic_config_name }}" when: borgmatic_large_repo - name: Add cron-job for borgmatic (normal-sized repo) + tags: molecule-idempotence-notest block: - - cron: + - name: Add cron job for create, check and prune + cron: name: "borgmatic" hour: "{{ 6 | random }}" minute: "{{ 59 | random }}" user: "root" cron_file: borgmatic job: "/usr/local/bin/borgmatic" - - cron: + - name: Ensure separate check cron job is absent + cron: name: "borgmatic-check" state: absent when: not borgmatic_large_repo diff --git a/tasks/yum.yml b/tasks/yum.yml index 3ee3f16..b02d132 100644 --- a/tasks/yum.yml +++ b/tasks/yum.yml @@ -1,6 +1,6 @@ --- - name: Install EPEL repo - yum: + yum: pkg: epel-release state: installed update_cache: yes \ No newline at end of file diff --git a/templates/config.yaml.j2 b/templates/config.yaml.j2 index c1e471c..3eead3d 100644 --- a/templates/config.yaml.j2 +++ b/templates/config.yaml.j2 @@ -1,8 +1,9 @@ # Full config: https://gist.github.com/coaxial/46e36d89d7b81887f7275d587fe04c44 -{% set archive_prefix = item | regex_replace("\\/$", "") | basename %} location: source_directories: - - {{ item }} +{% for dir in borg_source_directories %} + - {{ dir }} +{% endfor %} # Stay in same file system (do not cross mount points). one_file_system: {{ borg_one_file_system }} @@ -78,7 +79,7 @@ storage: # also specify a prefix in the retention section to avoid accidental pruning of # archives with a different archive name format. And you should also specify a # prefix in the consistency section as well. - archive_name_format: '{{ archive_prefix }}-{now}' + archive_name_format: '{hostname}-{now}' # Retention policy for how many backups to keep in each category. See # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. @@ -122,7 +123,7 @@ retention: # When pruning, only consider archive names starting with this prefix. # Borg placeholders can be used. See the output of "borg help placeholders" for # details. Default is "{hostname}-". - prefix: '{{ archive_prefix }}-' + prefix: '{hostname}-' # Consistency checks to run after backups. See # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and @@ -143,7 +144,7 @@ consistency: # When performing the "archives" check, only consider archive names starting with # this prefix. Borg placeholders can be used. See the output of # "borg help placeholders" for details. Default is "{hostname}-". - prefix: '{{ archive_prefix }}-' + prefix: '{hostname}-' # Shell commands or scripts to execute before and after a backup or if an error has occurred. # IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic. diff --git a/tests/inventory b/tests/inventory deleted file mode 100644 index 1eb4f3e..0000000 --- a/tests/inventory +++ /dev/null @@ -1,2 +0,0 @@ -[gce] -debian9 diff --git a/tests/playbook.yml b/tests/playbook.yml deleted file mode 100644 index 7100a70..0000000 --- a/tests/playbook.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- hosts: all - become: yes - roles: - - role: borgbackup - borg_encryption_passphrase: CHANGEME - borg_repository: m5vz9gp4@m5vz9gp4.repo.borgbase.com:repo - borg_source_directories: - - /srv/www - - /var/lib/automysqlbackup - borg_exclude_patterns: - - /srv/www/upload \ No newline at end of file diff --git a/vars/apt.yml b/vars/apt.yml index c6c4176..af83fac 100644 --- a/vars/apt.yml +++ b/vars/apt.yml @@ -4,9 +4,12 @@ borg_packages: - libacl1-dev - libacl1 - build-essential + - python-setuptools - python3-dev - python3-pip - python3-msgpack + - openssh-client + - cron python_bin: python3 pip_bin: pip3 diff --git a/vars/yum.yml b/vars/yum.yml index 93171cc..461aeac 100644 --- a/vars/yum.yml +++ b/vars/yum.yml @@ -8,6 +8,9 @@ borg_packages: - python36-pip - python36-wheel - python36-devel + - python-setuptools + - openssh-clients + - cronie python_bin: python3 pip_bin: pip3