Add testing with Molecule/Ansible-lint, undo #9 (#26)

* Add testing with Molecule/Ansible-lint and fix linting errors
* Undo PR #9 for now (sorry)

.travis.yml

@@ -0,0 +1,11 @@
+---
+sudo: required
+language: python
+services:
+  - docker
+before_install:
+  - sudo apt-get -qq update
+install:
+  - pip install molecule[lint,docker]
+script:
+  - molecule test

.yamllint

@@ -0,0 +1,33 @@
+---
+# Based on ansible-lint config
+extends: default
+
+rules:
+  braces:
+    max-spaces-inside: 1
+    level: error
+  brackets:
+    max-spaces-inside: 1
+    level: error
+  colons:
+    max-spaces-after: -1
+    level: error
+  commas:
+    max-spaces-after: -1
+    level: error
+  comments: disable
+  comments-indentation: disable
+  document-start: disable
+  empty-lines:
+    max: 3
+    level: error
+  hyphens:
+    level: error
+  indentation: disable
+  key-duplicates: enable
+  line-length: disable
+  new-line-at-end-of-file: disable
+  new-lines:
+    type: unix
+  trailing-spaces: disable
+  truthy: disable

README.md

@@ -10,6 +10,7 @@ An Ansible Role that sets up automated remote backups on the target machine. Use
 
 ### Optional Arguments
 - `borg_encryption_passphrase`: Password to use for repokey or keyfile. Empty if repo is unencrypted.
+- `borgmatic_config_name`: Name to use for the borgmatic config file. Defaults to `config.yml`
 - `borgmatic_large_repo`: Does repo-checking on a weekly basis instead of daily. Good for repos with 100GB+ size.
 - `borgmatic_failure_command`: Run this command when an error occurs. E.g. `curl -s -F "token=xxx" -F "user=xxx" -F "message=Error during backup" https://api.pushover.net/1/messages.json`
 - `borgmatic_before_backup_command`: Run this command before the backup. E.g. `dump-a-database /to/file.sql`
@@ -58,10 +59,21 @@ An Ansible Role that sets up automated remote backups on the target machine. Use
 ```
 
 ## Planned features
-- [ ] Testing via vagrant
+
+- [x] Testing
 - [ ] Multiple repos in one role-call instead of callng this role multiple times.
 - [ ] Support more OSs, like Red Hat/Fedora/CentOS, SuSE, Gentoo, Slackware, Arch, BSD
 
+
+## Contributing
+
+Pull requests (PR) are welcome, as long as they add features that are relevant for a meaningful number of users. All PRs are tested for style and functionality. To run tests locally (needs Docker):
+
+```
+$ pip install -r requirements-dev.txt
+$ molecule test
+```
+
 ## License
 
 MIT/BSD

defaults/main.yml

@@ -1,6 +1,7 @@
 ---
 borg_encryption_passphrase: ''
 borg_exclude_patterns: []
+borgmatic_config_name: config.yaml
 borgmatic_large_repo: false
 borgmatic_failure_command:
   - echo "`date` - Error while creating a backup."

molecule/default/Dockerfile.j2

@@ -0,0 +1,22 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+{% if item.env is defined %}
+{% for var, value in item.env.items() %}
+{% if value %}
+ENV {{ var }} {{ value }}
+{% endif %}
+{% endfor %}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates iproute2 && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash iproute && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash iproute && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi

molecule/default/INSTALL.rst

@@ -0,0 +1,22 @@
+*******
+Docker driver installation guide
+*******
+
+Requirements
+============
+
+* Docker Engine
+
+Install
+=======
+
+Please refer to the `Virtual environment`_ documentation for installation best
+practices. If not using a virtual environment, please consider passing the
+widely recommended `'--user' flag`_ when invoking ``pip``.
+
+.. _Virtual environment: https://virtualenv.pypa.io/en/latest/
+.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site
+
+.. code-block:: bash
+
+    $ pip install 'molecule[docker]'

molecule/default/molecule.yml

@@ -0,0 +1,26 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: centos-7
+    image: centos:7
+  # - name: centos-latest
+  #   image: centos:latest
+  - name: debian-oldstable
+    image: debian:oldstable
+  - name: debian-stable
+    image: debian:stable
+  # - name: ubuntu-latest
+  #   image: ubuntu:latest
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

molecule/default/playbook.yml

@@ -0,0 +1,22 @@
+---
+- name: Converge
+  hosts: all
+  pre_tasks:
+    - name: Install openssh
+      package:
+        name: openssh-server
+        state: present
+  roles:
+    - role: ansible-role-borgbackup
+      borg_encryption_passphrase: CHANGEME
+      borg_repository: m5vz9gp4@m5vz9gp4.repo.borgbase.com:repo
+      borg_source_directories:
+        - /srv/www
+        - /var/lib/automysqlbackup
+      borg_exclude_patterns:
+        - /srv/www/old-sites
+      borg_retention_policy:
+        keep_hourly: 3
+        keep_daily: 7
+        keep_weekly: 4
+        keep_monthly: 6

molecule/default/tests/test_default.py

@@ -0,0 +1,33 @@
+"""
+Validate host properties using Pytest after Ansible is finished. Uses Testinfra
+
+Possible tests:
+- https://testinfra.readthedocs.io/en/latest/modules.html#host
+"""
+
+import os
+# import pytest
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']
+).get_hosts('all')
+
+
+def test_borgmatic_config(host):
+    f = host.file('/etc/borgmatic/config.yaml')
+
+    assert f.exists
+    assert f.user == 'root'
+    assert f.group == 'root'
+
+
+# @pytest.mark.parametrize('file, content', [
+#   ("/etc/firewalld/zones/public.xml", "<service name=\"http\"/>"),
+#   ("/var/www/html/index.html", "Managed by Ansible")
+# ])
+# def test_files(host, file, content):
+#     file = host.file(file)
+
+#     assert file.exists
+#     assert file.contains(content)

requirements-dev.txt

@@ -0,0 +1,3 @@
+molecule[lint,docker]
+ansible-lint
+testinfra

tasks/main.yml

@@ -1,6 +1,9 @@
 ---
-- include_vars: '{{ ansible_pkg_mgr }}.yml'
+- name: Include OS-specific variables
-- include: "{{ ansible_pkg_mgr }}.yml"
+  include_vars: '{{ ansible_pkg_mgr }}.yml'
+
+- name: Run OS-specific setup
+  include: "{{ ansible_pkg_mgr }}.yml"
 
 - name: Install required System Packages
   package:
@@ -9,6 +12,7 @@
   with_items: "{{ borg_packages }}"
 
 - name: Update setuptools if needed
+  tags: skip_ansible_lint
   pip:
     name: setuptools
     state: latest
@@ -16,9 +20,8 @@
 
 - name: Install required Python Packages
   pip:
-    name: "{{ item }}"
+    name: "{{ borg_python_packages }}"
     executable: "{{ pip_bin }}"
-  with_items: "{{ borg_python_packages }}"
 
 - name: Ensure root has SSH key.
   user:
@@ -34,7 +37,7 @@
 - name: Create new repository for server
   borgbase:
     repository_name: "{{ bb_repo_name }}"
-    token: "{{ bb_token}}"
+    token: "{{ bb_token }}"
     new_ssh_key: "{{ bb_new_sshkey }}"
     ssh_key: "{{ bb_sshkey }}"
     append_only: "{{ bb_append }}"
@@ -52,9 +55,9 @@
     borg_repository: "{{ repo_creation['data']['repoPath'] }}"
   when: create_repo
 
-- name: Ensures /etc/borgmatic.d exists
+- name: Ensures /etc/borgmatic exists
   file:
-    path: /etc/borgmatic.d
+    path: /etc/borgmatic
     state: directory
     mode: 0700
     owner: root
@@ -62,42 +65,44 @@
 - name: Add Borgmatic Configuration
   template:
     src: config.yaml.j2
-    dest: "/etc/borgmatic.d/{{ item | regex_replace('\\/$', '') | basename }}.yaml"
+    dest: "/etc/borgmatic/{{ borgmatic_config_name }}"
     mode: 0600
-  with_items: "{{ borg_source_directories }}"
-
-- debug: msg="/etc/borgmatic.d/{{ item | regex_replace('\\/$', '') | basename }}.yaml"
-  with_items: "{{ borg_source_directories }}"
 
 - name: Add cron-job for borgmatic (large repo, create and check separate)
+  tags: molecule-idempotence-notest
   block:
-    - cron:
+    - name: Add cron job for regular create and prune
+      cron:
         name: "borgmatic"
         hour: "{{ 6 |random }}"
         minute: "{{ 59 |random }}"
         user: "root"
         cron_file: borgmatic
-        job: "/usr/local/bin/borgmatic --create --prune"
+        job: "/usr/local/bin/borgmatic -c /etc/borgmatic/{{ borgmatic_config_name }} --create --prune"
-    - cron:
+    - name: Add cron job for infrequent checks
+      cron:
         name: "borgmatic-check"
         day: "{{ 28 | random }}"
         hour: "{{ range(7, 24) | random }}"
         minute: "{{ 59 | random }}"
         user: "root"
         cron_file: borgmatic
-        job: "/usr/local/bin/borgmatic --check"
+        job: "/usr/local/bin/borgmatic -c /etc/borgmatic/{{ borgmatic_config_name }}"
   when: borgmatic_large_repo
 
 - name: Add cron-job for borgmatic (normal-sized repo)
+  tags: molecule-idempotence-notest
   block:
-    - cron:
+    - name: Add cron job for create, check and prune
+      cron:
         name: "borgmatic"
         hour: "{{ 6 | random }}"
         minute: "{{ 59 | random }}"
         user: "root"
         cron_file: borgmatic
         job: "/usr/local/bin/borgmatic"
-    - cron:
+    - name: Ensure separate check cron job is absent
+      cron:
         name: "borgmatic-check"
         state: absent
   when: not borgmatic_large_repo

templates/config.yaml.j2

@@ -1,8 +1,9 @@
 # Full config: https://gist.github.com/coaxial/46e36d89d7b81887f7275d587fe04c44
-{% set archive_prefix = item | regex_replace("\\/$", "") | basename %}
 location:
     source_directories:
-        - {{ item }}
+{% for dir in borg_source_directories %}
+        - {{ dir }}
+{% endfor %}
 
     # Stay in same file system (do not cross mount points).
     one_file_system: {{ borg_one_file_system }}
@@ -78,7 +79,7 @@ storage:
     # also specify a prefix in the retention section to avoid accidental pruning of
     # archives with a different archive name format. And you should also specify a
     # prefix in the consistency section as well.
-    archive_name_format: '{{ archive_prefix }}-{now}'
+    archive_name_format: '{hostname}-{now}'
 
 # Retention policy for how many backups to keep in each category. See
 # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
@@ -122,7 +123,7 @@ retention:
     # When pruning, only consider archive names starting with this prefix.
     # Borg placeholders can be used. See the output of "borg help placeholders" for
     # details. Default is "{hostname}-".
-    prefix: '{{ archive_prefix }}-'
+    prefix: '{hostname}-'
 
 # Consistency checks to run after backups. See
 # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and
@@ -143,7 +144,7 @@ consistency:
     # When performing the "archives" check, only consider archive names starting with
     # this prefix. Borg placeholders can be used. See the output of
     # "borg help placeholders" for details. Default is "{hostname}-".
-    prefix: '{{ archive_prefix }}-'
+    prefix: '{hostname}-'
 
 # Shell commands or scripts to execute before and after a backup or if an error has occurred.
 # IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.

tests/inventory

@@ -1,2 +0,0 @@
-[gce]
-debian9

tests/playbook.yml

@@ -1,12 +0,0 @@
----
-- hosts: all
-  become: yes
-  roles:
-    - role: borgbackup
-      borg_encryption_passphrase: CHANGEME
-      borg_repository: m5vz9gp4@m5vz9gp4.repo.borgbase.com:repo
-      borg_source_directories:
-        - /srv/www
-        - /var/lib/automysqlbackup
-      borg_exclude_patterns:
-        - /srv/www/upload

vars/apt.yml

@@ -4,9 +4,12 @@ borg_packages:
   - libacl1-dev
   - libacl1
   - build-essential
+  - python-setuptools
   - python3-dev
   - python3-pip
   - python3-msgpack
+  - openssh-client
+  - cron
 
 python_bin: python3
 pip_bin: pip3

vars/yum.yml

@@ -8,6 +8,9 @@ borg_packages:
   - python36-pip
   - python36-wheel
   - python36-devel
+  - python-setuptools
+  - openssh-clients
+  - cronie
 
 python_bin: python3
 pip_bin: pip3