Fix restic install and add backup role

Co-Authored-By: Claude.ai

17 files changed, 319 insertions, 0 deletions

diff --git a/host_vars/git.home.arpa.yml b/host_vars/git.home.arpa.yml
index bb002c3f..3dad37f5 100644
--- a/host_vars/git.home.arpa.yml
+++ b/host_vars/git.home.arpa.yml
diff --git a/host_vars/git.home.arpa.yml.example b/host_vars/git.home.arpa.yml.example
index 45b037b5..9eb71e29 100644
--- a/host_vars/git.home.arpa.yml.example
+++ b/host_vars/git.home.arpa.yml.example
@@ -18,3 +18,12 @@ cgit_hostname: "git.home.arpa"
 cgit_title: "git.example.com"
 cgit_description: "git repositories"
 cgit_ssh_user: "git"
+
+backup_name: "git"
+backup_paths:
+  - "/srv/git"
+backup_cron_schedule: "0 0 * * 0"
+backup_repo: "s3:http://{{ hostvars['rpi5.local'].network_ipv4_address }}:3900/backups"
+backup_access_key_id: "{{ hostvars['rpi5.local'].garage_access_key_id }}"
+backup_secret_access_key: "{{ hostvars['rpi5.local'].garage_secret_access_key }}"
+backup_password: "changeme"
diff --git a/molecule/git/converge.yml b/molecule/git/converge.yml
index 906de734..e6853c18 100644
--- a/molecule/git/converge.yml
+++ b/molecule/git/converge.yml
@@ -5,5 +5,9 @@
     - git.home.arpa
   vars:
     network_ipv4_address: "{{ ansible_facts['default_ipv4']['address'] }}"
+    backup_repo: "s3:https://example.com/backups"
+    backup_access_key_id: "GK_CHANGEME"
+    backup_secret_access_key: "CHANGEME"
   roles:
     - role: "cgit"
+    - role: "backup"
diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml
new file mode 100644
index 00000000..1983ce2b
--- /dev/null
+++ b/roles/backup/defaults/main.yml
@@ -0,0 +1,14 @@
+---
+backup_name: "backup"
+backup_paths: []
+backup_systemd_schedule: "daily"
+backup_cron_schedule: "0 0 * * *"
+backup_retention_daily: 7
+backup_retention_weekly: 4
+backup_retention_monthly: 6
+backup_repo: ""
+backup_access_key_id: ""
+backup_secret_access_key: ""
+backup_password: ""
+backup_user: "backup"
+backup_group: "backup"
diff --git a/roles/backup/handlers/main.yml b/roles/backup/handlers/main.yml
new file mode 100644
index 00000000..3360d709
--- /dev/null
+++ b/roles/backup/handlers/main.yml
@@ -0,0 +1,5 @@
+---
+- name: "Reload systemd"
+  become: true
+  ansible.builtin.systemd_service:
+    daemon_reload: true
diff --git a/roles/backup/meta/argument_specs.yml b/roles/backup/meta/argument_specs.yml
new file mode 100644
index 00000000..8dd86192
--- /dev/null
+++ b/roles/backup/meta/argument_specs.yml
@@ -0,0 +1,57 @@
+---
+argument_specs:
+  main:
+    options:
+      backup_name:
+        type: "str"
+        description: "Unique name for this backup job — used for systemd unit and script names"
+        default: "backup"
+      backup_paths:
+        type: "list"
+        elements: "str"
+        description: "List of paths to back up"
+        required: true
+      backup_systemd_schedule:
+        type: "str"
+        description: "systemd OnCalendar schedule (e.g. daily, weekly, *-*-* 02:00) — used on systemd hosts"
+        default: "daily"
+      backup_cron_schedule:
+        type: "str"
+        description: "cron schedule (e.g. 0 0 * * *) — used on OpenRC hosts"
+        default: "0 0 * * *"
+      backup_retention_daily:
+        type: "int"
+        description: "Number of daily snapshots to keep"
+        default: 7
+      backup_retention_weekly:
+        type: "int"
+        description: "Number of weekly snapshots to keep"
+        default: 4
+      backup_retention_monthly:
+        type: "int"
+        description: "Number of monthly snapshots to keep"
+        default: 6
+      backup_repo:
+        type: "str"
+        description: "Restic repository URL (e.g. s3:http://machine.local:3900/backups)"
+        required: true
+      backup_access_key_id:
+        type: "str"
+        description: "S3 access key ID"
+        required: true
+      backup_secret_access_key:
+        type: "str"
+        description: "S3 secret access key"
+        required: true
+      backup_password:
+        type: "str"
+        description: "Restic repository encryption password"
+        required: true
+      backup_user:
+        type: "str"
+        description: "System user to run the backup as"
+        default: "backup"
+      backup_group:
+        type: "str"
+        description: "System group for the backup user"
+        default: "backup"
diff --git a/roles/backup/meta/main.yml b/roles/backup/meta/main.yml
new file mode 100644
index 00000000..26db6058
--- /dev/null
+++ b/roles/backup/meta/main.yml
@@ -0,0 +1,22 @@
+---
+dependencies:
+  - role: "bash"
+  - role: "restic"
+galaxy_info:
+  author: "a14m"
+  description: "Backup a directory to S3 using restic"
+  license: "MIT"
+  min_ansible_version: "2.18"
+  platforms:
+    - name: "ArchLinux"
+      versions:
+        - "all"
+    - name: "Ubuntu"
+      versions:
+        - "noble"
+    - name: "Debian"
+      versions:
+        - "bookworm"
+    - name: "Alpine"
+      versions:
+        - "all"
diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml
new file mode 100644
index 00000000..3859a0b2
--- /dev/null
+++ b/roles/backup/tasks/main.yml
@@ -0,0 +1,51 @@
+---
+- name: "Create backup group"
+  become: true
+  ansible.builtin.group:
+    name: "{{ backup_group }}"
+    system: true
+
+- name: "Create backup user"
+  become: true
+  ansible.builtin.user:
+    name: "{{ backup_user }}"
+    group: "{{ backup_group }}"
+    system: true
+    shell: "/sbin/nologin"
+    create_home: false
+
+- name: "Create restic cache directory"
+  become: true
+  ansible.builtin.file:
+    path: "/var/cache/{{ backup_name }}-backup"
+    state: "directory"
+    owner: "{{ backup_user }}"
+    group: "{{ backup_group }}"
+    mode: "0700"
+
+- name: "Create credential store directory"
+  become: true
+  ansible.builtin.file:
+    path: "/etc/credstore"
+    state: "directory"
+    owner: "root"
+    group: "root"
+    mode: "0700"
+
+- name: "Deploy backup credentials"
+  become: true
+  ansible.builtin.template:
+    src: "backup.env.j2"
+    dest: "/etc/credstore/{{ backup_name }}-backup.env"
+    owner: "root"
+    group: "root"
+    mode: "0600"
+  no_log: true
+
+- name: "Schedule backup via systemd"
+  ansible.builtin.include_tasks: "systemd.yml"
+  when: ansible_facts['service_mgr'] == "systemd"
+
+- name: "Schedule backup via cron"
+  ansible.builtin.include_tasks: "openrc.yml"
+  when: ansible_facts['service_mgr'] == "openrc"
diff --git a/roles/backup/tasks/openrc.yml b/roles/backup/tasks/openrc.yml
new file mode 100644
index 00000000..12ea9686
--- /dev/null
+++ b/roles/backup/tasks/openrc.yml
@@ -0,0 +1,27 @@
+---
+- name: "Deploy backup script"
+  become: true
+  ansible.builtin.template:
+    src: "backup-openrc.sh.j2"
+    dest: "/usr/local/bin/{{ backup_name }}-backup"
+    owner: "root"
+    group: "root"
+    mode: "0755"
+
+- name: "Deploy backup crontab entry"
+  become: true
+  ansible.builtin.blockinfile:
+    path: "/etc/crontabs/root"
+    marker: "# {mark} ANSIBLE BACKUP {{ backup_name }} CRON"
+    block: "{{ backup_cron_schedule }} /usr/local/bin/{{ backup_name }}-backup"
+    create: true
+    owner: "root"
+    group: "root"
+    mode: "0600"
+
+- name: "Enable and start crond"
+  become: true
+  ansible.builtin.service:
+    name: "crond"
+    enabled: true
+    state: "started"
diff --git a/roles/backup/tasks/systemd.yml b/roles/backup/tasks/systemd.yml
new file mode 100644
index 00000000..bfa85f56
--- /dev/null
+++ b/roles/backup/tasks/systemd.yml
@@ -0,0 +1,37 @@
+---
+- name: "Deploy backup script"
+  become: true
+  ansible.builtin.template:
+    src: "backup-systemd.sh.j2"
+    dest: "/usr/local/bin/{{ backup_name }}-backup"
+    owner: "root"
+    group: "root"
+    mode: "0755"
+
+- name: "Deploy backup systemd service"
+  become: true
+  ansible.builtin.template:
+    src: "backup.service.j2"
+    dest: "/etc/systemd/system/{{ backup_name }}-backup.service"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  notify: "Reload systemd"
+
+- name: "Deploy backup systemd timer"
+  become: true
+  ansible.builtin.template:
+    src: "backup.timer.j2"
+    dest: "/etc/systemd/system/{{ backup_name }}-backup.timer"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  notify: "Reload systemd"
+
+- name: "Enable and start backup timer"
+  become: true
+  ansible.builtin.systemd_service:
+    name: "{{ backup_name }}-backup.timer"
+    enabled: true
+    state: "started"
+    daemon_reload: true
diff --git a/roles/backup/templates/backup-openrc.sh.j2 b/roles/backup/templates/backup-openrc.sh.j2
new file mode 100644
index 00000000..85dc8d1e
--- /dev/null
+++ b/roles/backup/templates/backup-openrc.sh.j2
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# shellcheck source=/dev/null
+source "/etc/credstore/{{ backup_name }}-backup.env"
+
+export RESTIC_REPOSITORY="${BACKUP_REPO}"
+export RESTIC_PASSWORD="${BACKUP_PASSWORD}"
+export AWS_ACCESS_KEY_ID="${BACKUP_ACCESS_KEY_ID}"
+export AWS_SECRET_ACCESS_KEY="${BACKUP_SECRET_ACCESS_KEY}"
+export CACHE_DIR="/var/cache/{{ backup_name }}-backup"
+
+# -p (--preserve-environment) -s (--shell): busybox su has no long-form flags
+exec su -p -s /bin/sh {{ backup_user }} <<'BACKUP_SU_EOF'
+restic --cache-dir "$CACHE_DIR" snapshots &>/dev/null || restic --cache-dir "$CACHE_DIR" init
+
+restic --cache-dir "$CACHE_DIR" backup {{ backup_paths | map('quote') | join(' ') }}
+
+restic --cache-dir "$CACHE_DIR" forget \
+  --keep-daily {{ backup_retention_daily }} \
+  --keep-weekly {{ backup_retention_weekly }} \
+  --keep-monthly {{ backup_retention_monthly }} \
+  --prune
+BACKUP_SU_EOF
diff --git a/roles/backup/templates/backup-systemd.sh.j2 b/roles/backup/templates/backup-systemd.sh.j2
new file mode 100644
index 00000000..24be22ea
--- /dev/null
+++ b/roles/backup/templates/backup-systemd.sh.j2
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# shellcheck source=/dev/null
+source "${CREDENTIALS_DIRECTORY}/{{ backup_name }}-backup.env"
+
+export RESTIC_REPOSITORY="${BACKUP_REPO}"
+export RESTIC_PASSWORD="${BACKUP_PASSWORD}"
+export AWS_ACCESS_KEY_ID="${BACKUP_ACCESS_KEY_ID}"
+export AWS_SECRET_ACCESS_KEY="${BACKUP_SECRET_ACCESS_KEY}"
+
+CACHE_DIR="/var/cache/{{ backup_name }}-backup"
+
+restic --cache-dir "$CACHE_DIR" snapshots &>/dev/null || restic --cache-dir "$CACHE_DIR" init
+
+restic --cache-dir "$CACHE_DIR" backup {{ backup_paths | map('quote') | join(' ') }}
+
+restic --cache-dir "$CACHE_DIR" forget \
+  --keep-daily {{ backup_retention_daily }} \
+  --keep-weekly {{ backup_retention_weekly }} \
+  --keep-monthly {{ backup_retention_monthly }} \
+  --prune
diff --git a/roles/backup/templates/backup.env.j2 b/roles/backup/templates/backup.env.j2
new file mode 100644
index 00000000..8557501e
--- /dev/null
+++ b/roles/backup/templates/backup.env.j2
@@ -0,0 +1,4 @@
+BACKUP_REPO="{{ backup_repo }}/{{ backup_name }}"
+BACKUP_PASSWORD="{{ backup_password }}"
+BACKUP_ACCESS_KEY_ID="{{ backup_access_key_id }}"
+BACKUP_SECRET_ACCESS_KEY="{{ backup_secret_access_key }}"
diff --git a/roles/backup/templates/backup.service.j2 b/roles/backup/templates/backup.service.j2
new file mode 100644
index 00000000..f35940b8
--- /dev/null
+++ b/roles/backup/templates/backup.service.j2
@@ -0,0 +1,13 @@
+[Unit]
+Description=Backup {{ backup_name }}
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/{{ backup_name }}-backup
+User={{ backup_user }}
+Group={{ backup_group }}
+StandardOutput=journal
+StandardError=journal
+LoadCredential={{ backup_name }}-backup.env:/etc/credstore/{{ backup_name }}-backup.env
diff --git a/roles/backup/templates/backup.timer.j2 b/roles/backup/templates/backup.timer.j2
new file mode 100644
index 00000000..74e5276d
--- /dev/null
+++ b/roles/backup/templates/backup.timer.j2
@@ -0,0 +1,9 @@
+[Unit]
+Description=Backup {{ backup_name }} timer
+
+[Timer]
+OnCalendar={{ backup_systemd_schedule }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/roles/restic/tasks/main.yml b/roles/restic/tasks/main.yml
index ae82d3bc..8640a521 100644
--- a/roles/restic/tasks/main.yml
+++ b/roles/restic/tasks/main.yml
@@ -10,3 +10,23 @@
     cmd: "mise use --global restic@{{ restic_version }}"
   changed_when: true
   when: restic_version not in restic_installed_versions.stdout
+
+# mise installs restic under whichever user ran this task's own home. When
+# that's root (e.g. containers connected to as root), the binary ends up
+# under /root, unreachable by other accounts (ex. backup user)
+# This copies the binary to a system-wide location so that any user can use it.
+- name: "Resolve installed restic path"
+  ansible.builtin.command:
+    cmd: "mise which restic"
+  register: restic_bin_path
+  changed_when: false
+
+- name: "Expose restic system-wide"
+  become: true
+  ansible.builtin.copy:
+    remote_src: true
+    src: "{{ restic_bin_path.stdout }}"
+    dest: "/usr/local/bin/restic"
+    mode: "0755"
+    owner: "root"
+    group: "root"
diff --git a/service.yml b/service.yml
index ee64d629..b26ae731 100644
--- a/service.yml
+++ b/service.yml
@@ -13,3 +13,4 @@
     - git.home.arpa
   roles:
     - role: "cgit"
+    - role: "backup"