From 631b8172073b2fb8de13d3a87123ba979e4b6310 Mon Sep 17 00:00:00 2001 From: Obsidian-MBPM4 Date: Tue, 31 Mar 2026 01:15:14 +0200 Subject: [PATCH] vault backup: 2026-03-31 01:15:14 Affected files: .obsidian/workspace.json 2 Personal/Home Lab/NAS/homelab_backup_architecture_first_draft.md --- .obsidian/workspace.json | 8 +- ...homelab_backup_architecture_first_draft.md | 665 ++++++++++++++++++ 2 files changed, 669 insertions(+), 4 deletions(-) create mode 100644 2 Personal/Home Lab/NAS/homelab_backup_architecture_first_draft.md diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index a436324..c1937f9 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -97,12 +97,12 @@ "state": { "type": "markdown", "state": { - "file": "2 Personal/Home Lab/NAS/Virtual Machine Hosting.md", + "file": "2 Personal/Home Lab/NAS/homelab_backup_architecture_first_draft.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "Virtual Machine Hosting" + "title": "homelab_backup_architecture_first_draft" } }, { @@ -491,6 +491,8 @@ }, "active": "f33efed5601c1085", "lastOpenFiles": [ + "2 Personal/Home Lab/NAS/Virtual Machine Hosting.md", + "2 Personal/Home Lab/NAS/homelab_backup_architecture_first_draft.md", "2 Personal/Home Lab/NAS/Maintenance Plan.md", "2 Personal/Home Lab/NAS/Photo Apps.md", "2 Personal/Home Lab/MAC/Software Management on MacOS.md", @@ -517,8 +519,6 @@ "0 Journal/0 Daily/2026-02-04.md", "99 Work/Jobhunt/Interview Questions.md", "Temporary/My Health Products.md", - "0 Journal/0 Daily/2026-01-28.md", - "2 Personal/Lists/Packlisten/Packliste - Skitour.md", "Attachments/Pasted image 20260121121234.png", "Attachments/ESPSomfyRTS 2026-01-18T16_26_16.backup", "Attachments/Pasted image 20260118150817.png", diff --git a/2 Personal/Home Lab/NAS/homelab_backup_architecture_first_draft.md b/2 Personal/Home Lab/NAS/homelab_backup_architecture_first_draft.md new file mode 100644 index 0000000..4c1a1d6 --- /dev/null +++ b/2 Personal/Home Lab/NAS/homelab_backup_architecture_first_draft.md @@ -0,0 +1,665 @@ +# Home Lab Backup Architecture - First Draft + +## Executive summary + +You do **not** have one backup problem. You have **five** different ones: + +1. **Primary shared data** on the Synology NAS +2. **Virtualized workloads** on Proxmox +3. **Standalone Linux servers** (VPS + Debian server) +4. **User endpoints** (Mac, Linux laptop, Windows PC) +5. **Mobile devices** (iPhone, Android) + +Treating all of them with one tool will make the whole system worse. + +My recommendation is a **layered architecture**: + +- **Synology NAS** = primary durable data store for family files/photos and backup landing zone +- **Local snapshots** = fast recovery from accidental deletion and ransomware-style mistakes +- **Proxmox Backup Server (PBS)** = proper backup system for VMs/LXCs +- **Offsite encrypted backup** = file/server copies to a remote target +- **Scheduled restore testing** = mandatory, not optional +- **Separate production and staging** = from day one + +--- + +## The main design decisions + +### 1) Snapshots are not backups + +Snapshots are for: +- accidental deletion +- "I changed something yesterday and want it back" +- quick rollback + +Backups are for: +- NAS failure +- host failure +- site disaster +- account compromise +- major corruption + +So the rule is: + +- **Use snapshots for short-term recovery** +- **Use true backups for 3-2-1** + +--- + +### 2) Do not store databases on a network share + +For **Nextcloud**, **Immich**, and **Authentik**: + +- Keep **PostgreSQL/MariaDB/Redis** on **local SSD storage** of the VM/LXC +- Keep only the **large user data / media blobs** on NAS-backed storage if needed +- Back up databases separately and consistently + +This is the clean split: + +- **App state + DB** -> local VM/LXC disk -> backed up by PBS / app-aware dumps +- **User files / photos** -> NAS share -> snapshot + offsite backup + +This reduces fragility a lot. + +--- + +### 3) Production and staging must be separated structurally + +Do not make staging just "another docker compose file on the same data". + +Recommended separation: + +- different subdomains +- different VM/LXC instances +- different databases +- different storage paths/shares +- different backup namespaces / repositories +- shorter retention on staging +- no staging write access to production data + +Best case: +- `prod` and `staging` live on separate Proxmox VMs/CTs +- staging can read from **sanitized copies** or test data only + +--- + +## Recommended target architecture + +```mermaid +flowchart TB + Users[Users
You / spouse / family] --> NC[Nextcloud on Proxmox
prod] + Users --> IM[Immich on Proxmox
prod] + Users --> AK[Authentik on Proxmox
prod] + + Laptops[Mac / Linux / Windows laptops] --> EPB[Endpoint backup services] + Phones[iPhone / Android] --> IM + Phones --> NC + + subgraph PVE[Proxmox cluster / host] + NC + IM + AK + STG[Staging VMs/LXCs] + PBS1[Local Proxmox Backup Server] + end + + subgraph NAS[Synology NAS] + SH1[Family files share] + SH2[Photos/media share] + SH3[App data shares] + SNAP[Synology snapshots] + ABB[Active Backup for Business / Time Machine targets] + HB[Hyper Backup jobs] + end + + NC --> SH1 + IM --> SH2 + NC --> SH3 + + SH1 --> SNAP + SH2 --> SNAP + SH3 --> SNAP + + PVE --> PBS1 + PBS1 --> OFFPBS[Offsite PBS
preferred for Proxmox backups] + + HB --> OFFSITE[Hetzner Storage Box or object storage] + + VPS[VPS services] --> RESTIC[Restic/Kopia backups] + Debian[Small Debian server] --> RESTIC + RESTIC --> OFFSITE + + EPB --> ABB + ABB --> SNAP + ABB --> HB +``` + +--- + +## What I would recommend for each layer + +## A. Synology NAS shared folders + +### Role +The NAS should be the **primary storage anchor** for user files, family data, and large photo/media payloads. + +### Local protection +Use: +- **Btrfs shared-folder snapshots** via Snapshot Replication +- **Recycle Bin** for user-facing convenience + +### Suggested snapshot policy +For important family shares: +- every **hour** for 48 hours +- every **day** for 30 days +- every **week** for 8 weeks +- every **month** for 6-12 months + +This gives you fast rollback without huge operational complexity. + +### Offsite protection +Use **Hyper Backup** from Synology to one of these: + +#### Option A - Hetzner Storage Box +**Pros** +- cheap +- European provider +- supports SFTP/rsync/Borg/WebDAV/SMB +- has its own snapshots + +**Cons** +- less elegant than S3-style cloud for some tools +- not object storage +- weaker ecosystem for immutability-style backup controls + +#### Option B - Backblaze B2 / S3-compatible object storage +**Pros** +- easy Hyper Backup integration +- clean cloud backup flow +- common backend for many tools + +**Cons** +- ongoing storage/egress economics need checking for your volume +- less aligned with your "Hetzner-like box" idea + +### My call +For your case: +- **Synology NAS -> Hyper Backup -> Hetzner Storage Box** is reasonable for NAS data +- if later you want stronger cloud-native backup properties, move NAS offsite backup to **B2/S3** + +--- + +## B. Proxmox VMs and containers + +### Do not use generic file copies for this +Use **Proxmox Backup Server**. + +### Why +PBS is the right tool for: +- deduplicated Proxmox backups +- restore of full VMs/CTs +- verification jobs +- prune/retention jobs +- remote sync between PBS instances + +### Best local deployment +Preferred order: + +1. **Dedicated small physical box for PBS** +2. **Your small Debian server repurposed for PBS**, if it can be dedicated +3. PBS in a VM only if you must + +The closer PBS is to "external to the main Proxmox failure domain", the better. + +### Offsite strategy for Proxmox +This is the important part: + +- If you want a **clean Proxmox-native offsite design**, use a **second PBS offsite** +- A plain **Hetzner Storage Box is good for files**, but it is **not the clean native target** for PBS remote sync + +### My call +For Proxmox workloads: +- **Local PBS on separate hardware (or your Debian box if dedicated)** +- **Offsite PBS in Hetzner / other remote host** for synced copies + +That is the most robust design in your whole setup. + +--- + +## C. VPS and the extra Debian server + +These are not Proxmox guests, so PBS is not the natural universal answer. + +Use: +- **restic** if you want simplicity, reliability, scripting, and wide backend support +- **Kopia** if you want a better GUI and endpoint UX + +### What to back up on each server +For each service: +- config files +- compose files / manifests +- secrets management exports where appropriate +- database dumps or consistent DB snapshots +- application data directories +- a machine bootstrap script or IaC definition + +### Recommendation +- **restic** for VPS and Linux servers is the safer default +- send encrypted backups to **Hetzner Storage Box** or **S3/B2** + +Why restic over trying to force one mega-tool? +- fewer moving parts +- easy cron/systemd timers +- easy restore scripting +- good fit for Linux servers and VPSs + +--- + +## D. Nextcloud, Immich, Authentik + +## 1. Nextcloud + +### Recommended layout +- app + DB on local VM/LXC storage +- file data on NAS-backed storage if needed +- back up the DB, config, and data separately + +### Identity +Use **Authentik via OIDC** if you want central auth. + +### Important note +Nextcloud is your **file platform**, not your only backup platform. +Do not confuse sync/share with backup. + +--- + +## 2. Immich + +### Recommended layout +- app + PostgreSQL on local VM/LXC storage +- `UPLOAD_LOCATION` media on NAS-backed storage +- use Immich's built-in **database backup job**, but also back up the underlying media storage + +### Identity +Immich supports **OIDC**, so Authentik can be used here too. + +### Important note +Immich's own docs explicitly warn that database backup alone is not enough; you must also back up the uploaded photos/videos. + +--- + +## 3. Authentik + +### Recommended layout +- keep Authentik small and boring +- PostgreSQL local +- media/config backed up +- include exported configuration / recovery procedure in runbook + +### Important note +Authentik removed its older integrated backup functionality, so you should treat it like a normal app stack: back up the DB and any relevant mounted state. + +--- + +## E. Private computers and family computers + +This is where many homelabs become annoying. + +You need something that: +- users do not hate +- works automatically +- restores cleanly +- you can operate without constant support calls + +### My recommendation is **not** one single tool for everything + +## For Macs +Use **Time Machine to Synology**. + +Why: +- native Mac UX +- low friction +- non-technical users understand it better +- good restore story for Mac users + +## For Windows PCs +Use **Synology Active Backup for Business**. + +Why: +- centralized management +- good Windows support +- easier for family members than a DIY CLI backup setup + +## For your Linux laptop +Use one of: +- **Kopia** if you want a GUI and easy restores +- **restic** if you want maximum control + +### Why I am not pushing UrBackup as the main answer +UrBackup is interesting and easy to set up, but it is weaker as a universal answer here because: +- Windows is its strongest path +- Mac is not really the center of gravity +- it adds another platform you must trust and operate + +For your environment, Synology-native endpoint backup plus Linux-specific tooling is the cleaner compromise. + +--- + +## F. iPhone and Android + +Here is the blunt answer: + +A clean, self-hosted, cross-platform, full-device backup experience for iPhone + Android + easy restore is **not realistically where you want to fight this battle**. + +### What I recommend instead + +#### Photos/videos +- **Immich mobile backup** for photos/videos + +#### Files/documents +- **Nextcloud mobile app** for file access and uploads + +#### Full device state restore +- **iCloud backup** for iPhones +- **Google/Android device backup** for Android + +This is one place where the practical answer beats the ideological self-hosting answer. + +--- + +## 3-2-1 mapping for your setup + +## Production family data (files/photos) + +### Copy 1 - primary +- Synology shared folders + +### Copy 2 - local secondary +- Synology snapshots / recycle bin / local backup target where appropriate + +### Copy 3 - offsite +- Hyper Backup to Hetzner Storage Box or S3/B2 + +--- + +## Proxmox workloads + +### Copy 1 - primary +- running VMs/CTs on Proxmox + +### Copy 2 - local backup +- local PBS datastore + +### Copy 3 - offsite +- remote PBS sync target + +--- + +## VPS / Debian server + +### Copy 1 - primary +- live server + +### Copy 2 - local or local-ish backup cache/repository if desired +- optional local repository + +### Copy 3 - offsite +- restic/Kopia to Hetzner Storage Box or object storage + +--- + +## Retention suggestions + +## Snapshots +- hourly x 48 +- daily x 30 +- weekly x 8 +- monthly x 6-12 + +## Proxmox backups +- nightly +- keep 7 daily +- keep 4 weekly +- keep 6 monthly +- keep 1-2 yearly for critical systems + +## Server file backups +- daily +- same retention as above unless data changes very fast + +## Staging +- keep short +- e.g. 3 daily, 2 weekly + +--- + +## Restore testing - mandatory design + +This is the part most people skip. + +You explicitly said you want testing. Good. Keep it formal. + +## Level 1 - automated integrity checks + +### PBS +- run **verify jobs** +- run **prune / GC** on schedule +- alert on failure + +### Restic / Kopia +- run backup check / metadata verification +- restore a few files automatically to a temp path +- verify checksums + +--- + +## Level 2 - scheduled app restore tests + +Monthly or quarterly: + +- restore latest VPS backup into an isolated test VM/network +- boot it +- run a smoke test script: + - service starts + - ports respond + - DB reachable + - app login page loads + - sample user file exists + +Do the same for: +- Nextcloud test restore +- Immich test restore +- Authentik test restore + +--- + +## Level 3 - disaster recovery rehearsal + +Quarterly or every 6 months: + +- simulate total loss of one service +- restore from documented procedure only +- measure: + - RTO: how long to restore + - RPO: how much data lost + - missing secrets / hidden manual steps + +That is how you find the lies in your own system. + +--- + +## Suggested restore-test automation pattern + +### For each service, create: +- backup source definition +- restore script +- smoke-test script +- teardown script + +Example flow: + +1. create isolated VM/LXC on test VLAN +2. pull latest backup +3. restore service +4. run health checks +5. record success/failure and timing +6. destroy test instance + +This can start simple with shell scripts and later become CI-driven. + +--- + +## Recommended first-version architecture + +If I had to choose a practical v1 for you now, I would do this: + +### Storage and data +- Synology NAS as primary family data store +- Btrfs snapshots on all critical shares +- Recycle Bin on user-facing shares +- Hyper Backup nightly to **Hetzner Storage Box** + +### Compute and apps +- Proxmox hosts production VMs/LXCs +- Nextcloud, Immich, Authentik each isolated in their own VM/LXC or at least separate stacks +- DBs local to compute +- large file/media data on NAS share if required + +### Proxmox backup +- deploy **PBS locally** on separate hardware if possible +- nightly backups of all prod VMs/CTs +- weekly verify jobs +- remote sync to **offsite PBS** + +### Other Linux systems +- restic to offsite storage +- documented DB dumps + config backups + +### Endpoints +- Mac -> Time Machine to Synology +- Windows -> Active Backup for Business +- Linux laptop -> Kopia or restic to NAS/offsite + +### Phones +- Immich for photos/videos +- Nextcloud for documents/files +- iCloud / Google backup for full-device state + +### Environments +- production and staging separated by VM/LXC, storage path, DNS, and credentials + +--- + +## Where this architecture is strong + +- low conceptual confusion +- good restore paths +- good separation of snapshots vs backups +- avoids forcing one backup tool onto every problem +- realistic for family usage +- expandable as you add more self-hosted services + +--- + +## Where it is still weak / uncertain + +The answer is uncertain because I do not know: +- your exact Synology model and whether all required packages/features are supported well +- whether your NAS volumes are Btrfs +- whether you can dedicate hardware to PBS +- your uplink bandwidth and expected offsite backup volume +- whether your family really needs full endpoint bare-metal restore, or mostly file-level recovery +- how much operational complexity you personally are willing to own long-term + +These details can change the final recommendation. + +--- + +## Biggest architectural traps to avoid + +1. **Putting all backups on the same NAS and calling it 3-2-1** +2. **Using snapshots as your only backup** +3. **Mounting NAS storage into apps without thinking about DB consistency** +4. **Using one universal backup tool for everything** +5. **No restore testing** +6. **Letting staging touch production data** +7. **Running your only PBS inside the same Proxmox failure domain without another copy** + +--- + +## My opinionated recommendation + +If you want the most solid path with a sane amount of complexity: + +- **Synology snapshots + Hyper Backup** for NAS data +- **PBS + offsite PBS** for Proxmox +- **restic** for VPS and Linux servers +- **Time Machine / ABB / Kopia** for endpoints depending on OS +- **Immich + Nextcloud + Authentik** with local DBs and NAS-backed large-data storage +- **formal restore tests** every month/quarter + +That is the first design I would trust with family data. + +--- + +## Good next steps + +1. Inventory every system and classify data into: + - critical + - important + - replaceable +2. Decide where PBS will run +3. Decide whether offsite for NAS data is **Hetzner Storage Box** or **B2/S3** +4. Separate prod and staging naming, storage paths, and credentials +5. Implement backup in this order: + - Synology snapshots + - PBS local backups + - offsite backups + - endpoint backups + - restore tests +6. Write one restore runbook per service + +--- + +## Sources / references + +- Proxmox Backup Server docs: https://pbs.proxmox.com/docs/ +- Proxmox PBS installation / Debian / Proxmox VE integration: https://www.proxmox.com/en/products/proxmox-backup-server/get-started +- Proxmox PBS sync jobs: https://pbs.proxmox.com/docs/managing-remotes.html +- Proxmox PBS maintenance tasks: https://pbs.proxmox.com/docs/maintenance.html +- Proxmox VE vzdump docs: https://pve.proxmox.com/pve-docs/chapter-vzdump.html +- Synology Snapshot Replication: https://kb.synology.com/en-global/DSM/help/SnapshotReplication/snapshots?version=7 +- Synology Snapshot Replication technical specs: https://www.synology.com/dsm/7.9/software_spec/snapshot_replication +- Synology snapshot recovery: https://kb.synology.com/DSM/tutorial/How_can_I_recover_files_from_snapshots +- Synology Recycle Bin restore: https://kb.synology.com/en-ro/DSM/tutorial/How_do_I_restore_files_deleted_from_Synology_NAS +- Synology Hyper Backup overview: https://kb.synology.com/en-global/DSM/help/HyperBackup/BackupApp_desc?version=7 +- Synology Hyper Backup feature page: https://www.synology.com/en-global/dsm/feature/hyper_backup +- Synology Hyper Backup Explorer: https://kb.synology.com/en-global/DSM/help/HyperBackupExplorer/hyperbackupexplorer?version=7 +- Hetzner Storage Box protocols: https://docs.hetzner.com/storage/storage-box/general/ +- Hetzner Storage Box SSH/rsync/Borg: https://docs.hetzner.com/storage/storage-box/access/access-ssh-rsync-borg/ +- Hetzner Storage Box overview: https://www.hetzner.com/storage/storage-box/ +- Restic docs: https://restic.readthedocs.io/en/latest/ +- Kopia docs/features: https://kopia.io/docs/features/ +- Kopia homepage: https://kopia.io/ +- UrBackup features: https://www.urbackup.org/features.html +- UrBackup admin manual: https://www.urbackup.org/administration_manual.html +- Synology Active Backup for Business PC/Mac: https://www.synology.com/en-global/dsm/feature/active-backup-business/pc +- Synology ABB Mac page: https://kb.synology.com/en-global/DSM/help/ActiveBackup/activebackup_business_pc_mac?version=7 +- Synology ABB requirements/limitations: https://kb.synology.com/en-af/DSM/help/ActiveBackup/activebackup_business_requireandlimit +- Synology Time Machine guide: https://kb.synology.com/en-us/DSM/tutorial/How_to_back_up_files_from_Mac_to_Synology_NAS_with_Time_Machine +- Windows File History: https://support.microsoft.com/en-us/windows/backup-and-restore-with-file-history-7bf065bf-f1ea-0a78-c1cf-7dcf51cc8bfc +- Nextcloud backup docs: https://docs.nextcloud.com/server/32/admin_manual/maintenance/backup.html +- Nextcloud desktop/mobile sync guidance: https://docs.nextcloud.com/server/latest/user_manual/en/files/desktop_mobile_sync.html +- Nextcloud WebDAV guidance: https://docs.nextcloud.com/server/latest/user_manual/en/files/access_webdav.html +- Nextcloud public shares: https://docs.nextcloud.com/server/latest/user_manual/en/files/sharing.html +- Nextcloud OIDC / auth docs: https://docs.nextcloud.com/server/stable/admin_manual/configuration_user/index.html +- Immich backup/restore: https://docs.immich.app/administration/backup-and-restore +- Immich quick start: https://docs.immich.app/overview/quick-start +- Immich custom file locations: https://docs.immich.app/guides/custom-locations +- Immich OAuth/OIDC: https://docs.immich.app/administration/oauth/ +- Authentik backup/restore: https://docs.goauthentik.io/sys-mgmt/ops/backup-restore/ +- Authentik release note about removal of integrated backups: https://docs.goauthentik.io/releases/2022.2/ +- Apple iPhone backup: https://support.apple.com/guide/iphone/back-up-iphone-iph3ecf67d29/ios +- Apple iPhone restore from backup: https://support.apple.com/en-us/118105 +- Android backup/restore: https://support.google.com/android/answer/2819582?hl=en +