'[sysadmin/kde-ansible] /: Add remaining backup scripts.'

[prev in list] [next in list] [prev in thread] [next in thread] 

List:       kde-commits
Subject:    [sysadmin/kde-ansible] /: Add remaining backup scripts.
From:       Nicolás_Alvarez <null () kde ! org>
Date:       2017-11-01 2:55:54
Message-ID: E1e9jC6-0005sp-OW () code ! kde ! org
[Download RAW message or body]

Git commit f2f058610c25111f314f5ffb57c5218b0f7bd09b by Nicolás Alvarez.
Committed on 01/11/2017 at 02:52.
Pushed by nalvarez into branch 'master'.

Add remaining backup scripts.

Most written by Ben Cooksley and installed directly; I'm just adding them
to the ansible playbooks.

M  +10   -0    backups.yml
A  +2    -0    host_vars/ange.kde.org.yml
A  +2    -0    host_vars/dalca.kde.org.yml
A  +2    -0    host_vars/elder.kde.org.yml
A  +2    -0    host_vars/kater.kde.org.yml
A  +2    -0    host_vars/library.kde.org.yml
A  +3    -0    host_vars/nellie.kde.org.yml
A  +2    -0    host_vars/orbi.kde.org.yml
A  +3    -0    host_vars/pirate.kde.org.yml
A  +3    -0    host_vars/racnoss.kde.org.yml
A  +2    -0    host_vars/rosetta.kde.org.yml
A  +16   -0    roles/kde-backup/templates/backup-ange.sh
A  +16   -0    roles/kde-backup/templates/backup-dalca.sh
A  +11   -0    roles/kde-backup/templates/backup-elder.sh
A  +16   -0    roles/kde-backup/templates/backup-kater.sh
A  +16   -0    roles/kde-backup/templates/backup-library.sh
A  +13   -0    roles/kde-backup/templates/backup-nellie.sh
A  +27   -0    roles/kde-backup/templates/backup-orbi.sh
A  +10   -0    roles/kde-backup/templates/backup-pirate.sh
A  +19   -0    roles/kde-backup/templates/backup-racnoss.sh
A  +25   -0    roles/kde-backup/templates/backup-rosetta.sh

https://commits.kde.org/sysadmin/kde-ansible/f2f058610c25111f314f5ffb57c5218b0f7bd09b

diff --git a/backups.yml b/backups.yml
index ed12584..7cb1b37 100644
--- a/backups.yml
+++ b/backups.yml
@@ -23,6 +23,16 @@
   - saba.kde.org
   - postbox.kde.org
   - identity.kde.org
+  - nellie.kde.org
+  - racnoss.kde.org
+  - elder.kde.org
+  - orbi.kde.org
+  - dalca.kde.org
+  - ange.kde.org
+  - pirate.kde.org
+  - library.kde.org
+  - kater.kde.org
+  - rosetta.kde.org
 
   vars_files:
     - vars/backups.yml
diff --git a/host_vars/ange.kde.org.yml b/host_vars/ange.kde.org.yml
new file mode 100644
index 0000000..70180dd
--- /dev/null
+++ b/host_vars/ange.kde.org.yml
@@ -0,0 +1,2 @@
+---
+backup_directory: backups
diff --git a/host_vars/dalca.kde.org.yml b/host_vars/dalca.kde.org.yml
new file mode 100644
index 0000000..70180dd
--- /dev/null
+++ b/host_vars/dalca.kde.org.yml
@@ -0,0 +1,2 @@
+---
+backup_directory: backups
diff --git a/host_vars/elder.kde.org.yml b/host_vars/elder.kde.org.yml
new file mode 100644
index 0000000..70180dd
--- /dev/null
+++ b/host_vars/elder.kde.org.yml
@@ -0,0 +1,2 @@
+---
+backup_directory: backups
diff --git a/host_vars/kater.kde.org.yml b/host_vars/kater.kde.org.yml
new file mode 100644
index 0000000..70180dd
--- /dev/null
+++ b/host_vars/kater.kde.org.yml
@@ -0,0 +1,2 @@
+---
+backup_directory: backups
diff --git a/host_vars/library.kde.org.yml b/host_vars/library.kde.org.yml
new file mode 100644
index 0000000..70180dd
--- /dev/null
+++ b/host_vars/library.kde.org.yml
@@ -0,0 +1,2 @@
+---
+backup_directory: backups
diff --git a/host_vars/nellie.kde.org.yml b/host_vars/nellie.kde.org.yml
new file mode 100644
index 0000000..568394f
--- /dev/null
+++ b/host_vars/nellie.kde.org.yml
@@ -0,0 +1,3 @@
+---
+backup_directory: backups
+hetzner_backup_host: pirate
diff --git a/host_vars/orbi.kde.org.yml b/host_vars/orbi.kde.org.yml
new file mode 100644
index 0000000..70180dd
--- /dev/null
+++ b/host_vars/orbi.kde.org.yml
@@ -0,0 +1,2 @@
+---
+backup_directory: backups
diff --git a/host_vars/pirate.kde.org.yml b/host_vars/pirate.kde.org.yml
new file mode 100644
index 0000000..568394f
--- /dev/null
+++ b/host_vars/pirate.kde.org.yml
@@ -0,0 +1,3 @@
+---
+backup_directory: backups
+hetzner_backup_host: pirate
diff --git a/host_vars/racnoss.kde.org.yml b/host_vars/racnoss.kde.org.yml
new file mode 100644
index 0000000..568394f
--- /dev/null
+++ b/host_vars/racnoss.kde.org.yml
@@ -0,0 +1,3 @@
+---
+backup_directory: backups
+hetzner_backup_host: pirate
diff --git a/host_vars/rosetta.kde.org.yml b/host_vars/rosetta.kde.org.yml
new file mode 100644
index 0000000..70180dd
--- /dev/null
+++ b/host_vars/rosetta.kde.org.yml
@@ -0,0 +1,2 @@
+---
+backup_directory: backups
diff --git a/roles/kde-backup/templates/backup-ange.sh \
b/roles/kde-backup/templates/backup-ange.sh new file mode 100755
index 0000000..5715efd
--- /dev/null
+++ b/roles/kde-backup/templates/backup-ange.sh
@@ -0,0 +1,16 @@
+LOCATION=/root/{{backup_directory}}
+
+# Cleanup old backups
+find $LOCATION/ -name '*gz' -mtime +7 | xargs rm -f
+
+# Backup system configuration, package listing and cronjobs
+dpkg -l > $LOCATION/dpkg.`date +%w`
+tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/
+tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/
+
+# Secure our backups
+chmod -R 700 $LOCATION
+
+# Transfer them to the backup server
+cd $LOCATION/..
+rsync --timeout=600 --delete -a backups -4 angebackup@gohma.kde.org:
diff --git a/roles/kde-backup/templates/backup-dalca.sh \
b/roles/kde-backup/templates/backup-dalca.sh new file mode 100755
index 0000000..91205a7
--- /dev/null
+++ b/roles/kde-backup/templates/backup-dalca.sh
@@ -0,0 +1,16 @@
+LOCATION=/root/{{backup_directory}}
+
+# Cleanup old backups
+find $LOCATION/ -name '*gz' -mtime +7 | xargs rm -f
+
+# Grab the system configuration, package listing, and cronjobs
+dpkg -l > $LOCATION/dpkg.`date +%w`
+tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/
+tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/
+
+# Secure our backups
+chmod -R 700 $LOCATION
+
+# Transfer them to the backup server
+cd $LOCATION/..
+rsync --timeout=600 --delete -a backups -4 dalcabackup@gohma.kde.org:
diff --git a/roles/kde-backup/templates/backup-elder.sh \
b/roles/kde-backup/templates/backup-elder.sh new file mode 100755
index 0000000..5916068
--- /dev/null
+++ b/roles/kde-backup/templates/backup-elder.sh
@@ -0,0 +1,11 @@
+LOCATION=/root/{{backup_directory}}
+
+# Backup configuration and list of installed packages
+tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/
+tar -czf $LOCATION/cronjobs.`date +%w`.tgz -C / var/spool/cron/
+dpkg -l > $LOCATION/dpkg.`date +%w`
+
+chmod -R 700 $LOCATION
+
+cd $LOCATION/..
+rsync --timeout=600 --delete -a backups -4 elderbackup@gohma.kde.org:
diff --git a/roles/kde-backup/templates/backup-kater.sh \
b/roles/kde-backup/templates/backup-kater.sh new file mode 100755
index 0000000..3aca1b0
--- /dev/null
+++ b/roles/kde-backup/templates/backup-kater.sh
@@ -0,0 +1,16 @@
+LOCATION=/root/{{backup_directory}}
+
+# Cleanup old backups
+find $LOCATION/ -name '*gz' -mtime +7 | xargs rm -f
+
+# Backup system config, package listing and cronjobs
+dpkg -l > $LOCATION/dpkg.`date +%w`
+tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/
+tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/
+
+# Secure our backups
+chmod -R 700 $LOCATION
+
+# Transfer them to the backup server
+cd $LOCATION/..
+rsync --timeout=600 --delete -a backups -4 katerbackup@gohma.kde.org:
diff --git a/roles/kde-backup/templates/backup-library.sh \
b/roles/kde-backup/templates/backup-library.sh new file mode 100755
index 0000000..40a47f0
--- /dev/null
+++ b/roles/kde-backup/templates/backup-library.sh
@@ -0,0 +1,16 @@
+LOCATION=/root/{{backup_directory}}
+
+# Cleanup old backups
+find $LOCATION/ -name '*gz' -mtime +7 | xargs rm -f
+
+# Grab the system configuration, package listing and cronjobs
+dpkg -l > $LOCATION/dpkg.`date +%w`
+tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/
+tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/
+
+# Secure our backups
+chmod -R 700 $LOCATION
+
+# Transfer them to the backup server now
+cd $LOCATION/..
+rsync --timeout=600 --delete -a backups -4 librarybackup@gohma.kde.org:
diff --git a/roles/kde-backup/templates/backup-nellie.sh \
b/roles/kde-backup/templates/backup-nellie.sh new file mode 100755
index 0000000..45b1759
--- /dev/null
+++ b/roles/kde-backup/templates/backup-nellie.sh
@@ -0,0 +1,13 @@
+LOCATION=/root/{{backup_directory}}
+
+# Backup homes
+tar -cJf $LOCATION/homes.`date +%w`.tar.xz -C / --exclude=home/sandbox/data/jobs \
--exclude=home/sandbox/data/cache --exclude=home/sandbox/logs \
--exclude=home/production/data/jobs --exclude=home/production/data/cache \
--exclude=home/production/logs --exclude=home/binaryfactory/data/jobs \
--exclude=home/binaryfactory/data/cache --exclude=home/binaryfactory/logs home/ +
+# Backup configuration and list of installed packages
+tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/
+tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/
+dpkg -l > $LOCATION/dpkg.`date +%w`
+
+# Transfer them to the backup server
+lftp -f ~/bin/backup-options
+
diff --git a/roles/kde-backup/templates/backup-orbi.sh \
b/roles/kde-backup/templates/backup-orbi.sh new file mode 100755
index 0000000..0ce4534
--- /dev/null
+++ b/roles/kde-backup/templates/backup-orbi.sh
@@ -0,0 +1,27 @@
+LOCATION=/root/{{backup_directory}}
+find $LOCATION -name '*xz' -mtime +21 | xargs rm -f
+
+# Backup website files, system configuration and user home directories
+tar -cJf $LOCATION/srv.`date +%w`.tar.xz -C / srv/
+tar -cJf $LOCATION/homes.`date +%w`.tar.xz -C / home/ 2>&1 | grep -v "file changed \
as we read it" +
+# Backup databases
+for DATABASE in `mysqlshow | grep -v "information_schema" | grep -v \
"performance_schema" | grep -v "mysql" | grep -v "sys" | tail -n+4 | awk '{print \
$2}'`; do +    mysqldump --opt --quick --single-transaction --skip-extended-insert  \
--events --create-options --set-charset  $DATABASE > $LOCATION/db.$DATABASE.`date \
+%w`.sql +    xz -f $LOCATION/db.$DATABASE.`date +%w`.sql
+done
+
+# Backup openldap
+/usr/sbin/slapcat -f /etc/ldap/slapd.conf -b dc=kde,dc=org | xz -3 > \
$LOCATION/ldap.`date +%w`.ldif.xz +
+# Grab a list of installed packages, the system config and crontabs
+tar -cJf $LOCATION/etc.`date +%w`.tar.xz -C / etc/
+tar -cJf $LOCATION/crontabs.`date +%w`.tar.xz -C / var/spool/cron/
+dpkg -l > $LOCATION/dpkg.`date +%w`
+
+# Secure our backups
+chmod -R 700 $LOCATION
+
+# Transfer them to safety
+cd $LOCATION/..
+rsync --timeout=600 --delete -a backups -4 orbibackup@gohma.kde.org:
diff --git a/roles/kde-backup/templates/backup-pirate.sh \
b/roles/kde-backup/templates/backup-pirate.sh new file mode 100755
index 0000000..cd23e25
--- /dev/null
+++ b/roles/kde-backup/templates/backup-pirate.sh
@@ -0,0 +1,10 @@
+LOCATION=/root/{{backup_directory}}
+
+# Backup configuration and list of installed packages
+tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/
+tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/
+dpkg -l > $LOCATION/dpkg.`date +%w`
+
+# Transfer to the backup server
+lftp -f ~/bin/backup-options
+
diff --git a/roles/kde-backup/templates/backup-racnoss.sh \
b/roles/kde-backup/templates/backup-racnoss.sh new file mode 100755
index 0000000..3a21048
--- /dev/null
+++ b/roles/kde-backup/templates/backup-racnoss.sh
@@ -0,0 +1,19 @@
+LOCATION=/root/{{backup_directory}}
+find $LOCATION -name "srv.*.tar.xz" -mtime +2 | xargs rm -f
+
+# Backup mirrors listing
+mb -b download export --format=postgresql > $LOCATION/mirrors-download.`date \
+%w`.sql +mb -b files export --format=postgresql > $LOCATION/mirrors-files.`date \
+%w`.sql +
+# Backup home directories
+tar -cJf $LOCATION/homes.`date +%w`.tar.xz -C / --exclude=home/marble/sentinel2 \
--exclude=home/marble/newstuff-tmp --exclude=home/neonarchives/user-moved-2017-04-11 \
--exclude=home/akademy/2017 home/ +
+# Backup the CDN
+tar -cJf $LOCATION/srv-cdn.`date +%w`.tar.xz -C / srv/www/cdn.kde.org/
+
+# Backup configuration and list of installed packages
+tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/
+tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/
+dpkg -l > $LOCATION/dpkg.`date +%w`
+
+lftp -f ~/bin/backup-options
diff --git a/roles/kde-backup/templates/backup-rosetta.sh \
b/roles/kde-backup/templates/backup-rosetta.sh new file mode 100755
index 0000000..058e4bf
--- /dev/null
+++ b/roles/kde-backup/templates/backup-rosetta.sh
@@ -0,0 +1,25 @@
+LOCATION=/root/{{backup_directory}}
+
+# Backup databases
+for DATABASE in `mysqlshow | grep -v "information_schema" | grep -v \
"performance_schema" | grep -v "mysql" | tail -n+4 | awk '{print $2}'`; do +    \
mysqldump --opt --quick --single-transaction --skip-extended-insert  --events \
--create-options --set-charset  $DATABASE > $LOCATION/db.$DATABASE.`date +%w`.sql +   \
xz -f $LOCATION/db.$DATABASE.`date +%w`.sql +done
+
+# Backup homes!
+tar -cJf /root/backups/homes.`date +%w`.tar.xz --exclude=home/scripty/prod -C / \
home/ +
+# Backup application data
+tar -cJf $LOCATION/srv.`date +%w`.tar.xz -C / --exclude=srv/l10n-svn srv/ 2>&1 | \
grep -v "file changed as we read it" +
+# Backup configuration and list of installed packages
+tar -czf $LOCATION/etc.`date +%w`.tgz -C / etc/
+tar -czf $LOCATION/crontabs.`date +%w`.tgz -C / var/spool/cron/
+dpkg -l > $LOCATION/dpkg.`date +%w`
+
+# Secure the backups
+chmod -R 700 $LOCATION
+
+# Transfer them to the backup server
+cd $LOCATION/..
+rsync --timeout=600 --delete -a backups -4 rosettabackup@gohma.kde.org:


[prev in list] [next in list] [prev in thread] [next in thread]
Configure | About | News | Add a list | Sponsored by KoreLogic