php/save.php
<?php
/*-----
# save: walters backup
#
# argumentsa
# c<clouds> set of cloulds: , separated list of clouds, e.g. -c -cwlklGo -cspzhNC,spwaGo etc
# arc | ele | sea set destination
# db mysqlDump databases in wk13 to /wkData/save
# www rclone sync /wkData/www to wlkl.ch/public_html (plus some further in both directions ...)
# bw rclone sync wkData to wlkl.ch/files
# bd rdiff-backup wkData to dest
# ba rdiff-backup wkArchive to dest
# bc rclone sync clouds to dest (with new ffid)
# d => db, www, bw, bd, bc else (if destination = arc for others without bw)
# a => ba
# l0d | l0a rclone sync from wkData | wkArchive to sea/lili (build create full set for lili - remove links before
# l9 rclone sync sea/lili to arc/lili (after deduping analysing and removing links etc. in sea/lili)
# lw | le rclone sync arc/lili to wlkl.ch://files/lili | ele/lili
#
# 28. 1.25 lili functions, syntax adapted
# 26. 1.25 syntax check warnings for rdiff-backup
# 16. 1.25 syntax rdiff-backup 2.2.6 und php 8.3.6
# 13. 1.25 neu implementierung in php, including cloud synchronisation and ffid
#
# save: walters backup from 2009 - 2024 in bash
# deimplemented p = Part: only mostly updated parts
# d = wkData: wkData (+ copy to backup2www und backup222data)
# a = wkArchive = wkArchive
# /... = Destination directory
# copyFrom www | data timstamp
# neues Verfahren: mit rdiff-backup
#
# 2.12.22 exclude lili
# 2.12.22 remove exclude of wwwWk13
# 27.11.22 use scp to upload 222www and 222data to wlkl.ch, compute bytes for copied files
# 23.01.20 saveSP --> save
# 7. 3.19 exclude zMysql (and skip chown, chmod)
# 26. 2.19 exclude wkArchive/s1*, s2*, sa*
# 9. 2.19 wk/extra
# 17. 1.19 added mysql dump of all DBs to /wkData/saveSP
# 8. 1.19 use rdiff-backup --list-changed-since for backup222www and backup222data
# 6. 7.18 d-> wkData. a-> wkArchive, a to wkArchive ohne Bilder etc.. vm.swappiness=0
# 15. 5.18 tst, tmp durchgehend excluded
# 5. 5.18 -D flag und etwas refactoring plus Seagate
# option --exclude-special-files eingefügt, damit keine symbolic links erstellt werden
# 20. 1.14 rdiff-backup
# 19.11.12 new directories
# 28.12.09 Walter Keller new
#
----- */
require_once 'cloud.php';
const MEDIA = '/media/walter'
, DESTA = [ 'sea' => MEDIA . '/Seagate1805' # destination, or where to put the backup - if not specified will look, what is mounted
, 'ele' => MEDIA . '/Elements0805'
, 'arc' => '/wkArchive']
, DESTS = [ ... DESTA
, 'dat' => '/wkData'
, 'wkf' => 'wlklFT:files'
, 'wkw' => 'wlklFT:wlkl.ch/public_html'
]
, LILIA = [ 'l0d' => ['dat', 'sea']
, 'l0a' => ['arc', 'sea']
, 'l9' => ['sea', 'arc']
, 'lw' => ['arc', 'wkf']
, 'le' => ['arc', 'ele']
]
, KEYP = '/ppKey'
, DIVP = '/ppDiv'
, MNTP = DIVP . "/mnt"
, RCCONLOG = "--config " . KEYP . "/cfg/rclone.conf --log-file " . DIVP . "/log/rclone.log"
, EXCL = ".* lost+fou* *backu* del* lili* log* old* proj* s[0-9]* install* nextcloud old* *run* tes* timeshi* tmp* tst* wk/extra wk/extraRe* zMysql" # the exclude (dirs) for /wkData
, RDBWARN = [2 => 'warning', 8 => 'file warning']
;
function sysEx($cmd, $ww=null) {
out("+++$cmd --- begin", toLocalTst('now'));
$rc = 'bad';
$ll = system($cmd, $rc);
if ($rc === 0)
out("+++ endedOK: $cmd ---", toLocalTst('now'));
elseif (isset($ww[$rc]))
out("+++ warning returnCode=$rc=$ww[$rc]: $cmd --- ", toLocalTst('now'));
else
err("--- failed rc=$rc: $cmd, lastLine $ll.", toLocalTst('now'));
}
function dbDump() {
$c = file_get_contents('/ppKey/cfg/mysqlOpt');
preg_match("/user=(\S+)\s+password=(\S+)\s/", $c, $m);
dbg1("c $c, match", $m);
$dbL = new PDO('mysql:host=localhost', $m[1], $m[2]);
foreach($dbL->query('show databases', PDO::FETCH_ASSOC) as $row) {
dbg1('db', $row);
if (! isset(['information_schema' => 1, 'performance_schema' => 1, 'sys' => 1][$db = $row['Database']]))
sysEx("mysqldump --defaults-file=/ppKey/cfg/mysqlOpt $db > /wkData/save/mysqlDumpDB$db.sql");
}
}
function syncCloud($aa, $dDir) {
$oId=true;
$oSync=true;
$f = new CloudFactory;
$bakP = "$dDir/backupCloud";
$synP = "/wkArchive/backupCloud/sync";
$aa or $aa = array_keys($f->key());
foreach ($aa as $one) {
out("--- $one", toLocalTst('now'));
$k = $f->key()[$one] ?? err("cloud $one unknown");
if ($oId) {
$cc = $f->make($one);
$ff = $cc->ff($ppl = $cc->pipeline('%,metaRowCsv,PHST RECUR'));
out(" ff $one", ($ppl->ppc->rowWrite2 ?? $ppl->ppc->rowWriter)->writeC, "rows,", sprintf('%7.2e', ftell($ff)), "bytes", toLocalTst('now'));
out(' creUpd', $cc->creUpd('%all,row1,PHST KRF WRIPA', "ffid-$one.csv", 'text/csv', fRewContClo($ff)), "{$cc->fun}d", toLocalTst('now'));
}
if ($oSync) {
$inc = is_readable($incFi= KEYP . "/cfg/rclone-$one.inc") ? "--include-from $incFi" : '';
@mkdir(MNTP . "/$one", 0755); # ignore error, it normally means exists already
@mkdir("$bakP", 0755, true); # ignore error, it normally means exists already
if ($k[1] === 'godr') {
sysEx("rclone " . RCCONLOG . " sync $inc $one: $synP/$one");
sysEx("rdiff-backup --api-version 201 backup $synP/$one $bakP/$one", RDBWARN);
} else {
sysEx("rclone -v " . RCCONLOG . " mount --read-only --daemon $inc $one: " . MNTP . "/$one");
sysEx("rdiff-backup --api-version 201 backup --no-eas " . MNTP . "/$one $bakP/$one", RDBWARN);
sysEx("fusermount -u " . MNTP . "/$one");
}
}
}
}
function rdiffBackup($fr, $dst) {
$ex = EXCL;
if ($fr === '/wkData') {
$dTo = "$dst/backupData";
} else {
$dTo = "$dst/backupArchive";
if ($dst === '/wkArchive')
$ex .= " bilder book music";
}
$c = 'rdiff-backup --api-version 201 backup --exclude-special-files';
foreach(preg_split('/\s+/', $ex, -1, PREG_SPLIT_NO_EMPTY) as $e)
$c .= " --exclude '$fr/$e'";
sysEx("$c $fr $dTo", RDBWARN);
}
function rcloneBackup($fr='/wkData', $dst='wlklFT:files') {
$ex = EXCL ; #. " *Trash*";
$dTo = ($fr === '/wkData') ?"$dst/backupData" : err("not implemented fr=$fr");
$c = "rclone " . RCCONLOG . " sync -v";
foreach(preg_split('/\s+/', $ex, -1, PREG_SPLIT_NO_EMPTY) as $e)
$c .= " --exclude '/$e/'"; # trailing slash to exclude dirs in rclone
sysEx("$c $fr $dTo");
}
function syncWWW() {
sysEx("rclone " . RCCONLOG . " sync -v --exclude '/*' /wkData/www wlklFT:wlkl.ch/public_html");
sysEx("rclone " . RCCONLOG . " sync -v --include '/*' /wkData/www/variant/wlkl/public_html wlklFT:wlkl.ch/public_html"); # root directory (without recursion) is special!
sysEx("rclone " . RCCONLOG . " sync -v --exclude '/*' wlklFT:eveline.keller.wlkl.ch/public_html /wkData/wwwWk13/eveline"); # wiki eveline wiki subdirs
sysEx("rclone " . RCCONLOG . " sync -v --include '/*' /wkData/www/variant/wk13/eveline /wkData/wwwWk13/eveline"); # wiki eveline root dir (without recursion) is special!
}
function liliDo($f) {
if (! $frto = LILIA[$f] ?? false)
err("bad lili func $f, valid:", array_keys(LILIA));
$fr = DESTS[$frto[0]];
$to = DESTS[$frto[1]];
if ($f[1] === '0') { # sync /wkData/... rsp /wkArchive to sea/lili
$e = '';
$eF = '/wkData/tmp/save-filter.txt';
foreach (['sp*', 'joomla*', 'pawa', 'pepr'] as $q)
foreach (explode(' ', 'administrator api bin cache cli components includes installat* language layouts libraries logs media modules plugins templates tmp') as $r)
$e .= "- $q/$r/\n";
$e .= "- wp*/wp-*/\n- extraRem*/\n";
foreach (explode(' ', $f === 'l0d' ? 'admin inf pc save sp wk www wwwWk13' : '199* 20*') as $r)
$e .= "+ /$r/**\n";
file_put_contents($eF, "$e- /**\n");
sysEx('rclone ' . RCCONLOG . " sync -v --delete-excluded --filter-from $eF $fr $to/lili$fr");
} else {
sysEx('rclone ' . RCCONLOG . " sync -v $fr/lili $to/lili"); # from Seagate to Arc (after jDupes, lili a ......)>
}
}
function liliSync($f, $d) {
if ($f === 'l2a') {
sysEx('rclone ' . RCCONLOG . ' sync -v ' . DESTS['sea'] .'/lili ' . DESTS['arc'] .'/lili'); # from Seagate to Arcwiki eveline root dir (without recursion) is special!
} else {
$e = ($f === 'l2w') ? 'wlklFT:files' : ($d !== 'arc' ? DESTS['arc'] : err("mismatch fun $f and dest $d"));
sysEx('rclone ' . RCCONLOG . ' sync -v ' . DESTS['arc'] .'/lili ' . "$e/lili"); # from arc to $e
}
}
function work($aa) {
$doDb = true;
$clds = [];
while (null !== $a = array_shift($aa)) {
if ('' === $a = trim($a)) {
} elseif (isset(DESTA[$a])) {
$dst = $a;
} else if ($a[0] === 'c') {
out("c set clouds to", $clds = preg_split('/[,\s]+/', substr($a, 2), -1, PREG_SPLIT_NO_EMPTY));
} elseif ($a[0] === 'l') {
liliDo($a); # === 'lilia' ? 'wkArchive' : 'wkData');
} elseif ($a === 'db') {
($doDb = false) or dbDump();
} elseif ($a === 'www') {
syncWWW();
} elseif ($a === 'bw') {
rcloneBackup();
} else { # the remaining actions need as destination, thus get default destination if necessary
if ( ! isset($dst)) {
foreach(DESTS as $dst => $v) {
if (is_dir($v) and is_writeable($v) and count($ff = scandir($v)) > 2)
break;
}
out("default dest $dst", count($ff));
}
if ($a === 'a') {
array_unshift($aa, 'ba');
} elseif ($a === 'd') {
array_unshift($aa, $doDb ? 'db' : '', 'www', $dst === 'arc' ? 'bw' : '', 'bd', 'bc');
} elseif ($a === 'bd') { # elseif ($fr = ($a === 'd' or $a === 'bd') ? '/wkData' : (($a === 'a' or $a === 'ba') ? '/wkArchive': false))
rdiffBackup('/wkData', DESTS[$dst]);
} elseif ($a === 'ba') {
rdiffBackup('/wkArchive', DESTS[$dst]);
} elseif ($a === 'bc') {
syncCloud($clds, DESTS[$dst]);
} else {
err("bad argument $a, valid are dests", array_keys(DESTS), "and actions d, a, db, www, bw, bd, ba, bc, l* and c<clouds>");
}
}
}
}
('walter' === ($g = posix_getgrgid(posix_getgid())) ['name']) or err('posix_getgid not walter group', $g);
work(($ea=envArgs()) ? $ea : ['d']);