--- psinib.pl 2003/01/21 19:50:47 1.5 +++ psinib.pl 2004/03/01 19:38:16 1.22 @@ -24,16 +24,21 @@ use Fcntl qw(LOCK_EX LOCK_NB); use Digest::MD5; use File::Basename; +use Getopt::Long; # configuration my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile my $DIR_TIME_FMT = '%Y%m%d'; # strftime format for backup dir +# define timeout for ping +my $PING_TIMEOUT = 5; + my $LOG = '/var/log/backup.log'; # add path here... #$LOG = '/tmp/backup.log'; # store backups in which directory my $BACKUP_DEST = '/backup/isis_backup'; +#my $BACKUP_DEST = '/tmp/backup/'; # files to ignore in backup my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt'); @@ -50,21 +55,93 @@ sleep 1; redo if ++$c < 10; # no response for 10 sec, bail out - print STDERR "can't take lock on $LOG -- another $0 running?\n"; + xlog("ABORT","can't take lock on $LOG -- another $0 running?"); exit 1; } # taint path: nmblookup should be there! $ENV{'PATH'} = "/usr/bin:/bin"; +my $use_ping = 1; # default: use syn tcp ping to verify that host is up +my $verbose = 1; # default verbosity level +my $quiet = 0; +my $email; + +my $result = GetOptions( + "ping!" => \$use_ping, "backupdest!" => \$BACKUP_DEST, + "verbose+" => \$verbose, "quiet+" => \$quiet, + "email=s" => \$email, +); + +$verbose -= $quiet; + my $mounts = shift @ARGV || 'mountscript'; # die "usage: $0 mountscript"; +my $basedir = $0; +$basedir =~ s,/?[^/]+$,,g; + +# default subject for e-mail messages +my @subjects = ('Backup needs your attention!'); +my $sub_nr = 0; +my $email_body; + +my $home_dir=$ENV{'HOME'}; +$home_dir = '/tmp' if (! -w $home_dir); + +if ($email) { + # It will use (and require) Tie::File only if --email=foo@bar.com + # arguement is used! + use Tie::File; + tie @subjects, 'Tie::File', "$basedir/subjects.txt" || xlog("CONFIG","Can't find $basedir/subjects.txt... using default (only one)"); + chdir; # this will change directory to HOME + if (open(SN,"$home_dir/.psinib.subject")) { + $sub_nr = ; + chomp($sub_nr); + close(SN); + } + $sub_nr++; + # skip comments in subjects.txt + while($subjects[$sub_nr] && $subjects[$sub_nr] =~ m/^#/) { + $sub_nr++; + } + $sub_nr = 0 if (! $subjects[$sub_nr]); + + if (open(SN,"> $home_dir/.psinib.subject")) { + print SN "$sub_nr\n"; + close (SN); + } else { + xlog("CONFIG","Can't open $home_dir/.psinib.subject -- I can't cycle subjects..."); + }; +} my @in_backup; # shares which are backeduped this run -my $p = new Net::Ping->new(); +# init Net::Ping object +my $ping; +if ($use_ping) { + $ping = new Net::Ping->new("syn", 2); + # ping will try tcp connect to netbios-ssn (139) + $ping->{port_num} = getservbyname("netbios-ssn", "tcp"); +} + +# do syn ping to cifs port +sub host_up { + my $ping = shift || return; + my $host_ip = shift || xlog("host_up didn't get IP"); + my $timeout = shift; + return 1 if (! $use_ping); + + $ping->ping($host_ip,$timeout); + my $return = 0; + + while (my ($host,$rtt,$ip) = $ping->ack) { + xlog("","HOST: $host [$ip] ACKed in $rtt seconds"); + $return = 1 if ($ip eq $host_ip); + } + return $return; +} my $backup_ok = 0; @@ -79,7 +156,7 @@ next if !/^\s*smbmount\s/; my (undef,$share,undef,$opt) = split(/\s+/,$_,4); - my ($user,$passwd,$workgroup); + my ($user,$passwd,$workgroup,$ip); foreach (split(/,/,$opt)) { my ($n,$v) = split(/=/,$_,2); @@ -95,6 +172,8 @@ } } elsif ($n =~ m#workgroup#i) { $workgroup = $v; + } elsif ($n =~ m#ip#i) { + $ip = $v; } } @@ -105,11 +184,11 @@ my $bl = "$BACKUP_DEST/$host/$dir/latest"; # latest backup my $bc = "$BACKUP_DEST/$host/$dir/$date_dir"; # current one my $real_bl; - if (-e $bl) { + if (-l $bl) { $real_bl=readlink($bl) || die "can't read link $bl: $!"; $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/"); - if (-e $bc && $real_bl eq $bc) { - print "$share allready backuped...\n"; + if (-l $bc && $real_bl eq $bc) { + xlog($share,"allready backuped..."); $backup_ok++; next; } @@ -117,22 +196,28 @@ } - print "working on $share\n"; - + xlog($share,"working on $share..."); - my $ip = get_ip($share); + # try to nmblookup IP + $ip = get_ip($share) if (! $ip); if ($ip) { xlog($share,"IP is $ip"); - if ($p->ping($ip)) { - snap_share($share,$user,$passwd,$workgroup); - $backup_ok++; + if (host_up($ping, $ip,$PING_TIMEOUT)) { + if (snap_share($share,$user,$passwd,$workgroup)) { + $backup_ok++; + } } } } close(M); -xlog("","$backup_ok backups completed of total ".($#in_backup+1)." this time (".int($backup_ok*100/($#in_backup+1))." %)"); +my $total = ($#in_backup + 1) || 0; +my $pcnt = ""; +$pcnt = "(".int($backup_ok*100/$total)." %)" if ($total > 0); +xlog("","$backup_ok backups completed of total $total this time".$pcnt); + +send_email(); 1; @@ -151,16 +236,41 @@ } } +# send e-mail with all messages +sub send_email { + return if (! $email || $email eq "" || !$email_body); + require Mail::Send; + my $msg = new Mail::Send; + $msg->to($email); + $msg->subject($subjects[$sub_nr]); + my $fn=$msg->open; + print $fn $email_body; + $fn->close; +} + # write entry to screen and log sub xlog { my $share = shift; my $t = strftime $LOG_TIME_FMT, localtime; my $m = shift || '[no log entry]'; - print STDERR $m,"\n"; + my $l = shift; + $l = 1 if (! defined $l); # default verbosity is 1 + if ($verbose >= $l) { + if (! $email) { + print STDERR $m,"\n"; + # don't e-mail mesages with verbosity < 1 + } elsif ($l < 1) { + $email_body .= $m."\n"; + } + } print L "$t $share\t$m\n"; } +# dump warn and dies into log +BEGIN { $SIG{'__WARN__'} = sub { xlog('WARN',$_[0],1) ; exit 1 } } +BEGIN { $SIG{'__DIE__'} = sub { xlog('DIE',$_[0],0) ; exit 1 } } + # split share name to host, dir and currnet date dir sub share2host_dir { @@ -172,7 +282,7 @@ $dir =~ s/^_+//; $dir =~ s/_+$//; } else { - print "Can't parse share $share into host and directory!\n"; + xlog($share,"Can't parse share $share into host and directory!",1); return; } return ($host,$dir,strftime $DIR_TIME_FMT, localtime); @@ -186,9 +296,9 @@ my %param = ( debug => 0 ); - $param{username} = shift; - $param{password} = shift; - $param{workgroup} = shift; + $param{username} = shift || warn "can't find username for share $share"; + $param{password} = shift || warn "can't find passwod for share $share"; + $param{workgroup} = shift || warn "can't find workgroup for share $share"; my ($host,$dir,$date_dir) = share2host_dir($share); @@ -198,28 +308,48 @@ my $bc = "$BACKUP_DEST/$host/$dir/$date_dir"; my $real_bl; - if (-e $bl) { + if (-l $bl) { $real_bl=readlink($bl) || die "can't read link $bl: $!"; $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/"); - } else { - print "no old backup, this is first run...\n"; + if (! -e $real_bl) { + xlog($share,"latest link $bl -> $real_bl not valid, removing it"); + unlink $bl || die "can't remove link $bl: $!"; + undef $real_bl; + } + } + if (! $real_bl) { + xlog($share,"no old backup, trying to find last backup"); + if (opendir(BL_DIR, "$BACKUP_DEST/$host/$dir")) { + my @bl_dirs = sort grep { !/^\./ && -d "$BACKUP_DEST/$host/$dir/$_" } readdir(BL_DIR); + closedir(BL_DIR); + $real_bl=pop @bl_dirs; + xlog($share,"using $real_bl as latest..."); + $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/"); + if ($real_bl eq $bc) { + xlog($share,"latest from today (possible partial backup)"); + rename $real_bl,$real_bl.".partial" || warn "can't reaname partial backup: $!"; + $real_bl .= ".partial"; + } + } else { + xlog($share,"this is first run..."); + } } - if (-e $bc && $real_bl && $real_bl eq $bc) { - print "$share allready backuped...\n"; - return; + if (-l $bc && $real_bl && $real_bl eq $bc) { + xlog($share,"allready backuped..."); + return 1; } die "You should really create BACKUP_DEST [$BACKUP_DEST] by hand! " if (!-e $BACKUP_DEST); if (! -e "$BACKUP_DEST/$host") { mkdir "$BACKUP_DEST/$host" || die "can't make dir for host $host, $BACKUP_DEST/$host: $!"; - print "created host directory $BACKUP_DEST/$host...\n"; + xlog($share,"created host directory $BACKUP_DEST/$host..."); } if (! -e "$BACKUP_DEST/$host/$dir") { mkdir "$BACKUP_DEST/$host/$dir" || die "can't make dir for share $share, $BACKUP_DEST/$host/$dir $!"; - print "created dir for share $share, $BACKUP_DEST/$host/$dir...\n"; + xlog($share,"created dir for this share $BACKUP_DEST/$host/$dir..."); } mkdir $bc || die "can't make dir for current backup $bc: $!"; @@ -235,6 +365,7 @@ my %file_atime; my %file_mtime; #my %file_md5; + %file_md5 = (); my @smb_files; my %smb_size; @@ -255,24 +386,24 @@ my $di = 0; while ($di <= $#dirs && $real_bl) { my $d=$dirs[$di++]; - opendir(DIR,"$bl/$d") || warn "opendir($bl/$d): $!\n"; + opendir(DIR,"$real_bl/$d") || warn "opendir($real_bl/$d): $!\n"; # read .backupignore if exists - if (-f "$bl/$d/.backupignore") { - open(I,"$bl/$d/.backupignore"); + if (-f "$real_bl/$d/.backupignore") { + open(I,"$real_bl/$d/.backupignore"); while() { chomp; push @ignore,norm_dir("$d/$_"); } close(I); -print STDERR "ignore: ",join("|",@ignore),"\n"; - link "$bl/$d/.backupignore","$bc/$d/.backupignore" || - warn "can't copy $bl/$d/.backupignore to current backup dir: $!\n"; +#print STDERR "ignore: ",join("|",@ignore),"\n"; + link "$real_bl/$d/.backupignore","$bc/$d/.backupignore" || + warn "can't copy $real_bl/$d/.backupignore to current backup dir: $!\n"; } # read .md5sum if exists - if (-f "$bl/$d/.md5sum") { - open(I,"$bl/$d/.md5sum"); + if (-f "$real_bl/$d/.md5sum") { + open(I,"$real_bl/$d/.md5sum"); while() { chomp; my ($md5,$f) = split(/\s+/,$_,2); @@ -286,7 +417,7 @@ next if ($f eq '.'); next if ($f eq '..'); my $pr = norm_dir("$d/$f"); # path relative - my $pf = norm_dir("$d/$f","$bl/"); # path full + my $pf = norm_dir("$d/$f","$real_bl/"); # path full if (grep(/^\Q$pr\E$/,@ignore) == 0) { if (-f $pf) { push @files,$pr; @@ -296,15 +427,16 @@ } elsif (-d $pf) { push @dirs,$pr; } else { - print STDERR "unknown type: $pf\n"; + xlog($share,"not file or directory: $pf",0); } } else { - print STDERR "ignored: $pr\n"; + xlog($share,"ignored: $pr"); } } } - xlog($share,($#files+1)." files and ".($#dirs+1)." dirs on local disk before backup"); + # local dir always include / + xlog($share,($#files+1)." files and ".($#dirs)." dirs on local disk before backup"); # read smb filesystem @@ -315,9 +447,16 @@ $di = 0; while ($di <= $#smb_dirs) { - my $d=$smb_dirs[$di++]; + my $d=$smb_dirs[$di]; my $pf = norm_dir($d,"smb:$share/"); # path full - my $D = $smb->opendir($pf) || warn "smb->opendir($pf): $!\n"; + my $D = $smb->opendir($pf); + if (! $D) { + xlog($share,"FATAL: $share [$pf] as $param{username}/$param{workgroup}: $!",0); + # remove failing dir + delete $smb_dirs[$di]; + return 0; # failed + } + $di++; my @clutter = $smb->readdir_struct($D); foreach my $item (@clutter) { @@ -335,15 +474,15 @@ } elsif ($item->[0] == main::SMBC_DIR) { push @smb_dirs,$pr; } else { - print STDERR "unknown type: $pf\n"; + xlog($share,"not file or directory [".$item->[0]."]: $pf",0); } } else { - print STDERR "smb ignored: $pr\n"; + xlog($share,"smb ignored: $pr"); } } } - xlog($share,($#smb_files+1)." files and ".($#smb_dirs+1)." dirs on remote share"); + xlog($share,($#smb_files+1)." files and ".($#smb_dirs)." dirs on remote share"); # sync dirs my $lc = List::Compare->new(\@dirs, \@smb_dirs); @@ -375,16 +514,16 @@ foreach my $f (@_) { #print "smb_copy $from/$f -> $to/$f\n"; - if (! open(F,"> $to/$f")) { - print STDERR "can't open new file $to/$f: $!\n"; - next; - } - my $md5 = Digest::MD5->new; my $fd = $smb->open("$from/$f"); if (! $fd) { - print STDERR "can't open smb file $from/$f: $!\n"; + xlog("WARNING","can't open smb file $from/$f: $!"); + next; + } + + if (! open(F,"> $to/$f")) { + xlog("WARNING","can't open new file $to/$f: $!"); next; } @@ -452,12 +591,13 @@ xlog($share,"$transfer bytes transfered..."); foreach (@ln_files) { - link "$bl/$_","$bc/$_" || warn "link $bl/$_ -> $bc/$_: $!\n"; + link "$real_bl/$_","$bc/$_" || warn "link $real_bl/$_ -> $bc/$_: $!\n"; } # remove files foreach (sort @files2erase) { unlink "$bc/$_" || warn "unlink $_: $!\n"; + delete $file_md5{$_}; } # remove not needed dirs (after files) @@ -470,29 +610,45 @@ unlink "$bc/$_/.md5sum" if (-e "$bc/$_/.md5sum"); } + # erase stale entries in .md5sum + my @md5_files = keys %file_md5; + $lc = List::Compare->new(\@md5_files, \@smb_files); + foreach my $file ($lc->get_Lonly) { + xlog("NOTICE","removing stale '$file' from .md5sum"); + delete $file_md5{$file}; + } + # create .md5sum my $last_dir = ''; my $md5; - foreach my $f (sort { $file_md5{$a}<=>$file_md5{$b} } keys %file_md5) { + foreach my $f (sort { $file_md5{$a} cmp $file_md5{$b} } keys %file_md5) { my $dir = dirname($f); my $file = basename($f); -print "$f -- $dir / $file<--\n"; +#print "$f -- $dir / $file<--\n"; if ($dir ne $last_dir) { close($md5) if ($md5); open($md5, ">> $bc/$dir/.md5sum") || warn "can't create $bc/$dir/.md5sum: $!"; $last_dir = $dir; -print STDERR "writing $last_dir/.md5sum\n"; +#print STDERR "writing $last_dir/.md5sum\n"; } print $md5 $file_md5{$f}," $file\n"; } - close($md5); + close($md5) if ($md5); # create leatest link -# symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n"; +#print "ln -s $bc $real_bl\n"; + if (-l $bl) { + unlink $bl || warn "can't remove old latest symlink $bl: $!\n"; + } + symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n"; + + # FIX: sanity check -- remove for speedup + xlog($share,"failed to create latest symlink $bl -> $bc...") if (readlink($bl) ne $bc || ! -l $bl); xlog($share,"backup completed..."); -} + return 1; +} __END__ #------------------------------------------------------------------------- @@ -503,10 +659,38 @@ =head1 SYNOPSIS -./psinib.pl +./psinib.pl [OPTION]... [mount script] =head1 DESCRIPTION +Option can be one of more of following: + +=over 8 + +=item C<--backupdest=dir> + +Specify backup destination directory (defaults is /data/ + +=item C<--noping> + +Don't use ping to check if host is up (default is ti use tcp syn to cifs +port) + +=item C<--verbose -v> + +Increase verbosity level. Defailt is 1 which prints moderate amount of data +on STDOUT and STDERR. + +=item C<--quiet -q> + +Decrease verbosity level + +=item C<--email=email@domain> + +Send e-mails instead of dumping errors to STDERR. Useful for cron jobs. + +=back + This script in current version support just backup of Samba (or Micro$oft Winblowz) shares to central disk space. Central disk space is organized in multiple directories named after: @@ -547,7 +731,7 @@ over network) =item - usage of C<.md5sum> files (compatible with command-line utility -C to keep file between snapshots hard-linked +C) to keep file between snapshots hard-linked =back @@ -591,12 +775,57 @@ server/c_WinNT_fonts/yyyymmdd/.... +=head2 Won't I run out of disk space? + +Of course you will... Snapshots and logfiles will eventually fill-up your disk. +However, you can do two things to stop that: + +=head3 Clean snapshort older than x days + +You can add following command to your C crontab: + + find /backup/isis_backup -type d -mindepth 3 -maxdepth 3 -mtime +11 -exec rm -Rf {} \; + +I assume that C is directory in which are your snapshots +and that you don't want to keep snapshots older than 11 days (that's +C<-mtime +11> part of command). + +=head3 Rotate your logs + +I will leave that to you. I relay on GNU/Debian's C to do it for me. + +=head2 What are I directories? + +If there isn't I symlink in snapshot directory, it's preatty safe to +assume that previous backup from that day failed. So, that directory will +be renamed to I and snapshot will be performed again, +linking same files (other alternative would be to erase that dir and find +second-oldest directory, but this seemed like more correct approach). + +=head2 I can't connect to any share + +Please verify that nmblookup (which is part of samba package) is in /bin or +/usr/bin. Also verify that nmblookup returns IP address for your server +using: + + $ nmblookup tvhouse + querying tvhouse on 192.168.34.255 + 192.168.34.30 tvhouse<00> + +If you don't get any output, your samba might not listen to correct interface +(see interfaces in smb.conf). + +=head2 Aren't backups boring? + +No! If you have subjects.txt in same directory as C you can get +various funny subjects in your mail. They change over time as long as you +ignore your backup. =head1 AUTHOR Dobrica Pavlinusic -L +LEwww.rot13.orgE~dpavlinE> =head1 LICENSE