/[psinib]/psinib.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /psinib.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.3 by dpavlin, Sat Jan 4 13:29:12 2003 UTC revision 1.10 by dpavlin, Tue Jul 15 17:41:45 2003 UTC
# Line 12  Line 12 
12  #  #
13  #  #
14  # usage:  # usage:
15  #       $ backup.pl mountscript  #       $ psinib.pl mountscript
16    
17  use strict 'vars';  use strict 'vars';
18  use Data::Dumper;  use Data::Dumper;
# Line 22  use List::Compare; Line 22  use List::Compare;
22  use Filesys::SmbClient;  use Filesys::SmbClient;
23  #use Taint;  #use Taint;
24  use Fcntl qw(LOCK_EX LOCK_NB);  use Fcntl qw(LOCK_EX LOCK_NB);
25    use Digest::MD5;
26    use File::Basename;
27    
28  # configuration  # configuration
29  my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile  my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile
30  my $DIR_TIME_FMT = '%Y%m%d';            # strftime format for backup dir  my $DIR_TIME_FMT = '%Y%m%d';            # strftime format for backup dir
31    
32  my $LOG = '/var/log/backup.log';        # add path here...  my $LOG = '/var/log/backup.log';        # add path here...
33  $LOG = '/tmp/backup.log';  #$LOG = '/tmp/backup.log';
34    
35  # store backups in which directory  # store backups in which directory
36  my $BACKUP_DEST = '/data/isis_backup';  my $BACKUP_DEST = '/backup/isis_backup';
37    
38  # files to ignore in backup  # files to ignore in backup
39  my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');  my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');
40    
41  # open log  # open log
42  open(L, "> $LOG") || die "can't open log $LOG: $!";  open(L, ">> $LOG") || die "can't open log $LOG: $!";
43  select((select(L), $|=1)[0]);   # flush output  select((select(L), $|=1)[0]);   # flush output
44    
45  # make a lock on logfile  # make a lock on logfile
# Line 62  my $mounts = shift @ARGV || Line 64  my $mounts = shift @ARGV ||
64    
65  my @in_backup;  # shares which are backeduped this run  my @in_backup;  # shares which are backeduped this run
66    
67  my $p = new Net::Ping->new();  my $p = new Net::Ping->new("tcp", 2);
68    # ping will try tcp connect to netbios-ssn (139)
69    $p->{port_num} = getservbyname("netbios-ssn", "tcp");
70    
71  my $backup_ok = 0;  my $backup_ok = 0;
72    
73  my $smb;  my $smb;
74  my %smb_atime;  my %smb_atime;
75  my %smb_mtime;  my %smb_mtime;
76    my %file_md5;
77    
78  open(M, $mounts) || die "can't open $mounts: $!";  open(M, $mounts) || die "can't open $mounts: $!";
79  while(<M>) {  while(<M>) {
# Line 97  while(<M>) { Line 102  while(<M>) {
102    
103          push @in_backup,$share;          push @in_backup,$share;
104    
105    
106            my ($host,$dir,$date_dir) = share2host_dir($share);
107            my $bl = "$BACKUP_DEST/$host/$dir/latest";      # latest backup
108            my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";   # current one
109            my $real_bl;
110            if (-l $bl) {
111                    $real_bl=readlink($bl) || die "can't read link $bl: $!";
112                    $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
113                    if (-l $bc && $real_bl eq $bc) {
114                            print "$share allready backuped...\n";
115                            $backup_ok++;
116                            next;
117                    }
118    
119            }
120    
121    
122          print "working on $share\n";          print "working on $share\n";
123    
124    
125          my $ip = get_ip($share);          my $ip = get_ip($share);
126    
127          if ($ip) {          if ($ip) {
# Line 117  xlog("","$backup_ok backups completed of Line 140  xlog("","$backup_ok backups completed of
140    
141  #-------------------------------------------------------------------------  #-------------------------------------------------------------------------
142    
143    
144  # get IP number from share  # get IP number from share
145  sub get_ip {  sub get_ip {
146          my $share = shift;          my $share = shift;
# Line 129  sub get_ip { Line 153  sub get_ip {
153          }          }
154  }  }
155    
156    
157    # write entry to screen and log
158  sub xlog {  sub xlog {
159          my $share = shift;          my $share = shift;
160          my $t = strftime $LOG_TIME_FMT, localtime;          my $t = strftime $LOG_TIME_FMT, localtime;
# Line 137  sub xlog { Line 163  sub xlog {
163          print L "$t $share\t$m\n";          print L "$t $share\t$m\n";
164  }  }
165    
166  sub snap_share {  # dump warn and dies into log
167    BEGIN { $SIG{'__WARN__'} = sub { xlog('WARN',$_[0]) ; warn $_[0] } }
168    BEGIN { $SIG{'__DIE__'} = sub { xlog('DIE',$_[0]) ; die $_[0] } }
169    
         my $share = shift;  
   
         my %param = ( debug => 0 );  
   
         $param{username} = shift;  
         $param{password} = shift;  
         $param{workgroup} = shift;  
170    
171    # split share name to host, dir and currnet date dir
172    sub share2host_dir {
173            my $share = shift;
174          my ($host,$dir);          my ($host,$dir);
175          if ($share =~ m#//([^/]+)/(.+)$#) {          if ($share =~ m#//([^/]+)/(.+)$#) {
176                  ($host,$dir) = ($1,$2);                  ($host,$dir) = ($1,$2);
# Line 157  sub snap_share { Line 181  sub snap_share {
181                  print "Can't parse share $share into host and directory!\n";                  print "Can't parse share $share into host and directory!\n";
182                  return;                  return;
183          }          }
184            return ($host,$dir,strftime $DIR_TIME_FMT, localtime);
185    }
186    
187    
188    # make a snapshot of a share
189    sub snap_share {
190    
191            my $share = shift;
192    
193          my $date_dir = strftime $DIR_TIME_FMT, localtime;          my %param = ( debug => 0 );
194    
195            $param{username} = shift || warn "can't find username for share $share";
196            $param{password} = shift || warn "can't find passwod for share $share";
197            $param{workgroup} = shift || warn "can't find workgroup for share $share";
198    
199            my ($host,$dir,$date_dir) = share2host_dir($share);
200    
201          # latest backup directory          # latest backup directory
202          my $bl = "$BACKUP_DEST/$host/$dir/latest";          my $bl = "$BACKUP_DEST/$host/$dir/latest";
# Line 166  sub snap_share { Line 204  sub snap_share {
204          my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";          my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";
205    
206          my $real_bl;          my $real_bl;
207          if (-e $bl) {          if (-l $bl) {
208                  $real_bl=readlink($bl) || die "can't read link $bl: $!";                  $real_bl=readlink($bl) || die "can't read link $bl: $!";
209                  $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");                  $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
210          } else {          } else {
211                  print "no old backup, this is first run...\n";                  print "no old backup, trying to find last backup, ";
212                    if (opendir(BL_DIR, "$BACKUP_DEST/$host/$dir")) {
213                            my @bl_dirs = sort grep { !/^\./ && -d "$BACKUP_DEST/$host/$dir/$_" } readdir(BL_DIR);
214                            closedir(BL_DIR);
215                            $real_bl=pop @bl_dirs;
216                            print "using $real_bl as latest...\n";
217                            $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
218                            if ($real_bl eq $bc) {
219                                    xlog($share,"latest from today (possible partial backup)");
220                                    rename $real_bl,$real_bl.".partial" || warn "can't reaname partial backup: $!";
221                                    $real_bl .= ".partial";
222                            }
223                    } else {
224                            print "this is first run...\n";
225                    }
226          }          }
227    
228          if (-e $bc && $real_bl && $real_bl eq $bc) {          if (-l $bc && $real_bl && $real_bl eq $bc) {
229                  print "$share allready backuped...\n";                  print "$share allready backuped...\n";
230                  return;                  return;
231          }          }
# Line 202  sub snap_share { Line 254  sub snap_share {
254          my %file_size;          my %file_size;
255          my %file_atime;          my %file_atime;
256          my %file_mtime;          my %file_mtime;
257          my %file_md5;          #my %file_md5;
258    
259          my @smb_files;          my @smb_files;
260          my %smb_size;          my %smb_size;
261          #my %smb_atime;          #my %smb_atime;
262          #my %smb_mtime;          #my %smb_mtime;
         my %smb_md5;  
   
263    
264          sub norm_dir {          sub norm_dir {
265                  my $foo = shift;                  my $foo = shift;
# Line 225  sub snap_share { Line 275  sub snap_share {
275          my $di = 0;          my $di = 0;
276          while ($di <= $#dirs && $real_bl) {          while ($di <= $#dirs && $real_bl) {
277                  my $d=$dirs[$di++];                  my $d=$dirs[$di++];
278                  opendir(DIR,"$bl/$d") || warn "opendir($bl/$d): $!\n";                  opendir(DIR,"$real_bl/$d") || warn "opendir($real_bl/$d): $!\n";
279    
280                  # read .backupignore if exists                  # read .backupignore if exists
281                  if (-f "$bl/$d/.backupignore") {                  if (-f "$real_bl/$d/.backupignore") {
282                          open(I,"$bl/$d/.backupignore");                          open(I,"$real_bl/$d/.backupignore");
283                          while(<I>) {                          while(<I>) {
284                                  chomp;                                  chomp;
285                                  push @ignore,norm_dir("$d/$_");                                  push @ignore,norm_dir("$d/$_");
286                          }                          }
287                          close(I);                          close(I);
288  print STDERR "ignore: ",join("|",@ignore),"\n";  #print STDERR "ignore: ",join("|",@ignore),"\n";
289                          link "$bl/$d/.backupignore","$bc/$d/.backupignore" ||                          link "$real_bl/$d/.backupignore","$bc/$d/.backupignore" ||
290                                  warn "can't copy $bl/$d/.backupignore to current backup dir: $!\n";                                  warn "can't copy $real_bl/$d/.backupignore to current backup dir: $!\n";
291                  }                  }
292    
293                  # read .md5sum if exists                  # read .md5sum if exists
294                  if (-f "$bl/$d/.md5sum") {                  if (-f "$real_bl/$d/.md5sum") {
295                          open(I,"$bl/$d/.md5sum");                          open(I,"$real_bl/$d/.md5sum");
296                          while(<I>) {                          while(<I>) {
297                                  chomp;                                  chomp;
298                                  my ($md5,$f) = split(/\s+/,$_,2);                                  my ($md5,$f) = split(/\s+/,$_,2);
# Line 256  print STDERR "ignore: ",join("|",@ignore Line 306  print STDERR "ignore: ",join("|",@ignore
306                          next if ($f eq '.');                          next if ($f eq '.');
307                          next if ($f eq '..');                          next if ($f eq '..');
308                          my $pr = norm_dir("$d/$f");     # path relative                          my $pr = norm_dir("$d/$f");     # path relative
309                          my $pf = norm_dir("$d/$f","$bl/");      # path full                          my $pf = norm_dir("$d/$f","$real_bl/"); # path full
310                          if (grep(/^\Q$pr\E$/,@ignore) == 0) {                          if (grep(/^\Q$pr\E$/,@ignore) == 0) {
311                                  if (-f $pf) {                                  if (-f $pf) {
312                                          push @files,$pr;                                          push @files,$pr;
# Line 285  print STDERR "ignore: ",join("|",@ignore Line 335  print STDERR "ignore: ",join("|",@ignore
335    
336          $di = 0;          $di = 0;
337          while ($di <= $#smb_dirs) {          while ($di <= $#smb_dirs) {
338                  my $d=$smb_dirs[$di++];                  my $d=$smb_dirs[$di];
339                  my $pf = norm_dir($d,"smb:$share/");    # path full                  my $pf = norm_dir($d,"smb:$share/");    # path full
340                  my $D = $smb->opendir($pf) || warn "smb->opendir($pf): $!\n";                  my $D = $smb->opendir($pf);
341                    if (! $D) {
342                            xlog($share,"FATAL: $share: $!");
343                            # remove failing dir
344                            delete $smb_dirs[$di];
345                            next;
346                    }
347                    $di++;
348    
349                  my @clutter = $smb->readdir_struct($D);                  my @clutter = $smb->readdir_struct($D);
350                  foreach my $item (@clutter) {                  foreach my $item (@clutter) {
# Line 350  print STDERR "ignore: ",join("|",@ignore Line 407  print STDERR "ignore: ",join("|",@ignore
407                                  next;                                  next;
408                          }                          }
409    
410                            my $md5 = Digest::MD5->new;
411    
412                          my $fd = $smb->open("$from/$f");                          my $fd = $smb->open("$from/$f");
413                          if (! $fd) {                          if (! $fd) {
414                                  print STDERR "can't open smb file $from/$f: $!\n";                                  print STDERR "can't open smb file $from/$f: $!\n";
# Line 359  print STDERR "ignore: ",join("|",@ignore Line 418  print STDERR "ignore: ",join("|",@ignore
418                          while (defined(my $b=$smb->read($fd,4096))) {                          while (defined(my $b=$smb->read($fd,4096))) {
419                                  print F $b;                                  print F $b;
420                                  $l += length($b);                                  $l += length($b);
421                                    $md5->add($b);
422                          }                          }
423    
424                          $smb->close($fd);                          $smb->close($fd);
425                          close(F);                          close(F);
426    
427                            $file_md5{$f} = $md5->hexdigest;
428    
429                          # FIX: this fails with -T                          # FIX: this fails with -T
430                          my ($a,$m) = ($smb->stat("$from/$f"))[10,11];                          my ($a,$m) = ($smb->stat("$from/$f"))[10,11];
431                          utime $a, $m, "$to/$f" ||                          utime $a, $m, "$to/$f" ||
# Line 417  print STDERR "ignore: ",join("|",@ignore Line 479  print STDERR "ignore: ",join("|",@ignore
479          xlog($share,"$transfer bytes transfered...");          xlog($share,"$transfer bytes transfered...");
480    
481          foreach (@ln_files) {          foreach (@ln_files) {
482                  link "$bl/$_","$bc/$_" || warn "link $bl/$_ -> $bc/$_: $!\n";                  link "$real_bl/$_","$bc/$_" || warn "link $real_bl/$_ -> $bc/$_: $!\n";
483          }          }
484    
485          # remove files          # remove files
# Line 430  print STDERR "ignore: ",join("|",@ignore Line 492  print STDERR "ignore: ",join("|",@ignore
492                  rmdir "$bc/$_" || warn "rmdir $_: $!\n";                  rmdir "$bc/$_" || warn "rmdir $_: $!\n";
493          }          }
494    
495            # remove old .md5sum
496          # FIX: create .md5sum          foreach (sort @dirs) {
497                    unlink "$bc/$_/.md5sum" if (-e "$bc/$_/.md5sum");
498            }
499    
500            # create .md5sum
501            my $last_dir = '';
502            my $md5;
503            foreach my $f (sort { $file_md5{$a} cmp $file_md5{$b} } keys %file_md5) {
504                    my $dir = dirname($f);
505                    my $file = basename($f);
506    #print "$f -- $dir / $file<--\n";
507                    if ($dir ne $last_dir) {
508                            close($md5) if ($md5);
509                            open($md5, ">> $bc/$dir/.md5sum") || warn "can't create $bc/$dir/.md5sum: $!";
510                            $last_dir = $dir;
511    #print STDERR "writing $last_dir/.md5sum\n";
512                    }
513                    print $md5 $file_md5{$f},"  $file\n";
514            }
515            close($md5);
516    
517          # create leatest link          # create leatest link
518    #print "ln -s $bc $real_bl\n";
519            if (-l $bl) {
520                    unlink $bl || warn "can't remove old latest symlink $bl: $!\n";
521            }
522          symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";          symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";
523    
524            # FIX: sanity check -- remove for speedup
525            xlog($share,"failed to create latest symlink $bl -> $bc...") if (readlink($bl) ne $bc || ! -l $bl);
526    
527          xlog($share,"backup completed...");          xlog($share,"backup completed...");
528  }  }
529    
# Line 493  transferring whole file over network (ju Line 581  transferring whole file over network (ju
581  over network)  over network)
582    
583  =item - usage of C<.md5sum> files (compatible with command-line utility  =item - usage of C<.md5sum> files (compatible with command-line utility
584  C<md5sum> to keep file between snapshots hard-linked  C<md5sum>) to keep file between snapshots hard-linked
585    
586  =back  =back
587    
# Line 501  C<md5sum> to keep file between snapshots Line 589  C<md5sum> to keep file between snapshots
589    
590  This section is not yet written.  This section is not yet written.
591    
592  =head1 BUGS and LIMITATIONS  =head1 HACKS, TRICKS, BUGS and LIMITATIONS
593    
594    This chapter will have all content that doesn't fit anywhere else.
595    
596    =head2 Can snapshots be more frequent than daily?
597    
598  There is not real reason why you can't take snapshot more often than  There is not real reason why you can't take snapshot more often than
599  one a day. Actually, if you are using B<psinib> to backup Windows workstations  once a day. Actually, if you are using B<psinib> to backup Windows
600  they tend to come-and-go, so running B<psinib> several times a day  workstations you already know that they tend to come-and-go during the day
601  increases your chance of having up-to-date backup (B<psinib> will not  (reboots probably ;-), so running B<psinib> several times a day increases
602  make multiple backups for same day if such backup already exists).  your chance of having up-to-date backup (B<psinib> will not make multiple
603    snapshots for same day, nor will it update snapshot for current day if
604    it already exists).
605    
606  However, changing that to produce backups which are, for example, hourly  However, changing B<psinib> to produce snapshots which are, for example, hourly
607  is a simple change of C<$DIR_TIME_FMT> which is currently set to  is a simple change of C<$DIR_TIME_FMT> which is currently set to
608  C<'%Y%m%d'> (see I<strftime> documentation for explanation of that  C<'%Y%m%d'> (see I<strftime> documentation for explanation of that
609  format). If you change that to C<'%Y%m%d-%H> you can have hourly snapshots  format). If you change that to C<'%Y%m%d-%H> you can have hourly snapshots
# Line 517  format). If you change that to C<'%Y%m%d Line 611  format). If you change that to C<'%Y%m%d
611  program will sound strange, but other than that it should work.  program will sound strange, but other than that it should work.
612  I<You have been warned>.  I<You have been warned>.
613    
614    =head2 Do I really need to share every directory which I want to snapshot?
615    
616    Actually, no. Due to usage of C<Filesys::SmbClient> module, you can also
617    specify sub-directory inside your share that you want to backup. This feature
618    is most useful if you want to use administrative shares (but, have in mind
619    that you have to enter your Win administrator password in unencrypted file on
620    disk to do that) like this:
621    
622            smbmount //server/c$/WinNT/fonts  /mnt  -o username=administrator%win  
623    
624    After that you will get directories with snapshots like:
625    
626            server/c_WinNT_fonts/yyyymmdd/....
627    
628    =head2 Won't I run out of disk space?
629    
630    Of course you will... Snapshots and logfiles will eventually fill-up your disk.
631    However, you can do two things to stop that:
632    
633    =head3 Clean snapshort older than x days
634    
635    You can add following command to your C<root> crontab:
636    
637            find /backup/isis_backup -type d -mindepth 3 -maxdepth 3 -mtime +11 -exec rm -Rf {} \;
638    
639    I assume that C</backup/isis_backup> is directory in which are your snapshots
640    and that you don't want to keep snapshots older than 11 days (that's
641    C<-mtime +11> part of command).
642    
643    =head3 Rotate your logs
644    
645    I will leave that to you. I relay on GNU/Debian's C<logrotate> to do it for me.
646    
647    =head2 What are I<YYYYMMDD.partial> directories?
648    
649    If there isn't I<latest> symlink in snapshot directory, it's preatty safe to
650    assume that previous backup from that day failed. So, that directory will
651    be renamed to I<YYYYMMDD.partial> and snapshot will be performed again,
652    linking same files (other alternative would be to erase that dir and find
653    second-oldest directory, but this seemed like more correct approach).
654    
655  =head1 AUTHOR  =head1 AUTHOR
656    
657  Dobrica Pavlinusic <dpavlin@rot13.org>  Dobrica Pavlinusic <dpavlin@rot13.org>
Legend:
Removed from v.1.3  
changed lines
  Added in v.1.10
  ViewVC Help
Powered by ViewVC 1.1.26