/[psinib]/psinib.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /psinib.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.2 by dpavlin, Sat Jan 4 12:14:54 2003 UTC revision 1.10 by dpavlin, Tue Jul 15 17:41:45 2003 UTC
# Line 12  Line 12 
12  #  #
13  #  #
14  # usage:  # usage:
15  #       $ backup.pl mountscript  #       $ psinib.pl mountscript
16    
17  use strict 'vars';  use strict 'vars';
18  use Data::Dumper;  use Data::Dumper;
# Line 22  use List::Compare; Line 22  use List::Compare;
22  use Filesys::SmbClient;  use Filesys::SmbClient;
23  #use Taint;  #use Taint;
24  use Fcntl qw(LOCK_EX LOCK_NB);  use Fcntl qw(LOCK_EX LOCK_NB);
25    use Digest::MD5;
26    use File::Basename;
27    
28  # configuration  # configuration
29  my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile  my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile
30  my $DIR_TIME_FMT = '%Y%m%d';            # strftime format for backup dir  my $DIR_TIME_FMT = '%Y%m%d';            # strftime format for backup dir
31    
32  my $LOG = '/var/log/backup.log';        # add path here...  my $LOG = '/var/log/backup.log';        # add path here...
33  $LOG = '/tmp/backup.log';  #$LOG = '/tmp/backup.log';
34    
35  # store backups in which directory  # store backups in which directory
36  my $BACKUP_DEST = '/data/isis_backup';  my $BACKUP_DEST = '/backup/isis_backup';
37    
38  # files to ignore in backup  # files to ignore in backup
39  my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');  my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');
40    
41  # open log  # open log
42  open(L, "> $LOG") || die "can't open log $LOG: $!";  open(L, ">> $LOG") || die "can't open log $LOG: $!";
43  select((select(L), $|=1)[0]);   # flush output  select((select(L), $|=1)[0]);   # flush output
44    
45  # make a lock on logfile  # make a lock on logfile
# Line 62  my $mounts = shift @ARGV || Line 64  my $mounts = shift @ARGV ||
64    
65  my @in_backup;  # shares which are backeduped this run  my @in_backup;  # shares which are backeduped this run
66    
67  my $p = new Net::Ping->new();  my $p = new Net::Ping->new("tcp", 2);
68    # ping will try tcp connect to netbios-ssn (139)
69    $p->{port_num} = getservbyname("netbios-ssn", "tcp");
70    
71  my $backup_ok = 0;  my $backup_ok = 0;
72    
73  my $smb;  my $smb;
74  my %smb_atime;  my %smb_atime;
75  my %smb_mtime;  my %smb_mtime;
76    my %file_md5;
77    
78  open(M, $mounts) || die "can't open $mounts: $!";  open(M, $mounts) || die "can't open $mounts: $!";
79  while(<M>) {  while(<M>) {
# Line 97  while(<M>) { Line 102  while(<M>) {
102    
103          push @in_backup,$share;          push @in_backup,$share;
104    
105    
106            my ($host,$dir,$date_dir) = share2host_dir($share);
107            my $bl = "$BACKUP_DEST/$host/$dir/latest";      # latest backup
108            my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";   # current one
109            my $real_bl;
110            if (-l $bl) {
111                    $real_bl=readlink($bl) || die "can't read link $bl: $!";
112                    $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
113                    if (-l $bc && $real_bl eq $bc) {
114                            print "$share allready backuped...\n";
115                            $backup_ok++;
116                            next;
117                    }
118    
119            }
120    
121    
122          print "working on $share\n";          print "working on $share\n";
123    
124    
125          my $ip = get_ip($share);          my $ip = get_ip($share);
126    
127          if ($ip) {          if ($ip) {
# Line 117  xlog("","$backup_ok backups completed of Line 140  xlog("","$backup_ok backups completed of
140    
141  #-------------------------------------------------------------------------  #-------------------------------------------------------------------------
142    
143    
144  # get IP number from share  # get IP number from share
145  sub get_ip {  sub get_ip {
146          my $share = shift;          my $share = shift;
# Line 129  sub get_ip { Line 153  sub get_ip {
153          }          }
154  }  }
155    
156    
157    # write entry to screen and log
158  sub xlog {  sub xlog {
159          my $share = shift;          my $share = shift;
160          my $t = strftime $LOG_TIME_FMT, localtime;          my $t = strftime $LOG_TIME_FMT, localtime;
# Line 137  sub xlog { Line 163  sub xlog {
163          print L "$t $share\t$m\n";          print L "$t $share\t$m\n";
164  }  }
165    
166  sub snap_share {  # dump warn and dies into log
167    BEGIN { $SIG{'__WARN__'} = sub { xlog('WARN',$_[0]) ; warn $_[0] } }
168          my $share = shift;  BEGIN { $SIG{'__DIE__'} = sub { xlog('DIE',$_[0]) ; die $_[0] } }
169    
         my %param = ( debug => 0 );  
   
         $param{username} = shift;  
         $param{password} = shift;  
         $param{workgroup} = shift;  
170    
171    # split share name to host, dir and currnet date dir
172    sub share2host_dir {
173            my $share = shift;
174          my ($host,$dir);          my ($host,$dir);
175          if ($share =~ m#//([^/]+)/(.+)$#) {          if ($share =~ m#//([^/]+)/(.+)$#) {
176                  ($host,$dir) = ($1,$2);                  ($host,$dir) = ($1,$2);
# Line 157  sub snap_share { Line 181  sub snap_share {
181                  print "Can't parse share $share into host and directory!\n";                  print "Can't parse share $share into host and directory!\n";
182                  return;                  return;
183          }          }
184            return ($host,$dir,strftime $DIR_TIME_FMT, localtime);
185    }
186    
187    
188    # make a snapshot of a share
189    sub snap_share {
190    
191            my $share = shift;
192    
193            my %param = ( debug => 0 );
194    
195            $param{username} = shift || warn "can't find username for share $share";
196            $param{password} = shift || warn "can't find passwod for share $share";
197            $param{workgroup} = shift || warn "can't find workgroup for share $share";
198    
199          my $date_dir = strftime $DIR_TIME_FMT, localtime;          my ($host,$dir,$date_dir) = share2host_dir($share);
200    
201          # latest backup directory          # latest backup directory
202          my $bl = "$BACKUP_DEST/$host/$dir/latest";          my $bl = "$BACKUP_DEST/$host/$dir/latest";
# Line 166  sub snap_share { Line 204  sub snap_share {
204          my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";          my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";
205    
206          my $real_bl;          my $real_bl;
207          if (-e $bl) {          if (-l $bl) {
208                  $real_bl=readlink($bl) || die "can't read link $bl: $!";                  $real_bl=readlink($bl) || die "can't read link $bl: $!";
209                  $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");                  $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
210          } else {          } else {
211                  print "no old backup, this is first run...\n";                  print "no old backup, trying to find last backup, ";
212                    if (opendir(BL_DIR, "$BACKUP_DEST/$host/$dir")) {
213                            my @bl_dirs = sort grep { !/^\./ && -d "$BACKUP_DEST/$host/$dir/$_" } readdir(BL_DIR);
214                            closedir(BL_DIR);
215                            $real_bl=pop @bl_dirs;
216                            print "using $real_bl as latest...\n";
217                            $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
218                            if ($real_bl eq $bc) {
219                                    xlog($share,"latest from today (possible partial backup)");
220                                    rename $real_bl,$real_bl.".partial" || warn "can't reaname partial backup: $!";
221                                    $real_bl .= ".partial";
222                            }
223                    } else {
224                            print "this is first run...\n";
225                    }
226          }          }
227    
228          if (-e $bc && $real_bl && $real_bl eq $bc) {          if (-l $bc && $real_bl && $real_bl eq $bc) {
229                  print "$share allready backuped...\n";                  print "$share allready backuped...\n";
230                  return;                  return;
231          }          }
# Line 202  sub snap_share { Line 254  sub snap_share {
254          my %file_size;          my %file_size;
255          my %file_atime;          my %file_atime;
256          my %file_mtime;          my %file_mtime;
257          my %file_md5;          #my %file_md5;
258    
259          my @smb_files;          my @smb_files;
260          my %smb_size;          my %smb_size;
261          #my %smb_atime;          #my %smb_atime;
262          #my %smb_mtime;          #my %smb_mtime;
         my %smb_md5;  
   
263    
264          sub norm_dir {          sub norm_dir {
265                  my $foo = shift;                  my $foo = shift;
# Line 225  sub snap_share { Line 275  sub snap_share {
275          my $di = 0;          my $di = 0;
276          while ($di <= $#dirs && $real_bl) {          while ($di <= $#dirs && $real_bl) {
277                  my $d=$dirs[$di++];                  my $d=$dirs[$di++];
278                  opendir(DIR,"$bl/$d") || warn "opendir($bl/$d): $!\n";                  opendir(DIR,"$real_bl/$d") || warn "opendir($real_bl/$d): $!\n";
279    
280                  # read .backupignore if exists                  # read .backupignore if exists
281                  if (-f "$bl/$d/.backupignore") {                  if (-f "$real_bl/$d/.backupignore") {
282                          open(I,"$bl/$d/.backupignore");                          open(I,"$real_bl/$d/.backupignore");
283                          while(<I>) {                          while(<I>) {
284                                  chomp;                                  chomp;
285                                  push @ignore,norm_dir("$d/$_");                                  push @ignore,norm_dir("$d/$_");
286                          }                          }
287                          close(I);                          close(I);
288  print STDERR "ignore: ",join("|",@ignore),"\n";  #print STDERR "ignore: ",join("|",@ignore),"\n";
289                          link "$bl/$d/.backupignore","$bc/$d/.backupignore" ||                          link "$real_bl/$d/.backupignore","$bc/$d/.backupignore" ||
290                                  warn "can't copy $bl/$d/.backupignore to current backup dir: $!\n";                                  warn "can't copy $real_bl/$d/.backupignore to current backup dir: $!\n";
291                  }                  }
292    
293                  # read .md5sum if exists                  # read .md5sum if exists
294                  if (-f "$bl/$d/.md5sum") {                  if (-f "$real_bl/$d/.md5sum") {
295                          open(I,"$bl/$d/.md5sum");                          open(I,"$real_bl/$d/.md5sum");
296                          while(<I>) {                          while(<I>) {
297                                  chomp;                                  chomp;
298                                  my ($md5,$f) = split(/\s+/,$_,2);                                  my ($md5,$f) = split(/\s+/,$_,2);
# Line 256  print STDERR "ignore: ",join("|",@ignore Line 306  print STDERR "ignore: ",join("|",@ignore
306                          next if ($f eq '.');                          next if ($f eq '.');
307                          next if ($f eq '..');                          next if ($f eq '..');
308                          my $pr = norm_dir("$d/$f");     # path relative                          my $pr = norm_dir("$d/$f");     # path relative
309                          my $pf = norm_dir("$d/$f","$bl/");      # path full                          my $pf = norm_dir("$d/$f","$real_bl/"); # path full
310                          if (grep(/^\Q$pr\E$/,@ignore) == 0) {                          if (grep(/^\Q$pr\E$/,@ignore) == 0) {
311                                  if (-f $pf) {                                  if (-f $pf) {
312                                          push @files,$pr;                                          push @files,$pr;
# Line 285  print STDERR "ignore: ",join("|",@ignore Line 335  print STDERR "ignore: ",join("|",@ignore
335    
336          $di = 0;          $di = 0;
337          while ($di <= $#smb_dirs) {          while ($di <= $#smb_dirs) {
338                  my $d=$smb_dirs[$di++];                  my $d=$smb_dirs[$di];
339                  my $pf = norm_dir($d,"smb:$share/");    # path full                  my $pf = norm_dir($d,"smb:$share/");    # path full
340                  my $D = $smb->opendir($pf) || warn "smb->opendir($pf): $!\n";                  my $D = $smb->opendir($pf);
341                    if (! $D) {
342                            xlog($share,"FATAL: $share: $!");
343                            # remove failing dir
344                            delete $smb_dirs[$di];
345                            next;
346                    }
347                    $di++;
348    
349                  my @clutter = $smb->readdir_struct($D);                  my @clutter = $smb->readdir_struct($D);
350                  foreach my $item (@clutter) {                  foreach my $item (@clutter) {
# Line 350  print STDERR "ignore: ",join("|",@ignore Line 407  print STDERR "ignore: ",join("|",@ignore
407                                  next;                                  next;
408                          }                          }
409    
410                            my $md5 = Digest::MD5->new;
411    
412                          my $fd = $smb->open("$from/$f");                          my $fd = $smb->open("$from/$f");
413                          if (! $fd) {                          if (! $fd) {
414                                  print STDERR "can't open smb file $from/$f: $!\n";                                  print STDERR "can't open smb file $from/$f: $!\n";
# Line 359  print STDERR "ignore: ",join("|",@ignore Line 418  print STDERR "ignore: ",join("|",@ignore
418                          while (defined(my $b=$smb->read($fd,4096))) {                          while (defined(my $b=$smb->read($fd,4096))) {
419                                  print F $b;                                  print F $b;
420                                  $l += length($b);                                  $l += length($b);
421                                    $md5->add($b);
422                          }                          }
423    
424                          $smb->close($fd);                          $smb->close($fd);
425                          close(F);                          close(F);
426    
427                            $file_md5{$f} = $md5->hexdigest;
428    
429                          # FIX: this fails with -T                          # FIX: this fails with -T
430                          my ($a,$m) = ($smb->stat("$from/$f"))[10,11];                          my ($a,$m) = ($smb->stat("$from/$f"))[10,11];
431                          utime $a, $m, "$to/$f" ||                          utime $a, $m, "$to/$f" ||
# Line 417  print STDERR "ignore: ",join("|",@ignore Line 479  print STDERR "ignore: ",join("|",@ignore
479          xlog($share,"$transfer bytes transfered...");          xlog($share,"$transfer bytes transfered...");
480    
481          foreach (@ln_files) {          foreach (@ln_files) {
482                  link "$bl/$_","$bc/$_" || warn "link $bl/$_ -> $bc/$_: $!\n";                  link "$real_bl/$_","$bc/$_" || warn "link $real_bl/$_ -> $bc/$_: $!\n";
483          }          }
484    
485          # remove files          # remove files
# Line 430  print STDERR "ignore: ",join("|",@ignore Line 492  print STDERR "ignore: ",join("|",@ignore
492                  rmdir "$bc/$_" || warn "rmdir $_: $!\n";                  rmdir "$bc/$_" || warn "rmdir $_: $!\n";
493          }          }
494    
495            # remove old .md5sum
496          # FIX: create .md5sum          foreach (sort @dirs) {
497                    unlink "$bc/$_/.md5sum" if (-e "$bc/$_/.md5sum");
498            }
499    
500            # create .md5sum
501            my $last_dir = '';
502            my $md5;
503            foreach my $f (sort { $file_md5{$a} cmp $file_md5{$b} } keys %file_md5) {
504                    my $dir = dirname($f);
505                    my $file = basename($f);
506    #print "$f -- $dir / $file<--\n";
507                    if ($dir ne $last_dir) {
508                            close($md5) if ($md5);
509                            open($md5, ">> $bc/$dir/.md5sum") || warn "can't create $bc/$dir/.md5sum: $!";
510                            $last_dir = $dir;
511    #print STDERR "writing $last_dir/.md5sum\n";
512                    }
513                    print $md5 $file_md5{$f},"  $file\n";
514            }
515            close($md5);
516    
517          # create leatest link          # create leatest link
518    #print "ln -s $bc $real_bl\n";
519            if (-l $bl) {
520                    unlink $bl || warn "can't remove old latest symlink $bl: $!\n";
521            }
522          symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";          symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";
523    
524            # FIX: sanity check -- remove for speedup
525            xlog($share,"failed to create latest symlink $bl -> $bc...") if (readlink($bl) ne $bc || ! -l $bl);
526    
527          xlog($share,"backup completed...");          xlog($share,"backup completed...");
528  }  }
529    
530    __END__
531  #-------------------------------------------------------------------------  #-------------------------------------------------------------------------
532    
533    
534    =head1 NAME
535    
536    psinib - Perl Snapshot Is Not Incremental Backup
537    
538    =head1 SYNOPSIS
539    
540    ./psinib.pl
541    
542    =head1 DESCRIPTION
543    
544    This script in current version support just backup of Samba (or Micro$oft
545    Winblowz) shares to central disk space. Central disk space is organized in
546    multiple directories named after:
547    
548    =over 4
549    
550    =item *
551    server which is sharing files to be backed up
552    
553    =item *
554    name of share on server
555    
556    =item *
557    dated directory named like standard ISO date format (YYYYMMDD).
558    
559    =back
560    
561    In each dated directory you will find I<snapshot> of all files on
562    exported share on that particular date.
563    
564    You can also use symlink I<latest> which will lead you to
565    last completed backup. After that you can use some other backup
566    software to transfer I<snapshot> to tape, CD-ROM or some other media.
567    
568    =head2 Design considerations
569    
570    Since taking of share snapshot every day requires a lot of disk space and
571    network bandwidth, B<psinib> uses several techniques to keep disk usage and
572    network traffic at acceptable level:
573    
574    =over 3
575    
576    =item - usage of hard-links to provide same files in each snapshot (as opposed
577    to have multiple copies of same file)
578    
579    =item - usage of file size, atime and mtime to find changes of files without
580    transferring whole file over network (just share browsing is transfered
581    over network)
582    
583    =item - usage of C<.md5sum> files (compatible with command-line utility
584    C<md5sum>) to keep file between snapshots hard-linked
585    
586    =back
587    
588    =head1 CONFIGURATION
589    
590    This section is not yet written.
591    
592    =head1 HACKS, TRICKS, BUGS and LIMITATIONS
593    
594    This chapter will have all content that doesn't fit anywhere else.
595    
596    =head2 Can snapshots be more frequent than daily?
597    
598    There is not real reason why you can't take snapshot more often than
599    once a day. Actually, if you are using B<psinib> to backup Windows
600    workstations you already know that they tend to come-and-go during the day
601    (reboots probably ;-), so running B<psinib> several times a day increases
602    your chance of having up-to-date backup (B<psinib> will not make multiple
603    snapshots for same day, nor will it update snapshot for current day if
604    it already exists).
605    
606    However, changing B<psinib> to produce snapshots which are, for example, hourly
607    is a simple change of C<$DIR_TIME_FMT> which is currently set to
608    C<'%Y%m%d'> (see I<strftime> documentation for explanation of that
609    format). If you change that to C<'%Y%m%d-%H> you can have hourly snapshots
610    (if your network is fast enough, that is...). Also, some of messages in
611    program will sound strange, but other than that it should work.
612    I<You have been warned>.
613    
614    =head2 Do I really need to share every directory which I want to snapshot?
615    
616    Actually, no. Due to usage of C<Filesys::SmbClient> module, you can also
617    specify sub-directory inside your share that you want to backup. This feature
618    is most useful if you want to use administrative shares (but, have in mind
619    that you have to enter your Win administrator password in unencrypted file on
620    disk to do that) like this:
621    
622            smbmount //server/c$/WinNT/fonts  /mnt  -o username=administrator%win  
623    
624    After that you will get directories with snapshots like:
625    
626            server/c_WinNT_fonts/yyyymmdd/....
627    
628    =head2 Won't I run out of disk space?
629    
630    Of course you will... Snapshots and logfiles will eventually fill-up your disk.
631    However, you can do two things to stop that:
632    
633    =head3 Clean snapshort older than x days
634    
635    You can add following command to your C<root> crontab:
636    
637            find /backup/isis_backup -type d -mindepth 3 -maxdepth 3 -mtime +11 -exec rm -Rf {} \;
638    
639    I assume that C</backup/isis_backup> is directory in which are your snapshots
640    and that you don't want to keep snapshots older than 11 days (that's
641    C<-mtime +11> part of command).
642    
643    =head3 Rotate your logs
644    
645    I will leave that to you. I relay on GNU/Debian's C<logrotate> to do it for me.
646    
647    =head2 What are I<YYYYMMDD.partial> directories?
648    
649    If there isn't I<latest> symlink in snapshot directory, it's preatty safe to
650    assume that previous backup from that day failed. So, that directory will
651    be renamed to I<YYYYMMDD.partial> and snapshot will be performed again,
652    linking same files (other alternative would be to erase that dir and find
653    second-oldest directory, but this seemed like more correct approach).
654    
655    =head1 AUTHOR
656    
657    Dobrica Pavlinusic <dpavlin@rot13.org>
658    
659    L<http://www.rot13.org/~dpavlin/>
660    
661    =head1 LICENSE
662    
663    This product is licensed under GNU Public License (GPL) v2 or later.
664    
665    =cut
Legend:
Removed from v.1.2  
changed lines
  Added in v.1.10
  ViewVC Help
Powered by ViewVC 1.1.26