/[psinib]/psinib.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /psinib.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.2 by dpavlin, Sat Jan 4 12:14:54 2003 UTC revision 1.8 by dpavlin, Tue Mar 4 21:08:43 2003 UTC
# Line 12  Line 12 
12  #  #
13  #  #
14  # usage:  # usage:
15  #       $ backup.pl mountscript  #       $ psinib.pl mountscript
16    
17  use strict 'vars';  use strict 'vars';
18  use Data::Dumper;  use Data::Dumper;
# Line 22  use List::Compare; Line 22  use List::Compare;
22  use Filesys::SmbClient;  use Filesys::SmbClient;
23  #use Taint;  #use Taint;
24  use Fcntl qw(LOCK_EX LOCK_NB);  use Fcntl qw(LOCK_EX LOCK_NB);
25    use Digest::MD5;
26    use File::Basename;
27    
28  # configuration  # configuration
29  my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile  my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile
30  my $DIR_TIME_FMT = '%Y%m%d';            # strftime format for backup dir  my $DIR_TIME_FMT = '%Y%m%d';            # strftime format for backup dir
31    
32  my $LOG = '/var/log/backup.log';        # add path here...  my $LOG = '/var/log/backup.log';        # add path here...
33  $LOG = '/tmp/backup.log';  #$LOG = '/tmp/backup.log';
34    
35  # store backups in which directory  # store backups in which directory
36  my $BACKUP_DEST = '/data/isis_backup';  my $BACKUP_DEST = '/backup/isis_backup';
37    
38  # files to ignore in backup  # files to ignore in backup
39  my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');  my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');
40    
41  # open log  # open log
42  open(L, "> $LOG") || die "can't open log $LOG: $!";  open(L, ">> $LOG") || die "can't open log $LOG: $!";
43  select((select(L), $|=1)[0]);   # flush output  select((select(L), $|=1)[0]);   # flush output
44    
45  # make a lock on logfile  # make a lock on logfile
# Line 62  my $mounts = shift @ARGV || Line 64  my $mounts = shift @ARGV ||
64    
65  my @in_backup;  # shares which are backeduped this run  my @in_backup;  # shares which are backeduped this run
66    
67  my $p = new Net::Ping->new();  my $p = new Net::Ping->new("tcp", 2);
68    # ping will try tcp connect to netbios-ssn (139)
69    $p->{port_num} = getservbyname("netbios-ssn", "tcp");
70    
71  my $backup_ok = 0;  my $backup_ok = 0;
72    
73  my $smb;  my $smb;
74  my %smb_atime;  my %smb_atime;
75  my %smb_mtime;  my %smb_mtime;
76    my %file_md5;
77    
78  open(M, $mounts) || die "can't open $mounts: $!";  open(M, $mounts) || die "can't open $mounts: $!";
79  while(<M>) {  while(<M>) {
# Line 97  while(<M>) { Line 102  while(<M>) {
102    
103          push @in_backup,$share;          push @in_backup,$share;
104    
105    
106            my ($host,$dir,$date_dir) = share2host_dir($share);
107            my $bl = "$BACKUP_DEST/$host/$dir/latest";      # latest backup
108            my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";   # current one
109            my $real_bl;
110            if (-e $bl) {
111                    $real_bl=readlink($bl) || die "can't read link $bl: $!";
112                    $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
113                    if (-e $bc && $real_bl eq $bc) {
114                            print "$share allready backuped...\n";
115                            $backup_ok++;
116                            next;
117                    }
118    
119            }
120    
121    
122          print "working on $share\n";          print "working on $share\n";
123    
124    
125          my $ip = get_ip($share);          my $ip = get_ip($share);
126    
127          if ($ip) {          if ($ip) {
# Line 117  xlog("","$backup_ok backups completed of Line 140  xlog("","$backup_ok backups completed of
140    
141  #-------------------------------------------------------------------------  #-------------------------------------------------------------------------
142    
143    
144  # get IP number from share  # get IP number from share
145  sub get_ip {  sub get_ip {
146          my $share = shift;          my $share = shift;
# Line 129  sub get_ip { Line 153  sub get_ip {
153          }          }
154  }  }
155    
156    
157    # write entry to screen and log
158  sub xlog {  sub xlog {
159          my $share = shift;          my $share = shift;
160          my $t = strftime $LOG_TIME_FMT, localtime;          my $t = strftime $LOG_TIME_FMT, localtime;
# Line 137  sub xlog { Line 163  sub xlog {
163          print L "$t $share\t$m\n";          print L "$t $share\t$m\n";
164  }  }
165    
166  sub snap_share {  # dump warn and dies into log
167    BEGIN { $SIG{'__WARN__'} = sub { xlog('WARN',$_[0]) ; warn $_[0] } }
168          my $share = shift;  BEGIN { $SIG{'__DIE__'} = sub { xlog('DIE',$_[0]) ; die $_[0] } }
169    
         my %param = ( debug => 0 );  
   
         $param{username} = shift;  
         $param{password} = shift;  
         $param{workgroup} = shift;  
170    
171    # split share name to host, dir and currnet date dir
172    sub share2host_dir {
173            my $share = shift;
174          my ($host,$dir);          my ($host,$dir);
175          if ($share =~ m#//([^/]+)/(.+)$#) {          if ($share =~ m#//([^/]+)/(.+)$#) {
176                  ($host,$dir) = ($1,$2);                  ($host,$dir) = ($1,$2);
# Line 157  sub snap_share { Line 181  sub snap_share {
181                  print "Can't parse share $share into host and directory!\n";                  print "Can't parse share $share into host and directory!\n";
182                  return;                  return;
183          }          }
184            return ($host,$dir,strftime $DIR_TIME_FMT, localtime);
185    }
186    
187          my $date_dir = strftime $DIR_TIME_FMT, localtime;  
188    # make a snapshot of a share
189    sub snap_share {
190    
191            my $share = shift;
192    
193            my %param = ( debug => 0 );
194    
195            $param{username} = shift || warn "can't find username for share $share";
196            $param{password} = shift || warn "can't find passwod for share $share";
197            $param{workgroup} = shift || warn "can't find workgroup for share $share";
198    
199            my ($host,$dir,$date_dir) = share2host_dir($share);
200    
201          # latest backup directory          # latest backup directory
202          my $bl = "$BACKUP_DEST/$host/$dir/latest";          my $bl = "$BACKUP_DEST/$host/$dir/latest";
# Line 170  sub snap_share { Line 208  sub snap_share {
208                  $real_bl=readlink($bl) || die "can't read link $bl: $!";                  $real_bl=readlink($bl) || die "can't read link $bl: $!";
209                  $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");                  $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
210          } else {          } else {
211                  print "no old backup, this is first run...\n";                  print "no old backup, trying to find last backup, ";
212                    if (opendir(BL_DIR, "$BACKUP_DEST/$host/$dir")) {
213                            my @bl_dirs = sort grep { !/^\./ && -d "$BACKUP_DEST/$host/$dir/$_" } readdir(BL_DIR);
214                            closedir(BL_DIR);
215                            $real_bl=pop @bl_dirs;
216                            print "using $real_bl as latest...\n";
217                            $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
218                            if ($real_bl eq $bc) {
219                                    xlog($share,"latest from today (possible partial backup)");
220                                    rename $real_bl,$real_bl.".partial" || warn "can't reaname partial backup: $!";
221                                    $real_bl .= ".partial";
222                            }
223                    } else {
224                            print "this is first run...\n";
225                    }
226          }          }
227    
228          if (-e $bc && $real_bl && $real_bl eq $bc) {          if (-e $bc && $real_bl && $real_bl eq $bc) {
# Line 202  sub snap_share { Line 254  sub snap_share {
254          my %file_size;          my %file_size;
255          my %file_atime;          my %file_atime;
256          my %file_mtime;          my %file_mtime;
257          my %file_md5;          #my %file_md5;
258    
259          my @smb_files;          my @smb_files;
260          my %smb_size;          my %smb_size;
261          #my %smb_atime;          #my %smb_atime;
262          #my %smb_mtime;          #my %smb_mtime;
         my %smb_md5;  
   
263    
264          sub norm_dir {          sub norm_dir {
265                  my $foo = shift;                  my $foo = shift;
# Line 225  sub snap_share { Line 275  sub snap_share {
275          my $di = 0;          my $di = 0;
276          while ($di <= $#dirs && $real_bl) {          while ($di <= $#dirs && $real_bl) {
277                  my $d=$dirs[$di++];                  my $d=$dirs[$di++];
278                  opendir(DIR,"$bl/$d") || warn "opendir($bl/$d): $!\n";                  opendir(DIR,"$real_bl/$d") || warn "opendir($real_bl/$d): $!\n";
279    
280                  # read .backupignore if exists                  # read .backupignore if exists
281                  if (-f "$bl/$d/.backupignore") {                  if (-f "$real_bl/$d/.backupignore") {
282                          open(I,"$bl/$d/.backupignore");                          open(I,"$real_bl/$d/.backupignore");
283                          while(<I>) {                          while(<I>) {
284                                  chomp;                                  chomp;
285                                  push @ignore,norm_dir("$d/$_");                                  push @ignore,norm_dir("$d/$_");
286                          }                          }
287                          close(I);                          close(I);
288  print STDERR "ignore: ",join("|",@ignore),"\n";  print STDERR "ignore: ",join("|",@ignore),"\n";
289                          link "$bl/$d/.backupignore","$bc/$d/.backupignore" ||                          link "$real_bl/$d/.backupignore","$bc/$d/.backupignore" ||
290                                  warn "can't copy $bl/$d/.backupignore to current backup dir: $!\n";                                  warn "can't copy $real_bl/$d/.backupignore to current backup dir: $!\n";
291                  }                  }
292    
293                  # read .md5sum if exists                  # read .md5sum if exists
294                  if (-f "$bl/$d/.md5sum") {                  if (-f "$real_bl/$d/.md5sum") {
295                          open(I,"$bl/$d/.md5sum");                          open(I,"$real_bl/$d/.md5sum");
296                          while(<I>) {                          while(<I>) {
297                                  chomp;                                  chomp;
298                                  my ($md5,$f) = split(/\s+/,$_,2);                                  my ($md5,$f) = split(/\s+/,$_,2);
# Line 256  print STDERR "ignore: ",join("|",@ignore Line 306  print STDERR "ignore: ",join("|",@ignore
306                          next if ($f eq '.');                          next if ($f eq '.');
307                          next if ($f eq '..');                          next if ($f eq '..');
308                          my $pr = norm_dir("$d/$f");     # path relative                          my $pr = norm_dir("$d/$f");     # path relative
309                          my $pf = norm_dir("$d/$f","$bl/");      # path full                          my $pf = norm_dir("$d/$f","$real_bl/"); # path full
310                          if (grep(/^\Q$pr\E$/,@ignore) == 0) {                          if (grep(/^\Q$pr\E$/,@ignore) == 0) {
311                                  if (-f $pf) {                                  if (-f $pf) {
312                                          push @files,$pr;                                          push @files,$pr;
# Line 350  print STDERR "ignore: ",join("|",@ignore Line 400  print STDERR "ignore: ",join("|",@ignore
400                                  next;                                  next;
401                          }                          }
402    
403                            my $md5 = Digest::MD5->new;
404    
405                          my $fd = $smb->open("$from/$f");                          my $fd = $smb->open("$from/$f");
406                          if (! $fd) {                          if (! $fd) {
407                                  print STDERR "can't open smb file $from/$f: $!\n";                                  print STDERR "can't open smb file $from/$f: $!\n";
# Line 359  print STDERR "ignore: ",join("|",@ignore Line 411  print STDERR "ignore: ",join("|",@ignore
411                          while (defined(my $b=$smb->read($fd,4096))) {                          while (defined(my $b=$smb->read($fd,4096))) {
412                                  print F $b;                                  print F $b;
413                                  $l += length($b);                                  $l += length($b);
414                                    $md5->add($b);
415                          }                          }
416    
417                          $smb->close($fd);                          $smb->close($fd);
418                          close(F);                          close(F);
419    
420                            $file_md5{$f} = $md5->hexdigest;
421    
422                          # FIX: this fails with -T                          # FIX: this fails with -T
423                          my ($a,$m) = ($smb->stat("$from/$f"))[10,11];                          my ($a,$m) = ($smb->stat("$from/$f"))[10,11];
424                          utime $a, $m, "$to/$f" ||                          utime $a, $m, "$to/$f" ||
# Line 417  print STDERR "ignore: ",join("|",@ignore Line 472  print STDERR "ignore: ",join("|",@ignore
472          xlog($share,"$transfer bytes transfered...");          xlog($share,"$transfer bytes transfered...");
473    
474          foreach (@ln_files) {          foreach (@ln_files) {
475                  link "$bl/$_","$bc/$_" || warn "link $bl/$_ -> $bc/$_: $!\n";                  link "$real_bl/$_","$bc/$_" || warn "link $real_bl/$_ -> $bc/$_: $!\n";
476          }          }
477    
478          # remove files          # remove files
# Line 430  print STDERR "ignore: ",join("|",@ignore Line 485  print STDERR "ignore: ",join("|",@ignore
485                  rmdir "$bc/$_" || warn "rmdir $_: $!\n";                  rmdir "$bc/$_" || warn "rmdir $_: $!\n";
486          }          }
487    
488            # remove old .md5sum
489          # FIX: create .md5sum          foreach (sort @dirs) {
490                    unlink "$bc/$_/.md5sum" if (-e "$bc/$_/.md5sum");
491            }
492    
493            # create .md5sum
494            my $last_dir = '';
495            my $md5;
496            foreach my $f (sort { $file_md5{$a} cmp $file_md5{$b} } keys %file_md5) {
497                    my $dir = dirname($f);
498                    my $file = basename($f);
499    #print "$f -- $dir / $file<--\n";
500                    if ($dir ne $last_dir) {
501                            close($md5) if ($md5);
502                            open($md5, ">> $bc/$dir/.md5sum") || warn "can't create $bc/$dir/.md5sum: $!";
503                            $last_dir = $dir;
504    #print STDERR "writing $last_dir/.md5sum\n";
505                    }
506                    print $md5 $file_md5{$f},"  $file\n";
507            }
508            close($md5);
509    
510          # create leatest link          # create leatest link
511    #print "ln -s $bc $real_bl\n";
512            if (-e $bl) {
513                    unlink $bl || warn "can't remove old latest symlink $bl: $!\n";
514            }
515          symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";          symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";
516    
517            # FIX: sanity check -- remove for speedup
518            xlog($share,"failed to create latest symlink...") if (readlink($bl) ne $bc || ! -e $bl);
519    
520          xlog($share,"backup completed...");          xlog($share,"backup completed...");
521  }  }
522    
523    __END__
524  #-------------------------------------------------------------------------  #-------------------------------------------------------------------------
525    
526    
527    =head1 NAME
528    
529    psinib - Perl Snapshot Is Not Incremental Backup
530    
531    =head1 SYNOPSIS
532    
533    ./psinib.pl
534    
535    =head1 DESCRIPTION
536    
537    This script in current version support just backup of Samba (or Micro$oft
538    Winblowz) shares to central disk space. Central disk space is organized in
539    multiple directories named after:
540    
541    =over 4
542    
543    =item *
544    server which is sharing files to be backed up
545    
546    =item *
547    name of share on server
548    
549    =item *
550    dated directory named like standard ISO date format (YYYYMMDD).
551    
552    =back
553    
554    In each dated directory you will find I<snapshot> of all files on
555    exported share on that particular date.
556    
557    You can also use symlink I<latest> which will lead you to
558    last completed backup. After that you can use some other backup
559    software to transfer I<snapshot> to tape, CD-ROM or some other media.
560    
561    =head2 Design considerations
562    
563    Since taking of share snapshot every day requires a lot of disk space and
564    network bandwidth, B<psinib> uses several techniques to keep disk usage and
565    network traffic at acceptable level:
566    
567    =over 3
568    
569    =item - usage of hard-links to provide same files in each snapshot (as opposed
570    to have multiple copies of same file)
571    
572    =item - usage of file size, atime and mtime to find changes of files without
573    transferring whole file over network (just share browsing is transfered
574    over network)
575    
576    =item - usage of C<.md5sum> files (compatible with command-line utility
577    C<md5sum>) to keep file between snapshots hard-linked
578    
579    =back
580    
581    =head1 CONFIGURATION
582    
583    This section is not yet written.
584    
585    =head1 HACKS, TRICKS, BUGS and LIMITATIONS
586    
587    This chapter will have all content that doesn't fit anywhere else.
588    
589    =head2 Can snapshots be more frequent than daily?
590    
591    There is not real reason why you can't take snapshot more often than
592    once a day. Actually, if you are using B<psinib> to backup Windows
593    workstations you already know that they tend to come-and-go during the day
594    (reboots probably ;-), so running B<psinib> several times a day increases
595    your chance of having up-to-date backup (B<psinib> will not make multiple
596    snapshots for same day, nor will it update snapshot for current day if
597    it already exists).
598    
599    However, changing B<psinib> to produce snapshots which are, for example, hourly
600    is a simple change of C<$DIR_TIME_FMT> which is currently set to
601    C<'%Y%m%d'> (see I<strftime> documentation for explanation of that
602    format). If you change that to C<'%Y%m%d-%H> you can have hourly snapshots
603    (if your network is fast enough, that is...). Also, some of messages in
604    program will sound strange, but other than that it should work.
605    I<You have been warned>.
606    
607    =head2 Do I really need to share every directory which I want to snapshot?
608    
609    Actually, no. Due to usage of C<Filesys::SmbClient> module, you can also
610    specify sub-directory inside your share that you want to backup. This feature
611    is most useful if you want to use administrative shares (but, have in mind
612    that you have to enter your Win administrator password in unencrypted file on
613    disk to do that) like this:
614    
615            smbmount //server/c$/WinNT/fonts  /mnt  -o username=administrator%win  
616    
617    After that you will get directories with snapshots like:
618    
619            server/c_WinNT_fonts/yyyymmdd/....
620    
621    =head2 Won't I run out of disk space?
622    
623    Of course you will... Snapshots and logfiles will eventually fill-up your disk.
624    However, you can do two things to stop that:
625    
626    =head3 Clean snapshort older than x days
627    
628    You can add following command to your C<root> crontab:
629    
630            find /backup/isis_backup -type d -mindepth 3 -maxdepth 3 -mtime +11 -exec rm -Rf {} \;
631    
632    I assume that C</backup/isis_backup> is directory in which are your snapshots
633    and that you don't want to keep snapshots older than 11 days (that's
634    C<-mtime +11> part of command).
635    
636    =head3 Rotate your logs
637    
638    I will leave that to you. I relay on GNU/Debian's C<logrotate> to do it for me.
639    
640    =head2 What are I<YYYYMMDD.partial> directories?
641    
642    If there isn't I<latest> symlink in snapshot directory, it's preatty safe to
643    assume that previous backup from that day failed. So, that directory will
644    be renamed to I<YYYYMMDD.partial> and snapshot will be performed again,
645    linking same files (other alternative would be to erase that dir and find
646    second-oldest directory, but this seemed like more correct approach).
647    
648    =head1 AUTHOR
649    
650    Dobrica Pavlinusic <dpavlin@rot13.org>
651    
652    L<http://www.rot13.org/~dpavlin/>
653    
654    =head1 LICENSE
655    
656    This product is licensed under GNU Public License (GPL) v2 or later.
657    
658    =cut
Legend:
Removed from v.1.2  
changed lines
  Added in v.1.8
  ViewVC Help
Powered by ViewVC 1.1.26