/[psinib]/psinib.pl
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /psinib.pl

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.1.1.1 by dpavlin, Sat Jan 4 11:42:56 2003 UTC revision 1.11 by dpavlin, Sun Oct 12 15:58:28 2003 UTC
# Line 12  Line 12 
12  #  #
13  #  #
14  # usage:  # usage:
15  #       $ backup.pl mountscript  #       $ psinib.pl mountscript
16    
17  use strict 'vars';  use strict 'vars';
18  use Data::Dumper;  use Data::Dumper;
# Line 21  use POSIX qw(strftime); Line 21  use POSIX qw(strftime);
21  use List::Compare;  use List::Compare;
22  use Filesys::SmbClient;  use Filesys::SmbClient;
23  #use Taint;  #use Taint;
24    use Fcntl qw(LOCK_EX LOCK_NB);
25    use Digest::MD5;
26    use File::Basename;
27    
28  # configuration  # configuration
29  my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile  my $LOG_TIME_FMT = '%Y-%m-%d %H:%M:%S'; # strftime format for logfile
30  my $DIR_TIME_FMT = '%Y%m%d';            # strftime format for backup dir  my $DIR_TIME_FMT = '%Y%m%d';            # strftime format for backup dir
31    
32  my $LOG = '/var/log/backup.log';        # add path here...  my $LOG = '/var/log/backup.log';        # add path here...
33  $LOG = '/tmp/backup.log';  #$LOG = '/tmp/backup.log';
34    
35  # store backups in which directory  # store backups in which directory
36  my $BACKUP_DEST = '/data/isis_backup';  my $BACKUP_DEST = '/backup/isis_backup';
37    
38  # files to ignore in backup  # files to ignore in backup
39  my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');  my @ignore = ('.md5sum', '.backupignore', 'backupignore.txt');
40    
41  # open log  # open log
42  open(L, "> $LOG") || die "can't open log $LOG: $!";  open(L, ">> $LOG") || die "can't open log $LOG: $!";
43  select((select(L), $|=1)[0]);   # flush output  select((select(L), $|=1)[0]);   # flush output
44    
45    # make a lock on logfile
46    
47    my $c = 0;
48    {
49            flock L, LOCK_EX | LOCK_NB and last;
50            sleep 1;
51            redo if ++$c < 10;
52            # no response for 10 sec, bail out
53            print STDERR "can't take lock on $LOG -- another $0 running?\n";
54            exit 1;
55    }
56    
57  # taint path: nmblookup should be there!  # taint path: nmblookup should be there!
58  $ENV{'PATH'} = "/usr/bin:/bin";  $ENV{'PATH'} = "/usr/bin:/bin";
59    
# Line 49  my $mounts = shift @ARGV || Line 64  my $mounts = shift @ARGV ||
64    
65  my @in_backup;  # shares which are backeduped this run  my @in_backup;  # shares which are backeduped this run
66    
67  my $p = new Net::Ping->new();  my $p = new Net::Ping->new("tcp", 2);
68    # ping will try tcp connect to netbios-ssn (139)
69    $p->{port_num} = getservbyname("netbios-ssn", "tcp");
70    
71  my $backup_ok = 0;  my $backup_ok = 0;
72    
73  my $smb;  my $smb;
74  my %smb_atime;  my %smb_atime;
75  my %smb_mtime;  my %smb_mtime;
76    my %file_md5;
77    
78  open(M, $mounts) || die "can't open $mounts: $!";  open(M, $mounts) || die "can't open $mounts: $!";
79  while(<M>) {  while(<M>) {
# Line 63  while(<M>) { Line 81  while(<M>) {
81          next if !/^\s*smbmount\s/;          next if !/^\s*smbmount\s/;
82          my (undef,$share,undef,$opt) = split(/\s+/,$_,4);          my (undef,$share,undef,$opt) = split(/\s+/,$_,4);
83    
84          my ($user,$passwd,$workgroup);          my ($user,$passwd,$workgroup,$ip);
85    
86          foreach (split(/,/,$opt)) {          foreach (split(/,/,$opt)) {
87                  my ($n,$v) = split(/=/,$_,2);                  my ($n,$v) = split(/=/,$_,2);
# Line 79  while(<M>) { Line 97  while(<M>) {
97                          }                          }
98                  } elsif ($n =~ m#workgroup#i) {                  } elsif ($n =~ m#workgroup#i) {
99                          $workgroup = $v;                          $workgroup = $v;
100                    } elsif ($n =~ m#ip#i) {
101                            $ip = $v;
102                  }                  }
103          }          }
104    
105          push @in_backup,$share;          push @in_backup,$share;
106    
107    
108            my ($host,$dir,$date_dir) = share2host_dir($share);
109            my $bl = "$BACKUP_DEST/$host/$dir/latest";      # latest backup
110            my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";   # current one
111            my $real_bl;
112            if (-l $bl) {
113                    $real_bl=readlink($bl) || die "can't read link $bl: $!";
114                    $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
115                    if (-l $bc && $real_bl eq $bc) {
116                            print "$share allready backuped...\n";
117                            $backup_ok++;
118                            next;
119                    }
120    
121            }
122    
123    
124          print "working on $share\n";          print "working on $share\n";
125    
126          my $ip = get_ip($share);          # try to nmblookup IP
127            $ip = get_ip($share) if (! $ip);
128    
129          if ($ip) {          if ($ip) {
130                  xlog($share,"IP is $ip");                  xlog($share,"IP is $ip");
# Line 104  xlog("","$backup_ok backups completed of Line 142  xlog("","$backup_ok backups completed of
142    
143  #-------------------------------------------------------------------------  #-------------------------------------------------------------------------
144    
145    
146  # get IP number from share  # get IP number from share
147  sub get_ip {  sub get_ip {
148          my $share = shift;          my $share = shift;
# Line 116  sub get_ip { Line 155  sub get_ip {
155          }          }
156  }  }
157    
158    
159    # write entry to screen and log
160  sub xlog {  sub xlog {
161          my $share = shift;          my $share = shift;
162          my $t = strftime $LOG_TIME_FMT, localtime;          my $t = strftime $LOG_TIME_FMT, localtime;
# Line 124  sub xlog { Line 165  sub xlog {
165          print L "$t $share\t$m\n";          print L "$t $share\t$m\n";
166  }  }
167    
168  sub snap_share {  # dump warn and dies into log
169    BEGIN { $SIG{'__WARN__'} = sub { xlog('WARN',$_[0]) ; warn $_[0] } }
170    BEGIN { $SIG{'__DIE__'} = sub { xlog('DIE',$_[0]) ; die $_[0] } }
171    
         my $share = shift;  
   
         my %param = ( debug => 0 );  
   
         $param{username} = shift;  
         $param{password} = shift;  
         $param{workgroup} = shift;  
172    
173    # split share name to host, dir and currnet date dir
174    sub share2host_dir {
175            my $share = shift;
176          my ($host,$dir);          my ($host,$dir);
177          if ($share =~ m#//([^/]+)/(.+)$#) {          if ($share =~ m#//([^/]+)/(.+)$#) {
178                  ($host,$dir) = ($1,$2);                  ($host,$dir) = ($1,$2);
# Line 144  sub snap_share { Line 183  sub snap_share {
183                  print "Can't parse share $share into host and directory!\n";                  print "Can't parse share $share into host and directory!\n";
184                  return;                  return;
185          }          }
186            return ($host,$dir,strftime $DIR_TIME_FMT, localtime);
187    }
188    
189          my $date_dir = strftime $DIR_TIME_FMT, localtime;  
190    # make a snapshot of a share
191    sub snap_share {
192    
193            my $share = shift;
194    
195            my %param = ( debug => 0 );
196    
197            $param{username} = shift || warn "can't find username for share $share";
198            $param{password} = shift || warn "can't find passwod for share $share";
199            $param{workgroup} = shift || warn "can't find workgroup for share $share";
200    
201            my ($host,$dir,$date_dir) = share2host_dir($share);
202    
203          # latest backup directory          # latest backup directory
204          my $bl = "$BACKUP_DEST/$host/$dir/latest";          my $bl = "$BACKUP_DEST/$host/$dir/latest";
# Line 153  sub snap_share { Line 206  sub snap_share {
206          my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";          my $bc = "$BACKUP_DEST/$host/$dir/$date_dir";
207    
208          my $real_bl;          my $real_bl;
209          if (-e $bl) {          if (-l $bl) {
210                  $real_bl=readlink($bl) || die "can't read link $bl: $!";                  $real_bl=readlink($bl) || die "can't read link $bl: $!";
211                  $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");                  $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
212          } else {          } else {
213                  print "no old backup, this is first run...\n";                  print "no old backup, trying to find last backup, ";
214                    if (opendir(BL_DIR, "$BACKUP_DEST/$host/$dir")) {
215                            my @bl_dirs = sort grep { !/^\./ && -d "$BACKUP_DEST/$host/$dir/$_" } readdir(BL_DIR);
216                            closedir(BL_DIR);
217                            $real_bl=pop @bl_dirs;
218                            print "using $real_bl as latest...\n";
219                            $real_bl="$BACKUP_DEST/$host/$dir/$real_bl" if (substr($real_bl,0,1) ne "/");
220                            if ($real_bl eq $bc) {
221                                    xlog($share,"latest from today (possible partial backup)");
222                                    rename $real_bl,$real_bl.".partial" || warn "can't reaname partial backup: $!";
223                                    $real_bl .= ".partial";
224                            }
225                    } else {
226                            print "this is first run...\n";
227                    }
228          }          }
229    
230          if (-e $bc && $real_bl && $real_bl eq $bc) {          if (-l $bc && $real_bl && $real_bl eq $bc) {
231                  print "$share allready backuped...\n";                  print "$share allready backuped...\n";
232                  return;                  return;
233          }          }
# Line 189  sub snap_share { Line 256  sub snap_share {
256          my %file_size;          my %file_size;
257          my %file_atime;          my %file_atime;
258          my %file_mtime;          my %file_mtime;
259          my %file_md5;          #my %file_md5;
260    
261          my @smb_files;          my @smb_files;
262          my %smb_size;          my %smb_size;
263          #my %smb_atime;          #my %smb_atime;
264          #my %smb_mtime;          #my %smb_mtime;
         my %smb_md5;  
   
265    
266          sub norm_dir {          sub norm_dir {
267                  my $foo = shift;                  my $foo = shift;
# Line 212  sub snap_share { Line 277  sub snap_share {
277          my $di = 0;          my $di = 0;
278          while ($di <= $#dirs && $real_bl) {          while ($di <= $#dirs && $real_bl) {
279                  my $d=$dirs[$di++];                  my $d=$dirs[$di++];
280                  opendir(DIR,"$bl/$d") || warn "opendir($bl/$d): $!\n";                  opendir(DIR,"$real_bl/$d") || warn "opendir($real_bl/$d): $!\n";
281    
282                  # read .backupignore if exists                  # read .backupignore if exists
283                  if (-f "$bl/$d/.backupignore") {                  if (-f "$real_bl/$d/.backupignore") {
284                          open(I,"$bl/$d/.backupignore");                          open(I,"$real_bl/$d/.backupignore");
285                          while(<I>) {                          while(<I>) {
286                                  chomp;                                  chomp;
287                                  push @ignore,norm_dir("$d/$_");                                  push @ignore,norm_dir("$d/$_");
288                          }                          }
289                          close(I);                          close(I);
290  print STDERR "ignore: ",join("|",@ignore),"\n";  #print STDERR "ignore: ",join("|",@ignore),"\n";
291                          link "$bl/$d/.backupignore","$bc/$d/.backupignore" ||                          link "$real_bl/$d/.backupignore","$bc/$d/.backupignore" ||
292                                  warn "can't copy $bl/$d/.backupignore to current backup dir: $!\n";                                  warn "can't copy $real_bl/$d/.backupignore to current backup dir: $!\n";
293                  }                  }
294    
295                  # read .md5sum if exists                  # read .md5sum if exists
296                  if (-f "$bl/$d/.md5sum") {                  if (-f "$real_bl/$d/.md5sum") {
297                          open(I,"$bl/$d/.md5sum");                          open(I,"$real_bl/$d/.md5sum");
298                          while(<I>) {                          while(<I>) {
299                                  chomp;                                  chomp;
300                                  my ($md5,$f) = split(/\s+/,$_,2);                                  my ($md5,$f) = split(/\s+/,$_,2);
# Line 243  print STDERR "ignore: ",join("|",@ignore Line 308  print STDERR "ignore: ",join("|",@ignore
308                          next if ($f eq '.');                          next if ($f eq '.');
309                          next if ($f eq '..');                          next if ($f eq '..');
310                          my $pr = norm_dir("$d/$f");     # path relative                          my $pr = norm_dir("$d/$f");     # path relative
311                          my $pf = norm_dir("$d/$f","$bl/");      # path full                          my $pf = norm_dir("$d/$f","$real_bl/"); # path full
312                          if (grep(/^\Q$pr\E$/,@ignore) == 0) {                          if (grep(/^\Q$pr\E$/,@ignore) == 0) {
313                                  if (-f $pf) {                                  if (-f $pf) {
314                                          push @files,$pr;                                          push @files,$pr;
# Line 272  print STDERR "ignore: ",join("|",@ignore Line 337  print STDERR "ignore: ",join("|",@ignore
337    
338          $di = 0;          $di = 0;
339          while ($di <= $#smb_dirs) {          while ($di <= $#smb_dirs) {
340                  my $d=$smb_dirs[$di++];                  my $d=$smb_dirs[$di];
341                  my $pf = norm_dir($d,"smb:$share/");    # path full                  my $pf = norm_dir($d,"smb:$share/");    # path full
342                  my $D = $smb->opendir($pf) || warn "smb->opendir($pf): $!\n";                  my $D = $smb->opendir($pf);
343                    if (! $D) {
344                            xlog($share,"FATAL: $share: $!");
345                            # remove failing dir
346                            delete $smb_dirs[$di];
347                            next;
348                    }
349                    $di++;
350    
351                  my @clutter = $smb->readdir_struct($D);                  my @clutter = $smb->readdir_struct($D);
352                  foreach my $item (@clutter) {                  foreach my $item (@clutter) {
# Line 337  print STDERR "ignore: ",join("|",@ignore Line 409  print STDERR "ignore: ",join("|",@ignore
409                                  next;                                  next;
410                          }                          }
411    
412                            my $md5 = Digest::MD5->new;
413    
414                          my $fd = $smb->open("$from/$f");                          my $fd = $smb->open("$from/$f");
415                          if (! $fd) {                          if (! $fd) {
416                                  print STDERR "can't open smb file $from/$f: $!\n";                                  print STDERR "can't open smb file $from/$f: $!\n";
# Line 346  print STDERR "ignore: ",join("|",@ignore Line 420  print STDERR "ignore: ",join("|",@ignore
420                          while (defined(my $b=$smb->read($fd,4096))) {                          while (defined(my $b=$smb->read($fd,4096))) {
421                                  print F $b;                                  print F $b;
422                                  $l += length($b);                                  $l += length($b);
423                                    $md5->add($b);
424                          }                          }
425    
426                          $smb->close($fd);                          $smb->close($fd);
427                          close(F);                          close(F);
428    
429                            $file_md5{$f} = $md5->hexdigest;
430    
431                          # FIX: this fails with -T                          # FIX: this fails with -T
432                          my ($a,$m) = ($smb->stat("$from/$f"))[10,11];                          my ($a,$m) = ($smb->stat("$from/$f"))[10,11];
433                          utime $a, $m, "$to/$f" ||                          utime $a, $m, "$to/$f" ||
# Line 404  print STDERR "ignore: ",join("|",@ignore Line 481  print STDERR "ignore: ",join("|",@ignore
481          xlog($share,"$transfer bytes transfered...");          xlog($share,"$transfer bytes transfered...");
482    
483          foreach (@ln_files) {          foreach (@ln_files) {
484                  link "$bl/$_","$bc/$_" || warn "link $bl/$_ -> $bc/$_: $!\n";                  link "$real_bl/$_","$bc/$_" || warn "link $real_bl/$_ -> $bc/$_: $!\n";
485          }          }
486    
487          # remove files          # remove files
# Line 417  print STDERR "ignore: ",join("|",@ignore Line 494  print STDERR "ignore: ",join("|",@ignore
494                  rmdir "$bc/$_" || warn "rmdir $_: $!\n";                  rmdir "$bc/$_" || warn "rmdir $_: $!\n";
495          }          }
496    
497            # remove old .md5sum
498          # FIX: create .md5sum          foreach (sort @dirs) {
499                    unlink "$bc/$_/.md5sum" if (-e "$bc/$_/.md5sum");
500            }
501    
502            # create .md5sum
503            my $last_dir = '';
504            my $md5;
505            foreach my $f (sort { $file_md5{$a} cmp $file_md5{$b} } keys %file_md5) {
506                    my $dir = dirname($f);
507                    my $file = basename($f);
508    #print "$f -- $dir / $file<--\n";
509                    if ($dir ne $last_dir) {
510                            close($md5) if ($md5);
511                            open($md5, ">> $bc/$dir/.md5sum") || warn "can't create $bc/$dir/.md5sum: $!";
512                            $last_dir = $dir;
513    #print STDERR "writing $last_dir/.md5sum\n";
514                    }
515                    print $md5 $file_md5{$f},"  $file\n";
516            }
517            close($md5) if ($md5);
518    
519          # create leatest link          # create leatest link
520    #print "ln -s $bc $real_bl\n";
521            if (-l $bl) {
522                    unlink $bl || warn "can't remove old latest symlink $bl: $!\n";
523            }
524          symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";          symlink $bc,$bl || warn "can't create latest symlink $bl -> $bc: $!\n";
525    
526            # FIX: sanity check -- remove for speedup
527            xlog($share,"failed to create latest symlink $bl -> $bc...") if (readlink($bl) ne $bc || ! -l $bl);
528    
529          xlog($share,"backup completed...");          xlog($share,"backup completed...");
530  }  }
531    
532    __END__
533  #-------------------------------------------------------------------------  #-------------------------------------------------------------------------
534    
535    
536    =head1 NAME
537    
538    psinib - Perl Snapshot Is Not Incremental Backup
539    
540    =head1 SYNOPSIS
541    
542    ./psinib.pl
543    
544    =head1 DESCRIPTION
545    
546    This script in current version support just backup of Samba (or Micro$oft
547    Winblowz) shares to central disk space. Central disk space is organized in
548    multiple directories named after:
549    
550    =over 4
551    
552    =item *
553    server which is sharing files to be backed up
554    
555    =item *
556    name of share on server
557    
558    =item *
559    dated directory named like standard ISO date format (YYYYMMDD).
560    
561    =back
562    
563    In each dated directory you will find I<snapshot> of all files on
564    exported share on that particular date.
565    
566    You can also use symlink I<latest> which will lead you to
567    last completed backup. After that you can use some other backup
568    software to transfer I<snapshot> to tape, CD-ROM or some other media.
569    
570    =head2 Design considerations
571    
572    Since taking of share snapshot every day requires a lot of disk space and
573    network bandwidth, B<psinib> uses several techniques to keep disk usage and
574    network traffic at acceptable level:
575    
576    =over 3
577    
578    =item - usage of hard-links to provide same files in each snapshot (as opposed
579    to have multiple copies of same file)
580    
581    =item - usage of file size, atime and mtime to find changes of files without
582    transferring whole file over network (just share browsing is transfered
583    over network)
584    
585    =item - usage of C<.md5sum> files (compatible with command-line utility
586    C<md5sum>) to keep file between snapshots hard-linked
587    
588    =back
589    
590    =head1 CONFIGURATION
591    
592    This section is not yet written.
593    
594    =head1 HACKS, TRICKS, BUGS and LIMITATIONS
595    
596    This chapter will have all content that doesn't fit anywhere else.
597    
598    =head2 Can snapshots be more frequent than daily?
599    
600    There is not real reason why you can't take snapshot more often than
601    once a day. Actually, if you are using B<psinib> to backup Windows
602    workstations you already know that they tend to come-and-go during the day
603    (reboots probably ;-), so running B<psinib> several times a day increases
604    your chance of having up-to-date backup (B<psinib> will not make multiple
605    snapshots for same day, nor will it update snapshot for current day if
606    it already exists).
607    
608    However, changing B<psinib> to produce snapshots which are, for example, hourly
609    is a simple change of C<$DIR_TIME_FMT> which is currently set to
610    C<'%Y%m%d'> (see I<strftime> documentation for explanation of that
611    format). If you change that to C<'%Y%m%d-%H> you can have hourly snapshots
612    (if your network is fast enough, that is...). Also, some of messages in
613    program will sound strange, but other than that it should work.
614    I<You have been warned>.
615    
616    =head2 Do I really need to share every directory which I want to snapshot?
617    
618    Actually, no. Due to usage of C<Filesys::SmbClient> module, you can also
619    specify sub-directory inside your share that you want to backup. This feature
620    is most useful if you want to use administrative shares (but, have in mind
621    that you have to enter your Win administrator password in unencrypted file on
622    disk to do that) like this:
623    
624            smbmount //server/c$/WinNT/fonts  /mnt  -o username=administrator%win  
625    
626    After that you will get directories with snapshots like:
627    
628            server/c_WinNT_fonts/yyyymmdd/....
629    
630    =head2 Won't I run out of disk space?
631    
632    Of course you will... Snapshots and logfiles will eventually fill-up your disk.
633    However, you can do two things to stop that:
634    
635    =head3 Clean snapshort older than x days
636    
637    You can add following command to your C<root> crontab:
638    
639            find /backup/isis_backup -type d -mindepth 3 -maxdepth 3 -mtime +11 -exec rm -Rf {} \;
640    
641    I assume that C</backup/isis_backup> is directory in which are your snapshots
642    and that you don't want to keep snapshots older than 11 days (that's
643    C<-mtime +11> part of command).
644    
645    =head3 Rotate your logs
646    
647    I will leave that to you. I relay on GNU/Debian's C<logrotate> to do it for me.
648    
649    =head2 What are I<YYYYMMDD.partial> directories?
650    
651    If there isn't I<latest> symlink in snapshot directory, it's preatty safe to
652    assume that previous backup from that day failed. So, that directory will
653    be renamed to I<YYYYMMDD.partial> and snapshot will be performed again,
654    linking same files (other alternative would be to erase that dir and find
655    second-oldest directory, but this seemed like more correct approach).
656    
657    =head1 AUTHOR
658    
659    Dobrica Pavlinusic <dpavlin@rot13.org>
660    
661    L<http://www.rot13.org/~dpavlin/>
662    
663    =head1 LICENSE
664    
665    This product is licensed under GNU Public License (GPL) v2 or later.
666    
667    =cut
Legend:
Removed from v.1.1.1.1  
changed lines
  Added in v.1.11
  ViewVC Help
Powered by ViewVC 1.1.26