lib/WebPAC/Validate.pm

package WebPAC::Validate;

use warnings;
use strict;

use blib;

use base 'WebPAC::Common';
use File::Slurp;
use List::Util qw/first/;
use Data::Dump qw/dump/;
use WebPAC::Normalize qw/_pack_subfields_hash/;
use Storable qw/dclone/;

=head1 NAME

WebPAC::Validate - provide simple validation for records

=head1 VERSION

Version 0.09

=cut

our $VERSION = '0.09';

=head1 SYNOPSIS

This module provide a simple way to validate your file against a simple
configuration file in following format:

  # field 10 doesn't have any subfields
  10
  # same with 101
  101
  # field 200 have valid subfields a-g
  # and field e is repeatable
  200 a b c d e* f g
  # field 205 can have only subfield a
  # and must exists
  205! a
  # while 210 can have a c or d
  210 a c d
  # field which is ignored in validation
  999-

=head1 FUNCTIONS

=head2 new

Create new validation object

  my $validate = new WebPAC::Validate(
        path => 'conf/validate/file',
  );

=cut

sub new {
        my $class = shift;
        my $self = {@_};
        bless($self, $class);

        my $log = $self->_get_logger();

        foreach my $p (qw/path/) {
                $log->logconfess("need $p") unless ($self->{$p});
        }

        my $v_file = read_file( $self->{path} ) ||
                $log->logdie("can't open validate path $self->{path}: $!");

        my $v;
        my $curr_line = 1;

        foreach my $l (split(/[\n\r]+/, $v_file)) {
                $curr_line++;

                # skip comments and whitespaces
                next if ($l =~ /^#/ || $l =~ /^\s*$/);

                $l =~ s/^\s+//;
                $l =~ s/\s+$//;

                my @d = split(/\s+/, $l);

                my $fld = shift @d;

                if ($fld =~ s/!$//) {
                        $self->{must_exist}->{$fld}++;
                } elsif ($fld =~ s/-$//) {
                        $self->{dont_validate}->{$fld}++;
                }

                $log->logdie("need field name in line $curr_line: $l") unless (defined($fld));

                if (@d) {
                        $v->{$fld} = [ map {
                                my $sf = $_;
                                if ( $sf =~ s/!(\*)?$/$1/ ) {
                                        $self->{must_exist_sf}->{ $fld }->{ $sf }++;
                                };
                                $sf;
                        } @d ];
                } else {
                        $v->{$fld} = 1;
                }

        }

        $log->debug("current validation rules: ", dump($v));

        $self->{rules} = $v;

        $log->info("validation uses rules from $self->{path}");

        $self ? return $self : return undef;
}

=head2 validate_errors

Validate record and return errors

  my @errors = $validate->validate_errors( $rec, $rec_dump );

=cut

sub validate_errors {
        my $self = shift;

        my $log = $self->_get_logger();

        my $rec = shift || $log->logdie("validate_errors need record");
        my $rec_dump = shift;

        $log->logdie("rec isn't HASH") unless (ref($rec) eq 'HASH');
        $log->logdie("can't find validation rules") unless (my $r = $self->{rules});

        my $errors;

        $log->debug("rec = ", sub { dump($rec) }, "keys = ", keys %{ $rec });

        my $fields;

        foreach my $f (keys %{ $rec }) {

                next if (!defined($f) || $f eq '' || $f eq '000');

                next if (defined( $self->{dont_validate}->{$f} ));

                # track field usage
                $fields->{$f}++;

                if ( ! defined($r->{$f}) ) {
                        $errors->{ $f }->{unexpected} = "this field is not expected";
                        next;
                }


                if (ref($rec->{$f}) ne 'ARRAY') {
                        $errors->{ $f }->{not_repeatable} = "probably bug in parsing input data";
                        next;
                }

                foreach my $v (@{ $rec->{$f} }) {
                        # can we have subfields?
                        if (ref($r->{$f}) eq 'ARRAY') {
                                # are values hashes? (has subfields)
                                if (! defined($v)) {
#                                       $errors->{$f}->{empty} = undef;
#                                       $errors->{dump} = $rec_dump if ($rec_dump);
                                } elsif (ref($v) ne 'HASH') {
                                        $errors->{$f}->{missing_subfield} = join(",", @{ $r->{$f} }) . " required";
                                        next;
                                } else {

                                        my $h = dclone( $v );

                                        my $sf_repeatable;

                                        delete($v->{subfields}) if (defined($v->{subfields}));

                                        my $subfields;

                                        foreach my $sf (keys %{ $v }) {

                                                $subfields->{ $sf }++;

                                                # is non-repeatable but with multiple values?
                                                if ( ! first { $_ eq $sf.'*' } @{$r->{$f}} ) {
                                                        if ( ref($v->{$sf}) eq 'ARRAY' ) {
                                                                $sf_repeatable->{$sf}++;
                                                        };
                                                        if (! first { $_ eq $sf } @{ $r->{$f} }) {
                                                                $errors->{ $f }->{subfield}->{extra}->{$sf}++;
                                                        }
                                                }

                                        }
                                        if (my @r_sf = sort keys( %$sf_repeatable )) {

                                                foreach my $sf (@r_sf) {
                                                        $errors->{$f}->{subfield}->{extra_repeatable}->{$sf}++;
                                                        $errors->{$f}->{dump} = _pack_subfields_hash( $h, 1 );
                                                }

                                        }

                                        if ( defined( $self->{must_exist_sf}->{$f} ) ) {
                                                foreach my $sf (sort keys %{ $self->{must_exist_sf}->{$f} }) {
#warn "====> $f $sf must exist\n";
                                                        $errors->{$f}->{subfield}->{missing}->{$sf}++
                                                                unless defined( $subfields->{$sf} );
                                                }
                                        }

                                }
                        } elsif (ref($v) eq 'HASH') {
                                $errors->{$f}->{unexpected_subfields}++;
                                $errors->{$f}->{dump} = _pack_subfields_hash( $v, 1 );
                        }
                }
        }

        foreach my $must (sort keys %{ $self->{must_exist} }) {
                next if ($fields->{$must});
                $errors->{$must}->{missing}++;
                $errors->{dump} = $rec_dump if ($rec_dump);
        }

        if ($errors) {
                $log->debug("errors: ", $self->report_error( $errors ) );

                my $mfn = $rec->{'000'}->[0] || $log->logconfess("record ", dump( $rec ), " doesn't have MFN");
                $self->{errors}->{$mfn} = $errors;
        }

        #$log->logcluck("return from this function is ARRAY") unless wantarray;

        return $errors;
}

=head2 reset_errors

Clean all accumulated errors for this input

  $validate->reset_errors;

=cut

sub reset_errors {
        my $self = shift;
        delete ($self->{errors});
}

=head2 all_errors

Return hash with all errors

  print dump( $validate->all_errors );

=cut

sub all_errors {
        my $self = shift;
        return $self->{errors};
}

=head2 report_error

Produce nice humanly readable report of single error

  print $validate->report_error( $error_hash );

=cut

sub report_error {
        my $self = shift;

        my $h = shift || die "no hash?";

        sub _unroll {
                my ($self, $tree, $accumulated) = @_;

                my $log = $self->_get_logger();

                $log->debug("# ",
                        ( $tree                 ? "tree: $tree "                                        : '' ),
                        ( $accumulated  ? "accumulated: $accumulated "          : '' ),
                );

                my $results;

                if (ref($tree) ne 'HASH') {
                        return ("$accumulated\t($tree)", undef);
                }

                my $dump;

                foreach my $k (sort keys %{ $tree }) {

                        if ($k eq 'dump') {
                                $dump = $tree->{dump};
#                               warn "## dump: ",dump($dump),"\n";
                                next;
                        }

                        $log->debug("current: $k");

                        my ($new_results, $new_dump) = $self->_unroll($tree->{$k},
                                $accumulated ? "$accumulated\t$k" : $k
                        );

                        $log->debug(
                                ( $new_results          ? "new_results: " . dump($new_results) ." "     : '' ),
                        );

                        push @$results, $new_results if ($new_results);
                        $dump = $new_dump if ($new_dump);

                }

                $log->debug(
                        ( $results              ? "results: " . dump($results) ." "     : '' ),
                );

                if ($#$results == 0) {
                        return ($results->[0], $dump);
                } else {
                        return ($results, $dump);
                }
        }


        sub _reformat {
                my $l = shift;
                $l =~ s/\t/ /g;
                $l =~ s/_/ /;
                return $l;
        }

        my $out = '';

        for my $f (sort keys %{ $h }) {
                $out .= "$f: ";
                
                my ($r, $d) = $self->_unroll( $h->{$f} );
                my $e;
                if (ref($r) eq 'ARRAY') {
                        $e .= join(", ", map { _reformat( $_ ) } @$r);
                } else {
                        $e .= _reformat( $r );
                }
                $e .= "\n\t$d" if ($d);

                $out .= $e . "\n";
        }
        return $out;
}


=head2 report

Produce nice humanly readable report of errors

  print $validate->report;

=cut

sub report {
        my $self = shift;
        my $e = $self->{errors} || return;

        my $out;
        foreach my $mfn (sort { $a <=> $b } keys %$e) {
                $out .= "MFN $mfn\n" . $self->report_error( $e->{$mfn} ) . "\n";
        }

        return $out;

}

=head1 AUTHOR

Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>

=head1 COPYRIGHT & LICENSE

Copyright 2006 Dobrica Pavlinusic, All Rights Reserved.

This program is free software; you can redistribute it and/or modify it
under the same terms as Perl itself.

=cut

1; # End of WebPAC::Validate
1	dpavlin	514	package WebPAC::Validate;
2
3			use warnings;
4			use strict;
5
6			use blib;
7
8			use base 'WebPAC::Common';
9			use File::Slurp;
10	dpavlin	515	use List::Util qw/first/;
11	dpavlin	652	use Data::Dump qw/dump/;
12	dpavlin	643	use WebPAC::Normalize qw/_pack_subfields_hash/;
13			use Storable qw/dclone/;
14	dpavlin	514
15			=head1 NAME
16
17			WebPAC::Validate - provide simple validation for records
18
19			=head1 VERSION
20
21	dpavlin	677	Version 0.09
22	dpavlin	514
23			=cut
24
25	dpavlin	677	our $VERSION = '0.09';
26	dpavlin	514
27			=head1 SYNOPSIS
28
29			This module provide a simple way to validate your file against a simple
30			configuration file in following format:
31
32			# field 10 doesn't have any subfields
33			10
34			# same with 101
35			101
36			# field 200 have valid subfields a-g
37	dpavlin	645	# and field e is repeatable
38			200 a b c d e* f g
39	dpavlin	514	# field 205 can have only subfield a
40	dpavlin	645	# and must exists
41			205! a
42	dpavlin	514	# while 210 can have a c or d
43			210 a c d
44	dpavlin	677	# field which is ignored in validation
45			999-
46	dpavlin	514
47			=head1 FUNCTIONS
48
49			=head2 new
50
51			Create new validation object
52
53			my $validate = new WebPAC::Validate(
54	dpavlin	643	path => 'conf/validate/file',
55	dpavlin	514	);
56
57			=cut
58
59			sub new {
60			my $class = shift;
61			my $self = {@_};
62			bless($self, $class);
63
64			my $log = $self->_get_logger();
65
66			foreach my $p (qw/path/) {
67			$log->logconfess("need $p") unless ($self->{$p});
68			}
69
70			my $v_file = read_file( $self->{path} ) \|\|
71			$log->logdie("can't open validate path $self->{path}: $!");
72
73			my $v;
74			my $curr_line = 1;
75
76			foreach my $l (split(/[\n\r]+/, $v_file)) {
77			$curr_line++;
78
79	dpavlin	516	# skip comments and whitespaces
80			next if ($l =~ /^#/ \|\| $l =~ /^\s*$/);
81
82	dpavlin	514	$l =~ s/^\s+//;
83			$l =~ s/\s+$//;
84
85			my @d = split(/\s+/, $l);
86
87	dpavlin	516	my $fld = shift @d;
88	dpavlin	514
89	dpavlin	645	if ($fld =~ s/!$//) {
90			$self->{must_exist}->{$fld}++;
91	dpavlin	677	} elsif ($fld =~ s/-$//) {
92			$self->{dont_validate}->{$fld}++;
93	dpavlin	645	}
94
95	dpavlin	516	$log->logdie("need field name in line $curr_line: $l") unless (defined($fld));
96
97	dpavlin	514	if (@d) {
98	dpavlin	646	$v->{$fld} = [ map {
99			my $sf = $_;
100			if ( $sf =~ s/!(\*)?$/$1/ ) {
101			$self->{must_exist_sf}->{ $fld }->{ $sf }++;
102			};
103			$sf;
104			} @d ];
105	dpavlin	514	} else {
106	dpavlin	515	$v->{$fld} = 1;
107	dpavlin	514	}
108
109			}
110
111	dpavlin	652	$log->debug("current validation rules: ", dump($v));
112	dpavlin	514
113			$self->{rules} = $v;
114
115	dpavlin	516	$log->info("validation uses rules from $self->{path}");
116
117	dpavlin	514	$self ? return $self : return undef;
118			}
119
120	dpavlin	515	=head2 validate_errors
121
122			Validate record and return errors
123
124	dpavlin	653	my @errors = $validate->validate_errors( $rec, $rec_dump );
125	dpavlin	515
126			=cut
127
128			sub validate_errors {
129			my $self = shift;
130
131			my $log = $self->_get_logger();
132
133			my $rec = shift \|\| $log->logdie("validate_errors need record");
134	dpavlin	653	my $rec_dump = shift;
135	dpavlin	515
136			$log->logdie("rec isn't HASH") unless (ref($rec) eq 'HASH');
137			$log->logdie("can't find validation rules") unless (my $r = $self->{rules});
138
139	dpavlin	652	my $errors;
140	dpavlin	515
141	dpavlin	652	$log->debug("rec = ", sub { dump($rec) }, "keys = ", keys %{ $rec });
142	dpavlin	515
143	dpavlin	645	my $fields;
144
145	dpavlin	515	foreach my $f (keys %{ $rec }) {
146
147	dpavlin	516	next if (!defined($f) \|\| $f eq '' \|\| $f eq '000');
148	dpavlin	515
149	dpavlin	677	next if (defined( $self->{dont_validate}->{$f} ));
150
151			# track field usage
152	dpavlin	645	$fields->{$f}++;
153
154			if ( ! defined($r->{$f}) ) {
155	dpavlin	666	$errors->{ $f }->{unexpected} = "this field is not expected";
156	dpavlin	515	next;
157			}
158
159	dpavlin	645
160	dpavlin	515	if (ref($rec->{$f}) ne 'ARRAY') {
161	dpavlin	666	$errors->{ $f }->{not_repeatable} = "probably bug in parsing input data";
162	dpavlin	515	next;
163			}
164
165			foreach my $v (@{ $rec->{$f} }) {
166			# can we have subfields?
167			if (ref($r->{$f}) eq 'ARRAY') {
168			# are values hashes? (has subfields)
169	dpavlin	652	if (! defined($v)) {
170	dpavlin	666	# $errors->{$f}->{empty} = undef;
171	dpavlin	656	# $errors->{dump} = $rec_dump if ($rec_dump);
172	dpavlin	652	} elsif (ref($v) ne 'HASH') {
173	dpavlin	666	$errors->{$f}->{missing_subfield} = join(",", @{ $r->{$f} }) . " required";
174	dpavlin	515	next;
175			} else {
176	dpavlin	643
177			my $h = dclone( $v );
178
179	dpavlin	645	my $sf_repeatable;
180
181	dpavlin	643	delete($v->{subfields}) if (defined($v->{subfields}));
182
183	dpavlin	646	my $subfields;
184
185	dpavlin	515	foreach my $sf (keys %{ $v }) {
186	dpavlin	643
187	dpavlin	646	$subfields->{ $sf }++;
188
189	dpavlin	645	# is non-repeatable but with multiple values?
190			if ( ! first { $_ eq $sf.'*' } @{$r->{$f}} ) {
191			if ( ref($v->{$sf}) eq 'ARRAY' ) {
192			$sf_repeatable->{$sf}++;
193			};
194			if (! first { $_ eq $sf } @{ $r->{$f} }) {
195	dpavlin	666	$errors->{ $f }->{subfield}->{extra}->{$sf}++;
196	dpavlin	645	}
197	dpavlin	515	}
198	dpavlin	643
199	dpavlin	515	}
200	dpavlin	645	if (my @r_sf = sort keys( %$sf_repeatable )) {
201
202	dpavlin	652	foreach my $sf (@r_sf) {
203	dpavlin	666	$errors->{$f}->{subfield}->{extra_repeatable}->{$sf}++;
204	dpavlin	670	$errors->{$f}->{dump} = _pack_subfields_hash( $h, 1 );
205	dpavlin	652	}
206
207	dpavlin	645	}
208	dpavlin	646
209			if ( defined( $self->{must_exist_sf}->{$f} ) ) {
210			foreach my $sf (sort keys %{ $self->{must_exist_sf}->{$f} }) {
211	dpavlin	647	#warn "====> $f $sf must exist\n";
212	dpavlin	666	$errors->{$f}->{subfield}->{missing}->{$sf}++
213	dpavlin	652	unless defined( $subfields->{$sf} );
214	dpavlin	646	}
215			}
216
217	dpavlin	515	}
218			} elsif (ref($v) eq 'HASH') {
219	dpavlin	666	$errors->{$f}->{unexpected_subfields}++;
220	dpavlin	670	$errors->{$f}->{dump} = _pack_subfields_hash( $v, 1 );
221	dpavlin	515	}
222			}
223			}
224
225	dpavlin	645	foreach my $must (sort keys %{ $self->{must_exist} }) {
226			next if ($fields->{$must});
227	dpavlin	666	$errors->{$must}->{missing}++;
228	dpavlin	653	$errors->{dump} = $rec_dump if ($rec_dump);
229	dpavlin	645	}
230
231	dpavlin	652	if ($errors) {
232	dpavlin	671	$log->debug("errors: ", $self->report_error( $errors ) );
233	dpavlin	652
234	dpavlin	653	my $mfn = $rec->{'000'}->[0] \|\| $log->logconfess("record ", dump( $rec ), " doesn't have MFN");
235	dpavlin	652	$self->{errors}->{$mfn} = $errors;
236			}
237
238	dpavlin	515	#$log->logcluck("return from this function is ARRAY") unless wantarray;
239
240	dpavlin	652	return $errors;
241	dpavlin	515	}
242
243	dpavlin	653	=head2 reset_errors
244
245			Clean all accumulated errors for this input
246
247			$validate->reset_errors;
248
249			=cut
250
251			sub reset_errors {
252			my $self = shift;
253			delete ($self->{errors});
254			}
255
256			=head2 all_errors
257
258			Return hash with all errors
259
260			print dump( $validate->all_errors );
261
262			=cut
263
264			sub all_errors {
265			my $self = shift;
266			return $self->{errors};
267			}
268
269	dpavlin	671	=head2 report_error
270	dpavlin	664
271	dpavlin	671	Produce nice humanly readable report of single error
272	dpavlin	664
273	dpavlin	671	print $validate->report_error( $error_hash );
274	dpavlin	664
275			=cut
276
277	dpavlin	671	sub report_error {
278	dpavlin	664	my $self = shift;
279
280	dpavlin	671	my $h = shift \|\| die "no hash?";
281
282			sub _unroll {
283	dpavlin	666	my ($self, $tree, $accumulated) = @_;
284	dpavlin	664
285	dpavlin	666	my $log = $self->_get_logger();
286
287	dpavlin	665	$log->debug("# ",
288			( $tree ? "tree: $tree " : '' ),
289			( $accumulated ? "accumulated: $accumulated " : '' ),
290			);
291	dpavlin	664
292	dpavlin	665	my $results;
293	dpavlin	664
294	dpavlin	665	if (ref($tree) ne 'HASH') {
295			return ("$accumulated\t($tree)", undef);
296	dpavlin	664	}
297
298	dpavlin	665	my $dump;
299	dpavlin	664
300	dpavlin	665	foreach my $k (sort keys %{ $tree }) {
301
302	dpavlin	664	if ($k eq 'dump') {
303	dpavlin	665	$dump = $tree->{dump};
304	dpavlin	670	# warn "## dump: ",dump($dump),"\n";
305	dpavlin	664	next;
306			}
307
308	dpavlin	665	$log->debug("current: $k");
309	dpavlin	664
310	dpavlin	671	my ($new_results, $new_dump) = $self->_unroll($tree->{$k},
311	dpavlin	665	$accumulated ? "$accumulated\t$k" : $k
312			);
313
314			$log->debug(
315			( $new_results ? "new_results: " . dump($new_results) ." " : '' ),
316			);
317
318			push @$results, $new_results if ($new_results);
319			$dump = $new_dump if ($new_dump);
320
321	dpavlin	664	}
322	dpavlin	665
323			$log->debug(
324			( $results ? "results: " . dump($results) ." " : '' ),
325			);
326
327			if ($#$results == 0) {
328			return ($results->[0], $dump);
329			} else {
330			return ($results, $dump);
331			}
332	dpavlin	664	}
333
334	dpavlin	666
335	dpavlin	671	sub _reformat {
336	dpavlin	666	my $l = shift;
337			$l =~ s/\t/ /g;
338			$l =~ s/_/ /;
339			return $l;
340			}
341
342	dpavlin	671	my $out = '';
343	dpavlin	666
344	dpavlin	671	for my $f (sort keys %{ $h }) {
345			$out .= "$f: ";
346
347			my ($r, $d) = $self->_unroll( $h->{$f} );
348			my $e;
349			if (ref($r) eq 'ARRAY') {
350			$e .= join(", ", map { _reformat( $_ ) } @$r);
351			} else {
352			$e .= _reformat( $r );
353	dpavlin	666	}
354	dpavlin	671	$e .= "\n\t$d" if ($d);
355
356			$out .= $e . "\n";
357	dpavlin	664	}
358	dpavlin	671	return $out;
359			}
360	dpavlin	664
361	dpavlin	671
362			=head2 report
363
364			Produce nice humanly readable report of errors
365
366			print $validate->report;
367
368			=cut
369
370			sub report {
371			my $self = shift;
372			my $e = $self->{errors} \|\| return;
373
374			my $out;
375			foreach my $mfn (sort { $a <=> $b } keys %$e) {
376			$out .= "MFN $mfn\n" . $self->report_error( $e->{$mfn} ) . "\n";
377			}
378
379	dpavlin	664	return $out;
380	dpavlin	671
381	dpavlin	664	}
382
383	dpavlin	514	=head1 AUTHOR
384
385			Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>
386
387			=head1 COPYRIGHT & LICENSE
388
389			Copyright 2006 Dobrica Pavlinusic, All Rights Reserved.
390
391			This program is free software; you can redistribute it and/or modify it
392			under the same terms as Perl itself.
393
394			=cut
395
396			1; # End of WebPAC::Validate