--- trunk/lib/WebPAC/Normalize.pm	2006/06/26 16:39:51	536
+++ trunk/lib/WebPAC/Normalize.pm	2006/06/29 21:19:08	543
@@ -1,9 +1,12 @@
 package WebPAC::Normalize;
 use Exporter 'import';
 @EXPORT = qw/
-	set_rec set_lookup
-	get_ds clean_ds
+	_set_rec _set_lookup
+	_get_ds _clean_ds
+
 	tag search display
+	marc21
+
 	rec1 rec2 rec
 	regex prefix suffix surround
 	first lookup join_with
@@ -14,6 +17,7 @@
 
 #use base qw/WebPAC::Common/;
 use Data::Dumper;
+use Encode qw/from_to/;
 
 =head1 NAME
 
@@ -21,11 +25,11 @@
 
 =head1 VERSION
 
-Version 0.04
+Version 0.06
 
 =cut
 
-our $VERSION = '0.04';
+our $VERSION = '0.06';
 
 =head1 SYNOPSIS
 
@@ -38,22 +42,33 @@
 C<perl -c normalize.pl>.
 
 Normalisation can generate multiple output normalized data. For now, supported output
-types (on the left side of definition) are: C<tag>, C<display> and C<search>.
+types (on the left side of definition) are: C<tag>, C<display>, C<search> and
+C<marc21>.
 
 =head1 FUNCTIONS
 
+Functions which start with C<_> are private and used by WebPAC internally.
+All other functions are available for use within normalisation rules.
+
 =head2 data_structure
 
 Return data structure
 
-  my $ds = WebPAC::Normalize(
+  my $ds = WebPAC::Normalize::data_structure(
   	lookup => $lookup->lookup_hash,
 	row => $row,
 	rules => $normalize_pl_config,
+	marc_encoding => 'utf-8',
   );
 
+Options C<lookup>, C<row>, C<rules> and C<log> are mandatory while all
+other are optional.
+
 This function will B<die> if normalizastion can't be evaled.
 
+Since this function isn't exported you have to call it with 
+C<WebPAC::Normalize::data_structure>.
+
 =cut
 
 sub data_structure {
@@ -63,28 +78,121 @@
 	die "need normalisation argument" unless ($arg->{rules});
 
 	no strict 'subs';
-	set_lookup( $arg->{lookup} );
-	set_rec( $arg->{row} );
-	clean_ds();
+	_set_lookup( $arg->{lookup} );
+	_set_rec( $arg->{row} );
+	_clean_ds( %{ $arg } );
 	eval "$arg->{rules}";
 	die "error evaling $arg->{rules}: $@\n" if ($@);
-	return get_ds();
+
+	return _get_ds();
 }
 
-=head2 set_rec
+=head2 _set_rec
 
 Set current record hash
 
-  set_rec( $rec );
+  _set_rec( $rec );
 
 =cut
 
 my $rec;
 
-sub set_rec {
+sub _set_rec {
 	$rec = shift or die "no record hash";
 }
 
+=head2 _get_ds
+
+Return hash formatted as data structure
+
+  my $ds = _get_ds();
+
+=cut
+
+my $out;
+my $marc21;
+my $marc_encoding;
+
+sub _get_ds {
+	return $out;
+}
+
+=head2 _clean_ds
+
+Clean data structure hash for next record
+
+  _clean_ds();
+
+=cut
+
+sub _clean_ds {
+	my $a = {@_};
+	$out = undef;
+	$marc21 = undef;
+	$marc_encoding = $a->{marc_encoding};
+}
+
+=head2 _set_lookup
+
+Set current lookup hash
+
+  _set_lookup( $lookup );
+
+=cut
+
+my $lookup;
+
+sub _set_lookup {
+	$lookup = shift;
+}
+
+=head2 _get_marc21_fields
+
+Get all fields defined by calls to C<marc21>
+
+	$marc->add_fields( WebPAC::Normalize:_get_marc21_fields() );
+
+
+
+We are using I<magic> which detect repeatable fields only from
+sequence of field/subfield data generated by normalization.
+
+Repeatable field is created if there is second occurence of same subfield or
+if any of indicators are different. This is sane for most cases except for
+non-repeatable fields with repeatable subfields.
+
+B<TODO>: implement exceptions to magic
+
+=cut
+
+sub _get_marc21_fields {
+	my @m;
+	my $last;
+	foreach my $row (@{ $marc21 }) {
+		if ($last &&
+			$last->[0] eq $row->[0] &&		# check if field is same
+			$last->[1] eq $row->[1] &&		# check for i1
+			$last->[2] eq $row->[2]			# and for i2
+		) {
+			push @$last, ( $row->[3] , $row->[4] );
+			warn "## ++ added $row->[0] ^$row->[3] to $last->[0]\n";
+			next;
+		} elsif ($last) {
+			push @m, $last;
+		}
+
+		$last = $row;
+	}
+
+	push @m, $last if ($last);
+
+	return @m;
+}
+
+=head1 Functions to create C<data_structure>
+
+Those functions generally have to first in your normalization file.
+
 =head2 tag
 
 Define new tag for I<search> and I<display>.
@@ -94,8 +202,6 @@
 
 =cut
 
-my $out;
-
 sub tag {
 	my $name = shift or die "tag needs name as first argument";
 	my @o = grep { defined($_) && $_ ne '' } @_;
@@ -137,43 +243,32 @@
 	$out->{$name}->{search} = \@o;
 }
 
-=head2 get_ds
-
-Return hash formatted as data structure
-
-  my $ds = get_ds();
-
-=cut
-
-sub get_ds {
-	return $out;
-}
-
-=head2 clean_ds
+=head2 marc21
 
-Clean data structure hash for next record
+Save value for MARC field
 
-  clean_ds();
+  marc21('900','a', rec('200','a') );
 
 =cut
 
-sub clean_ds {
-	$out = undef;
+sub marc21 {
+	my $f = shift or die "marc21 needs field";
+	die "marc21 field must be numer" unless ($f =~ /^\d+$/);
+
+	my $sf = shift or die "marc21 needs subfield";
+
+	foreach (@_) {
+		my $v = $_;		# make var read-write for Encode
+		next unless (defined($v) && $v !~ /^\s*$/);
+		from_to($v, 'iso-8859-2', $marc_encoding) if ($marc_encoding);
+		push @{ $marc21 }, [ $f, ' ', ' ', $sf => $v ];
+	}
 }
 
-=head2 set_lookup
+=head1 Functions to extract data from input
 
-Set current lookup hash
-
-  set_lookup( $lookup );
-
-=cut
-
-my $lookup;
-
-sub set_lookup {
-	$lookup = shift;
-}
+This function should be used inside functions to create C<data_structure> described
+above.
 
 =head2 rec1