--- trunk/lib/WebPAC/Normalize.pm	2005/07/17 00:04:25	14
+++ trunk/lib/WebPAC/Normalize.pm	2006/06/30 18:48:33	550
@@ -1,535 +1,551 @@
 package WebPAC::Normalize;
+use Exporter 'import';
+@EXPORT = qw/
+	_set_rec _set_lookup
+	_get_ds _clean_ds
+
+	tag search display
+	marc marc_indicators marc_repeatable_subfield
+
+	rec1 rec2 rec
+	regex prefix suffix surround
+	first lookup join_with
+/;
 
 use warnings;
 use strict;
-use Data::Dumper;
-use Storable;
+
+#use base qw/WebPAC::Common/;
+use Data::Dump qw/dump/;
+use Encode qw/from_to/;
+
+# debugging warn(s)
+my $debug = 0;
+
 
 =head1 NAME
 
-WebPAC::Normalize - normalisation of source file
+WebPAC::Normalize - describe normalisaton rules using sets
 
 =head1 VERSION
 
-Version 0.01
+Version 0.06
 
 =cut
 
-our $VERSION = '0.01';
+our $VERSION = '0.06';
 
 =head1 SYNOPSIS
 
-This package contains code that could be helpful in implementing different
-normalisation front-ends.
+This module uses C<conf/normalize/*.pl> files to perform normalisation
+from input records using perl functions which are specialized for set
+processing.
+
+Sets are implemented as arrays, and normalisation file is valid perl, which
+means that you check it's validity before running WebPAC using
+C<perl -c normalize.pl>.
+
+Normalisation can generate multiple output normalized data. For now, supported output
+types (on the left side of definition) are: C<tag>, C<display>, C<search> and
+C<marc>.
 
 =head1 FUNCTIONS
 
-=head2 new
+Functions which start with C<_> are private and used by WebPAC internally.
+All other functions are available for use within normalisation rules.
 
-Create new normalisation object
+=head2 data_structure
 
-  my $n = new WebPAC::Normalize::Something(
-	cache_data_structure => './cache/ds/',
-	lookup_regex => $lookup->regex,
+Return data structure
+
+  my $ds = WebPAC::Normalize::data_structure(
+  	lookup => $lookup->lookup_hash,
+	row => $row,
+	rules => $normalize_pl_config,
+	marc_encoding => 'utf-8',
   );
 
-Optional parameter C<cache_data_structure> defines path to directory
-in which cache file for C<data_structure> call will be created.
+Options C<lookup>, C<row>, C<rules> and C<log> are mandatory while all
+other are optional.
+
+This function will B<die> if normalizastion can't be evaled.
 
-Recommended parametar C<lookup_regex> is used to enable parsing of lookups
-in structures.
+Since this function isn't exported you have to call it with 
+C<WebPAC::Normalize::data_structure>.
 
 =cut
 
-sub new {
-	my $class = shift;
-        my $self = {@_};
-        bless($self, $class);
+sub data_structure {
+	my $arg = {@_};
 
-	$self->setup_cache_dir( $self->{'cache_data_structure'} );
+	die "need row argument" unless ($arg->{row});
+	die "need normalisation argument" unless ($arg->{rules});
 
-	$self ? return $self : return undef;
-}
+	no strict 'subs';
+	_set_lookup( $arg->{lookup} );
+	_set_rec( $arg->{row} );
+	_clean_ds( %{ $arg } );
+	eval "$arg->{rules}";
+	die "error evaling $arg->{rules}: $@\n" if ($@);
 
-=head2 setup_cache_dir
+	return _get_ds();
+}
 
-Check if specified cache directory exist, and if not, disable caching.
+=head2 _set_rec
 
- $setup_cache_dir('./cache/ds/');
+Set current record hash
 
-If you pass false or zero value to this function, it will disable
-cacheing.
+  _set_rec( $rec );
 
 =cut
 
-sub setup_cache_dir {
-	my $self = shift;
+my $rec;
 
-	my $dir = shift;
+sub _set_rec {
+	$rec = shift or die "no record hash";
+}
 
-	my $log = $self->_get_logger();
+=head2 _get_ds
 
-	if ($dir) {
-		my $msg;
-		if (! -e $dir) {
-			$msg = "doesn't exist";
-		} elsif (! -d $dir) {
-			$msg = "is not directory";
-		} elsif (! -w $dir) {
-			$msg = "not writable";
-		}
+Return hash formatted as data structure
 
-		if ($msg) {
-			undef $self->{'cache_data_structure'};
-			$log->warn("cache_data_structure $dir $msg, disabling...");
-		} else {
-			$log->debug("using cache dir $dir");
-		}
-	} else {
-		$log->debug("disabling cache");
-		undef $self->{'cache_data_structure'};
-	}
+  my $ds = _get_ds();
+
+=cut
+
+my ($out,$marc_record, $marc_encoding, $marc_repeatable_subfield, $marc_indicators);
+
+sub _get_ds {
+	return $out;
 }
 
+=head2 _clean_ds
 
-=head2 data_structure
+Clean data structure hash for next record
+
+  _clean_ds();
 
-Create in-memory data structure which represents normalized layout from
-C<conf/normalize/*.xml>.
+=cut
 
-This structures are used to produce output.
+sub _clean_ds {
+	my $a = {@_};
+	($out,$marc_record, $marc_encoding, $marc_repeatable_subfield, $marc_indicators) = ();
+	$marc_encoding = $a->{marc_encoding};
+}
 
- my @ds = $webpac->data_structure($rec);
+=head2 _set_lookup
 
-B<Note: historical oddity follows>
+Set current lookup hash
 
-This method will also set C<< $webpac->{'currnet_filename'} >> if there is
-C<< <filename> >> tag and C<< $webpac->{'headline'} >> if there is
-C<< <headline> >> tag.
+  _set_lookup( $lookup );
 
 =cut
 
-sub data_structure {
-	my $self = shift;
+my $lookup;
+
+sub _set_lookup {
+	$lookup = shift;
+}
 
-	my $log = $self->_get_logger();
+=head2 _get_marc_fields
 
-	my $rec = shift;
-	$log->logconfess("need HASH as first argument!") if ($rec !~ /HASH/o);
+Get all fields defined by calls to C<marc>
 
-	my $cache_file;
+	$marc->add_fields( WebPAC::Normalize:_get_marc_fields() );
 
-	if (my $cache_path = $self->{'cache_data_structure'}) {
-		my $id = $rec->{'000'};
-		$id = $rec->{'000'}->[0] if ($id =~ m/^ARRAY/o);
-		unless (defined($id)) {
-			$log->warn("Can't use cache_data_structure on records without unique identifier in field 000");
-			undef $self->{'cache_data_structure'};
-		} else {
-			$cache_file = "$cache_path/$id";
-			if (-r $cache_file) {
-				my $ds_ref = retrieve($cache_file);
-				if ($ds_ref) {
-					$log->debug("cache hit: $cache_file");
-					my $ok = 1;
-					foreach my $f (qw(current_filename headline)) {
-						if ($ds_ref->{$f}) {
-							$self->{$f} = $ds_ref->{$f};
-						} else {
-							$ok = 0;
-						}
-					};
-					if ($ok && $ds_ref->{'ds'}) {
-						return @{ $ds_ref->{'ds'} };
-					} else {
-						$log->warn("cache_data_structure $cache_path corrupt. Use rm $cache_path/* to re-create it on next run!");
-						undef $self->{'cache_data_structure'};
-					}
-				}
-			}
-		}
-	}
 
-	undef $self->{'currnet_filename'};
-	undef $self->{'headline'};
 
-	my @sorted_tags;
-	if ($self->{tags_by_order}) {
-		@sorted_tags = @{$self->{tags_by_order}};
-	} else {
-		@sorted_tags = sort { $self->_sort_by_order } keys %{$self->{'import_xml'}->{'indexer'}};
-		$self->{tags_by_order} = \@sorted_tags;
-	}
+We are using I<magic> which detect repeatable fields only from
+sequence of field/subfield data generated by normalization.
 
-	my @ds;
+Repeatable field is created if there is second occurence of same subfield or
+if any of indicators are different. This is sane for most cases except for
+non-repeatable fields with repeatable subfields.
 
-	$log->debug("tags: ",sub { join(", ",@sorted_tags) });
+You can change behaviour of that using C<marc_repeatable_subfield>.
 
-	foreach my $field (@sorted_tags) {
+=cut
 
-		my $row;
+sub _get_marc_fields {
 
-#print "field $field [",$self->{'tag'},"] = ",Dumper($self->{'import_xml'}->{'indexer'}->{$field}->{$self->{'tag'}});
 
-		foreach my $tag (@{$self->{'import_xml'}->{'indexer'}->{$field}->{$self->{'tag'}}}) {
-			my $format = $tag->{'value'} || $tag->{'content'};
+	# first, sort all existing fields 
+	# XXX might not be needed, but modern perl might randomize elements in hash
+	my @sorted_marc_record = sort {
+		$a->[0] . $a->[3] cmp $b->[0] . $b->[3]	
+	} @{ $marc_record };
 
-			$log->debug("format: $format");
+	# output marc fields
+	my @m;
 
-			my @v;
-			if ($self->{'lookup_regex'} && $format =~ $self->{'lookup_regex'}) {
-				@v = $self->fill_in_to_arr($rec,$format);
-			} else {
-				@v = $self->parse_to_arr($rec,$format);
-			}
-			next if (! @v);
+	# count unique field-subfields (used for offset when walking to next subfield)
+	my $u;
+	map { $u->{ $_->[0] . $_->[3]  }++ } @sorted_marc_record;
 
-			if ($tag->{'sort'}) {
-				@v = $self->sort_arr(@v);
-			}
+	if ($debug) {
+		warn "## marc_repeatable_subfield ", dump( $marc_repeatable_subfield ), $/;
+		warn "## marc_record ", dump( $marc_record ), $/;
+		warn "## sorted_marc_record ", dump( \@sorted_marc_record ), $/;
+		warn "## subfield count ", dump( $u ), $/;
+	}
 
-			# use format?
-			if ($tag->{'format_name'}) {
-				@v = map { $self->apply_format($tag->{'format_name'},$tag->{'format_delimiter'},$_) } @v;
-			}
+	my $len = $#sorted_marc_record;
+	my $visited;
+	my $i = 0;
+	my $field;
 
-			if ($field eq 'filename') {
-				$self->{'current_filename'} = join('',@v);
-				$log->debug("filename: ",$self->{'current_filename'});
-			} elsif ($field eq 'headline') {
-				$self->{'headline'} .= join('',@v);
-				$log->debug("headline: ",$self->{'headline'});
-				next; # don't return headline in data_structure!
-			}
+	foreach ( 0 .. $len ) {
 
-			# delimiter will join repeatable fields
-			if ($tag->{'delimiter'}) {
-				@v = ( join($tag->{'delimiter'}, @v) );
-			}
+		# find next element which isn't visited
+		while ($visited->{$i}) {
+			$i = ($i + 1) % ($len + 1);
+		}
 
-			# default types 
-			my @types = qw(display swish);
-			# override by type attribute
-			@types = ( $tag->{'type'} ) if ($tag->{'type'});
-
-			foreach my $type (@types) {
-				# append to previous line?
-				$log->debug("type: $type ",sub { join(" ",@v) }, $row->{'append'} || 'no append');
-				if ($tag->{'append'}) {
-
-					# I will delimit appended part with
-					# delimiter (or ,)
-					my $d = $tag->{'delimiter'};
-					# default delimiter
-					$d ||= " ";
-
-					my $last = pop @{$row->{$type}};
-					$d = "" if (! $last);
-					$last .= $d . join($d, @v);
-					push @{$row->{$type}}, $last;
-
-				} else {
-					push @{$row->{$type}}, @v;
-				}
-			}
+		# mark it visited
+		$visited->{$i}++;
+
+		my $row = $sorted_marc_record[$i];
 
+		# field and subfield which is key for
+		# marc_repeatable_subfield and u
+		my $fsf = $row->[0] . $row->[3];
+
+		if ($debug > 1) {
+
+			print "### field so far [", $#$field, "] : ", dump( $field ), " ", $field ? 'T' : 'F', $/;
+			print "### this [$i]: ", dump( $row ),$/;
+			print "### sf: ", $row->[3], " vs ", $field->[3],
+				$marc_repeatable_subfield->{ $row->[0] . $row->[3] } ? ' (repeatable)' : '', $/,
+				if ($#$field >= 0);
 
 		}
 
-		if ($row) {
-			$row->{'tag'} = $field;
+		# if field exists
+		if ( $#$field >= 0 ) {
+			if (
+				$row->[0] ne $field->[0] ||		# field
+				$row->[1] ne $field->[1] ||		# i1
+				$row->[2] ne $field->[2]		# i2
+			) {
+				push @m, $field;
+				warn "## saved/1 ", dump( $field ),$/ if ($debug);
+				$field = $row;
+
+			} elsif (
+				( $row->[3] lt $field->[-2] )		# subfield which is not next (e.g. a after c)
+				||
+				( $row->[3] eq $field->[-2] &&		# same subfield, but not repeatable
+					! $marc_repeatable_subfield->{ $fsf }
+				)
+			) {
+				push @m, $field;
+				warn "## saved/2 ", dump( $field ),$/ if ($debug);
+				$field = $row;
 
-			# TODO: name_sigular, name_plural
-			my $name = $self->{'import_xml'}->{'indexer'}->{$field}->{'name'};
-			$row->{'name'} = $name ? $self->_x($name) : $field;
-
-			# post-sort all values in field
-			if ($self->{'import_xml'}->{'indexer'}->{$field}->{'sort'}) {
-				$log->warn("sort at field tag not implemented");
+			} else {
+				# append new subfields to existing field
+				push @$field, ( $row->[3], $row->[4] );
 			}
-
-			push @ds, $row;
-
-			$log->debug("row $field: ",sub { Dumper($row) });
+		} else {
+			# insert first field
+			$field = $row;
 		}
 
+		if (! $marc_repeatable_subfield->{ $fsf }) {
+			# make step to next subfield
+			$i = ($i + $u->{ $fsf } ) % ($len + 1);
+		}
 	}
 
-	if ($cache_file) {
-		store {
-			ds => \@ds,
-			current_filename => $self->{'current_filename'},
-			headline => $self->{'headline'},
-		}, $cache_file;
-		$log->debug("created storable cache file $cache_file");
+	if ($#$field >= 0) {
+		push @m, $field;
+		warn "## saved/3 ", dump( $field ),$/ if ($debug);
 	}
 
-	return @ds;
-
+	return @m;
 }
 
-=head2 apply_format
+=head1 Functions to create C<data_structure>
+
+Those functions generally have to first in your normalization file.
 
-Apply format specified in tag with C<format_name="name"> and
-C<format_delimiter=";;">.
+=head2 tag
 
- my $text = $webpac->apply_format($format_name,$format_delimiter,$data);
+Define new tag for I<search> and I<display>.
+
+  tag('Title', rec('200','a') );
 
-Formats can contain C<lookup{...}> if you need them.
 
 =cut
 
-sub apply_format {
-	my $self = shift;
+sub tag {
+	my $name = shift or die "tag needs name as first argument";
+	my @o = grep { defined($_) && $_ ne '' } @_;
+	return unless (@o);
+	$out->{$name}->{tag} = $name;
+	$out->{$name}->{search} = \@o;
+	$out->{$name}->{display} = \@o;
+}
 
-	my ($name,$delimiter,$data) = @_;
+=head2 display
 
-	my $log = $self->_get_logger();
+Define tag just for I<display>
 
-	if (! $self->{'import_xml'}->{'format'}->{$name}) {
-		$log->warn("<format name=\"$name\"> is not defined in ",$self->{'import_xml_file'});
-		return $data;
-	}
+  @v = display('Title', rec('200','a') );
 
-	$log->warn("no delimiter for format $name") if (! $delimiter);
+=cut
 
-	my $format = $self->_x($self->{'import_xml'}->{'format'}->{$name}->{'content'}) || $log->logdie("can't find format '$name'");
+sub display {
+	my $name = shift or die "display needs name as first argument";
+	my @o = grep { defined($_) && $_ ne '' } @_;
+	return unless (@o);
+	$out->{$name}->{tag} = $name;
+	$out->{$name}->{display} = \@o;
+}
 
-	my @data = split(/\Q$delimiter\E/, $data);
+=head2 search
 
-	my $out = sprintf($format, @data);
-	$log->debug("using format $name [$format] on $data to produce: $out");
+Prepare values just for I<search>
 
-	if ($self->{'lookup_regex'} && $out =~ $self->{'lookup_regex'}) {
-		return $self->lookup($out);
-	} else {
-		return $out;
-	}
+  @v = search('Title', rec('200','a') );
 
+=cut
+
+sub search {
+	my $name = shift or die "search needs name as first argument";
+	my @o = grep { defined($_) && $_ ne '' } @_;
+	return unless (@o);
+	$out->{$name}->{tag} = $name;
+	$out->{$name}->{search} = \@o;
 }
 
-=head2 parse
+=head2 marc
 
-Perform smart parsing of string, skipping delimiters for fields which aren't
-defined. It can also eval code in format starting with C<eval{...}> and
-return output or nothing depending on eval code.
+Save value for MARC field
 
- my $text = $webpac->parse($rec,'eval{"v901^a" eq "Deskriptor"}descriptor: v250^a', $i);
+  marc('900','a', rec('200','a') );
 
 =cut
 
-sub parse {
-	my $self = shift;
+sub marc {
+	my $f = shift or die "marc needs field";
+	die "marc field must be numer" unless ($f =~ /^\d+$/);
 
-	my ($rec, $format_utf8, $i) = @_;
+	my $sf = shift or die "marc needs subfield";
 
-	return if (! $format_utf8);
+	foreach (@_) {
+		my $v = $_;		# make var read-write for Encode
+		next unless (defined($v) && $v !~ /^\s*$/);
+		from_to($v, 'iso-8859-2', $marc_encoding) if ($marc_encoding);
+		my ($i1,$i2) = defined($marc_indicators->{$f}) ? @{ $marc_indicators->{$f} } : (' ',' ');
+		push @{ $marc_record }, [ $f, $i1, $i2, $sf => $v ];
+	}
+}
 
-	my $log = $self->_get_logger();
+=head2 marc_repeatable_subfield
 
-	$log->logconfess("need HASH as first argument!") if ($rec !~ /HASH/o);
+Save values for MARC repetable subfield
 
-	$i = 0 if (! $i);
+  marc_repeatable_subfield('910', 'z', rec('909') );
 
-	my $format = $self->_x($format_utf8) || $log->logconfess("can't convert '$format_utf8' from UTF-8 to ",$self->{'code_page'});
+=cut
 
-	my @out;
+sub marc_repeatable_subfield {
+	my ($f,$sf) = @_;
+	die "marc_repeatable_subfield need field and subfield!\n" unless ($f && $sf);
+	$marc_repeatable_subfield->{ $f . $sf }++;
+	marc(@_);
+}
 
-	$log->debug("format: $format");
+=head2 marc_indicators
 
-	my $eval_code;
-	# remove eval{...} from beginning
-	$eval_code = $1 if ($format =~ s/^eval{([^}]+)}//s);
+Set both indicators for MARC field
 
-	my $filter_name;
-	# remove filter{...} from beginning
-	$filter_name = $1 if ($format =~ s/^filter{([^}]+)}//s);
+  marc_indicators('900', ' ', 1);
 
-	my $prefix;
-	my $all_found=0;
+Any indicator value other than C<0-9> will be treated as undefined.
 
-	while ($format =~ s/^(.*?)(v|s)(\d+)(?:\^(\w))?//s) {
+=cut
 
-		my $del = $1 || '';
-		$prefix ||= $del if ($all_found == 0);
+sub marc_indicators {
+	my $f = shift || die "marc_indicators need field!\n";
+	my ($i1,$i2) = @_;
+	die "marc_indicators($f, ...) need i1!\n" unless(defined($i1));
+	die "marc_indicators($f, $i1, ...) need i2!\n" unless(defined($i2));
 
-		# repeatable index
-		my $r = $i;
-		$r = 0 if (lc("$2") eq 's');
+	$i1 = ' ' if ($i1 !~ /^\d$/);
+	$i2 = ' ' if ($i2 !~ /^\d$/);
+	@{ $marc_indicators->{$f} } = ($i1,$i2);
+}
 
-		my $found = 0;
-		my $tmp = $self->get_data(\$rec,$3,$4,$r,\$found);
 
-		if ($found) {
-			push @out, $del;
-			push @out, $tmp;
-			$all_found += $found;
-		}
-	}
+=head1 Functions to extract data from input
 
-	return if (! $all_found);
+This function should be used inside functions to create C<data_structure> described
+above.
 
-	my $out = join('',@out);
+=head2 rec1
 
-	if ($out) {
-		# add rest of format (suffix)
-		$out .= $format;
+Return all values in some field
 
-		# add prefix if not there
-		$out = $prefix . $out if ($out !~ m/^\Q$prefix\E/);
+  @v = rec1('200')
 
-		$log->debug("result: $out");
-	}
+TODO: order of values is probably same as in source data, need to investigate that
 
-	if ($eval_code) {
-		my $eval = $self->fill_in($rec,$eval_code,$i) || return;
-		$log->debug("about to eval{$eval} format: $out");
-		return if (! $self->_eval($eval));
-	}
-	
-	if ($filter_name && $self->{'filter'}->{$filter_name}) {
-		$log->debug("about to filter{$filter_name} format: $out");
-		$out = $self->{'filter'}->{$filter_name}->($out);
-		return unless(defined($out));
-		$log->debug("filter result: $out");
-	}
+=cut
 
-	return $out;
+sub rec1 {
+	my $f = shift;
+	return unless (defined($rec) && defined($rec->{$f}));
+	if (ref($rec->{$f}) eq 'ARRAY') {
+		return map { 
+			if (ref($_) eq 'HASH') {
+				values %{$_};
+			} else {
+				$_;
+			}
+		} @{ $rec->{$f} };
+	} elsif( defined($rec->{$f}) ) {
+		return $rec->{$f};
+	}
 }
 
-=head2 parse_to_arr
+=head2 rec2
 
-Similar to C<parse>, but returns array of all repeatable fields
+Return all values in specific field and subfield
 
- my @arr = $webpac->parse_to_arr($rec,'v250^a');
+  @v = rec2('200','a')
 
 =cut
 
-sub parse_to_arr {
-	my $self = shift;
+sub rec2 {
+	my $f = shift;
+	return unless (defined($rec && $rec->{$f}));
+	my $sf = shift;
+	return map { $_->{$sf} } grep { ref($_) eq 'HASH' && $_->{$sf} } @{ $rec->{$f} };
+}
 
-	my ($rec, $format_utf8) = @_;
+=head2 rec
 
-	my $log = $self->_get_logger();
+syntaxtic sugar for
 
-	$log->logconfess("need HASH as first argument!") if ($rec !~ /HASH/o);
-	return if (! $format_utf8);
+  @v = rec('200')
+  @v = rec('200','a')
 
-	my $i = 0;
-	my @arr;
+=cut
 
-	while (my $v = $self->parse($rec,$format_utf8,$i++)) {
-		push @arr, $v;
+sub rec {
+	if ($#_ == 0) {
+		return rec1(@_);
+	} elsif ($#_ == 1) {
+		return rec2(@_);
 	}
-
-	$log->debug("format '$format_utf8' returned ",--$i," elements: ", sub { join(" | ",@arr) }) if (@arr);
-
-	return @arr;
 }
 
-=head2 fill_in_to_arr
+=head2 regex
 
-Similar to C<fill_in>, but returns array of all repeatable fields. Usable
-for fields which have lookups, so they shouldn't be parsed but rather
-C<fill_id>ed.
+Apply regex to some or all values
 
- my @arr = $webpac->fill_in_to_arr($rec,'[v900];;[v250^a]');
+  @v = regex( 's/foo/bar/g', @v );
 
 =cut
 
-sub fill_in_to_arr {
-	my $self = shift;
-
-	my ($rec, $format_utf8) = @_;
-
-	my $log = $self->_get_logger();
+sub regex {
+	my $r = shift;
+	my @out;
+	#warn "r: $r\n", dump(\@_);
+	foreach my $t (@_) {
+		next unless ($t);
+		eval "\$t =~ $r";
+		push @out, $t if ($t && $t ne '');
+	}
+	return @out;
+}
 
-	$log->logconfess("need HASH as first argument!") if ($rec !~ /HASH/o);
-	return if (! $format_utf8);
+=head2 prefix
 
-	my $i = 0;
-	my @arr;
+Prefix all values with a string
 
-	while (my @v = $self->fill_in($rec,$format_utf8,$i++)) {
-		push @arr, @v;
-	}
+  @v = prefix( 'my_', @v );
 
-	$log->debug("format '$format_utf8' returned ",--$i," elements: ", sub { join(" | ",@arr) }) if (@arr);
+=cut
 
-	return @arr;
+sub prefix {
+	my $p = shift or die "prefix needs string as first argument";
+	return map { $p . $_ } grep { defined($_) } @_;
 }
 
-=head2 sort_arr
+=head2 suffix
 
-Sort array ignoring case and html in data
+suffix all values with a string
 
- my @sorted = $webpac->sort_arr(@unsorted);
+  @v = suffix( '_my', @v );
 
 =cut
 
-sub sort_arr {
-	my $self = shift;
+sub suffix {
+	my $s = shift or die "suffix needs string as first argument";
+	return map { $_ . $s } grep { defined($_) } @_;
+}
+
+=head2 surround
 
-	my $log = $self->_get_logger();
+surround all values with a two strings
 
-	# FIXME add Schwartzian Transformation?
+  @v = surround( 'prefix_', '_suffix', @v );
 
-	my @sorted = sort {
-		$a =~ s#<[^>]+/*>##;
-		$b =~ s#<[^>]+/*>##;
-		lc($b) cmp lc($a)
-	} @_;
-	$log->debug("sorted values: ",sub { join(", ",@sorted) });
+=cut
 
-	return @sorted;
+sub surround {
+	my $p = shift or die "surround need prefix as first argument";
+	my $s = shift or die "surround needs suffix as second argument";
+	return map { $p . $_ . $s } grep { defined($_) } @_;
 }
 
+=head2 first
 
-=head2 _sort_by_order
+Return first element
 
-Sort xml tags data structure accoding to C<order=""> attribute.
+  $v = first( @v );
 
 =cut
 
-sub _sort_by_order {
-	my $self = shift;
-
-	my $va = $self->{'import_xml'}->{'indexer'}->{$a}->{'order'} ||
-		$self->{'import_xml'}->{'indexer'}->{$a};
-	my $vb = $self->{'import_xml'}->{'indexer'}->{$b}->{'order'} ||
-		$self->{'import_xml'}->{'indexer'}->{$b};
-
-	return $va <=> $vb;
+sub first {
+	my $r = shift;
+	return $r;
 }
 
-=head2 _x
-
-Convert strings from C<conf/normalize> encoding into application specific
-(optinally specified using C<code_page> to C<new> constructor.
+=head2 lookup
 
- my $text = $n->_x('normalize text string');
+Consult lookup hashes for some value
 
-This is a stub so that other modules doesn't have to implement it.
+  @v = lookup( $v );
+  @v = lookup( @v );
 
 =cut
 
-sub _x {
-	my $self = shift;
-	return shift;
+sub lookup {
+	my $k = shift or return;
+	return unless (defined($lookup->{$k}));
+	if (ref($lookup->{$k}) eq 'ARRAY') {
+		return @{ $lookup->{$k} };
+	} else {
+		return $lookup->{$k};
+	}
 }
 
+=head2 join_with
 
-=head1 AUTHOR
-
-Dobrica Pavlinusic, C<< <dpavlin@rot13.org> >>
-
-=head1 COPYRIGHT & LICENSE
+Joins walues with some delimiter
 
-Copyright 2005 Dobrica Pavlinusic, All Rights Reserved.
-
-This program is free software; you can redistribute it and/or modify it
-under the same terms as Perl itself.
+  $v = join_with(", ", @v);
 
 =cut
 
-1; # End of WebPAC::DB
+sub join_with {
+	my $d = shift;
+	return join($d, grep { defined($_) && $_ ne '' } @_);
+}
+
+# END
+1;