trunk/t/3-normalize.t

#!/usr/bin/perl -w

use strict;
use lib 'lib';

use Test::More tests => 357;

use utf8;

BEGIN {
        use_ok( 'WebPAC::Test' );
        use_ok( 'WebPAC::Normalize' );
}

cmp_ok(_debug(1), '==', 1, '_debug level');
cmp_ok(_debug(0), '==', 0, '_debug level');

diag "debug level for $0 is $debug" if ($debug);
if ($debug > 2) {
        diag "debug level for WebPAC::Normalize is ", _debug( $debug - 2 );
}

my $rec1 = {
        '200' => [{
                'a' => '200a',
                'b' => '200b',
                },{
                'c' => '200c',
                'd' => '200d',
                },{
                'a' => '200a*2',
                'd' => '200d*2',
                }],
        '201' => [{
                'x' => '201x',
                'y' => '201y',
                }],
        '900' => [
                '900-no_subfield'
                ],
        '901' => [{
                'a' => '900a',
                }],
        '902' => [{
                'z' => '900',
                }],
};

my $rec2 = {
 '675' => [ {
              'a' => '159.9'
            } ],
 '210' => [ {
              'c' => 'New York University press',
              'a' => 'New York',
              'd' => 'cop. 1988'
            } ],
 '700' => [ {
              'a' => 'Haynal',
              'b' => 'André'
            } ],
 '801' => [ 'FFZG' ],
 '991' => [ '8302' ],
 '000' => [ 1 ],
 '702' => [ {
              'a' => 'Holder',
              'b' => 'Elizabeth'
            } ],
 '215' => [ {
              'c' => 'ilustr',
              'a' => 'xix, 202 str',
              'd' => '23cm'
            } ],
 '990' => [
            '2140',
            '88',
            'HAY'
          ],
 '200' => [ {
              'e' => 'from Freud and Ferenczi to Michael balint',
              'a' => 'Controversies in psychoanalytic method',
              'g' => 'translated by Elizabeth Holder on the basisi of a first draft by Archie Hooton ; with a preface by Daniel N. Stern',
              'f' => 'by André E. Haynal'
            } ],
 '610' => [ 'povijest psihoanalize' ],
 '994' => [ {
              'c' => '',
              'a' => 'PS',
              'b' => 'MG'
            } ],
 '320' => [ 'Kazalo' ],
 '101' => [ 'ENG' ],
 '686' => [ '2140' ],
 '300' => [ 'Prijevod djela: ' ],
};


my $lookup_hash1 = {
        'db1' => {
                'input1' => {
                        'key1' => { 1 => 1 },
                        'key2' => { 2 => 1 },
                },
                'input2' => {
                        'key3' => { 3 => 1 },
                        'key4' => { 4 => 1 },
                },
        },
        'db2' => {
                'input3' => {
                        'key5' => { 5 => 1 },
                        'key6' => { 6 => 1 },
                },
        }
};

my $lookup_hash2 = {
        'db3' => {
                'input4' => {
                        'key7' => { 7 => 1 },
                        'key8' => { 8 => 1 },
                },
        }
};

sub test {
        print dump( @_ ), ("-" x 78), "\n";
        ok( defined(@_) );
}

# how much of string evaled to display?
my $max_eval_output = 170;

sub dump_error {
        my ($msg,$code) = @_;

        my @l = split(/[\n\r]/, $code);
        my $out = "$msg\n";

        foreach my $i ( 0 .. $#l ) {
                $out .= sprintf("%2d: %s\n", $i, $l[$i]);
        }

        return $out;
}

sub test_s {
        my $t = shift || die;

        my $eval_t = $t;
        $eval_t =~ s/[\n\r\s]+/ /gs;
        $eval_t = substr($eval_t,0,$max_eval_output) . '...' if (length($eval_t) > $max_eval_output);
        $eval_t =~ s/\\/\\\\/gs;

        my @__ret;
        eval "\@__ret = $t";
        ok(! $@, $@ ? dump_error($@, $t) : "eval: $eval_t = " . dump(@__ret));
        return \@__ret;
}

{
        no strict 'subs';
        use WebPAC::Normalize;

        ok(! _set_lookup( undef ), "set_lookup(undef)");

        _set_ds( $rec1 );

        cmp_ok( join(",", rec2('200','a') ), 'eq', '200a,200a*2', 'join rec2' );
        cmp_ok( join(",", rec2('200','a'), rec2('200','b') ), 'eq', '200a,200a*2,200b', 'join rec2 rec2' );
        cmp_ok( join(" * ", sort(rec1('200'), rec1('201') )), 'eq', '200a * 200a*2 * 200b * 200c * 200d * 200d*2 * 201x * 201y', 'join sort rec1 rec1' );
        diag "is_deeply checks\n";
        is_deeply( \[ rec1('200') ], \[ qw/200a 200b 200c 200d 200a*2 200d*2/ ] );
        is_deeply( \[ regex( 's/0/o/g', rec1('200') ) ],  \[ qw/2ooa 2oob 2ooc 2ood 2ooa*2 2ood*2/ ]);
        is_deeply( \[ grep { /\*/ } regex( 's/0/o/g', rec1('200') ) ], \[ qw/2ooa*2 2ood*2/ ]);
        is_deeply( \[ rec('902') ], \[ '900' ] );

        cmp_ok( rec('902'), 'eq', rec('902','z'), 'rec sf' );

        # simple list manipulatons
        cmp_ok( join('', prefix('ab', 'cd') ), 'eq', 'abcd', 'prefix');
        cmp_ok( join('-', prefix('', 'x', 'y') ), 'eq', 'x-y', 'prefix empty');
        cmp_ok( join('-', prefix(0, 'x', 'y') ), 'eq', '0x-0y', 'prefix 0');

        cmp_ok( join('', suffix('xy', 'cd') ), 'eq', 'cdxy', 'suffix');
        cmp_ok( join('-', suffix('', 'x', 'y' ) ), 'eq', 'x-y', 'suffix empty');
        cmp_ok( join('-', suffix(0, 'x', 'y' ) ), 'eq', 'x0-y0', 'suffix 0');

        cmp_ok( join('', surround('->', '<-', 'a','b','c') ), 'eq', '->a<-->b<-->c<-', 'surround');
        cmp_ok( join('-', surround('', '', 'x','y','z') ), 'eq', 'x-y-z', 'surround empty');
        cmp_ok( join('-', surround(0, 0, 'x','y','z') ), 'eq', '0x0-0y0-0z0', 'surround 0 0');

        # count
        my @el;
        for my $i ( 0 .. 10 ) {
                cmp_ok( count( @el ), '==', $i, "count($i)");
                push @el, "element $i";
        }

        # lookups

        throws_ok { _set_load_row() } qr/CODE/, 'empty _set_load_row()';

        ok(_set_load_row(sub {
                my ($database,$input,$mfn) = @_;
                diag "load_row( $database, $input, $mfn )" if ($debug);
                cmp_ok( $#_, '==', 2, 'have 3 arguments');
                ok($database, '_load_row database');
                ok($input, '_load_row input');
                ok($mfn, '_load_row mfn');
                return {
                        '900' => [{ x => '900x-' . $mfn , y => '900y-' . $mfn }],
                }

        }), '_set_load_row');

        my @v = qw/foo bar baz aaa bbb ccc ddd/;

        my @accumulated;

        for my $i ( 0 .. $#v ) {

                my $mfn = 1000 + $i;

                ok(WebPAC::Normalize::_set_config({ '_mfn' => $mfn }), "_set_config _mfn=$mfn");

                my $size = $#v + 1;

                cmp_ok(
                        save_into_lookup('db','input','key', sub { @v }),
                        '==', $size, "save_into_lookup $size values"
                );

                ok(my $l = WebPAC::Normalize::_get_lookup(), '_get_lookup');
                diag "_get_lookup = ", dump($l) if ($debug);

                my @lookup;

                ok(@lookup = lookup(
                                sub {
                                        diag "in show" if ($debug);
                                        rec('900','x');
                                },
                                'db','input','key',
                                sub {
                                        return @v;
                                }
                        ),
                "lookup db/input/key");

                push @accumulated, '900x-' . $mfn;

                is_deeply(\@lookup, \@accumulated, "lookup db/input/key");

                shift @v;

        }

        ok(my $l = WebPAC::Normalize::_get_lookup(), '_get_lookup');
        diag "_get_lookup = ", dump($l) if ($debug);

        is_deeply( $l, {
                db => {
                        input => {
                                key => {
                                        foo => { 1000 => 1 },
                                        bar => { 1000 => 1, 1001 => 1 },
                                        baz => { 1000 => 1, 1001 => 1, 1002 => 1 },
                                        aaa => { 1000 => 1, 1001 => 1, 1002 => 1, 1003 => 1 },
                                        bbb => { 1000 => 1, 1001 => 1, 1002 => 1, 1003 => 1, 1004 => 1 },
                                        ccc => { 1000 => 1, 1001 => 1, 1002 => 1, 1003 => 1, 1004 => 1, 1005 => 1 },
                                        ddd => { 1000 => 1, 1001 => 1, 1002 => 1, 1003 => 1, 1004 => 1, 1005 => 1, 1006 => 1 },
                                },
                        },
                },
        }, 'lookup data');

#######

        diag "lookup_hash1 = ", dump($lookup_hash1) if ($debug);
        ok(_set_lookup( $lookup_hash1 ), '_set_lookup $lookup_hash1');

        throws_ok { _set_load_row() } qr/CODE/, 'empty _set_load_row()';

        ok(_set_load_row(sub {
                my ($database,$input,$mfn) = @_;
                diag "load_row( $database, $input, $mfn )";
                cmp_ok( $#_, '==', 2, 'have 3 arguments');
                ok($database, 'database');
                ok($input, 'input');
                ok($mfn, 'mfn');

        }), '_set_load_row');


#       cmp_ok(lookup(
#               sub {
#                       'found'
#               },
#               'db1','input1','key1',
#               sub {
#                       rec('200','a')
#               }
#       ), 'eq', 'found', 'lookup db1/input1/key1');


#       cmp_ok(
#               lookup( 
#               ),
#       'eq', 'lookup 1 i lookup 2', 'join lookup regex rec2');

        # check join_with operations

        sub test_join_with_2 {
                my ($a,$b,$e) = @_;

                cmp_ok(
                        join_with(" <1> ",
                                rec('201',$a),
                                rec('201',$b),
                        ), 
                'eq', $e, "join_with $a <1> $b = $e");
        }

        test_join_with_2('_','_','');
        test_join_with_2('x','_','201x');
        test_join_with_2('_','x','201x');
        test_join_with_2('x','y','201x <1> 201y');

        sub test_join_with_3 {
                my ($a,$b,$c,$e) = @_;

                cmp_ok(
                        join_with(" <1> ", rec('201',$a),
                                join_with(" <2> ", rec('201',$b),
                                        rec('201',$c),
                                )
                        ), 
                'eq', $e, "join_with $a <1> $b <2> $c = $e");
        };

        test_join_with_3('_','_','_','');
        test_join_with_3('x','_','_','201x');
        test_join_with_3('_','x','_','201x');
        test_join_with_3('_','_','x','201x');
        test_join_with_3('x','y','_','201x <1> 201y');
        test_join_with_3('x','_','y','201x <1> 201y');
        test_join_with_3('_','x','y','201x <2> 201y');
        test_join_with_3('x','_','y','201x <1> 201y');
        test_join_with_3('x','y','x','201x <1> 201y <2> 201x');

        # test lookups

        _set_lookup( $lookup_hash2 );

        throws_ok { lookup() } qr/need/, 'empty lookup';

        #is_deeply( \[ lookup( prefix( '00', rec('902') ) ) ], \[ 'lookup' ], 'lookup prefix' );

        #ok(! lookup('non-existent'), 'lookup non-existant' );

        _set_ds( $rec2 );

        test_s(qq{
                search_display('Title',
                        rec('200','a')
                );
        });
        test_s(qq{
                search_display('Who',
                        join_with(" ",
                                rec('702','a'),
                                rec('702','b')
                        )
                );
        });

        test_s(qq{
                display('Publisher',
                        rec('210','c')
                )
        });
        
        test_s(qq{
                search('Year',
                        regex( 's/[^\\d]+//',
                                rec('210','d')
                        )
                )
        });

        ok(my $ds = _get_ds(), "get_ds");
        diag "ds = ", dump($ds) if ($debug);


        sub test_check_ds {

                my $t = shift;

                ok($ds = _get_ds(), 'get_ds');
                diag dump( $ds ) if ($debug);

                ok( $ds && $ds->{something}, 'get_ds->something exists' );
                ok( $ds && $ds->{something}->{$t}, 'get_ds->something->'.$t.' exists') if ($t);
                ok( $ds && !$ds->{empty}, 'get_ds->empty doesn\'t' );

                return $ds;
        }

        _clean_ds();
        test_s(qq{ search('something', '42'); });
        test_s(qq{ search('empty', ''); });
        test_check_ds('search');

        _clean_ds();
        test_s(qq{ display('something', '42'); });
        test_s(qq{ display('empty', ''); });
        test_check_ds('display');

        _clean_ds();
        test_s(qq{ search_display('something', '42'); });
        test_s(qq{ search_display('empty', ''); });
        test_check_ds('search');
        test_check_ds('display');

        _clean_ds();
        test_s(qq{ sorted('something', '42'); });
        test_s(qq{ sorted('empty', ''); });
        test_check_ds('sorted');

        _clean_ds();
        my $n = read_file( "$abs_path/data/normalize.pl" );
        $n .= "\n1;\n";
        #diag "normalize code:\n$n\n";
        test_s( $n );

        ok($ds = _get_ds(), "get_ds");
        diag "ds = ", dump($ds) if ($debug);

        my $rec = {
                '200' => [{
                        'a' => '200a',
                        'b' => '200b',
                }],
        };
        my $rules = qq{ search('mixed', rec('200') ) };
        
        _clean_ds();
        _set_ds( $rec );
        test_s( $rules );
        ok($ds = _get_ds(), "get_ds");
        is_deeply( $ds, {
                'mixed' => {
                        'search' => [ '200a', '200b' ],
                }
        }, 'correct get_ds');

        ok(my $ds2 = WebPAC::Normalize::data_structure(
                row => $rec,
                rules => $rules,
        ), 'data_structure');
        is_deeply( $ds, $ds2, 'data_structure(s) same');

        # wird and non-valid structure which is supported anyway
        _clean_ds();
        _set_ds({
                '200' => [{
                        'a' => '200a',
                },
                        '200-solo'
                ]
        });
        test_s(qq{ search('mixed', rec('200') ) });
        ok($ds = _get_ds(), "get_ds");
        is_deeply( $ds, {
                'mixed' => {
                        'search' => [ '200a', '200-solo' ],
                }
        }, 'correct get_ds');

        #
        # MARC
        #
        #_debug( 4 );

        test_s(qq{ marc_indicators('900',1,2) });
        test_s(qq{ marc('900','a', rec('200') ) });
        my $marc;
        ok($marc = WebPAC::Normalize::MARC::_get_marc_fields(), "_get_marc_fields");
        diag dump( $marc ) if ($debug);

        is_deeply( $marc, [
                [ '900', 1, 2, 'a', '200a' ],
                [ '900', 1, 2, 'a', '200-solo' ]
        ], 'correct marc with indicators');

        test_s(qq{ marc_indicators('900',' ',9) });
        test_s(qq{ marc_repeatable_subfield('900','a', rec('200') ) });

        ok($marc = WebPAC::Normalize::MARC::_get_marc_fields(), "_get_marc_fields");
        diag dump( $marc ) if ($debug);

        is_deeply( $marc, [
                [ '900', 1, 2, 'a', '200a', 'a', '200-solo' ],
                [ '900', ' ', 9, 'a', '200a', 'a', '200-solo' ]
        ], 'correct marc with repetable subfield');

        #
        # test magic re-ordering of input data
        #

        sub test_rec_rules {
                my ($msg, $rec, $rules, $struct) = @_;

                _clean_ds();
                _set_ds($rec);

                foreach my $r (split(/;\s*$/, $rules)) {
                        $r =~ s/[\s\n\r]+/ /gs;
                        $r =~ s/^\s+//gs;
                        $r =~ s/\s+$//gs;
                        diag "rule: $r" if $debug;
                        test_s($r) if ($r);
                }

                ok(my $marc = WebPAC::Normalize::MARC::_get_marc_fields(), "_get_marc_fields");
                diag dump( $marc ) if $debug;
                diag "expects:\n", dump($struct) if ($debug > 1);
                is_deeply( $marc, $struct, $msg );
        }

        test_rec_rules(
                'correct marc with repetable subfield',
                {
                        '200' => [{
                                'a' => '200a-1',
                                'b' => '200b-1',
                                'c' => '200c-1',
                        }, {
                                'a' => '200a-2',
                                'b' => '200b-2',
                        }, {
                                'a' => '200a-3',
                        }],
                },
                qq{
                        marc_indicators('900',1 ,0);
                        marc('900','a', rec('200','a') );
                        marc('900','b', rec('200','b') );
                        marc('900','c', rec('200','c') );
                },
                [
                        [ '900', 1, 0, 'a', '200a-1', 'b', '200b-1', 'c', '200c-1' ],
                        [ '900', 1, 0, 'a', '200a-2', 'b', '200b-2' ],
                        [ '900', 1, 0, 'a', '200a-3' ],
                ],
        );


        test_rec_rules(
                'marc_repeatable_subfield',
                {
                        '200' => [{
                                'a' => '200a-1',
                                'b' => '200b-1',
                                'c' => '200c-1',
                        }, {
                                'a' => '200a-2',
                                'b' => '200b-2',
                                'c' => '200c-2',
                        }, {
                                'a' => '200a-3',
                                'c' => '200c-3',
                        }],
                },
                qq{
                        marc_indicators('900',1 ,0);
                        marc_repeatable_subfield('900','a', rec('200','a') );
                        marc('900','b', rec('200','b') );
                        marc('900','c', rec('200','c') );
                },
                [
                        [ '900', 1, 0, 'a', '200a-1', 'a', '200a-2', 'a', '200a-3', 'b', '200b-1', 'c', '200c-1' ],
                        [ '900', 1, 0, 'b', '200b-2', 'c', '200c-2' ],
                        [ '900', 1, 0, 'c', '200c-3' ],
                ],
        );

        test_rec_rules(
                'marc_compose',
                { '200' => [{ a => 'foo ; bar', b => 42, c => 'baz' }] },
                qq{
                        marc_compose('900',
                                'c', rec(200,'b'),
                                'b', rec(200,'a'),
                                'a', rec(200,'c'),
                        );
                },
                [
                        [ '900', ' ', ' ', 'c', 42, 'b', 'foo ; bar', 'a', 'baz' ]
                ],
        );

        test_rec_rules(
                'marc_compose with + subfields',
                { '200' => [{ a => 'foo ; bar', b => 42, c => 'baz' }] },
                qq{
                        marc_compose('900',
                                'a', rec(200,'a'),
                                '+', prefix(" * ", rec(200,'c')),
                                'b', rec(200,'b'),
                                '+', prefix(" : ", rec(200,'c')),
                        );
                },
                [
                        [ '900', ' ', ' ', 'a', 'foo ; bar * baz', 'b', '42 : baz' ]
                ],
        );

        #
        # test rules
        #
        sub test_rule {
                my ($msg, $rec, $rule, $struct) = @_;
                _clean_ds();
                _set_ds( $rec );
                $rule =~ s/\\/\\/gs;
                my $r = test_s( $rule );
                diag "for ", dump($rec), " got:\n", dump($r), "\nexpect:\n" if ($debug > 1);
                diag dump($struct) if ($debug);
                is_deeply( $r, $struct, $msg );
        }

        # test split_rec_on
        test_rule(
                'split_rec_on',
                { '200' => [{ a => 'foo ; bar', b => 42, c => 'baz' }] },
                qq{ split_rec_on('200','a', qr/\\s*;\\s*/, 1) },
                [ 'foo' ],
        );
        test_rule(
                'split_rec_on',
                { '200' => [{ a => 'foo ; bar', b => 42, c => 'baz' }] },
                qq{ split_rec_on('200','a', qr/\\s*;\\s*/, 2) },
                [ 'bar' ],
        );
        test_rule(
                'split_rec_on no part',
                { '200' => [{ a => 'foo ; bar', b => 42, c => 'baz' }] },
                qq{ split_rec_on('200','a', qr/\\s*;\\s*/) },
                [ 'foo', 'bar' ],
        );
        test_rule(
                'split_rec_on no record',
                {},
                qq{ split_rec_on('200','a', qr/\\s*;\\s*/) },
                [ '' ],
        );

        test_rec_rules(
                'marc_compose+split_rec_on',
                { '200' => [{ a => 'foo ! bar', b => 42, c => 'baz' }] },
                qq{
                        marc_compose('900',
                                'a', split_rec_on(200,'a', qr/\\s*!\\s*/, 1),
                                'c', rec(200,'c'),
                                'a', split_rec_on(200,'a', qr/\\s*!\\s*/, 2),
                                'b', rec(200,'b'),
                        );
                },
                [
                        [ '900', ' ', ' ',
                                'a', 'foo',
                                'c', 'baz',
                                'a', 'bar',
                                'b', 42,
                        ]
                ],
        );

        cmp_ok(marc_leader('06',42), '==', 42, 'marc_leader');
        cmp_ok(marc_leader('11',5), '==', 5, 'marc_leader');
        ok(marc_leader(), 'marc_leader get');
        diag "leader: ", dump(marc_leader()) if ($debug);
        is_deeply(marc_leader(), { '06' => 42, 11 => 5 }, "marc_leader full");

        test_rule(
                'rec1(000)',
                { '000' => [ 42 ]},
                qq{ rec('000') },
                [ 42 ],
        );

        test_rec_rules(
                'marc(001,rec(000))',
                { '000' => [ 42 ]},
                qq{
                        marc('001', rec('000') );
                },
                [
                        [ '001', 42, ]
                ],
        );

        test_rec_rules(
                'marc_remove subfield',
                { '200' => [{ a => 42, b => 'bar', c => 'baz' }] },
                qq{
                        marc('900', 'a', rec('200','a') );
                        marc('900', 'b', rec('200','b') );
                        marc_remove('900','b');
                        marc('900', 'b', rec('200','c') );
                        marc_remove('900','a');
                },
                [
                        [ '900', ' ', ' ', 'b', 'baz' ],
                ],
        );

        test_rec_rules(
                'marc_remove field',
                { '200' => [{ a => 42, b => 'bar', c => 'baz' }] },
                qq{
                        marc('900', 'a', rec('200','a') );
                        marc('900', 'b', rec('200','b') );
                        marc('901', 'b', rec('200','b') );
                        marc('901', 'c', rec('200','c') );
                        marc_remove('900');
                },
                [
                        [ '901', ' ', ' ', 'b', 'bar', 'c', 'baz' ],
                ],
        );

        test_s(qq{ marc_remove('*'); });
        ok(! WebPAC::Normalize::MARC::_get_marc_fields(), 'marc_remove(*)');

        test_rec_rules(
                'marc_duplicate',
                { '200' => [{ a => 42, b => 'bar', c => 'baz', d => 'bing', e => 'bong' }] },
                qq{
                        marc_leader('06',42);
                        marc_leader('11',0);
                        marc('900', 'a', rec('200','a') );
                        marc('900', 'b', rec('200','b') );
                        marc_duplicate;
                        marc_leader('11',1);
                        marc_remove('900','b');
                        marc('900', 'b', rec('200','c') );
                        marc_duplicate;
                        marc_leader('11',2);
                        marc_remove('900','b');
                        marc('900', 'b', rec('200','d') );
                        marc_duplicate;
                        marc_leader('11',3);
                        marc_remove('900','b');
                        marc('900', 'b', rec('200','e') );
                },
                [
                        # this will return FIRST record
                        [ '900', ' ', ' ', 'a', 42, 'b', 'bar' ],
                ],
        );

        cmp_ok( marc_count(), '==', 3, 'marc_count' );

        my $i = 0;
        foreach my $v ( qw/bar baz bing bong/ ) {

                ok($marc = WebPAC::Normalize::MARC::_get_marc_fields( offset => $i ),
                        "_get_marc_fields( offset => $i )"
                );
                diag "marc $i = ", dump( $marc ) if ($debug);
                is_deeply( $marc,
                        [ [ '900', ' ', ' ', 'a', 42, 'b', $v ] ],
                        "MARC copy $i has $v",
                );
                is_deeply(WebPAC::Normalize::MARC::_get_marc_leader(), { '06' => 42, 11 => $i }, "_get_marc_leader copy $i");
                $i++;
        }

        test_rec_rules(
                'marc_original_order',
                {
                        '200' => [ {
                                a => [ 'a1', 'a2' ], b => [ 'b1', 'b2' ], c => [ 'c1', 'c2' ],
                                subfields => [ qw/a 0 b 0 a 1 b 1 c 0 c 1/ ],
                        }, {
                                a => [ 'a3', 'a4', 'a5' ], b => 'b3', c => 'c3',
                                subfields => [ qw/a 0 a 1 b 0 c 0 a 2/ ],
                        } ],
                },
                qq{
                        marc_original_order(900,200);
                },
                [
                        [ '900', ' ', ' ', 'a', 'a1', 'b', 'b1', 'a', 'a2', 'b', 'b2', 'c', 'c1', 'c', 'c2', ],
                        [ '900', ' ', ' ', 'a', 'a3', 'a', 'a4', 'b', 'b3', 'c', 'c3', 'a', 'a5', ],
                ],
        );

        test_rule(
                'rec1 skips subfields',
                {
                        '200' => [ {
                                a => [ 'a1', 'a2' ], b => [ 'b1', 'b2' ], c => [ 'c1', 'c2' ],
                                subfields => [ qw/a 0 b 0 a 1 b 1 c 0 c 1/ ],
                        }, {
                                a => [ 'a3', 'a4', 'a5' ], b => 'b3', c => 'c3',
                                subfields => [ qw/a 0 a 1 b 0 c 0 a 2/ ],
                        } ],
                },
                qq{
                        rec1(200);
                },
                ['a1', 'b1', 'a2', 'b2', 'c1', 'c2', 'a3', 'a4', 'b3', 'c3', 'a5' ],
        );

        is_deeply(
                [ _pack_subfields_hash({
                        a => [ 'a1', 'a2' ], b => [ 'b1', 'b2' ], c => [ 'c1', 'c2' ],
                        subfields => [ qw/a 0 b 0 a 1 b 1 c 0 c 1/ ],
                }) ],
                ['a1', 'b1', 'a2', 'b2', 'c1', 'c2'],
                '_pack_subfields_hash( $h )'
        );

        cmp_ok(
                _pack_subfields_hash({
                        a => [ 'a1', 'a2' ], b => [ 'b1', 'b2' ], c => [ 'c1', 'c2' ],
                        subfields => [ qw/a 0 b 0 a 1 b 1 c 0 c 1/ ],
                }, 1),
                'eq',
                '^aa1^bb1^aa2^bb2^cc1^cc2',
                '_pack_subfields_hash( $h, 1 )'
        );

        _clean_ds();
        test_s(qq{
                marc_fixed('008', 0, 'abcdef');
                marc_fixed('000', 5, '5');
                marc_fixed('000', 10, 'A');
                marc_fixed('000', 0, '0');
        });
        ok( my $m = WebPAC::Normalize::MARC::_get_marc_fields(), '_get_marc_fields');
        diag dump( $m );
        is_deeply( WebPAC::Normalize::MARC::_get_marc_fields(),
                [
                        ["008", "abcdef"], 
                        #        0....5....10
                        ["000", "0    5    A"]
                ]
        );

        test_s(qq{ isbn_13( '1558607013', '978-1558607019' ) });
        test_s(qq{ isbn_10( '1558607013', '978-1558607019' ) });

        is_deeply(
                [ isbn_13( '1558607013', '978-1558607019' ) ],
                [ '978-1-55860-701-9', '978-1-55860-701-9', ],
        'isbn_13' );

        is_deeply(
                [ isbn_10( '1558607013', '978-1558607019' ) ],
                [ '1-55860-701-3', '1-55860-701-3' ],
        'isbn_10' );

        # frec

        $rec = {
                        '200' => [ {
                                a => [ 'a1', 'a2' ], b => [ 'b1', 'b2' ], c => [ 'c1', 'c2' ],
                                subfields => [ qw/a 0 b 0 a 1 b 1 c 0 c 1/ ],
                                i1 => '0', i2 => '1',
                        }, {
                                a => [ 'a3', 'a4', 'a5' ], b => 'b3', c => 'c3',
                                subfields => [ qw/a 0 a 1 b 0 c 0 a 2/ ],
                        } ],
        };

        test_rule( 'frec', $rec, qq{ frec(200) }, [ 'a1' ] );
        test_rule( 'frec', $rec, qq{ frec(200,'a') }, [ 'a1' ] );
        test_rule( 'frec', $rec, qq{ frec(200,'b') }, [ 'b1' ] );
        test_rule( 'frec', $rec, qq{ frec(200,'c') }, [ 'c1' ] );

        $rec->{'900'} = $rec->{'200'};
        foreach my $sf ( qw/a b c/ ) {
                ok( frec_eq( '200' => $sf, '900' => $sf ), "frec_eq 200 == 900 $sf");
                ok( ! frec_ne( '200' => $sf, '900' => $sf ), "! frec_ne 200 == 900 $sf");
        }

        foreach my $sf ( qw/a b/ ) {
                ok( ! frec_eq( '200' => $sf, '200' => 'c' ), "! frec_eq 200 $sf == 200 c");
                ok( frec_ne( '200' => $sf, '200' => 'c' ), "frec_ne 200 $sf == 200 c");
        }

        test_rule( 'rec(200,i1)', $rec, qq{ rec(200,'i1') }, [ '0' ] );
        test_rule( 'rec(200,i2)', $rec, qq{ rec(200,'i2') }, [ '1' ] );

        my $hash = { a => '[a]', 'b' => '[b]', subfields => [ 'a', 0, 'b', 0 ] };
        is_deeply([ _pack_subfields_hash( $hash ) ], [ '[a]', '[b]' ], '_pack_subfields_hash' );
        ok( $hash->{subfields}, 'subfields exist' );
        cmp_ok( _pack_subfields_hash( $hash, 1 ), 'eq', '^a[a]^b[b]', '_pack_subfields_hash' );
        ok( $hash->{subfields}, 'subfields exist' );

        $rec = { 'arr' => [ 1, 2, 3 ] };
        test_rule( 'rec_array', $rec, qq{ rec_array('arr') }, $rec->{arr} );

        _clean_ds();
        _set_ds( $rec );
        test_s(q{
                        row( 'table', e => $_ ) foreach ( rec_array('arr') );
        });
        ok( my $rows = _get_ds->{_rows}->{table}, 'ds have _rows' );

        foreach my $i ( 1 .. 3 ) {
                cmp_ok( $rows->[ $i - 1 ]->{e}, '==', $i, "e $i" );
        }

        test_rule( 'utf-8'
                , { '900' => [{ a => 'Čev', b => 'ić' }] }
                , qq{ join_with('', rec(900,'a'), 'apč', rec(900,'b') ) }
                , [ "\x{10C}evap\x{10D}i\x{107}" ]
        );

}
