#!/usr/bin/env perl
#
# filter.pl - converts the SPL XML data into a CSV file with specific extracted fields
# 
# Usage: find . -name '*.xml' | filter.pl > filtered.txt
#
# Darren Hardy, (c) January 2009
#

# Example XML record:
#
# <transaction>
#  <itemNumber>743237</itemNumber>
#  <bibNumber>2077437</bibNumber>
#  <ckodate>2007-02-07</ckodate>
#  <ckotime>10:56:00</ckotime>
#  <ckidate>2007-03-03</ckidate>
#  <ckitime>09:00:00</ckitime>
#  <collcode>nacd</collcode>
#  <itemtype>accd</itemtype>
#  <barcode>0010043433720</barcode>
#  <title>Putumayo presents Arabic groove</title>
#  <callNumber>CD 789.3096 P987</callNumber>
#  <deweyClass>789.3096</deweyClass>
#  <subjects>
#   <subject>Songs Arabic</subject>
#   <subject>World music</subject>
#   <subject>Popular music Arab countries 1991 2000</subject>
#  </subjects>
# </transaction>

my $do_utf16 = 0; # use this if using the UTF-16 version of the XML files

if ($do_utf16) {
    use Encode;
}

# this list must be in the SAME order as the XML record
my @tags = (
    'ckodate',
    'ckidate',
    'itemtype',
    'barcode'
);

print STDOUT "# ", join(',', @tags), "\n";

while (<>) {
    chop;
    my $fn = $_;
    print STDOUT "# file: $fn\n";
    print STDERR "opening $fn...\n";
    if ($do_utf16) {
        open XML, "<:encoding(utf16)", $fn || die "$fn: $!";
    } else {
        open XML, "<$fn" || die "$fn: $!";
    }
    my @record = ();
    LINE: while (<XML>) {
        if ($do_utf16) {
            $_ = Encode::encode("UTF8", $_);            
        }
        
        if (/^\s*<transaction>\s*$/o) {
            @record = ();
            next LINE;
        } 

        if (/^\s*<\/transaction>\s*$/o) {
            print STDOUT join(',', @record), "\n";
            @record = ();
            next LINE;
        } 

        # unrolled regex for performance reasons (nFiles ~ O(10,000))
        if (
            (/^\s*<ckodate>(.*)<\/ckodate>\s*$/o) or 
            (/^\s*<ckidate>(.*)<\/ckidate>\s*$/o) or 
            (/^\s*<itemtype>(.*)<\/itemtype>\s*$/o) or 
            (/^\s*<barcode>(.*)<\/barcode>\s*$/o)) {
            push @record, $1;
            next LINE;
        }
        
        # too slow for very large data sets
        # foreach $tag (@tags) {
        #   if (/^\s*<$tag>(.*)<\/$tag>\s*$/) {
        #       push @record, $1;
        #       next LINE;
        #   }
        # }
    }
    close XML;
}

exit(0);
