KSx::Analysis::StripAccents - Remove accents and fold to lowercase

    my $stripper = KSx::Analysis::StripAccents->new;

    my $polyanalyzer = KinoSearch::Analysis::PolyAnalyzer->new(
        analyzers => [ $tokenizer, $stripper, $stemmer ],
    );

use strict;
use warnings;

package KSx::Analysis::StripAccents;
use base qw( KinoSearch::Analysis::Analyzer );

our $VERSION = '0.05';

use Encode qw 'encode decode';
use Text::Unaccent 'unac_string_utf16';

sub analyze_batch {
    my ( $self, $batch ) = @_;

    # lc and unaccent all of the terms, one by one
    while ( my $token = $batch->next ) {
        # I have to use UTF-16BE, since, although it’s not documented,
        # Text::Unaccent only supports big-endian. And I have to encode it,
        # since it doesn’t support Perl’s Unicode strings. (And it’ll con-
        # vert it to UTF-16 behind the scenes anyway, if I don’t.)
        $token->set_text(
            lc uc decode 'utf-16be', unac_string_utf16
                encode 'UTF-16BE', $token->get_text );
        # We have an ‘lc uc’ there, since some letters won’t be normalised
        # properly without it; e.g., ‘Σσς’ should be normalised to three
        # instances of the same character (‘σσσ’ as opposed to ‘σσς’).
    }

    $batch->reset;
    return $batch;
}

*transform = *analyze_batch;

1;

__END__