Peptide::Pubmed - extract peptide sequences from MEDLINE article abstracts.

  use Peptide::Pubmed;
  $parser = Peptide::Pubmed->new;
  $in = { 
     PMID      	=> q[15527327],
     Author	=> q[Doe JJ, Smith Q],
     Journal	=> q[J Biological Foo. 2004;8(2):123-30.],
     Title	=> q[Foo, bar and its significance in phage display.],
     Abstract	=> 
      q[Peptide sequences EYHHYNK and Arg-Gly-Asp, but not ACCCGTNA or VEGFRI.],
     Mesh      	=> q[Genes, p53/genetics; Humans; Bar],
     Chemical	=> q[Multienzyme Complexes; Peptide Library; Foo],
    };
  $parser->parse_abstract($in);

  # get the peptide sequences in 1 letter symbols (select all words where the 
  # combined word/abstract score is above threshold: 
  # WordAbstScore >= WordAbstScoreMin):
  @seqs = $parser->get_seqs;
  print "@seqs\n"; # prints: 'EYHHYNK RGD'

  # same as above, set threshold explicitly:
  $parser->WordAbstScoreMin(0.4);
  @seqs = $parser->get_seqs;

  # set low threshold to get more peptide sequences (but at a cost of getting 
  # more false positives) 
  $parser->WordAbstScoreMin(-1);
  @seqs = $parser->get_seqs;
  print "@seqs\n"; # prints: 'EYHHYNK RGD ACCCGTNA VEGFRI'

  # reset threshold back:
  $parser->WordAbstScoreMin(0.4);

  # get more data for the abstract:
  $abst = $parser->get_abst;
  print "$abst->{AbstScore}\n"; # abstract score, in the [0,1] interval
  print "$abst->{AbstMtext}\n"; # abstract with sequences marked up: 
  # 'Peptide sequences <mark>EYHHYNK</mark> and <mark>Arg-Gly-Asp,</mark> 
  # but not ACCCGTNA or VEGFRI.'

  # get more data for the words, in addition to peptide sequences:
  @words = $parser->get_words;
  for my $word (@words) {
      # combined word/abstract score, in the [0,1] interval
      print "$word->{WordAbstScore}\n"; 
      # word as found in the abstract, eg 'Arg-Gly-Asp,'
      print "$word->{WordOrig}\n";
      # peptide sequence in 1 letter symbols, eg 'RGD'	
      print "$word->{WordSequence}\n";	
  }

  # There are no mandatory input fields. This will work too, but may give lower score.
  $in = { 
	 Abstract => 
          q[Peptide sequences EYHHYNK and Arg-Gly-Asp, but not ACCCGTNA or VEGFRI.],
	};
  $parser->parse_abstract($in);
  @words = $parser->get_words;

  # No peptide sequences are found in empty input:
  $in = undef;
  $parser->parse_abstract($in);
  @words = $parser->get_words;

 ADAM OVERVIEW                                                                 

     ADAM is an abbreviation database generated from the 2006             
     baseline of MEDLINE. It covers frequently used abbreviations         
     and their definitions (or long-forms), including both                
     acronyms and non-acronym abbreviations.                              
     Reference

     Zhou W, Torvik VI, Smalheiser NR. ADAM: Another Database 
     of Abbreviations in MEDLINE. Bioinformatics 2006; 22(22): 2813-2818.

 ADAM Copyright                                                                

     University of Illinois at Chicago, 2006.                             

 ADAM License

     By using this software, you expressly agree that your use will be
     noncommercial, that you will not use this software to make money, and that
     you will not distribute the software to anyone else or let anyone else use
     it.  Moreover, you will give credit to the University of Illinois and Dr.
     Smalheiser as the author of the software.  The software is provided "as is"
     and without warranties of any kind, express or implied, including but not
     limited to the implied warranties of merchantability and fitness, and
     statutory warranties of noninfringement.

  Args        : %named_parameters: 

		AbstScoreMin : print abstract data if AbstScore >= this threshold.
		WordScoreMin : print word data if WordScore >=  this threshold.
		WordAbstScoreMin : print word data if WordAbstScore >= this threshold.
		Allowed values: real in [0,1] interval, or -1.
		Set to -1 to undefine these, and print all that satisfy 
		*PrintMin*, *PrintMax* thresholds. 

		WordNumPrintMin : min number of words to print per abstract.
		Allowed values: 1 and -1
		WordNumPrintMax : max number of words to print per abstract.
		Allowed values: non-negative integer, or -1.
		Set to -1 to undefine these, and print all that satisfy 
		*ScoreMin* thresholds. 

		If WordNumPrintMin = 1, at least one line is printed
		per each abstract for which AbstScore satisfies
		AbstScoreMin threshold. That is, 1 line is printed
		even if no sequences were found (many Word* vars will
		be empty or 0 when this happens) or if no sequences
		satisfy WordScore / WordScoreMin threshold. If there
		are sequences found, at least 1 line is printed (which
		may or may not satisfy WordScore / WordScoreMin
		threshold), and the rest are printed only if they
		satisfy it. If WordNumPrintMax = N, where N is a
		positive integer, only the first N words are printed
		(_not_ the best N words by score). A special case is
		(WordNumPrintMin => 1, WordNumPrintMax => 1), which
		prints exactly 1 line per abstract. Word* vars are not
		computed, which make the code much faster. An even
		more special case is (AbstScoreMin => -1, WordScoreMin
		=> -1, WordAbstScoreMin => -1, WordNumPrintMin => 1,
		WordNumPrintMax => 1), which prints all abstracts, 1
		abstract per line, and computes Abst*, but not Word*
		vars, thus allowing to separate computation into 2
		steps: compute and select abstracts with good
		AbstScore, then compute and select abstracts and words
		with good WordAbstScore, WordScore.

		in_col : columns for input
		kmer_length : for computing WordPropProtein:	=> 3,

		print_col : columns for printing
		print_colf : column formats for rdb table, eg [qw(50S 10N)]; 
		default: undef, to be determined automatically from col names.
		verbose : verbosity level for diagnostic msgs. Allowed values: 0..4.
  Example     : my $parser = Peptide::Pubmed->new(verbose => $verbose) or 
			carp "not ok: new Peptide::Pubmed" and return;
  Description : bare constructor. All work done by init().
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $abst = $parser->get_abst;
  Description : get the output data for Abst* variables.
  Returns     : a ref to hash with Abst* variables if successful, 
		ref to empty hash otherwise.

  Args        : none
  Example     : # get data for words likely to contain peptide sequences:
		@words = $parser->get_words;
  Description : get the output data for Words* variables.
  Returns     : an array where each element is a ref to hash with
		Word* variables output data for 1 word. Only words
		that satisfy the threshold: combined word/abstract score 
		WordAbstScore greater than or equal to WordAbstScoreMin, 
		are included, otherwise an empty array is returned. 
		To return all words, set WordAbstScoreMin to -1.

  Args        : none
  Example     : @seqs = $parser->get_seqs;
  Description : 
  Returns     : a list with sequences for all words in the abstract, 
		empty list if none found.

  Args        : (optional) new value
  Example     : $parser->WordAbstScoreMin(0.4);
		$parser->WordAbstScoreMin;
  Description : assign to WordAbstScoreMin a new value, if called with an argument.
  Returns     : WordAbstScoreMin value (after assignment)

  Args        : none
  Example     : $parser->init or return;
  Description : check to see if the entry is ok. Initialize several fields used for 
		printing, such as column definitions.
  Returns     : always TRUE

  Arg[1]      : string of items, one item per line.
  Arg[2]      : (optional) %named_parameters:
		lc : convert to lowercase? 1 = yes, 0 = no. Default = 1.
  Example     : my %words = map { $_ => 1 } splitln("foo\nbar\n", lc => 0);
		my %words = map { $_ => 1 } $this->splitln("foo\nbar\n");
  Description : parse input string into list of items, remove leading and trailing 
		whitespace, remove empty lines, convert to lc. Can be used as a 
		method call or direct call.
  Returns     : list of items

  Args        : %named_parameters:
		mandatory:
		in_fname : input file name.
		out_fname : output file name.
  Example     : $parser->parse_file(in_fname => $in_fname, out_fname => $out_fname);
  Description : reads input and write output, both in rdb
		format. Input data is 1 abstract per line. Prints data
		for all sequences found, 1 sequence per line. Skips
		abstracts and words based on rules described in
		new(). Because it prints data for 1 word per line, the
		same abstract data is printed for all sequences from
		the same abstract.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : input rdb file handle
  Example     : my ($comment, $ra_col, $ra_colf) = read_rdb_header($in_fh);
  Description : Reads header of an rdb table. 
  Returns     : Returns comment (or empty string), ref to array of column names, 
		reference to array of column definitions.

  Args        : output rdb file handle, comment (or undef), ref to array of 
		column names, reference to array of column definitions.
  Example     : print_rdb_header($out_fh, $comment, $self->{print_col}, 
		$self->{print_colf});
  Description : Prints header of an rdb table. If comment is undefined, 
		it is not printed.
  Returns     : always TRUE.

  Args        : ref to hash with input data for one pubmed abstract
  Example     : $parser->parse_abstract($rh_pm_in);
  Description : does all parsing for 1 abstract by calling init_abstract, AbstVars.
  Returns     : ref to hash with parsed data if successful, FALSE if error. 
		If no sequences are found, the hash will have a single sequence 
		consisting of an empty string (this is not considered an error).

  Args        : ref to hash with input data with field names 'Title', 'Abstract', etc 
		(optionally, with prefix 'Abst', eg 'AbstTitle', 'AbstAbstract'). 
		See field names listed in in_col.
  Example     : $rh_pm_in = { Title => q[Some title.], Abstract	=> q[Some abstract.] };
		$self->init_abstract($rh_pm_in) or return;
  Description : Reads the input data and stores in 'Abst*' fields
		('Abst' prefix is added if needed). Adds prefix 'Abst'
		to field names if needed. Converts to lower case and
		stores text in AbstLc* fields. Undefined fields in the
		input are converted to empty strings and
		stored. Currently, there are no mandatory fields:
		undefined $rh_pm_in is not considered an error. In
		such case, all fields will be empty strings. No
		sequences will be found, and the default score (zero)
		will be assigned.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->AbstVars or return;
  Description : Calls all Abst* methods that compute the corresponding variables, 
		eg AbstNumAb. Calls Words which in turn calls Word* methods.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->AbstNumAb or return;
  Description : See package DESCRIPTION above.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->AbstNumBind or return;
  Description : See package DESCRIPTION above.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->AbstNumDigest or return;
  Description : See package DESCRIPTION above.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->AbstNumMHC or return;
  Description : See package DESCRIPTION above.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->AbstNumPeptide or return;
  Description : See package DESCRIPTION above.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->AbstNumPhage or return;
  Description : See package DESCRIPTION above.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->AbstNumProtease or return;
  Description : See package DESCRIPTION above.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->AbstScore or return;
  Description : Computes AbstScore based on the results of Abst* vars in the abstract.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->Words or return;
  Description : Parses all peptide sequences from abstract text in AbstAbstract. 
		Calls the necessary methods, and stores results in array ref 
		$parser->{words}.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : string with a peptide sequence
  Example     : $str = $self->aa3_to_aa1($str)
  Description : finds the first string that looks like protein
		sequence in 3 letter amino acid symbols or full
		names. Cleans (remove non-informative parens, -, /,
		etc), and returns the cleaned sequence. Currently,
		full names are also handled here, but this may change.
  Returns     : sequence (TRUE) if a sequence is found, FALSE otherwise.

  Args        : string with a peptide sequence
  Example     : $str = $self->to_aa1($str)
  Description : finds the first string that looks like peptide sequence in uppercase 
		1 letter amino acid symbols. Cleans (remove non-informative parens, 
		-, /, etc), and returns the cleaned sequence.
  Returns     : sequence (TRUE) if a sequence is found, FALSE otherwise.

  Args        : string with a peptide sequence in 1 letter aa symbols
  Example     : $str = clean_seq($str);
  Description : for sequence in 1 letter aa symbols, remove extra parens, etc. 
		Simple motifs are ok: A(B/C/D)E...
  Returns     : cleaned string.

  Args        : string with a sequence to clean up.
  Example     : $_ = clean_orphan_parens($_);
  Description : removes orphan parens from a string.
  Returns     : cleaned string.

  Args        : string with a sequence.
  Example     : $_ = parse_slashes($_);
  Description : handles slashes. Changes A/B/C/etc to (A/B/C/etc)
		which means any of A, B, C, etc. Returns the resulting
		string. Exceptions are as follows, based on the the
		fact that the result would make no sense. If X occurs
		next to '/' or if the result would contain a repeated
		character. For exceptions, splits the string on
		slashes, and returns the first longest string.
  Returns     : see above

  Args        : string with a sequence.
  Example     : print 1 if has_repeats_at_slashes($_);
  Description : see below
  Returns     : TRUE if the string has a repeated char next to a series of 
		slashes and chars, FALSE otherwise.

  Args        : $rh_word - ref to hash with data for the word
  Example     : $parser->WordVars($rh_word) or return;
  Description : computes variables for WordScore, eg WordPropDegen, WordIsDNA, etc.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : $word : string
  Example     : $rh_word->{WordPropProtein} = $self->WordPropProtein($word);
  Description : See package DESCRIPTION above.
  Returns     : sum of log10(proportion(k-mers))

  Args        : $rh_word - ref to hash with word data.
  Example     : $parser->WordScore($rh_word) or return;
  Description : See package DESCRIPTION above.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : $rh_word - ref to hash with word data.
  Example     : $parser->WordScoreForMatch($rh_word) or return;
  Description : Calls WordIsGeneOrDict to compute WordIsDict, WordIsGene. 
		Changes WordScore based on these vars, as well as based on WordIsDNA, 
		WordPropProtein, etc.
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : $rh_word - ref to hash with word data.
  Example     : $parser->WordIsGeneOrDict($rh_word) or return;
  Description : Computes WordIsDict, WordIsGene and related vars. 
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->WordAbstScore or return;
  Description : Computes WordAbstScore based on the results of
		AbstScore and WordScore vars for all words in the
		abstract. AbstMaxWordScore is high if either AbstScore
		is high or WordScore is high, and higher if both are
		high. Instead of using simply AbstMaxWordScore =
		AbstScore * WordScore, AbstMaxWordScore is also added
		to WordScore with a smaller weight. This is because
		probability that a given word is a sequence increases
		if another word in the same abstract is a
		sequence. The weight for AbstMaxWordScore is smaller
		than that for WordScore in order not to make all words
		have equal AbstMaxWordScore. WordAbstScore = 0 for
		WordScore = 0, to reduce obvious noise (otherwise, all
		words that are clearly not peptide sequences will get
		positive WordAbstScore when they occur in an abstract
		with at least one sequence). Skip transform to (0,1)
		because the scores below are all in (0,1), and the
		weights add to 1, so WordAbstScore is already in
		(0,1).
  Returns     : TRUE if successful, FALSE otherwise.

  Args        : none
  Example     : $parser->AbstMtext
  Description : Marks all words that are putative sequences with
		'<mark>...</mark>' tags. This is done only for words
		for which WordScore is at least WordScoreMin.
  Returns     : always TRUE

  Args[1]     : $number - score or proportion.
  Example     : $rh_word->{WordScore} = formatScoreProp($rh_word->{WordScore});
  Description : changes $number to be in (0,1) interval, formats $number 
		for nice printing
  Returns     : formatted score

  Args[1]     : $key - field name
  Args[2]     : $val - value
  Example     : # print WordScore for the first GETRAPL sequence.
		# print $parser->find(WordSequence => 'GETRAPL')->{WordScore};
  Description :  finds the first word for which the value of field $key 
		is equal to $val (string eq, not numeric ==)
  Returns     : returns this word (hash ref) if successful, FALSE otherwise.

package Peptide::Pubmed;

use vars qw($VERSION);
$VERSION = '1.02';

# for printing diagnostic msgs when the module is loaded. 
# use $obj->{verbose} for object related diagnostic msgs.
our $VERBOSE; 
BEGIN { 
    $VERBOSE ||= $ENV{VERBOSE} || $ENV{TEST_VERBOSE} || 0;
}

use Peptide::DictionaryAbbreviations;
use Peptide::DictionaryEnglish;
use Peptide::DictionaryFrequent;
use Peptide::DictionaryScientific;
use Peptide::Gene;
use Peptide::Sequence;
use Peptide::NonSequence;
use Data::Dumper;
use Carp;
use warnings;
use strict;

use constant LOG_10_2 => log(2) / log(10); # empirically derived weight for WordPropProtein
our $MAX_PEPTIDE_LENGTH = 20; # Somewhat arbitrary cutoff, consistent with IUPAC. Find the peptides sequences not longer than this cutoff.

# word / score assignment
#

# for WordScore()

my %Dict = map { $_ => 1 } 
  splitln(Peptide::DictionaryAbbreviations->words),
  splitln(Peptide::DictionaryEnglish->words),
  splitln(Peptide::DictionaryFrequent->words),
  splitln(Peptide::DictionaryScientific->words);
my %Gene = map { $_ => 1 } 
  splitln(Peptide::Gene->words);

# for WordPropProtein()

my %log_prop_sequence = map /(\S+)\s+(\S+)\s*\n/g, Peptide::Sequence->log_prop_kmers;
my %log_prop_nonsequence = map /(\S+)\s+(\S+)\s*\n/g, Peptide::NonSequence->log_prop_kmers;

if ($VERBOSE > 3) { 
    foreach (sort keys %log_prop_sequence) {
	print join("\t", $_, $log_prop_sequence{$_}, $log_prop_nonsequence{$_}), "\n";
    }
}

{
    my %args_default 
      = ( 
	 AbstScoreMin		=> 0.2,
	 WordScoreMin		=> 0.2,  
	 WordAbstScoreMin	=> 0.2, 
	 WordNumPrintMin   	=> -1, 
	 WordNumPrintMax   	=> -1,
	 in_col			=> [ qw(PMID Author Journal Title Abstract Mesh Chemical) ],
	 kmer_length		=> 3,
	 print_col		=> [ qw(
AbstPMID AbstAuthor AbstJournal AbstTitle AbstAbstract AbstMesh AbstChemical 

AbstNumAb AbstNumAllCap AbstNumBind AbstNumDigest AbstNumMHC AbstNumPeptide AbstNumPhage AbstNumProtease AbstNumWords AbstPropAllCap AbstScore AbstComment AbstMtext 

WordAbstScore WordIsDNA WordIsDict WordIsGene WordSeqLen WordOrig WordPropDegen WordPropProtein WordScore WordSequence WordAaSymbols WordIdx

)],
	 print_colf		=> undef,
	 verbose		=> 1,
	);

    sub new {
	my $class = shift;
	my %args = @_;
	foreach (sort keys %args) {
	    exists $args_default{$_} or carp "not ok: arg=$_ is not allowed in new()" and 
	      return;
	}
	my $self = bless { %args_default, %args  }, ref($class) || $class;
	$self->init or return;
	return $self;
    }
}

sub get_abst {
    my ($self) = @_;
    my $abst = {};
    foreach my $key (grep /^Abst/, @{ $self->{abst_col} } ) {
	$abst->{$key} = $self->{abst}{$key} if exists $self->{abst}{$key};
    }
    return $abst;
}

sub get_words {
    my ($self) = @_;
    my @words = ();
    foreach my $rh_word ( @{ $self->{words} } ) {
	next unless $rh_word->{WordAbstScore} >= $self->{WordAbstScoreMin};
	my $rh_word_out = {};
	foreach my $key (grep /^Word/, @{ $self->{words_col} } ) {
	    $rh_word_out->{$key} = $rh_word->{$key} if exists $rh_word->{$key};
	}
	push @words, $rh_word_out;
    }
    return @words;
}

sub get_seqs {
    my ($self) = @_;
    my @seqs = ();
    foreach my $rh_word ( @{ $self->{words} } ) {
	next unless $rh_word->{WordAbstScore} >= $self->{WordAbstScoreMin};
	push @seqs, $rh_word->{WordSequence} if $rh_word->{WordSequence};
    }
    return @seqs;
}

sub WordAbstScoreMin {
    my ($self, $val) = @_;
    if (@_ == 2) {
	$self->{WordAbstScoreMin} = $val;
    }
    return $self->{WordAbstScoreMin};
}

sub init {
    my ($self) = @_;
    unless (defined $self->{print_colf}) {
	# assign to numeric or text
	$self->{print_colf} = [ 
			       map { 
				   /^(Abst|Word|AbstWord)(Idx|Is|Num|Prop|Score)([A-Z]|$)/ ? 
				     "10N" : "50S" 
				 } 
			       @{ $self->{print_col} } 
			      ]
    }
    # cols to be printed are either words cols, or abstract cols
    $self->{abst_col} = [ grep { /^Abst/ } @{ $self->{print_col} } ];
    $self->{words_col} = [ grep { /^Word/ } @{ $self->{print_col} } ];
    if ($self->{verbose} > 2) {
	foreach (sort keys %{ $self }) {
	    warn "$_ => $self->{$_}";
	}
    }
    return 1;
}

    
sub splitln {
    ref $_[0] and shift;
    my $str = shift;
    my %args = (lc => 1, @_);
    my @lines = 
      grep { $_ ne '' }
	map { s/^\s+|\s+$//g; $args{lc} ? lc $_ : $_ } 
	  split /\n/, $str;
    return @lines;
}

sub parse_file {
    my ($self, %args) = @_;
    foreach (qw(in_fname out_fname)) {
	$self->{$_} = $args{$_};
    }
    my $in_fh;
    my $out_fh;
    open $in_fh, $self->{in_fname} or carp "not ok: open $self->{in_fname}: $!" and return;
    open $out_fh, ">$self->{out_fname}" or 
      carp "not ok: open $self->{out_fname}: $!" and return;
    my ($comment, $ra_col, $ra_colf) = read_rdb_header($in_fh);
    print_rdb_header($out_fh, $comment, $self->{print_col}, $self->{print_colf});
    while (<$in_fh>) { # for 1 pubmed entry (=abstract)
	chomp;
	print STDERR "processing line $.\n" if ($self->{verbose} > 1 and not $. % 500);
	my $rh_pm_in = {}; # input data for 1 abstract
	@{$rh_pm_in}{ @$ra_col } = split "\t", $_;
	my $rh_pm_out = $self->parse_abstract($rh_pm_in) or 
	  carp "not ok: parse_abstract" and return;
	next unless $rh_pm_out->{abst}{AbstScore} >= $self->{AbstScoreMin};
	my %seen;
	my $num_words = 0;
	foreach my $rh_word (@{ $rh_pm_out->{words} }) {
	    $num_words++;
	    last if $self->{WordNumPrintMax} != -1 and $num_words > $self->{WordNumPrintMax};
	    if ($self->{WordNumPrintMin} == -1 || 
		($self->{WordNumPrintMin} == 1 && $num_words > $self->{WordNumPrintMin}) ) {
		next unless $rh_word->{WordScore} >= $self->{WordScoreMin} && 
		  $rh_word->{WordAbstScore} >= $self->{WordAbstScoreMin};
		next if defined $rh_word->{WordSequence} and 
		  $seen{ $rh_word->{WordSequence} }++;	    
	    }
	    my %line = map { tr/\t/ /; $_ } # tab is used as the delimiter
	      ( 
	       map( { $_ => defined $rh_word->{$_} ? $rh_word->{$_} : '' } 
		   @{ $self->{words_col} }), 
	       map( { $_ => defined $rh_pm_out->{abst}{$_} ? $rh_pm_out->{abst}{$_} : '' } 
		   @{ $self->{abst_col} }),
	      );
	    print $out_fh join("\t", @line{ @{ $self->{print_col} } }), "\n";
	}
    }
    close $out_fh or carp "not ok: close $self->{out_fname}: $!" and return;
    close $in_fh or carp "not ok: close $self->{in_fname}: $!" and return;
    return 1;
}

sub read_rdb_header {
    my ($in_fh) = @_;
    my ($comment, $ra_col, $ra_colf);
    while (<$in_fh>) {
	$comment .= $_ and next if /^\#/;	# table comments (optional)
	chomp;
	@$ra_col = split /\t/;			# column names (eg, Hit, Expect)
	last unless defined ($_ = <$in_fh> );
	chomp;
	@$ra_colf = split /\t/;			# column formats (eg, 10N, 50S)
	last;
    }
    return $comment || '', $ra_col, $ra_colf;
}

sub print_rdb_header {
    my ($out_fh, $comment, $ra_col, $ra_colf) = @_;
    print $out_fh $comment if defined $comment;
    print $out_fh join( "\t", @$ra_col  ), "\n";
    print $out_fh join( "\t", @$ra_colf ), "\n" if defined $ra_colf;
    return 1;
}

sub parse_abstract { 
    my ($self, $rh_pm_in) = @_;
    my $rh_pm_out = {};
    $self->init_abstract($rh_pm_in) or return;
    $self->AbstVars or return;
    %$rh_pm_out = map { $_ => $self->{$_} } qw(words abst);
    carp((caller(0))[3], "(@_)", ' ', Data::Dumper->Dump([$self], ['self'])) 
      if $self->{verbose} > 2;
    return $rh_pm_out; 
}

{
    my %abst_default = 
      (
       AbstComment	=> undef,
       AbstMtext	=> undef,
       AbstNumAb	=> 0,
       AbstNumAllCap	=> 0,
       AbstNumBind	=> 0,
       AbstNumDigest	=> 0,
       AbstNumMHC	=> 0,
       AbstNumPeptide	=> 0,
       AbstNumPhage	=> 0,
       AbstNumProtease	=> 0,
       AbstNumWords	=> 0,
       AbstPropAllCap	=> 0,
       AbstScore	=> 0,
      );

    sub init_abstract {
	my ($self, $rh_pm_in) = @_;
	# reset abst and words data from the prev parsed abstract, if any
	$self->{abst} = { %abst_default };
	$self->{words} = []; 
	my @in_col = @{ $self->{in_col} };
	foreach (@{ $self->{in_col} }) {
	    $self->{abst}{"Abst$_"} = do {
		if (defined $rh_pm_in->{$_}) {
		    $rh_pm_in->{$_}
		} elsif (defined $rh_pm_in->{"Abst$_"}) {
		    $rh_pm_in->{"Abst$_"}
		} else {
		    ''
		}
	    };
	}
	# mandatory field:
	#my @missing = grep { $self->{abst}{$_} eq '' } qw(AbstAbstract);
	#@missing and carp "not ok: missing fields: @missing" and return;
	$self->{abst}{AbstAllText} = 
	  join(";; ", 
	       map "$_: $self->{abst}{$_}", 
	       qw(AbstTitle AbstAbstract AbstMesh AbstChemical)
	      ) || '';
	$self->{abst}{AbstLcAllText} = lc $self->{abst}{AbstAllText};
	$self->{abst}{AbstLcAbstract} = lc $self->{abst}{AbstAbstract};
	return 1;
    }
}

sub AbstVars {
    my ($self) = @_;
    $self->AbstNumAb or return;
    # AbstNumAllCap is computed in AbstScore()
    $self->AbstNumBind or return;
    $self->AbstNumDigest or return;
    $self->AbstNumMHC or return;
    $self->AbstNumPeptide or return;
    $self->AbstNumPhage or return;
    $self->AbstNumProtease or return;
    $self->AbstScore or return;
    $self->Words or return;
    $self->WordAbstScore or return;
    $self->AbstMtext or return;
    foreach (qw(AbstMaxWordScore AbstPropAllCap AbstScore)) {
	$self->{abst}{$_} = formatScoreProp( $self->{abst}{$_} );
    }
    foreach my $rh_word ( @{ $self->{words} }) {
	foreach (qw(WordScore WordAbstScore WordPropDNA WordPropDegen)) {
	    $rh_word->{$_} = formatScoreProp( $rh_word->{$_} );
	}
    }
    if ($self->{verbose} > 0) {
	$self->{abst}{AbstComment} .= join '; ', map { 
	    defined $self->{abst}{"ra_$_"} && 
	      ref $self->{abst}{"ra_$_"} eq 'ARRAY' ? 
		"$_='" . join(", ", @{ $self->{abst}{"ra_$_"} }) . "'" : "$_=''" 
	    } 
	  qw(
	     AbstNumAb AbstNumBind AbstNumDigest AbstNumMHC AbstNumPeptide 
	     AbstNumPhage AbstNumProtease 
	    );
    }
    return 1;
}

sub AbstNumAb {
    my ($self) = @_;
    return unless defined $self->{abst}{AbstLcAllText};
    (my $var = (caller(0))[3]) =~ s/.*:://;
    my @matches;
    for ($self->{abst}{AbstLcAllText}) {
	push @matches,
	  m[
	    	    adjuvant|
	    	    antibod|
	    	    antigen|
	    	    antiser|
	    	    epitope|
	    	    freund|
	    	    heavy\ chain|
	    	    immun|
	    	    light\ chain|
	    	    mimotope|
	    	    monoclonal|
	    	    phagotope|
	    	    polyclonal|
	    	    recombinant\ fab|
	    	    vaccin|
	    	    variable\ region|
	    	    variable\ regions
	   	   ]gx,
	     m[
	       	       \b(
	       	       mab|
	       	       mabs|
	       	       rfab|
	       	       sera|
	       	       serum
	      	      )\b
	      	      ]gx;
	$self->{abst}{$var} = @matches;
	$self->{abst}{"ra_$var"} = [ @matches ];
    }
    return 1;
}

sub AbstNumBind {
    my ($self) = @_;
    return unless defined $self->{abst}{AbstLcAllText};
    (my $var = (caller(0))[3]) =~ s/.*:://;
    my @matches;
    for ($self->{abst}{AbstLcAllText}) {
	push @matches,
	  m[
	    	    abolish|
	    	    activate|
	    	    adher|
	    	    adhes|
	    	    affinit|
	    	    agonise|
	    	    agonist|
	    	    agonize|
	    	    associat|
	    	    bind|
	    	    block|
	    	    bound|
	    	    compet|
	    	    disrupt|
	    	    dissociat|
	    	    inhibit|
	    	    interact|
	    	    interfere|
	    	    ligand|
	    	    prevent|
	    	    recogn|
	    	    regulat|
	    	    suppress
	   	   ]gx,
	     m[
	       	       \b(
	       	       ec50|
	       	       ic50|
	       	       kd|
	       	       ki
	      	      )\b
	      	      ]gx;
	$self->{abst}{$var} = @matches;
	$self->{abst}{"ra_$var"} = [ @matches ];
    }
    return 1;
}

{
    # older papers are more likely to have data on protein digestion 
    # into peptides which are relevant only to the protein, 
    # rather than to the peptides.
    # for these time intervals, multiply by this factor
    my %AbstNumDigestScoreForYear;
    for (1900..2100) {
	if ($_ > 1993) {
	    $AbstNumDigestScoreForYear{$_} = 1;
	} elsif ($_ > 1991) {
	    $AbstNumDigestScoreForYear{$_} = 1.5;
	} elsif ($_ > 1990) {
	    $AbstNumDigestScoreForYear{$_} = 2;
	} elsif ($_ > 1989) {
	    $AbstNumDigestScoreForYear{$_} = 3;
	} else { # <= 1989
	    $AbstNumDigestScoreForYear{$_} = 4;
	} ## stopped here 
    }

    sub AbstNumDigest {
	my ($self) = @_;
	return unless defined $self->{abst}{AbstLcAllText};
	(my $var = (caller(0))[3]) =~ s/.*:://;
	my @matches;
	for ($self->{abst}{AbstLcAllText}) {
	    # do not shorten to 'trypt' to avoid confusion with tryptophan
	    push @matches,
	      m[
				borohydride|
				complete\ amino\ acid\ sequence|
				chromatogra|
				cnbr|
				cyanogen\ bromide|
				dansyl|
				digest|
				edman|
				hplc|
				lsims|
				maldi-tof|
				mass\ spec|
				matrix-assisted\ laser\ desorption|
				nabh4|
				peptide\ map|
				protease\ digestion|
				proteolytic\ digestion|
				rplc|
				spectrometry,\ mass|
				spectrum analysis,\ mass|
				thermolysin|
				thermolytic|
				tryptic|
				trypsin|
				v-8\ prote|
				v8\ prote
	       	       ]gx,
		 m[
		   		   \b(
		   		   ms
		  		  )\b
		  		  ]gx;
	    $self->{abst}{$var} = @matches;
	    $self->{abst}{"ra_$var"} = [ @matches ];
	    for ($self->{abst}{AbstJournal}) {    
		$self->{abst}{$var} *= $AbstNumDigestScoreForYear{$1} if 
		  defined $_ and /\b((19|20)\d\d)\b/ and $AbstNumDigestScoreForYear{$1};
	    }
	    $self->{abst}{$var} = sprintf "%.0f", $self->{abst}{$var};
	}
	return 1;
    }
}

sub AbstNumMHC {
    my ($self) = @_;
    return unless defined $self->{abst}{AbstLcAllText};
    (my $var = (caller(0))[3]) =~ s/.*:://;
    my @matches;
    for ($self->{abst}{AbstLcAllText}) {
	push @matches,
	  m[
	    	    alloge|
	    	    allograft|
	    	    antibod|
	    	    antigen|
	    	    b\ cell|
	    	    dendri|
	    	    epitope|
	    	    histocompatibility|
	    	    immun|
	    	    present|
	    	    restrict|
	    	    tolero|
	    	    t\ cell
	   	   ]gx,
	     m[
	       	       \b(
	       	       cd\d{1,2}|
	       	       class\ i{1,2}|
	       	       hla|
	       	       mhc|
	       	       self|
	       	       tcr
	      	      )\b
	      	      ]gx;
	$self->{abst}{$var} = @matches;
	$self->{abst}{"ra_$var"} = [ @matches ];
    }
    return 1;
}

sub AbstNumPeptide {
    my ($self) = @_;
    return unless defined $self->{abst}{AbstLcAllText};
    (my $var = (caller(0))[3]) =~ s/.*:://;
    my @matches;
    my @matches_neg;
    ##cc TODO: possibly add matches to /protein/ and /fragment/ in the same sentence.
    for ($self->{abst}{AbstLcAllText}) {
	# step 1, mandatory matches
	push @matches,
	  m[
	    	    benzyloxycarbonyl|
	    	    hormone|
	    	    peptid
	   	   ]gx;
	next unless @matches;
	push @matches_neg,
	  m[
	    	    peptidase|
	    	    polypeptide
	   	   ]gx;
	next unless (scalar(@matches) - scalar(@matches_neg)) > 0;
	# step 2, optional matches
	push @matches,
	  m[
	    	    consensus|
	    	    constrain|
	    	    \bcyclic|
	    	    librar|
	    	    linear|
	    	    motif|
	    	    sequenc|
	    	    synthe
	   	   ]gx,
	     m[
	       	       \b(
	       	       ac|
	       	       acetyl|
	       	       amide|
	       	       boc|
	       	       ch2nh|
	       	       conh|
	       	       cyclo|
	       	       dansyl|
	       	       h2n|
	       	       nh|
	       	       nh2|
	       	       ome|
	       	       tos
	      	      )\b
	      	      ]gx;
	$self->{abst}{$var} = scalar(@matches) - scalar(@matches_neg);
	$self->{abst}{"ra_$var"} = [ @matches ];
    }
    return 1;
}

sub AbstNumPhage {
    my ($self) = @_;
    return unless defined $self->{abst}{AbstLcAllText};
    (my $var = (caller(0))[3]) =~ s/.*:://;
    my @matches;
    my @matches_neg;
    for ($self->{abst}{AbstLcAllText}) {
	# step 1, mandatory matches
	push @matches,
	  m[
	    	    bio-pan|
	    	    biopan|
	    	    phagemid|
	    	    phage\b|
	    	    phages\b	    
	   	   ]gx,
	     m[
	       	       \b(
	       	       panned|		# match panning related, but not spanning, pan-caspase, etc
	       	       panning| 
	       	       pannings| 
	       	       pans
	      	      )\b
	      	      ]gx;
	next unless @matches;
	push @matches_neg,
	  m[
	    	    macrophage
	   	   ]gx;
	next unless (scalar(@matches) - scalar(@matches_neg)) > 0;
	# step 2, optional matches
	push @matches,
	  m[
	    	    clone|   
	    	    cloni|	    
	    	    display|
	    	    librar|
	    	    screen|
	    	    select	    
	   	   ]gx,
	     m[
	       	       \b(
	       	       round|
	       	       rounds
	      	      )\b
	      	      ]gx;
	$self->{abst}{$var} = scalar(@matches) - scalar(@matches_neg);
	$self->{abst}{"ra_$var"} = [ @matches ];

    }
    return 1;
}

sub AbstNumProtease {
    my ($self) = @_;
    return unless defined $self->{abst}{AbstLcAllText};
    (my $var = (caller(0))[3]) =~ s/.*:://;
    my @matches;
    for ($self->{abst}{AbstLcAllText}) {
	push @matches,
	  m[
	    	    aminobenzoyl|
	    	    caspase|
	    	    cathepsin|
	    	    convertase|
	    	    dinitrophenyl|
	    	    nitroanilide|
	    	    peptidase|
	    	    protease|
	    	    proteasom|
	    	    proteinase|
	    	    proteol
	   	   ]gx;
	next unless @matches;
	# match: MMP, MMPs MMP1, MMP-1, but not MMPFOO
	# match: Dnp-Met
	push @matches,
	  m[
	    	    (?:\b|[^a-z])(
	    	    abz|
	    	    amc
	    	    dnp|
	    	    dpa|
	    	    eddnp|
	    	    fmk|
	    	    mca|
	    	    mmp|
	    	    mmps
	   	   )(?:\b|[^a-z]) 
	   	   ]gx;
	$self->{abst}{$var} = @matches;
	$self->{abst}{"ra_$var"} = [ @matches ];
    }
    return 1;
}

sub AbstScore {
    my ($self) = @_;
    my $rh_abst = $self->{abst};
    # Split with saving the original whitespace, which is then added back 
    # during join after marking up is done, in AbstMtext(). 
    # This preserves the original character positions.
    @{ $rh_abst->{ra_WordOrig} } = split /(\s+)/, $rh_abst->{AbstAbstract};
    
    $rh_abst->{AbstNumWords} = grep /\S/, @{ $rh_abst->{ra_WordOrig} };
    $rh_abst->{AbstNumAllCap} = @{[grep { /[A-Z]/ and not /[a-z]/ } 
				     @{ $rh_abst->{ra_WordOrig} } ]};
    # initialize:
    $rh_abst->{AbstScore} = 0;
    # linear terms:
    $rh_abst->{AbstScore} += 
      (-1) * $rh_abst->{AbstNumDigest} + $rh_abst->{AbstNumPeptide} + $rh_abst->{AbstNumPhage};
    warn "\$rh_abst->{AbstScore}=$rh_abst->{AbstScore}" if $self->{verbose} > 2;

    # interaction terms (score increases more if both vars increase simultaneously)
    $rh_abst->{AbstScore} += 
      $rh_abst->{AbstNumBind} * 
	($rh_abst->{AbstNumPeptide} + $rh_abst->{AbstNumPhage}) + 
	  $rh_abst->{AbstNumProtease} * 
	    ($rh_abst->{AbstNumPeptide} + $rh_abst->{AbstNumPhage}) + 
	      $rh_abst->{AbstNumPeptide} * $rh_abst->{AbstNumPhage};
    warn "\$rh_abst->{AbstScore}=$rh_abst->{AbstScore}" if $self->{verbose} > 2;

    # abstracts with too many all cap words are more likely to have all cap 
    # non-sequence words rather than true sequences.
    $rh_abst->{AbstPropAllCap} = $rh_abst->{AbstNumAllCap} / ($rh_abst->{AbstNumWords} || 1);
    if ($rh_abst->{AbstPropAllCap} > 0.3 || $rh_abst->{AbstNumAllCap} > 30) { 
	$rh_abst->{AbstScore} *= 0.01; # strong penalty if above threshold
    } else {
	$rh_abst->{AbstScore} *= (1 - $rh_abst->{AbstPropAllCap});
    }
    warn "\$rh_abst->{AbstScore}=$rh_abst->{AbstScore}" if $self->{verbose} > 2;
    # AbstScore < 0 can happen for hi AbstNumDigest:
    $rh_abst->{AbstScore} = 0 if $rh_abst->{AbstScore} < 0; 
    # transform (0, +inf) to (0,1):
    $rh_abst->{AbstScore} = $rh_abst->{AbstScore} / (1 + $rh_abst->{AbstScore});
    warn "\$rh_abst->{AbstScore}=$rh_abst->{AbstScore}" if $self->{verbose} > 2;
    carp((caller(0))[3], "(@_)", ' ', Data::Dumper->Dump([$self], ['self'])) 
      if $self->{verbose} > 2;
    return 1;
}

{
 
    my %word_default = 
      (
       WordAbstScore	=> 0,
       WordAaSymbols	=> 0,
       WordIsDNA	=> 0,
       WordIsDNALen	=> 0,
       WordIsDict	=> 0, 
       WordIsGene	=> 0,
       WordIsRomanLen	=> 0,
       WordNumDegen	=> 0,
       WordOrig		=> undef,
       WordPropDNA	=> 0,
       WordPropDegen	=> 0,
       WordPropProtein	=> 0,
       WordSeqLen	=> 0,
       WordScore	=> 0,
       WordSequence	=> undef,
       WordSubLenMax	=> 0,
       ##cc provide defaults for all , to be used for 3 letter and full names of aa.
      );

    sub Words {
	my ($self) = @_;
	my @words; # each el is a ref to hash with data 
	if ($self->{WordNumPrintMin} == 1 and $self->{WordNumPrintMax} == 1) { 
	    # need exactly 1 empty word
	    $self->{words} = [ { %word_default } ];
	    return 1;
	}
	my $idx = 0;
	my @WordOrig = @{ $self->{abst}{ra_WordOrig} };
	foreach (@WordOrig) {
	    my $rh_word = { %word_default };
	    $rh_word->{WordOrig}	= $_;
	    $rh_word->{WordLcOrig}	= lc $_;
	    $rh_word->{WordIdx} = $idx++;
	    if (not /\S/) {
		# do nothing
	    }
	    elsif ( $_ = $self->aa3_to_aa1( $rh_word->{WordOrig} ) ) {
		$rh_word->{WordAaSymbols} = 3; # 3 letter aa symbols (and full names, too)
		$rh_word->{WordSequence} = $_;
		$self->WordVars($rh_word) or return;
	    }
	    elsif ( $_ = $self->to_aa1( $rh_word->{WordOrig} ) ) {
		$rh_word->{WordAaSymbols} = 1; # 1 letter aa symbols:
		$rh_word->{WordSequence} = $_;
		$self->WordVars($rh_word) or return;
	    }
	    # else: has non-whitespace chars, but is not a sequence: do nothing
	    $self->WordScore($rh_word) or return;
	    push @words, $rh_word;
	}
	if ($self->{WordNumPrintMin} == 1 and not @words) { # need at least 1 word
	    push @words, { %word_default };
	}
	$self->{words} = [ @words ];
	return 1;
    }
}

{
    # See more: Nomenclature and Symbolism for Amino Acids and Peptides
    # http://www.chem.qmul.ac.uk/iupac/AminoAcid/A2021.html
    # Amino acids below are converted to X for blasting, no need to worry here.

    # keep order for correct substitution below. Generally, longer patterns go first, so that less non-replaced chars remain, so that there is less to clean up later.
    my @aa3_to_aa1 = 
      ( 
       # full names:
       # Omitted: B, U, Z
       alanine		=> 'A',
       cysteine 	=> 'C',
       cystein		=> 'C', # alternative spelling
       aspartic 	=> 'D',
       aspartate 	=> 'D',
       glutamic 	=> 'E',
       glutamate 	=> 'E',
       phenylalanine 	=> 'F',
       glycine		=> 'G',
       histidine 	=> 'H',
       isoleucine 	=> 'I',
       lysine		=> 'K',
       leucine		=> 'L',
       methionine 	=> 'M',
       asparagine 	=> 'N',
       proline		=> 'P',
       glutamine 	=> 'Q',
       arginine 	=> 'R',
       serine		=> 'S',
       threonine 	=> 'T',
       selenocysteine	=> 'U', 
       valine		=> 'V',
       tryptophane 	=> 'W', # alternative spelling
       tryptophan 	=> 'W',
       tyrosine 	=> 'Y',

       # full names, 'yl' endings:
       # matches:
       # glycyl-L-prolyl-L-glutamic acid;
       # arginyl glycyl aspartyl serine
       alanyl		=> 'A',
       cysteinyl       	=> 'C',
       aspartyl 	=> 'D',
       glutamyl 	=> 'E',
       phenylalanyl 	=> 'F',
       glycyl		=> 'G',
       histidyl 	=> 'H',
       isoleucyl 	=> 'I',
       lysyl		=> 'K',
       leucyl		=> 'L',
       methionyl 	=> 'M',
       asparaginyl 	=> 'N',
       prolyl		=> 'P',
       glutaminyl 	=> 'Q',
       arginyl	 	=> 'R',
       seryl		=> 'S',
       threonyl 	=> 'T',
       selenocysteinyl	=> 'U',
       valyl		=> 'V',
       tryptophyl 	=> 'W',
       tyrosyl	 	=> 'Y',

       # 3 letter aa symbols to 1 letter.
       Ala => 'A',
       Cys => 'C',
       Asp => 'D',
       Glu => 'E',
       Phe => 'F',
       Gly => 'G',
       His => 'H',
       Ile => 'I',
       Lys => 'K',
       Leu => 'L',
       Met => 'M',
       Asn => 'N',
       Pro => 'P',
       Gln => 'Q',
       Arg => 'R',
       Ser => 'S',
       Thr => 'T',
       Sec => 'U',
       Val => 'V',
       Trp => 'W',       
       Tyr => 'Y',

       # non-standard amino acids are all converted to X
       Aad => 'X',
       Abu => 'X',
       Aib => 'X',
       Ahx => 'X',
       Ape => 'X',
       Apm => 'X',
       Asx => 'X', # B in http://www.chem.qmul.ac.uk/iupac/AminoAcid/A2021.html
       Cit => 'X',
       Dab => 'X',
       Dap => 'X',
       Dpm => 'X',
       Dpr => 'X',
       Glx => 'X', # Z in http://www.chem.qmul.ac.uk/iupac/AminoAcid/A2021.html
       Hyl => 'X',
       Hyp => 'X',
       Nle => 'X',
       Nva => 'X',
       Orn => 'X',
       Pyr => 'X',

       # unknown amino acids
       Xaa => 'X', # keep order as shown, so that these replacements are correct: 'Asn-x-Glu-x-x-(aromatic)-x-x-Gly', 
       # unknown amino acids, not in IUPAC, but frequently used:
       Xxx => 'X', # such as Gly-Xxx-Gly, but some XXX or xxx may refer to X{3}, eg Gly-xxx-Gly
       Yaa => 'X',
       Zaa => 'X',

      );

    my %aa3_to_aa1 = @aa3_to_aa1;
    my @aa3_to_aa1_keys = grep exists $aa3_to_aa1{$_}, @aa3_to_aa1;
    # print to redo sort:
#     for (sort { $aa3_to_aa1{$a} cmp $aa3_to_aa1{$b} } keys %aa3_to_aa1) { 
# 	next unless length $_ == 3; 
# 	print "$_ => '$aa3_to_aa1{$_}',\n";
#     }
    my $aa3_to_aa1 = join "|", @aa3_to_aa1_keys;
    my $aa3_to_aa1_re = qr!$aa3_to_aa1!;

    # enable parsing lowercase 3 letter symbols, eg 'NAc-lys-gly-gln-OH'. 
    my %aa3_to_aa1_lc = map { lc($_) => $aa3_to_aa1{$_} } keys %aa3_to_aa1;
    my @aa3_to_aa1_keys_lc = map { lc $_ } @aa3_to_aa1_keys;
    my $aa3_to_aa1_lc = join "|", @aa3_to_aa1_keys_lc;
    my $aa3_to_aa1_lc_re = qr!$aa3_to_aa1_lc!;

    sub aa3_to_aa1 {
	my ($self, $str) = @_;
	# first try more frequent, uppercase, then lowercase.
	# tag with _BeginTag_ and _EndTag_ the replaced amino acids, 
	# then use the tag to remove the rest 
	# of the chars, except for a set of allowed chars that did not 
	# need 3 letter to 1 letter conversion and thus did not get tagged.
	# proceed if > 1 replacements , eg > 1 aa.
	return unless 
	  ( 
	   ($_ = $str and s!($aa3_to_aa1_re)!_BeginTag_$aa3_to_aa1{$1}_EndTag_!g > 1) or 
	   ($_ = $str and s!\b($aa3_to_aa1_lc_re)\b!_BeginTag_$aa3_to_aa1_lc{$1}_EndTag_!g > 1)
	  );
	# warn "tag aa3_to_aa1: \$str=$str;" if $self->{verbose} > 2;
	# warn "tag aa3_to_aa1: \$_=$_;" if $self->{verbose} > 2;
	# handle x and X here, so that the method returns TRUE only if 3 letter 
	# symbols occurred. x and X do not count.
	s!\bx\b!_BeginTag_X_EndTag_!g;
	# neg lookbehind: tag each X, except those already tagged
	# (eg, converted from Xaa to X) (= preceded by _), or 
	# those where X means something other than amino acid 
	# (eg, Met(OX) means Met, oxidized) (= preceded by non-X upper case letter).
	s!(?<\![_A-WYZ])X!_BeginTag_X_EndTag_!g;
	# warn "tag X: \$_=$_;" if $self->{verbose} > 2;
	# change 'X(3)' to 'XXX'
	# repeat only Xs, because X(n) usually refers to X repeated n times, 
	# while for other aa, such as G,
	# G(n) refers to the position within the protein.
	# Some cases where X(n) refers to position n are eliminated later if n is too large, 
	# eg X(20) results in sequence length > 20 and is assigned lower score.
	s!_BeginTag_X_EndTag_\W*(\d)\W*!'_BeginTag_X_EndTag_' x $1!eg;
	$_ = join '', grep { defined $_ } m!([\-/()])|_BeginTag_(.)_EndTag_!g;
	return clean_seq($_);
    }
}


{

    # p = phospho, eg pY = phosphotyrosine
    # x and X are both frequently used in motifs as any aa.
    # B, Z, U are not included because they do not occur very frequently in aa sequences in abstracts.
    my $aa1_allowed = '\-/()ACDEFGHIKLMNPQRSTVWYXpx';
    my $aa1_re = qr!
				# not immed preceded by other letters     	
				# optional N-terminus
				(?:N\-|HN\-|H2N\-|H\(2\)N\-|[\W\d]|^)  
				# sequence, >= 2 aa
				([$aa1_allowed]{2,})
				# not immed followed by other letters
				(?:[\W\d]|\-.*|$) 
				!x;

    sub to_aa1 {
	my ($self, $str) = @_;
	for ($str) {
	    if (m!$aa1_re!) {
		return clean_seq($1);
	    }
	}
	return;
    }
}

sub clean_seq {
    my ($str) = @_;
    for ($str) {
	s!\-NH[()\d]*$!!;      	# remove C/N terminal mark
	s!(^[CN]'|[CN]'$)!!g;
	s!FMK\W*$!!;		# remove aa modifications, eg fluorophores
	tr!\-!!d;		# delete connecting '-', eg change 'G-E-T' to 'GET'
	tr!p!!d;		# delete p=phospho, not P=proline. lowercase only.	
	$_ = uc $_;		# do this after resolving: p=phospho, not P=proline; 
				# Xxx = X, not XXX = X{3}
	s!X\W*(\d)\W*!'X' x $1!eg;
	$_ = clean_orphan_parens($_);
	s!^/+!!;		# clean up beginning and end
	s!/+$!!;		# clean up beginning and end
	s!/+!/!g;		# clean up repeated separator chars
	$_ = parse_slashes($_);
	$_ = clean_orphan_parens($_);
	return $_;
    }
}

sub clean_orphan_parens {
    my ($str) = @_;
    for (my $i = 0; $i < 100; $i++) { # magic number 100 to prevent inf loops
	my $num_changes = 0;
	$num_changes += s!^\)+!!g;		# delete wrong sided parens
	$num_changes += s!\(+$!!g;		# delete wrong sided parens
	$num_changes += s!\(+!\(!g;		# change '((GET)' to '(GET)'
	$num_changes += s!\)+!\)!g;		# change '(GET))' to '(GET)'
	$num_changes += s!\([^A-Z]*\)!!g;	# remove '(/)', '()'
	$num_changes += s!^\((.)\)!$1!;		# change '(G)ETRAPL' to 'GETRAPL'
	# change 'GETR(A)PL' to 'GETRAPL'
	# change 'GET(RAP)L' to 'GETRAPL'
	$num_changes += s!\(([^()/]*)\)!$1!g;
	# change '(GETR(A)PL)' to 'GETR(A)PL' - delete outer parens in 2 steps:
	$num_changes += s!\(([^()]+)\(!$1\(!g;	# change '(GETR(A)PL)' to 'GETR(A)PL)'
	$num_changes += s!\)([^()]+)\)!\)$1!g;	# change 'GETR(A)PL)' to 'GETR(A)PL'
	my $num_parens = tr!()!!;
	if ($num_parens == 1) {
	    $num_changes += tr!()!!d;	       	# change 'GE(TRAPL' to 'GETRAPL'
	}
	last unless $num_changes; # clean stubborn sequences, eg: 
	# 'G(ET(RA)PL' => 'G(ETRAPL' (after iteration 1), 'GETRAPL' (after iteration 2).
    }
    return $_;
}

sub parse_slashes {
    $_ = $_[0];
    return $_ unless m!/!;
    if (m!/X|X/! or has_repeats_at_slashes($_) ) {
	my $str = '';
	for (split m!/+!) {
	    $str = $_  if length($str) < length($_); # change PXXP/PXPXP to PXPXP
	}
	return $str;
    } else {
	s!(\w(/\w)+)!($1)!g;	# change PXXP/GXPXP to PXX(P/G)XPXP
	return $_;
    }	
}

sub has_repeats_at_slashes {
    for ($_[0]) {
	for (m!(\w(?:/\w)+)!g) {
	    my %seen;
	    for (/\w/g) {
		return 1 if $seen{$_}++;
	    }
	}
    }
    return;
}

sub WordVars {
    my ($self, $rh_word) = @_;
    return unless $rh_word and ref $rh_word eq 'HASH';
    $_ = $rh_word->{WordSequence};
    # keep allowed chars in kmers(): 'a'..'z' in sync with 
    # WordPropProtein() args : lc tr/A-Z//cd;
    ( $rh_word->{WordAlpha} = $_ ) =~ tr/A-Z//cd;
    
    if ( $rh_word->{WordSequence} ) {
	$rh_word->{WordSeqLen} = length $rh_word->{WordAlpha};
	$rh_word->{WordNumDegen} = tr/X//;
	$rh_word->{WordNumNotDegen} = $rh_word->{WordSeqLen} - $rh_word->{WordNumDegen};
	$rh_word->{WordPropDegen} = $rh_word->{WordNumDegen} / ($rh_word->{WordSeqLen} || 1);
	$rh_word->{WordLcSequence} = lc $rh_word->{WordSequence};
    }
    
    if ($rh_word->{WordAaSymbols} == 1) {
	# Compute WordPropProtein only for 1 letter symbols, because
	# it is important for classification if 'GET' looks like an english word, 
	# but not important if 'Gly-Glu-Thr' does. 
	$rh_word->{WordPropProtein} = $self->WordPropProtein(lc $rh_word->{WordAlpha});
	$rh_word->{WordIsDNALen} = tr/ACTGN//;
	$rh_word->{WordPropDNA} = $rh_word->{WordIsDNALen} / $rh_word->{WordSeqLen}
	  if $rh_word->{WordSeqLen};
	# if all or mostly dna, eg, '(A/C)TGG' or 'WCCGGGCGGCCCC'
	$rh_word->{WordIsDNA} = 1 if $rh_word->{WordPropDNA} >= 0.9;
    }
    return 1;
}

{
    my $re;

    
    sub WordPropProtein {
	my ($self, $str) = @_;
	my $WordPropProtein = 0;
	return $WordPropProtein if (length($str) < $self->{kmer_length});
	$re ||= qr/(?=(.{$self->{kmer_length}}))/;
	foreach ($str =~ /$re/g) {
	    unless (defined $log_prop_sequence{$_}) {
		carp "not ok: undefined log_prop_sequence for $_";
		next;
	    }
	    unless (defined $log_prop_nonsequence{$_}) {
		carp "not ok: undefined log_prop_nonsequence for $_";
		next;
	    }
	    $WordPropProtein += ($log_prop_sequence{$_} - $log_prop_nonsequence{$_});
	}
	$WordPropProtein = sprintf("%.1f", $WordPropProtein);
	return $WordPropProtein;
    }   
}

{
    # WordScore factors for different values of vars, eg 
    # for WordAaSymbols = 3, multiply WordScore by 1000.

    # WordScoreForAaSymbols:
    # Keep the ratio of 3 letter or full name symbols relative to 1 letter symbols high, 
    # because even for very short words they are more likely to be a sequence, eg: 
    # 'Pro-Ala-Pro' relative to 'PAP'.
    my %WordScoreForAaSymbols = 
      ( 
       0 => 0,
       1 => 1, 
       3 => 1000, 
       5 => 10000, # full names (not implemented)
      );
    # 
    my %WordScoreForSeqLen =
      (
       0 => 0,
       1 => 0,
       2 => 0.0001,
       3 => 0.1,
       4 => 0.5,
       5 => 1,
      );

    # as WordSeqLen increases between 5 and $MAX_PEPTIDE_LENGTH, 
    # WordScore factor increases linearly from 1 to 5
    foreach (5..$MAX_PEPTIDE_LENGTH) {
	$WordScoreForSeqLen{$_} = 1 + 4 * ($_ - 5) / ($MAX_PEPTIDE_LENGTH - 5);
    }
    my $max_penalized_length = 6; # after this length, flat rate penalty applies:
    my $WordScoreForMatchLenMin = 3**(-$max_penalized_length);
    # up to $max_penalized_length, penalize by this factor (divide score by 3 for each char)
    my %WordScoreForMatchLen = map { ($_ => 3**(-$_)) } 0..$max_penalized_length;

    sub WordScore {
	my ($self, $rh_word) = @_;
	ref $rh_word eq 'HASH' or 
	  carp "not ok: expected WordScore arg[1]  = hash ref, got $rh_word" and return;
	$rh_word->{WordScore} = 1; # initialize WordScore
	unless ($rh_word->{WordSequence} and $rh_word->{WordOrig}) {
	    $rh_word->{WordScore} = 0;
	}
	warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
	return 1 unless $rh_word->{WordScore};
	$rh_word->{WordScore} *= $WordScoreForAaSymbols{ $rh_word->{WordAaSymbols} };
	warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
	if (exists $WordScoreForSeqLen{ $rh_word->{WordSeqLen} }) {
	    $rh_word->{WordScore} *= $WordScoreForSeqLen{ $rh_word->{WordSeqLen} };
	} else { # if length > $MAX_PEPTIDE_LENGTH, discard regardless of anything else
	    $rh_word->{WordScore} = 0;
	} 	
	warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
	return 1 unless $rh_word->{WordScore};
	
	if (
	    $rh_word->{WordPropDegen} > 0.95 || 
	    $rh_word->{WordNumNotDegen} <= 1 ||
	    ($rh_word->{WordNumNotDegen} <= 2 && $rh_word->{WordSeqLen} >= 6) # <= 2 of 6 aa
	   ) {
	    $rh_word->{WordScore} = 0;
	}
	elsif (
	       $rh_word->{WordPropDegen} > 0.7 ||
	       ($rh_word->{WordNumNotDegen} <= 2 && $rh_word->{WordSeqLen} >= 5)
	      ) {
	    $rh_word->{WordScore} *= 0.001;
	}
	elsif ($rh_word->{WordPropDegen} > 0.6) {
	    $rh_word->{WordScore} *= 0.01;
	}
	warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
	return 1 unless $rh_word->{WordScore};

	$self->WordScoreForMatch($rh_word) or return;
	warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
	return 1 unless $rh_word->{WordScore};
	# normalize for the ease of use, so that (after all transformations), 
	# at WordScore = 0.5, approximately 50% of the words are
	# true positives by manual inspection
	$rh_word->{WordScore} = (1/6) * $rh_word->{WordScore};
	warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
	# transform (0, +inf) to (0,1):
	$rh_word->{WordScore} = $rh_word->{WordScore} / ( 1 + $rh_word->{WordScore} );
	warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
	return 1;
    }

    
sub WordScoreForMatch {	
    my ($self, $rh_word) = @_;
    ref $rh_word eq 'HASH' or 
      carp "not ok: expected WordScoreForMatch arg[1]: hash ref, got: $rh_word" and 
	return;
    # no need to check this for 3 and full letter symbols:
    return 1 unless $rh_word->{WordAaSymbols} == 1;
    $self->WordIsGeneOrDict($rh_word) or return;
    warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
    # roman numeral < 50 that ends within 0 or 1 char of the end of word, eg 
    # TFIII or TFIIA. WordIsRomanLen = length of the roman 
    # numeral (minus 1 if there is 1 char between the numeral and the 
    # end of the word).
    while ($rh_word->{WordOrig} =~ /(X{0,3})(I[XV]|V?I{0,3})([A-Z])?\b/g) {
	my $next_roman_len = length($1) + length($2) - ($3 ? length($3) : 0 );
	next unless $next_roman_len > 0; # skip 0 or -1 (not a roman numeral).
	$rh_word->{WordIsRomanLen} += $next_roman_len;
	$rh_word->{WordIsRoman} = 1 if $rh_word->{WordIsRomanLen} >= 2; # eg, HYNKIIIA
    }
    # for these vars with values: 0/1, penalize based on length:
    foreach (qw(WordIsDNA WordIsDict WordIsGene WordIsRoman)) {
	next unless $rh_word->{$_};
	# limit the penalty by the sequence length, eg
	# if 'SDS-PAGE' is entered into dictionary twice, 
	# as both 'SDS-PAGE' and 'SDSPAGE' 
	$rh_word->{"${_}Len"} = $rh_word->{WordSeqLen} 
	  if $rh_word->{"${_}Len"} > $rh_word->{WordSeqLen};
	$rh_word->{WordScore} *= 
	  ( $WordScoreForMatchLen{ $rh_word->{"${_}Len"} } || 
	    $WordScoreForMatchLenMin ); 
	warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
    }
    # if the word was not flagged by WordIsGene WordIsDict:
    unless ($rh_word->{WordIsGene} || $rh_word->{WordIsDict}) {
	# penalize for similarity to a gene symbol, eg KLF4. 
	foreach ($rh_word->{WordOrig} =~ m!([^A-Z]|\b)[A-Z]{2,3}\d*\b!g) {
	    $rh_word->{WordScore} *= 0.1;
	    warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
	    # penalize for similarity to abbreviations, or to gene/protein interactions, eg 
	    # 'GM-CSF', 'HPLC/FPLC', 'MAP/MAPKK'.
	    $rh_word->{WordScore} *= 0.01 
	      if $rh_word->{WordSubLenMax} && # if the word could be split into subwords and
		$rh_word->{WordSubLenMax} <= 4; # all sub-words are short
	    warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
	}
    }
    # WordPropProtein is a good predictor if abs(WordPropProtein) > 8.
    # WordPropProtein is very high for WordIsDNA, eg 'CGGTTAAAA', or 
    # WordIsRoman, eg 'HYNKIIIA', thus not used.
    unless(
	     $rh_word->{WordIsDNA} || $rh_word->{WordIsDict} || 
	     $rh_word->{WordIsGene} || $rh_word->{WordIsRoman}
	  ) {
	$rh_word->{WordScore} *= 
	  (10 ** (LOG_10_2 * $rh_word->{WordPropProtein}) );
    }
    warn "WordScore = $rh_word->{WordScore}; WordOrig = $rh_word->{WordOrig}" if $self->{verbose} > 2;
    carp((caller(0))[3], "(@_)", ' ', Data::Dumper->Dump([$rh_word], ['rh_word'])) 
      if $self->{verbose} > 2;
    return 1;
}
}

sub WordIsGeneOrDict {	
    my ($self, $rh_word) = @_;
    ref $rh_word eq 'HASH' or 
      carp "not ok: expected WordIsGeneOrDict arg[1]: hash ref, got: $rh_word" and 
	return;
    # select words for matching. Do not penalize more than once for the 
    # same subword, unless it actually occurs more than once.
    my @subwords = split /[^A-Za-z]+/, lc $rh_word->{WordLcOrig};
    my @subwords_long;
    if (@subwords != 1) { # if could split; otherwise just handle $rh_word->{WordLcOrig} below.
	# select sub-words that are long in absolute terms and also long relative 
	# to the original word 
	# (eg, select 'RAS' in 'RAS/EGFR' , but not in 'RAS/GETRAPLGETRAPL'
	foreach (@subwords) {
	    my $length = length $_;
	    push @subwords_long, $_ 
	      if $length > 2 and ( $length / $rh_word->{WordSeqLen} ) > 0.25;
	    # WordSubLenMax = max sub-word length, if could split; otherwise 0.
	    $rh_word->{WordSubLenMax} = $length if $rh_word->{WordSubLenMax} < $length;
	}
    }
    foreach my $subword ( $rh_word->{WordLcSequence}, 
	      # do not penalize more than once for 1 occurrence:
	      ( $rh_word->{WordLcOrig} eq $rh_word->{WordLcSequence} ? 
		() : $rh_word->{WordLcOrig} ), 
	      @subwords_long ) {
	foreach my $bad (qw(Dict Gene)) {
	    # if ( eval { ${$bad}{$subword} } ) { # eval does not work - ??
	    my $is_bad = do {
		if	($bad eq 'Dict') { $Dict{$subword} }
		elsif	($bad eq 'Gene') { $Gene{$subword} }
	    };
	    if ($is_bad) {
		$rh_word->{"WordIs$bad"} = 1;
		$rh_word->{"WordIs${bad}Len"} += length($subword);
	    }
	}
    }
    return 1;
}

sub WordAbstScore {
    my ($self) = @_;
    my $rh_abst = $self->{abst};
    # max WordScore for all words in this abstract:
    $rh_abst->{AbstMaxWordScore} = 0;
    foreach my $rh_word (@{ $self->{words} }) {
	$rh_abst->{AbstMaxWordScore} = $rh_word->{WordScore} 
	  if $rh_abst->{AbstMaxWordScore} < $rh_word->{WordScore};
    }	
    
    foreach my $rh_word (@{ $self->{words} }) {
	$rh_word->{WordAbstScore} = $rh_word->{WordScore} > 0 ?
	  $rh_abst->{AbstScore} *
	    (0.1 * $rh_abst->{AbstMaxWordScore} + 0.9 * $rh_word->{WordScore})
	      : 0; 
    }
    if ($self->{verbose} > 0) {
	$rh_abst->{AbstComment} .= 
	  sprintf("AbstMaxWordScore=%.4f; ", $rh_abst->{AbstMaxWordScore});
    }
    carp((caller(0))[3], "(@_)", ' ', Data::Dumper->Dump([$self], ['self'])) 
      if $self->{verbose} > 2;
    return 1;
}

sub AbstMtext {
    my ($self) = @_;
    if (
	$self->{words} and 
	ref $self->{words} eq 'ARRAY'
       ) {
	$self->{abst}{AbstMtext} = 
	  join '', map {
	      if (! defined $_->{WordOrig} ) { () } # happens if WordNumPrintMin = 1
	      elsif (
		     $_->{WordScore}		>= $self->{WordScoreMin} && 
		     $_->{WordAbstScore}	>= $self->{WordAbstScoreMin}
		    ) 
		{ "<mark>$_->{WordOrig}</mark>"}
	      else { $_->{WordOrig} }
	    } 
	    @{ $self->{words} } ;
    }
    else {
	$self->{abst}{AbstMtext} = $self->{abst}{AbstAbstract} || '';
    }
    return 1;
}

sub formatScoreProp {
    my ($number) = @_;
    return unless defined $number;
    # number should be between 0 and 1,
    # In methods in this package, the input $number is supposed to be always in (0,1) - 
    # maybe issue warning otherwise?
    $number = 0 if $number < 0;
    $number = 1 if $number > 1;
    $number = sprintf "%.4f", $number;
}

sub find {
    my ($self, $key, $val) = @_;
    foreach ( @{ $self->{words} } ) {
	return $_ if defined $_->{$key} and $_->{$key} eq $val;
    }
    return;
}

1;