UMLS::Similarity::vector - Perl module for computing semantic relatedness

 @inproceedings{PatwardhanP06,
  title={{Using WordNet-based Context Vectors to Estimate 
          the Semantic Relatedness of Concepts}},
  author={Patwardhan, S. and Pedersen, T.},
  booktitle={Proceedings of the EACL 2006 Workshop Making Sense
             of Sense - Bringing Computational Linguistics and 
             Psycholinguistics Together},
  volume={1501},
  pages={1-8},
  year={2006},
  month={April},
  address={Trento, Italy}
 }

  use UMLS::Interface;
  use UMLS::Similarity::vector;

  my $umls = UMLS::Interface->new(); 
  die "Unable to create UMLS::Interface object.\n" if(!$umls);

  my $vector = UMLS::Similarity::vector->new($umls);
  die "Unable to create measure object.\n" if(!$vector);

  my $cui1 = "C0018563";
  my $cui2 = "C0037303";

  $ts1 = $umls->getTermList($cui1);
  my $term1 = pop @{$ts1};

  $ts2 = $umls->getTermList($cui2);
  my $term2 = pop @{$ts2};

  my $value = $vector->getRelatedness($cui1, $cui2);

  print "The similarity between $cui1 ($term1) and $cui2 ($term2) is $value\n";

	9
	LONG<>file<>1 1 1
	Of<>a<>1 1 1
	This<>is<>1 1 1
	a<>LONG<>1 1 1
	file<>.<>1 1 1
	first<>line<>1 1 1
	is<>the<>1 1 1
	line<>Of<>1 1 1
	the<>first<>1 1 1

	. 1 0
	LONG 2 0 8
	Of 3 8 8
	This 4 16 8
	a 5 24 8
	file 6 32 8
	first 7 40 8
	is 8 48 9
	line 9 57 8
	the 10 65 9

	2: 6 1
	3: 5 1
	4: 8 1
	5: 2 1
	6: 1 1
	7: 9 1
	8: 10 1
	9: 3 1
	10: 7 1

    1. cui1/term1 cui2/term2 
       without --dictfile option and without --config option, 
       use the UMLS definition of the default config file. 

    2. cui1/term1 cui2/term2  --dictfile ./sample/dictfile
       --dictfile option is set and without --config option, 
       definitions only come from dictfile. 

    3. cui1/term1 cui2/term2  --config ./sample/leskmeasure.config
       without --dictfile option, --config option is set, 
       definitions only come from UMLS by the config file. 

    4. cui1/term1 cui2/term2  --dictfile ./sample/dictfile --config ./sample/leskmeasure.config
       --dictfile option is set, --config option is set, 
       definitions come from dictfile and UMLS. If the associated term 
       for each CUI is defined in the dictfile, the associated terms' 
       definition are also included.  

    For example, the original defintion for 'cat' is, 
    cat: a feline pet, feline 

    And then, the word vector for feline and pet in the doubledef file is: 
    feline: small to medium-sized cats, cougar cheetah
    pet: cat dog bird fish

    The final definition for cat is to combine the original definition for cat, and 
    then add the definition for feline(only add once) and pet. 

    cat: a feline pet small to medium-sized cats cougar cheetah cat dog bird fish

    For every unique word of the final definition, the doubledef finds the vector from the 
    co-occurrence matrix of each word and build the vector for cat.  

    For example, the definition for iraq and france are: 

    iraq : saddam hussein
    france : jacques chirac

    In the --compoundfile file, "saddam hussein" and "jacques chirac" are compounds:

    jacques_chirac
    saddam_hussein

    So, the compound words in the definition could be detected:

    iraq : saddam_hussein
    france : jacques_chirac

   use UMLS::Similarity::vector;
   $measure = UMLS::Similarity::vector->new($interface);

   $relatedness = $measure->getRelatedness('C0005767', 'C0007634');

      1. CUI which refers to the CUIs definition

      2. TERM which refers to the terms associated with the CUI

  If you have any trouble installing and using UMLS-Similarity, 
  please contact us via the users mailing list :

      umls-similarity@yahoogroups.com

  You can join this group by going to:

      http://tech.groups.yahoo.com/group/umls-similarity/

  You may also contact us directly if you prefer :

      Bridget T. McInnes: bthomson at cs.umn.edu 

      Ted Pedersen : tpederse at d.umn.edu

  Bridget T McInnes <bthomson at cs.umn.edu>
  Siddharth Patwardhan <sidd at cs.utah.edu>
  Serguei Pakhomov <pakh0002 at umn.edu>
  Ted Pedersen <tpederse at d.umn.edu>
  Ying Liu <liux0935 at umn.edu> 

# UMLS::Similarity::vector.pm
#
# Module implementing the vector semantic relatedness measure 
# based on the measure proposed by Patwardhan and Pedersen 
# (2006)
#
# Copyright (c) 2009-2011,
#
# Bridget T McInnes, University of Minnesota, Twin Cities
# bthomson at umn.edu
#
# Siddharth Patwardhan, University of Utah, Salt Lake City
# sidd at cs.utah.edu
#
# Serguei Pakhomov, University of Minnesota, Twin Cities
# pakh002 at umn.edu
#
# Ted Pedersen, University of Minnesota, Duluth
# tpederse at d.umn.edu
#
# Ying Liu, University of Minnesota
# liux0935 at umn.edu
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to 
#
# The Free Software Foundation, Inc., 
# 59 Temple Place - Suite 330, 
# Boston, MA  02111-1307, USA.



package UMLS::Similarity::vector;

use strict;
use warnings;

use UMLS::Similarity;
use Lingua::Stem::En;
use UMLS::Similarity::ErrorHandler;


use vars qw($VERSION);
$VERSION = '0.11';

my $debug        = 0;
my $defraw_option= 0;

my $compoundfile = "";
my $vectormatrix = "";
my $vectorindex  = "";
my $dictfile     = "";
my $doubledef    = "";
my $stoplist	 = "";
my $stem         = "";
my $stopregex	 = "";
my $debugfile    = "";
my $config       = "";

my %index         = ();
my %reverse_index = ();
my %position      = ();
my %length        = ();
my %dictionary    = ();
my %ddef          = ();
my %stopwords     = ();
my %complist 	  = ();


local(*DEBUG);

sub new
{
    my $className = shift;

    return undef if(ref $className);

    if($debug) { print STDERR "In UMLS::Similarity::vector->new()\n"; }

    my $interface = shift;
    my $params    = shift;

    my $self = {};
    
    # Bless the object.
    bless($self, $className);
    
    # The backend interface object.
    $self->{'interface'} = $interface;
    
    #  check the configuration file if defined
    my $errorhandler = UMLS::Similarity::ErrorHandler->new("vector",  $interface);
    if(!$errorhandler) {
	print STDERR "The UMLS::Similarity::ErrorHandler did not load properly\n";
	exit;
    }
    
    $params = {} if(!defined $params);

    $vectorindex  = $params->{'vectorindex'};
    $vectormatrix = $params->{'vectormatrix'};
    $config       = $params->{'config'};
    $dictfile     = $params->{'dictfile'};
    $doubledef    = $params->{'doubledef'};
    $compoundfile = $params->{'compoundfile'};
    $debugfile	  = $params->{'debugfile'};
    $stoplist	  = $params->{'stoplist'};
    $stem         = $params->{'stem'};
    
    my $defraw       = $params->{'defraw'};
    
    if(defined $defraw) { 
	$defraw_option = 1;
    }
   
	# read in the doubledef for --doubledef option 
    if (defined $doubledef) {
	open(DDEF, "$doubledef")
	    or die("Error: cannot open doubledef file ($doubledef).\n");
	
	while(<DDEF>) {
	    chomp;
	    if($_=~/^\s*$/) { next; }
	    
	    my @defs = split (":", $_);
	    my $concept = $defs[0]; 
	    $concept =~ s/^\s+//;
	    $concept =~ s/\s+$//;
	    my $definition = $defs[1];	
	    $ddef{$concept} = $definition;
	}
	close DDEF;
    }


    # read in the dictfile for --dictfile option 
    if (defined $dictfile) {
	open(DICT, "$dictfile")
	    or die("Error: cannot open dictionary file ($dictfile).\n");
	
	while(<DICT>) {
	    chomp;
	    
	    if($_=~/^\s*$/) { next; }
	    
	    my @defs = split (":", $_);
	    my $concept = $defs[0]; 
	    $concept =~ s/^\s+//;
	    $concept =~ s/\s+$//;
	    my $definition = $defs[1];	
	    $dictionary{$concept} = $definition;
	}
	close DICT;
    }

    #  if the vector index is not defined - get the default
    if(! (defined $vectorindex) ) { 
	foreach my $path (@INC) {
	    if(-e $path."/UMLS/vector-index.dat") { 
		$vectorindex = $path."/UMLS/vector-index.dat";
	    }
	    elsif(-e $path."\\UMLS\\vector-index.dat") { 
		$vectorindex =  $path."\\UMLS\\vector-index.dat";
	    }
	}
    }

    #  check to make certain that you found the vector index file
    if(! (defined $vectorindex) ) { 
	print STDERR "Error: can not find the default vector index file (vector-index.dat).\n";
	exit;
    }

    #  if the vector matrix is not defined - get the default
    if(! (defined $vectormatrix) ) { 
	foreach my $path (@INC) {
	    if(-e $path."/UMLS/vector-matrix.dat") { 
		$vectormatrix = $path."/UMLS/vector-matrix.dat";
	    }
	    elsif(-e $path."\\UMLS\\vector-matrix.dat") { 
		$vectormatrix =  $path."\\UMLS\\vector-matrix.dat";
	    }
	}
    }
    
    #  check to make certain that you found the vector matrix file
    if(! (defined $vectormatrix) ) { 
	print STDERR "Error: can not find the default vector matrix file (vector-matrix.dat).\n";
	exit;
    }
	
    # read in the vectorindex file
    open(INDX, "<$vectorindex")
        or die("Error: cannot open file '$vectorindex' for output index.\n");
    
    while (my $line = <INDX>)
    {
	chomp($line);
	my @terms = split(' ', $line);
	
	$index{$terms[0]} = $terms[1];
	$reverse_index{$terms[1]} = $terms[0];
	$position{$terms[1]} = $terms[2]; 
	$length{$terms[1]} = $terms[3]; 
    }
    close INDX;
    
    if(defined $debugfile) { 
	if(-e $debugfile) {
	    print "Debug file $debugfile already exists! Overwrite (Y/N)? ";
	    my $reply = <STDIN>;
	    chomp $reply;
	    $reply = uc $reply;
	    exit 0 if ($reply ne "Y");
	}
	
	open(DEBUG, ">$debugfile") || die "Could not open debug file: $debugfile\n";
    }
    
    if (defined $stoplist) {
	
	open(STP, "$stoplist")
	    or die("Error: cannot open stop list file ($stoplist).\n");
	
	$stopregex  = "(";
	while(<STP>) {
	    chomp;
	    if($_ ne ""){
        	$_=~s/\///g;
        	$stopregex .= "$_|";
		}
	}   
	chop $stopregex; $stopregex .= ")";
	close STP;
	
    }

	if(defined $compoundfile) {
    
	#replace the compound words in the definition
	open(LST, "$compoundfile") or die ("Error: cannot open file $compoundfile for input.\n");

	# read the compound txt and put them in the hash array. 
	while (my $line = <LST>)
	{
		chomp($line);
		my $lower_case = lc($line);
		my @string = split('_', $lower_case);
		my $head = shift(@string);

		my $rest = join (' ', @string);
		push (@{$complist{$head}}, $rest);
	}
	close LST;
		
	# sort the compound txt 
	foreach my $h (sort (keys (%complist)) )
	{
		my @sort_list = sort(@{$complist{$h}});
		for my $i (0..$#sort_list)
		{
			$complist{$h}[$i] = $sort_list[$i];
		}
	}
	}
    return $self;
}


sub getRelatedness
{
    my $self = shift;
    
    return undef if(!defined $self || !ref $self);
    
    my $concept1 = shift;
    my $concept2 = shift;
    
    if(defined $debugfile) {
	print DEBUG "$concept1<>$concept2\n";
    }
   
    my $interface = $self->{'interface'};
    
    my $d1 = "";
    my $d2 = "";
    
    if (!defined $dictfile) {
	if ($concept1 =~ /C[0-9]+/)
	{	
	    my $defs1 = $interface->getExtendedDefinition($concept1);
	    if(defined $debugfile) {
		print DEBUG "DEFINITIONS FOR CONCEPT 1 $concept1: \n";
	    }
	    my $i = 1;
	    foreach my $def (@{$defs1}) {
		if(defined $debugfile) {
		    print DEBUG "$i. $def\n";
		    $i++;
		}
		$def=~/(C[0-9]+) ([A-Za-z]+) ([A-Za-z0-9]+) ([A-Za-z0-9\.]+) \s*\:\s*(.*?)$/;
		$d1 .= $5 . " ";
	    }

	    #if the definition is empty, return -1
	    if ($d1 eq "") 
	    {
			return -1;
	    }
	}
	if($concept2 =~ /C[0-9]+/)
	{
	    my $defs2 = $interface->getExtendedDefinition($concept2);
	    if(defined $debugfile) {
		print DEBUG "DEFINITIONS FOR CONCEPT 2 $concept2: \n";
	    }
	    my $i = 1;
	    foreach my $def (@{$defs2}) {
		if(defined $debugfile) {
		    print DEBUG "$i. $def\n";
		    $i++;
		}
		$def=~/(C[0-9]+) ([A-Za-z]+) ([A-Za-z0-9]+) ([A-Za-z0-9\.]+) \s*\:\s*(.*?)$/;
		$d2 .= $5 . " ";
	    }
	    
	    #if the definition is empty, return -1
	    if ($d2 eq "") 
	    {
			return -1;
	    }
	}
    } # end of without --dictfile option 
    
    if (defined $dictfile)
    { 
	my $defs1;
	my $defs2;
	my $term1;
	my $term2;
	my $term1_def = "";
	my $term2_def = "";

	my @dictfile_term1;
	my @dictfile_term2;
	
	if(defined $debugfile) { print DEBUG "DEFINITIONS FOR CONCEPT 1: $concept1\n"; }

	# the input format is cui#term	
	if($concept1 =~ /^(C[0-9]+)(\#)(.*?)$/)
	{
	    my $cui1 = $1;
	    $term1 = $3;
	    
	    $defs1 = $interface->getExtendedDefinition($cui1);
	    $term1_def = $dictionary{$term1} if (defined $dictionary{$term1});
	    
	    my $i = 1;
	    foreach my $extendeddef (@{$defs1}) {
		if (defined $debugfile) {
		    print DEBUG "$i. $extendeddef\n";
		    $i++;
		}
		
		#  seperate definition from the other information 
		#  sent by the getExtendedDefinition function
		$extendeddef=~/(C[0-9]+) ([A-Za-z]+) ([A-Za-z0-9]+) ([A-Za-z0-9\.]+) \s*\:\s*(.*?)$/;
		my $def = $5;
		
		#  store the definition in the string d1
		$d1 .= $def . " "; 
	    }	   
	    
	    $d1 .= $term1_def . " " if $term1_def ne ""; 

		# check the if the cui's assoicated terms are defined in the dictfile
	    #@dictfile_term1 = $interface->getTermList($cui1);			
		#foreach my $t (@dictfile_term1)
		#{
			#if (defined $dictionary{$t})
			#{
				#my $term1_def = $dictionary{$t};
				#$d1 .= "$term1_def" . " ";
			#}
		#}

	    if(defined $debugfile)
	    {
			print DEBUG "$i. $term1_def\n" if (defined $term1_def);
	    }
	    
	    #if the definition is empty, return -1
	    if ($d1 eq "") 
	    {
			return -1;
	    }
	}
	else #the input concept is a term 
	{
	    if (defined $dictionary{$concept1}) {
		$d1 = $dictionary{$concept1};
		if (defined $debugfile) {
		    print DEBUG "$concept1: $d1\n"; 
		}
	    }
	    else{
		if (defined $debugfile) {
		    print DEBUG "$concept1: not defined\n"; 
		}
		return -1; }
	}
	
	if(defined $debugfile) { print DEBUG "DEFINITIONS FOR CONCEPT 2: $concept2\n"; }
	
	if($concept2 =~ /^(C[0-9]+)(\#)(.*?)$/)
	{
	    my $cui2 = $1;
	    $term2 = $3;
	    
	    $defs2 = $interface->getExtendedDefinition($cui2);
	    $term2_def = $dictionary{$term2} if (defined $dictionary{$term2});
	    
	    my $i = 1;
	    foreach my $extendeddef (@{$defs2}) {
		if (defined $debugfile) {
		    print DEBUG "$i. $extendeddef\n";
		    $i++;
		}
		
		#  seperate definition from the other information 
		#  sent by the getExtendedDefinition function
		$extendeddef=~/(C[0-9]+) ([A-Za-z]+) ([A-Za-z0-9]+) ([A-Za-z0-9\.]+) \s*\:\s*(.*?)$/;
		my $def = $5;
		
		#  store the definition in the string d1
		$d2 .= $def . " "; 
	    }	   
	    
	    $d2 .= $term2_def . " " if $term2_def ne ""; 
	   

		# check the if the cui's assoicated terms are defined in the dictfile
		#@dictfile_term2 = $interface->getTermList($cui2);			
		#foreach my $t (@dictfile_term2)
		#{
			#if (defined $dictionary{$t})
			#{
				#my $term2_def = $dictionary{$t}; 
				#$d2 .= "$term2_def" . " ";
			#}
		#}

	    if(defined $debugfile)
	    {
			print DEBUG "$i. $term2_def\n" if (defined $term2_def);
	    }
	    
	    #if the definition is empty, return -1
	    if ($d2 eq "") 
	    {
			return -1;
	    }
	}
	else
	{
	    if (defined $dictionary{$concept2}) {
		$d2 = $dictionary{$concept2};
		if (defined $debugfile) {
		    print DEBUG "$concept2: $d2\n"; }
	    }
	    else{
		if (defined $debugfile) {
		    print DEBUG "$concept2: not defined\n"; }
		return -1; }
	}
	
    } #end of defined --dictfile option


	# if define --doubledef option
	if (defined $doubledef)
	{
		my @def1 = split(/\s/, $d1);	
		my @def2 = split(/\s/, $d2);	
	
		my %unique = (); # for every word, only check its definition once	
		foreach my $w (@def1)
		{
			$unique{$w}++;	
			if ((defined $ddef{$w}) and ($unique{$w}==1))
			{
				my $def = $ddef{$w}; 
				if (defined $debugfile) 
				{
		    		print DEBUG "ddef1 $w: $def\n"; 
				}
				$d1 .= "$def" . " ";
			}
		}		

		%unique = ();	
		foreach my $w (@def2)
		{
			$unique{$w}++;	
			if ((defined $ddef{$w}) and ($unique{$w}==1))
			{
				my $def = $ddef{$w}; 
				if (defined $debugfile) 
				{
		    		print DEBUG "ddef2 $w: $def\n"; 
				}
				$d2 .= "$def" . " ";
			}
		}		

		if (defined $debugfile)
		{
			print DEBUG "after --doubledef processing\n";
			print DEBUG "concept 1: $d1\n";
			print DEBUG "concept 2: $d2\n";
		}


	} #end of defined --doubledef option
   
 
    #  if the --defraw option is not set clean up the defintions
    if($defraw_option == 0) {
	$d1 = lc($d1); $d2 = lc($d2);
	
	# remove punctuation doesn't contain '<' and '>' and '_'    
	$d1=~s/[\.\,\?\/\'\"\;\:\[\]\{\}\!\@\#\$\%\^\&\*\(\)\-\+\-\=]//g;
	$d2=~s/[\.\,\?\/\'\"\;\:\[\]\{\}\!\@\#\$\%\^\&\*\(\)\-\+\-\=]//g;

	if (defined $debugfile)
	{
		print DEBUG "after --defraw processing\n";
		print DEBUG "concept 1: $d1\n";
		print DEBUG "concept 2: $d2\n";
	}
    }
   

	# find compound words
	if(defined $compoundfile)
	{
		$d1 = findCompoundWord($d1, \%complist);
		$d2 = findCompoundWord($d2, \%complist);
		if (defined $debugfile)
		{
			print DEBUG "after --compoundfile processing\n";
			print DEBUG "concept 1: $d1\n";
			print DEBUG "concept 2: $d2\n";
		}
	}


    # if --stopword option is set remove stop words
    if (defined $stoplist) 
    {
	my @def1 = split(/\s/, $d1);	
	my @def2 = split(/\s/, $d2);	
	my @new_def1 = ();
	my @new_def2 = ();
	foreach my $w (@def1) {
	    if (!($w =~ /$stopregex/)) {
		push (@new_def1, $w);}
	}
	foreach my $w (@def2) {
	    if (!($w =~ /$stopregex/)) {
		push (@new_def2, $w);}
	}
	
	$d1 = join (" ", @new_def1);	
	$d2 = join (" ", @new_def2);	

	if (defined $debugfile)
	{
		print DEBUG "after --stoplist processing\n";
		print DEBUG "concept 1: $d1\n";
		print DEBUG "concept 2: $d2\n";
	}
    } #end of stoplist option


    if(defined $stem) 
    {
		my @def_words1 = split(/\s/, $d1);
		my $stemmed_words1 = Lingua::Stem::En::stem({ -words => \@def_words1, -locale => 'en'});
		$d1 = join(" ", @{$stemmed_words1});
		
		my @def_words2 = split(/\s/, $d2);
		my $stemmed_words2 = Lingua::Stem::En::stem({ -words => \@def_words2, -locale => 'en'});
		$d2 = join(" ", @{$stemmed_words2});

		if (defined $debugfile)
		{
			print DEBUG "after --stem processing\n";
			print DEBUG "concept 1: $d1\n";
			print DEBUG "concept 2: $d2\n";
		}
    }
    

    open(MATX, "<$vectormatrix")
        or die("Error: cannot open file '$vectormatrix' for output index.\n");
    
    my %vector1 = ();
    my %vector2 = ();
    my @defs1 = split(" ", $d1);	
    my @defs2 = split(" ", $d2);	
   
    my $def1_length = 0 ;
    
    foreach my $def_term1 (@defs1){
	if (defined $index{$def_term1})
	{
	    my $index_term = $index{$def_term1};
	    my $p = $position{$index_term};
	    my $l = $length{$index_term};
	    
	    if (($p==0) and (!defined $l))
	    {
			next;
	    }
	    else
	    {
			$def1_length++;
			
			my ($data, $n);
			seek MATX, $p, 0;
			if (($n = read MATX, $data, $l) != 0)
			{
				if (defined $debugfile) {
				print DEBUG "$def_term1: ";
				}
				
				chomp($data);
				my @word_vector = split (' ', $data);
				my $index = shift @word_vector;
				$index =~ m/^(\d+)\:$/;
				
				if ($index_term == $1)
				{
					for (my $z=0; $z<@word_vector; )
					{
						$vector1{$word_vector[$z]} += $word_vector[$z+1];
						
						if (defined $debugfile) { 
						if(defined $word_vector[$z]) {
							print DEBUG "$reverse_index{$word_vector[$z]} ";
						}
						} 	
					
						$z += 2;
					}
				
					if (defined $debugfile) 
					{
						print DEBUG "\n";
					} 	
				}
				else 
				{
					print STDERR "$def_term1 is not a correct word!\n";
					exit;
				}
			}	
	    }
	}
    } # end of def1
    
    if (defined $debugfile) {
	print DEBUG "def1 length: $def1_length\n";
    } 	
    
    my $def2_length = 0 ;
    foreach my $def_term2 (@defs2)
    {

	if (defined $index{$def_term2})
	{
	    my $index_term = $index{$def_term2};
            my $p = $position{$index_term};
            my $l = $length{$index_term};
	    
	    if (($p==0) and (!defined $l))
	    {
			next;
	    }
	    else
	    {
			$def2_length++;
		
            my ($data, $n);
            seek MATX, $p, 0;
            if (($n = read MATX, $data, $l) != 0)
            {
		    	if (defined $debugfile) {
				print DEBUG "$def_term2: ";
		    	}
		    	chomp($data);
		    	my @word_vector = split (' ', $data);
		    	my $index = shift @word_vector;
                $index =~ m/^(\d+)\:$/;
		    
                if ($index_term == $1)
		    	{
                  	for (my $z=0; $z<@word_vector; )
                    {
			    		$vector2{$word_vector[$z]} += $word_vector[$z+1];
			    
			    		if (defined $debugfile) {
						if(defined $word_vector[$z]) {
				    	print DEBUG "$reverse_index{$word_vector[$z]} "; }
			    		} 	

			    		$z += 2;
                    }
			
					if (defined $debugfile) {
			    	print DEBUG "\n"; } 	
                }
                else
                {
					print STDERR "$def_term2 is not a correct word!\n";
					exit;
                }
           }
	    }
	}
    }
    
    
    if (defined $debugfile) {
	print DEBUG "def2 length: $def2_length\n";
    } 	
    
    #  normalize
    my $vec1 = &norm(\%vector1);
    my $vec2 = &norm(\%vector2);
    
    #  cosine
    my $score = &_inner($vec1, $vec2);
    
    return $score;
}

sub findCompoundWord
{
	my $def = shift;
	my $ref_complist = shift;
	my $new_def = "";

	my @words = split(' ', $def);
	my $size_line = @words;
    for (my $i=0; $i<$size_line; $i++)
    {
        my $w = $words[$i];
        my $flag_print_w = 0;
        my $flag_comp = 0;
        if(defined $ref_complist->{$w})
        {
            # get the compound list start with word $w
            my @comps = @{$ref_complist->{$w}};
            foreach my $c (@comps)
            {
                #compare the rest of the compound word  
                my @string = split(' ', $c);

                my $count = 1;
                foreach my $s (@string)
                {
                    if (($i+$count)<$size_line)
                    {
                        if ($s eq $words[$i+$count])
                        {
                            $flag_comp = 1;
                            $count++;
                        }
                        else
                        {
                            $flag_comp = 0;
                            last;
                        }
                    }
                } # test one compound word start by $w           
				
				# connect the compound word     
                if ($flag_comp==1)
                {
                    unshift(@string, "$w");
                    my $comp = join('_', @string);
					$new_def .= "$comp ";
					if (defined $debugfile)
					{ 
                    	print DEBUG "compounds: $comp\n";
					}
                    my $skip = @string-1;
                    $i = $i + $skip;
                    last;
                }
            } # test all the compound word start by $w  

            # print out the $w if it doesn't match any compound words
            if (($flag_print_w==0) and ($flag_comp==0))
            {
                $new_def .= "$w ";
                $flag_print_w = 1;
            }

        } # end of defined compound word start by $w

        if(!defined $ref_complist->{$w})
        {
            $new_def .= "$w ";
        }
	} # end of one definition 
	
	return $new_def;
	
}


# Subroutine to normalize a vector.
sub norm
{
    my $vec = shift;
    my $out = {};
    my $lent = 0;
    my $ind = 0;

    return {} if(!defined $vec);
    foreach $ind (keys %{$vec})
    {
	$lent += (($vec->{$ind}) * ($vec->{$ind}));
    }
    $lent = sqrt($lent);
    if($lent)
    {
	foreach $ind (keys %{$vec})
	{
	    $out->{$ind} = $vec->{$ind}/$lent;
	}    
    }

    return $out;
}


# Subroutine to find the dot-product of two vectors.
sub _inner
{
    my $a = shift;
    my $b = shift;
    my $ind;
    my $dotProduct = 0;

    return 0 if(!defined $a || !defined $b);
    foreach $ind (keys %{$a})
    {
	$dotProduct += $a->{$ind} * $b->{$ind} if(defined $a->{$ind} && defined $b->{$ind});
    }

    return $dotProduct;
}

1;

__END__