9ekddlmZddlmZddlmZddlmZmZm Z ddl m Z m ZddlmZmZ ddlZgd Ze d ZGd d e ZGd de ZGdde ZGddeZGddeZGddeZGddeZGddeZGddeZGddeZeZ eZ!exZ"Z#eZ$eZ%eZ&eZ'eZ(eZ)eZ*y#e$rdZYwxYw)) annotations) defaultdict) zip_longest)AnySequenceTypeVar)BaseBaseSimilarity)SimFuncTestFuncN)HammingMLIPNS LevenshteinDamerauLevenshteinJaro JaroWinklerStrCmp95NeedlemanWunschGotoh SmithWatermanhammingmlipns levenshteindamerau_levenshteinjaro jaro_winklerstrcmp95needleman_wunschgotohsmith_watermanTc<eZdZdZ d ddZddZy)rz Compute the Hamming distance between the two or more sequences. The Hamming distance is the number of differing items in ordered sequences. https://en.wikipedia.org/wiki/Hamming_distance NcX||_|xs |j|_||_||_yN)qval_ident test_functruncateexternal)selfr&r(r)r*s D/usr/lib/python3/dist-packages/textdistance/algorithms/edit_based.py__init__zHamming.__init__*s) "1dkk    cj|}j|}|t|tsJ|Sjrt nt }tfd||DS)Nc3>K|]}j| ywr%)r().0esr+s r, z#Hamming.__call__..?sFr~t~~r**Fs)_get_sequences quick_answer isinstanceintr)ziprsum)r+ sequencesresult_zips` r,__call__zHamming.__call__6se'D''3 """I.  fc* **MmmsFT95EFFFr.)r NFT) r&r7r(TestFunc | Noner)boolr*r?returnNoner:Sequence[object]r@r7)__name__ __module__ __qualname____doc__r-r=r.r,rr"sP%) ! !# ! !  !  ! Gr.rcFeZdZdZ d ddZd dZd dZd dZy) ra Compute the absolute Levenshtein distance between the two sequences. The Levenshtein distance is the minimum number of edit operations necessary for transforming one sequence into the other. The edit operations allowed are: * deletion: ABC -> BC, AC, AB * insertion: ABC -> ABCD, EABC, AEBC.. * substitution: ABC -> ABE, ADC, FBC.. https://en.wikipedia.org/wiki/Levenshtein_distance TODO: https://gist.github.com/kylebgorman/1081951/9b38b7743a3cb5167ab2c6608ac8eea7fc629dca NcJ||_|xs |j|_||_yr%)r&r'r(r*)r+r&r(r*s r,r-zLevenshtein.__init__Ps"  "1dkk  r.c |r|st|t|zS|j|d|dr||dd|ddSt||dd||||dd}||dd|dd}t||dzS)Nr )lenr(min)r+s1s2dss r, _recursivezLevenshtein._recursiveZsr7SW$ $ >>"R&"R& )3BCR) )  CR"  RW   CR"Sb' "1ay1}r.ct|dz}t|dz}d}trtj|}n t|}td|D]o}||gdg|dz zz}}td|D]O}||dz} ||dz dz} |j ||dz ||dz } ||dz | z} t | | | ||<Qq|dS)zp source: https://github.com/jamesturk/jellyfish/blob/master/jellyfish/_jellyfish.py#L18 r NrrL)rMnumpyarangeranger(rN) r+rOrProwscolsprevcurrcdeletion insertiondistedits r,_cycledzLevenshtein._cycledks 2w{2w{ ,,t$C+Cq$ 8AaSA3$(#33#D1d^ 87Q;AJN ~~bQiAE;AE{$h/T8Y7A  8 82wr.c|j||\}}|j||}|t|tsJ|S|j ||Sr%)r4r5r6r7rbr+rOrPr;s r,r=zLevenshtein.__call__sV$$R,B""2r*  fc* **M||B##r.)r NT)r&r7r(r>r*r?r@rArO Sequence[T]rPrfr@r7)rDrErFrGr-rSrbr=rHr.r,rrBsL %) !!#! !  !"0$r.rcTeZdZdZ d d dZd dZd dZd dZd dZy) ra Compute the absolute Damerau-Levenshtein distance between the two sequences. The Damerau-Levenshtein distance is the minimum number of edit operations necessary for transforming one sequence into the other. The edit operations allowed are: * deletion: ABC -> BC, AC, AB * insertion: ABC -> ABCD, EABC, AEBC.. * substitution: ABC -> ABE, ADC, FBC.. * transposition: ABC -> ACB, BAC If `restricted=False`, it will calculate unrestricted distance, where the same character can be touched more than once. So the distance between BA and ACB is 2: BA -> AB -> ACB. https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance NcX||_|xs |j|_||_||_yr%)r&r'r(r* restricted)r+r&r(r*ris r,r-zDamerauLevenshtein.__init__s) "1dkk  $r.c tjt|dzt|dzgt}t dt|dzD] }|dz||d<t dt|dzD] }|dz|d|<t |D]\}}t |D]\}}t|j || }t||dz |dz|||dz dz||dz |dz |z|||<|r|sa|j |||dz szt|||||dz |dz |z|||<|t|dz t|dz S)Nr dtyperL)rUzerosrMr7rW enumerater(rN r+rOrPrQijcs1cs2costs r,_numpyzDamerauLevenshtein._numpys KKR1c"gk2# >r3r7Q;' A1uAaDH r3r7Q;' A1uAbE!H  m FAs#B- 3t~~c3778a!eHQK!OaDQK!Oa!eHQUOd*!Q~~c2a!e95aDGa!eHQUOd*!Q  *R1~c"gk**r.c ^i}i}t|}t|}||z}||d<tt|dzD]}|||df<|||df<tt|dzD]} ||d| f<| |d| f<t|dD]\}} d} t|dD]\} } |j| d} | }|j | | rd}| } nd}t ||dz | dz f|z||| dz fdz||dz | fdz|| dz |dz f|| z zdz | |z z||| f<||| <|||fS)zKhttps://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance )rLrLr rLrstart)rMrWrogetr(rN)r+rOrPrQdalen1len2maxdistrqrrrsdbrti1j1rus r,_pure_python_unrestrictedz,DamerauLevenshtein._pure_python_unrestricteds)+2w2w+& s2w{# AAaeHAadG s2w{# AAb!eHAadG  !, FAsB#Ba0 3VVC^>>#s+DBDa!eQUlOd*aQhK!Oa!eQhK!Ob1fb1fn%R014B? !Q$ BsG# &t}r.c i}tdt|dzD] }|dz||df<tdt|dzD] }|dz|d|f<t|D]\}}t|D]\}}t|j || }t ||dz |fdz|||dz fdz||dz |dz f|z|||f<|r|s]|j |||dz sv|j ||dz |st |||f||dz |dz f|z|||f<|t|dz t|dz fS)za https://www.guyrutenberg.com/2008/12/15/damerau-levenshtein-distance-in-python/ rLr rm)rWrMror7r(rNrps r,_pure_python_restrictedz*DamerauLevenshtein._pure_python_restricteds)+r3r7Q;' A1uAaeH r3r7Q;' A1uAb!eH  m FAs#B- 3t~~c3778a!eQhK!OaQhK!Oa!eQUlOd*!Q$~~c2a!e95~~bQi5adGa!eQUlOd*!Q$#  .R1c"gk)**r.c|j||\}}|j||}||S|jr|j||S|j ||Sr%)r4r5rirrrds r,r=zDamerauLevenshtein.__call__sd$$R,B""2r*  M ??//B7 7--b"55r.)r NTT) r&r7r(r>r*r?rir?r@rAre) rDrErFrGr-rvrrr=rHr.r,rrsa&%) % %# % %  %  %+B'R#+J 6r.rcFeZdZdZ d ddZddZd d dZy) ra Computes the Jaro-Winkler measure between two strings. The Jaro-Winkler measure is designed to capture cases where two strings have a low Jaro score, but share a prefix. and thus are likely to match. https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance https://github.com/Yomguithereal/talisman/blob/master/src/metrics/jaro.js https://github.com/Yomguithereal/talisman/blob/master/src/metrics/jaro-winkler.js c<||_||_||_||_yr%)r&long_tolerance winklerizer*)r+rrr&r*s r,r-zJaroWinkler.__init__6s! ,$  r.cyNr rHr+r:s r,maximumzJaroWinkler.maximumBr.c|j||\}}|j||}||St|}t|}|r|syt||}t ||}|dzdz }|dkrd}dg|z} dg|z} d} t |D]Y\} } t d| |z }t| |z|dz }t ||dzD]!}| |r ||| k(sdx| | <| |<| dz } Y[| sydx}}t | D]6\} }|s t ||D]}| |s |dz}n|| |k7s2|dz }8|dz}| |z | |z z}|| |z | z z }|dz}|js|S|dkr|St|d }d} | |kr!|| || k(r| dz } | |kr || || k(r| r|| |zd |z zz }|jr|d kr|S| | dzks d| z|| zkr|S| | z dz ||z| dzz dzz }|d |z |zz }|S) Nrmr rFTffffff??) r4r5rMrNmaxrorWrr)r+rOrP prefix_weightr;s1_lens2_lenmin_len search_ranges1_flagss2_flags common_charsrqs1_chlowhirrk trans_counts1_fweighttmps r,r=zJaroWinkler.__call__Es$$R,B""2r*  MRRVff%66* $)Q. ! L7V#7V# !"  HAua\)*CQ%vz2B3Q' {r!u~044HQK(1+ A%L   K * %GAtq&)A{Ea5BqE>1$K %  &)>><+-==! M S=M O !e1A FA!e1A a-'3<8 8F ""glM 1q5 A $4w{$BMa!#!a%(?!(CD3<3&& r.N)FTr T) rr?rr?r&r7r*r?r@rArB)皙?)rOrfrPrfrfloatr@rrDrErFrGr-rr=rHr.r,rr*sU  % ! ! ! !  !  !Nr.rc6eZdZ d dfd ZxZS)rc,t||d||y)NF)rrr&r*)superr-)r+rr&r* __class__s r,r-z Jaro.__init__s# )  r.)Fr T)rr?r&r7r*r?r@rA)rDrErFr- __classcell__)rs@r,rrs> %           r.rcdeZdZdZ d d dZd dZd dZd dZd dZd dZ d d Z y)ra0 Computes the Needleman-Wunsch measure between two strings. The Needleman-Wunsch generalizes the Levenshtein distance and considers global alignment between two strings. Specifically, it is computed by assigning a score to each alignment between two input strings and choosing the score of the best alignment, that is, the maximal score. An alignment between two strings is a set of correspondences between the characters of between them, allowing for gaps. https://en.wikipedia.org/wiki/Needleman%E2%80%93Wunsch_algorithm Ncr||_||_|r||_||_y|j|_||_yr%)r&gap_costsim_funcr'r*r+rrr&r*s r,r-zNeedlemanWunsch.__init__s9   $DM! !KKDM  r.cPttt| |jzSr%)rmaprMrrs r,minimumzNeedlemanWunsch.minimums CY'((4==88r.c4ttt|Sr%)rrrMrs r,rzNeedlemanWunsch.maximum3sI&''r.c&d|j|zS)z'Get distance between sequences rL) similarityrs r,distancezNeedlemanWunsch.distancesODOOY///r.cz|j|}|j|}|dk(ry|j||z ||z z S)z!Get distance from 0 to 1 r)rrrr+r:rrs r,normalized_distancez#NeedlemanWunsch.normalized_distancesN$,, *$,, * a< y)G3'8IJJr.cz|j|}|j|}|dk(ry|j||z |dzz S)z#Get similarity from 0 to 1 rr rm)rrrrs r,normalized_similarityz%NeedlemanWunsch.normalized_similaritysM$,, *$,, * a<+g5'A+FFr.cts td|j||\}}tjt |dzt |dzft }t t |dzD]}||jz ||df<t t |dzD]}||jz |d|f<t|dD]y\}}t|dD]e\}}||dz |dz f|j||z}||dz |f|jz } |||dz f|jz } t|| | |||f<g{||jddz |jddz fS)Nz2Please, install numpy for Needleman-Wunsch measurer rkr) rU ImportErrorr4rnrMrrWrrorrshape) r+rOrPdist_matrqrrc1c2matchdeleteinserts r,r=zNeedlemanWunsch.__call__sRS S$$R,B ;; Wq[#b'A+ &  s2w{# 2A 4==01HQTN 2s2w{# 2A 4==01HQTN 2r1% A ? ?q)A-x~~a/@1/DDEEr.rrrBrrrHr.r,rrsU   ! ! ! !  !  !(Fr.rcReZdZdZ d ddZd dZd dZd dZy) rzGotoh score Gotoh's algorithm is essentially Needleman-Wunsch with affine gap penalties: https://www.cs.umd.edu/class/spring2003/cmsc838t/papers/gotoh1982.pdf Nc||_||_||_|r||_||_y|j|_||_yr%)r&gap_opengap_extrr'r*)r+rrrr&r*s r,r-zGotoh.__init__9s@    $DM! !KKDM  r.c6ttt| Sr%rrs r,rz Gotoh.minimumJsCY'(((r.c4ttt|Sr%rrs r,rz Gotoh.maximumMrr.c zts td|j||\}}t|}t|}tj|dz|dzft }tj|dz|dzft }tj|dz|dzft }d|d<t d|d<t d|d<t d|dzD]Y}t d||df<|j |j|dz zz ||df<t d||df<|j ||df<[t d|dzD]Y} t d|d| f<t d|d| f<|j |d| f<|j |j| dz zz |d| f<[t|dD]\}} t|dD]\} } |j| | } t||dz | dz f| z||dz | dz f| z||dz | dz f| z||| f<t||dz | f|jz ||dz | f|jz ||| f<t||| dz f|jz ||| dz f|jz ||| f<d|jD\}} t||| f||| f||| fS) Nz'Please, install numpy for Gotoh measurer rkr)rrz-infrxc3&K|] }|dz  yw)r NrH)r1ns r,r3z!Gotoh.__call__..s+!A+s) rUrr4rMrnrrWrrrorrr) r+rOrPlen_s1len_s2d_matp_matq_matrqrrrrsim_vals r,r=zGotoh.__call__PsGH H$$R,B RR VaZ!4EB VaZ!4EB VaZ!4EBd Fmd Fmd q&1*% )A-E!Q$K==.4<<1q5+AAE!Q$K-E!Q$K==.E!Q$K  ) q&1*% BA-E!Q$K-E!Q$K==.E!Q$K==.4<<1q5+AAE!Q$K  B  !, FAs#Ba0 3--S1!!a%Q,''1!a%Q,''1!a%Q,''1ad "!a%(Odmm3!a%(Odll2ad "!QU(Odmm3!QU(Odll2ad   ",u{{+15A;ad U1a4[99r.)r g?Nr T) rr7rrrr r&r7r*r?r@rArBr)rDrErFrGr-rrr=rHr.r,rr2sf  !!! !  !  ! !")(0:r.rcLeZdZUdZdZded<d d dZd dZed dZ ddZ y )rz`strcmp95 similarity http://cpansearch.perl.org/src/SCW/Text-JaroWinkler-0.1/strcmp95.c )$)AE)rI)rO)rU)BV)rr)rr)rr)rr)rr)rr)rY)rr)CG)rF)Wr)rr)XK)SZ)rr)Qr)rr)MN)Lr)rr)PR)rJ)2r)5r)8r)1r)rr)0r)rr)rr)rrztuple[tuple[str, str], ...]sp_mxc ||_||_yr%) long_stringsr*)r+rr*s r,r-zStrCmp95.__init__s(  r.cyrrHrs r,rzStrCmp95.maximumrr.c4dt|cxkxrdkScS)Nr[)ord)chars r, _in_rangezStrCmp95._in_ranges3t9!r!!!!r.c|jj}|jj}|j||}||St|}t|}t t }|j D]\}}d|||f<d|||f<||kDr|} |} n|} |} dg| z} dg| z} td| dzdz } d} |dz }t|D]Y\}}t|| z d}t|| z|}t||dzD]$}| |dk(s |||k(sd| |<d| |<| dz } Y[| dk(rydx}}t|D]9\}}| |s t||D]}| |dk7s |dz}n||k7s5|dz };|dz}d}| | kDrt|D]q}| |dk7r |j||s!t|D]C}| |dk7r |j||s!||||f|vr.||||||fz }d| |<qs|dz | z}||z ||z z}|| |z | z z }|dz }|dkr|St| d }d}t||D]*\}}||k\rn ||k7rn|jrn|dz },|r||d zd |z zz }|js|S| d kr|S| |dzks d| z| |zkr|S|djr|S| |z dz ||z|dzz dzz }|d |z |zz }|S) Nrrrmr rg$@g@rrrr)stripupperr5rMrr7rrrorNrWrr8isdigitr)r+rOrPr;rradjwtrrrminvs1_flags2_flagnum_comyl1rqrlowlimhilimrrrn_transn_siminum_simrrress r,r=zStrCmp95.__call__s XXZ    XXZ   ""2r*  MRRC  jj FBE"b&ME"b&M  F?!LD!LD# $# $1la/!34 qjm FAs\)1-FL(#.E6519- 1:?r!u|!"GAJ!"GAJqLG    a<Gm FAs1:1f% 1:?AA be|1  Q, '>6] 1:?~~be,v AqzQ >>"Q%0 1r!u~U2 eBqE2a5L11F!"GAJ   4-')6!Gf$447W$//# S=M aL B  HCAvcz{{} FA  a#gv. .F   M 19M a!e q7{TAX5M a5==?M{Q6F?QU#:Q#>?3<3&& r.N)FT)rr?r*r?r@rArB)r@r?)rOstrrPrr@r) rDrErFrGr__annotations__r-r staticmethodrr=rHr.r,rrs:*E &!""xr.rcDeZdZdZ d ddZddZd dZy) ra! Compute the Hamming distance between the two or more sequences. The Hamming distance is the number of differing items in ordered sequences. http://www.sial.iias.spb.su/files/386-386-1-PB.pdf https://github.com/Yomguithereal/talisman/blob/master/src/metrics/mlipns.js c<||_||_||_||_yr%)r& threshold maxmismatchesr*)r+rrr&r*s r,r-zMLIPNS.__init__ s!  "*  r.cyrrHrs r,rzMLIPNS.maximum+rr.c^|j|}|j|}||Sd}t|}tt t |}t |rU||jkrF|syd||z |z z |jkry|dz }|dz}|dz}t |r||jkrF|syy)Nrr ) r4r5rrrrMallrr)r+r:r; mismatcheshammaxlens r,r=zMLIPNS.__call__.s'D''3 """I.  M gi#Si())nt/A/A!AFSLF**dnn< !OJ 1HC aKF)nt/A/A!Ar.N)g?rmr T) rrrr7r&r7r*r?r@rArBrrrHr.r,rrsR"& ! ! ! ! !  !r.r)+ __future__r collectionsr itertoolsrtypingrrrbaser _Baser _BaseSimilaritytypesr r rUr__all__r"rrrrrrrrrrrrdameraurrrrr!r rrrHr.r,r0s4"$!))C$  CLGeG@I$%I$XY6Y6xi/iX  ;  TFoTFn3FO3FlN:ON:bQQh,_,^ )m 2 44  v} "$  :  EsC55C?>C?