q&fXddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z m Z m Z mZmZmZddlmZmZddlmZmZmZej0ZdadadadadZdZd Z d Z!Gd d Z"d Z#dZ$dZ%GddZ&y)N)_)binhex)errorpycompatrevlogsshpeerutilwireprotov1peer)hashutilprocutil) constants contentstore metadatastorecttj|j}tj j ||dd|dd|S)Nrr sha1digestospathjoin)reponamefileidpathhashs F/usr/lib/python3/dist-packages/hgext/remotefilelog/fileserverclient.py getcachekeyr/sC8==&--/0H 77<<(2A, b AActtj|j}tj j ||SNr)rrrs r getlocalkeyr#4s38==&--/0H 77<<" %%r cBGfdd|j|_y)Nc|eZdZejdZejdZdZfdZfdZ xZ S)!peersetup..remotefilepeercl|jdstjdfd}d|fS)N x_rfl_getfiles5configured remotefile server does not support getfilecv|jdd\}}t|rtj||S)Nr)splitintr LookupError)dcodedatarnodes rdecodez?peersetup..remotefilepeer.x_rfl_getfile..decodeBs8WWUA. dt9++D$== r sfilesnodecapablerAbort)selfrr1r2s `` r x_rfl_getfilez/peersetup..remotefilepeer.x_rfl_getfile;s;<< 01kkL  "D169 9r c`|jdstjdd}d|i|fS)Nsx_rfl_getflogheadss:configured remotefile server does not support getflogheadsc,|r|jdSgS)N )r+)r.s rr2zDpeersetup..remotefilepeer.x_rfl_getflogheads..decodeRs)*qwwu~22r spathr4)r7rr2s rx_rfl_getflogheadsz4peersetup..remotefilepeer.x_rfl_getflogheadsJs;<< 56kk,  3T?F* *r c|dk7rytj|jvryt|dsytj|j j vry|jd}|r|g}ng}|jtj|j jr;dj|j j}d|z}|j||j jr;dj|j j}d|z}|j|dj||d<y)Ns getbundle _localrepos bundlecapsr*sincludepattern=sexcludepattern=,) rNETWORK_CAP_LEGACY_SSH_GETFILES capabilitieshasattrSHALLOWREPO_REQUIREMENTr> requirementsgetappendBUNDLE2_CAPABLITYincludepatternrexcludepattern)r7commandopts bundlecapspatterns includecap excludecaps r_updatecallstreamoptsz7peersetup..remotefilepeer._updatecallstreamoptsWs,&99((*+4.11334-0J(\     i99 :-- ::doo&D&DE/(: !!*--- ::doo&D&DE/(: !!*-"&))J"7D r cJ|j||t| ||fi|Sr")rPsuper _sendrequest)r7rJargsrK __class__remotefilepeers rrSz.peersetup..remotefilepeer._sendrequests4  & &w 5;!% r ct|}t|ds%|j|tj|t||fi|S)NrS)rRrBrPr byteskwargs _callstream)r7rJrK supertyperUrVs rrYz-peersetup..remotefilepeer._callstreamsKnd3I9n5**7H4H4H4NO:7KdK Kr ) __name__ __module__ __qualname__r batchabler8r<rPrSrY __classcell__)rUrVs@rrVr&:sL  " " : # :  " " + # +& 8P   L Lr rV)rU)uipeerrVs @r peersetuprb9s!OLOLb$DNr c0eZdZdZdZdZdZddZdZy) cacheconnectionzThe connection for communicating with the remote cache. Performs gets and sets by communicating with an external process that has the cache-specific implementation. cJdx|_x|_|_d|_d|_y)NF)pipeopipeipipee subprocess connectedr7s r__init__zcacheconnection.__init__s%/33 3TZ$*r c|jrtjtdt j |\|_|_|_|_d|_ y)Nscache connection already openT) rfrr6rr popen4rgrhrirj)r7 cachecommands rconnectzcacheconnection.connectsI ::++a @AB B>Foo ? ; DJ DOr c~d}|jr |jjd||jd|_||jd|_||j d|_ |j jd|_d|_y#t$rYwxYw#t$rY)wxYw)NcD |jy#t$rYywxYwr")close Exception)pipes rtryclosez'cacheconnection.close..trycloses"    s  sexit F)rjrgwritertrfrhri communicate)r7rvs rrszcacheconnection.closes  >>    + TZZ DJ TZZ DJ TZZ DJ ++-#DO!    s#B!8B0! B-,B-0 B<;B<c|jr: |jj||r|jjyyy#t$r|j YywxYwr")rjrgrwflushIOErrorrs)r7requestrzs rr|zcacheconnection.requestsW >>    )JJ$$&     s7AA$#A$c|jsy |jjdd}|s|j|S#t$r|jYSwxYw)N)rjrfreadlinersr{)r7results r receivelinezcacheconnection.receivelines\~~ ZZ((*3B/F    JJL  s/AAANT) r[r\r]__doc__rlrprsr|rr rrdrds   8 r rdc |j5}g}|D],}|j|jd|||ddd.t|D]Y\} }|| j } d|| <||} |dd} |t j dt| | fz| | |[ dddy#1swYyxYw)Nr(r3s%d %s)commandexecutorrF callcommand enumeraterioBytesIOlen) remotereceivemissing progresstickmissedidmap batchsizeefuturesmirfile_r1s r_getfilesbatchrs    !Q A NN $a1ST7&K  f% DAq !!#AGAJ!HEST7D 2::i3q61+&=>t L N  s BB66B?c|jdd}|j}|j}|t|kr|} t t|| |z} | }|| | D]$} | dd} || } d| | fz}|j |&|j || | D]} | dd} || } ||| | ||t|kr|j d|j y)Nx_rfl_getfilesrr%s%s r;)rY_pipeo_pipeirminrwrz)rrrrrsteprrfrgstartend missingid versionidr sshrequests r_getfiles_optimisticrs () A MME MME c&k/#f+ut|, c* $I!#$I#D"i%66J KK #  $   c* I!#$I#D 5$ 2 N   c&k/* KK KKMr cj |jd|j |j} fd}tj|}d|_|j D]} | dd} | } ||| | ||j jd jy)NrcxD]$}|dd}|}d||fz}j|&jy)Nrr)rwrz)rrrrrrrfs rwriterz"_getfiles_threaded..writersM $I!#$I#D"i%66J KK #  $  r )targetTrr;) rYrr threadingThreaddaemonrrrwrz) rrrrrrrgr writerthreadrrrrfs `` @r_getfiles_threadedrs () MME MME##62LL cdO YudI.   KK KKMr cPeZdZdZdZdZdZdZdZdZ dZ d d Z d Z d Z y )fileserverclientz:A client for requesting files from the remote file server.c@|j}||_||_|jdd|_|jr$t j |j|_|j dd|_|j dd|_t|_ y)N remotefilelogs cacheprocessscacheprocess.includepathsdebug) r`repoconfig cacheprocessr expandpath configboolcacheprocesspasspath debugoutputrd remotecache)r7rr`s rrlzfileserverclient.__init__4s WW II&6H    $0A0A BD %'MM 9% !==)98D*,r c<||_||_||_||_yr") datastore historystore writedata writehistory)r7rrrrs rsetstorezfileserverclient.setstoreFs "("(r ct|jjj|jjSr")rconnectionpoolrE fallbackpathrks r_connectzfileserverclient._connectLs'yy''++DII,B,BCCr c  |jjs|j|j}|j}|j}t |}d|z}i}|j }|D]3\} } t|| | } |jr|| dzz }|| dzz }| || <5|j||jjtd|} | jdg} |j}|sMt| }|D]}||vs| j!||jj#tdnX|dk(rnR|j%d r/|j'd }| j)t+|d | j!|t,t | z a|t | z }| j|||jj/d d ||||t1j2d } | r|jj4}d|j_ |j75}|j8}|j;t<j>rtA|tBstEjFd|jjId d}|jjKd d}|dk(rtL}ntN}|||jP| j(| ||n|j;dr|j;drd}nd}|jjId d|}|jjSdt | ztU||jP| j(| ||ntEjFdddd|jj/dd| jV|z ||z  ||j_d t | dj[| fz}|j|| j]|j_|jj`t1j2|y#1swYxYw#tX$r1|jj/dd| jV|z ||z wxYw#||j_wxYw#t1j2|wxYw)!zTakes a list of filename/node pairs and fetches them from the server. Files are stored in the local cache. A list of nodes that the server couldn't find is returned. If the connection fails, an exception is raised. sget %d r*r;s downloading)totalrs@warning: cache connection closed early - falling back to server 0s_hits__rrs"remote cache hit rate is %r of %r )hitrFs"remotefilelog requires ssh serverss getfilessteps getfilestypesthreadedr(sbatchd s batchsizes1requesting %d files from remotefilelog server... s>configured remotefilelog server does not support remotefilelogNsremotefilefetchlogsSuccess ) fetched_filestotal_to_fetchsFail s set %d %s )1rrjrprrrnamerrr|r` makeprogressrupdatersetrFwarn startswithr+ incrementr, fetchmisseslogrumaskverboserrar5rr@ isinstance _sshv1peerrr6 configintrrrrdebugrposrtrcompletemarkrepor)r7fileidscacherrrr|rrrrfullidprogressrr missedsetparts fromcacheoldumaskrconnrr getfilestype _getfiles batchdefaultrs rr|zfileserverclient.requestOs )) LLN  NN yyG &99 !HD" 44F((4%<' v~ %G E&M  !  g77''.(9'G))+IK !&1I$ 1 i01 6 D ##I.!-""3uQx=1 MM) $-2 s6{" CK'  /   2    88E?W ''//"'E.3D!%!>>%EE$.fj#A&+kk$I'"!"$(77#4#4 0/$D,077>> 0/,L ,{:,> ,@ % & $ 3 3 ( 2 2 & % $ $^^,<=%~~h7/2 /1 (,(9(9 0, )I!GGMM!=?B6{!K+ & $ 3 3 ( 2 2 & % ) #(++!C#a3jGGKK-$&.llY&>',y'8  '.DGGO*c&k5::f;M-NN g&       tyy~~ . HHX ]33v!GGKK-!&.llY&>',y'8   '.DGGO HHX sJ *S 8Q:E"Q.*7Q:"A6S .Q73Q:::R44R77S  S S$c|jdd}|s(tjtdtdt |}|j |}t ||k7r6tjtdtdt ||fz|jj|t|tj|y)Nr~s error downloading file contents:sconnection closed earlysonly received %s of %s bytes) rr ResponseErrorrr,readrraddremotefilelognoderzlib decompress)r7rufilenamer1linesizer0s rrzfileserverclient.receivemissings}}s#%%56,- 4yyy t9 %%5612c$i5FF  ++ c$i!6 r c|jrAd|j|jjfz}|jj |yGdd}||_y)Ns%s %sc&eZdZdZdZddZdZy)-fileserverclient.connect..simplecachec g|_d|_y)NT) missingidsrjrks rrlz6fileserverclient.connect..simplecache.__init__ s&(DO%)DNr cyr"rrks rrsz3fileserverclient.connect..simplecache.close sr c|jd}|ddk7ry|dd|_|jjdy)Nr;rsgetrr~r)r+rrF)r7valuerzliness rr|z5fileserverclient.connect..simplecache.requests>!KK.EQx6)&+AbkDOOO**40r cjt|jdkDr|jjdSy)Nr)rrpoprks rrz9fileserverclient.connect..simplecache.receivelines+4??+a/#22155r Nr)r[r\r]rlrsr|rrr r simplecachers*1 r r)rr_pathrrp)r7cmdrs rrpzfileserverclient.connectsT   d//1E1EFFC    $ $S )   ( +}D r c trdttttttz ttz dztfz}|j r|j j||j jd|jddttttdz|jjr|jjyy)NsN%d files fetched over %d fetches - (%d misses, %0.2f%% hit ratio) over %0.2fs gY@sremotefilelog.prefetch%s%%i)remotefilelogfetchedremotefilelogfetchesremotefilelogfetchmissesremotefilelogfetchtime) fetchesfetchedrfloat fetchcostrr`rrreplacerrjrs)r7msgs rrszfileserverclient.closes Bg +,uW~=E  C S! GGKK) D%(%,%,)4'04'7      % %    " " $ &r c|j}g}|D]F\}}|dk(st|dk(s|j|s+|j|t |fH|j } |j } |r8tj|j} tj|j} t} |r | j| j||r | j| j|t| D cgc]!\} }||jj k(s d#c}} } | rP| Dcgc]#\}}||jj k7s ||f%} }}|j"j%d| zd| r+t&dz at&dk\r^t&d krUt&dk(r<|j"j)d d }|r|j"j+|d z|j-t/| Dcgc]\}}|t1|f} }}t2t| z at5j4}|j7| } | r*t9j:t=d t| zt>t5j4|z z a|jA|yycc}} wcc}}wcc}}w)z.downloads the given file versions to the caches.hgtags*NsDremotefilelog not fetching %d null revs - this is likely hiding bugssremotefilelog-ext)rrrs fetchwarningr;sunable to download %d files)!rr shallowmatchrFrrrrunioncontentstoreshareddatastoresrunionmetadatastoresharedhistorystoresrr getmissingnullidr` develwarnr rr logstacktracesortedrr timer|rr6rr  _lfsprefetch)r7rforce fetchdata fetchhistoryr idstocheckrrrrrunusednullidsf fetchwarningrs rprefetchzfileserverclient.prefetch8syy  /HD"  "r7b=((.   tSWo . /NN (( $668M8MNI(;;))LU    i22:> ?    l55jA B$. Ijfb" 8H8H2HT I  %/!Ar29I9I3IBJ GG  57>?,    qLG"}2b=#'77>>(/$L$ \E%9:""$:@:LMhdB4R/MJM s: &GIIKEj1Jkk45JG u, ,I   g &/  J0Ns!J3 &J3 9!J9J9J?c tr t|jjdsytjj |jsyg}|jjj |D]\}}t|}|jj|}|j|tjzsO|j|}tjj|}|j} j!| r|j#|t%|dkDrG|jjj&j)| t+ fd|DsJyy)Nlfslocalblobstorerc3\K|]#}j|j%ywr")hasoid).0pstores r z0fileserverclient._lfsprefetch..s >$'Dzz$&"9"99||D)OO//5eegyy~OOA& ' x=1  IINN - - 7 7% H<8<< << r c ddl}|jjdddjd|j Dy)Nrrs"excess remotefilelog fetching: %s r c3FK|]}tj|ywr")rsysbytes)r.ss rr1z1fileserverclient.logstacktrace..sLaX&&q)Ls!) tracebackr`rr format_stack)r7rEs rrzfileserverclient.logstacktraces6   3 HHL93I3I3KL L r N)FTF)r[r\r]rrlrrr|rrprsr(rrrr rrr1sCD-$) D[z &->%6BGI'V=( r r)'rrrrrmercurial.i18nrmercurial.noderr mercurialrrrr r r mercurial.utilsr r rrr sshv1peerrr r r rr2rr#rbrdrrrrrr rrMs  #         B & R$jBBJ>@>m m r