
    9                         S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
r
SSKJr  SSKJr  SS	KJr  SS
KJr  SSKJr  Sr " S S\5      r\R,                  S 5       r " S S\5      rg)z(Wrapper for use in daisy-chained copies.    )absolute_import)division)print_function)unicode_literalsN)BadRequestException)CloudApi)	constants)parallelism_framework_util)CryptoKeyWrapperFromKeyi  @c                   (    \ rS rSrSrSS jrS rSrg)BufferWrapper)   z<Wraps the download file pointer to use our in-memory buffer.c                 b    Xl         [        US5      (       a  UR                  U l        gX l        g)zProvides a buffered write interface for a file download.

Args:
  daisy_chain_wrapper: DaisyChainWrapper instance to use for buffer and
                       locking.
modeN)daisy_chain_wrapperhasattrr   )selfr   r   s      ,platform/gsutil/gslib/daisy_chain_wrapper.py__init__BufferWrapper.__init__,   s*      3"F++%**dii    c                     U R                   R                     U R                   R                  U R                   R                  :  a
   SSS5        O! SSS5        [        R
                  " S5        Mp  [        U5      nU(       ad  U R                   R                     U R                   R                  R                  U5        U R                   =R                  U-  sl        SSS5        gg! , (       d  f       N= f! , (       d  f       g= f)z>Waits for space in the buffer, then writes data to the buffer.Nr   )	r   lockbytes_bufferedmax_buffer_sizetimesleeplenbufferappend)r   datadata_lens      r   writeBufferWrapper.write9   s    
##(($$33&&667
 )(7 )
 jjm  4yH##((  ''..t4  //8;/ )(  )( )(s   0C)AC:)
C7:
D)r   r   N)b)__name__
__module____qualname____firstlineno____doc__r   r#   __static_attributes__ r   r   r   r   )   s    D<r   r   c              #   b   #    U R                  US9nUv   U(       a  U R                  5         g g 7f)N)timeout)acquirerelease)r   r.   results      r   AcquireLockWithTimeoutr2   I   s+     <<<(&,LLN s   -/c                   j    \ rS rSrSrSS\S4S jrSS jrSS jrS r	\
R                  4S	 jrS
 rSrg)DaisyChainWrapperQ   a  Wrapper class for daisy-chaining a cloud download to an upload.

This class instantiates a BufferWrapper object to buffer the download into
memory, consuming a maximum of max_buffer_size. It implements intelligent
behavior around read and seek that allow for all of the operations necessary
to copy a file.

This class is coupled with the XML and JSON implementations in that it
expects that small buffers (maximum of constants.TRANSFER_BUFFER_SIZE) in
size will be used.
FNc                 h   SU l         [        R                  " 5       U l        SU l        SU l        X`l        SU l        SU l        [        R                  " 5       U l        [        R                  " 5       U l        X l        Xl        X@l        [!        U5      U l        X0l        SU l        SU l        XPl        [,        R.                  " 5       U l        [,        R.                  " 5       U l        U R5                  U R*                  S9  U R0                  R7                  S5      (       d  [9        S5      eSU l        g)a   Initializes the daisy chain wrapper.

Args:
  src_url: Source CloudUrl to copy from.
  src_obj_size: Size of source object.
  gsutil_api: gsutil Cloud API to use for the copy.
  compressed_encoding: If true, source object has content-encoding: gzip.
  progress_callback: Optional callback function for progress notifications
      for the download thread. Receives calls with arguments
      (bytes_transferred, total_size).
  download_chunk_size: Integer number of bytes to download per
      GetObjectMedia request. This is the upper bound of bytes that may be
      unnecessarily downloaded if there is a break in the resumable upload.
  decryption_key: Base64-encoded decryption key for the source object,
      if any.
Raises:
  Exception: if the download thread doesn't start within 60 seconds
r   i   N)progress_callback<   z1Could not start download thread after 60 seconds.r   )positioncollectionsdequer   r   r   _download_chunk_sizelast_position	last_datar
   
CreateLockr   download_exception_locksrc_obj_sizesrc_urlcompressed_encodingr   decryption_tuple
gsutil_apidownload_exceptiondownload_threadr7   	threadingEventdownload_startedstop_downloadStartDownloadThreadwait	Exception_unused_data_from_previous_read)r   rB   rA   rE   rC   r7   download_chunk_sizedecryption_keys           r   r   DaisyChainWrapper.__init__^   s   6 DM##%DKD&D 3 DDN +557DI $>#H#H#JD $L23NCD !O #DD.%OO-D"*Dt/E/EF  %%b))IJJ+.D(r   c                 ~   ^  U 4S jn[         R                  " UUU4S9T l        T R                  R                  5         g)zCStarts the download thread for the source object (from start_byte).c                   > TR                   R                  5          U TR                  -   TR                  :  Ga,  TR                  R                  TR                  R                  TR                  R                  [        T5      TR                  U U TR                  -   S-
  TR                  R                  TR                  [        R                  R                  TR                  R                  UTR                   S9  TR"                  R%                  5       (       a  TR"                  R'                  5         gU TR                  -  n U TR                  -   TR                  :  a  GM,  TR                  R                  TR                  R                  TR                  R                  [        T5      TR                  U TR                  R                  TR                  [        R                  R                  TR                  R                  UTR                   S9  g! [(         a/  nTR*                     UTl        e ! , (       d  f        SnAg= fSnAff = f)a  Downloads the source object in chunks.

This function checks the stop_download event and exits early if it is set.
It should be set when there is an error during the daisy-chain upload,
then this function can be called again with the upload's current position
as start_byte.

Args:
  start_byte: Byte from which to begin the download.
  progress_callback: Optional callback function for progress
      notifications. Receives calls with arguments
      (bytes_transferred, total_size).
   )	rC   
start_byteend_byte
generationobject_sizedownload_strategyproviderr7   rD   N)rC   rV   rX   rY   rZ   r[   r7   rD   )rJ   setr<   rA   rE   GetObjectMediarB   bucket_nameobject_namer   rC   rX   r   DownloadStrategyONE_SHOTschemerD   rK   is_setclearrN   r@   rF   )rV   r7   er   s      r   PerformDownload>DaisyChainWrapper.StartDownloadThread.<locals>.PerformDownload   s   " !%4444t7H7HH
//
(
(ll&&ll&&D!"&":":#!D$=$==A00++ ( 9 9 B B||** 1#44 ) 6 &&(($$&
11
1*% 4444t7H7HH& 	&&LL$$LL$$$ $ 8 8!||..))&77@@\\((/!22 	' 	4  ))$%$
!
 *))s=   DH 8,H 'B2H 
I$I0	H99
I	III)targetargsN)rH   ThreadrG   start)r   rV   r7   rf   s   `   r   rL   %DaisyChainWrapper.StartDownloadThread   sA    7t %++?2<2C2EFD 	 r   c                    U R                   U R                  :X  d  US:X  a  gUb  U[        R                  :  a"  [	        SU< S[        R                  < S35      eU R
                  (       a$  U R
                  SU nU R
                  US U l        OSn U R                     U R                  (       a
   SSS5        O[        U R                  S5      (       a  U R                  (       a  U R                  eO*U R                  R                  5       (       d  [        S	5      eSSS5        [        R                  " S5        M  U R                     U(       d(  U R                  R!                  5       nUSU nX1S U l        U R                   U l        X l        ['        U5      nU =R                   U-  sl         U =R(                  U-  sl        SSS5        WU:  a  [	        S
U< SU< S35      eU$ ! , (       d  f       N= f! , (       d  f       N;= f)z9Exposes a stream from the in-memory buffer to the upload.r    NzInvalid HTTP read size z+ during daisy chain operation, expected <= .r      zDownload thread died suddenly.z<Invalid read during daisy chain operation, got data of size z, expected size )r9   rA   r	   TRANSFER_BUFFER_SIZEr   rO   r   r   r2   r@   rF   rG   is_aliverN   r   r   popleftr=   r>   r   r   )r   amt
valid_datar!   r"   s        r   readDaisyChainWrapper.read   s   }})))SAX 
{cI:::"I$B$BDE E ++77=j

.
.st
4 * jYY[[ Y $D$@$@"EE&& +++ '
 ''0022>??  	

1  
 {{""$$3Z
/3Dz,==d!nZh
mmxm
X%' 
( #~%-s45 5 K Y 
s    G*;A#G*
A<G;*
G8;
H	c                 h    U R                      U R                  sS S S 5        $ ! , (       d  f       g = fN)r   r9   r   s    r   tellDaisyChainWrapper.tell"  s    	]] 
s   #
1c                    SnU[         R                  :X  aT  U(       a  [        SU-  5      eU R                     U R                  U l        SU l        U R                  U l        SSS5        gU[         R                  :X  Ga  U R                     XR                  :X  a  OXR
                  :X  ap  U R
                  U l        U R                  (       aM  U R                  R                  U R                  5        U =R                  [        U R                  5      -  sl        OSnSSS5        U(       Ga^  U R                  R                  5         U R                  R!                  5       (       a  U R                     U R                  (       aI  U =R                  [        U R                  R#                  5       5      -  sl        U R                  (       a  MI  SSS5        [$        R&                  " S5        U R                  R!                  5       (       a  M  U R                     Xl        [(        R*                  " 5       U l	        SU l        SU l        SU l        U R                  R-                  5         SSS5        U R/                  UU R0                  S9  gg[        SU-  5      e! , (       d  f       g= f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       Nl= f)z)Sets current read position of the stream.Fz_Invalid seek during daisy chain operation. Non-zero offset %s from os.SEEK_END is not supportedNTr   )rV   r7   z:Daisy-chain download wrapper does not support seek mode %s)osSEEK_ENDIOErrorr   r9   r=   r>   rA   SEEK_SETr   
appendleftr   r   rK   r\   rG   rr   rs   r   r   r:   r;   rd   rL   r7   )r   offsetwhencerestart_downloads       r   seekDaisyChainWrapper.seek&  s*   	0289: 	: 99!]]))	 9
 
2;;	99]]"
))),,$-^^ KK""4>>23t~~#66 "
# & 
  ""++--yy%%!!S)<)<)>%??! %%%  **Q-	 ""++-- YY -#))+$+ !$
 $
$.



"
"
$  	  F373I3I 	! 	K% 
*  #%+, - -] 9 92 y
 Ys2   *JBJ,3AJ>AK
J),
J;>
K
Kc                     g)NTr,   rz   s    r   seekableDaisyChainWrapper.seekable_  s    r   )r<   rO   r   r   rC   rD   rF   r@   rJ   rG   rE   r>   r=   r   r   r9   r7   rA   rB   rK   )r   Nry   )r&   r'   r(   r)   r*   _DEFAULT_DOWNLOAD_CHUNK_SIZEr   rL   rv   r{   r~   r   r   r   r+   r,   r   r   r4   r4   Q   sB    
  $)!%#?"F/P@!D8t !# 7-rr   r4   )r*   
__future__r   r   r   r   r:   
contextlibr~   rH   r   gslib.cloud_apir   r   gslib.utilsr	   r
   gslib.utils.encryption_helperr   r   objectr   contextmanagerr2   r4   r,   r   r   <module>r      sq    / &  % '   	   / $ ! 2 A  1 <F <@  O Or   