
    vs                     R   S SK r S SKJr  S SKJrJr  S SKJr  S SKJ	r	  S SKJ
r
  S SKJr  S SKJr  S S	KJr  S S
KJr  S SKJrJr   " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\
5      r " S S\	5      r " S S\	5      rg)    N)six)seekablereadable)IN_MEMORY_UPLOAD_TAG)Task)SubmissionTask)CreateMultipartUploadTask)CompleteMultipartUploadTask)get_callbacks)get_filtered_dict)DeferredOpenFileChunksizeAdjusterc                   0    \ rS rSrSS jrS rS rS rSrg)	AggregatedProgressCallback   c                 *    Xl         X l        SU l        g)a  Aggregates progress updates for every provided progress callback

:type callbacks: A list of functions that accepts bytes_transferred
    as a single argument
:param callbacks: The callbacks to invoke when threshold is reached

:type threshold: int
:param threshold: The progress threshold in which to take the
    aggregated progress and invoke the progress callback with that
    aggregated progress total
r   N)
_callbacks
_threshold_bytes_seen)self	callbacks	thresholds      $lib/third_party/s3transfer/upload.py__init__#AggregatedProgressCallback.__init__   s     $#    c                     U =R                   U-  sl         U R                   U R                  :  a  U R                  5         g g N)r   r   _trigger_callbacks)r   bytes_transferreds     r   __call__#AggregatedProgressCallback.__call__-   s7    --t.##% /r   c                 F    U R                   S:  a  U R                  5         gg)z@Flushes out any progress that has not been sent to its callbacksr   N)r   r   r   s    r   flush AggregatedProgressCallback.flush2   s!    a##%  r   c                 X    U R                    H  nU" U R                  S9  M     SU l        g )N)r    r   )r   r   )r   callbacks     r   r   -AggregatedProgressCallback._trigger_callbacks7   s&    Ht'7'78 (r   )r   r   r   N)i   )	__name__
__module____qualname____firstlineno__r   r!   r%   r   __static_attributes__ r   r   r   r      s     &
&
r   r   c                   F    \ rS rSrSrS rSS jrS rS rS r	S	 r
S
 rSrg)InterruptReader=   a  Wrapper that can interrupt reading using an error

It uses a transfer coordinator to propagate an error if it notices
that a read is being made while the file is being read from.

:type fileobj: file-like obj
:param fileobj: The file-like object to read from

:type transfer_coordinator: s3transfer.futures.TransferCoordinator
:param transfer_coordinator: The transfer coordinator to use if the
    reader needs to be interrupted.
c                     Xl         X l        g r   _fileobj_transfer_coordinator)r   fileobjtransfer_coordinators      r   r   InterruptReader.__init__J   s    %9"r   Nc                     U R                   R                  (       a  U R                   R                  eU R                  R                  U5      $ r   )r6   	exceptionr5   read)r   amounts     r   r<   InterruptReader.readN   s:     %%//,,666}}!!&))r   c                 :    U R                   R                  U5        g r   )r5   seek)r   wheres     r   r@   InterruptReader.seekX   s    5!r   c                 6    U R                   R                  5       $ r   )r5   tellr$   s    r   rD   InterruptReader.tell[   s    }}!!##r   c                 8    U R                   R                  5         g r   )r5   closer$   s    r   rG   InterruptReader.close^   s    r   c                     U $ r   r/   r$   s    r   	__enter__InterruptReader.__enter__a   s    r   c                 $    U R                  5         g r   )rG   )r   argskwargss      r   __exit__InterruptReader.__exit__d   s    

r   r4   r   )r*   r+   r,   r-   __doc__r   r<   r@   rD   rG   rJ   rO   r.   r/   r   r   r1   r1   =   s*    :*"$r   r1   c                   b    \ rS rSrSrSS jr\S 5       rS rS r	S r
S	 rS
 rS rS rS rSrg)UploadInputManagerh   a"  Base manager class for handling various types of files for uploads

This class is typically used for the UploadSubmissionTask class to help
determine the following:

    * How to determine the size of the file
    * How to determine if a multipart upload is required
    * How to retrieve the body for a PutObject
    * How to retrieve the bodies for a set of UploadParts

The answers/implementations differ for the various types of file inputs
that may be accepted. All implementations must subclass and override
public methods from this class.
Nc                 (    Xl         X l        X0l        g r   )_osutilr6   _bandwidth_limiter)r   osutilr8   bandwidth_limiters       r   r   UploadInputManager.__init__w   s    %9""3r   c                     [        S5      e)zDetermines if the source for the upload is compatible with manager

:param upload_source: The source for which the upload will pull data
    from.

:returns: True if the manager can handle the type of source specified
    otherwise returns False.
zmust implement _is_compatible()NotImplementedErrorclsupload_sources     r   is_compatible UploadInputManager.is_compatible|   s     ""CDDr   c                     [        S5      e)a  Whether the body it provides are stored in-memory

:type operation_name: str
:param operation_name: The name of the client operation that the body
    is being used for. Valid operation_names are ``put_object`` and
    ``upload_part``.

:rtype: boolean
:returns: True if the body returned by the manager will be stored in
    memory. False if the manager will not directly store the body in
    memory.
z%must implement store_body_in_memory())NotImplementedr   operation_names     r   stores_body_in_memory(UploadInputManager.stores_body_in_memory   s     DEEr   c                     [        S5      e)zProvides the transfer size of an upload

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request
z&must implement provide_transfer_size()r\   r   transfer_futures     r   provide_transfer_size(UploadInputManager.provide_transfer_size   s     ""JKKr   c                     [        S5      e)a  Determines where a multipart upload is required

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request

:type config: s3transfer.manager.TransferConfig
:param config: The config associated to the transfer manager

:rtype: boolean
:returns: True, if the upload should be multipart based on
    configuartion and size. False, otherwise.
z*must implement requires_multipart_upload()r\   r   rk   configs      r   requires_multipart_upload,UploadInputManager.requires_multipart_upload   s     ""NOOr   c                     [        S5      e)a  Returns the body to use for PutObject

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request

:type config: s3transfer.manager.TransferConfig
:param config: The config associated to the transfer manager

:rtype: s3transfer.utils.ReadFileChunk
:returns: A ReadFileChunk including all progress callbacks
    associated with the transfer future.
z$must implement get_put_object_body()r\   rj   s     r   get_put_object_body&UploadInputManager.get_put_object_body   s     ""HIIr   c                     [        S5      e)a  Yields the part number and body to use for each UploadPart

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request

:type chunksize: int
:param chunksize: The chunksize to use for this upload.

:rtype: int, s3transfer.utils.ReadFileChunk
:returns: Yields the part number and the ReadFileChunk including all
    progress callbacks associated with the transfer future for that
    specific yielded part.
z)must implement yield_upload_part_bodies()r\   )r   rk   	chunksizes      r   yield_upload_part_bodies+UploadInputManager.yield_upload_part_bodies   s     ""MNNr   c                     [        XR                  5      nU R                  (       a$  U R                  R                  XR                  SS9nU$ )NF)enabled)r1   r6   rW   get_bandwith_limited_stream)r   r7   s     r   _wrap_fileobj UploadInputManager._wrap_fileobj   sH    !'+E+EF""--II33U J DGr   c                 D    [        US5      nU(       a  [        U5      /$ / $ )Nprogress)r   r   )r   rk   r   s      r   _get_progress_callbacks*UploadInputManager._get_progress_callbacks   s&    !/:>	 .y9::	r   c                 D    U Vs/ s H  o"R                   PM     sn$ s  snf r   )r%   )r   aggregated_progress_callbacksr(   s      r   _get_close_callbacks'UploadInputManager._get_close_callbacks   s    /LM/L8/LMMMs   )rW   rV   r6   r   )r*   r+   r,   r-   rQ   r   classmethodra   rg   rl   rq   rt   rx   r}   r   r   r.   r/   r   r   rS   rS   h   sP    4
 	E 	EFLPJO Nr   rS   c                   ^    \ rS rSrSr\S 5       rS rS rS r	S r
S rS	 rS
 rS rS rSrg)UploadFilenameInputManager   zUpload utility for filenamesc                 6    [        U[        R                  5      $ r   )
isinstancer   string_typesr^   s     r   ra   (UploadFilenameInputManager.is_compatible   s    -)9)9::r   c                     g)NFr/   re   s     r   rg   0UploadFilenameInputManager.stores_body_in_memory   s    r   c                     UR                   R                  U R                  R                  UR                   R                  R
                  5      5        g r   )metarl   rV   get_file_size	call_argsr7   rj   s     r   rl   0UploadFilenameInputManager.provide_transfer_size   s=    22LL&&$$..668	9r   c                 H    UR                   R                  UR                  :  $ r   )r   sizemultipart_thresholdro   s      r   rq   4UploadFilenameInputManager.requires_multipart_upload   s    ##((F,F,FFFr   c                     U R                  U5      u  p#U R                  U5      nU R                  U5      nU R                  U5      nUR                  R
                  nU R                  R                  X&UXES9$ )Nr7   
chunk_sizefull_file_sizer   close_callbacks)&_get_put_object_fileobj_with_full_sizer}   r   r   r   r   rV   #open_file_chunk_reader_from_fileobj)r   rk   r7   	full_sizer   r   r   s          r   rt   .UploadFilenameInputManager.get_put_object_body   s    !HH $$W-00A	33I>##(( ||??Y @ B 	Br   c           	   #     #    UR                   R                  nU R                  X5      n[        SUS-   5       H  nU R	                  U5      nU R                  U5      nX%S-
  -  nU R                  UR                   R                  R                  UX#S9u  pU R                  U	5      n	U R                  R                  XXUS9nX[4v   M     g 7f)N   )
start_byte	part_sizer   r   )r   r   _get_num_partsranger   r   '_get_upload_part_fileobj_with_full_sizer   r7   r}   rV   r   )r   rk   rw   r   	num_partspart_numberr   r   r   r7   r   read_file_chunks               r   rx   3UploadFilenameInputManager.yield_upload_part_bodies  s     (--22''C	 IM2K44_EI"77	BO"Ao6J "&!M!M$$..66:# "N "DG ((1G #llNN( / O 1O ..) 3s   C
Cc                 B    [        XU R                  R                  S9nU$ )N)open_function)r   rV   open)r   r7   r   s      r   _get_deferred_open_file2UploadFilenameInputManager._get_deferred_open_file  s!    "t||/@/@Br   c                     UR                   R                  R                  nUR                   R                  nU R	                  US5      U4$ )Nr   )r   r   r7   r   r   r   rk   r7   r   s       r   r   AUploadFilenameInputManager._get_put_object_fileobj_with_full_size#  sB    !&&0088##((++GQ7==r   c                 <    US   nUS   nU R                  X5      U4$ )Nr   r   )r   )r   r7   rN   r   r   s        r   r   BUploadFilenameInputManager._get_upload_part_fileobj_with_full_size(  s-    L)
+,	++G@)KKr   c                     [        [        R                  " UR                  R                  [        U5      -  5      5      $ r   )intmathceilr   r   float)r   rk   r   s      r   r   )UploadFilenameInputManager._get_num_parts-  s3    IIo**//%	2BBCE 	Er   r/   N)r*   r+   r,   r-   rQ   r   ra   rg   rl   rq   rt   rx   r   r   r   r   r.   r/   r   r   r   r      sH    &; ;9
GB&/2
>
L
Er   r   c                   @    \ rS rSrSr\S 5       rS rS rS r	S r
Srg	)
UploadSeekableInputManageri2  z&Upload utility for an open file objectc                 <    [        U5      =(       a    [        U5      $ r   )r   r   r^   s     r   ra   (UploadSeekableInputManager.is_compatible4  s    &B8M+BBr   c                     US:X  a  gg)N
put_objectFTr/   re   s     r   rg   0UploadSeekableInputManager.stores_body_in_memory8  s    \)r   c                    UR                   R                  R                  nUR                  5       nUR	                  SS5        UR                  5       nUR	                  U5        UR                   R                  XC-
  5        g )Nr      )r   r   r7   rD   r@   rl   )r   rk   r7   start_positionend_positions        r   rl   0UploadSeekableInputManager.provide_transfer_size>  se    !&&0088 !Q||~^$22)	+r   c                 l    UR                  US   5      n[        R                  " U5      [        U5      4$ )Nr   )r<   r   BytesIOlen)r   r7   rN   datas       r   r   BUploadSeekableInputManager._get_upload_part_fileobj_with_full_sizeJ  s1     ||F;/0 {{4 #d)++r   c                     UR                   R                  R                  nUR                  5       UR                   R                  -   nX#4$ r   )r   r   r7   rD   r   r   s       r   r   AUploadSeekableInputManager._get_put_object_fileobj_with_full_sizeY  s?    !&&0088 ||~ 4 4 9 99}r   r/   N)r*   r+   r,   r-   rQ   r   ra   rg   rl   r   r   r.   r/   r   r   r   r   2  s-    0C C
+,r   r   c                   n   ^  \ rS rSrSrSU 4S jjr\S 5       rS rS r	S r
S rS	 rSS
 jrS rSrU =r$ )UploadNonSeekableInputManageria  z7Upload utility for a file-like object that cannot seek.c                 <   > [         [        U ]  XU5        SU l        g )Nr   )superr   r   _initial_data)r   rX   r8   rY   	__class__s       r   r   &UploadNonSeekableInputManager.__init__c  s!    +T;*;	= r   c                     [        U5      $ r   )r   r^   s     r   ra   +UploadNonSeekableInputManager.is_compatibleh  s    &&r   c                     g)NTr/   re   s     r   rg   3UploadNonSeekableInputManager.stores_body_in_memoryl  s    r   c                     g r   r/   rj   s     r   rl   3UploadNonSeekableInputManager.provide_transfer_sizeo  s     	r   c                 2   UR                   R                  b#  UR                   R                  UR                  :  $ UR                   R                  R                  nUR                  nU R                  X4S5      U l        [        U R                  5      U:  a  gg)NFT)r   r   r   r   r7   _readr   r   )r   rk   rp   r7   r   s        r   rq   7UploadNonSeekableInputManager.requires_multipart_uploadt  s    $$0"'',,0J0JJJ "&&0088..	!ZZEBt!!"Y.r   c                     U R                  U5      nU R                  U5      nUR                  R                  R                  nU R                  U R                  UR                  5       -   X#5      nS U l        U$ r   )r   r   r   r   r7   
_wrap_datar   r<   )r   rk   r   r   r7   bodys         r   rt   1UploadNonSeekableInputManager.get_put_object_body  sm    00A	33I>!&&0088/M
 "r   c              #     #    UR                   R                  R                  nSn U R                  U5      nU R	                  U5      nUS-  nU R                  X25      nU(       d  g U R                  XuU5      nS nXH4v   M[  7f)Nr   r   )r   r   r7   r   r   r   r   )	r   rk   rw   file_objectr   r   r   part_contentpart_objects	            r   rx   6UploadNonSeekableInputManager.yield_upload_part_bodies  s     %**44<< 44_EI"77	BO1K::k=L//:K  L** s   A?Bc                 l   [        U R                  5      S:X  a  UR                  U5      $ U[        U R                  5      ::  a,  U R                  SU nU(       a  U R                  US U l        U$ U[        U R                  5      -
  nU R                  UR                  U5      -   nU(       a  SU l        U$ )a  
Reads a specific amount of data from a stream and returns it. If there
is any data in initial_data, that will be popped out first.

:type fileobj: A file-like object that implements read
:param fileobj: The stream to read from.

:type amount: int
:param amount: The number of bytes to read from the stream.

:type truncate: bool
:param truncate: Whether or not to truncate initial_data after
    reading from it.

:return: Generator which generates part bodies from the initial data.
r   Nr   )r   r   r<   )r   r7   r=   truncater   amount_to_reads         r   r   #UploadNonSeekableInputManager._read  s    $ t!!"a'<<'' S++,,%%gv.D %)%7%7%@"K
  #d&8&8"99!!GLL$@@ !$Dr   c                     U R                  [        R                  " U5      5      nU R                  R	                  U[        U5      [        U5      X#S9$ )ah  
Wraps data with the interrupt reader and the file chunk reader.

:type data: bytes
:param data: The data to wrap.

:type callbacks: list
:param callbacks: The callbacks associated with the transfer future.

:type close_callbacks: list
:param close_callbacks: The callbacks to be called when closing the
    wrapper for the data.

:return: Fully wrapped data.
r   )r}   r   r   rV   r   r   )r   r   r   r   r7   s        r   r   (UploadNonSeekableInputManager._wrap_data  sN      $$S[[%67||??D	#d) @ B 	Br   )r   r   )T)r*   r+   r,   r-   rQ   r   r   ra   rg   rl   rq   rt   rx   r   r   r.   __classcell__)r   s   @r   r   r   a  sJ    A!
 ' '
"+&(TB Br   r   c                   V    \ rS rSrSr/ SQrS/rS r SS jrS r	S	 r
S
 rS rS rSrg)UploadSubmissionTaski  z.Task for submitting tasks to execute an upload)SSECustomerKeySSECustomerAlgorithmSSECustomerKeyMD5RequestPayerr   c                     [         [        [        /nUR                  R                  R
                  nU H  nUR                  U5      (       d  M  Us  $    [        SU< S[        U5      < S35      e)a?  Retrieves a class for managing input for an upload based on file type

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The transfer future for the request

:rtype: class of UploadInputManager
:returns: The appropriate class to use for managing a specific type of
    input for uploads.
zInput z
 of type: z is not supported.)	r   r   r   r   r   r7   ra   RuntimeErrortype)r   rk   upload_manager_resolver_chainr7   upload_manager_clss        r   _get_upload_input_manager_cls2UploadSubmissionTask._get_upload_input_manager_cls  sn     '&))
% "&&0088"?!//88)) #@ g() 	)r   Nc                    U R                  U5      " X0R                  U5      nUR                  R                  c  UR	                  U5        UR                  XR5      (       d  U R                  XX4UU5        gU R                  XX4UU5        g)aj  
:param client: The client associated with the transfer manager

:type config: s3transfer.manager.TransferConfig
:param config: The transfer config associated with the transfer
    manager

:type osutil: s3transfer.utils.OSUtil
:param osutil: The os utility associated to the transfer manager

:type request_executor: s3transfer.futures.BoundedExecutor
:param request_executor: The request executor associated with the
    transfer manager

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The transfer future associated with the
    transfer request that tasks are being submitted for
N)r   r6   r   r   rl   rq   _submit_upload_request_submit_multipart_request)r   clientrp   rX   request_executorrk   rY   upload_input_managers           r   _submitUploadSubmissionTask._submit  s    (  $AA 224E G
 $$, 66G $==) )''/$& **/$&r   c                    UR                   R                  nU R                  US5      nU R                  R	                  U[        U R                  UUR                  U5      UR                  UR                  UR                  S.SS9US9  g )Nr   )r  r7   bucketkey
extra_argsT)r8   main_kwargsis_finaltag)
r   r   _get_upload_task_tagr6   submitPutObjectTaskrt   r  r  r	  )	r   r  rp   rX   r  rk   r  r   put_object_tags	            r   r   +UploadSubmissionTask._submit_upload_request2  s    #((22	 22 ,0 	""))%)%?%?$3GG' )'..$=="+"6"6   	* 	
r   c                 d   UR                   R                  nU R                  R                  U[	        U R                  UUR
                  UR                  UR                  S.S95      n/ n	U R                  UR                  5      n
U R                  US5      nUR                   R                  n[        5       nUR                  UR                  U5      nUR                  X^5      nU H`  u  nnU	R                  U R                  R                  U[!        U R                  UUUR
                  UR                  UU
S.SU0S9US95        Mb     U R#                  UR                  5      nU R                  R                  U[%        U R                  UUR
                  UR                  US.UU	S.S	S
95        g )N)r  r  r  r	  )r8   r
  upload_part)r  r7   r  r  r   r	  	upload_id)r8   r
  pending_main_kwargsr  )r  partsT)r8   r
  r  r  )r   r   r6   r  r	   r  r  r	  _extra_upload_part_argsr  r   r   adjust_chunksizemultipart_chunksizerx   appendUploadPartTask_extra_complete_multipart_argsr
   )r   r  rp   rX   r  rk   r  r   create_multipart_futurepart_futuresextra_part_argsupload_part_tagr   adjusterrw   part_iteratorr   r7   complete_multipart_extra_argss                      r   r   .UploadSubmissionTask._submit_multipart_requestL  s    $((22	 #'"<"<"C"C%%)%?%?$'..$=="+"6"6	#
 66y7K7KL 33 -1 ##(($&--f.H.H$O	,EE( %2 K**11$"-1-G-G&,'.&/&6&6#,==+6*9% ()@- (! 2  %2, )-(K(K  )"% 	""))'%)%?%?$'..$=="?	 "9)% 	
r   c                 ,    [        XR                  5      $ r   )r   UPLOAD_PART_ARGSr   r	  s     r   r  ,UploadSubmissionTask._extra_upload_part_args  s     !-B-BCCr   c                 ,    [        XR                  5      $ r   )r   COMPLETE_MULTIPART_ARGSr(  s     r   r  3UploadSubmissionTask._extra_complete_multipart_args  s     -I-IJJr   c                 B    S nUR                  U5      (       a  [        nU$ r   )rg   r   )r   r  rf   r  s       r   r  )UploadSubmissionTask._get_upload_task_tag  s"    55nEE&C
r   r/   r   )r*   r+   r,   r-   rQ   r'  r+  r   r  r   r   r  r  r  r.   r/   r   r   r   r     sG    8 	)2 48%&N
4K
ZD
Kr   r   c                       \ rS rSrSrS rSrg)r  i  z Task to do a nonmultipart uploadc                 f    U nUR                   " SX4US.UD6  SSS5        g! , (       d  f       g= f)a  
:param client: The client to use when calling PutObject
:param fileobj: The file to upload.
:param bucket: The name of the bucket to upload to
:param key: The name of the key to upload to
:param extra_args: A dictionary of any extra arguments that may be
    used in the upload.
)BucketKeyBodyNr/   )r   )r   r  r7   r  r  r	  r   s          r   _mainPutObjectTask._main  s+     NV4N:N WWs   "
0r/   Nr*   r+   r,   r-   rQ   r4  r.   r/   r   r   r  r    s    *
Or   r  c                       \ rS rSrSrS rSrg)r  i  z+Task to upload a part in a multipart uploadc           
      x    U nUR                   " SX4XVUS.UD6n	SSS5        W	S   n
XS.$ ! , (       d  f       N= f)a[  
:param client: The client to use when calling PutObject
:param fileobj: The file to upload.
:param bucket: The name of the bucket to upload to
:param key: The name of the key to upload to
:param upload_id: The id of the upload
:param part_number: The number representing the part of the multipart
    upload
:param extra_args: A dictionary of any extra arguments that may be
    used in the upload.

:rtype: dict
:returns: A dictionary representing a part::

    {'Etag': etag_value, 'PartNumber': part_number}

    This value can be appended to a list to be used to complete
    the multipart upload.
)r1  r2  UploadId
PartNumberr3  NETag)r;  r:  r/   )r  )r   r  r7   r  r  r  r   r	  r   responseetags              r   r4  UploadPartTask._main  sS    * )) )") ()H 
 88 Ws   +
9r/   Nr6  r/   r   r   r  r    s
    59r   r  )r   botocore.compatr   s3transfer.compatr   r   s3transfer.futuresr   s3transfer.tasksr   r   r	   r
   s3transfer.utilsr   r   r   r   objectr   r1   rS   r   r   r   r   r  r  r/   r   r   <module>rE     s      0 3 ! + 6 8 * . @ B(f (VvN vNrNE!3 NEb,!; ,^AB$6 ABH@> @FOD O9T 9r   