
    1                        S r SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSKJ	r	  SSKJ
r
  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKr " S S\R>                  " \R@                  \!5      5      r" " S S\"5      r# " S S\"5      r$ " S S\$5      r% " S S\$5      r&g)zGClasses that represent and execute different upload strategies for GCS.    )absolute_import)division)unicode_literalsN)encoding_helper)transfer)errors)
retry_util)metadata_util)apis)gcs_resource_reference)resource_reference)s3_resource_reference)	copy_util)log)
properties)retry)scaled_integerc                   \    \ rS rSrSr  S
S jrS rS rS r\	R                  S 5       rS	rg)_Upload*   z1Base class shared by different upload strategies.Nc	                     Xl         X l        X0l        X@l        XPl        X`l        Xpl        Xl        [        R                  " SS5      U l
        g)a  Initializes an _Upload instance.

Args:
  gcs_api (gcs_api.GcsApi): The API used to execute the upload request.
  http_client: An httplib2.Http-like object.
  source_stream (io.IOBase): Yields bytes to upload.
  destination_resource (resource_reference.ObjectResource|UnknownResource):
    Metadata for the destination object.
  should_gzip_in_flight (bool): Should gzip encode upload in flight.
  request_config (gcs_api.GcsRequestConfig): Tracks additional request
    preferences.
  posix_to_set (PosixAttributes|None): Set as custom metadata on target.
  source_resource (FileObjectResource|ObjectResource|None): Contains the
    source StorageUrl and source object metadata for daisy chain transfers.
    Can be None if source is pure stream.
storagev1N)_gcs_api_http_client_source_stream_destination_resource_should_gzip_in_flight_request_config_posix_to_set_source_resourcer   GetMessagesModule	_messages)	selfgcs_apihttp_clientsource_streamdestination_resourceshould_gzip_in_flightrequest_configposix_to_setsource_resources	            5lib/googlecloudsdk/api_lib/storage/gcs_json/upload.py__init___Upload.__init__-   sG    6 M#'!5"7)%+++It<DN    c                 b   [        U R                  [        R                  5      (       a  U R                  R
                  R                  (       a`  [        U R                  R                  S5      (       a:  [        R                  " U R                  R                  R                  5      Ul        g g g g )Nacl)
isinstancer!   r   ObjectResourcer   resource_argspreserve_aclhasattrmetadatacopydeepcopyr2   )r$   destination_metadatas     r-   J_copy_acl_from_source_if_source_is_a_cloud_object_and_preserve_acl_is_trueR_Upload._copy_acl_from_source_if_source_is_a_cloud_object_and_preserve_acl_is_trueT   s     	4((*<*K*KLL  ..;;D))22E:: "&



(
(
,
," ; < 	Mr0   c           
         Sn[        U R                  [        R                  5      (       a+  [        R
                  " U R                  R                  5      nO[        U R                  [        R                  5      (       a}  U R                  R                  (       ab  [        R                  " U R                  R                  R                  5        VVs0 s H  u  p4U[        R                  " U5      _M     snn5      nU(       a;  [        R                  " UU R                  R                   R"                  5      Ul        ggs  snnf )z>Update object metadata with contexts from the source resource.N)r3   r!   r   GcsObjectResourcer
   6parse_custom_contexts_dict_from_resource_contexts_dictcontextsr   S3ObjectResourcetags+get_contexts_dict_from_custom_contexts_dictitems!get_context_value_dict_from_valuer   DictToMessager#   ObjectContextsValue)r$   object_metadatacontexts_to_setkeyvalues        r-   %_update_object_metadata_with_contexts-_Upload._update_object_metadata_with_contextsa   s   O5GG  
N
N##,, 
 
4EE
 
 
			#	#EE #'"7"7"<"<"B"B"DG #EJC !CC
 #EG  	 !0!>!>

..


-
-"o Gs   #E 
c                    U R                   R                  (       a?  [        U R                  R                  R
                  U R                   R                  5      nOSnU R                  R                  U R                  R                  R                  U R                  R                  R                  S9n[        U R                  [        R                  5      (       ai  U R                  R                  (       aN  [         R"                  " U R                  R                  U R                  R                  R$                  5      Ul        U R)                  U5        U R+                  U5        [,        R.                  " UU R                   U R                  U R0                  [,        R2                  R4                  S9  U R                  R	                  UR6                  U[8        R:                  " U R                   5      U R                   R<                  US9$ )z>Get an insert request that includes validated object metadata.N)namebucket)attributes_resourcer+   method_type)rR   objectifGenerationMatchifMetagenerationMatchpredefinedAcl)r   predefined_acl_stringgetattrr#   StorageObjectsInsertRequestPredefinedAclValueValuesEnumrH   r   storage_urlresource_namebucket_namer3   r!   r   r4   custom_fieldsr   DictToAdditionalPropertyMessageMetadataValuer8   rN   r<   r
   *update_object_metadata_from_request_configr    
MethodTypeOBJECT_INSERTrR   r   get_generation_match_value!precondition_metageneration_match)r$   predefined_aclrJ   s      r-   _get_validated_insert_request%_Upload._get_validated_insert_request   s   11
..
4
4''



4
46n
 nnn++''33AA))55AA , CO 	4((*<*K*KLL++

9
9##11nn##11  	..?SS << 11''!,,:: >>55%%#>>  "   BB$ 6 & &r0   c                     g)z1Performs an upload and returns an Object message.N r$   s    r-   run_Upload.run        	r0   )	r   r   r   r#   r    r   r   r!   r   )NN)__name__
__module____qualname____firstlineno____doc__r.   r<   rN   ri   abcabstractmethodrn   __static_attributes__rl   r0   r-   r   r   *   s>    9 %=NB+&Z 	 	r0   r   c                       \ rS rSrSrS rSrg)SimpleUpload   z&Uploads objects with a single request.c                 p   U R                   R                  n[        R                  " U R                  UR
                  U R                  UR                  S9nU R                  Ul	        [        R                  Ul        U R                  R                  R                  R                  U R!                  5       US9$ )N)gzip_encoded
total_sizeupload)r   r5   r   Uploadr   content_typer   sizer   
bytes_httpSIMPLE_UPLOADstrategyr   clientobjectsInsertri   )r$   r5   apitools_uploads      r-   rn   SimpleUpload.run   s    ((66Moo""00 %%	'O
 "&!2!2O'55O==''..**,_ / F Fr0   rl   N)rq   rr   rs   rt   ru   rn   rx   rl   r0   r-   rz   rz      s    .Fr0   rz   c                   T    \ rS rSrSrS rS r\R                  S 5       r	S r
S rSrg	)
_BaseRecoverableUpload   z7Common logic for strategies allowing retries in-flight.c           
         U R                   R                  n[        USS5      n[        R                  R
                  R                  R                  5       n[        R                  " U R                  UR                  S[        R                  " [        R                  R
                  R                  R                  5       5      U R                   UUS9n[        R"                  Ul        U$ )z9Returns an apitools upload class used for a new transfer.r   NF)auto_transfer	chunksizer}   r~   num_retries)r   r5   rZ   r   VALUESr   max_retriesGetIntr   r   r   r   r   ParseIntegerupload_chunk_sizeGetr   RESUMABLE_UPLOADr   )r$   r5   r   r   r   s        r-   _get_upload"_BaseRecoverableUpload._get_upload   s    ((66M=&$/D##++77>>@Koo"" --%%77;;=?00!O  (88Or0   c                     U R                   R                  (       dG  U R                  R                  R                  R                  U R                  5       U R                   S9  gg)z1Inserts a a new object at the upload destination.r   N)_apitools_uploadinitializedr   r   r   r   ri   rm   s    r-   _initialize_upload)_BaseRecoverableUpload._initialize_upload   sN      ,,
mm""))

,
,
.t7L7L * N -r0   c                     g)zBResponsible for pushing bytes to GCS with an appropriate strategy.Nrl   rm   s    r-   *_call_appropriate_apitools_upload_strategyA_BaseRecoverableUpload._call_appropriate_apitools_upload_strategy   rp   r0   c                 N   [        U[        R                  5      (       d  gU R                  R	                  5         U R                  R
                  U R                  :  a"  U R                  R
                  U l        SUl        [        R                  " SR                  X5      5        g)z.Returns True if the failure should be retried.Fr   z.Retrying upload after exception: {}. Trace: {}T)r3   r   RetryableApiErrorr   RefreshResumableUploadStateprogress_last_progress_byteretrialr   debugformat)r$   exc_type	exc_valueexc_tracebackstates        r-   _should_retry_resumable_upload5_BaseRecoverableUpload._should_retry_resumable_upload   s     i!9!9::557%%(@(@@!%!6!6!?!?demII !6(:<r0   c                    U R                  5       U l        U R                  U R                  l        [        R
                  " U R                  5        U R                  5         U R                  R                  U l         [        R                  " U R                  U R                  S9nU R"                  R$                  R&                  R)                  U R"                  R$                  R&                  R+                  S5      U5      $ ! [        R                   a*  n[        R                  " SR!                  U5      5      eSnAff = f)zAUploads with in-flight retry logic and returns an Object message.)targetshould_retry_ifz7Max retrial attempts reached. Aborting upload.Error: {}Nr   )r   r   r   r   r	   set_retry_funcr   r   r   retryerr   r   r   MaxRetrialsExceptionr   ResumableUploadAbortErrorr   r   r   r   ProcessHttpResponseGetMethodConfig)r$   http_responsees      r-   rn   _BaseRecoverableUpload.run   s    ,,.D'+'8'8D$d334#44==D! ((@@==?m =='';;$$44X>O O %% !,,fQi! !!s   =)D E%D<<E)r   r   N)rq   rr   rs   rt   ru   r   r   rv   rw   r   r   rn   rx   rl   r0   r-   r   r      s5    ?"N 	 	 Or0   r   c                       \ rS rSrSrS rSrg)StreamingUploadi  zHUploads objects from a stream with support for error recovery in-flight.c                 6    U R                   R                  5       $ )z5Calls StreamInChunks since the final size is unknown.)r   StreamInChunksrm   s    r-   r   :StreamingUpload._call_appropriate_apitools_upload_strategy  s      //11r0   rl   N)rq   rr   rs   rt   ru   r   rx   rl   r0   r-   r   r     s
    P2r0   r   c                   V   ^  \ rS rSrSr    SU 4S jjrU 4S jrU 4S jrS rSr	U =r
$ )	ResumableUploadi  zDUploads objects with support for resuming between runs of a command.c                 N   > [         [        U ]  UUUUUUUU	S9  Xl        Xl        g)a  Initializes a ResumableUpload instance.

See super class for arguments not described below.

New Args:
  serialization_data (dict): JSON used by apitools to resume an upload.
  tracker_callback (Callable[[dict]|None]): Function that writes a tracker
    file with serialization data.
)r+   r,   N)superr   r.   _serialization_data_tracker_callback)r$   r%   r&   r'   r(   r)   r*   r+   serialization_datar,   tracker_callback	__class__s              r-   r.   ResumableUpload.__init__  s?    . 
/4)!' * 	  2-r0   c           	        > [         R                  R                  R                  R	                  5       nU R
                  br  [        R                  R                  U R                  [        R                  " U R
                  5      U R                  R                  R                  SU R                  US9$ [         TU ]G  5       $ )zCCreates a new transfer object, or gets one from serialization data.F)r   r}   r   )r   r   r   r   r   r   r   r   FromDatar   jsondumpsr   r   httpr   r   r   )r$   r   r   s     r-   r   ResumableUpload._get_upload;  s    ##++77>>@K+__%%



**T--
.
--


#
#22! & # # 9d/11r0   c                    > U R                   c  [        TU ]  5         U R                  b&  U R                  U R                  R
                  5        gg)zEInserts an object if not already inserted, and writes a tracker file.N)r   r   r   r   r   r   )r$   r   s    r-   r   "ResumableUpload._initialize_uploadJ  sG    'It/1)
T22EEF *r0   c                     U R                   (       a  U R                  R                  5       $ U R                  R                  5       $ )zDCalls StreamMedia, or StreamInChunks when the final size is unknown.)r   r   r   StreamMediarm   s    r-   r   :ResumableUpload._call_appropriate_apitools_upload_strategyR  s7    """"1133 ""..00r0   )r   r   )NNNN)rq   rr   rs   rt   ru   r.   r   r   r   rx   __classcell__)r   s   @r-   r   r     s/    L ".H2G1 1r0   r   )'ru   
__future__r   r   r   rv   r9   r   apitools.base.pyr   r   googlecloudsdk.api_lib.storager   r	   'googlecloudsdk.api_lib.storage.gcs_jsonr
   googlecloudsdk.api_lib.utilr   ,googlecloudsdk.command_lib.storage.resourcesr   r   r   +googlecloudsdk.command_lib.storage.tasks.cpr   googlecloudsdk.corer   r   googlecloudsdk.core.utilr   r   sixwith_metaclassABCMetarU   r   rz   r   r   r   rl   r0   r-   <module>r      s    N &  ' 
   , % 1 5 A , O K N A # * * 3 
H	c  f5 H	VF7 F"BOW BOJ2, 2G1, G1r0   