
    1                     X   S r SSKrSSKrSSKJr  SSKrSSKJr  SSKJr  SSKJ	r
  SSKJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  / SQrSrSrSrSrSr Sr!Sr"Sr#Sr$Sr%Sr&S\'S\(4S jr)S\*S \*S\(4S! jr+ " S" S#\RX                  5      r- " S$ S%\.5      r/g)&zLatency Diagnostic.    N)List)api_factory)	cloud_api)errors)request_config_factory)statistics_util)storage_url)
diagnostic)resource_reference)log)progress_tracker)files)scaled_integer)r   i   i  i      uploaddownloaddeletemetadataMeanzStandard deviationz90th percentilez50th percentilezLatency DiagnosticzSuccessful trialstime_in_secondsreturnc                     U S-  nUS S3$ )z*Formats a time in seconds as milliseconds.i  z.2fms )r   time_in_millisecondss     Elib/googlecloudsdk/command_lib/storage/diagnose/latency_diagnostic.py_format_as_millisecondsr   4   s    (4/ %R	((    object_sizeobject_numberc                 @    [         R                  " U 5      nSU SU S3$ )zEReturns the payload description for the given object size and number.zobject size z at index [])r   FormatInteger)r   r    scaled_object_sizes      r   _get_payload_descriptionr%   :   s*    %33K@*+;}oQ	GGr   c                       \ rS rSrSr SS\R                  S\\   4S jjr	\
S\4S j5       rS	 rS
\S\SS4S jr  SS jrS r  SS jr  SS jrS rS r\
S\R,                  4S j5       rSrg)LatencyDiagnostic@   zDiagnostic to measure the latency of various operations.

This diagnostic test will upload, download, and delete objects of different
sizes and record the latency of each operation.
Ntest_bucket_urlobject_sizesc                 ^   U(       a  UO[         U l        [        U R                  5      U l        Xl        S U l        / U l        [        R                  " U R                  R                  5      U l
        [        5       U l        0 U l        S[        [        R                   " 5       5      -   U l        g )Nlatency_diagnostics_)_DEFAULT_OBJECT_SIZESr*   lenobject_count
bucket_urltemp_dir_filesr   get_apischeme_api_client	DummyFile_discard_sink_resultstruuiduuid4object_prefix)selfr)   r*   s      r   __init__LatencyDiagnostic.__init__G   s    
 )5:ODD--.D%ODMDK"**4??+A+ABD"D DL 0#djjl2CCDr   r   c                     [         $ N)_DIAGNOSTIC_NAMEr=   s    r   nameLatencyDiagnostic.name[   s    r   c                     U R                  U R                  U R                  5      nU(       d  [        R                  " S5      eg)z4Creates the test files to be used in the diagnostic.zFailed to create test files.N)_create_test_filesr*   r<   r
   DiagnosticIgnorableError)r=   is_dones     r   _pre_processLatencyDiagnostic._pre_process_   s:    %%d&7&79K9KLG//0NOO r   operation_titler    c                     U R                   R                  U5      (       d  0 U R                   U'   U R                   U   R                  U5      (       d  0 U R                   U   U'   gg)zCreates an entry in the result dictionary for the given operation.

Args:
  operation_title: The title of the operation.
  object_number: The number of the object being operated on.
N)r8   get)r=   rL   r    s      r   _create_result_entry&LatencyDiagnostic._create_result_entryf   sY     <<O,,&(dll?#<<(,,];;57dll?#M2 <r   c                    U R                  [        U5        U R                  [           U   n [        R                  " XV5         [
        R                  " U5       nU R                  R                  XsU5        SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f! [        R                   a	    XV;   a  Xe	 e f = f)a8  Uploads an object and records the latency.

Args:
  object_number: The number of the object being uploaded.
  file_path: The path to the file to upload.
  object_resource: The object resource to upload.
  request_config: The request config to use for the upload.
  iteration: The iteration number of the upload.
N)rO   _UPLOAD_OPERATION_TITLEr8   r
   time_recorder
file_utils
FileReaderr5   upload_object
api_errorsCloudApiError)r=   r    	file_pathobject_resourcerequest_config	iterationresults_dictfiles           r   _upload_object LatencyDiagnostic._upload_objectu   s     	5}E<< 78GL##I<""9-



(
(
O . =<-- =< ## 		"#	sA   B+ BB	8B B+ 	
B	B
B($B+ (B+ +Cc                 ^   U R                  [        U5        U R                  [           U   n [        R                  " X45         U R
                  R                  U R                  R                  U5        SSS5        g! , (       d  f       g= f! [        R                   a	    X4;   a  XC	 e f = f)zFetches object metadata and records the latency.

Args:
  object_number: The number of the object being fetched.
  object_name: The name of the object to fetch metadata for.
  iteration: The iteration number of the fetch.
N)rO   _METADATA_OPERATION_TITLEr8   r
   rS   r5   get_object_metadatar0   bucket_namerW   rX   )r=   r    object_namer\   r]   s        r   _fetch_object_metadata(LatencyDiagnostic._fetch_object_metadata   s     	7G<< 9:=IL##I<,,OO''	
 =<< ## 		"#s)   B 1A>5B >
BB B B,c                 z   U R                  [        U5        U R                  [           U   n [        R                  " XE5         U R
                  R                  UU R                  U[        R                  R                  S9  SSS5        g! , (       d  f       g= f! [        R                   a	    XE;   a  XT	 e f = f)a  Downloads an object and records the latency.

Args:
  object_number: The number of the object being downloaded.
  object_resource: The object resource to download.
  request_config: The request config to use for the download.
  iteration: The iteration number of the download.
)download_strategyN)rO   _DOWNLOAD_OPERATION_TITLEr8   r
   rS   r5   download_objectr7   r   DownloadStrategyONE_SHOTrW   rX   )r=   r    rZ   r[   r\   r]   s         r   _download_object"LatencyDiagnostic._download_object   s     	7G<< 9:=IL##I<(('88AA	 	) 	
 =<< ## 		"#s)   B ?BB 
BB B B:c                 4   U R                  [        U5        U R                  [           U   n [        R                  " XE5         U R
                  R                  X#5        SSS5        g! , (       d  f       g= f! [        R                   a	    XE;   a  XT	 e f = f)a   Deletes an object and records the latency.

Args:
  object_number: The number of the object being deleted.
  object_url: The object url to delete.
  request_config: The request config to use for the delete.
  iteration: The iteration number of the delete.
N)	rO   _DELETE_OPERATION_TITLEr8   r
   rS   r5   delete_objectrW   rX   )r=   r    
object_urlr[   r\   r]   s         r   _delete_object LatencyDiagnostic._delete_object   s     	5}E<< 78GL##I<&&zB =<<## 		"#s)   A: A) A: )
A73A: 7A: :Bc                 *   [        [        5       GHj  n[        R                  " SU 35         [        U R                  5       GH*  nU R
                  U   nU R                  U   nUR                  [        R                  R                  5      S   n[        R                  " [        R                  R                  U R                  R                   U5      n["        R$                  " XdS9n[&        R(                  " UR                  [&        R*                  US9n U R-                  UUUUU5        U R/                  X'R0                  U5        U R3                  X'X5        U R5                  X&X5        GM-     SSS5        GMm     g! [6        R8                   a5  n	[:        R<                  " SUR0                   SU SU	 35         Sn	A	GM  Sn	A	f[>         a.  n	[@        RB                  " S	UR0                   SU SU	 35      eSn	A	ff = f! , (       d  f       GM  = f)
z|Runs the diagnostic.

Uploads, downloads, and deletes objects of different sizes and records the
latency of each operation.
zRunning latency iteration )size)content_typerx   z,Failed to run full operation set for object z in iteration z. NzUnexpected error for object )"range_ITERATION_COUNTr   ProgressTrackerr/   r2   r*   splitospathsepr	   CloudUrlProviderPrefixGCSr0   rd   r   ObjectResourcer   get_request_configDEFAULT_CONTENT_TYPEr_   rf   rD   rn   rt   rW   rX   r   debug	Exceptionr
   rH   )
r=   r\   r    rY   	file_sizere   rs   rZ   r[   es
             r   _runLatencyDiagnostic._run   s    +,	++&yk
2 #4#4#45Mkk-0)''6) "4R8+"++((,,oo))* /==/ 2DD))1FF.
 ''33Y !! >G 6	  -V )) II#(()	{"QCI   55#(()	{"QCI _ sC   C*HAF -H G?	)G	=HG?	)G:	:G?	?H
H	c                 .   U R                   bL   U R                   R                  5         U R                  U R                  R                  U R                  5        g g ! [         a/  n[        R                  " U R
                   SU 35         S nANfS nAff = f)Nz" : Failed to clean up temp files. )
r1   CloseOSErrorr   warningrD   _clean_up_objectsr0   
url_stringr<   )r=   r   s     r   _post_processLatencyDiagnostic._post_process  s|    }} I T__779K9KL !  Ityyk!CA3GHHIs   A 
B%%BBc           
        ^ / nU R                   R                  5        GH  u  p#UR                  5        GH  nU R                   U   U   R                  5       nU R                  U   n[
        R                  " U5      nU(       d  [        R                  " SU SU S35        Mo  [        U5      n0 n	U S[         3U	[        '   [        U5      U-  m[        T5      U	[        '   [        R                   " [        U4S jU 5       5      U-  5      n
[        U
5      U	["        '   [        [$        R&                  " [)        U5      S5      5      U	[*        '   [        [$        R&                  " [)        U5      S5      5      U	[,        '   [.        R0                  " UU	[3        U R                  U   U5      S9nUR5                  U5        GM     GM     [.        R6                  " U R8                  U5      $ )	NzNo successful trials for z on object size z. Skipping statistics./c              3   2   >#    U  H  oT-
  S -  v   M     g7f)   Nr   ).0xmeans     r   	<genexpr>+LatencyDiagnostic.result.<locals>.<genexpr>4  s     0ATas   2   Z   )payload_description)r8   itemskeysvaluesr*   r   r#   r   r   r.   r{   _SUCCESSFUL_TRIALS_TITLEsumr   _MEAN_TITLEmathsqrt_STANDARD_DEVIATION_TITLEr   find_percentilelist_PERCENTILE_50TH_TITLE_PERCENTILE_90TH_TITLEr
   DiagnosticOperationResultr%   appendDiagnosticResultrD   )r=   operation_resultsrL   object_number_to_latency_dictr    trialsr   r$   
num_trialscumulative_result_dictstandard_deviationoperation_resultr   s               @r   resultLatencyDiagnostic.result  s   :>,,:L:L:N68==?-o.}=DDF''6+99+F
++)/): ;)**@B [
!# l!,-. 	78 6{Z'.Ed.K{+!YY000:=

 $$67 	89
 $//VbA 	56 $//VbA 	56 &??" 8!!-0-!
 	  !12] @ ;Ob &&tyy2CDDr   )	r5   r7   r2   r8   r0   r/   r<   r*   r1   rA   )r   N)__name__
__module____qualname____firstlineno____doc__r	   r   r   intr>   propertyr9   rD   rJ   rO   r_   rf   rn   rt   r   r   r
   r   r   __static_attributes__r   r   r   r'   r'   @   s     !%D"++D ID( C  P8 814882*6,:x	M 3Ej11 3E 3Er   r'   c                   (    \ rS rSrSrSrS rS rSrg)r6   iR  zCA dummy file-like object that throws away everything written to it.bwc                     g rA   r   )r=   agrskwargss      r   writeDummyFile.writeW      r   c                     g rA   r   rC   s    r   closeDummyFile.closeZ  r   r   r   N)	r   r   r   r   r   moder   r   r   r   r   r   r6   r6   R  s    K	$		r   r6   )0r   r   r~   typingr   r:   googlecloudsdk.api_lib.storager   r   r   rW   r   "googlecloudsdk.command_lib.storager   r	   +googlecloudsdk.command_lib.storage.diagnoser
   ,googlecloudsdk.command_lib.storage.resourcesr   googlecloudsdk.corer   googlecloudsdk.core.consoler   googlecloudsdk.core.utilr   rT   r   r-   r{   rR   rj   rq   rb   r   r   r   r   rB   r   floatr9   r   r   r%   
Diagnosticr'   objectr6   r   r   r   <module>r      s       	   6 4 ? A > : B K # 8 8 3 ;  " & " & 0 * * ' . )U )s )H# Hc Hc HOE
-- OEd		 		r   