
                            S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
r
SSKrSSKrSSKrSSKrSSKrSSKJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKrSSKJr  SSKJr  SSK J!r!  SSK"J#r#  SSK$J%s  J&r&  SSK'J(r(  SSK)J*r*  SSK)J+r,  SSK)J-r-  SSK)J.r.  SSK)J/r/  SSK0J1r2  SSK3J4r4  SSK3J5r5  SSK6J7r7  SSK8J9r9  SSK:J;r;  SSK:J<r<  SSK=J>r>  SSK=J?r?  SS KJ@r@JArA  \@" \A" S!S!S"5      5        SS#KBJCrC  S$rDS%S&S'S(S)S*S+S,S-R                  \5      /	rF\G" \!" S.S/S0R                  \D\R                  \5      S15      \!" S.S/S2R                  \D\R                  \5      S15      \!" S.S/S3R                  \D\R                  \5      S15      \!" S.S/S4R                  \D\R                  \5      S15      /5      rK\	R                  " S55      rMS6 rNS7 rO " S8 S9\P5      rQ\." S:5      \CR                  " S;\CR                  " SS<9S=9 " S> S?\&R                  5      5       5       rU " S@ SA\P5      rV " SB SC\P5      rW\." S:5       " SD SE\&R                  5      5       rYg)F)Unit tests for analytics data collection.    )absolute_import)print_function)division)unicode_literalsN)
exceptions)http_wrapper)BucketStorageUri)metrics)VERSION)ApiSelector)
GcsJsonApi)MetricsCollector)Metric)MockLoggingHandler)	SkipForS3)HAS_S3_CREDS)ObjectToURI)SetBotoConfigForTest)SkipForParFile)unittest)storage_v1_messages)FileMessage)RetryableErrorMessage)START_CALLBACK_PER_BYTES)LogAndHandleRetries)IS_LINUX)
IS_WINDOWS)ONE_KIB)ONE_MIB)add_moveMovedModulemockzunittest.mock)r#   zXa=b&c=d&cd1=cmd1+action1&cd10=0&cd2=x%2Cy%2Cz&cd3=opta%2Coptb&cd6=CommandException&cm1=0za=bzc=dzcd1=cmd1 action1z	cd2=x,y,zzcd3=opta,optbzcd6=CommandExceptionzcm1=0zev=0zel={0}https://example.comPOSTz,{0}&cm2=3&ea=cmd1+action1&ec={1}&el={2}&ev=0user-agent-007z){0}&cm2=2&ea=Exception&ec={1}&el={2}&ev=0z*{0}&cm2=1&ea=ValueError&ec={1}&el={2}&ev=0z*{0}&ea=CommandException&ec={1}&el={2}&ev=0z(\[Metric.*\])c                 $     U " U0 UD6  g!    g= f)aC  Calls the given function with the arguments and ignores exceptions.

In these tests, we often force a failure that doesn't matter in order to
check that a metric was collected.

Args:
  func: The function to call.
  *args: Any arguments to call the function with.
  **kwargs: Any named arguments to call the function with.
N )funcargskwargss      +platform/gsutil/gslib/tests/test_metrics.py_TryExceptAndPassr-   h   s    	$&	s    c                  v   [         R                  " SS// SQSS/S9  [        [        5       S5      n [        [	        5       S5      n[         R
                  " U 5        [         R
                  " U 5        [         R
                  " U5        [         R                  " [        R                  R                  S5      5        g	)
z'Logs all the common metrics for a test.cmd1action1)z-yvalue)z-z )z-xr3   optbr3   optar3   )command_namesubcommandsglobal_optssub_optsr   testN)
r   LogCommandParamsr   	Exception
ValueErrorLogRetryableErrorLogFatalErrorgslib	exceptionCommandException)retry_msg_1retry_msg_2s     r,   _LogAllTestMetricsrG   y   s    	(1{(4%1<$@	B
 &ik15+%jlA6+	K(	K(	K(	88@A    c                       \ rS rSrSrS rSrg)RetryableErrorsQueue   zGEmulates Cloud API status queue, processes only RetryableErrorMessages.c                 \    [        U[        5      (       a  [        R                  " U5        g g N)
isinstancer   r   r@   )selfstatus_items     r,   putRetryableErrorsQueue.put   s#    +455, 6rH   r(   N)__name__
__module____qualname____firstlineno____doc__rQ   __static_attributes__r(   rH   r,   rJ   rJ      s
    O-rH   rJ   z:Do not try spawning the interpreter nested in the archive.	time.timereturn_valuenewc                      ^  \ rS rSrSrU 4S jrU 4S jrS rS rS r	S r
S	 r\R                  R                  \S
5      S 5       rS rSrU =r$ )TestMetricsUnitTests   r   c                   > [         [        U ]  5         [        R                  " 5       U l        [        R                  " SSSSS.5        [        R                  " 5       U l        [        5       U l	        [        R                  " S5      R                  [        R                  5        [        R                  " S5      R                  U R                  5        g )Nr$   r&   bdacr   )superr_   setUpr   GetCollectororiginal_collector_instanceStartTestCollector	collectorr   log_handlerlogging	getLoggersetLevelDEBUG
addHandlerrO   	__class__s    r,   rh   TestMetricsUnitTests.setUp   s    	
+- (8'D'D'FD$ ''(=?O1414)*+
 &224DN)+D i ))'--8i ++D,<,<=rH   c                 f   > [         [        U ]  5         [        R                  " U R
                  S9  g N)original_instance)rg   r_   tearDownr   StopTestCollectorrj   rs   s    r,   ry   TestMetricsUnitTests.tearDown   s*    	
.0 &&::<rH   c           	      
   U R                  U R                  [        R                  " 5       5        [        R
                  R                  [        R                  SSS.S9   [        R                  " 5         U R                  [        R                  5        U R                  U R                  [        R                  " 5       5        SSS5        [        R
                  " SSS9   [        R                  " 5         U R                  [        R                  5        U R                  S[        R                  " 5       5        SSS5        [        R
                  R                  [        R                  SS	S.S9   [        R                  " 5         U R                  [        R                  5        U R                  S[        R                  " 5       5        SSS5        [        R
                  R                  [        R                  S
S	0S9   [        R
                  " SSS9   [        R                  " 5         U R                  [        R                  5        U R                  S[        R                  " 5       5        SSS5        SSS5        [        R
                  R                  [        R                  S
S	0S9   [        R
                  " SSS9   [        R                  (       a  SnOSn[        R
                  " U5       nS UR                  l        ["        R$                  UR                  R&                  l        [        R                  " 5         U R                  [        R                  5        U R                  S[        R                  " 5       5        SUR                  R&                  l        [        R                  " 5         U R                  [        R                  5        U R                  U R                  [        R                  " 5       5        U R                  S[)        UR*                  5      5        U R                  S[)        UR                  R&                  R*                  5      5        SSS5        SSS5        SSS5        g! , (       d  f       GN= f! , (       d  f       GN_= f! , (       d  f       GN= f! , (       d  f       GNL= f! , (       d  f       GNV= f! , (       d  f       Ny= f! , (       d  f       N= f! , (       d  f       g= f)z/Tests enabling/disabling of metrics collection.1555)CLOUDSDK_WRAPPERGA_CID)valuesNzboto.config.getboolTrZ   r3   r   zos.path.existsFz__builtin__.openzbuiltins.openc                     U $ rM   r(   )ss    r,   <lambda>4TestMetricsUnitTests.testDisabling.<locals>.<lambda>   s    qrH   mock_cid   )assertEqualrl   r   ri   r#   patchdictosenviron_CheckAndSetDisabledCacheassertFalse_disabled_cache
assertTruesixPY2r[   	__enter__r   _DISABLED_TEXTreadlencall_args_list)rO   builtin_open	mock_opens      r,   testDisabling"TestMetricsUnitTests.testDisabling   se   T^^%5%B%B%DE 
14',! 
 

 002
'778
t~~'7'D'D'FG
 
)	=002
oo&667
t-::<= 
> 
14')! 
 

 002
oo&667
t-::<=
 
-?,D	E::&U;224(889/<<>? < 
F 
-?,D	E::&T:77+,(,ZZ%-8)
 
 
* 6=5K5K)
 
 
%
%
2

4
4
6
//*::
;


4!1!>!>!@
A 6@)
 
 
%
%
2

4
4
6


+;;
<


4>>+;+H+H+J
K 

1c)":":;
<


1c)"8"8"="="L"LM
N# & ; 
F	EE
 
 
>	=
 
 <; 
F	E &% ;: 
F	Es   A$R  AR1AR$SAR61S/S<1S+6E1S'S+/S< 
R
R!$
R36
S	 S
S
S($S++
S9	5S<<
T
c                   ^	 / SQn/ SQm	[        UT	-   5      n[        R                  " SSS9   U R                  SU R                  R                  5       5        SSS5        [        R                  " SSS9   U R                  SR                  U Vs/ s H  o3S	-   PM	     sn5      U R                  R                  5       5        SSS5        [        R                  " SS
S9   U R                  SR                  U Vs/ s H  o3S	-   PM	     sn5      U R                  R                  5       5        SSS5        S n[        R                  " SUS9   U R                  SU R                  R                  5       5        SSS5        U	4S jn[        R                  " SUS9   U R                  SU R                  R                  5       5        SSS5        U	4S jn[        R                  " SUS9   U R                  SU R                  R                  5       5        U	4S jn[        R                  " SUS9   U R                  SR                  T	 Vs/ s H  o3S	-   PM	     sn5      U R                  R                  5       5        SSS5        S n[        R                  " SUS9   U R                  SU R                  R                  5       5        SSS5        SSS5        g! , (       d  f       GN= fs  snf ! , (       d  f       GN3= fs  snf ! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNk= fs  snf ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g= f)z6Tests the validation of potentially PII config values.)check_hashescontent_languagedisable_analytics_prompthttps_validate_certificatesjson_api_version(parallel_composite_upload_component_size#parallel_composite_upload_threshold
prefer_api%sliced_object_download_component_size sliced_object_download_thresholdtab_completion_time_logstoken_cacheuse_magicfile)debugdefault_api_versionhttp_socket_timeoutmax_retry_delaynum_retriesoauth2_refresh_retriesparallel_process_countparallel_thread_countresumable_thresholdrsync_buffer_lines%sliced_object_download_max_componentssoftware_update_check_periodtab_completion_timeouttask_estimation_thresholdzboto.config.get_valueNrZ   r3   zinvalid string,z:INVALID   £c                     U S:X  a#  US:X  a  gUS:X  a  gUS:X  a  gUS:X  a  g	US
;   a  gU S:X  a  US:X  a  gU S:X  a  US:X  a  gg)NGSUtilr   if_fast_else_skipr   chir   v3r   xml)r   r   r   TrueOAuth2r   file_systemBotor   r3   r(   )sectioncategorys     r,   MockValidStringsHTestMetricsUnitTests.testConfigValueValidation.<locals>.MockValidStrings  sp    	H	~%$))))|# 4 4	H	]!:	F	x+HHrH   side_effectzcheck_hashes:if_fast_else_skip,content_language:chi,disable_analytics_prompt:True,https_validate_certificates:True,json_api_version:v3,prefer_api:xml,tab_completion_time_logs:True,token_cache:file_system,use_magicfile:Truec                    > UT;   a  gg)N1999r3   r(   _r   int_categoriess     r,   MockValidSmallIntsJTestMetricsUnitTests.testConfigValueValidation.<locals>.MockValidSmallInts2      	^	#rH   an  debug:1999,default_api_version:1999,http_socket_timeout:1999,max_retry_delay:1999,num_retries:1999,oauth2_refresh_retries:1999,parallel_process_count:1999,parallel_thread_count:1999,resumable_threshold:1999,rsync_buffer_lines:1999,sliced_object_download_max_components:1999,software_update_check_period:1999,tab_completion_timeout:1999,task_estimation_threshold:1999c                    > UT;   a  gg)N2001r3   r(   r   s     r,   MockValidLargeIntsJTestMetricsUnitTests.testConfigValueValidation.<locals>.MockValidLargeIntsB  r   rH   a  debug:INVALID,default_api_version:INVALID,http_socket_timeout:INVALID,max_retry_delay:INVALID,num_retries:INVALID,oauth2_refresh_retries:INVALID,parallel_process_count:INVALID,parallel_thread_count:INVALID,resumable_threshold:2001,rsync_buffer_lines:2001,sliced_object_download_max_components:INVALID,software_update_check_period:INVALID,tab_completion_timeout:INVALID,task_estimation_threshold:2001c                    > UT;   a  gg)Nz10.28r3   r(   r   s     r,   MockNonIntegerValueKTestMetricsUnitTests.testConfigValueValidation.<locals>.MockNonIntegerValueT  s    ~%rH   c                     US;   a  gg)N)r   r   r   r   10MiBr3   r(   )r   r   s     r,   MockDataSizeValueITestMetricsUnitTests.testConfigValueValidation.<locals>.MockDataSizeValue_  s     < < rH   zparallel_composite_upload_component_size:10485760,parallel_composite_upload_threshold:10485760,sliced_object_download_component_size:10485760,sliced_object_download_threshold:10485760)sortedr#   r   r   rl   _ValidateAndGetConfigValuesjoin)
rO   string_and_bool_categoriesall_categoriesr   r   r   r   r   r   r   s
            @r,   testConfigValueValidation.TestMetricsUnitTests.testConfigValueValidation   s   "N 6GHN 
+$	?
r4>>EEGH 
@ 
+:J	K

((.I.hz).I
J
..
4
4
68 
L 
+$	?

((.I.hz).I
J
..
4
4
68 
@& 
+9I	J
7 ..
4
4
68 
K
 
+9K	L
+ ..
4
4
68 
M
 
+9K	L
J ..
4
4
6	8
 ::-;NOHHNKN+NKLNN668	: P ::-;LM8 NN668	: NA 
M	LA 
@	?
 J 
L	K J 
@	?2 
K	J 
M	LH L PO NMA 
M	Ls   +K5K.K)$K.L:L $L+L+L)(AM".M 	L;$M ; M"+MM"
K&)K..
K= L
L
L&)
L8;M  
M	
M"
M	M""
M0c                 r   U R                  / U R                  R                  5        [        5         [        R
                  " SS9  U R                  / U R                  R                  5        U R                  R                  5         U R                  [        [        U R                  R                  5      5        g)z4Tests the collection of command and error GA events.cmd2)r8   N)	r   rl   _metricsrG   r   r=   _CollectCommandAndErrorMetricsCOMMAND_AND_ERROR_TEST_METRICSsetrO   s    r,   #testCommandAndErrorEventsCollection8TestMetricsUnitTests.testCommandAndErrorEventsCollectiono  s    R001&1 	R001NN11330013rH   c                 T   SU R                   R                  [        R                  S   '   [        R
                  " SSS0S9   [        R                  " S;0 SS_S	S_S
S_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS/_6  SSS5        [        [        R                  " 5       S 5      n[        [        R                  " 5       S 5      n[        R                  " U5        [        R                  " U5        [        R                  " U5        [        S!S"S SS#9n[        S!S"SSS$9nS%=Ul        Ul        S%=Ul        Ul        [        R                  " US&9  [        R                  " US&9  U R!                  U R                   R"                  R$                  S'   R'                  5       S5        [        R
                  " SSS(0S9   U R                   R)                  5         SSS5        U R                   R*                  S    R,                  nS)[        R.                  4S*S+S,S-S.S/S0S1S2S3S4S5S6S7S8/n[0        (       a  UR3                  S95        U H8  u  pxU R5                  S:R7                  [        R                  U   U5      U5        M:     g! , (       d  f       GN= f! , (       d  f       N= f)<z4Test the collection of PerformanceSummary GA events.cpCommand Namez)gslib.metrics.system_util.GetDiskCountersz	fake-disk)r   r   r   r   r   r   rZ   uses_fanT
uses_sliceavg_throughput
   is_daisy_chainhas_file_dstFhas_cloud_dsthas_file_srchas_cloud_srctotal_bytes_transferredd   total_elapsed_timethread_idle_time(   thread_execution_timenum_processesr   num_threads   num_objects_transferredprovider_typesgsNr   srcdst)size)finished   )file_message)r	  r	  )r   r   r   r   r   r   Event Category)Event ActionCloudToCloud%2CDaisyChain)zExecution Time10)Parallelism Strategyboth)Source URL Typecloud)Provider Typesr  )Num Processes2)Num Threads3)#Number of Files/Objects Transferredr  )!Size of Files/Objects Transferred100)Average Overall Throughputr  )Num Retryable Service Errorsr}   )zNum Retryable Network Errorsr  )Thread Idle Time Percentz0.8)Slowest Thread Throughputr  )Fastest Thread Throughputr  )Disk I/O Time20{0}={1}r(   )rl   	ga_paramsr   _GA_LABEL_MAPr#   r   LogPerformanceSummaryParamsr   apitools_exceptionsCommunicationErrorsocketerrorr@   r   	thread_id
process_idr   perf_sum_paramsthread_throughputsGetThroughput _CollectPerformanceSummaryMetricr   body_GA_PERFSUM_CATEGORYr   appendassertInformat)	rO   service_retry_msgnetwork_retry_msgstart_file_msgend_file_msgmetric_bodylabel_and_value_pairslabel	exp_values	            r,   %testPerformanceSummaryEventCollection:TestMetricsUnitTests.testPerformanceSummaryEventCollection~  s    GKDNNW22>BC 
?"-/A!B
D)) A4 A59A9;A :>A 8=	A
 9=A 8=A 9=A CFA >@A <>A ACA 9:A 78A CDA ;?A
D( ...0!5-fllna@/0/0/0 !qs;NueR$?L899N|5:;;N 7''^D''\B&&99 ;> 	??L}
 
?"-/C!D
F
nn557
F
 ..))!,11K	77785 ($ 44,--+++!$ x""#:;1
mmI$$W%:%:5%A9M! 2E
D 
DP
F 
Fs   AJJ
J
J'c                    [        U R                  R                  SSS/SS9  U R                  SU R                  R
                  R                  [        R                  S   5      5        U R                  SU R                  R
                  R                  [        R                  S	   5      5        U R                  R
                  R                  5         U R                  R                  S
SS9  U R                  SU R                  R
                  R                  [        R                  S   5      5        U R                  S
U R                  R
                  R                  [        R                  S   5      5        U R                  R
                  R                  5         [        U R                  R                  SSS/SS9  U R                  SU R                  R
                  R                  [        R                  S   5      5        g)z+Tests the collection of command parameters.aclr   -aT)collect_analyticszacl setr   re   Command-Level OptionslistlsCommand Aliasiamgetdummy_bucketziam getN)
r-   command_runnerRunNamedCommandr   rl   r#  rH  r   r$  clearr   s    r,   testCommandCollection*TestMetricsUnitTests.testCommandCollection  s   d))99eT](,. 	  $$W%:%:>%JKM 	  $$!!"9:	<= 	NN""$''$'G  $$W%:%:>%JKM 	  $$W%:%:?%KLN 	NN""$d))99e^4(,. 	  $$W%:%:>%JKMrH   )HandleExceptionsAndRebuildHttpConnectionsc                    [        5       n[        R                  " SS[        5       SSS5      n[        R                  " SS[        R
                  " 5       SSS5      n[        SUS9n[        SUS9nU" U5        U R                  U R                  R                  S   S5        U" U5        U R                  U R                  R                  S   S5        U" U5        [        R                  (       a*  U R                  U R                  R                  S   S5        O)U R                  U R                  R                  S	   S5        [        Xc5        [        Xd5        U R                  U R                  R                  S   S
5        [        R                  (       a*  U R                  U R                  R                  S   S5        gU R                  U R                  R                  S	   S5        g)z@Tests the collection of a retryable error in the retry function.NF)is_data_transferstatus_queueTr?   r	  r   SocketErrorOSErrorr  )rJ   r	   ExceptionRetryArgsr?   r(  r)  r   r   rl   retryable_errorsr   r   r-   )rO   mock_default_retry
mock_queuevalue_error_retry_argssocket_error_retry_argsmetadata_retry_funcmedia_retry_funcs          r,   testRetryableErrorCollection1TestMetricsUnitTests.testRetryableErrorCollection  s   
 &'J)<<dJL$d4*==dFLLND$6-u;EG*D8BD ./T^^44\BAF./T^^44\BAF/0
ww
t~~66}EqI
t~~66yA1E &?&@T^^44\BAF
ww
t~~66}EqI
t~~66yA1ErH   c                    [         R                  " [        S5      [        5       S9n[        R
                  " U5      nU" 5         U R                  R                  S   nU R                  SUS   5        U R                  R                  5         U R                  SUR                  5        [         R                  " [        S5      [        5       S9n[        R
                  " U5      nU" 5         U R                  SUR                  5        U R                  R                  S   nU R                  SUS   5        U R                  R                  5         [         R                  R                  [        S	S
S9   [        R                   " 5         [        R"                  " 5         [        R$                  " 5         [        R&                  " 5         [        R(                  " 5         [        R*                  " S5        U R                  R                  S   nSnS H  nU R                  SU-  X5   5        US-  nM!     U R                  R                  5         SSS5        g! , (       d  f       g= f)z>Tests the exception catching decorator CaptureAndLogException.mock_exc_fn)rS   r   r   z;Exception captured in mock_exc_fn during metrics collectionr   r	  mock_err_fnz;Exception captured in mock_err_fn during metrics collectionri   znot a collectorrZ   zinvalid argument)Shutdownr=   r@   rA   r%  'CheckAndMaybePromptForAnalyticsEnablingz2Exception captured in %s during metrics collectionN)r#   	MagicMockstrr>   r   CaptureAndLogExceptionrm   messagesr3  resetr   
call_count	TypeErrorr   objectr   rb  r=   r@   rA   r%  rc  )rO   r`  
wrapped_fndebug_messagesra  message_index	func_names          r,   testExceptionCatchingDecorator3TestMetricsUnitTests.testExceptionCatchingDecorator  s    ..#m*<-6[:K//<JL%%..w7NMMO #%Q../..#m*<-6[:K//<JLQ../%%..w7NMMO #% 
		+)(9 
 
;  !))+556HI''009nmC) 	@9L)	+ 	C +
; 
; 
;s   2CI
I#)rl   rm   rj   )rS   rT   rU   rV   rW   rh   ry   r   r   r   r=  rM  r#   r   rk  r	   r]  rp  rX   __classcell__rt   s   @r,   r_   r_      sg     2>(<?OB|:|3J!XMB ::\#NO F P FD1 1rH   r_   c                   $    \ rS rSrSrS rS rSrg)&_JSONForceHTTPErrorCopyCallbackHandleriD  zDTest callback handler that raises an arbitrary HTTP error exception.c                 *    Xl         X l        SU l        g )NF)_startover_at_byte_http_error_numstarted_over_once)rO   startover_at_bytehttp_error_nums      r,   __init__/_JSONForceHTTPErrorCopyCallbackHandler.__init__G  s    /)"DrH   c                     XR                   :  a=  U R                  (       d+  SU l        [        R                  " SU R                  0SS5      egg)z<Forcibly exits if the transfer has passed the halting point.TstatusN)rw  ry  r&  	HttpErrorrx  rO   r   unused_total_sizes      r,   call+_JSONForceHTTPErrorCopyCallbackHandler.callM  sN    #:#::""#d))8T5I5I*J*.6 6 # 	;rH   )rx  rw  ry  NrS   rT   rU   rV   rW   r|  r  rX   r(   rH   r,   ru  ru  D  s    L#6rH   ru  c                   *    \ rS rSrSr SS jrS rSrg)_ResumableUploadRetryHandleriV  zFTest callback handler for causing retries during a resumable transfer.c                 B    Xl         X l        X0l        X@l        SU l        g )Nr   )_retry_at_byte_exception_to_raise_exception_args_num_retries_retries_made)rO   retry_at_byteexception_to_raiseexc_argsr   s        r,   r|  %_ResumableUploadRetryHandler.__init__Y  s$    
 (1##DrH   c                     XR                   :  aI  U R                  U R                  :  a.  U =R                  S-  sl        U R                  " U R                  6 egg)z(Cause a single retry at the retry point.r	  N)r  r  r  r  r  r  s      r,   r  !_ResumableUploadRetryHandler.callf  sT    #6#66T...
A$$d&:&:;; 	/ 	7rH   )r  r  r  r  r  N)r	  r  r(   rH   r,   r  r  V  s    N 	
<rH   r  c                     ^  \ rS rSrSrU 4S jrU 4S jrSS jrS rS r	\
R                  " S	\
R                  " SS
9S9S 5       r\R                  " S5      \
R                  " S	\
R                  " SS
9S9S 5       5       rS rS rS rS rS rSS jrS r\" S5      S 5       r\" S5      S 5       rS r\R8                  " \S5      S 5       rSrU =r $ )TestMetricsIntegrationTestsin  z0Integration tests for analytics data collection.c                    > [         [        U ]  5         [        R                  " 5       U l        [        R                  " SSSSS.5        [        R                  " 5       U l        g )Nr$   r&   rb   rc   rd   )rg   r  rh   r   ri   rj   rk   rl   rs   s    r,   rh   !TestMetricsIntegrationTests.setUpr  sW    	
%t24 (8'D'D'FD$ ''(=?O1414)*+
 &224DNrH   c                 f   > [         [        U ]  5         [        R                  " U R
                  S9  g rw   )rg   r  ry   r   rz   rj   rs   s    r,   ry   $TestMetricsIntegrationTests.tearDown  s*    	
%t57 &&::<rH   r   c                 x    U R                  S/U-   SUSS0S9n[        R                  U5      R                  5       $ )a  Runs the gsutil command to check for metrics log output.

The env value is set so that the metrics collector in the subprocess will
use testing parameters and output the metrics collected to the debugging
log, which lets us check for proper collection in the stderr.

Args:
  cmd: The command to run, as a list.
  expected_status: The expected return code.

Returns:
  The string of metrics output.
z-dTGSUTIL_TEST_ANALYTICSr  )return_stderrexpected_statusenv_vars)	RunGsUtilMETRICS_LOG_REsearchgroup)rO   cmdr  stderrs       r,   _RunGsUtilWithAnalyticsOutput9TestMetricsIntegrationTests._RunGsUtilWithAnalyticsOutput  sJ     ^^TFSL*.,;&=s%C  EF   (..00rH   c                     [         R                  R                  " U0 UD6nU R                  UR                  5        UR                  5       $ )aT  Runs mock.patch.object with the given args, and returns the mock object.

This starts the patcher, returns the mock object, and registers the patcher
to stop on test teardown.

Args:
  *args: The args to pass to mock.patch.object()
  **kwargs: The kwargs to pass to mock.patch.object()

Returns:
  Mock, The result of starting the patcher.
)r#   r   rk  
addCleanupstopstart)rO   r*   r+   patchers       r,   _StartObjectPatch-TestMetricsIntegrationTests._StartObjectPatch  s9     jj00GOOGLL!==?rH   c                 j    U R                  SR                  [        R                  U   U5      U5        g)z;Checks for a correct key=value pair in a log output string.r"  N)r3  r4  r   r$  )rO   
param_namer<  metrics_to_searchs       r,   _CheckParameterValue0TestMetricsIntegrationTests._CheckParameterValue  s,    MM..z:IFrH   rY   rZ   r\   c                    U R                  [        S5      n[        R                  " 5       nUR	                  5         U R                  [        S5      n[        UR                  S5      Ul        U R                  R                  5         U R                  SUR                  5        [        5         [        R                  " 5         UR                  nU R                  S[!        U5      5        US   nU R#                  SUS   S   5        [%        [&        R(                  5      [%        US   S   S   R+                  [,        R.                  5      5      -
  nU R                  [%        5       U5        [        UR                  S5       n[0        R2                  " U5      nS	S	S	5        U R                  [4        [%        W5      5        g	! , (       d  f       N.= f)
z5Tests the subprocess creation by Popen in metrics.py.PopenNamedTemporaryFilewbr   r	  
PYTHONPATHenvrbN)r  
subprocesstempfiler  closeopennamer[   rl   ReportMetricsr   ri  rG   r   rb  r   r   r3  r   syspathsplitr   pathseppickleloadr   )rO   
popen_mockmetrics_filetemp_file_mock	call_listr*   missing_pathsreported_metricss           r,   testMetricsReporting0TestMetricsIntegrationTests.testMetricsReporting  sg    ''
G<J ..0L++H6JKN"&|'8'8$"?N 	NN  "Q
--. ))IQI'Q<DMM,Q/ ]a5;;BJJGHIMSUM* 
l	&,\2 
'3S9I5JK 
'	&s   G  
Gzexample.com is currently not accepting post requests, tracking issue: https://github.com/GoogleCloudPlatform/gsutil/issues/1800c                   ^  [         R                  " 5       nUR                  nUR                  5         S nT R	                  X25        U 4S jn[
        R                  " / SQS9  U" [        R                  UR                  5        [        UR                  S5       nUR                  5       nSSS5        [        R                  (       a*  SR                  [        [
        R                  [         5      nO8SR                  [        [
        R                  [         5      R#                  S	5      nT R%                  UW5        T R%                  S
U5        U" [        R&                  UR                  5        [        UR                  S5       nUR                  5       nSSS5        T R)                  US5        U" [        R*                  UR                  5        [        UR                  S5       nUR                  5       nSSS5        T R)                  US5        g! , (       d  f       GNo= f! , (       d  f       N= f! , (       d  f       ND= f)zFTests the metrics posting process as performed in metrics_reporter.py.c                 R     [         R                  " U 5        g ! [         a     g f = frM   )r   unlinkrT  )	file_paths    r,   MetricsTempFileCleanupNTestMetricsIntegrationTests.testMetricsPosting.<locals>.MetricsTempFileCleanup  s%    
		) s    
&&c                    > [         R                  " SS/SS/S9  [         R                  " [        R                  R                  S5      5        TR                  R                  SU US9  TR                  / TR                  R                  5        g )	Nr/   r0   r4   r6   )r8   r9   r;   r<   T)wait_for_report	log_levellog_file_path)
r   r=   rA   rB   rC   rD   rl   r  r   r   )r  r  rO   s     r,   CollectMetricAndSetLogLevelSTestMetricsIntegrationTests.testMetricsPosting.<locals>.CollectMetricAndSetLogLevel  s~    F,5;)5|(DF EOO<<VDE nn""4-61> # @ r4>>223rH   r1   )r:   r  Ns   Metric(endpoint=u'https://example.com', method=u'POST', body='{0}&cm2=0&ea=cmd1+action1&ec={1}&el={2}&ev=0', user_agent=u'user-agent-007')zMetric(endpoint='https://example.com', method='POST', body='{0}&cm2=0&ea=cmd1+action1&ec={1}&el={2}&ev=0', user_agent='user-agent-007')utf_8s   RESPONSE: 200rH   )r  r  r  r  r  r   r=   rn   rq   r  r   r   r   r4  GLOBAL_DIMENSIONS_URL_PARAMS_GA_COMMANDS_CATEGORYr   encoder3  INFOr   WARN)rO   r  metrics_file_namer  r  metrics_loglog_textexpected_responses   `       r,   testMetricsPosting.TestMetricsIntegrationTests.testMetricsPosting  s    ..0L$)) 	OO*>
4  *M N  |/@/@A	l	&+!!#h 
'
ww--3V*G,I,I. ++162N292O2O29,;<BF7O  	MM#X.MM"H-l.?.?@	l	&+!!#h 
'Xs#l.?.?@	l	&+!!#h 
'Xs#7 
'	&( 
'	&
 
'	&s$   HH%'H6
H"%
H36
Ic                    U R                  [        S5      n[        5       Ul        U R                  R
                  R                  S5        U R                  R                  5         U R                  UR                  5        g)z?Tests that metrics reporting error does not throw an exception.r  zdummy metricN)
r  r  rT  r   rl   r   r2  r  r   called)rO   r  s     r,   testMetricsReportingWithFail8TestMetricsIntegrationTests.testMetricsReportingWithFail  sX    ''
G<J$YJNN"">2NN  "OOJ%%&rH   c                    U R                  / SQSS9nU R                  S[        R                  U5        U R                  SSU5        U R                  SSU5        U R                  S	S
U5        U R                  S/5      nU R                  S[        R                  U5        U R                  SSU5        U R                  SSU5        g)z!Tests the collection of commands.)-mr@  r   rA  r	  r  r  r  zacl+setzGlobal Optionszd%2CmrC  re   verversionrF  N)r  r  r   r  )rO   metrics_lists     r,   rM  1TestMetricsIntegrationTests.testCommandCollection%  s    55"A 6 7L.0M0M*,niF.F5sLI55ug>L.0M0M*,niFoulCrH   c           	      p   U R                   [        R                  :w  a  [        R                  " S5      $ U R                  5       nU R                  USSS9nSU R                  R                  [        R                  S   '   [        [        [        R                  " 5       [        5       U R                   5      nSUR"                  l        SUR"                  l        UR)                  5       n[*        R,                  " UR.                  UR0                  R.                  UR2                  S	9n[*        R,                  " UR0                  U R5                  S
5      UR6                  S9n[8        R:                  R=                  [>        S[@        RB                  " 5       S9   [E        URF                  UU5        SSS5        [H        RJ                  (       a*  U RM                  U R                  RN                  S   S5        O)U RM                  U R                  RN                  S   S5        [8        R:                  R=                  [>        S[P        RR                  " SSS5      S9   [E        URT                  URV                  URX                  5        SSS5        U RM                  U R                  RN                  S   S5        U RM                  U R                  RZ                  R\                  S5        U RM                  U R                  RZ                  R^                  S5        g! , (       d  f       GNn= f! , (       d  f       N= f)zFTests that retryable errors are collected on JSON metadata operations.+Retryable errors are only collected in JSONfoo   bar)
bucket_uriobject_namecontentsrsyncr   r   r	  )r  bucketcontentTyperk  )r  r  r  _MakeRequestNoRetryr   NrS  rT  unusedr  )0test_apir   JSONr   skipCreateBucketCreateObjectrl   r#  r   r$  r   r
   rn   ro   rJ   default_provider
api_clientr   max_retry_waitget_keyapitools_messagesObjectr  r  content_typeMakeTempNamer  r#   r   rk  r	   r(  r)  r-   
CopyObjectr   r   r   rV  r&  r  DeleteObjectbucket_namer  r,  num_retryable_network_errorsnum_retryable_service_errors)rO   r  
object_uri
gsutil_apikeysrc_obj_metadatadst_obj_metadatas          r,   $testRetryableErrorMetadataCollection@TestMetricsIntegrationTests.testRetryableErrorMetadataCollection7  s    }}(((]]HII""$J""j/4,2 # 4J GNDNNW22>BC ,g.?.?.A02D4I4IKJ )*J%+,J( 


C(//SXX7:zz<?<L<LN )//&&x($002 
		<0'-||~ 
 
7 
--/?(*
7
 ww
t~~66}EqI t~~66yA1E 
		<0':'D'D'8(= 
 
> 
//1G1G"..0	
> 	T^^44[A1E 	&&CCQH&&CCQH/
7 
7
> 
>s   L,L'
L$'
L5c                    U R                   [        R                  :w  a  [        R                  " S5      $ SS[        [        5      4/nU R                  5       n[        S-  nU R                  SU-  S9nU R                  [        R                  " [        S[        R                  S5      5      S9n[        U5         U R!                  S	S
XT[#        U5      /5      nU R%                  S[&        R(                  U5        U R%                  SSU5        U R%                  SSU5        U R%                  SSU5        SSS5        U R                  [        R                  " [+        SS5      5      S9n[        U5         U R!                  S	S
XT[#        U5      /5      nU R%                  S[&        R(                  U5        U R%                  SSU5        U R%                  SSU5        U R%                  SSU5        SSS5        U R                  [        R                  " [+        SS5      5      S9n[        U5         U R!                  SS	S
XT[#        U5      /5      nU R%                  S[&        R(                  U5        U R%                  SSU5        U R%                  SSU5        U R%                  SSU5        SSS5        g! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       g= f)zCTests that retryable errors are collected on JSON media operations.r  r   r   r      a)r     )r  r  r  r   z--testcallbackfiler  r  BadStatusCodeErrorzRetryable Errorsr}   r  Ni  !ResumableUploadStartOverExceptionr  )r  r   r  r   r  re  r   r   r   CreateTempFiler  dumpsr  r&  r  r   r  surir  r   _GA_ERRORRETRY_CATEGORYru  )rO   boto_config_for_testr  	halt_sizefpathtest_callback_filer  s          r,   !testRetryableErrorMediaCollection=TestMetricsIntegrationTests.testRetryableErrorMediaCollectiono  s    }}(((]]HII%'<c'lKL""$J )1,I	)9:E ,,fll$Q(;(N(N%C	E7F, G 
2	377
$&8
z
9 	l  0 ' ? ?O
0D ,.
 2CF
 > ,. 
4 ,,DQLM - O	2	377
$&8
z
9 	l  0 ' ? ?O
 C ,.  2CF
 > ,. 
4 ,,DQLM - O	2	377
*,>
z
9 	l  0 ' ? ?O
 C ,.  2CF
 > ,. 
4	3C 
4	3  
4	3" 
4	3s'   :A9J/3A9K,A:K/
J>
K
K c                    ^  U 4S jnT R                  S/SS9nU" U5        T R                  SS/SS9nU" U5        T R                  5       nT R                  S[        U5      [        U5      /SS9nU" U5        g)	z&Tests that fatal errors are collected.c                 n   > TR                  S[        R                  U 5        TR                  SSU 5        g )Nr  r  rD   )r  r   _GA_ERRORFATAL_CATEGORY)
log_outputrO   s    r,   CheckForCommandExceptionVTestMetricsIntegrationTests.testFatalErrorCollection.<locals>.CheckForCommandException  s3    
 0 ' ? ?M
0BJOrH   zinvalid-commandr	  r  mbz-invalid-optionr   N)r  r   r  )rO   r)  r  r  s   `   r,   testFatalErrorCollection4TestMetricsIntegrationTests.testFatalErrorCollection  s    P
 557H6IFG 6 IL\*55t=N6OFG 6 IL\*""$J55	tJj!12A 6 GL\*rH   c                   ^ ^ UU 4S jnU(       a,  U" S5      nT R                  US5        T R                  US5        U" S5      nT R                  US5        U" S5      nU" S5      nT R                  Xv5        T R                  US5        T R                  US5        Sn[        (       a  U" S	5      nT R                  US5        XgU4$ )
a   Checks number metrics for PerformanceSummary tests.

Args:
  metrics_to_search: The string of metrics to search.
  multithread: False if the the metrics were collected in a non-multithread
               situation.

Returns:
  (slowest_throughput, fastest_throughput, io_time) as floats.
c                    > [         R                  " [        R                  U    S-   T5      nU(       d.  TR	                  S[        R                  U    < SU < ST< 35        [        UR                  S5      5      $ )Nz=(\d+\.?\d*)&zCould not find z (z) in metrics string r	  )rer  r   r$  failfloatr  )r  extracted_matchr  rO   s     r,   _ExtractNumberMetricVTestMetricsIntegrationTests._GetAndCheckAllNumberMetrics.<locals>._ExtractNumberMetric  sk    		




+.>
>
o 		"":.
<MO	P ?((+,,rH   r  r   r	  r  r  r  Nr   )assertGreaterEqualassertLessEqualassertGreaterr   )	rO   r  multithreadr4  r   
throughputslowest_throughputfastest_throughputio_times	   ``       r,   _GetAndCheckAllNumberMetrics8TestMetricsIntegrationTests._GetAndCheckAllNumberMetrics  s    - -.HI
.2
+Q/%&BCJz1%-.IJ-.IJ.C)1-)1-Gx$_5g
gq) G<<rH   c                    U R                  5       nU R                  5       n[        nU R                  USU-  S9  [        (       a  SOSn[	        SS[        U5      4S/5         U R                  SS	X/5      nU R                  S
[        R                  U5        U R                  SSU5        U R                  SSU5        U R                  SSU5        U R                  S[        U5      U5        U R                  SSU5        U R                  SSU5        U R                  SUU5        U R                  SSU5        U R                  U5      u    pg[        (       a  U R                  US5        SSS5        g! , (       d  f       g= f)z?Tests PerformanceSummary collection in a file-to-file transfer.r  tmpdirr  r	     r   r   )r   r   7r  r  r  r  
FileToFiler  fanr  filer  r  rD  r  r  r  r   N)CreateTempDirr    r  r   r   re  r  r  r   r1  r>  r   r6  )rO   tmpdir1tmpdir2	file_sizeprocess_countr  r   r=  s           r,    testPerformanceSummaryFileToFile<TestMetricsIntegrationTests.testPerformanceSummaryFileToFile  s     "G  "GIw	1AB $AM	*B #M 2 4GI 
J 77'
+-l
 0'2N2N ,.
lK
 6|L
 16<H
]1C ,.
sLA
 0&,G
 CY ,.
 Eq ,. 99,Goq!	 	+/
J 
J 
Js   "DE33
Fz$No slice parallelism support for S3.c                 4   U R                  5       nU R                  5       nSnU R                  USU-  S9  U R                  USU-  S9  [        (       a  SOSn[	        SS[        U5      4S	S
/5         U R                  SU[        U5      /5      nU R                  S[        R                  U5        U R                  SSU5        U R                  SSU5        U R                  S[        U5      U5        U R                  SSU5        U R                  SSUR                  -   U5        U R                  SSU-  U5        U R                  SSU5        U R                  U5      u    pg[        (       a  U R                  US5        SSS5        g! , (       d  f       g= f)z@Tests PerformanceSummary collection in a file-to-cloud transfer.rC  r  rA     br	  r   r   r   )r   r   r  )r   r   r}   r  r  r  FileToCloudr  slicer  r  r  r  file%2Cr  r  r   N)r   rH  r  r   r   re  r  r  r  r   r1  schemer>  r   r6  )rO   r  rB  rK  rL  r  r   r=  s           r,   !testPerformanceSummaryFileToCloud=TestMetricsIntegrationTests.testPerformanceSummaryFileToCloud  s    ""$J!FIvy0@Avy0@A#AM		+S-?@0> 

 77FD,
-/l
 0'2N2N ,.
|L
 6N
]1C ,.
sLA
 0)j>O>O2O ,.
 C !I|=
 Eq ,.99,Goq!	 	+1
 
 
s   2DF		
Fc                    U R                  5       nSnU R                  USU-  S9nU R                  5       n[        (       a  SOSn[	        SS[        U5      4SS	S
/5         U R                  SS[        U5      U/5      nU R                  S[        R                  U5        U R                  SSU5        U R                  SSU5        U R                  S[        U5      U5        U R                  SSU5        U R                  SSUR                  -   U5        U R                  SSU5        U R                  SUU5        U R                  U5      u    px[        (       a  U R                  US5        SSS5        g! , (       d  f       g= f)z@Tests PerformanceSummary collection in a cloud-to-file transfer.rC  r  r  r  r	     r   r   )r   r   5)r   r   r}   )r   test_assume_fast_crcmodr   r  r   r  r  CloudToFiler  r  r  r  rZ  r  rS  r  r}   r  r   N)r   r  r  r   r   re  r  r  r  r   r1  rT  r>  r   r6  )	rO   r  rK  r  r!  rL  r  r   r=  s	            r,   !testPerformanceSummaryCloudToFile=TestMetricsIntegrationTests.testPerformanceSummaryCloudToFile:  s    ""$JI""j,09,< # >J !E#AM		+S-?@0;5	 
 77tJ'
/1l
 0'2N2N ,.
|L
 6M
]1C ,.
sLA
 0)j>O>O2O ,.
 Es ,.
 CY ,.99,Goq!	 	+3
 
 
s    DE55
Fc                    U R                  5       nU R                  5       nSnU R                  USU-  S9nU R                  SS[        U5      [        U5      /5      nU R	                  USS9u  pgnU R                  Xg5        U R                  S[        R                  U5        U R                  S	S
U5        U R                  SSU5        U R                  SSU5        U R                  SSU5        U R                  SSU5        U R                  SUR                  U5        U R                  SSU5        U R                  SUU5        g)zATests PerformanceSummary collection in a cloud-to-cloud transfer.rC  r  rX  r   z-DF)r9  r  r  r  r  noner  r  r  r}   r  r  r  r  N)
r   r  r  r  r>  r   r  r   r1  rT  )	rO   bucket1_uribucket2_urirK  key_urir  r;  r<  r   s	            r,   "testPerformanceSummaryCloudToCloud>TestMetricsIntegrationTests.testPerformanceSummaryCloudToCloud`  s_   ##%K##%KI;)-	)9   ;G 55	tT']	k		L
 
+
+Le
+
L'<.0L0L*,n.I*,4flK/,GosLAmS,?.0B0B*,CS*,A9*,rH   z(Test requires both S3 and GS credentialsc                    U R                  SS9nU R                  SS9nU R                  USS9nU R                  USS9nU R                  S[        U5      [        U5      /5      nU R	                  SS	U5        U R	                  S
SU5        U R                  S[        U5      [        U5      /5      nU R	                  SS	U5        U R	                  S
SU5        g)z/Tests the collection of daisy-chain operations.s3)providerr  s   foorX  r  r  r  r  r  zgs%2Cs3r   N)r   r  r  r  r  )rO   	s3_bucket	gs_bucketunused_s3_keygs_keyr  s         r,   %testCrossProviderDaisyChainCollectionATestMetricsIntegrationTests.testCrossProviderDaisyChainCollection  s     !!4!0I!!4!0I%%V%LM)fEF55	$y/4	?35Ln.I*,.	<H55	tF|T)_-/Ln.I*,.	<HrH   )rl   rj   )r   )T)!rS   rT   rU   rV   rW   rh   ry   r  r  r  r#   r   rd  r  r   r  r  r  rM  r  r#  r,  r>  rM  r   rU  r]  rd  
skipUnlessr   rm  rX   rr  rs  s   @r,   r  r  n  s   85<1(" ::kt~~1=> L ? LD ==  S  T::kt~~1=>A$ ? TA$F	'D$6Hp>.@+*,=\!,F 34", 5",H 34#, 5#,J,B |%OPI QIrH   r  )ZrW   
__future__r   r   r   r   rn   r   r  r0  r(  r  r  r  pprintr   apitools.base.pyr   r&  r	   boto.storage_urir
   rB   r   r   gslib.cs_api_mapr   gslib.exceptiongslib.gcs_json_apir   gslib.metricsr   gslib.metrics_tupler    gslib.tests.mock_logging_handlerr   gslib.tests.testcaseteststestcase)gslib.tests.testcase.integration_testcaser   gslib.tests.utilr   r   r  r   r   r   "gslib.third_party.storage_apitoolsr   r  gslib.thread_messager   r   gslib.utils.constantsr   gslib.utils.retry_utilr   gslib.utils.system_utilr   r   gslib.utils.unit_utilr   r    r!   r"   	six.movesr#   r  r4  GLOBAL_PARAMETERSr   r  r  r'  r   compiler  r-   rG   rk  rJ   r   rd  GsUtilUnitTestCaser_   ru  r  GsUtilIntegrationTestCaser  r(   rH   r,   <module>r     sO    0 & %  '  	  	   
   
 > ) -   (  ) * & ? ' ' ? ) 0 1 + % W , 6 : 6 , . ) ) % VV_	5 6 ! 
 
5$k?GVX__W-E  "%
v6==('*G*G	&	(
 v3::('*I*I	&	(
 v4;;('*I*I	&	(
 v4;;('*I*I	&	(!& " 0 -.	"B-6 - LMKT^^;<m866 m = Nmd6V 6$<6 <0 LMdI("D"D dI NdIrH   