
    u                       S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKJr  SSKJ	r	  SSK
r
SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSS	KJr  SS
KJr  SSKJr  SSKJr  SSKrSSK J!r!  SSK J"r"  SSK#J$r$  SSK#J%r%  SSK&J'r'  SSK(J)r)  SSK*J+r+  SSK,J-r-  SSK.J/r/  SSK0J1r1  SSK2J3r4  SSK5J6r6  SSK7J8r8  SSK7J9r9  SSK:J;r;  SSK<J=r=  SSK<J>r>  SSK?J@r@  SSKAJBrB  SSKAJCrC  SS KAJDrD  SS!KAJErE  SS"KAJFrF  SS#KGJHrH  SS$KGJIrI  SS%KGJJrJ  SS&KGJKrK  SS'KGJLrL  S(rMS)\M-   S*-   rN\	" S+S,5      rO\	" S-S.5      rP\	" S/S05      rQ\	" S1S25      rR\	" S3S45      rS0 rT " S5 S6\U5      rV " S7 S8\V5      rWSFS9 jrXSFS: jrYSFS; jrZSFS< jr[SFS= jr\S> r]S? r^ " S@ SA\_5      r`   SGSB jra " SC SD\$5      rbSE rcg)Hz%Contains the perfdiag gsutil command.    )absolute_import)print_function)division)unicode_literalsN)defaultdict)
namedtuple)	cStringIO)http_client)range)NotFoundException)ServiceException)Command)DummyArgChecker)CommandArgument)config)ApiSelector)CommandException)FilePart)StorageUrlFromString)storage_v1_messages)	text_util)GetMaxRetryDelay)ResumableThreshold)UTF8)GetCloudApiInstance)GetDownloadSerializationData)"CalculateB64EncodedMd5FromContents)CheckFreeSpace)GetDiskCounters)GetFileSize)IS_LINUX)IsRunningInCiEnvironment)DivideAndCeil)HumanReadableToBytes)MakeBitsHumanReadable)MakeHumanReadable)
Percentilez
  gsutil perfdiag [-i in.json]
  gsutil perfdiag [-o out.json] [-n objects] [-c processes]
      [-k threads] [-p parallelism type] [-y slices] [-s size] [-d directory]
      [-t tests] [-j ratio] gs://<bucket_name>...
z
<B>SYNOPSIS</B>
a!  


<B>DESCRIPTION</B>
  The ``perfdiag`` command runs a suite of diagnostic tests for a given Cloud
  Storage bucket.

  The ``bucket_name`` parameter must name an existing bucket to which the user
  has write permission. Several test files will be uploaded to and downloaded
  from this bucket. All test files will be deleted at the completion of the
  diagnostic if it finishes successfully. For a list of relevant permissions,
  see `Cloud IAM permissions for gsutil commands
  <https://cloud.google.com/storage/docs/access-control/iam-gsutil>`_.

  gsutil performance can be influenced by a number of factors originating
  at the client, server, or network level. Some examples include the
  following:
  
   + CPU speed
   + Available memory
   + The access path to the local disk
   + Network bandwidth
   + Contention and error rates along the path between gsutil and Google servers
   + Operating system buffering configuration
   + Firewalls and other network elements
  
  The `perfdiag` command is provided so that customers can run a known
  measurement suite when troubleshooting performance problems.


<B>PROVIDING DIAGNOSTIC OUTPUT TO THE CLOUD STORAGE TEAM</B>
  If the Cloud Storage team asks you to run a performance diagnostic
  please use the following command, and email the output file (output.json)
  to the @google.com address provided by the Cloud Storage team:

    gsutil perfdiag -o output.json gs://your-bucket

  Additional resources for discussing ``perfdiag`` results include the
  `Stack Overflow tag for Cloud Storage
  <https://stackoverflow.com/questions/tagged/google-cloud-storage>`_ and
  the `gsutil GitHub repository
  <https://github.com/GoogleCloudPlatform/gsutil/issues>`_.


<B>OPTIONS</B>
  -n          Sets the number of objects to use when downloading and uploading
              files during tests. Defaults to 5.

  -c          Sets the number of `processes
              <https://en.wikipedia.org/wiki/Process_(computing)>`_ to use
              while running throughput experiments. The default value is 1.

  -k          Sets the number of `threads
              <https://en.wikipedia.org/wiki/Thread_(computing)>`_ per process
              to use while running throughput experiments. Each process will
              receive an equal number of threads. The default value is 1.

              NOTE: All specified threads and processes will be created, but may
              not by saturated with work if too few objects (specified with ``-n``)
              and too few components (specified with ``-y``) are specified.

  -p          Sets the type of parallelism to be used (only applicable when
              threads or processes are specified and threads * processes > 1). The
              default is to use ``fan``. Must be one of the following:

              fan
                 Use one thread per object. This is akin to using gsutil ``-m cp``,
                 with sliced object download / parallel composite upload
                 disabled.

              slice
                 Use Y (specified with ``-y``) threads for each object, transferring
                 one object at a time. This is akin to using parallel object
                 download / parallel composite upload, without ``-m``. Sliced
                 uploads not supported for s3.

              both
                 Use Y (specified with ``-y``) threads for each object, transferring
                 multiple objects at a time. This is akin to simultaneously
                 using sliced object download / parallel composite upload and
                 ``gsutil -m cp``. Parallel composite uploads not supported for s3.

  -y          Sets the number of slices to divide each file/object into while
              transferring data. Only applicable with the slice (or both)
              parallelism type. The default is 4 slices.

  -s          Sets the size (in bytes) for each of the N (set with ``-n``) objects
              used in the read and write throughput tests. The default is 1 MiB.
              This can also be specified using byte suffixes such as 500K or 1M.
              
              NOTE: these values are interpreted as multiples of 1024 (K=1024,
              M=1024*1024, etc.)
              
              NOTE: If ``rthru_file`` or ``wthru_file`` are performed, N (set with
              ``-n``) times as much disk space as specified will be required for
              the operation.

  -d          Sets the directory to store temporary local files in. If not
              specified, a default temporary directory will be used.

  -t          Sets the list of diagnostic tests to perform. The default is to
              run the ``lat``, ``rthru``, and ``wthru`` diagnostic tests. Must be a
              comma-separated list containing one or more of the following:

              lat
                 For N (set with ``-n``) objects, write the object, retrieve its
                 metadata, read the object, and finally delete the object.
                 Record the latency of each operation.

              list
                 Write N (set with ``-n``) objects to the bucket, record how long
                 it takes for the eventually consistent listing call to return
                 the N objects in its result, delete the N objects, then record
                 how long it takes listing to stop returning the N objects.

              rthru
                 Runs N (set with ``-n``) read operations, with at most C
                 (set with -c) reads outstanding at any given time.

              rthru_file
                 The same as ``rthru``, but simultaneously writes data to the disk,
                 to gauge the performance impact of the local disk on downloads.

              wthru
                 Runs N (set with ``-n``) write operations, with at most C
                 (set with ``-c``) writes outstanding at any given time.

              wthru_file
                 The same as wthru, but simultaneously reads data from the disk,
                 to gauge the performance impact of the local disk on uploads.

  -m          Adds metadata to the result JSON file. Multiple ``-m`` values can be
              specified. Example:

                  gsutil perfdiag -m "key1:val1" -m "key2:val2" gs://bucketname

              Each metadata key will be added to the top-level "metadata"
              dictionary in the output JSON file.

  -o          Writes the results of the diagnostic to an output file. The output
              is a JSON file containing system information and performance
              diagnostic results. The file can be read and reported later using
              the ``-i`` option.

  -i          Reads the JSON output file created using the ``-o`` command and prints
              a formatted description of the results.

  -j          Applies gzip transport encoding and sets the target compression
              ratio for the generated test files. This ratio can be an integer
              between 0 and 100 (inclusive), with 0 generating a file with
              uniform data, and 100 generating random data. When you specify
              the ``-j`` option, files being uploaded are compressed in-memory and
              on-the-wire only. See `cp -j
              <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_
              for specific semantics.


<B>MEASURING AVAILABILITY</B>
  The ``perfdiag`` command ignores the boto num_retries configuration parameter.
  Instead, it always retries on HTTP errors in the 500 range and keeps track of
  how many 500 errors were encountered during the test. The availability
  measurement is reported at the end of the test.

  Note that HTTP responses are only recorded when the request was made in a
  single process. When using multiple processes or threads, read and write
  throughput measurements are performed in an external process, so the
  availability numbers reported won't include the throughput measurements.


<B>NOTE</B>
  The ``perfdiag`` command runs a series of tests that collects system information,
  such as the following: 
  
  + Retrieves requester's IP address.
  + Executes DNS queries to Google servers and collects the results.
  + Collects network statistics information from the output of ``netstat -s`` and
    evaluates the BIOS product name string.
  + If a proxy server is configured, attempts to connect to it to retrieve
    the location and storage class of the bucket being used for performance
    testing.
  
  None of this information will be sent to Google unless you proactively choose to
  send it.
FileDataTuplezsize md5 dataFanDownloadTuplez6need_to_slice object_name file_name serialization_dataSliceDownloadTuplez<object_name file_name serialization_data start_byte end_byteFanUploadTuplez9need_to_slice file_name object_name use_file gzip_encodedSliceUploadTuplez@file_name object_name use_file file_start file_size gzip_encodedc                       \ rS rSrSrSrg)Errori2  z%Base exception class for this module. N__name__
__module____qualname____firstlineno____doc____static_attributes__r/       *platform/gsutil/gslib/commands/perfdiag.pyr.   r.   2  s    -r7   r.   c                       \ rS rSrSrSrg)InvalidArgumenti7  z)Raised on invalid arguments to functions.r/   Nr0   r/   r7   r8   r:   r:   7  s    1r7   r:   c                    [        X5      nUR                  (       a2  U R                  UR                  UR                  UR
                  5        gU R                  UR                  X1R                  UR
                  5        g)zFunction argument to apply for performing fanned parallel downloads.

Args:
  cls: The calling PerfDiagCommand class instance.
  args: A FanDownloadTuple object describing this download.
  thread_state: gsutil Cloud API instance to use for the operation.
N)r   need_to_slicePerformSlicedDownloadobject_name	file_nameserialization_dataDownloadclsargsthread_state
gsutil_apis       r8   _DownloadObjectrG   <  sa     #35*	d.."557 LL!!:~~((*r7   c                     [        X5      nU R                  UR                  X1R                  UR                  UR
                  UR                  5        g)zFunction argument to apply for performing sliced downloads.

Args:
  cls: The calling PerfDiagCommand class instance.
  args: A SliceDownloadTuple object describing this download.
  thread_state: gsutil Cloud API instance to use for the operation.
N)r   rA   r>   r?   r@   
start_byteend_byterB   s       r8   _DownloadSlicerK   M  s>     #35*,,t^^&&Hr7   c                 *   [        X5      nUR                  (       a<  U R                  UR                  UR                  UR
                  UUR                  S9  gU R                  UR                  UR                  UUR
                  UR                  S9  g)zFunction argument to apply for performing fanned parallel uploads.

Args:
  cls: The calling PerfDiagCommand class instance.
  args: A FanUploadTuple object describing this upload.
  thread_state: gsutil Cloud API instance to use for the operation.
gzip_encodedN)r   r<   PerformSlicedUploadr?   r>   use_filerN   UploadrB   s       r8   _UploadObjectrR   Z  s     #35*	DNN ,, MM&)-):):	  < JJt~~}} --	  /r7   c           
          [        X5      nU R                  UR                  UR                  UUR                  UR
                  UR                  UR                  S9  g)zFunction argument to apply for performing sliced parallel uploads.

Args:
  cls: The calling PerfDiagCommand class instance.
  args: A SliceUploadTuple object describing this upload.
  thread_state: gsutil Cloud API instance to use for the operation.
rM   N)r   rQ   r?   r>   rP   
file_start	file_sizerN   rB   s       r8   _UploadSlicerV   q  sO     #35***T^^]]__^^++  -r7   c                 <    [        X5      nU R                  X5        g)a  Function argument to apply for performing parallel object deletions.

Args:
  cls: The calling PerfDiagCommand class instance.
  object_name: The object name to delete from the test bucket.
  thread_state: gsutil Cloud API instance to use for the operation.
N)r   Delete)rC   r>   rE   rF   s       r8   _DeleteWrapperrY     s     #35***[%r7   c                 L    U R                   R                  [        U5      5        gz9Simple exception handler to allow post-completion status.Nloggererrorstr)rC   es     r8   _PerfdiagExceptionHandlerra     s    **3q6r7   c                     g Nr/   )_s    r8   _DummyTrackerCallbackre     s    r7   c                   (    \ rS rSrSrSrS rS rSrg)	DummyFilei  zDA dummy, file-like object that throws away everything written to it.bwc                     g rc   r/   )selfrD   kwargss      r8   writeDummyFile.write      r7   c                     g rc   r/   rj   s    r8   closeDummyFile.close  rn   r7   r/   N)	r1   r2   r3   r4   r5   moderl   rq   r6   r/   r7   r8   rg   rg     s    L 
$		r7   rg   c                    US-  n[         R                  " [        X5      5      nSnXQ:  aW  [        X1U-
  5      n[        USU-
  -  5      nXg-
  nU R	                  USU 5        U R	                  SU-  5        XV-  nXQ:  a  MV  gg)aW  Writes data into a file like object.

Args:
  fp: A file like object to write the data to.
  file_size: The amount of data to write to the file.
  random_ratio: The percentage of randomly generated data to write. This can
      be any number between 0 and 100 (inclusive), with 0 producing uniform
      data, and 100 producing random data.
  max_unique_random_bytes: The maximum number of bytes to generate
                           pseudo-randomly before beginning to repeat
                           bytes. The default was chosen as the next prime
                           larger than 5 MiB.
g      Y@r      N   x)osurandomminintrl   )	fprU   random_ratiomax_unique_random_bytesrandom_bytestotal_bytes_written	num_bytesnum_bytes_seqnum_bytes_randoms	            r8   _GenerateFileDatar     s    $ %,C	CD,'+9L-LMI 	Q%567M 0HH\++,-HHTM!"$ 	'r7   c                   ,   \ rS rSrSr\R                  " S/ SQ\SSSSSS\R                  \R                  /\R                  \R                  " S5      /S	9r\R                  " S/ S
S\0 S9rSrSrSrSrSrSrSrSrSrSr\\\\\\4r\\\4r\\\4r\R>                  RA                  SS\RB                  RD                  RF                  RH                  5      r%\R>                  RM                  SSS5      r'Sr(Sr)Sr*Sr+   S>S jr,S  r-     S?S! jr.S" r/S# r0\1Rd                  S$ 5       r3S% r4S& r5 " S' S(\6Rn                  5      r8S) r9S* r:S+ r; S@S, jr< S@S- jr=S@S. jr>S@S/ jr?S0 r@    SAS2 jrA    SBS3 jrBS4 rCS5 rDS6 rES7 rFS8 rGS9 rHS: rIS; rJS< rKS=rLg1)CPerfDiagCommandi  z*Implementation of gsutil perfdiag command.perfdiag)diag
diagnosticperfperformancer   ru   zn:c:k:p:y:s:d:t:m:i:o:j:F)command_name_aliasesusage_synopsismin_argsmax_argssupported_sub_argsfile_url_okprovider_url_okurls_start_arggs_api_supportgs_default_apiargparse_argumentscommand_helpzRun performance diagnostic)	help_namehelp_name_aliases	help_typehelp_one_line_summary	help_textsubcommand_help_text)r   i   i     rthru
rthru_filewthru
wthru_filelatlistfanslicebothCredentialsgs_hostgs_portP      
   i @  g      N@c                    U R                   R                  SU5        U(       a  [        R                  OSn[        R                  " U[        R                  US9nUR                  5       u  px[        R                  (       a*  [        U[        5      (       a  UR                  [        5      nU(       a:  UR                  (       a)  [        SUR                  SR                  U5      4-  5      eU(       a  U$ UR                  $ )aK  Executes a command in a subprocess.

Args:
  cmd: List containing the command to execute.
  raise_on_error: Whether or not to raise an exception when a process exits
      with a non-zero return code.
  return_output: If set to True, the return value of the function is the
      stdout of the process.
  mute_stderr: If set to True, the stderr of the process is not printed to
      the console.

Returns:
  The return code of the process or the stdout if return_output is set.

Raises:
  Exception: If raise_on_error is set to True and any process exits with a
  non-zero return code.
zRunning command: %sN)stdoutstderrz8Received non-zero return code (%d) from subprocess '%s'. )r]   debug
subprocessPIPEPopencommunicatesixPY3
isinstancebytesdecoder   
returncoder   join)	rj   cmdraise_on_errorreturn_outputmute_stderrr   p
stdoutdatard   s	            r8   _ExecPerfDiagCommand._Exec  s    . 	KK+S1 +Z__FZ__VDAmmoOZ
ww	J	&	&&&t,
!,, 034<<#2OP Q Q&:8ALL8r7   c                     U R                   U R                  -  [        S5      :  a  U R                  R	                  S5        gg)zBOutputs a warning message if a large amount of data is being used.2GiBz2This is a large operation, and could take a while.N)num_objectsthru_filesizer$   r]   inforp   s    r8   _WarnIfLargeData PerfDiagCommand._WarnIfLargeData<  s:    $,,,/CF/KK
kkKL Lr7   c                    [         R                  " SUU R                  SS9u  pg[        R                  " US5       n[        XU5        SSS5        U(       d  U(       ai  [        US5       nU(       a  [        U5      OSnU(       a  [        U5      OSn	U(       a  UR                  5       OSn
[        XU
5      [        U'   SSS5        U R                  R                  U5        U$ ! , (       d  f       N= f! , (       d  f       N<= f)a  Creates a temporary file of the given size and returns its path.

Args:
  file_size: The size of the temporary file to create.
  mem_metadata: If true, store md5 and file size in memory at
                temp_file_dict[fpath].md5, tempfile_data[fpath].file_size.
  mem_data: If true, store the file data in memory at
            temp_file_dict[fpath].data
  prefix: The prefix to use for the temporary file. Defaults to
          gsutil_test_file.
  random_ratio: The percentage of randomly generated data to write. This
      can be any number between 0 and 100 (inclusive), with 0 producing
      uniform data, and 100 producing random data.

Returns:
  The file path of the created temporary file.
z.binF)suffixprefixdirtextwbNrb)tempfilemkstemp	directoryrw   fdopenr   openr    r   readr(   temp_file_dicttemporary_filesadd)rj   rU   mem_metadatamem_datar   r|   fdfpathr{   md5datas              r8   _MakeTempFilePerfDiagCommand._MakeTempFileA  s    .   (.%)^^&+-IB 
2t	|4 
 xt'3KO	8D04$$rwwy$ -id Cu	  	U#L 
	 s   C$*AC5$
C25
Dc                    0 U l         [        5       U l        [        5       U l        SU l        SU l        [        [        5      U l        SU l	        SU l
        U R                  U R                  ;   aF  / U l        U R                   H/  nU R                  USSS9nU R                  R!                  U5        M1     U R                  R#                  U R$                  U R&                  U R(                  U R*                  45      (       Ga:  U R                  SSSS9U l        U R                  R#                  U R$                  U R&                  45      (       aY  U R                  U R.                  SSU R0                  S9U l        [4        R6                  R9                  U R2                  5      U l        U R                  R#                  U R(                  U R*                  45      (       Ga^  / U l        / U l        [A        U RB                  5      nX0R.                  U RD                  -  :  a  U RF                  RI                  SU RD                  [K        U R.                  5      4-  5        U RM                  5         [O        U RD                  5       H{  nU R                  U R.                  SU R0                  S9nU R<                  R!                  U5        U R>                  R!                  [4        R6                  R9                  U5      5        M}     O=[Q        S	[K        U R.                  U RD                  -  5      < S
[K        U5      < S35      e[S        5       U l*        U RF                  RW                  U RY                  5       5        g)z?Performs setup operations needed before diagnostics can be run.r   FT)r   r   i  P )r   r   r|   z)
Creating %d local files each of size %s.)r   r|   z1Not enough free disk space for throughput files: z" of disk space required, but only z available.N)-resultssetr   temporary_objectstotal_requestsrequest_errorsr   rz   error_responses_by_codeconnection_breaksteardown_completedLAT
diag_testslatency_filestest_lat_file_sizesr   appendintersectionRTHRUWTHRU
RTHRU_FILE
WTHRU_FILEtcp_warmup_filer   gzip_compression_ratiomem_thru_file_namerw   pathbasenamemem_thru_object_namethru_file_namesthru_object_namesr   r   r   r]   r   r&   r   r   r   rg   discard_sink	addFilter_PerfdiagFilter)rj   rU   r   free_disk_spacerd   r?   s         r8   _SetUpPerfDiagCommand._SetUpi  s    DL5D UDDD#.s#3D D#D xx4??"d//)""94$"O!!%( 0
 ##	TZZ$//BD D "//=A9= 0 ?d 
	%	%tzz4::&>	?	?"&"4"444	 #5 #6
 %'GG$4$4T5L5L$M! 
	%	%t&H	I	I  "!#(80043C3CCC
++

:!243E3E!FGHI 


!))*a**""!!88 + :I   ''	2""))"''*:*:9*EF + ! !!3!3d6F6F!FG 134 4 "D 	KK$..01r7   c                    U R                   (       d  [        R                  5          U R                   H  n[        R
                  " U5        M     U R                  (       a   [        R                  " U R                  5        U R                   H  nU R                  X R                  5        M      SU l         g! [         a     NBf = f)zDPerforms operations to clean things up after performing diagnostics.TN)r   r   clearr   rw   removedelete_directoryrmdirr   OSErrorr   rX   rF   )rj   r   r>   s      r8   	_TearDownPerfDiagCommand._TearDown  s    ""))E
))E
 *  
((4>>
": //+K1 0"D=  s   AB7 7
CCc              #     #    U R                   R                  SU5        [        R                  " 5       nSv   [        R                  " 5       nX!   R                  XC-
  5        U R                   R                  SU5        g7f)a  A context manager that measures time.

A context manager that prints a status message before and after executing
the inner command and times how long the inner command takes. Keeps track of
the timing, aggregated by the given key.

Args:
  key: The key to insert the timing value into a dictionary bucket.
  bucket: A dictionary to place the timing value in.

Yields:
  For the context manager.
z%s starting...Nz%s done.)r]   r   timer   )rj   keybuckett0t1s        r8   _TimePerfDiagCommand._Time  s[      	KK%s+	B		B
KrwKKZ%s   A=A?c                 B   SnSnSnSnSnU(       dX  [        [        R                  " 5       SU-  -  S-   [        5       5      n U" 5       nU =R                  S-  sl        SnU(       d  MX  U$ ! [	        U R
                  5       Ga  nUS-  nX@R                  :  a!  U R                  R                  S5         SnAU$ [        U[        5      (       a  UR                  S:  ag  U R                  UR                  ==   S-  ss'   U =R                  S-  sl        U =R                  S-  sl        US-  n[        R                  " U5        Oe X0R                   :  a!  U R                  R                  S	5         SnAU$  SnAGNU =R"                  S-  sl         SnAGN8SnAff = f)
zRuns an operation with retry logic.

Args:
  func: The function to run.

Returns:
  True if the operation succeeds, False if aborted.
Fr   N   ru   Tz,Reached maximum total retries. Not retrying.i  z3Reached maximum server error retries. Not retrying.)ry   randomr   r   tuple
exceptionsMAX_TOTAL_RETRIESr]   r   r   r   statusr   r   r  sleepMAX_SERVER_ERROR_RETRIESr   )	rj   funcsuccessserver_error_retriedtotal_retriedi
return_val
next_sleepr`   s	            r8   _RunOperationPerfDiagCommand._RunOperation  s    GM	AJv}}!Q$/!35E5GHj&V
q  g4 ) 4??# &111
++

I
J
   a)**XX_((2a721$1$ A% JJz"!$A$AAKKEG  B
 
 
 A
%
 
 '&s%   A, ,F/F;B7F>FFc                   ^ ^	^
^^^ [        [        5      T R                  S'   [        T R                  5       GH-  nT R
                  R                  SUS-   5        T R                   H  n[        U   m	T R                  R                  5       m[        R                  R                  U5      Tl        T	R                  m
[!        T
5      nT R
                  R                  SX2T5        [#        T5      mU	U
U U4S jnT R%                  U5        U
U U4S jnT R%                  U5      n['        U5      mU
U UU4S jnT R%                  U5        U
U U4S jnT R%                  U5        M     GM0     g	)
zRuns latency tests.latencyz 
Running latency iteration %d...ru   zN
File of size %s located on disk at '%s' being diagnosed in the cloud at '%s'.c            	        > [         R                  " TR                  5      n TR                  ST-  TR                  S   5         TR
                  R                  U TTTR                  S/S9  S S S 5        g ! , (       d  f       g = f)Nz	UPLOAD_%dr(  name)sizeproviderfields)r   BytesIOr   r  r   rF   UploadObjectr,  )io_fp	file_datarU   rj   upload_targets    r8   _Upload1PerfDiagCommand._RunLatencyTests.<locals>._Upload;  si    ++inn-%zz+	14<<	3JKOO(()6.726--17	 ) : LKKs   )A66
Bc                     > TR                  ST -  TR                  S   5         TR                  R                  TR                  TR
                  TR                  / SQS9sS S S 5        $ ! , (       d  f       g = f)NzMETADATA_%dr(  )r*  contentType	mediaLinkr+  r,  r-  )r  r   rF   GetObjectMetadatabucket_namer>   r,  rU   rj   urls   r8   	_Metadata3PerfDiagCommand._RunLatencyTests.<locals>._MetadataF  s\    zz-)3T\\)5LM??44C	 5 E NMMs   <A**
A8c            	        > TR                  ST -  TR                  S   5         TR                  R                  TR                  TR
                  TR                  TR                  TS9  S S S 5        g ! , (       d  f       g = f)NzDOWNLOAD_%dr(  )r,  r@   )r  r   rF   GetObjectMediar:  r>   r   r,  )rU   rj   r@   r<  s   r8   	_Download3PerfDiagCommand._RunLatencyTests.<locals>._DownloadR  sd    zz-)3T\\)5LMOO**!!#5 + 7 NMMs   AA33
Bc                     > TR                  ST -  TR                  S   5         TR                  R                  TR                  TR
                  TR                  S9  S S S 5        g ! , (       d  f       g = f)Nz	DELETE_%dr(  r,  )r  r   rF   DeleteObjectr:  r>   r,  r;  s   r8   _Delete1PerfDiagCommand._RunLatencyTests.<locals>._Delete]  sX    zz+	14<<	3JKOO((),26-- ) A LKKs   :A''
A5N)r   r   r   r   r   r]   r   r   r   
bucket_urlClonerw   r   r   r>   r+  r&    StorageUrlToUploadObjectMetadatar%  r   )rj   r"  r   readable_file_sizer3  r=  download_metadatarA  rF  r1  rU   r@   r2  r<  s   `        @@@@@r8   _RunLatencyTests PerfDiagCommand._RunLatencyTests'  s4    *$/DLL4##$
kk:AEB%%%"5)	oo##%''**51NN	.y90	> 9=	: 	: 	7#	E !..y99:KL	7 	7 	9%	A 	7#k & %r7   c                       \ rS rSrS rSrg)PerfDiagCommand._PerfdiagFilterie  c                 ~    UR                  5       nSU;   =(       d    SU;   =(       d    SU;   =(       d    SU;   (       + $ )NzCopying file:///zCopying gs://zComputing CRCzgsutil -m perfdiag)
getMessage)rj   recordmsgs      r8   filter&PerfDiagCommand._PerfdiagFilter.filterg  sM    c%, L/S2H L"c)L/Cs/JM Mr7   r/   N)r1   r2   r3   r4   rU  r6   r/   r7   r8   r  rP  e  s    Mr7   r  c                 L    U R                   R                  [        U5      5        gr[   r\   )rj   r`   s     r8   ra   )PerfDiagCommand._PerfdiagExceptionHandlerm  s    KKc!fr7   c                 .   / n[        [        U5      5       H1  nU(       a  X6   OSnUR                  [        XU   UXF   5      5        M3     U R	                  [
        U[        S[        U R                  R                  U R                  U R                  S9  g)aQ  Performs a parallel download of multiple objects using the fan strategy.

Args:
  need_to_slice: If True, additionally apply the slice strategy to each
                 object in object_names.
  object_names: A list of object names to be downloaded. Each object must
                already exist in the test bucket.
  file_names: A list, corresponding by index to object_names, of file names
              for downloaded data. If None, discard downloaded data.
  serialization_data: A list, corresponding by index to object_names,
                      of serialization data for each object.
Nr   r   arg_checkerparallel_operations_overrideprocess_countthread_count)r   lenr   r)   ApplyrG   ra   r   ParallelOverrideReasonPERFDIAG	processesthreads)rj   r<   object_names
file_namesr@   rD   r"  r?   s           r8   PerformFannedDownload%PerfDiagCommand.PerformFannedDownloadq  s     D3|$%#-*-4i
kk
=q/9-023 &
 	JJ!#G#%)%@%@%I%Inn\\  #r7   c                 (   U(       a0  [        US5       nUR                  U R                  5        SSS5        [        U R                  U R                  5      n/ n[        U R                  5       HE  nXu-  n[        US-   U-  S-
  U R                  S-
  5      n	UR                  [        XUX5      5        MG     U R                  [        U[        S[        U R                  R                  U R                  U R                   S9  g! , (       d  f       N= f)a  Performs a download of an object using the slice strategy.

Args:
  object_name: The name of the object to download.
  file_name: The name of the file to download data to, or None if data
             should be discarded.
  serialization_data: The serialization data for the object.
abNru   rZ  r[  )r   truncater   r#   
num_slicesr   ry   r   r*   ra  rK   ra   r   rb  rc  rd  re  )
rj   r>   r?   r@   r{   component_sizerD   r"  rI   rJ   s
             r8   r=   %PerfDiagCommand.PerformSlicedDownload  s     	4 B
D&&' !"4#5#5tGND4??#%ja!e/!3T5G5G!5KLh
kk
[5G'34 $ 	JJ!#G#%)%@%@%I%Inn\\  # ! s   D
Dc                    / n[        [        U5      5       H$  nUR                  [        XU   X7   XE5      5        M&     U R	                  [
        U[        S[        U R                  R                  U R                  U R                  S9  g)a
  Performs a parallel upload of multiple files using the fan strategy.

The metadata for file_name should be present in temp_file_dict prior
to calling. Also, the data for file_name should be present in temp_file_dict
if use_file is specified as False.

Args:
  need_to_slice: If True, additionally apply the slice strategy to each
                 file in file_names.
  file_names: A list of file names to be uploaded.
  object_names: A list, corresponding by by index to file_names, of object
                names to upload data to.
  use_file: If true, use disk I/O, otherwise read upload data from memory.
  gzip_encoded: Flag for if the file will be uploaded with the gzip
                transport encoding. If true, a lock is used to limit
                resource usage.
rZ  r[  N)r   r`  r   r+   ra  rR   ra   r   rb  rc  rd  re  )rj   r<   rg  rf  rP   rN   rD   r"  s           r8   PerformFannedUpload#PerfDiagCommand.PerformFannedUpload  s|    . D3z?#
kk
1|!12 $ 	JJ!#G#%)%@%@%I%Inn\\  #r7   c                   ^ ^^^ [        T R                  T R                  5      n[        T R                  5       Vs/ s H  nT[	        U5      -   PM     nn/ n	[        T R                  5       HG  nXv-  n
[        U[        U   R                  U
-
  5      nU	R                  [        XU   UXU5      5        MI      T R                  [        U	[        S[        T R                  R                  T R                   T R"                  S9  / m[        T R                  5       H3  n[$        R&                  R)                  X   S9nTR                  U5        M5     UUUU 4S jnT R+                  U5        T R                  [,        U[        S[        T R                  R                  T R                   T R"                  S9  gs  snf ! T R                  [,        U[        S[        T R                  R                  T R                   T R"                  S9  f = f)a  Performs a parallel upload of a file using the slice strategy.

The metadata for file_name should be present in temp_file_dict prior
to calling. Also, the data from for file_name should be present in
temp_file_dict if use_file is specified as False.

Args:
  file_name: The name of the file to upload.
  object_name: The name of the object to upload to.
  use_file: If true, use disk I/O, otherwise read upload data from memory.
  gsutil_api: CloudApi instance to use for operations in this thread.
  gzip_encoded: Flag for if the file will be uploaded with the gzip
                transport encoding. If true, a semaphore is used to limit
                resource usage.
rZ  r[  )r*  c                     > [         R                  " 5       n TU l        TR                  R                  U l        TR                  TU TR                  S9  g NrD  )apitools_messagesObjectr*  rH  r:  r  ComposeObjectr,  )dst_obj_metadatarF   r>   request_componentsrj   s    r8   _Compose5PerfDiagCommand.PerformSlicedUpload.<locals>._Compose  sN    ,335 +"&//"="=  !3!1*.-- 	! 	9r7   N)r#   r   rm  r   r_   ry   r   r+  r   r,   ra  rV   ra   r   rb  rc  rd  re  rv  ComposeRequestSourceObjectsValueListEntryr%  rY   )rj   r?   r>   rP   rF   rN   rn  r"  component_object_namesrD   component_startsrc_obj_metadatar{  rz  s   ` ` `        @r8   rO   #PerfDiagCommand.PerformSlicedUpload  s   , #4#5#5tGN&+DOO&<&<c!f&<   D4??#*o>))499OKMn
kk
9Q&?*LJK	 $$%
jj


#%I%'+'B'B'K'K||  % T__%!,,HH+. I 0 	 	!!"23	 &9 9 " jj

 
#%I%'+'B'B'K'K||  %WV jj

 
#%I%'+'B'B'K'K||  %s   F69B2F; ;AHc           
      	   U(       a  SOSnU(       a  SOSnU R                   R                  SU< SU R                  < S[        U R                  5      < S35        U R                  5         U R                  U R                  U R                  U R                  S	.U R                  U'   U(       Gak  U R                  nU R                  n/ n[        U R                  5       GH6  nU R                  R                  U R                  U   5        U R                  U R                   ;   aT  U R"                  R%                  U R&                  R(                  U R                  U   S
S/U R&                  R*                  S9nO8U R-                  U R                  U   U R                  U   U R"                  U5      n[.        R0                  " U R                  U   5        [3        U R                  U   S5      R5                  5         UR7                  [9        U5      5        GM9     OU R                  R                  U R:                  5        U R-                  U R<                  U R:                  U R"                  U5      nSnU R:                  /U R                  -  n[9        U5      /U R                  -  n[.        R>                  RA                  U RB                  5      n	U R                  R                  U	5        U R-                  U RB                  XR"                  5        U RE                  XR"                  5        [F        RF                  " 5       n
U R                  S:X  a\  U R                  S:X  aL  [        U R                  5       H2  nU(       a  XG   OSnU RE                  XW   U R"                  UXg   5        M4     OU R                  U RH                  U RJ                  4;   a-  U R                  U RJ                  :H  nU RM                  XUU5        OZU R                  U RN                  :X  a@  [        U R                  5       H'  nU(       a  XG   OSnU RQ                  XW   UXg   5        M)     [F        RF                  " 5       nX-
  nU R                  U R                  -  nX-  nXR                  U   S'   XR                  U   S'   UU R                  U   S'   g)zRuns read throughput tests.read_throughput_fileread_throughputwith file I/O z
Running read throughput tests  ( objects of size ))rU   rd  re  parallelismr+  r7  r-  r,  rk  Nru   	time_tooktotal_bytes_copiedbytes_per_second))r]   r   r   r&   r   r   rd  re  parallel_strategyr   r   r   r   r   r   r   r   rF   r9  rH  r:  schemerQ   rw   unlinkr   rq   r   r   r   r   r   r   r   rA   r  FANBOTHrh  SLICEr=   )rj   rP   	test_namefile_io_stringrg  rf  r@   r"  obj_metadatawarmup_obj_namer  r?   r<   r  r  r  r  s                    r8   _RunReadThruTests!PerfDiagCommand._RunReadThruTests  s   *2&8II(0_bNKK	))+<, 	!" 	 ''^^<<--	DLL ''j++lT%%&!""4#9#9!#<=??doo-::oo))$$Q'k*--	 ; /, T%9%9!%<%)%;%;A%>%-/, 			$&&q)*T!!!$d+113!!">|"LM' '.   !:!:;[[!8!8!%!:!:DOO!)+l j//043C3CCl9,GH ,,- gg&&t';';<O/KK$$oGMM/??3	B~~t||q0T%%&!%-JM4	lot	(+	- '
 
		DHHdii#8	8//499<""=
#5	7!!TZZ/t''(A'/jmT)

$
$\_i%7%:< ) 
BI++d.>.>>)5+4LLK(4FLL012BDLL./r7   c           
         U(       a  SOSnU(       a  SOSnU R                   R                  SU< SU R                  < S[        U R                  5      < S35        U R                  5         U R                  U R                  U R                  U R                  U R                  U R                  S	.U R                  U'   [        R                  R                  U R                  5      nU R                   R#                  U5        U R%                  U R                  X@R&                  5        U(       a  U R(                  nU R*                  nOTU R,                  /U R                  -  n[/        U R                  5       Vs/ s H  opR0                  [3        U5      -   PM     nnU H  nU R                   R#                  U5        M      [4        R4                  " 5       n	U R                  S
:X  aX  U R                  S
:X  aH  [/        U R                  5       H.  nU R%                  XW   Xg   U R&                  UU R                  S9  M0     OU R                  U R6                  U R8                  4;   a7  U R                  U R8                  :H  n
U R;                  U
UUUU R                  S9  OaU R                  U R<                  :X  aG  [/        U R                  5       H.  nU R?                  XW   Xg   UU R&                  U R                  S9  M0     [4        R4                  " 5       nX-
  nU R                  U R                  -  nX-  nXR                  U   S'   XR                  U   S'   XR                  U   S'   gs  snf )zRuns write throughput tests.write_throughput_filewrite_throughputr  r  z 
Running write throughput tests r  r  r  )rU   rd  re  r  gzip_encoded_writesr   ru   rM   r  r  r  N) r]   r   r   r&   r   r   rd  re  r  r  r   r   rw   r   r   r   r   r   rQ   rF   r   r   r   r   r   r_   r  r  r  rq  r  rO   )rj   rP   r  r  r  rg  rf  r"  r>   r  r<   r  r  r  r  s                  r8   _RunWriteThruTests"PerfDiagCommand._RunWriteThruTestss  s   +3'9KI(0_bNKK	))+<, 	!" 	 ''^^<<--#77"&"="=DLL gg&&t';';<O/KK$$oG''j++l ++,t/?/??j6;D<L<L6M6M
#
#c!f
,6M   $
  - $ 
B~~t||q0T%%&!JM OOO!%!9!9	 	 	; ' 
		DHHdii#8	8//499<  !+!-!).2.F.F	 	! 	H
 !!TZZ/t''(A

"
":=#/?#+#'??040H0H	 # J ) 
BI++d.>.>>)5+4LLK(4FLL012BLL./Ms   9!M%c           
        ^ ^^^^^^ ST R                   0T R                  S'   / n/ nSR                  [        S5       Vs/ s H'  n[        R
                  " [        R                  5      PM)     sn5      nSU-   S-   m[        T R                   5       H}  nT R                  SSSTS	9n[        R                  R                  U5      nUR                  U5        UR                  [        S
XVS
S
5      5        T R                  R                  U5        M     T R                   R#                  ST R                   5        T R%                  [&        U[(        [*        S9  / m/ m[,        R,                  " 5       n[/        U5      m[/        5       mUUU 4S jmUUUU4S jnT R                   R#                  ST R0                  R2                  T R                   5        TT-
  (       ai  T R5                  U5        TT-
  (       aB  [,        R,                  " 5       U-
  T R6                  :  a  T R                   R9                  S5        OTT-
  (       a  Mi  [,        R,                  " 5       n	[;        T5      TTX-
  S.T R                  S   S'   U Vs/ s H  ofPM     nnT R                   R#                  ST R                   5        T R%                  [<        U[(        [*        S9  T R                   R#                  ST R0                  R2                  T R                   5        / m/ m[,        R,                  " 5       n[/        U5      mT(       ak  UUU4S jn
T R5                  U
5        T(       aB  [,        R,                  " 5       U-
  T R6                  :  a  T R                   R9                  S5        O	T(       a  Mk  [,        R,                  " 5       n	[;        T5      TTX-
  S.T R                  S   S'   gs  snf s  snf )z0Runs eventual consistency listing latency tests.	num_fileslistingr  r   zgsutil-perfdiag-list--r   T)r   r   r   Fz'
Writing %s objects for listing test...)r\  c            
        > [         R                   " 5       n [        TR                  R                  TR                  R
                  TSTR                  S/S95      n[        U5      TR                  :  a&  TR                  R                  STR                  5        [         R                   " 5       nTR                  X -
  5        [        U Vs/ s H  o3R                  R                  PM     sn5      $ s  snf )z>Lists and returns objects in the bucket. Also records latency./z
items/name)r   	delimiterr,  r-  z5Listing produced more than the expected %d object(s).)r  r   rF   ListObjectsrH  r:  r,  r`  r   r]   warningr   r   r   r*  )r  objectsr  objlist_latencieslist_prefixrj   s       r8   _List,PerfDiagCommand._RunListTests.<locals>._List  s    99;b
//
%
%doo&A&A-803/3}}.:^	 & =>g 
W((	(C	 99;bBG$737C((--73443s   C2c                  p   > T" 5       n TR                  U T-  5        TR                  [        T5      5        g rc   )updater   r`  )namesr  expected_objects
files_seenfound_objectss    r8   _ListAfterUpload7PerfDiagCommand._RunListTests.<locals>._ListAfterUpload  s1    ge5#334M*+r7   z5Listing bucket %s waiting for %s objects to appear...z)Maximum time reached waiting for listing.)num_listing_callsr  files_seen_after_listingr  insertz'Deleting %s objects for listing test...z8Listing bucket %s waiting for %s objects to disappear...c                  j   > T" 5       n TR                  U 5        TR                  [        T5      5        g rc   )intersection_updater   r`  )r  r  r  r  s    r8   _ListAfterDelete7PerfDiagCommand._RunListTests.<locals>._ListAfterDelete  s+    ))%0#m,-r7   deleteN)r   r   r   r   r  choicestringascii_lowercaser   rw   r   r   r   r+   r   r   r]   r   ra  rR   ra   r   r  r   rH  r:  r%  MAX_LISTING_WAIT_TIMEr  r`  rY   )rj   list_objectsrD   rd   	random_idr   r>   total_start_timer  total_end_timer  r  r  r  r  r  r  s   `          @@@@@@r8   _RunListTestsPerfDiagCommand._RunListTests  s^   *D,<,<=DLL LD 8=b	B	1v--	.	BDI)I5;K4##$  *..2(3 ! 5e GG$$U+k+&
kk.E5IJ
  - % 	KK?%%' 	JJ}(*  ,
 NJyy{<(EM5", ,
 	KKL__00$2B2BD
]
*
)*	M	)99;))D,F,FF
++

I
J
 ]
*
* YY[N !0($.#6	)DLLH% ,88<KK<D8KK>%%'JJ~(*  ,
 	KKO__00$2B2BDNJyy{%M
.
 )*	99;))D,F,FF
++

I
J
 - YY[N !0($.#6	)DLLH% 	CF 9s   .O>ONc                   ^ ^^^^^^ SmTc  [         U   R                  mT R                  R                  5       nX(l        [        U5      m U(       a  [        XT5      mO.[         U   R                  XUT-    n	[        R                  " U	5      mUUUUU U4S jmU4S jn
T(       a  U
OTnT R                  U5      T(       a  TR                  5         $ $ ! T(       a  TR                  5         f f = f)aS  Performs an upload to the test bucket.

The file is uploaded to the bucket referred to by self.bucket_url, and has
name object_name.

Args:
  file_name: The path to the local file, and the key to its entry in
             temp_file_dict.
  object_name: The name of the remote object.
  gsutil_api: Cloud API instance to use for this upload.
  use_file: If true, use disk I/O, otherwise read everything from memory.
  file_start: The first byte in the file to upload to the object.
              (only should be specified for sliced uploads)
  file_size: The size of the file to upload.
             (only should be specified for sliced uploads)
  gzip_encoded: Flag for if the file will be uploaded with the gzip
                transport encoding. If true, a lock is used to limit
                resource usage.
Returns:
  Uploaded Object Metadata.
Nc            
         > T [        5       :  a   TR                  TTTR                  T / SQTS9$ TR                  TTTR                  T / SQ[        TS9$ )N)r*  r7  r+  )r,  r+  r-  rN   )r,  r+  r-  tracker_callbackrN   )r   r/  r,  UploadObjectResumablere   )rU   r{   rF   rN   rj   r2  s   r8   _InnerUpload,PerfDiagCommand.Upload.<locals>._InnerUploadX  sq    )++(()626--.70M6B ) D D 11}}24' 2 ) )r7   c                  |   > [         R                  R                     T " 5       sS S S 5        $ ! , (       d  f       g = frc   )gslibcommand!concurrent_compressed_upload_lock)r  s   r8   _InnerUploadLock0PerfDiagCommand.Upload.<locals>._InnerUploadLockm  s!    ]]<< =<<s   -
;)r   r+  rH  rI  r>   rJ  r   r   r   r.  r%  rq   )rj   r?   r>   rF   rP   rT   rU   rN   
upload_urlr   r  upload_funcr  r{   r2  s   `  `  ``    @@@r8   rQ   PerfDiagCommand.Upload,  s    : 
B +00i&&(J(4Z@M$	iY7i(--ji9OP[[) )*  )5$,k,	

 


 
s   A.C C-c                   ^ ^^^^^^ Sm Ub  [        US5      mTR                  T5        OT R                  mUUUUU UU4S jnT R                  U5        T(       a  TR	                  5         gg! T(       a  TR	                  5         f f = f)a  Downloads an object from the test bucket.

Args:
  object_name: The name of the object (in the test bucket) to download.
  gsutil_api: CloudApi instance to use for the download.
  file_name: Optional file name to write downloaded data to. If None,
             downloaded data is discarded immediately.
  serialization_data: Optional serialization data, used so that we don't
                      have to get the metadata before downloading.
  start_byte: The first byte in the object to download.
              (only should be specified for sliced downloads)
  end_byte: The last byte in the object to download.
            (only should be specified for sliced downloads)
Nzr+bc            
      l   > TR                  TR                  R                  TTTR                  TT TS9  g )N)r,  rI   rJ   r@   )r@  rH  r:  r,  )rJ   r{   rF   r>   rj   r@   rI   s   r8   _InnerDownload0PerfDiagCommand.Download.<locals>._InnerDownload  s9    !!$//"="="-"$+/==-7+35G 	" 	Ir7   )r   seekr   r%  rq   )	rj   r>   rF   r?   r@   rI   rJ   r  r{   s	   ``` ``` @r8   rA   PerfDiagCommand.Downloadw  sw    * 
B		)U#

I I (	

 


 
s   A
A. .Bc                 ^   ^ ^^  UUU 4S jnT R                  U5        g! [         a     gf = f)zDeletes an object from the test bucket.

Args:
  object_name: The name of the object to delete.
  gsutil_api: Cloud API instance to use for this delete.
c                  d   > T R                  TR                  R                  TTR                  S9  g ru  )rE  rH  r:  r,  )rF   r>   rj   s   r8   _InnerDelete,PerfDiagCommand.Delete.<locals>._InnerDelete  s,     ; ; +)- 	  	8r7   N)r%  r   )rj   r>   rF   r  s   ``` r8   rX   PerfDiagCommand.Delete  s/    	8
 & 
s    
,,c                 X   Sn/ n[        SS5       nUR                  5       SS nU HB  nUR                  5       u      pgUS   R                  5       (       d  M1  UR	                  U5        MD     SSS5        0 n[        SS5       nU Hu  nUR                  5       SS n	U	u    pgppppXr;   d  M&  [        U5      U-  n[        U5      U-  n[        U
5      n
[        U5      n[        U5      n[        U5      nXXX4X'   Mw     SSS5        U$ ! , (       d  f       N= f! , (       d  f       U$ = f)	a  Retrieves disk I/O statistics for all disks.

Adapted from the psutil module's psutil._pslinux.disk_io_counters:
  http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py

Originally distributed under under a BSD license.
Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola.

Returns:
  A dictionary containing disk names mapped to the disk counters from
  /disk/diskstats.
i   z/proc/partitionsrr  Nz/proc/diskstats   )r   	readlinessplitisdigitr   rz   )rj   sector_size
partitionsflineslinerd   r*  retdictvaluesreadsrbytesrtimewriteswbyteswtimes                   r8   _GetDiskCounters PerfDiagCommand._GetDiskCounters  s)     KJ	 #	&!kkmABe$

1a8


D
!  
' G		%$cr"HNE1EfVv;,&v;,&e*%v;&e*%e*% &%G'-  
& N) 
'	& 
&	% Ns%   AD	D	%D,AD	
D
D)c                     U R                  SS/SSS9nUR	                  5       R                  5       nSnSnSnSnUR                  S5       H  nS	U;   d  S
U;   a  SnU(       a-  Uc*  SU;   d  SU;   d  SU;   a  SR                  S U 5       5      nU(       a'  Uc$  SU;   d  SU;   a  SR                  S U 5       5      nU(       d  M|  Ub  M  SU;   d  SU;   d  SU;   d  M  SR                  S U 5       5      nM     0 n [        U5      US'   [        U5      US'   [        U5      US'   U$ ! [         a    U R                  R                  S5         gf = f! [        [        4 a    SUS'   SUS'   SUS'    U$ f = f)zTries to parse out TCP packet information from netstat output.

Returns:
   A dictionary containing TCP information, or None if netstat is not
   available.
netstat-sTF)r   r   zGnetstat not found on your system; some measurement data will be missingN
ztcp:ztcp statisticszsegments retransmitedzretransmit timeoutszsegments retransmittedr  c              3   P   #    U  H  o[         R                  ;   d  M  Uv   M     g 7frc   r  digits.0cs     r8   	<genexpr>/PerfDiagCommand._GetTcpStats.<locals>.<genexpr>  s      GDq4FD   &	&zsegments receivedzpackets receivedc              3   P   #    U  H  o[         R                  ;   d  M  Uv   M     g 7frc   r   r  s     r8   r  r    s     E$Qv}}2Dqq$r  zsegments send outzpackets sentzsegments sentc              3   P   #    U  H  o[         R                  ;   d  M  Uv   M     g 7frc   r   r  s     r8   r  r    s     Ad6==.@11dr  tcp_retransmittcp_receivedtcp_sent)r   r
  r]   r  striplowerr  r   rz   
ValueError	TypeError)rj   netstat_output	found_tcpr
  r  r  r  results           r8   _GetTcpStatsPerfDiagCommand._GetTcpStats  s   zz9d"30416 " 8n $))+113NINLH$$T* 
4+t3	 ."d*.Ct.K#t+ GD GG ,$&*<*DwwE$EE )($&.D*@d"77AdAA- +0 F !$^!4f"<0f^x=fZ MW  
kk 1 2L 	"  !%f#f^fZM s#   D,  *E ,%EEE:9E:c                    0 n[         R                  [         R                  [         R                  [         R                  4n[
        R                  R                  SSS5      US'   [
        R                  R                  SSS5      n[
        R                  R                  SSS5      n[        U5      US	'   [
        R                  R                  SS
U(       a  SOS5      (       a  U R                  R                  S5         [         R                  " [         R                  " 5       5      US'   U R                  US'   [        R                   " S[        R"                  " 5       5      US'   SSU R$                  /n U R'                  USSS9n[(        R*                  " SU5      nU(       a  UR-                  S5      OSUS'   [2        (       a   [5        SS5       nUR7                  5       S:H  US'   SSS5        UR                  SS5      (       a<  [         R                  " 5       n	SSSSS U	/n [9        5       n
U R'                  USU
S9US!'   U R:                  R=                  U R>                  R@                  S"S#/U R>                  RB                  S$9nURD                  US%'   URF                  US&'    [        R                  " 5       n[         R                  " U R$                  5        [        R                  " 5       nX-
  US''    [         RH                  " U R$                  5      u  pnXS('   / US)'   U H1  n [         RJ                  " U5      u  pnUS)   RM                  U	5        M3      / S*QnU R'                  USSS9n[(        R*                  " S+U5      nU(       a  UR-                  S,5      OSUS-'   0 US.'   U Hj  n [         R                   " 5       n[        R                  " 5       nURO                  UU RP                  45        [        R                  " 5       nX-
  US.   U'   Ml     U(       a  Sn [        R                  " 5       n[         R                  " U5      n[        R                  " 5       nX-
  US/'    [         R                   " 5       n[        R                  " 5       nURO                  U=(       d    UU45        [        R                  " 5       nX-
  US0'    [R        RT                  " 5       US1'    [Y        [Z        R\                  " 5       5      US2'   SnSnSnSn [5        S3S5       nU H  nURa                  S45      (       a&  [c        SRe                  S5 U 5       5      5      S6-  nM?  URa                  S75      (       a&  [c        SRe                  S8 U 5       5      5      S6-  nM{  URa                  S95      (       a&  [c        SRe                  S: U 5       5      5      S6-  nM  URa                  S;5      (       d  M  [c        SRe                  S< U 5       5      5      S6-  nM     SSS5        UUUUS=.US>'   0 US?'   [k        [        5       HY  n[m        [        U5      nURo                  5       (       d  M*  [q        U[r        Rt                  5      (       a  S@U;   a  MQ  UUS?   U'   M[     0 USA'   / SBQnU H\  n [5        US5       nURw                  5       nSSS5        WRy                  5       USA   [Z        Rz                  R}                  U5      '   M^     XR~                  SC'   g! U a	    SUS'    GNf = f! [.        [0        4 a	    SUS'    GNf = f! , (       d  f       GNc= f! [0         a     GNsf = f! [.        [0        4 a	    SUS!'    GN<f = f! U a     GNf = f! U a    / n/ US('    GNf = f! U a     GM  f = f! [.        [0        4 a	    SUS-'    GN$f = f! U a     GM+  f = f! U a     GNyf = f! U a     GN#f = f! [V         a	    SUS1'    GN f = f! [^        [0        4 a	    SUS2'    GNf = f! , (       d  f       GN= f! [f        [h        4 a     GN,f = f! , (       d  f       GN= f! [f         a     GM  f = f)DzCollects system information.Boto	is_secureTboto_https_enabledproxyN
proxy_portr   using_proxy
proxy_rdnsFzDNS lookups are disallowed in this environment, so some information is not included in this perfdiag run. To allow local DNS lookups while using a proxy, set proxy_rdns to False in your boto file.
ip_addressr  tempdir%a, %d %b %Y %H:%M:%S +0000gmt_timestampnslookupz-type=CNAME)r   r   z = (?P<googserv>[^.]+)\.googservgoogserv_routez/sys/class/dmi/id/product_namer  zGoogle Compute Engine
on_gcegcloudcompute	instancesr   z	--filter=gce_instance_infolocationstorageClassr  bucket_locationbucket_storageClassgoogle_host_dns_latencygoogserv_ipsgoogserv_hostnames)r"  z	-type=TXTzo-o.myaddr.google.com.ztext\s+=\s+"(?P<dnsip>[\.\d]+)"dnsip
dns_o-o_ipgoogle_host_connect_latenciesproxy_dns_latencyproxy_host_connect_latency	cpu_countload_avgz/proc/meminfoMemTotalc              3   P   #    U  H  o[         R                  ;   d  M  Uv   M     g 7frc   r   r  s     r8   r  2PerfDiagCommand._CollectSysInfo.<locals>.<genexpr>  s     $K1V]]8JQQr    MemFreec              3   P   #    U  H  o[         R                  ;   d  M  Uv   M     g 7frc   r   r  s     r8   r  r:    s     #Jt!FMM7IAAtr  Buffersc              3   P   #    U  H  o[         R                  ;   d  M  Uv   M     g 7frc   r   r  s     r8   r  r:    s     &M$Qv}}:Lqq$r  Cachedc              3   P   #    U  H  o[         R                  ;   d  M  Uv   M     g 7frc   r   r  s     r8   r  r:    s     %LAfmm9Kaar  )	mem_totalmem_freemem_buffers
mem_cachedmeminfogsutil_configr  tcp_proc_values)z/proc/sys/net/core/rmem_defaultz/proc/sys/net/core/rmem_maxz/proc/sys/net/core/wmem_defaultz/proc/sys/net/core/wmem_maxz!/proc/sys/net/ipv4/tcp_timestampsz/proc/sys/net/ipv4/tcp_sackz%/proc/sys/net/ipv4/tcp_window_scalingsysinfo)@socketr^   herrorgaierrortimeoutbotor   getgetintboolr]   r   gethostbynamegethostnamer   r  strftimegmtimeXML_API_HOSTr   researchgroupr   r
  r!   r   readliner"   rF   	GetBucketrH  r:  r  r*  r+  gethostbyname_exgethostbyaddrr   connectXML_API_PORTmultiprocessingr6  NotImplementedErrorr   rw   
getloadavgAttributeError
startswithrz   r   IOErrorr  r   getattrisupperr   r   string_typesr   r  r   r   r   )rj   rI  socket_errors
proxy_hostr  r   nslookup_cname_outputmr{   hostnamer   bucket_infor  r  rd   
ipaddrlistgoogserv_ipnslookup_txt_outputsockproxy_iprB  rC  rD  rE  r  r  attr
attr_valuestats_to_checkfnamevalues                                  r8   _CollectSysInfoPerfDiagCommand._CollectSysInfo  s   G \\6==&//^^%M %)KKOOFK$NG ! $7J##FL!<J!*-GM{{v|ZTUKK
kkLM!$2263E3E3GHgl
 GI  $}}-J-1[[] <GO
 }d&7&7
8C%"jj7;59 ) ; ))/1F
Ga9:!''*"5g x2C8B!{{}0II'(
 9
 
Xu	%	%%%'KhO	, 12+*.**SCGAL +5 +N'%
& //++DOO,G,G4>3O59__5K5K , MK "-!5!5G%0%=%=G!"
99;b4,,-99;b+-7g'(
#"("9"9$:K:K"LxJ *n %'G !!$*$8$8$E!j$%,,X6 "!?c JJs5937 ' 9 ))68K
La23aggg.gl 02G+,!}}YY[k4#4#456YY[@B/0= " hYY[''
3YY[')w#$}}YY[h,*j9:YY[02,-
",668gk
! 1gj
 IHKJ%D__Z((RWW$K$KKLIy))BGG#Jt#JJKHy))rww&M$&MMN Kx((bgg%L%LLMJ  &$ " 	GI  "GOF64(j	J8H8H!I!I!%!3)3 &  "$GN  %&&(% >Ckkm!"277#3#3E#:;	   &LLI  ! gl!& g& %"$g% 98  !'* 	,)+'%
&	,&  
  #j "gn#   g& ! gl!        "!gk" G$ ! gj! &% Z  
>   so   ,\ 8A\, 	] ],] )]+ +A^ ='^ /-^(!A	^6 5A&_)A_  1A"_- _: -!` `> #C`,5(`,`> /a';a;a'\)(\),]]
]] ] 
]('](+^^^^^%$^%(^32^36____ _*)_*-_76_7:```)(`),
`;6`> ;`> >aa
a$	a''
a65a6c                   ^ [        U5      n[        [        U5      5      U-  m[        R                  " [        U4S jU 5       5      U-  5      n[
        R                  " [        U5      R                  S5      SSS9  [
        R                  " STS-  -  R                  S5      SSS9  [
        R                  " SUS-  -  R                  S	5      SSS9  [
        R                  " S[        US
5      S-  -  R                  S5      SSS9  [
        R                  " S[        US5      S-  -  R                  S5      S5        g)zAPrints out mean, standard deviation, median, and 90th percentile.c              3   2   >#    U  H  oT-
  S -  v   M     g7f)r  Nr/   )r  xmeans     r8   r  0PerfDiagCommand._DisplayStats.<locals>.<genexpr>  s     8At8a-s      r  r   endz%.1fr;  	      g      ?r  g?N)
r`  floatsummathsqrtr   print_to_fdr_   rjustr'   )rj   trialsnstdevr~  s       @r8   _DisplayStatsPerfDiagCommand._DisplayStats  s
   FAV!DIIc8881<=E#a&,,q/2376TD[188;RSI6UT\299"=rsK6Z%<t%CDKKBO!# 6Z%<t%CDKKBOr7   c                 %   [         R                  " 5         [         R                  " S5        [         R                  " SR                  S5      5        [         R                  " S5        SU R                  ;   Ga  [         R                  " 5         [         R                  " S5        [         R                  " SR                  S5      5        [         R                  " S5        [         R                  " S5        [         R                  " S5        [	        U R                  S   5       GH  n[	        U R                  S   U   5      nUR                  S	5      u  p4[        U5      nUS
:X  ad  [         R                  " SR                  S5      SSS9  [         R                  " [        U5      R                  S5      SSS9  U R                  U5        US:X  ad  [         R                  " SR                  S5      SSS9  [         R                  " [        U5      R                  S5      SSS9  U R                  U5        US:X  ad  [         R                  " SR                  S5      SSS9  [         R                  " [        U5      R                  S5      SSS9  U R                  U5        US:X  d  GM  [         R                  " SR                  S5      SSS9  [         R                  " [        U5      R                  S5      SSS9  U R                  U5        GM     SU R                  ;   Ga  [         R                  " 5         [         R                  " S5        [         R                  " SR                  S5      5        [         R                  " S5        U R                  S   n[         R                  " SU R                  < S[        US   5      < S[        US   5      < S35        [         R                  " S[        US   S-  5      -  5        S U;   a  [         R                  " S!US    -  5        S"U R                  ;   Ga  [         R                  " 5         [         R                  " S5        [         R                  " S#R                  S5      5        [         R                  " S5        U R                  S"   n[         R                  " SU R                  < S[        US   5      < S[        US   5      < S35        [         R                  " S[        US   S-  5      -  5        S U;   a  [         R                  " S!US    -  5        S$U R                  ;   Ga  [         R                  " 5         [         R                  " S5        [         R                  " S%R                  S5      5        [         R                  " S5        U R                  S$   n[         R                  " SU R                  < S[        US   5      < S[        US   5      < S35        [         R                  " S&[        US   S-  5      -  5        S U;   a  [         R                  " S!US    -  5        S'U R                  ;   Ga  [         R                  " 5         [         R                  " S5        [         R                  " S(R                  S5      5        [         R                  " S5        U R                  S'   n[         R                  " SU R                  < S[        US   5      < S[        US   5      < S35        [         R                  " S&[        US   S-  5      -  5        S U;   a  [         R                  " S!US    -  5        S)U R                  ;   Ga  [         R                  " 5         [         R                  " S5        [         R                  " S*R                  S5      5        [         R                  " S5        U R                  S)   n	U	S+   n
U	S,   n[         R                  " S-U	S.   -  5        [         R                  " S/U
S0   -  5        [         R                  " S1U
S2   -  5        [         R                  " S3S4R                  S5 U
S6    5       5      -  5        [         R                  " S7S4R                  [        [        U
S8   5      5      -  5        [         R                  " S9U	S.   -  5        [         R                  " S/US0   -  5        [         R                  " S1US2   -  5        [         R                  " S3S4R                  S: US6    5       5      -  5        [         R                  " S7S4R                  [        [        US8   5      5      -  5        S;U R                  ;   Ga  [         R                  " 5         [         R                  " S5        [         R                  " S<R                  S5      5        [         R                  " S5        U R                  S;   n[         R                  " S=US>   -  5        [         R                  " S?US@   -  5        [         R                  " SAU R                  SB   -  5        [         R                  " SCU R                  R                  SDSE5      -  5        [         R                  " SFU R                  R                  SGSE5      -  5        SHU;   a  USH   nSIn [         R"                  " USJ5      nU(       a]  [&        R(                  " U5      n[*        R*                  R-                  U5      n[         R                  " SKUR/                  SL5      -  5        SMU;   aR  [         R                  " SNUSM   -  5        USM   (       a,  [         R                  " SOUSP   R1                  SQSR5      -  5        [         R                  " SSUST   -  5        [         R                  " SUUSV   -  5        [         R                  " SWUSX   -  5        [         R                  " SYSZR                  US[   5      -  5        [         R                  " S\SZR                  US]   5      -  5        [         R                  " S^US_   -  5        [         R                  " S`USa   -  5        [         R                  " SbUSc   -  5         [         R                  " Sd[        USe   Sf   5      -  5        [         R                  " Sg[        USe   Sh   USe   Si   -   USe   Sj   -   5      -  5        SkU;   aG  SlU;   aA  USk   nUSl   nSm H0  n USnU-     USnU-     -
  n[         R                  " SoUU4-  5        M2     O[         R                  " Sp5        SqU;   GaE  SrU;   Ga>  [         R                  " SsSS9  USq   nUSr   n[         R                  " SStR                  Su5      SS9  Sv H'  n[         R                  " UR                  S5      SS9  M)     [         R                  " 5         [	        U5       H  nUU   nUU   nUu  nnnnnn Uu  n!n"n#n$n%n&[         R                  " SUR                  Su5      SS9  U!U-
  U"U-
  U#U-
  U$U-
  U%U-
  U&U -
  /n'U' H0  n[         R                  " [        U5      R                  S5      SS9  M2     [         R                  " 5         M     SwU;   aN  [         R                  " SxSS9  [4        R6                  " USw   5       H  n([         R                  " SyU(-  5        M     SzU;   a  [         R                  " S{USz   -  5        S|U;   a  [         R                  " S}US|   -  5        S~U;   a  [         R                  " SUS~   S-  -  5        SU;   aW  [         R                  " S5        [4        R6                  " US   5       H$  u  n)n*[         R                  " SU)U*S-  4-  5        M&     SU;   a  [         R                  " SUS   S-  -  5        SU;   a  [         R                  " SUS   S-  -  5        SU R                  ;   Ga  SU R                  ;   Ga  [         R                  " 5         [         R                  " S5        [         R                  " SR                  S5      5        [         R                  " S5        [        U R                  S   5      n+[        U R                  S   5      n,[        U R                  S   5      n-U+S:  a  U+U,-
  [9        U+5      -  S-  OSn.[         R                  " SU+-  5        [         R                  " SU,-  5        [         R                  " SU--  5        [         R                  " SU.-  5        SU R                  ;   au  [	        [4        R6                  " U R                  S   5      5      n/U/(       aB  [         R                  " S5        [         R                  " SQR                  S U/ 5       5      5        U R:                  (       aw  [=        U R:                  S5       n0[>        R@                  " U R                  U0SS9  SISISI5        [         R                  " 5         [         R                  " SU R:                  -  5        [         R                  " 5         gI! [$         a     GNf = f! [2         a     GNf = f! [2         a     GM  f = f! , (       d  f       N= f)z/Displays results collected from diagnostic run.zN==============================================================================zDIAGNOSTIC RESULTSN   r(  zN------------------------------------------------------------------------------LatencyzOOperation       Size  Trials  Mean (ms)  Std Dev (ms)  Median (ms)  90th % (ms)zO=========  =========  ======  =========  ============  ===========  ===========rd   METADATAMetadatar  r  r   r  DOWNLOADrA   UPLOADrQ   DELETErX   r  zWrite ThroughputzCopied rU   z& file(s) for a total transfer size of r  .zWrite throughput: %s/s.r     r  zParallelism strategy: %sr  zWrite Throughput With File I/Or  zRead ThroughputzRead throughput: %s/s.r  zRead Throughput With File I/Or  Listingr  r  zAfter inserting %s objects:r  z0  Total time for objects to appear: %.2g secondsr  z"  Number of listing calls made: %sr  z)  Individual listing call latencies: [%s], c              3   ,   #    U  H
  nS U-  v   M     g7fz%.2gsNr/   r  r   s     r8   r  2PerfDiagCommand._DisplayResults.<locals>.<genexpr>       G.FsWs].F   r  z'  Files reflected after each call: [%s]r  zAfter deleting %s objects:c              3   ,   #    U  H
  nS U-  v   M     g7fr  r/   r  s     r8   r  r    r  r  rI  zSystem InformationzIP Address: 
  %sr  zTemporary Directory: 
  %sr  zBucket URI: 
  %s
bucket_urizgsutil Version: 
  %sgsutil_versionUnknownzboto Version: 
  %sboto_versionr!  Nr   zMeasurement time: 
 %sz%Y-%m-%d %I:%M:%S %p %Zr%  zRunning on GCE: 
  %szGCE Instance:
	%sr)  r  z
	zBucket location: 
  %sr,  zBucket storage class: 
  %sr-  zGoogle Server: 
  %sr$  z!Google Server IP Addresses: 
  %sz
  r/  zGoogle Server Hostnames: 
  %sr0  z#Google DNS thinks your IP is: 
  %sr2  zCPU Count: 
  %sr6  zCPU Load Average: 
  %sr7  zTotal Memory: 
  %srF  rB  zFree Memory: 
  %srC  rD  rE  netstat_endnetstat_start)sentreceived
retransmitztcp_%sz!TCP segments %s during test:
  %dzQTCP segment counts not available because "netstat" was not found during test runsdisk_counters_enddisk_counters_startzDisk Counter Deltas:
diskr  )r  r  r  r  r  r  rH  zTCP /proc values:
z
   %s = %sr  zBoto HTTPS Enabled: 
  %sr  z$Requests routed through proxy: 
  %sr.  zALatency of the DNS lookup for Google Storage server (ms): 
  %.1fg     @@r3  z7Latencies connecting to Google Storage server IPs (ms):z  %s = %.1fr4  z@Latency of the DNS lookup for the configured proxy (ms): 
  %.1fr5  z8Latency connecting to the configured proxy (ms): 
  %.1fr   r   zIn-Process HTTP Statisticsr   r   d   zTotal HTTP requests made: %dzHTTP 5xx errors: %dzHTTP connections broken: %dzAvailability: %.7g%%r   zError responses by code:c              3   ,   #    U  H
  nS U-  v   M     g7f)z  %s: %sNr/   r  s     r8   r  r  2  s     )O,Q*q.,r  wr  )indentzOutput file written to '%s'.)!r   r  centerr   sortedr  rz   r  r&   r  r   r%   r   mapr_   rO  r  strptimer  calendartimegmdatetimefromtimestamprT  replacer  r   	iteritemsr  output_filer   jsondump)1rj   r  r  opnumbytes
write_thruwrite_thru_file	read_thruread_thru_filer  r  r  r   	ts_string	timetuple	localtimelocaldtnetstat_afternetstat_beforetcp_typedelta
disk_afterdisk_beforecolnamedisknamebeforeafterreads1writes1rbytes1wbytes1rtime1wtime1reads2writes2rbytes2wbytes2rtime2wtime2deltasitemipr(  total	numerrors	numbreaksavailabilitysorted_codesr  s1                                                    r8   _DisplayResultsPerfDiagCommand._DisplayResults  s   (#.55b9:(#DLL H%I,,R01H%%& %& Y/0#Y/45yy~x=



 0 0 3RS
A


 1( ; A A! D "$') 

V
$



 0 0 3RS
A


 1( ; A A! D "$') 

V
$>


q 123
?


 1( ; A A! D "$') 

V
$>


q 123
?


 1( ; A A! D "$') 

V
$7 1: T\\)H%.55b9:H%<< 23j.z+/FGZ(<=>@A 
# ,>!?!!CDFG 
*	$8(78 	9 $,,.H%<CCBGHH%%<=o.{/KL_-ABCEF 
# 1C!Dq!HIKL 
/	)8-m<= 	> DLL(H%-44R89H%,,01i.y/EFY';<=?@ 
" +=!>!BCEF 
)	#8'67 	8 -H%;BB2FGH%||$:;n.~k/JK^,@ABDE 
" 0B!Ca!GHJK 
.	(8,];< 	= DLL H%I,,R01H%Y'gx fx f9#K01 2=+! @"#678 9699Gf5E.FGGHJ 499Sf%?@ABCE 87;;OOP=+! @"#678 9699Gf5E.FGGHJ 499Sf%?@ABCE DLL H%077;<H%\\)$d043EEF9DOKL04<<3MMN4 LL,,-=yIJ K2 LL,,^YGH I 
D	 )			mmI/LM) ooi0)%%33I>'


 9 ' 0 01J K!L M 
T	6hGH>


 5 $%8 9 A A$ O!P Q5=N8OOP: !678 93d;K6LLM@#[[n)=>@ A=#[[.B)CDF GB ./ 0/${2CCD6j9IIJ	50i1MN O 	Q 	3/Y
0K04Y0N1O04Y0M1N OO 	P 
$	?d#:]+o.:H"8h#67#Hx$789E!!"F#+U"3#4 5	 ; 	%	& 
	$)>$)F6C@-.
01b&,,q/s;
G 

a 0c
:
 	z*Hx(&X&%@F
=67GWff@E
=67GWff


HNN1$53
?vow0'G2C&&6/& e!!#e*"2"21"53? 



! + 
d	"3=MM$'8"9:D


t 3
4 ; 
	%:"#789 	: 
$	E"=12 	3 
#d	* 9:VCE	G 
)D	0E	G==.M)NOKB


Wv5E0F F
G P 
	$ 34v=?	A 
&	-H/069;	= 4<<',<,LH%8??CDH%$,,/01edll#345idll#678i"QY 	)U5\9,/ :UBC1I=>9IEF2\AB	"dll	2MM$,,'@ABD


 :
;


		)O,)O O
P  #&!		$,,!, ':T=M=MMNW  	
	F    n '&sV   	AJ A"AJ ,AJ%H!AJ7J
AJJAJJ
AJ"J!AJ"J%
AJ4J3AJ4J7
AKc                 r     [        U5      nUS:  a  [        U5      eU$ ! [         a    [        U5      ef = f)a>  Tries to convert val argument to a positive integer.

Args:
  val: The value (as a string) to convert to a positive integer.
  msg: The error message to place in the CommandException on an error.

Returns:
  A valid positive integer.

Raises:
  CommandException: If the supplied value is not a valid positive integer.
ru   )rz   r   r  )rj   valrT  s      r8   _ParsePositiveInteger%PerfDiagCommand._ParsePositiveInteger<  sA    "Hc	qs##j "S!!"s     6c                    SU l         SU l        SU l        SU l        SU l        SU l        [        R                  " 5       U l        SU l	        [        U R                  5      U l        SU l        SU l        0 U l        SU l        SU l        U R$                  (       GaR  U R$                   GHA  u  pUS:X  a  U R'                  US	5      U l         US
:X  a  U R'                  US5      U l        US:X  a  U R'                  US5      U l        US:X  aB  UR)                  5       U R*                  ;   a  UR)                  5       U l        O[-        SU-  5      eUS:X  a  U R'                  US5      U l        US:X  a   [/        U5      U l        US:X  a[  X l        [2        R4                  R7                  U R                  5      (       d'  SU l	        [2        R8                  " U R                  5        US:X  a|  [        5       U l        UR;                  5       R=                  S5       HJ  nUR)                  5       U R>                  ;  a  [-        SU-  5      eU R                  RA                  U5        ML     US:X  a@  UR=                  S5      n[C        U5      S:w  a  [-        SU-  5      eUu  pVX`R                  U'   US:X  a$  [2        R4                  RE                  U5      U l        US:X  a  [2        R4                  RE                  U5      U l        [2        R4                  RG                  U R                  5      (       d  [-        SU-  5      e [I        U R                  S 5       n[J        RL                  " U5      U l'        U RP                  RS                  S!U R                  5        SSS5          gUS#:X  d  GM  SU l         [U        U5      U l        U R"                  S%:  d  U R"                  S:  d  GM9  [-        S&5      e   U R                  S:  d  U R                  S:  a#  U R                  (       d  U RV                  U l        O<U R                  S:X  a,  U R                  S:X  a  U R                  (       a  [-        S'5      eU RX                  (       d  U R[                  5         []        U RX                  S%   5      U l/        U R^                  R`                  U l1        U R^                  Re                  5       (       d:  U R^                  Rg                  5       (       a  [-        S(U RX                  S%   -  5      eU R
                  [/        S)5      :  a?  U Rh                  U R                  ;   d  U Rj                  U R                  ;   a  [-        S*5      eU R                  U Rl                  U Rn                  4;   nU Rb                  S+:H  =(       a0    U R                  Rq                  U Rj                  U Rr                  5      n	U(       a  U	(       a  [-        S,5      eU Rt                  Rw                  U R^                  Rx                  U R^                  R`                  S-/S.9  [z        R|                  [~        R                  [~        R                  [~        R                  [z        R                  [        /U lE        g! [0         a    [-        S5      ef = f! , (       d  f         g= f! [0         a    [-        S"U-  5      ef = f! [0         a    S$U l         GN4f = f)/z&Parses arguments for perfdiag command.r   ru   N   r   Fr  z-nz,The -n parameter must be a positive integer.z-cz,The -c parameter must be a positive integer.z-kz,The -k parameter must be a positive integer.z-pz)'%s' is not a valid parallelism strategy.z-yz,The -y parameter must be a positive integer.r  zInvalid -s parameter.z-dTz-t,z8List of test names (-t) contains invalid test name '%s'.z-m:r  z,Invalid metadata key-value combination '%s'.z-oz-izInvalid input file (-i): '%s'.r  zRead input file: '%s'.z'Could not decode input file (-i): '%s'.z-jr  r   z7The -j parameter must be between 0 and 100 (inclusive).zrCannot specify parallelism strategy (-p) without also specifying multiple threads and/or processes (-c and/or -k).zOThe perfdiag command requires a URL that specifies a bucket.
"%s" is not valid.r   z~For in-memory tests maximum file size is 2GiB. For larger file sizes, specify rthru_file and/or wthru_file with the -t option.s3zQSliced uploads are not available for s3. Use -p fan or sequential uploads for s3.idr8  )Fr   rd  re  r  rm  r   r   
gettempdirr   r  r   DEFAULT_DIAG_TESTSr   r  
input_filemetadata_keysr  r   sub_optsr  r  PARALLEL_STRATEGIESr   r$   r  rw   r   existsmakedirsr  r  ALL_DIAG_TESTSr   r`  abspathisfiler   r  loadr   r]   r   rz   r  rD   $RaiseWrongNumberOfArgumentsExceptionr   rH  r  r,  
IsCloudUrlIsBucketr   r   r  r  r   r   rF   r[  r:  r
   HTTPExceptionrJ  r^   rL  rM  BadStatusLiner   r  )
rj   oar  piecesr  rx  r  perform_sliceslice_not_availables
             r8   
_ParseArgsPerfDiagCommand._ParseArgsQ  s    DDNDL!DDO D((*DN!D$112DODDOD$D"%D}}}--$!9!77?A$
955?A$.933?A$,9WWY$222%&WWYD""#N#$$% & &9 66?A$/9<!5a!8D 9.//$(D!KK'9E$/779??3/i (;(;;$ &79B&C D DOO	*	 0
 9773<&[A">BD D*#$)

S
!9WW__Q/$
9GGOOA.$/00"#Ca#GHH&doos+q!YYq\dlkk7I , 9%)$
"-*-a&D' ))A-))C/"IK KC  J 	dllQ.8N8N#xxd	1	!2t7M7M>? ? 99
//1*499Q<8DOOO**DM??%%''DOO,D,D,F,F &(,		!5 6 6 	1&99	t	&$***GLM M **tzz499.EEM==D0 I??77

8<I  , H I I 	OOdoo99'+'='=&*V  - 	!!6<<113CDOe  <"#:;;<8 ,+   &"#L#$$% & &&  -*,D'-sI   0Z:[AZ/[-[Z,/
Z>	9[>[[[21[2c                    U R                  5         U R                  (       a  U R                  5         g[        R                  R                  SSS5        U R                  R                  SU R                  U R                  U R                  U R                  U R                  [        U R                  5      SR                  U R                   5      U R"                  U R$                  5
         U R'                  5         U R)                  5         U R+                  5       nU(       a  XR,                  S   S'   [.        (       a  [1        5       U R,                  S   S	'   [3        U R                  5      U R,                  S
'   SU R,                  S'   U R4                  U R,                  S'   U R6                  U R                   ;   a  U R9                  5         U R:                  U R                   ;   a  U R=                  5         U R>                  U R                   ;   a  U RA                  SS9  U RB                  U R                   ;   a  U R=                  SS9  U RD                  U R                   ;   a  U RA                  5         U RF                  U R                   ;   a  U RI                  5         U R+                  5       nU(       a  XR,                  S   S'   [.        (       a  [1        5       U R,                  S   S'   U RJ                  U R,                  S'   U RL                  U R,                  S'   U RN                  U R,                  S'   U RP                  U R,                  S'   [R        RT                  U R,                  S'   [        RV                  U R,                  S'   U RY                  5         U R                  5         U RY                  5         g! U RY                  5         f = f)z3Called by gsutil when the command is being invoked.r   r  num_retries0zNumber of iterations to run: %d
Base bucket URI: %s
Number of processes: %d
Number of threads: %d
Parallelism strategy: %s
Throughput file size: %s
Diagnostics to run: %s
Gzip compression ratio: %s
Gzip transport encoding writes: %sr  rI  r  r  r  r   json_formatmetadataT)rP   r  r  r   r   r   r   r  r  )-r
  r  r  rN  r   r   r]   r   r   rH  rd  re  r  r&   r   r   r   r   r  r  ry  r  r   r!   r   r_   r  r   rM  r   r  r   r  r   r   LISTr  r   r   r   r   r  VERSION__version__r  )rj   r  s     r8   
RunCommandPerfDiagCommand.RunCommand  s   OO
 	KKOOFM3/KK	- /3.>.>d&<&<$,,-		$//0J##T%=%=?3
kkm ((*n	3AY0	9H9JY 56#&t#7dll< $.dll=!!%!3!3dll:	T__	$	t	&  
DOO	+.	DOO	+-	t	&!	doo	% ((*n	1?Y.	7F7HY 34'+':':dll#$'+':':dll#$040L0Ldll,-*.*@*@dll&'',}}dll#$%)%5%5dll>"
nn

 nn nns   +J<N8 8O
) rH  r   r  r   r   r   r   r  r   r  r  r   r   r   r  r   rm  r  r  rd  r,  r   r   r   r   r   r   re  r   r   r   r   )TFF)r   FFgsutil_test_filer  )F)Fr   NF)NNr   N)Mr1   r2   r3   r4   r5   r   CreateCommandSpec	_SYNOPSISr   XMLJSONr   MakeNCloudBucketURLsArgumentcommand_specHelpSpec_DETAILED_HELP_TEXT	help_specr   r   r   r   r   r   r  r  r  r  r  r  r  rN  r   rO  gs
connectionGSConnectionDefaultHostrV  rP  r_  r  r  KEY_BUFFER_SIZEr  r   r   r   r  r  
contextlibcontextmanagerr  r%  rM  loggingFilterr  ra   rh  r=   rq  rO   r  r  r  rQ   rA   rX   r  r  ry  r  r  r  r
  r  r6   r/   r7   r8   r   r     si   2 **H3!oo{'7'78 %%)FFqIJL, 8#) %*
%*#	$ 	#
%	$ :uj#tD. uc* eT* 	!%!3!3!@!@!L!LN, ##M9bA,   /   	!9FM  !&"-!$&PN2`(#T & &*,\<$|M M#:#D (-##T (-I%VVCpFCPoj I\ "&)V$'R7r]&~vp	"*JZOr7   r   c                     U R                  5       (       aN  U R                  5       (       a9  [        R                  " 5       nU R                  Ul        U R                  Ul        U$ [        SU -  5      e)NzENon-cloud URL upload target %s was created in perfdiag implemenation.)	r  IsObjectrv  rw  r>   r*  r:  r  r   )storage_urlr2  s     r8   rJ  rJ  0	  sl    +"6"6"8"8%,,.M$00M&22M
 57BC D Dr7   rc   )r   r  i P )dr5   
__future__r   r   r   r   r  collectionsr   r   r%  r  r  r'  r  r`  rw   r  rW  rJ  r  r   r   r  rN  boto.gs.connectionr   	six.movesr	   r
   r   r  gslib.cloud_apir   r   gslib.commandr   r   gslib.command_argumentr   gslib.commandsr   gslib.cs_api_mapr   gslib.exceptionr   gslib.file_partr   gslib.storage_urlr   "gslib.third_party.storage_apitoolsr   rv  gslib.utilsr   gslib.utils.boto_utilr   r   gslib.utils.constantsr   gslib.utils.cloud_api_helperr   r   gslib.utils.hashing_helperr   gslib.utils.system_utilr   r   r    r!   r"   gslib.utils.unit_utilr#   r$   r%   r&   r'   r  r  r(   r)   r*   r+   r,   r   	Exceptionr.   r:   rG   rK   rR   rV   rY   ra   re   objectrg   r   r   rJ  r/   r7   r8   <module>rB     s   , & %  '  # "       	  	        
  !    - , ! ) 2 ! ( , $ 2 W ! 2 4 & < E I 2 3 / , < / 6 7 3 ,	wy v ?O<
 <>   BD  ?A FH  I 
e 
*"
H/.-$	&
	 	 !"#&.5%Hcg cL;Dr7   