
    #                        S r SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
Jr  SSK
Jr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  S rS r\R:                  " \R<                  R>                  \R<                  R@                  5       " S S\RB                  5      5       r"SSS.\"l#        g)zFCommand to send SSH commands into a Cloud TPU Queued Resource's Nodes.    )absolute_import)division)unicode_literalsN)base)
exceptions)
completers)flags)	ssh_utils)ssh)util)log)
propertiesc                 d    U R                  SS9nUR                  SSSS9  UR                  SSS9  g	)
z.Argument group for running commands using SSH.z3These arguments are used to run commands using SSH.)helpz	--commandTa/        Command to run on the Cloud TPU VM.

      Runs the command on the target Cloud TPU Queued Resource's nodes and then exits.

      Note: in the case of a TPU Pod, it will only run the command in the
      workers specified with the `--worker` flag (defaults to worker all if not
      set).
      )requiredr   z--output-directorya(        Path to the directory to output the logs of the commands.

      The path can be relative or absolute. The directory must already exist.

      If not specified, standard output will be used.

      The logs will be written in files named {WORKER_ID}.log. For example:
      "2.log".
      N)add_argument_groupadd_argument)parsercommand_groups     0lib/surface/compute/tpus/queued_resources/ssh.pyAddCommandArgGroupr   &   sW    ++@ , - 
   	
      c                     U R                  SSSS9  U R                  S[        R                  SSS9  U R                  S	[        R                  S
SS9  U R                  SSSS9  g)z>Additional flags and positional args to be passed to *ssh(1)*.z
--ssh-flagappenda        Additional flags to be passed to *ssh(1)*. It is recommended that flags
      be passed using an assignment operator and quotes. Example:

        $ {command} example-instance --zone=us-central1-a --ssh-flag="-vvv" --ssh-flag="-L 80:localhost:80"

      This flag will replace occurences of ``%USER%'' and ``%TPU%'' with
      their dereferenced values. For example, passing ``80:%TPU%:80`` into
      the flag is equivalent to passing ``80:162.222.181.197:80'' to *ssh(1)*
      if the external IP address of 'example-instance' is 162.222.181.197.

      If connecting to the instance's external IP, then %TPU% is replaced
      with that, otherwise it is replaced with the internal IP.
      )actionr   user_queued_resourcez	[USER@]QRa        Specifies the Cloud TPU Queued Resource to send SSH command to.

      ``USER'' specifies the username with which to SSH. If omitted, the user
      login name is used.

      ``QR'' specifies the name of the Cloud TPU Queued Resource to send SSH command to.
      )	completermetavarr   ssh_argszW          Flags and positionals passed to the underlying ssh implementation.
          zW        $ {command} example-instance --zone=us-central1-a -- -vvv -L 80:%TPU%:80
      )nargsr   examplez--node0aS            TPU node(s) to connect to. The supported value is a single 0-based
          index of the node(s) in the case of a TPU Pod. When also using the
          `--command` flag, it additionally supports a comma-separated list
          (e.g. '1,4,6'), range (e.g. '1-3'), or special keyword ``all" to
          run the command concurrently on each of the specified node(s).

          Note that when targeting multiple nodes, you should run 'ssh-add'
          with your private key prior to executing the gcloud command. Default:
          'ssh-add ~/.ssh/google_compute_engine'.
          )defaultr   N)r   r   InstancesCompleterargparse	REMAINDER)r   s    r   
AddSSHArgsr'   G   s    
  & 	--
	   	
  	 	
  r   c                   :    \ rS rSrSrSrSrSr\S 5       r	S r
Srg)	Ssh   z1Send SSH commands to a Cloud TPU Queued Resource.T@   c                    [         R                  R                  U5        [        U5        [        R
                  " UU R                  U R                  U R                  S9  [        U5        [        R                  " USSS9  g)zPSet up arguments for this command.

Args:
  parser: An argparse.ArgumentParser.
)
enable_iapenable_batchingenable_batching_defaulttpur   )resource_typeoperation_typeN)r
   BaseSSHCLIHelperArgsr'   tpu_ssh_utilsAddTPUSSHArgs_ENABLE_IAP_ENABLE_BATCHINGDEFAULT_BATCH_SIZEr   r	   AddZoneFlag)clsr   s     r   r4   Ssh.Args   sf     ##F+v??,, # 6 6	 v	fE%Hr   c                    [         R                   " 5       n[        R                  " UR                  5      u  p4UR                  c6  [
        R                  R                  R                  R                  SS9Ul        UR                  (       a  [        R                  R                  [        R                  R                  [        R                  R                  UR                  5      5      5      n[        R                  R                  U5      (       d&  [         R"                  " SSR%                  U5      5      e[&        R(                  " U R+                  5       5      nUR                  XAR                  5      nSUR                  ;   n[,        R.                  " UR0                  UR2                  R4                  5      n	/ n
Sn[7        U	5      n[8        R:                  " UR<                  [7        U	5      5      nS /U-  n[?        U	5       H  u  nnU
RA                  [B        RD                  " [8        RF                  URH                  UUU R+                  5       U RJ                  UUU4S95        U
S   RM                  5         US	-  nX:X  d  M  [,        RN                  " X5        Sn/ n
M     US:  a  [,        RN                  " X5        U Vs/ s H
  nUc  M  UPM     nn[7        U5      U:  a%  [P        RR                  " S
R%                  U5      5        [8        R:                  " UR<                  U RT                  5      n[8        RV                  " UUU5        [P        RX                  R[                  S[         R                   " 5       U-
  -  5        g s  snf )NT)r   z--output_directoryzJFailed to find directory {}. Please create it or specify another directory@r   )targetargs   z@Could not prepare all {} nodes, attempting to ssh into the rest.z!Completed execution in %s seconds).timer
   GetUserAndInstancer   zoner   VALUEScomputeGetoutput_directoryospathabspath
expandvars
expanduserisdirr   InvalidArgumentExceptionformatqueued_resource_utilsTPUQueuedResourceReleaseTrackqr_ssh_utilsParseNodeFlagnoder0   nodeSpeclenr5   ParseBatchSize
batch_size	enumerater   	threadingThreadPrepareNodeForSSHnodeIdr8   startWaitForNodeBatchCompletionr   warningr9   SSHIntoPreppedNodesstatusPrint)selfr@   
start_timeuserqr_nameoutput_directory_pathqueued_resource_clientqueued_resourceusername_requested
node_specsprep_nodes_threadscurrent_batch_size	num_nodesprep_node_batch_sizeprepped_nodesindexrW   prepped_nodessh_batch_sizes                      r   RunSsh.Run   s    J001J1JKMD yy##++0044d4Cdi ggoo
''

RWW//0E0EF
G WW]]01111 ..4f5J.K
 	
 3DD -00))DO 9 99 ++		?&&//J JI(77Z FY&M ,t


"44++##%''$	 ""$A		3//	
 / -2 A--.@P
 *)L 	)  
 =I%	kk6)$
 #1100N %% JJ+tyy{Z/GH)s   NN N)__name__
__module____qualname____firstlineno____doc__r7   r8   r9   classmethodr4   rx   __static_attributes__rz   r   r   r)   r)      s0    9+I I"Zr   r)   z/SSH into a Cloud TPU Queued Resource's node(s).a  
        To run an SSH command in a Cloud TPU Queued Resource's first node and
        first worker (for example, to print the time since last boot), run:

            $ {command} my-qr --command="last boot"

        To run the same command in all nodes and workers in a Cloud TPU Queued
        Resource (with the default batch size), run:

            $ {command} my-qr --command="last boot" --worker=all --node=all

        To run the same command in all nodes and workers in a Cloud TPU Queued
        Resource but batching the request in groups of 4, run:

            $ {command} my-qr --command="last boot" --worker=all --node=all --batch-size=4
        )briefEXAMPLES)$r   
__future__r   r   r   r%   os.pathrJ   r]   rC   googlecloudsdk.callioper   r   "googlecloudsdk.command_lib.computer   r	   r
   8googlecloudsdk.command_lib.compute.tpus.queued_resourcesr   rU   r   rR   .googlecloudsdk.command_lib.compute.tpus.tpu_vmr5   googlecloudsdk.corer   r   r   r'   ReleaseTracksrT   ALPHAGACommandr)   detailed_helprz   r   r   <module>r      s    M &  '     ( . 9 4 8 X b O # *B<~ D%%++T->->-A-ABs$,, s Csn ? r   