
                             S r SSKJr  SSKJr  SSKJr  SSKrSSKJr  SSKJ	r	  SSK
Jr  SS	K
Jr   " S
 S\R                  5      rg)zBase class for Spark Job.    )absolute_import)division)unicode_literalsN)encoding)arg_parsers)base)utilc                   H    \ rS rSrSr\S 5       r\S 5       r\S 5       rSr	g)	SparkBase   z.Submit a Java or Scala Spark job to a cluster.c                    U R                  S[        R                  " 5       S/ SS9  U R                  S[        R                  " 5       S/ SS9  U R                  S[        R                  " 5       S	/ S
S9  U R                  S[        R                  SS9  U R                  S[        R
                  " 5       SSS9  U R                  S[        R                  S9  U R                  S[        R
                  " 5       SSS9  g)z@Parses command-line arguments specific to submitting Spark jobs.z--jarsJARzWComma separated list of jar files to be provided to the executor and driver classpaths.)typemetavardefaulthelpz--filesFILEziComma separated list of files to be placed in the working directory of both the app driver and executors.z
--archivesARCHIVEzComma separated list of archives to be extracted into the working directory of each executor. Must be one of the following file formats: .zip, .tar, .tar.gz, or .tgz.job_argsz Arguments to pass to the driver.)nargsr   z--propertieszPROPERTY=VALUEzList of key value pairs to configure Spark. For a list of available properties, see: https://spark.apache.org/docs/latest/configuration.html#available-properties.)r   r   r   z--properties-file)r   z--driver-log-levelszPACKAGE=LEVELznList of package to log4j log level pairs to configure driver logging. For example: root=FATAL,com.example=INFON)add_argumentr   ArgListargparse	REMAINDERArgDictjob_utilPROPERTIES_FILE_HELP_TEXT)parsers    5lib/googlecloudsdk/command_lib/dataproc/jobs/spark.pyArgsSparkBase.Args"   s0      "0  2   "@  B   "  	   /  1   " 8	  9 //  1   "B	  D    c                 `    U R                   U R                  U R                  U R                  S.$ )z=Returns a dict of files by their type (jars, archives, etc.).main_jarjarsarchivesfilesr$   )argss    r   GetFilesByTypeSparkBase.GetFilesByTypeS   s*     MM		MM	 r"   c           
      L   U R                  UR                  =(       d    / US   US   US   UR                  US   US9n[        R                  " UR
                  UR                  5      nU(       a.  [        R                  " X`R                   R                  SS9Ul        XQl
        g)	z/Populates the sparkJob member of the given job.r'   r(   r&   r%   )r)   archiveUrisfileUrisjarFileUris	mainClassmainJarFileUriloggingConfigT)
sort_itemsN)SparkJobr   
main_classr   BuildJobProperties
propertiesproperties_filer   DictToAdditionalPropertyMessagePropertiesValuesparkJob)messagesjobfiles_by_typelogging_configr)   	spark_jobjob_propertiess          r   ConfigureJobSparkBase.ConfigureJob\   s     !!]] b!*-w'!&)//$Z0$ " I 00--N %EE
++;;i Lr"    N)
__name__
__module____qualname____firstlineno____doc__staticmethodr    r*   rB   __static_attributes__rD   r"   r   r   r      sB    6.D .D`    r"   r   )rI   
__future__r   r   r   r   apitools.base.pyr   googlecloudsdk.callioper   (googlecloudsdk.command_lib.dataproc.jobsr   job_baser	   r   JobBaser   rD   r"   r   <module>rR      s6       &  '  % / E ES   Sr"   