
    3                         S r SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSKJ	r	  SSK
Jr  SSK
Jr  SS	K
Jr  SS
KJr  SSKJr  SSKrSSKr " S S\R(                  " \R*                  \5      5      rg)z1Utilities for building the dataproc clusters CLI.    )absolute_import)division)unicode_literalsN)encoding)	constants)
exceptions)storage_helpers)log)filesc                   N   ^  \ rS rSrSrU 4S jrS rS rS
S jrS r	S r
S	rU =r$ )JobBase#   zBase class for Jobs.c                 X   > [         [        U ]
  " U0 UD6  0 U l        / U l        S U l        g N)superr   __init__files_by_typefiles_to_stage_staging_dir)selfargskwargs	__class__s      4lib/googlecloudsdk/command_lib/dataproc/jobs/base.pyr   JobBase.__init__&   s/    	'4!4262DDD    c                    [         R                  R                  U5      u  p#[        R                  R
                  R                  R                  USS9nU=(       d    UR                  (       + nU(       d  U$ [         R                  R                  U5      (       d%  [        R                  " SR                  U5      5      eU R                  c%  [        R                  " SR                  U5      5      e[         R                  R!                  U5      nU R"                  R%                  U5        [        R                  R
                  R                  R'                  U R                  U5      nU$ )z?Validate file URI and register it for uploading if it is local.F)allow_fragmentszFile Not Found: [{0}].zCould not determine where to stage local file {0}. When submitting a job to a cluster selected via --cluster-labels, either
- a staging bucket must be provided via the --bucket argument, or
- all provided files must be non-local.)ospath
splitdrivesixmovesurllibparseurlsplitschemeexistsr   Errorformatr   r   ArgumentErrorbasenamer   appendurljoin)r   file_strdrive_uriis_localr,   staged_files           r   _GetStagedFileJobBase._GetStagedFile,   s   ww!!(+HE
))


 
 
)
)(E
)
JC &CJJH o77>>(##KK077ABB  $$4 5;F84D	F F ww)Hx())""((001B1BHMKr   c                 :   [         R                  " U R                  5       H{  u  pU(       d  M  [        U[         R                  5      (       a   U R                  U5      U R                  U'   MM  U Vs/ s H  o0R                  U5      PM     nnX@R                  U'   M}     U R                  (       af  [        R                  " SR                  U R                  U R                  5      5        [        R                  " U R                  U R                  5        ggs  snf )z5Validate file URIs and upload them if they are local.zStaging local files {0} to {1}.N)r"   	iteritemsr   
isinstancestring_typesr5   r   r
   infor*   r   r	   Upload)r   	file_typefile_or_filesfstaged_filess        r   ValidateAndStageFilesJobBase.ValidateAndStageFilesI   s    $'MM$2D2D$E 	mS%5%566(,(;(;M(J9%8EF1++A.F(49% %F 	hh0778K8K8<8I8IK LT00$2C2CD  Gs   2Dc                 (   Uc  Uc  gUcQ  UR                   (       a  UR                   R                  nO)UR                  (       a  UR                  R                  nOgSnUb  UR                  nUb  UnSR                  U[        R                  UUS9nU$ )z6Determine the GCS directory to stage job resources in.N
unresolvedz;gs://{bucket}/{prefix}/{environment}/jobs/{job_id}/staging/)bucketprefixenvironmentjob_id)configconfigBucketvirtualClusterConfigstagingBucketclusterUuidr*   r   GCS_METADATA_PREFIX)r   clustercluster_poolrH   rE   rG   staging_dirs          r   GetStagingDirJobBase.GetStagingDirZ   s    ~'/~	,,''--;;
 K''k kELL00#	 	M 	  r   c           
      b   U(       d  gUR                   R                  R                  R                  n[        R
                  " UR                  5        VVs/ s H  u  pEXC" U5      4PM     snn5      nUR                  [        R                  " UUR                   R                  5      S9$ s  snnf )z$Build LoggingConfig from parameters.N)driverLogLevels)	LoggingConfigDriverLogLevelsValueAdditionalPropertyValueValueValuesEnumcollectionsOrderedDictitemsr   DictToAdditionalPropertyMessage)r   messagesdriver_logging
value_enumkeyvaluerI   s          r   BuildLoggingConfigJobBase.BuildLoggingConfigv   s    ((==$$%9%9 $$4B4H4H4JK4Jjc#z% 	!4JKMF!! @@""779 " : : 	Ls   B+
c                 X    U R                   R                  U R                  U5      5        g r   )r   updateGetFilesByType)r   r   s     r   PopulateFilesByTypeJobBase.PopulateFilesByType   s!    d11$78r   )r   r   r   r   )__name__
__module____qualname____firstlineno____doc__r   r5   rA   rR   rc   rh   __static_attributes____classcell__)r   s   @r   r   r   #   s*    :E"8:9 9r   r   )rn   
__future__r   r   r   abcrZ   r   apitools.base.pyr   googlecloudsdk.api_lib.dataprocr   r   r	   googlecloudsdk.corer
   googlecloudsdk.core.utilr   r"   six.moves.urllib.parsewith_metaclassABCMetaobjectr    r   r   <module>r|      sS    8 &  ' 
  	 % 5 6 ; # * 
 b9c  f5 b9r   