
    P                        S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
r
SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKJr  SSKJr  SS	KJr  SS
KJr  SSKrSSKJr  SSKJr  SSKJr  SSK J!r!  SSK"J#r#  SSK$J%r%  SSK&J'r'  SSK(J)r)  SSK(J*r*  SSK+J,r,  SSK-J.s  J/r/  SSK-J0r0  SSK-J1r2  SSK-J3r3  SSK-J4r4  SSK-J5r5  SSK-J6r6  SSK-J7r7  SSK8J9s  J:s  J;r<  SSK=J>r>  SSK?J@r@  SSK?JArA  SSKBJCrC  SS KDJErE  SS!KDJFrF  SS"KGJHrH  SS#KGJIrI  SS$KGJJrJ  SS%KGJKrK  SS&KGJLrL  SS'KMJNrN  SSKOrOSS(KPJQrQ  \	R                  " S)5      rSS* rTS+ rUS, rVS- rW\6R                  " \/R                  S.5       " S/ S0\,R                  5      5       r[g)1z1Contains gsutil base integration test case class.    )absolute_import)division)print_function)unicode_literalsN)config)StorageResponseError)DeleteMarker)BucketStorageUri)BotoTranslation)PreconditionException)Preconditions)DiscardMessagesQueue)CommandException)
GcsJsonApi)KmsApi)GOOG_PROJ_ID_HDR)PopulateProjectId)base)InvokedFromParFile)ObjectToURI)RUN_S3_TESTS)SetBotoConfigForTest)SetEnvironmentForTest)unittest)USING_JSON_API)UTF8)#Base64Sha256FromBase64EncryptionKey)CryptoKeyWrapperFromKey)Base64ToHexHash)CreateCustomMetadata) GetValueFromObjectCustomMetadata)
ATIME_ATTR)GID_ATTR)	MODE_ATTR)
MTIME_ATTR)UID_ATTR)Retry)rangezintegration-testc                 `    [         (       a  [        (       a  [        R                  " U 5      $ S $ )zGSkips the test if running S3 tests, or if prefer_api isn't set to json.c                     U $ N funcs    <platform/gsutil/gslib/tests/testcase/integration_testcase.py<lambda>SkipForXML.<locals>.<lambda>U           )r   r   r   skipreasons    r/   
SkipForXMLr7   P   s     	<<==  r3   c                 J    [         (       a  [        R                  " U 5      $ S $ )Nc                     U $ r+   r,   r-   s    r/   r0   SkipForJSON.<locals>.<lambda>\   r2   r3   )r   r   r4   r5   s    r/   SkipForJSONr;   X   s    ^==  r3   c                 J    [         (       d  [        R                  " U 5      $ S $ )Nc                     U $ r+   r,   r-   s    r/   r0   SkipForGS.<locals>.<lambda>c   r2   r3   r   r   r4   r5   s    r/   	SkipForGSr@   _   s    	==  r3   c                 J    [         (       a  [        R                  " U 5      $ S $ )Nc                     U $ r+   r,   r-   s    r/   r0   SkipForS3.<locals>.<lambda>j   r2   r3   r?   r5   s    r/   	SkipForS3rD   f   s    \==  r3   zNot running integration tests.c                     ^  \ rS rSrSrSrSrSrSrSr	Sr
U 4S	 jr\" \S
SS9U 4S j5       rS rS r     S1S jrS rS rS rS2S jrS rS rS r  S3S jr  S3S jrS r            S4S jrS5S jr            S6S jr        S7S jr       S8S jr S  r! S9S! jr"S" r#S# r$       S:S$ jr%S;S% jr&\'RP                  S& 5       r)S' r*S( r+S) r,S<S* jr-S+ r.S, r/S- r0S. r1S/ r2S0r3U =r4$ )=GsUtilIntegrationTestCaseo   z(Base class for gsutil integration tests.zgs-discussion@googlegroups.com@00b4903a97d097895ab58ef505d535916a712215b79c3e54932c2eb502ad97f5zgsutiltesting123@gmail.com@00b4903a97f0baa2680740f5adb90b2dcf9c8b878abd84ba1bdba653de949619z
google.comz#nonexistent-bucket-foobar.gmail.comc                 <  > [         [        U ]  5         / U l        [        R
                  R                  SSS5      U l        [        [        [        R                  " 5       [        5       S5      U l        [        [        [        R                  " 5       [        U R                  5      U l        [#        [        R                  " 5       5      U l        [&        R(                  U l        [
        R,                  " SSS5      U l        [&        R0                  (       a  SU l        gg)	z1Creates base configuration for integration tests.GSUtildefault_api_version1gsuse_gcloud_storageFz,nonexistentbucket-asf801rj3r9as90mfnnkjxpo02N)superrF   setUpbucket_urisbotor   	get_valueapi_versionr   r
   logging	getLoggerr   json_apir   default_providerxml_apir   kms_apiutilUSE_MULTIREGIONAL_BUCKETSmultiregional_bucketsgetbool_use_gcloud_storager   nonexistent_bucket_name)self	__class__s    r/   rQ   GsUtilIntegrationTestCase.setUp   s    	
#T02D {{,,X7L-02D /1B1B1D35t=DM"#3W5F5F5H#79N9NPDL'++-.DL!%!?!?D%~~h8L.3 5D 
8 " r3         triestimeout_secsc                   > [         [        U ]  5         U R                  (       a  U R                  S   n U R	                  U5      nU(       a  S nU H]  n [        U[        5      (       a4  UR                  5       R                  UR                  UR                  S9  MM  UR                  5         M_     U(       a  UeU R	                  U5      nU(       a  M  UR#                  5         U R                  R                  5         U R                  (       a  M  g g ! [
         a7  nUR                  S:X  a!  U R                  R                  5          S nAGM;  e S nAff = f! [
         aq  nUR                  S:X  a	  Un S nAGM*  UR                  S:X  aB  UR                  S:X  d  UR                  S:X  a"  U R!                  XR                  5         S nAGM|  e S nAff = f)Ni  )
version_idi  ObjectUnderActiveHoldRetentionPolicyNotMet)rP   rF   tearDownrR   _ListBucketr   statuspop
isinstancer	   
get_bucket
delete_keynamerl   delete
error_code._ClearHoldsOnObjectAndWaitForRetentionDurationdelete_bucket)rb   
bucket_uribucket_listeerrorkrc   s         r/   ro   "GsUtilIntegrationTestCase.tearDown   s   	
#T35


##B'j	&&z2 A!\**##%00<=LL 1 J hhj @ +&&z2I KJ  
e 


 "  88s?





 

" & 
 xx3
 eSall6M&M&'ll6M&M AAff& & 1sI   D AEE
E*EEE
G"G;AGGGc                    U R                   R                  UR                  U/ SQS9nSR                  X5      nUR                  (       a  U R                  SSSU/5        UR                  (       a  U R                  SSSU/5        U R                   R                  UR                  S/S9R                  nUb  UR                  OS
nUS::  a  [        R                  " U5        g	[        SR                  U5      5      e)a  Removes Holds on test objects and waits till retention duration is over.

This method makes sure that object is not under active Temporary Hold or
Release Hold. It also waits (up to 1 minute) till retention duration for the
object is over. This is necessary for cleanup, otherwise such test objects
cannot be deleted.

It's worth noting that tests should do their best to remove holds and wait
for objects' retention period on their own and this is just a fallback.
Additionally, Tests should not use retention duration longer than 1 minute,
preferably only few seconds in order to avoid lengthening test execution
time unnecessarily.

Args:
  bucket_uri: bucket's uri.
  object_name: object's name.
)timeCreatedtemporaryHoldeventBasedHoldfieldsz{}{}	retentiontempreleaseeventretentionPolicyNr   <   zfRetention duration is too large for bucket "{}". Use shorter durations for Retention duration in tests)rX   GetObjectMetadatabucket_nameformatr   	RunGsUtilr   	GetBucketr   retentionPeriodtimesleepr   )rb   r{   object_nameobject_metadata
object_uriretention_policyretention_periods          r/   ry   HGsUtilIntegrationTestCase._ClearHoldsOnObjectAndWaitForRetentionDuration   s    & mm55A 6 CO z7J$$
nnk69jAB%%
nnk7IzBC}}..z/E/E7H 78 / 99H  ,7 )88=>  2
jj!" '(.z(:< <r3   c                     [         R                  " 5       n[        XE05      Ul        US:X  a  U R                  R                  UUUUS9  gU R                  R                  UUUUS9  g)ak  Sets a custom metadata attribute for an object.

Args:
  provider: Provider string for the bucket, ex. 'gs' or 's3.
  bucket_name: The name of the bucket the object is in.
  object_name: The name of the object itself.
  attr_name: The name of the custom metadata attribute to set.
  attr_value: The value of the custom metadata attribute to set.

Returns:
  None
rN   providerN)apitools_messagesObjectr    metadatarX   PatchObjectMetadatarZ   )rb   r   r   r   	attr_name
attr_valueobj_metadatas          r/   !_SetObjectCustomMetadataAttribute;GsUtilIntegrationTestCase._SetObjectCustomMetadataAttribute  sp     %++-L0)1HIL4
mm''(3(419 ( ;
 ll&&{'2'308 ' :r3   c	                    [         R                  " 5       n	[         R                  R                  / S9U	l        Ub  [	        [
        U0U	R                  S9  Ub  [	        [        U0U	R                  S9  Ub  [	        [        U0U	R                  S9  Ub  [	        [        U0U	R                  S9  Ub  [	        [        U0U	R                  S9  US:X  a  U R                  R                  UUU	US9  gU R                  R                  UUU	US9  g)z#Sets POSIX metadata for the object.additionalPropertiesN)entriescustom_metadatarN   r   )r   r   MetadataValuer   r    r"   r$   r%   r&   r#   rX   r   rZ   )
rb   r   r   r   atimemtimeuidgidmoder   s
             r/   SetPOSIXMetadata*GsUtilIntegrationTestCase.SetPOSIXMetadata  s    %++-L-44BB C !LJ#6+7+@+@BIt#4+7+@+@BJ#6+7+@+@B
Hc?+7+@+@B
Hc?+7+@+@B4
mm''(3(419 ( ;
 ll&&{'2'308 ' :r3   c                     UR                   S:X  a  SOSnU R                  SSSU< S[        < 3SSU< S[        < 3SSU< S[        < 3SSU< S[
        < 3SSU< S[        < 3[        U5      /5        g)	z}Uses the setmeta command to clear POSIX attributes from user metadata.

Args:
  obj: The object to clear POSIX metadata for.
rN   googamzsetmetaz-hzx-z-meta-N)schemer   r"   r%   r&   r#   r$   suri)rb   objprovider_meta_strings      r/   ClearPOSIXMetadata,GsUtilIntegrationTestCase.ClearPOSIXMetadataD  sd     &)ZZ4%76UNN4.
;T.
;T.94.94.	:S	 r3   c                 l    [         R                  " SS5      =(       d    [         R                  " SS5      $ )NCredentialsgs_service_key_fileGoogleComputeservice_account)r   
has_option)rb   s    r/   !_ServiceAccountCredentialsPresent;GsUtilIntegrationTestCase._ServiceAccountCredentialsPresentU  s2     m-BC Bo/@ACr3   c                     UR                   S:X  a.  [        S UR                  5       R                  5        5       5      $ [        UR	                  SS95      $ )Ns3c              3   $   #    U  H  ov   M     g 7fr+   r,   ).0vs     r/   	<genexpr>8GsUtilIntegrationTestCase._ListBucket.<locals>.<genexpr>d  s     ED!Ds   T)all_versions)r   listrt   list_versionslist_bucket)rb   r{   s     r/   rp   %GsUtilIntegrationTestCase._ListBucket`  sN    D  EZ224BBDEEE
&&D&9::r3   c                    ^ ^^^^ UUU U4S jmT R                   (       a  [        [        SSS9U4S j5       nU" 5       $ T" 5       $ )a  Checks (with retries) that 'ls bucket_uri/**' returns num_objects.

This is a common test pattern to deal with eventual listing consistency for
tests that rely on a set of objects to be listed.

Args:
  bucket_uri: storage_uri for the bucket.
  num_objects: number of objects expected in the bucket.
  versioned: If True, perform a versioned listing.

Raises:
  AssertionError if number of objects does not match expected value.

Returns:
  Listing split across lines.
c                     > T(       a  SS/OS/n T(       a  [        T5      S-   /O[        T5      /nTR                  X-   SS9R                  S5      nTR                  [	        U5      TS-   5        U$ )Nlsz-az/**Treturn_stdout
rf   )r   r   splitassertEquallen)commandb_urilistingr{   num_objectsrb   	versioneds      r/   _CheckBucketFGsUtilIntegrationTestCase.AssertNObjectsInBucket.<locals>._CheckBuckety  sl     )tvg,7tJ%'(d:>N=OewdCII$Og
s7|[1_5nr3      rf   rg   c                     > T " 5       $ r+   r,   )r   s   r/   _Check1AGsUtilIntegrationTestCase.AssertNObjectsInBucket.<locals>._Check1  s    ~r3   )r^   r'   AssertionError)rb   r{   r   r   r   r   s   ```` @r/   AssertNObjectsInBucket0GsUtilIntegrationTestCase.AssertNObjectsInBucketg  sE    $  !!^115 6 Y^r3   c           
          [        S/5         U R                  SU/SSS9nSSS5        U R                  [        U5      R	                  S5      WSU< S[        U5      < S	U< 35        g! , (       d  f       NO= f)
a  Strongly consistent check that the correct CSEK encryption key is used.

This check forces use of the JSON API, as encryption information is not
returned in object metadata via the XML API.

Args:
  object_uri_str: uri for the object.
  encryption_key: expected CSEK key.
rK   
prefer_apijsonstatTr   force_gsutilNasciizObject z/ did not use expected encryption key with hash z. Actual object: )r   r   assertInr   decoderb   object_uri_strencryption_keystdouts       r/   AssertObjectUsesCSEK.GsUtilIntegrationTestCase.AssertObjectUsesCSEK  sw     
?@	A~~v~6,0+/  1f 
B 	MM+N;BB7K	<^L		 
B	As   A**
A8c                     [        S/5         U R                  SU/SSS9nSSS5        U R                  WSU-  5        g! , (       d  f       N$= f)a  Strongly consistent check that the correct KMS encryption key is used.

This check forces use of the JSON API, as encryption information is not
returned in object metadata via the XML API.

Args:
  object_uri_str: uri for the object.
  encryption_key: expected CMEK key.
r   r   Tr   NzKMS key:\s+%s)r   r   assertRegexr   s       r/   AssertObjectUsesCMEK.GsUtilIntegrationTestCase.AssertObjectUsesCMEK  sX     
?@	A~~v~6,0+/  1f 
B 	V->?	 
B	As	   ?
Ac                     [        S/5         U R                  SU/SSS9nSSS5        U R                  SW5        U R                  SU5        g! , (       d  f       N3= f)zChecks that no CSEK or CMEK attributes appear in `stat` output.

This check forces use of the JSON API, as encryption information is not
returned in object metadata via the XML API.

Args:
  object_uri_str: uri for the object.
r   r   Tr   NzEncryption key SHA256zKMS key)r   r   assertNotIn)rb   r   r   s      r/   AssertObjectUnencrypted1GsUtilIntegrationTestCase.AssertObjectUnencrypted  sc     
?@	A~~v~6,0+/  1f 
B 	,f5Y' 
B	As   A
Ac                     [         R                  R                  US9nU R                  UUSS9nU(       a  U R	                  SS[        U5      /SS9  U R                  UUUS9  U$ )	a  Creates a test bucket with Retention Policy.

The bucket and all of its contents will be deleted after the test.

Args:
  retention_period_in_seconds: Retention duration in seconds
  is_locked: Indicates whether Retention Policy should be locked
             on the bucket or not.
  bucket_name: Create the bucket with this name. If not provided, a
               temporary test bucket name is constructed.

Returns:
  StorageUri for the created bucket.
)r   T)r   r   prefer_json_apir   lockystdin)$expected_retention_period_in_secondsexpected_is_locked)r   BucketRetentionPolicyValueCreateBucketr   r   VerifyRetentionPolicy)rb   retention_period_in_seconds	is_lockedr   r   r{   s         r/   CreateBucketWithRetentionPolicy9GsUtilIntegrationTestCase.CreateBucketWithRetentionPolicy  s    & *00EE3 F 5""{4D37 # 9J 
nnk64
+;<CnH 	-H$  &
 r3   c                    U R                   R                  UR                  S/S9R                  nUc  U R	                  US5        gU R	                  UR
                  U5        U R	                  UR                  U5        U R                  UR                  5      nU R                  [        R                  R                  5       5      nU R                  UUS-
  5        g)a)  Verifies the Retention Policy on a bucket.

Args:
  bucket_uri: Specifies the bucket.
  expected_retention_period_in_seconds: Specifies the expected Retention
                                        Period of the Retention Policy on
                                        the bucket. Setting this field to
                                        None, implies that no Retention
                                        Policy should be present.
  expected_is_locked: Indicates whether the Retention Policy should be
                      locked or not.
r   r   Nr   )rX   r   r   r   r   r   isLockedDateTimeToSecondseffectiveTimedatetimenowassertGreater)rb   r{   r  r  actual_retention_policyeffective_time_in_secondscurrent_time_in_secondss          r/   r  /GsUtilIntegrationTestCase.VerifyRetentionPolicy  s      #mm55(9': 6 <<KO  ,3
.5
.>>;=
.779KL
 #'"8"8
!
/
/#1 $ 6 6x7H7H7L7L7N O
20257r3   c                 \    [        [        R                  " UR                  5       5      5      $ r+   )intr   mktime	timetuple)rb   datetime_objs     r/   r  +GsUtilIntegrationTestCase.DateTimeToSeconds  s    t{{<113455r3   c                 x  ^^^^ U(       d  U R                   nTc;  U R                  (       d  US:X  a  SmO![        R                  R	                  SSS5      m[
        R                  " U	5      n	[
        R                  " U
5      n
U(       a*  SR                  XU
/5      n[        R                  " U5      nOU R                  SU	U
S9nU(       aN  US	:X  aH  U R                  UUTTUUUUS
9n[        R                  " SUR                  R                  5       -  SS9mT$ [        R                  " U< SUR                  5       < 3SS9mUS	:X  a  SU R                  [         [#        5       0mO0 mU(       d  STS'   [%        [&        SSS9UUUU4S j5       nU" 5         U R(                  R+                  T5        U(       a  TR-                  S5        US	:w  a  U(       d  SnTR/                  SU5        [1        U5       HA  nU R3                  TU R                  S5      SR5                  U5      R7                  S5      S9  MC     T$ )a  Creates a test bucket.

The bucket and all of its contents will be deleted after the test.

Args:
  bucket_name: Create the bucket with this name. If not provided, a
               temporary test bucket name is constructed.
  test_objects: The number of objects that should be placed in the bucket.
                Defaults to 0.
  storage_class: Storage class to use. If not provided we us standard.
  retention_policy: Retention policy to be used on the bucket.
  provider: Provider to use - either "gs" (the default) or "s3".
  prefer_json_api: If True, use the JSON creation functions where possible.
  versioning_enabled: If True, set the bucket's versioning attribute to
      True.
  bucket_policy_only: If True, set the bucket's iamConfiguration's
      bucketPolicyOnly attribute to True.
  bucket_name_prefix: Unicode string to be prepended to bucket_name
  bucket_name_suffix: Unicode string to be appended to bucket_name
  location: The location/region in which the bucket should be created.
  public_access_prevention: String value of public access prevention. Valid
      values are "enforced" and "unspecified".

Returns:
  StorageUri for the created bucket.
Nr   rK   !test_cmd_regional_bucket_locationzus-central1 bucket)prefixsuffixrN   )r   test_objectsstorage_classlocationversioning_enabledr   bucket_policy_onlypublic_access_preventiongs://%sFsuppress_consec_slashesz://zx-goog-api-versionObjectWriterzx-amz-object-ownershipre   rf   rg   c                     >  TR                  TT=(       d    STS9  g! [         a<  n U R                  S:X  a&  U R                  (       a  SU R                  ;   a   Sn A ge Sn A ff = f)zCreates a bucket, retrying with exponential backoff on error.

Parallel tests can easily run into bucket creation quotas.
Retry with exponential backoff so that we create them as fast as we
reasonably can.

Returns:
  StorageUri for the created bucket
r$  )r)  r*  headersi  zalready ownN)create_bucketr   rq   body)r}   r{   r3  r*  r)  s    r/   #_CreateBucketWithExponentialBackoffSGsUtilIntegrationTestCase.CreateBucket.<locals>._CreateBucketWithExponentialBackoffj  s]      }*2.b)0 	! 	2 "  88s?qvv-166*A

s    
A$1AAA$Tz<?xml version="1.0" encoding="UTF-8"?><PublicAccessBlockConfiguration><BlockPublicAcls>False</BlockPublicAcls></PublicAccessBlockConfiguration>publicAccessBlockr   	test {:d}r   )r{   r   contents)rY   r^   rS   r   getsixensure_textjoinr\   MakeBucketNameValidMakeTempNameCreateBucketJsonstorage_urirv   lowerrU   r   r   r'   r   rR   appendconfigure_versioningset_subresourcer(   CreateObjectr   encode)rb   r   r(  r)  r   r   r  r+  r,  bucket_name_prefixbucket_name_suffixr*  r-  json_bucketr6  xml_bodyir{   r3  s      `       `     @@r/   r  &GsUtilIntegrationTestCase.CreateBucket  sG   N &&h 		#	#x4'7 ;;??8#F#02 );<);<GG,>
?Ak,,[9k%%h-?-? & Ak 8t+))!#%/+/#; * =k ##I0@0@0F0F0H$H<ACj!!x9J9J9L"M:?AJ 4  0 0
-/g
 g -;()
qq9 :8 ()J'%%d+4 86h   !4h?< 
:$($5$5e$<!,!3!3A!6!=!=g!F  H ! r3   c                 *    U R                  UUSSS9nU$ )a  Creates a versioned test bucket.

The bucket and all of its contents will be deleted after the test.

Args:
  bucket_name: Create the bucket with this name. If not provided, a
               temporary test bucket name is constructed.
  test_objects: The number of objects that should be placed in the bucket.
                Defaults to 0.

Returns:
  StorageUri for the created bucket with versioning enabled.
T)r   r(  r  r+  )r  )rb   r   r(  r{   s       r/   CreateVersionedBucket/GsUtilIntegrationTestCase.CreateVersionedBucket  s,    " ""{0<376: # <J r3   c                    U=(       d    U R                  5       nUbh  [        U[        R                  [        R                  45      (       d#  [        SR                  [        U5      5      5      e[        R                  " U5      nU(       a  UR                  S:X  a  U(       d  U(       d  U(       a  U=(       d    U R                  S5      nU R                  UUR                  UUUU
UUS9nUR                  U5      n[        UR                  5      UR                  R!                  S5      4nUR#                  SUR$                  SUS9  U$ U=(       d    U R                  5       nU=(       d    U R                  S5      nUR                  U5      nUbA  UR                  S:X  a   Ub   UR'                  US	[)        U5      0S
9  OUR'                  U5        USL=(       d    USL=(       d    USL=(       d    U	SLnU(       a*  U R/                  UR                  UR                  USUUU	US9  U$ ! [*         a  nUR,                  S:X  a   SnANwe SnAff = f)a  Creates a test object.

Args:
  bucket_uri: The URI of the bucket to place the object in. If not
      specified, a new temporary bucket is created.
  object_name: The name to use for the object. If not specified, a temporary
      test object name is constructed.
  contents: The contents to write to the object. If not specified, the key
      is not written to, which means that it isn't actually created
      yet on the server.
  prefer_json_api: If true, use the JSON creation functions where possible.
  encryption_key: AES256 encryption key to use when creating the object,
      if any.
  mode: The POSIX mode for the object. Must be a base-8 3-digit integer
      represented as a string.
  mtime: The modification time of the file in POSIX time (seconds since
      UTC 1970-01-01). If not specified, this defaults to the current
      system time.
  uid: A POSIX user ID.
  gid: A POSIX group ID.
  storage_class: String representing the storage class to use for the
      object.
  gs_idempotent_generation: For use when overwriting an object for which
      you know the previously uploaded generation. Create GCS object
      idempotently by supplying this generation number as a precondition
      and assuming the current object is correct on precondition failure.
      Defaults to 0 (new object); to disable, set to None.
  kms_key_name: Fully-qualified name of the KMS key that should be used to
      encrypt the object. Note that this is currently only valid for 'gs'
      objects.

Returns:
  A StorageUri for the created object.
Nz-contents must be either none or bytes, not {}rN   r   )r:  r   r   r   r   r)  gs_idempotent_generationkms_key_namez
"'T)md5zx-goog-if-generation-match)r3  i  )r   r   r   r   r   )r  rs   r<  binary_type	text_type	TypeErrorr   typeensure_binaryr   r@  CreateObjectJsonr   clone_replace_namer   md5Hashstrip_update_from_values
generationset_contents_from_stringstrr   rq   r   )rb   r{   r   r:  r  r   r   r   r   r   r)  rS  rT  json_objectr   rU  key_urir}   custom_metadata_presents                      r/   rG  &GsUtilIntegrationTestCase.CreateObject  sV   ^ 2t002J3??CMM"BCCGNNN  	""8,hZ&&$.	Nl;4#4#4U#;k)) ,,!'%#;# * %k 00=j
 [001  &&w/1c$$T%0%;%;%)), % .
 2t002J9!2!25!9K++K8G			d	"'?'K
	

*
*8/K367O3P4- + . 	((2#4/  B53D  B"$ B25T/ 
J--&22'"&"' # #!%  ' N% $ 	XX_		s   %H* *
I4I
	I

Ic	                 R   [         R                  " U=(       d    U R                  S5      5      n[        R                  " UR                  5       S9n	U(       a  X9l        U(       a  XIl        U(       a"  [        R                  R                  SS9U	l	        U(       a  Xil
        U(       d  U(       a^  [        R                  R                  5       n
U(       a&  U
R                  5       U
l        SU
R                  l        U(       a  Xl        Xl        U R"                  R%                  XS9nU R&                  R)                  [*        R,                  " SU-  SS95        [/        U5       HA  nU R1                  UU R                  S	5      S
R3                  U5      R5                  S5      S9  MC     U$ )aq  Creates a test bucket using the JSON API.

The bucket and all of its contents will be deleted after the test.

Args:
  bucket_name: Create the bucket with this name. If not provided, a
               temporary test bucket name is constructed.
  test_objects: The number of objects that should be placed in the bucket.
                Defaults to 0.
  storage_class: Storage class to use. If not provided we use standard.
  location: Location to use.
  versioning_enabled: If True, set the bucket's versioning attribute to
      True.
  retention_policy: Retention policy to be used on the bucket.
  bucket_policy_only: If True, set the bucket's iamConfiguration's
      bucketPolicyOnly attribute to True.
  public_access_prevention: String value of public access prevention. Valid
      values are "enforced" and "unspecified".

Returns:
  Apitools Bucket for the created bucket.
r%  )rv   T)enabled)r   r.  Fr/  r   r9  r   )r   r   r:  )r\   r?  r@  r   r	  rC  storageClassr*  VersioningValue
versioningr   IamConfigurationValueBucketPolicyOnlyValuebucketPolicyOnlyrh  publicAccessPreventioniamConfigurationrX   r  rR   rD  rS   rB  r(   r[  r   rH  )rb   r   r(  r)  r*  r+  r   r,  r-  bucket_metadata
iam_configr%  rM  s                r/   rA  *GsUtilIntegrationTestCase.CreateBucketJson#  sk   > **; ,G+/+<+<X+FHK'..K4E4E4GHO%2"!)$5$<$<$L$L %M %o (8%5$++AACj	&0&F&F&H
#.2
##+	!,D))3& ]]'''NF 	[016	89 < 
(,(9(9%(@%0%7%7%:%A%A'%J  L ! Mr3   c	           	      R   U=(       d    U R                  5       R                  nU=(       d    U R                  S5      n[        US9n	[        R
                  R                  / S9n
Ub  [        [        U0U
5        [        R
                  " UU
USUUS9n[        U5      n U R                  R                  [        R                  " U5      USUU	S9$ ! [         aM    Uc  e [        S	S
U4/5         U R                  R!                  X#5      sSSS5        s $ ! , (       d  f        g= ff = f)a  Creates a test object (GCS provider only) using the JSON API.

Args:
  contents: The contents to write to the object.
  bucket_name: Name of bucket to place the object in. If not specified,
      a new temporary bucket is created. Assumes the given bucket name is
      valid.
  object_name: The name to use for the object. If not specified, a temporary
      test object name is constructed.
  encryption_key: AES256 encryption key to use when creating the object,
      if any.
  mtime: The modification time of the file in POSIX time (seconds since
      UTC 1970-01-01). If not specified, this defaults to the current
      system time.
  storage_class: String representing the storage class to use for the
      object.
  gs_idempotent_generation: For use when overwriting an object for which
      you know the previously uploaded generation. Create GCS object
      idempotently by supplying this generation number as a precondition
      and assuming the current object is correct on precondition failure.
      Defaults to 0 (new object); to disable, set to None.
  kms_key_name: Fully-qualified name of the KMS key that should be used to
      encrypt the object. Note that this is currently only valid for 'gs'
      objects.

Returns:
  An apitools Object for the created object.
r   )	gen_matchr   Nzapplication/octet-stream)rv   r   r%  contentTyperi  
kmsKeyNamerN   )r   encryption_tuplepreconditionsrK   decryption_key1)rA  rv   r@  r   r   r   r   r    r%   r   rX   UploadObjectr<  BytesIOr   r   r   )rb   r:  r   r   r   r   r)  rS  rT  ry  r   r   encryption_keywrappers                r/   r[  *GsUtilIntegrationTestCase.CreateObjectJsond  s5   J =!6!6!8!=!=K9!2!25!9K!,DEM'..<< = !OJ.@'.. ."!O 4NCI]]''H(=(7159N6C	 ( E E
 ! I	!	)(,=~!N !" #}}..{H# # # #Is*   0C D&,D
D&
D"	D&"D&c                     U R                   S:X  a  U R                  OU R                  nUR                  UUU R                   US9$ )a  Retrieves and verifies an object's metadata attribute.

Args:
  bucket_name: The name of the bucket the object is in.
  object_name: The name of the object itself.
  fields: List of attributes strings. Custom attributes begin "metadata/".

Returns:
  Apitools object.
rN   )r   r   )rY   rX   rZ   r   )rb   r   r   r   
gsutil_apis        r/   GetObjectMetadataWithFields5GsUtilIntegrationTestCase.GetObjectMetadataWithFields  sP     **d2 --8< ''(3151F1F/5 ( 7 7r3   c                     U R                  XSU-  /S9n[        XcUS9u  pxU R                  XW5        U R                  XH5        g)a  Retrieves and verifies an object's custom metadata attribute.

Args:
  bucket_name: The name of the bucket the object is in.
  object_name: The name of the object itself.
  attr_name: The name of the custom metadata attribute.
  expected_value: The expected retrieved value for the attribute.
  expected_present: True if the attribute must be present in the
      object metadata, False if it must not be present.

Returns:
  None
zmetadata/%sr   )default_valueN)r  r!   r   )	rb   r   r   r   expected_valueexpected_presentr   attr_presentvalues	            r/   VerifyObjectCustomAttribute5GsUtilIntegrationTestCase.VerifyObjectCustomAttribute  sV    & //-)*C)D 0 FH:>;L%4^+r3   c                     U R                  SS[        U5      /SS9n[        R                  " S5      n[        R                  " UU5      nUR                  S5      nU R                  [        U5      U5        g )Npublicaccesspreventionr;  Tr   z:\s+(?P<pap_val>.+)$pap_val)r   r   recompilesearchgroupr   rb  )rb   r{   r  r   public_access_prevention_republic_access_prevention_matchpublic_access_prevention_vals          r/   !VerifyPublicAccessPreventionValue;GsUtilIntegrationTestCase.VerifyPublicAccessPreventionValue  sz    ^^5u!*-/*.  0F #%**-D"E%'YY/J/5&7"#A#G#G$ SZ!=>r3   c                    U R                  US[        U5      /SS9n[        R                  " SR	                  [        U5      5      5      n[        R
                  " XT5      nUR                  S5      nU R                  Xs5        g)z5Verifies if <command> get returns the expected value.r;  Tr   z{}: (?P<actual>.+)$actualN)r   r   r  r  r   r  r  r   )rb   r{   r   expectedr   output_regexoutput_matchr  s           r/   VerifyCommandGet*GsUtilIntegrationTestCase.VerifyCommandGet  sq    ^^WeT*-=>*.  0F::3::4
;KLML99\2L)FV&r3   c	                    [         R                  " XS9n	[         R                  " XS9n
[         R                  " XS9nUS   R	                  [
        R                  S5      nUS   R	                  [
        R                  S5      nU
R                  nUb]  [        [        R                  U5      nU R                  [        U5      [        U5      SR                  X^S	R                  U5      U5      S
9  / nU(       a  UR                  U5        U(       a  UR                  U5        U(       a  UR                  U5        [!        U5      S:X  a  US   $ U(       a  [#        U5      $ g)al  Runs the gsutil command.

Args:
  cmd: The command to run, as a list, e.g. ['cp', 'foo', 'bar']
  return_status: If True, the exit status code is returned.
  return_stdout: If True, the standard output of the command is returned.
  return_stderr: If True, the standard error of the command is returned.
  expected_status: The expected return code. If not specified, defaults to
                   0. If the return code is a different value, an exception
                   is raised.
  stdin: A string of data to pipe to the process as standard input.
  env_vars: A dictionary of variables to extend the subprocess's os.environ
            with.
  force_gsutil: If True, will always run the command using gsutil,
    irrespective of the value provided for use_gcloud_storage.

Returns:
  If multiple return_* values were specified, this method returns a tuple
  containing the desired return values specified by the return_* arguments
  (in the order those parameters are specified in the method definition).
  If only one return_* value was specified, that value is returned directly
  rather than being returned within a 1-tuple.
)r   )env_varsr  r   r   rf   Nz3Expected status {}, got {}.
Command:
{}

stderr:
{} )msg)r\   GetGsutilCommandGetGsutilSubprocessCommunicateWithTimeoutreplaceoslinesep
returncodemapr<  r=  r   r  r   r>  rD  r   tuple)rb   cmdreturn_statusr   return_stderrexpected_statusr  r  r   full_gsutil_commandprocessc_outr   stderrrq   toreturns                   r/   r   #GsUtilIntegrationTestCase.RunGsUtil  s'   @ //O&&':NG''=E1Xbjj$/F1Xbjj$/FF"%c

f+
o
HOOsxx}f>  ? Hoofoofoof
8}a[	8_ 
r3   c                 P  ^ ^^^ [         R                  /S/-   U-   n[        5       (       a  SmOSm[        [        R
                  5      /U-   nSR                  U5      m[        [        SSS9UUUU 4S j5       n[        S/5         U" 5         S	S	S	5        g	! , (       d  f       g	= f)
zRuns the gsutil command in tab completion mode.

Args:
  cmd: The command to run, as a list, e.g. ['cp', 'foo', 'bar']
  expected_results: The expected tab completion results for the given input.
z--testexceptiontracesrf      r  r   rg   c                    > Sn Sn[         R                  " SS9 nU (       a  SR                  TUR                  S9nOSR                  TUR                  S9n[        R
                  R                  5       n[        T5      US'   SUS	'   S
U;   a  US
   US	'   TUS'   [        [        T5      5      US'   [        R                  " X4SS9  UR                  5       R                  [        R                  " 5       5      nSSS5        U(       a:  U (       a!  [        S5        [        U5        [        S5        UR!                  S5      nO/ nT	R#                  UT5        g! , (       d  f       Nd= f)z/Runs the tab completion operation with retries.FN)rw   z {cs} 1>{fn} 2>{fn} 8>{fn} 9>{fn})csfnz{cs} 8>{fn}_ARGCOMPLETEz"'@><=;|&(:_ARGCOMPLETE_COMP_WORDBREAKSCOMP_WORDBREAKS	COMP_LINE
COMP_POINTT)envshellz'---------------------------------------)tempfileNamedTemporaryFiler   rv   r  environcopyrb  r   
subprocesscallreadr   localegetpreferredencodingprintr   r   )
hacky_debuggingresults_stringtab_complete_result_filecmd_str_with_result_redirectr  resultsargcomplete_start_idxcmd_strexpected_resultsrb   s
         r/   _RunTabCompletionKGsUtilIntegrationTestCase.RunGsUtilTabCompletion.<locals>._RunTabCompletion6  s_    on&&3
 177!9!>!> 8 @ ' *7)=)=5:: *> *<
&jjoo!"78N.?*+#034E0F#,
-"KG-L4TJ1668??'')+/2 

9
:


9
: &&v.
w 01C s   CE
E%)rK   tab_completion_timeout120N)
gslibGSUTIL_PATHr   rb  sys
executabler>  r'   r   r   )rb   r  r  r  r  r  s   ` ` @@r/   RunGsUtilTabCompletion0GsUtilIntegrationTestCase.RunGsUtilTabCompletion#  s     
!8 9
9C
?C
  !C'chhsmG
>302 402h 
JK	L 
M	L	Ls   B
B%c           
   #     #    S/nS H6  nUR                  SU[        R                  R                  SUS 5      45        M8     UR                  SS[        R                  R                  SSS 5      45        S H6  nUR                  SU[        R                  R                  SUS 5      45        M8     [	        USS	9   [        S S
S
SS.5         S v   S S S 5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f7f)N)Testsbypass_anonymous_access_warningTrue)gs_hostgs_json_hostgs_json_host_headergs_postgs_json_portr   Botohttps_validate_certificates)json_api_versionr   rK   F)use_existing_config_r  )DEVSHELL_CLIENT_PORTAWS_SECRET_ACCESS_KEYAWS_ACCESS_KEY_ID!CLOUDSDK_AUTH_DISABLE_CREDENTIALS)rD  rS   r   r;  r   r   )rb   boto_config_for_testcreds_config_keyapi_config_keys       r/   SetAnonymousBotoCreds/GsUtilIntegrationTestCase.SetAnonymousBotoCredsn  s     Q 
8!!=2B#';;??=3CT$K#L M8
 	.	!>	E	GH =!!8^#';;??8^37$9#: ; =
 
2	N
 !"&#&"/5" 	 		 
O	N
	 	 
O	Ns6   B9D;C3C"C3	D"
C0	,C33
D=Dc           
          U R                  U[        [        [        R                  " U5      R
                  5      SS S5      5        g)zVerifies the mode of the file specified at path.

Args:
  path: The path of the file on the local file system.
  expected_mode: The expected mode as a 3-digit base-8 number.

Returns:
  None
N   )r   r  octr  r   st_mode)rb   pathexpected_modes      r/   _VerifyLocalMode*GsUtilIntegrationTestCase._VerifyLocalMode  s5     	]CBGGDM,A,A(B23(G$KLr3   c                 d    U R                  U[        R                  " U5      R                  5        g)zVerifies the uid of the file specified at path.

Args:
  path: The path of the file on the local file system.
  expected_uid: The expected uid of the file.

Returns:
  None
N)r   r  r   st_uid)rb   r  expected_uids      r/   _VerifyLocalUid)GsUtilIntegrationTestCase._VerifyLocalUid  "     	\2774=#7#78r3   c                 d    U R                  U[        R                  " U5      R                  5        g)zVerifies the gid of the file specified at path.

Args:
  path: The path of the file on the local file system.
  expected_gid: The expected gid of the file.

Returns:
  None
N)r   r  r   st_gid)rb   r  expected_gids      r/   _VerifyLocalGid)GsUtilIntegrationTestCase._VerifyLocalGid  r  r3   c                 ~    Ub  U R                  X5        Ub  U R                  X5        Ub  U R                  X5        gg)aQ  Verifies the uid, gid, and mode of the file specified at path.

Will only check the attribute if the corresponding method parameter is not
None.

Args:
  path: The path of the file on the local file system.
  gid: The expected gid of the file.
  uid: The expected uid of the file.
  mode: The expected mode of the file.

Returns:
  None
N)r  r  r  )rb   r  r   r   r   s        r/   VerifyLocalPOSIXPermissions5GsUtilIntegrationTestCase.VerifyLocalPOSIXPermissions  sC     
4%

4%
D' r3   c                     / n[         R                  " U5       H=  u  p4nU H1  nUR                  [         R                  R	                  X65      5        M3     M?     SR	                  U5      R                  SS5      $ )zPerform a flat listing over directory.

Args:
  directory: The directory to list

Returns:
  Listings with path separators canonicalized to '/', to make assertions
  easier for Linux vs Windows.
r   \/)r  walkrD  r  r>  r  )rb   	directoryresultdirpathr  	filenamesfs          r/   FlatListDir%GsUtilIntegrationTestCase.FlatListDir  sb     F!#!3I!bggll7./  "4 99V$$T3//r3   c                 :    U R                  S[        US5      /SS9$ )z.Perform a flat listing over bucket_url_string.r   z**Tr   )r   r   )rb   bucket_url_strings     r/   FlatListBucket(GsUtilIntegrationTestCase.FlatListBucket  s*    >>4&7!>?(,  . .r3   c                 $    UR                  U5      $ )zWrapper for StorageUri.clone_replace_key().

Args:
  storage_uri: URI representing the object to be cloned
  key: key for the new StorageUri to represent
)clone_replace_key)rb   rB  keys      r/   StorageUriCloneReplaceKey3GsUtilIntegrationTestCase.StorageUriCloneReplaceKey  s     ((--r3   c                 $    UR                  U5      $ )zWrapper for StorageUri.clone_replace_name().

Args:
  storage_uri: URI representing the object to be cloned
  key: new object name
)r\  )rb   rB  rv   s      r/   StorageUriCloneReplaceName4GsUtilIntegrationTestCase.StorageUriCloneReplaceName  s     ))$//r3   c                 $    UR                  U5      $ )zWrapper for StorageUri.set_contents_from_string().

Args:
  storage_uri: URI representing the object
  contents: String of the new contents of the object
)ra  )rb   rB  r:  s      r/   StorageUriSetContentsFromString9GsUtilIntegrationTestCase.StorageUriSetContentsFromString  s     //99r3   )r`   rU   rR   rX   r[   r^   ra   rZ   )NNNNN)F)NN)Nr   NNNFFFr$  r$  NN)Nr   )NNNFNNNNNNr   N)Nr   NNFNFN)NNNNNNN)T)FFFr   NNFr+   )NNN)5__name__
__module____qualname____firstlineno____doc__GROUP_TEST_ADDRESSGROUP_TEST_IDUSER_TEST_ADDRESSUSER_TEST_IDDOMAIN_TESTra   rQ   r'   r   ro   ry   r   r   r   r   rp   r   r   r   r   r  r  r  r  rP  rG  rA  r[  r  r  r  r  r   r  
contextlibcontextmanagerr  r  r  r  r
  r  r  r  r!  r$  __static_attributes____classcell__)rc   s   @r/   rF   rF   o   s    17H 2 I +
 B:< 		QQ75 85n)<V:> "! %:N"	C;"H*@ ($ 1526#N BF/3!7F6  $ !!%$( #(&+&+&(&( ,0Ob0 ## #("&!%,- $nb $($%%) $*/(,*/04?F $(#'&*!%)04$(?IB7. 48,4
?' $## !";zIV ! !F
M
9
9(,0 .
.0: :r3   rF   )\r*  
__future__r   r   r   r   r0  r  r  rV   r  r  signalr  r  r  	threadingr   rS   r   boto.exceptionr   boto.s3.deletemarkerr	   boto.storage_urir
   r  gslib.boto_translationr   gslib.cloud_apir   r   gslib.discard_messages_queuer   gslib.exceptionr   gslib.gcs_json_apir   gslib.kms_apir   gslib.project_idr   r   gslib.tests.testcaser   gslib.tests.utiltestsr\   r   r   r   r   r   r   r   r   6gslib.third_party.storage_apitools.storage_v1_messagesthird_partystorage_apitoolsstorage_v1_messagesr   gslib.utils.constantsr   gslib.utils.encryption_helperr   r   gslib.utils.hashing_helperr   gslib.utils.metadata_utilr    r!   gslib.utils.posix_utilr"   r#   r$   r%   r&   gslib.utils.retry_utilr'   r<  	six.movesr(   rW   LOGGERr7   r;   r@   rD   
skipUnlessRUN_INTEGRATION_TESTSGsUtilTestCaserF   r,   r3   r/   <module>rS     s   8 &  % '     	 	   
      / - -  2 1 ) = , )   - . %   / 0 ) 1 2 % + R R & M A 6 : F - + , - + ( 
 			-	.
 
T//57I: 3 3 I:7I:r3   