1573 lines
67 KiB
Python
1573 lines
67 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright 2016 Google Inc. All Rights Reserved.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
|
# copy of this software and associated documentation files (the
|
|
# "Software"), to deal in the Software without restriction, including
|
|
# without limitation the rights to use, copy, modify, merge, publish, dis-
|
|
# tribute, sublicense, and/or sell copies of the Software, and to permit
|
|
# persons to whom the Software is furnished to do so, subject to the fol-
|
|
# lowing conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included
|
|
# in all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
|
|
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
|
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
|
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
# IN THE SOFTWARE.
|
|
"""Tests for gsutil UI functions."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import print_function
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import os
|
|
import pickle
|
|
|
|
import crcmod
|
|
import six
|
|
from six.moves import queue as Queue
|
|
|
|
from gslib.cs_api_map import ApiSelector
|
|
from gslib.parallel_tracker_file import ObjectFromTracker
|
|
from gslib.parallel_tracker_file import WriteParallelUploadTrackerFile
|
|
from gslib.storage_url import StorageUrlFromString
|
|
import gslib.tests.testcase as testcase
|
|
from gslib.tests.testcase.integration_testcase import SkipForS3
|
|
from gslib.tests.util import HaltingCopyCallbackHandler
|
|
from gslib.tests.util import HaltOneComponentCopyCallbackHandler
|
|
from gslib.tests.util import ObjectToURI as suri
|
|
from gslib.tests.util import SetBotoConfigForTest
|
|
from gslib.tests.util import TailSet
|
|
from gslib.tests.util import TEST_ENCRYPTION_KEY1
|
|
from gslib.tests.util import TEST_ENCRYPTION_KEY2
|
|
from gslib.tests.util import unittest
|
|
from gslib.thread_message import FileMessage
|
|
from gslib.thread_message import FinalMessage
|
|
from gslib.thread_message import MetadataMessage
|
|
from gslib.thread_message import ProducerThreadMessage
|
|
from gslib.thread_message import ProgressMessage
|
|
from gslib.thread_message import SeekAheadMessage
|
|
from gslib.tracker_file import DeleteTrackerFile
|
|
from gslib.tracker_file import GetSlicedDownloadTrackerFilePaths
|
|
from gslib.tracker_file import GetTrackerFilePath
|
|
from gslib.tracker_file import TrackerFileType
|
|
from gslib.ui_controller import BytesToFixedWidthString
|
|
from gslib.ui_controller import DataManager
|
|
from gslib.ui_controller import MainThreadUIQueue
|
|
from gslib.ui_controller import MetadataManager
|
|
from gslib.ui_controller import UIController
|
|
from gslib.ui_controller import UIThread
|
|
from gslib.utils.boto_util import UsingCrcmodExtension
|
|
from gslib.utils.constants import START_CALLBACK_PER_BYTES
|
|
from gslib.utils.constants import UTF8
|
|
from gslib.utils.copy_helper import PARALLEL_UPLOAD_STATIC_SALT
|
|
from gslib.utils.copy_helper import PARALLEL_UPLOAD_TEMP_NAMESPACE
|
|
from gslib.utils.hashing_helper import GetMd5
|
|
from gslib.utils.parallelism_framework_util import PutToQueueWithTimeout
|
|
from gslib.utils.parallelism_framework_util import ZERO_TASKS_TO_DO_ARGUMENT
|
|
from gslib.utils.retry_util import Retry
|
|
from gslib.utils.unit_util import HumanReadableWithDecimalPlaces
|
|
from gslib.utils.unit_util import MakeHumanReadable
|
|
from gslib.utils.unit_util import ONE_KIB
|
|
|
|
DOWNLOAD_SIZE = 300
|
|
UPLOAD_SIZE = 400
|
|
# Ensures at least one progress callback is made
|
|
HALT_SIZE = START_CALLBACK_PER_BYTES * 2
|
|
# After waiting this long, assume the UIThread is hung.
|
|
THREAD_WAIT_TIME = 5
|
|
|
|
|
|
def JoinThreadAndRaiseOnTimeout(ui_thread, thread_wait_time=THREAD_WAIT_TIME):
|
|
"""Joins the ui_thread and ensures it has not timed out.
|
|
|
|
Args:
|
|
ui_thread: the UIThread to be joined.
|
|
thread_wait_time: the time to wait to join
|
|
Raises:
|
|
Exception: Warns UIThread is still alive.
|
|
"""
|
|
ui_thread.join(thread_wait_time)
|
|
if ui_thread.is_alive():
|
|
raise Exception('UIThread is still alive')
|
|
|
|
|
|
def _FindAppropriateDescriptionString(metadata):
|
|
"""Returns the correspondent string (objects or files) for the operation type.
|
|
|
|
Args:
|
|
metadata: Describes whether this is a metadata operation.
|
|
Returns:
|
|
' objects' if a metadata operation; ' files' otherwise.
|
|
"""
|
|
return ' objects' if metadata else ' files'
|
|
|
|
|
|
# TODO: migrate CheckUiOutput functions to integration_testcase
|
|
# and call them directly from the adapted tests so we do not have to duplicate
|
|
# code.
|
|
def CheckUiOutputWithMFlag(test_case,
|
|
content,
|
|
num_objects,
|
|
total_size=0,
|
|
metadata=False):
|
|
"""Checks if the UI output works as expected with the -m flag enabled.
|
|
|
|
Args:
|
|
test_case: Testcase used to maintain the same assert structure.
|
|
content: The output provided by the UI.
|
|
num_objects: The number of objects processed.
|
|
total_size: The total size transferred in the operation. Used for data
|
|
operations only.
|
|
metadata: Indicates whether this is a metadata operation.
|
|
"""
|
|
description_string = _FindAppropriateDescriptionString(metadata)
|
|
# We must have transferred 100% of our data.
|
|
test_case.assertIn('100% Done', content)
|
|
# All files should be completed.
|
|
files_completed_string = str(num_objects) + '/' + str(num_objects)
|
|
test_case.assertIn(files_completed_string + description_string, content)
|
|
final_message = 'Operation completed over %s objects' % num_objects
|
|
if not metadata:
|
|
# The total_size must also been successfully transferred.
|
|
total_size_string = BytesToFixedWidthString(total_size)
|
|
test_case.assertIn(total_size_string + '/' + total_size_string, content)
|
|
final_message += '/%s' % HumanReadableWithDecimalPlaces(total_size)
|
|
test_case.assertIn(final_message, content)
|
|
|
|
|
|
def CheckUiOutputWithNoMFlag(test_case,
|
|
content,
|
|
num_objects,
|
|
total_size=0,
|
|
metadata=False):
|
|
"""Checks if the UI output works as expected with the -m flag not enabled.
|
|
|
|
Args:
|
|
test_case: Testcase used to maintain the same assert structure.
|
|
content: The output provided by the UI.
|
|
num_objects: The number of objects processed.
|
|
total_size: The total size transferred in the operation. Used for data
|
|
operations only.
|
|
metadata: Indicates whether this is a metadata operation.
|
|
"""
|
|
description_string = _FindAppropriateDescriptionString(metadata)
|
|
# All files should be completed.
|
|
files_completed_string = str(num_objects)
|
|
test_case.assertIn(files_completed_string + description_string, content)
|
|
final_message = 'Operation completed over %s objects' % num_objects
|
|
if not metadata:
|
|
# The total_size must also been successfully transferred.
|
|
total_size_string = BytesToFixedWidthString(total_size)
|
|
test_case.assertIn(total_size_string + '/' + total_size_string, content)
|
|
final_message += '/%s' % HumanReadableWithDecimalPlaces(total_size)
|
|
test_case.assertIn(final_message, content)
|
|
|
|
|
|
def CheckBrokenUiOutputWithMFlag(test_case,
|
|
content,
|
|
num_objects,
|
|
total_size=0,
|
|
metadata=False):
|
|
"""Checks if the UI output fails as expected with the -m flag enabled.
|
|
|
|
Args:
|
|
test_case: Testcase used to maintain the same assert structure.
|
|
content: The output provided by the UI.
|
|
num_objects: The number of objects processed.
|
|
total_size: The total size transferred in the operation. Used for data
|
|
operations only.
|
|
metadata: Indicates whether this is a metadata operation.
|
|
"""
|
|
description_string = _FindAppropriateDescriptionString(metadata)
|
|
# We must not have transferred 100% of our data.
|
|
test_case.assertNotIn('100% Done', content)
|
|
# We cannot have completed a file.
|
|
files_completed_string = str(num_objects) + '/' + str(num_objects)
|
|
test_case.assertNotIn(files_completed_string + description_string, content)
|
|
if not metadata:
|
|
total_size_string = BytesToFixedWidthString(total_size)
|
|
zero = BytesToFixedWidthString(0)
|
|
# Zero bytes must have been transferred in the beginning.
|
|
test_case.assertIn(zero + '/' + total_size_string, content)
|
|
# The total_size must have not been successfully transferred.
|
|
test_case.assertNotIn(total_size_string + '/' + total_size_string, content)
|
|
final_message_prefix = 'Operation completed over'
|
|
test_case.assertNotIn(final_message_prefix, content)
|
|
|
|
|
|
def CheckBrokenUiOutputWithNoMFlag(test_case,
|
|
content,
|
|
num_objects,
|
|
total_size=0,
|
|
metadata=False):
|
|
"""Checks if the UI output fails as expected with the -m flag not enabled.
|
|
|
|
Args:
|
|
test_case: Testcase used to maintain the same assert structure.
|
|
content: The output provided by the UI.
|
|
num_objects: The number of objects processed.
|
|
total_size: The total size transferred in the operation. Used for data
|
|
operations only.
|
|
metadata: Indicates whether this is a metadata operation.
|
|
"""
|
|
description_string = _FindAppropriateDescriptionString(metadata)
|
|
# 0 files should be completed.
|
|
no_files_string = str(0)
|
|
test_case.assertIn(no_files_string + description_string, content)
|
|
# We cannot have completed a file.
|
|
files_completed_string = str(num_objects)
|
|
test_case.assertNotIn(files_completed_string + description_string, content)
|
|
if not metadata:
|
|
total_size_string = BytesToFixedWidthString(total_size)
|
|
zero = BytesToFixedWidthString(0)
|
|
# Zero bytes must have been transferred in the beginning.
|
|
test_case.assertIn(zero + '/' + total_size_string, content)
|
|
# The total_size must have not been successfully transferred.
|
|
test_case.assertNotIn(total_size_string + '/' + total_size_string, content)
|
|
final_message_prefix = 'Operation completed over'
|
|
test_case.assertNotIn(final_message_prefix, content)
|
|
|
|
|
|
class TestUi(testcase.GsUtilIntegrationTestCase):
|
|
"""Integration tests for UI functions."""
|
|
|
|
def test_ui_download_single_objects_with_m_flag(self):
|
|
"""Tests UI for a single object download with the -m flag enabled.
|
|
|
|
This test indirectly tests the correctness of ProducerThreadMessage in the
|
|
UIController.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
file_contents = b'd' * DOWNLOAD_SIZE
|
|
object_uri = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='foo',
|
|
contents=file_contents)
|
|
fpath = self.CreateTempFile()
|
|
stderr = self.RunGsUtil(['-m', 'cp', suri(object_uri), fpath],
|
|
return_stderr=True)
|
|
CheckUiOutputWithMFlag(self, stderr, 1, total_size=DOWNLOAD_SIZE)
|
|
|
|
def test_ui_download_single_objects_with_no_m_flag(self):
|
|
"""Tests UI for a single object download with the -m flag not enabled.
|
|
|
|
The UI should behave differently from the -m flag option because in the
|
|
latter we have a ProducerThreadMessage that allows us to know our progress
|
|
percentage and total number of files.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
file_contents = b'd' * DOWNLOAD_SIZE
|
|
object_uri = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='foo',
|
|
contents=file_contents)
|
|
fpath = self.CreateTempFile()
|
|
stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True)
|
|
CheckUiOutputWithNoMFlag(self, stderr, 1, total_size=DOWNLOAD_SIZE)
|
|
|
|
def test_ui_upload_single_object_with_m_flag(self):
|
|
"""Tests UI for a single object upload with -m flag enabled.
|
|
|
|
This test indirectly tests the correctness of ProducerThreadMessage in the
|
|
UIController.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
file_contents = b'u' * UPLOAD_SIZE
|
|
fpath = self.CreateTempFile(file_name='sample-file.txt',
|
|
contents=file_contents)
|
|
stderr = self.RunGsUtil(
|
|
['-m', 'cp', suri(fpath), suri(bucket_uri)], return_stderr=True)
|
|
|
|
CheckUiOutputWithMFlag(self, stderr, 1, total_size=UPLOAD_SIZE)
|
|
|
|
def test_ui_upload_single_object_with_no_m_flag(self):
|
|
"""Tests UI for a single object upload with -m flag not enabled.
|
|
|
|
The UI should behave differently from the -m flag option because in the
|
|
latter we have a ProducerThreadMessage that allows us to know our progress
|
|
percentage and total number of files.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
file_contents = b'u' * UPLOAD_SIZE
|
|
fpath = self.CreateTempFile(file_name='sample-file.txt',
|
|
contents=file_contents)
|
|
stderr = self.RunGsUtil(
|
|
['cp', suri(fpath), suri(bucket_uri)], return_stderr=True)
|
|
|
|
CheckUiOutputWithNoMFlag(self, stderr, 1, total_size=UPLOAD_SIZE)
|
|
|
|
def test_ui_download_multiple_objects_with_m_flag(self):
|
|
"""Tests UI for a multiple object download with the -m flag enabled.
|
|
|
|
This test indirectly tests the correctness of ProducerThreadMessage in the
|
|
UIController.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
num_objects = 7
|
|
argument_list = ['-m', 'cp']
|
|
total_size = 0
|
|
for i in range(num_objects):
|
|
file_size = DOWNLOAD_SIZE // 3
|
|
file_contents = b'd' * file_size
|
|
object_uri = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='foo' + str(i),
|
|
contents=file_contents)
|
|
total_size += file_size
|
|
argument_list.append(suri(object_uri))
|
|
|
|
fpath = self.CreateTempDir()
|
|
argument_list.append(fpath)
|
|
stderr = self.RunGsUtil(argument_list, return_stderr=True)
|
|
|
|
CheckUiOutputWithMFlag(self, stderr, num_objects, total_size=total_size)
|
|
|
|
def test_ui_download_multiple_objects_with_no_m_flag(self):
|
|
"""Tests UI for a multiple object download with the -m flag not enabled.
|
|
|
|
The UI should behave differently from the -m flag option because in the
|
|
latter we have a ProducerThreadMessage that allows us to know our progress
|
|
percentage and total number of files.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
num_objects = 7
|
|
argument_list = ['cp']
|
|
total_size = 0
|
|
for i in range(num_objects):
|
|
file_size = DOWNLOAD_SIZE // 3
|
|
file_contents = b'd' * file_size
|
|
object_uri = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='foo' + str(i),
|
|
contents=file_contents)
|
|
total_size += file_size
|
|
argument_list.append(suri(object_uri))
|
|
|
|
fpath = self.CreateTempDir()
|
|
argument_list.append(fpath)
|
|
stderr = self.RunGsUtil(argument_list, return_stderr=True)
|
|
|
|
CheckUiOutputWithNoMFlag(self, stderr, num_objects, total_size=total_size)
|
|
|
|
def test_ui_upload_mutliple_objects_with_m_flag(self):
|
|
"""Tests UI for a multiple object upload with -m flag enabled.
|
|
|
|
This test indirectly tests the correctness of ProducerThreadMessage in the
|
|
UIController.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
num_objects = 7
|
|
argument_list = ['-m', 'cp']
|
|
total_size = 0
|
|
for i in range(num_objects):
|
|
file_size = UPLOAD_SIZE // 3
|
|
file_contents = b'u' * file_size
|
|
fpath = self.CreateTempFile(file_name='foo' + str(i),
|
|
contents=file_contents)
|
|
total_size += file_size
|
|
argument_list.append(suri(fpath))
|
|
|
|
argument_list.append(suri(bucket_uri))
|
|
stderr = self.RunGsUtil(argument_list, return_stderr=True)
|
|
|
|
CheckUiOutputWithMFlag(self, stderr, num_objects, total_size=total_size)
|
|
|
|
def test_ui_upload_mutliple_objects_with_no_m_flag(self):
|
|
"""Tests UI for a multiple object upload with -m flag not enabled.
|
|
|
|
The UI should behave differently from the -m flag option because in the
|
|
latter we have a ProducerThreadMessage that allows us to know our progress
|
|
percentage and total number of files.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
num_objects = 7
|
|
argument_list = ['cp']
|
|
total_size = 0
|
|
for i in range(num_objects):
|
|
file_size = UPLOAD_SIZE // 3
|
|
file_contents = b'u' * file_size
|
|
fpath = self.CreateTempFile(file_name='foo' + str(i),
|
|
contents=file_contents)
|
|
total_size += file_size
|
|
argument_list.append(suri(fpath))
|
|
|
|
argument_list.append(suri(bucket_uri))
|
|
stderr = self.RunGsUtil(argument_list, return_stderr=True)
|
|
|
|
CheckUiOutputWithNoMFlag(self, stderr, num_objects, total_size=total_size)
|
|
|
|
@SkipForS3('No resumable upload support for S3.')
|
|
def test_ui_resumable_upload_break_with_m_flag(self):
|
|
"""Tests UI for upload resumed after a connection break with -m flag.
|
|
|
|
This was adapted from test_cp_resumable_upload_break.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
fpath = self.CreateTempFile(contents=b'a' * HALT_SIZE)
|
|
boto_config_for_test = [
|
|
('GSUtil', 'resumable_threshold', str(ONE_KIB)),
|
|
('GSUtil', 'parallel_composite_upload_component_size', str(ONE_KIB))
|
|
]
|
|
test_callback_file = self.CreateTempFile(
|
|
contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5)))
|
|
|
|
with SetBotoConfigForTest(boto_config_for_test):
|
|
stderr = self.RunGsUtil([
|
|
'-m', 'cp', '--testcallbackfile', test_callback_file, fpath,
|
|
suri(bucket_uri)
|
|
],
|
|
expected_status=1,
|
|
return_stderr=True)
|
|
self.assertIn('Artifically halting upload', stderr)
|
|
CheckBrokenUiOutputWithMFlag(self, stderr, 1, total_size=HALT_SIZE)
|
|
stderr = self.RunGsUtil(
|
|
['-m', 'cp', fpath, suri(bucket_uri)], return_stderr=True)
|
|
self.assertIn('Resuming upload', stderr)
|
|
CheckUiOutputWithMFlag(self, stderr, 1, total_size=HALT_SIZE)
|
|
|
|
@SkipForS3('No resumable upload support for S3.')
|
|
def test_ui_resumable_upload_break_with_no_m_flag(self):
|
|
"""Tests UI for upload resumed after a connection break with no -m flag.
|
|
|
|
This was adapted from test_cp_resumable_upload_break.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
fpath = self.CreateTempFile(contents=b'a' * HALT_SIZE)
|
|
boto_config_for_test = [
|
|
('GSUtil', 'resumable_threshold', str(ONE_KIB)),
|
|
('GSUtil', 'parallel_composite_upload_component_size', str(ONE_KIB))
|
|
]
|
|
test_callback_file = self.CreateTempFile(
|
|
contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5)))
|
|
|
|
with SetBotoConfigForTest(boto_config_for_test):
|
|
stderr = self.RunGsUtil([
|
|
'cp', '--testcallbackfile', test_callback_file, fpath,
|
|
suri(bucket_uri)
|
|
],
|
|
expected_status=1,
|
|
return_stderr=True)
|
|
self.assertIn('Artifically halting upload', stderr)
|
|
CheckBrokenUiOutputWithNoMFlag(self, stderr, 1, total_size=HALT_SIZE)
|
|
stderr = self.RunGsUtil(['cp', fpath, suri(bucket_uri)],
|
|
return_stderr=True)
|
|
self.assertIn('Resuming upload', stderr)
|
|
CheckUiOutputWithNoMFlag(self, stderr, 1, total_size=HALT_SIZE)
|
|
|
|
def _test_ui_resumable_download_break_helper(self,
|
|
boto_config,
|
|
gsutil_flags=None):
|
|
"""Helper function for testing UI on a resumable download break.
|
|
|
|
This was adapted from _test_cp_resumable_download_break_helper.
|
|
|
|
Args:
|
|
boto_config: List of boto configuration tuples for use with
|
|
SetBotoConfigForTest.
|
|
gsutil_flags: List of flags to run gsutil with, or None.
|
|
"""
|
|
if not gsutil_flags:
|
|
gsutil_flags = []
|
|
bucket_uri = self.CreateBucket()
|
|
file_contents = b'a' * HALT_SIZE
|
|
object_uri = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='foo',
|
|
contents=file_contents)
|
|
fpath = self.CreateTempFile()
|
|
test_callback_file = self.CreateTempFile(
|
|
contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5)))
|
|
|
|
with SetBotoConfigForTest(boto_config):
|
|
gsutil_args = (gsutil_flags + [
|
|
'cp', '--testcallbackfile', test_callback_file,
|
|
suri(object_uri), fpath
|
|
])
|
|
stderr = self.RunGsUtil(gsutil_args,
|
|
expected_status=1,
|
|
return_stderr=True)
|
|
self.assertIn('Artifically halting download.', stderr)
|
|
if '-q' not in gsutil_flags:
|
|
if '-m' in gsutil_flags:
|
|
CheckBrokenUiOutputWithMFlag(self, stderr, 1, total_size=HALT_SIZE)
|
|
else:
|
|
CheckBrokenUiOutputWithNoMFlag(self, stderr, 1, total_size=HALT_SIZE)
|
|
tracker_filename = GetTrackerFilePath(StorageUrlFromString(fpath),
|
|
TrackerFileType.DOWNLOAD,
|
|
self.test_api)
|
|
self.assertTrue(os.path.isfile(tracker_filename))
|
|
gsutil_args = gsutil_flags + ['cp', suri(object_uri), fpath]
|
|
stderr = self.RunGsUtil(gsutil_args, return_stderr=True)
|
|
if '-q' not in gsutil_args:
|
|
self.assertIn('Resuming download', stderr)
|
|
|
|
with open(fpath, 'rb') as f:
|
|
self.assertEqual(f.read(), file_contents, 'File contents differ')
|
|
if '-q' in gsutil_flags:
|
|
self.assertEqual('', stderr)
|
|
elif '-m' in gsutil_flags:
|
|
CheckUiOutputWithMFlag(self, stderr, 1, total_size=HALT_SIZE)
|
|
else:
|
|
CheckUiOutputWithNoMFlag(self, stderr, 1, total_size=HALT_SIZE)
|
|
|
|
def test_ui_resumable_download_break_with_m_flag(self):
|
|
"""Tests UI on a resumable download break with -m flag.
|
|
|
|
This was adapted from test_cp_resumable_download_break.
|
|
"""
|
|
self._test_ui_resumable_download_break_helper(
|
|
[('GSUtil', 'resumable_threshold', str(ONE_KIB))], gsutil_flags=['-m'])
|
|
|
|
def test_ui_resumable_download_break_with_no_m_flag(self):
|
|
"""Tests UI on a resumable download break with no -m flag.
|
|
|
|
This was adapted from test_cp_resumable_download_break.
|
|
"""
|
|
self._test_ui_resumable_download_break_helper([
|
|
('GSUtil', 'resumable_threshold', str(ONE_KIB))
|
|
])
|
|
|
|
def test_ui_resumable_download_break_with_q_flag(self):
|
|
"""Tests UI on a resumable download break with -q flag but no -m flag.
|
|
|
|
This was adapted from test_cp_resumable_download_break, and the UI output
|
|
should be empty.
|
|
"""
|
|
self._test_ui_resumable_download_break_helper(
|
|
[('GSUtil', 'resumable_threshold', str(ONE_KIB))], gsutil_flags=['-q'])
|
|
|
|
def test_ui_resumable_download_break_with_q_and_m_flags(self):
|
|
"""Tests UI on a resumable download break with -q and -m flags.
|
|
|
|
This was adapted from test_cp_resumable_download_break, and the UI output
|
|
should be empty.
|
|
"""
|
|
self._test_ui_resumable_download_break_helper(
|
|
[('GSUtil', 'resumable_threshold', str(ONE_KIB))],
|
|
gsutil_flags=['-m', '-q'])
|
|
|
|
def _test_ui_composite_upload_resume_helper(self, gsutil_flags=None):
|
|
"""Helps testing UI on a resumable upload with finished components.
|
|
|
|
Args:
|
|
gsutil_flags: List of flags to run gsutil with, or None.
|
|
"""
|
|
if not gsutil_flags:
|
|
gsutil_flags = []
|
|
bucket_uri = self.CreateBucket()
|
|
dst_url = StorageUrlFromString(suri(bucket_uri, 'foo'))
|
|
|
|
file_contents = b'foobar'
|
|
file_name = 'foobar'
|
|
source_file = self.CreateTempFile(contents=file_contents,
|
|
file_name=file_name)
|
|
src_url = StorageUrlFromString(source_file)
|
|
|
|
# Simulate an upload that had occurred by writing a tracker file
|
|
# that points to a previously uploaded component.
|
|
tracker_file_name = GetTrackerFilePath(dst_url,
|
|
TrackerFileType.PARALLEL_UPLOAD,
|
|
self.test_api, src_url)
|
|
tracker_prefix = '123'
|
|
|
|
# Create component 0 to be used in the resume; it must match the name
|
|
# that will be generated in copy_helper, so we use the same scheme.
|
|
encoded_name = (PARALLEL_UPLOAD_STATIC_SALT + source_file).encode(UTF8)
|
|
content_md5 = GetMd5()
|
|
content_md5.update(encoded_name)
|
|
digest = content_md5.hexdigest()
|
|
component_object_name = (tracker_prefix + PARALLEL_UPLOAD_TEMP_NAMESPACE +
|
|
digest + '_0')
|
|
|
|
component_size = 3
|
|
object_uri = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name=component_object_name,
|
|
contents=file_contents[:component_size])
|
|
existing_component = ObjectFromTracker(component_object_name,
|
|
str(object_uri.generation))
|
|
existing_components = [existing_component]
|
|
|
|
WriteParallelUploadTrackerFile(tracker_file_name, tracker_prefix,
|
|
existing_components)
|
|
|
|
try:
|
|
# Now "resume" the upload.
|
|
with SetBotoConfigForTest([
|
|
('GSUtil', 'parallel_composite_upload_threshold', '1'),
|
|
('GSUtil', 'parallel_composite_upload_component_size',
|
|
str(component_size))
|
|
]):
|
|
gsutil_args = (
|
|
gsutil_flags +
|
|
['cp', source_file, suri(bucket_uri, 'foo')])
|
|
stderr = self.RunGsUtil(gsutil_args, return_stderr=True)
|
|
self.assertIn('Found 1 existing temporary components to reuse.', stderr)
|
|
self.assertFalse(
|
|
os.path.exists(tracker_file_name),
|
|
'Tracker file %s should have been deleted.' % tracker_file_name)
|
|
read_contents = self.RunGsUtil(['cat', suri(bucket_uri, 'foo')],
|
|
return_stdout=True)
|
|
self.assertEqual(read_contents.encode(UTF8), file_contents)
|
|
if '-m' in gsutil_flags:
|
|
CheckUiOutputWithMFlag(self, stderr, 1, total_size=len(file_contents))
|
|
else:
|
|
CheckUiOutputWithNoMFlag(self,
|
|
stderr,
|
|
1,
|
|
total_size=len(file_contents))
|
|
finally:
|
|
# Clean up if something went wrong.
|
|
DeleteTrackerFile(tracker_file_name)
|
|
|
|
@SkipForS3('No resumable upload support for S3.')
|
|
def test_ui_composite_upload_resume_with_m_flag(self):
|
|
"""Tests UI on a resumable upload with finished components and -m flag."""
|
|
self._test_ui_composite_upload_resume_helper(gsutil_flags=['-m'])
|
|
|
|
@SkipForS3('No resumable upload support for S3.')
|
|
def test_ui_composite_upload_resume_with_no_m_flag(self):
|
|
"""Tests UI on a resumable upload with finished components and no -m flag.
|
|
"""
|
|
self._test_ui_composite_upload_resume_helper()
|
|
|
|
@unittest.skipUnless(UsingCrcmodExtension(),
|
|
'Sliced download requires fast crcmod.')
|
|
@SkipForS3('No sliced download support for S3.')
|
|
def _test_ui_sliced_download_partial_resume_helper(self, gsutil_flags=None):
|
|
"""Helps testing UI for sliced download with some finished components.
|
|
|
|
This was adapted from test_sliced_download_partial_resume_helper.
|
|
|
|
Args:
|
|
gsutil_flags: List of flags to run gsutil with, or None.
|
|
"""
|
|
if not gsutil_flags:
|
|
gsutil_flags = []
|
|
bucket_uri = self.CreateBucket()
|
|
object_uri = self.CreateObject(bucket_uri=bucket_uri,
|
|
object_name='foo',
|
|
contents=b'abc' * HALT_SIZE)
|
|
fpath = self.CreateTempFile()
|
|
test_callback_file = self.CreateTempFile(
|
|
contents=pickle.dumps(HaltOneComponentCopyCallbackHandler(5)))
|
|
|
|
boto_config_for_test = [
|
|
('GSUtil', 'resumable_threshold', str(HALT_SIZE)),
|
|
('GSUtil', 'sliced_object_download_threshold', str(HALT_SIZE)),
|
|
('GSUtil', 'sliced_object_download_max_components', '3')
|
|
]
|
|
|
|
with SetBotoConfigForTest(boto_config_for_test):
|
|
gsutil_args = gsutil_flags + [
|
|
'cp', '--testcallbackfile', test_callback_file,
|
|
suri(object_uri),
|
|
suri(fpath)
|
|
]
|
|
|
|
stderr = self.RunGsUtil(gsutil_args,
|
|
return_stderr=True,
|
|
expected_status=1)
|
|
if '-m' in gsutil_args:
|
|
CheckBrokenUiOutputWithMFlag(self,
|
|
stderr,
|
|
1,
|
|
total_size=(len('abc') * HALT_SIZE))
|
|
else:
|
|
CheckBrokenUiOutputWithNoMFlag(self,
|
|
stderr,
|
|
1,
|
|
total_size=(len('abc') * HALT_SIZE))
|
|
# Each tracker file should exist.
|
|
tracker_filenames = GetSlicedDownloadTrackerFilePaths(
|
|
StorageUrlFromString(fpath), self.test_api)
|
|
for tracker_filename in tracker_filenames:
|
|
self.assertTrue(os.path.isfile(tracker_filename))
|
|
gsutil_args = gsutil_flags + ['cp', suri(object_uri), fpath]
|
|
|
|
stderr = self.RunGsUtil(gsutil_args, return_stderr=True)
|
|
self.assertIn('Resuming download', stderr)
|
|
self.assertIn('Download already complete', stderr)
|
|
|
|
# Each tracker file should have been deleted.
|
|
tracker_filenames = GetSlicedDownloadTrackerFilePaths(
|
|
StorageUrlFromString(fpath), self.test_api)
|
|
for tracker_filename in tracker_filenames:
|
|
self.assertFalse(os.path.isfile(tracker_filename))
|
|
|
|
with open(fpath, 'r') as f:
|
|
self.assertEqual(f.read(), 'abc' * HALT_SIZE, 'File contents differ')
|
|
if '-m' in gsutil_args:
|
|
CheckUiOutputWithMFlag(self,
|
|
stderr,
|
|
1,
|
|
total_size=(len('abc') * HALT_SIZE))
|
|
else:
|
|
CheckUiOutputWithNoMFlag(self,
|
|
stderr,
|
|
1,
|
|
total_size=(len('abc') * HALT_SIZE))
|
|
|
|
@SkipForS3('No resumable upload support for S3.')
|
|
def test_ui_sliced_download_partial_resume_helper_with_m_flag(self):
|
|
"""Tests UI on a resumable download with finished components and -m flag.
|
|
"""
|
|
self._test_ui_sliced_download_partial_resume_helper(gsutil_flags=['-m'])
|
|
|
|
@SkipForS3('No resumable upload support for S3.')
|
|
def _test_ui_sliced_download_partial_resume_helper_with_no_m_flag(self):
|
|
"""Tests UI on a resumable upload with finished components and no -m flag.
|
|
"""
|
|
self._test_ui_sliced_download_partial_resume_helper()
|
|
|
|
def test_ui_hash_mutliple_objects_with_no_m_flag(self):
|
|
"""Tests UI for a multiple object hashing with no -m flag enabled.
|
|
|
|
This test indirectly tests the correctness of ProducerThreadMessage in the
|
|
UIController.
|
|
"""
|
|
num_objects = 7
|
|
argument_list = ['hash']
|
|
total_size = 0
|
|
for i in range(num_objects):
|
|
file_size = UPLOAD_SIZE // 3
|
|
file_contents = b'u' * file_size
|
|
fpath = self.CreateTempFile(file_name='foo' + str(i),
|
|
contents=file_contents)
|
|
total_size += file_size
|
|
argument_list.append(suri(fpath))
|
|
|
|
stderr = self.RunGsUtil(argument_list, return_stderr=True)
|
|
CheckUiOutputWithNoMFlag(self, stderr, num_objects, total_size)
|
|
|
|
def test_ui_rewrite_with_m_flag(self):
|
|
"""Tests UI output for rewrite and -m flag.
|
|
|
|
Adapted from test_rewrite_stdin_args.
|
|
"""
|
|
if self.test_api == ApiSelector.XML:
|
|
return unittest.skip('Rewrite API is only supported in JSON.')
|
|
object_uri = self.CreateObject(contents=b'bar',
|
|
encryption_key=TEST_ENCRYPTION_KEY1)
|
|
stdin_arg = suri(object_uri)
|
|
|
|
boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
|
|
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
|
|
with SetBotoConfigForTest(boto_config_for_test):
|
|
stderr = self.RunGsUtil(['-m', 'rewrite', '-k', '-I'],
|
|
stdin=stdin_arg,
|
|
return_stderr=True)
|
|
self.AssertObjectUsesCSEK(stdin_arg, TEST_ENCRYPTION_KEY2)
|
|
num_objects = 1
|
|
total_size = len(b'bar')
|
|
CheckUiOutputWithMFlag(self, stderr, num_objects, total_size)
|
|
|
|
def test_ui_rewrite_with_no_m_flag(self):
|
|
"""Tests UI output for rewrite and -m flag not enabled.
|
|
|
|
Adapted from test_rewrite_stdin_args.
|
|
"""
|
|
if self.test_api == ApiSelector.XML:
|
|
return unittest.skip('Rewrite API is only supported in JSON.')
|
|
object_uri = self.CreateObject(contents=b'bar',
|
|
encryption_key=TEST_ENCRYPTION_KEY1)
|
|
stdin_arg = suri(object_uri)
|
|
|
|
boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2),
|
|
('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)]
|
|
with SetBotoConfigForTest(boto_config_for_test):
|
|
stderr = self.RunGsUtil(['rewrite', '-k', '-I'],
|
|
stdin=stdin_arg,
|
|
return_stderr=True)
|
|
self.AssertObjectUsesCSEK(stdin_arg, TEST_ENCRYPTION_KEY2)
|
|
num_objects = 1
|
|
total_size = len(b'bar')
|
|
CheckUiOutputWithNoMFlag(self, stderr, num_objects, total_size)
|
|
|
|
def test_ui_setmeta_with_m_flag(self):
|
|
"""Tests a recursive setmeta command with m flag has expected UI output.
|
|
|
|
Adapted from test_recursion_works on test_setmeta.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
object1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo')
|
|
object2_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo')
|
|
stderr = self.RunGsUtil([
|
|
'-m', 'setmeta', '-h', 'content-type:footype',
|
|
suri(object1_uri),
|
|
suri(object2_uri)
|
|
],
|
|
return_stderr=True)
|
|
|
|
for obj_uri in [object1_uri, object2_uri]:
|
|
stdout = self.RunGsUtil(['stat', suri(obj_uri)], return_stdout=True)
|
|
self.assertIn('footype', stdout)
|
|
CheckUiOutputWithMFlag(self, stderr, 2, metadata=True)
|
|
|
|
def test_ui_setmeta_with_no_m_flag(self):
|
|
"""Tests a recursive setmeta command with no m flag has expected UI output.
|
|
|
|
Adapted from test_recursion_works on test_setmeta.
|
|
"""
|
|
bucket_uri = self.CreateBucket()
|
|
object1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo')
|
|
object2_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo')
|
|
stderr = self.RunGsUtil([
|
|
'setmeta', '-h', 'content-type:footype',
|
|
suri(object1_uri),
|
|
suri(object2_uri)
|
|
],
|
|
return_stderr=True)
|
|
|
|
for obj_uri in [object1_uri, object2_uri]:
|
|
stdout = self.RunGsUtil(['stat', suri(obj_uri)], return_stdout=True)
|
|
self.assertIn('footype', stdout)
|
|
CheckUiOutputWithNoMFlag(self, stderr, 2, metadata=True)
|
|
|
|
def test_ui_acl_with_m_flag(self):
|
|
"""Tests UI output for an ACL command with m flag enabled.
|
|
|
|
Adapted from test_set_valid_acl_object.
|
|
"""
|
|
get_acl_prefix = ['-m', 'acl', 'get']
|
|
set_acl_prefix = ['-m', 'acl', 'set']
|
|
obj_uri = suri(self.CreateObject(contents=b'foo'))
|
|
acl_string = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True)
|
|
inpath = self.CreateTempFile(contents=acl_string.encode(UTF8))
|
|
stderr = self.RunGsUtil(set_acl_prefix + ['public-read', obj_uri],
|
|
return_stderr=True)
|
|
CheckUiOutputWithMFlag(self, stderr, 1, metadata=True)
|
|
acl_string2 = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True)
|
|
stderr = self.RunGsUtil(set_acl_prefix + [inpath, obj_uri],
|
|
return_stderr=True)
|
|
CheckUiOutputWithMFlag(self, stderr, 1, metadata=True)
|
|
acl_string3 = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True)
|
|
|
|
self.assertNotEqual(acl_string, acl_string2)
|
|
self.assertEqual(acl_string, acl_string3)
|
|
|
|
def test_ui_acl_with_no_m_flag(self):
|
|
"""Tests UI output for an ACL command with m flag not enabled.
|
|
|
|
Adapted from test_set_valid_acl_object.
|
|
"""
|
|
get_acl_prefix = ['acl', 'get']
|
|
set_acl_prefix = ['acl', 'set']
|
|
obj_uri = suri(self.CreateObject(contents=b'foo'))
|
|
acl_string = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True)
|
|
inpath = self.CreateTempFile(contents=acl_string.encode(UTF8))
|
|
stderr = self.RunGsUtil(set_acl_prefix + ['public-read', obj_uri],
|
|
return_stderr=True)
|
|
CheckUiOutputWithNoMFlag(self, stderr, 1, metadata=True)
|
|
acl_string2 = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True)
|
|
stderr = self.RunGsUtil(set_acl_prefix + [inpath, obj_uri],
|
|
return_stderr=True)
|
|
CheckUiOutputWithNoMFlag(self, stderr, 1, metadata=True)
|
|
acl_string3 = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True)
|
|
|
|
self.assertNotEqual(acl_string, acl_string2)
|
|
self.assertEqual(acl_string, acl_string3)
|
|
|
|
def _test_ui_rsync_bucket_to_bucket_helper(self, gsutil_flags=None):
|
|
"""Helper class to test UI output for rsync command.
|
|
|
|
Args:
|
|
gsutil_flags: List of flags to run gsutil with, or None.
|
|
|
|
Adapted from test_bucket_to_bucket in test_rsync.
|
|
"""
|
|
if not gsutil_flags:
|
|
gsutil_flags = []
|
|
# Create 2 buckets with 1 overlapping object, 1 extra object at root level
|
|
# in each, and 1 extra object 1 level down in each, where one of the objects
|
|
# starts with "." to test that we don't skip those objects. Make the
|
|
# overlapping objects named the same but with different content, to test
|
|
# that we detect and properly copy in that case.
|
|
bucket1_uri = self.CreateBucket()
|
|
bucket2_uri = self.CreateBucket()
|
|
self.CreateObject(bucket_uri=bucket1_uri,
|
|
object_name='obj1',
|
|
contents=b'obj1')
|
|
self.CreateObject(bucket_uri=bucket1_uri,
|
|
object_name='.obj2',
|
|
contents=b'.obj2',
|
|
mtime=10)
|
|
self.CreateObject(bucket_uri=bucket1_uri,
|
|
object_name='subdir/obj3',
|
|
contents=b'subdir/obj3')
|
|
self.CreateObject(bucket_uri=bucket1_uri,
|
|
object_name='obj6',
|
|
contents=b'obj6_',
|
|
mtime=100)
|
|
# .obj2 will be replaced and have mtime of 10
|
|
self.CreateObject(bucket_uri=bucket2_uri,
|
|
object_name='.obj2',
|
|
contents=b'.OBJ2')
|
|
self.CreateObject(bucket_uri=bucket2_uri,
|
|
object_name='obj4',
|
|
contents=b'obj4')
|
|
self.CreateObject(bucket_uri=bucket2_uri,
|
|
object_name='subdir/obj5',
|
|
contents=b'subdir/obj5')
|
|
self.CreateObject(bucket_uri=bucket2_uri,
|
|
object_name='obj6',
|
|
contents=b'obj6',
|
|
mtime=100)
|
|
|
|
# Use @Retry as hedge against bucket listing eventual consistency.
|
|
@Retry(AssertionError, tries=3, timeout_secs=1)
|
|
def _Check1():
|
|
"""Tests rsync works as expected."""
|
|
gsutil_args = (gsutil_flags +
|
|
['rsync', suri(bucket1_uri),
|
|
suri(bucket2_uri)])
|
|
stderr = self.RunGsUtil(gsutil_args, return_stderr=True)
|
|
num_objects = 3
|
|
total_size = len('obj1') + len('.obj2') + len('obj6_')
|
|
CheckUiOutputWithNoMFlag(self, stderr, num_objects, total_size)
|
|
listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri))
|
|
listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri))
|
|
# First bucket should have un-altered content.
|
|
self.assertEqual(listing1,
|
|
set(['/obj1', '/.obj2', '/subdir/obj3', '/obj6']))
|
|
# Second bucket should have new objects added from source bucket (without
|
|
# removing extraneeous object found in dest bucket), and without the
|
|
# subdir objects synchronized.
|
|
self.assertEqual(
|
|
listing2, set(['/obj1', '/.obj2', '/obj4', '/subdir/obj5', '/obj6']))
|
|
# Assert that the src/dest objects that had same length but different
|
|
# content were correctly synchronized (bucket to bucket rsync uses
|
|
# checksums).
|
|
self.assertEqual(
|
|
'.obj2',
|
|
self.RunGsUtil(['cat', suri(bucket1_uri, '.obj2')],
|
|
return_stdout=True))
|
|
self.assertEqual(
|
|
'.obj2',
|
|
self.RunGsUtil(['cat', suri(bucket2_uri, '.obj2')],
|
|
return_stdout=True))
|
|
self.assertEqual(
|
|
'obj6_',
|
|
self.RunGsUtil(['cat', suri(bucket2_uri, 'obj6')],
|
|
return_stdout=True))
|
|
|
|
_Check1()
|
|
|
|
def test_ui_rsync_bucket_to_bucket_with_m_flag(self):
|
|
"""Tests UI output for rsync with -m flag enabled works as expected."""
|
|
self._test_ui_rsync_bucket_to_bucket_helper(gsutil_flags=['-m'])
|
|
|
|
def test_ui_rsync_bucket_to_bucket_with_no_m_flag(self):
|
|
"""Tests UI output for rsync with -m flag not enabled works as expected."""
|
|
self._test_ui_rsync_bucket_to_bucket_helper()
|
|
|
|
|
|
class TestUiUnitTests(testcase.GsUtilUnitTestCase):
|
|
"""Unit tests for UI functions."""
|
|
|
|
upload_size = UPLOAD_SIZE
|
|
start_time = 10000
|
|
|
|
def test_ui_seek_ahead_message(self):
|
|
"""Tests if a seek ahead message is correctly printed."""
|
|
status_queue = Queue.Queue()
|
|
stream = six.StringIO()
|
|
# No time constraints for displaying messages.
|
|
start_time = self.start_time
|
|
ui_controller = UIController(0, 0, 0, 0, custom_time=start_time)
|
|
ui_thread = UIThread(status_queue, stream, ui_controller)
|
|
num_objects = 10
|
|
total_size = 1024**3
|
|
PutToQueueWithTimeout(status_queue,
|
|
SeekAheadMessage(num_objects, total_size, start_time))
|
|
|
|
# Adds a file. Because this message was already theoretically processed
|
|
# by the SeekAheadThread, the number of files reported by the UIController
|
|
# should not change.
|
|
fpath = self.CreateTempFile(file_name='sample-file.txt', contents=b'foo')
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(StorageUrlFromString(suri(fpath)),
|
|
None,
|
|
start_time + 10,
|
|
size=UPLOAD_SIZE,
|
|
message_type=FileMessage.FILE_UPLOAD,
|
|
finished=False))
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(StorageUrlFromString(suri(fpath)),
|
|
None,
|
|
start_time + 20,
|
|
size=UPLOAD_SIZE,
|
|
message_type=FileMessage.FILE_UPLOAD,
|
|
finished=True))
|
|
|
|
PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT)
|
|
JoinThreadAndRaiseOnTimeout(ui_thread)
|
|
content = stream.getvalue()
|
|
expected_message = (
|
|
'Estimated work for this command: objects: %s, total size: %s\n' %
|
|
(num_objects, MakeHumanReadable(total_size)))
|
|
self.assertIn(expected_message, content)
|
|
# This ensures the SeekAheadMessage did its job.
|
|
self.assertIn('/' + str(num_objects), content)
|
|
# This ensures a FileMessage did not affect the total number of files
|
|
# obtained by the SeekAheadMessage.
|
|
self.assertNotIn('/' + str(num_objects + 1), content)
|
|
|
|
def test_ui_seek_ahead_zero_size(self):
|
|
"""Tests the case where the SeekAheadThread returns total size of 0."""
|
|
current_time_ms = self.start_time
|
|
status_queue = Queue.Queue()
|
|
stream = six.StringIO()
|
|
ui_controller = UIController(custom_time=current_time_ms)
|
|
ui_thread = UIThread(status_queue, stream, ui_controller)
|
|
PutToQueueWithTimeout(status_queue,
|
|
SeekAheadMessage(100, 0, current_time_ms))
|
|
for i in range(100):
|
|
current_time_ms += 200
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(StorageUrlFromString('gs://foo%s' % i),
|
|
StorageUrlFromString('bar%s' % i),
|
|
current_time_ms,
|
|
message_type=FileMessage.FILE_DOWNLOAD))
|
|
for i in range(100):
|
|
current_time_ms += 200
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(StorageUrlFromString('gs://foo%s' % i),
|
|
StorageUrlFromString('bar%s' % i),
|
|
current_time_ms,
|
|
finished=True,
|
|
message_type=FileMessage.FILE_DOWNLOAD))
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
ProducerThreadMessage(100, 0, current_time_ms, finished=True))
|
|
PutToQueueWithTimeout(status_queue, FinalMessage(current_time_ms))
|
|
PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT)
|
|
JoinThreadAndRaiseOnTimeout(ui_thread)
|
|
self.assertIn('100/100', stream.getvalue())
|
|
|
|
def test_ui_empty_list(self):
|
|
"""Tests if status queue is empty after processed by UIThread."""
|
|
status_queue = Queue.Queue()
|
|
stream = six.StringIO()
|
|
ui_controller = UIController()
|
|
ui_thread = UIThread(status_queue, stream, ui_controller)
|
|
for i in range(10000): # pylint: disable=unused-variable
|
|
PutToQueueWithTimeout(status_queue, 'foo')
|
|
PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT)
|
|
JoinThreadAndRaiseOnTimeout(ui_thread)
|
|
self.assertEqual(0, status_queue.qsize())
|
|
|
|
def test_ui_controller_shared_states(self):
|
|
"""Tests that UIController correctly integrates messages.
|
|
|
|
This test ensures UIController correctly shares its state, which is used by
|
|
both UIThread and MainThreadUIQueue. There are multiple ways of checking
|
|
that. One such way is to create a ProducerThreadMessage on the
|
|
MainThreadUIQueue, simulate a upload with messages coming from the UIThread,
|
|
and check if the output has the percentage done and number of files
|
|
(both happen only when a ProducerThreadMessage or SeekAheadMessage is
|
|
called).
|
|
"""
|
|
ui_thread_status_queue = Queue.Queue()
|
|
stream = six.StringIO()
|
|
# No time constraints for displaying messages.
|
|
start_time = self.start_time
|
|
ui_controller = UIController(0, 0, 0, 0, custom_time=start_time)
|
|
main_thread_ui_queue = MainThreadUIQueue(stream, ui_controller)
|
|
ui_thread = UIThread(ui_thread_status_queue, stream, ui_controller)
|
|
PutToQueueWithTimeout(
|
|
main_thread_ui_queue,
|
|
ProducerThreadMessage(1, UPLOAD_SIZE, start_time, finished=True))
|
|
fpath = self.CreateTempFile(file_name='sample-file.txt', contents=b'foo')
|
|
PutToQueueWithTimeout(
|
|
ui_thread_status_queue,
|
|
FileMessage(StorageUrlFromString(suri(fpath)),
|
|
None,
|
|
start_time + 10,
|
|
size=UPLOAD_SIZE,
|
|
message_type=FileMessage.FILE_UPLOAD,
|
|
finished=False))
|
|
PutToQueueWithTimeout(
|
|
ui_thread_status_queue,
|
|
FileMessage(StorageUrlFromString(suri(fpath)),
|
|
None,
|
|
start_time + 20,
|
|
size=UPLOAD_SIZE,
|
|
message_type=FileMessage.FILE_UPLOAD,
|
|
finished=True))
|
|
PutToQueueWithTimeout(ui_thread_status_queue, FinalMessage(start_time + 50))
|
|
PutToQueueWithTimeout(ui_thread_status_queue, ZERO_TASKS_TO_DO_ARGUMENT)
|
|
JoinThreadAndRaiseOnTimeout(ui_thread)
|
|
content = stream.getvalue()
|
|
CheckUiOutputWithMFlag(self, content, 1, UPLOAD_SIZE)
|
|
|
|
def test_ui_throughput_calculation_with_components(self):
|
|
"""Tests throughput calculation in the UI.
|
|
|
|
This test takes two different values, both with a different size and
|
|
different number of components, and see if throughput behaves as expected.
|
|
"""
|
|
status_queue = Queue.Queue()
|
|
stream = six.StringIO()
|
|
# Creates a UIController that has no time constraints for updating info,
|
|
# except for having to wait at least 2 seconds (considering the time
|
|
# informed by the messages) to update the throughput. We use a value
|
|
# slightly smaller than 2 to ensure messages that are 2 seconds apart from
|
|
# one another will be enough to calculate throughput.
|
|
start_time = self.start_time
|
|
ui_controller = UIController(sliding_throughput_period=2,
|
|
update_message_period=1,
|
|
first_throughput_latency=0,
|
|
custom_time=start_time)
|
|
# We use start_time to have a reasonable set of values for the time messages
|
|
# processed by the UIController. However, the start_time does not influence
|
|
# this test, as the throughput is calculated based on the time
|
|
# difference between two messages, which is fixed in this test.
|
|
|
|
ui_thread = UIThread(status_queue, stream, ui_controller)
|
|
fpath1 = self.CreateTempFile(file_name='sample-file.txt', contents=b'foo')
|
|
fpath2 = self.CreateTempFile(file_name='sample-file2.txt', contents=b'FOO')
|
|
|
|
def _CreateFileVariables(alpha, component_number, src_url):
|
|
"""Creates size and component_size for a given file."""
|
|
size = 1024**2 * 60 * alpha # this is 60*alpha MiB
|
|
component_size = size / component_number
|
|
return (size, component_number, component_size, src_url)
|
|
|
|
# Note: size1 and size2 do not actually correspond to the actual sizes of
|
|
# fpath1 and fpath2. However, the UIController only uses the size sent on
|
|
# the message, so we should be able to pretend they are much larger on size.
|
|
(size1, component_num_file1, component_size_file1,
|
|
src_url1) = (_CreateFileVariables(1, 3,
|
|
StorageUrlFromString(suri(fpath1))))
|
|
|
|
(size2, component_num_file2, component_size_file2,
|
|
src_url2) = (_CreateFileVariables(10, 4,
|
|
StorageUrlFromString(suri(fpath2))))
|
|
|
|
for file_message_type, component_message_type, operation_name in (
|
|
(FileMessage.FILE_UPLOAD, FileMessage.COMPONENT_TO_UPLOAD,
|
|
'Uploading'), (FileMessage.FILE_DOWNLOAD,
|
|
FileMessage.COMPONENT_TO_DOWNLOAD, 'Downloading')):
|
|
# Testing for uploads and downloads
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url1,
|
|
None,
|
|
start_time + 100,
|
|
size=size1,
|
|
message_type=file_message_type))
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url2,
|
|
None,
|
|
start_time + 150,
|
|
size=size2,
|
|
message_type=file_message_type))
|
|
|
|
for i in range(component_num_file1):
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url1,
|
|
None,
|
|
start_time + 200 + i,
|
|
size=component_size_file1,
|
|
component_num=i,
|
|
message_type=component_message_type))
|
|
for i in range(component_num_file2):
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url2,
|
|
None,
|
|
start_time + 250 + i,
|
|
size=component_size_file2,
|
|
component_num=i,
|
|
message_type=component_message_type))
|
|
|
|
progress_calls_number = 4
|
|
for j in range(1, progress_calls_number + 1):
|
|
# We will send progress_calls_number ProgressMessages for each
|
|
# component.
|
|
base_start_time = (start_time + 300 + (j - 1) *
|
|
(component_num_file1 + component_num_file2))
|
|
|
|
for i in range(component_num_file1):
|
|
# Each component has size equal to
|
|
# component_size_file1/progress_calls_number
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
ProgressMessage(size1,
|
|
j * component_size_file1 / progress_calls_number,
|
|
src_url1,
|
|
base_start_time + i,
|
|
component_num=i,
|
|
operation_name=operation_name))
|
|
|
|
for i in range(component_num_file2):
|
|
# Each component has size equal to
|
|
# component_size_file2/progress_calls_number
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
ProgressMessage(size2,
|
|
j * component_size_file2 / progress_calls_number,
|
|
src_url2,
|
|
base_start_time + component_num_file1 + i,
|
|
component_num=i,
|
|
operation_name=operation_name))
|
|
|
|
# Time to finish the components and files.
|
|
for i in range(component_num_file1):
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url1,
|
|
None,
|
|
start_time + 500 + i,
|
|
finished=True,
|
|
size=component_size_file1,
|
|
component_num=i,
|
|
message_type=component_message_type))
|
|
for i in range(component_num_file2):
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url2,
|
|
None,
|
|
start_time + 600 + i,
|
|
finished=True,
|
|
size=component_size_file2,
|
|
component_num=i,
|
|
message_type=component_message_type))
|
|
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url1,
|
|
None,
|
|
start_time + 700,
|
|
size=size1,
|
|
finished=True,
|
|
message_type=file_message_type))
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url2,
|
|
None,
|
|
start_time + 800,
|
|
size=size2,
|
|
finished=True,
|
|
message_type=file_message_type))
|
|
|
|
PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT)
|
|
JoinThreadAndRaiseOnTimeout(ui_thread)
|
|
content = stream.getvalue()
|
|
# There were 2-second periods when no progress was reported. The
|
|
# throughput here will be 0. We will use BytesToFixedWidthString(0)
|
|
# to ensure that any changes to the function are applied here as well.
|
|
zero = BytesToFixedWidthString(0)
|
|
self.assertIn(zero + '/s', content)
|
|
file1_progress = (size1 / (component_num_file1 * progress_calls_number))
|
|
file2_progress = (size2 / (component_num_file2 * progress_calls_number))
|
|
# There were 2-second periods when only two progresses from file1
|
|
# were reported. The throughput here will be file1_progress.
|
|
self.assertIn(BytesToFixedWidthString(file1_progress) + '/s', content)
|
|
# There were 2-second periods when only two progresses from file2
|
|
# were reported. The throughput here will be file2_progress.
|
|
self.assertIn(BytesToFixedWidthString(file2_progress) + '/s', content)
|
|
# For each loop iteration, there are two 2-second periods when
|
|
# one progress from each file is reported: in the middle of the
|
|
# iteration, and in the end of the iteration along with the beginning
|
|
# of the following iteration, on a total of
|
|
# 2 * progress_calls_number - 1 occurrences (-1 due to only 1 occurrence
|
|
# on the last iteration).
|
|
# The throughput here will be (file1_progress + file2_progress) / 2.
|
|
average_progress = BytesToFixedWidthString(
|
|
(file1_progress + file2_progress) / 2)
|
|
self.assertEqual(content.count(average_progress + '/s'),
|
|
2 * progress_calls_number - 1)
|
|
|
|
def test_ui_throughput_calculation_with_no_components(self):
|
|
"""Tests throughput calculation in the UI.
|
|
|
|
This test takes two different values, both with a different size and
|
|
different number of components, and see if throughput behaves as expected.
|
|
"""
|
|
status_queue = Queue.Queue()
|
|
stream = six.StringIO()
|
|
# Creates a UIController that has no time constraints for updating info,
|
|
# except for having to wait at least 2 seconds(considering the time informed
|
|
# by the messages) to update the throughput. We use a value slightly smaller
|
|
# than 2 to ensure messages that are 2 seconds apart from one another will
|
|
# be enough to calculate throughput.
|
|
start_time = self.start_time
|
|
ui_controller = UIController(sliding_throughput_period=2,
|
|
update_message_period=1,
|
|
first_throughput_latency=0,
|
|
custom_time=start_time)
|
|
# We use start_time to have a reasonable set of values for the time messages
|
|
# processed by the UIController. However, the start_time does not influence
|
|
# much this test, as the throughput is calculated based on the time
|
|
# difference between two messages, which is fixed in this text.
|
|
|
|
ui_thread = UIThread(status_queue, stream, ui_controller)
|
|
fpath1 = self.CreateTempFile(file_name='sample-file.txt', contents=b'foo')
|
|
fpath2 = self.CreateTempFile(file_name='sample-file2.txt', contents=b'FOO')
|
|
|
|
# Note: size1 and size2 do not actually correspond to the actual sizes of
|
|
# fpath1 and fpath2. However, the UIController only uses the size sent on
|
|
# the message, so we should be able to pretend they are much larger on size.
|
|
size1 = 1024**2 * 60
|
|
src_url1 = StorageUrlFromString(suri(fpath1))
|
|
size2 = 1024**2 * 600
|
|
src_url2 = StorageUrlFromString(suri(fpath2))
|
|
|
|
for file_message_type, operation_name in ((FileMessage.FILE_UPLOAD,
|
|
'Uploading'),
|
|
(FileMessage.FILE_DOWNLOAD,
|
|
'Downloading')):
|
|
# Testing for uploads and downloads
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url1,
|
|
None,
|
|
start_time + 200,
|
|
size=size1,
|
|
message_type=file_message_type))
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url2,
|
|
None,
|
|
start_time + 301,
|
|
size=size2,
|
|
message_type=file_message_type))
|
|
progress_calls_number = 4
|
|
for j in range(1, progress_calls_number + 1):
|
|
# We will send progress_calls_number ProgressMessages for each file.
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
ProgressMessage(size1,
|
|
j * size1 / 4,
|
|
src_url1,
|
|
start_time + 300 + j * 2,
|
|
operation_name=operation_name))
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
ProgressMessage(size2,
|
|
j * size2 / 4,
|
|
src_url2,
|
|
start_time + 300 + j * 2 + 1,
|
|
operation_name=operation_name))
|
|
|
|
# Time to finish the files.
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url1,
|
|
None,
|
|
start_time + 700,
|
|
size=size1,
|
|
finished=True,
|
|
message_type=file_message_type))
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(src_url2,
|
|
None,
|
|
start_time + 800,
|
|
size=size2,
|
|
finished=True,
|
|
message_type=file_message_type))
|
|
|
|
PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT)
|
|
JoinThreadAndRaiseOnTimeout(ui_thread)
|
|
content = stream.getvalue()
|
|
# There were 2-second periods when no progress was reported. The
|
|
# throughput here will be 0. We will use BytesToFixedWidthString(0)
|
|
# to ensure that any changes to the function are applied here as well.
|
|
zero = BytesToFixedWidthString(0)
|
|
self.assertIn(zero + '/s', content)
|
|
file1_progress = (size1 / progress_calls_number)
|
|
file2_progress = (size2 / progress_calls_number)
|
|
# For each loop iteration, there are two 2-second periods when
|
|
# one progress from each file is reported: in the middle of the
|
|
# iteration, and in the end of the iteration along with the beginning
|
|
# of the following iteration, on a total of
|
|
# 2 * progress_calls_number - 1 occurrences (-1 due to only 1 occurrence
|
|
# on the last iteration).
|
|
# The throughput here will be (file1_progress + file2_progress) / 2.
|
|
average_progress = BytesToFixedWidthString(
|
|
(file1_progress + file2_progress) / 2)
|
|
self.assertEqual(content.count(average_progress + '/s'),
|
|
2 * progress_calls_number - 1)
|
|
|
|
def test_ui_metadata_message_passing(self):
|
|
"""Tests that MetadataMessages are being correctly received and processed.
|
|
|
|
This also tests the relation and hierarchy between different estimation
|
|
sources, as represented by the EstimationSource class.
|
|
"""
|
|
status_queue = Queue.Queue()
|
|
stream = six.StringIO()
|
|
# Creates a UIController that has no time constraints for updating info,
|
|
# except for having to wait at least 2 seconds(considering the time informed
|
|
# by the messages) to update the throughput. We use a value slightly smaller
|
|
# than 2 to ensure messages that are 2 seconds apart from one another will
|
|
# be enough to calculate throughput.
|
|
start_time = self.start_time
|
|
ui_controller = UIController(sliding_throughput_period=2,
|
|
update_message_period=1,
|
|
first_throughput_latency=0,
|
|
custom_time=start_time)
|
|
num_objects = 200
|
|
ui_thread = UIThread(status_queue, stream, ui_controller)
|
|
for i in range(num_objects):
|
|
if i < 100:
|
|
PutToQueueWithTimeout(status_queue,
|
|
MetadataMessage(start_time + 0.1 * i))
|
|
elif i < 130:
|
|
if i == 100:
|
|
# Sends an estimation message
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
ProducerThreadMessage(130, 0, start_time + 0.1 + 0.1 * i))
|
|
PutToQueueWithTimeout(
|
|
status_queue, MetadataMessage(start_time + 10 + 0.2 * (i - 100)))
|
|
elif i < 150:
|
|
if i == 130:
|
|
# Sends a SeekAheadMessage
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
SeekAheadMessage(190, 0, start_time + 10.1 + 0.2 * (i - 100)))
|
|
PutToQueueWithTimeout(
|
|
status_queue, MetadataMessage(start_time + 16 + 0.5 * (i - 130)))
|
|
elif i < num_objects:
|
|
if i == 150:
|
|
# Sends a final ProducerThreadMessage
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
ProducerThreadMessage(200,
|
|
0,
|
|
start_time + 16.1 + 0.5 * (i - 130),
|
|
finished=True))
|
|
PutToQueueWithTimeout(status_queue,
|
|
MetadataMessage(start_time + 26 + (i - 150)))
|
|
PutToQueueWithTimeout(status_queue, FinalMessage(start_time + 100))
|
|
PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT)
|
|
JoinThreadAndRaiseOnTimeout(ui_thread)
|
|
content = stream.getvalue()
|
|
# We should not have estimated the number of objects as 130 in the UI.
|
|
self.assertNotIn('/130 objects', content)
|
|
# We should have estimated the number of objects as 190 in the UI.
|
|
self.assertIn('/190 objects', content)
|
|
# We should have estimated the number of objects as 200 in the UI.
|
|
self.assertIn('/200 objects', content)
|
|
# We should have calculated the throughput at all moments.
|
|
# First 100 elements.
|
|
self.assertIn('10.00 objects/s', content)
|
|
# At one exact point between first and second round of elements.
|
|
self.assertEqual(content.count('7.50 objects/s'), 1)
|
|
# Next 30 elements.
|
|
self.assertIn('5.00 objects/s', content)
|
|
# At one exact point between second and third round of elements.
|
|
self.assertEqual(content.count('3.50 objects/s'), 1)
|
|
# Next 20 elements.
|
|
self.assertIn('2.00 objects/s', content)
|
|
# At one exact point between third and fourth round of elements.
|
|
self.assertEqual(content.count('1.50 objects/s'), 1)
|
|
# Final 50 elements.
|
|
self.assertIn('1.00 objects/s', content)
|
|
CheckUiOutputWithMFlag(self, content, 200, metadata=True)
|
|
|
|
def test_ui_manager(self):
|
|
"""Tests the correctness of the UI manager.
|
|
|
|
This test ensures a DataManager is created whenever a data message appears,
|
|
regardless of previous MetadataMessages.
|
|
"""
|
|
stream = six.StringIO()
|
|
start_time = self.start_time
|
|
ui_controller = UIController(custom_time=start_time)
|
|
status_queue = MainThreadUIQueue(stream, ui_controller)
|
|
# No manager has been created.
|
|
self.assertEqual(ui_controller.manager, None)
|
|
PutToQueueWithTimeout(status_queue, ProducerThreadMessage(2, 0, start_time))
|
|
# Still no manager has been created.
|
|
self.assertEqual(ui_controller.manager, None)
|
|
PutToQueueWithTimeout(status_queue, MetadataMessage(start_time + 1))
|
|
# Now we have a MetadataManager.
|
|
self.assertIsInstance(ui_controller.manager, MetadataManager)
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(StorageUrlFromString('foo'), None, start_time + 2))
|
|
# Now we have a DataManager since the DataManager overwrites the
|
|
# MetadataManager.
|
|
self.assertIsInstance(ui_controller.manager, DataManager)
|
|
|
|
def test_ui_BytesToFixedWidthString(self):
|
|
"""Tests the correctness of BytesToFixedWidthString."""
|
|
self.assertEqual(' 0.0 B', BytesToFixedWidthString(0, decimal_places=1))
|
|
self.assertEqual(' 0.00 B', BytesToFixedWidthString(0, decimal_places=2))
|
|
self.assertEqual(' 2.3 KiB',
|
|
BytesToFixedWidthString(2.27 * 1024, decimal_places=1))
|
|
self.assertEqual(' 1023 KiB',
|
|
BytesToFixedWidthString(1023.2 * 1024, decimal_places=1))
|
|
self.assertEqual(' 1.0 MiB',
|
|
BytesToFixedWidthString(1024**2, decimal_places=1))
|
|
self.assertEqual('999.1 MiB',
|
|
BytesToFixedWidthString(999.1 * 1024**2, decimal_places=1))
|
|
|
|
def test_ui_spinner(self):
|
|
stream = six.StringIO()
|
|
start_time = self.start_time
|
|
ui_controller = UIController(update_spinner_period=1,
|
|
custom_time=start_time)
|
|
status_queue = MainThreadUIQueue(stream, ui_controller)
|
|
PutToQueueWithTimeout(status_queue,
|
|
ProducerThreadMessage(1, len('foo'), start_time))
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(StorageUrlFromString('foo'),
|
|
None,
|
|
start_time,
|
|
message_type=FileMessage.FILE_UPLOAD))
|
|
current_spinner = ui_controller.manager.GetSpinner()
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
ProgressMessage(1, len('foo'), StorageUrlFromString('foo'),
|
|
start_time + 1.2))
|
|
old_spinner1 = current_spinner
|
|
current_spinner = ui_controller.manager.GetSpinner()
|
|
# Spinner must have changed since more than 1 second has passed.
|
|
self.assertNotEqual(old_spinner1, current_spinner)
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
ProgressMessage(2, len('foo'), StorageUrlFromString('foo'),
|
|
start_time + 2))
|
|
old_spinner2 = current_spinner
|
|
current_spinner = ui_controller.manager.GetSpinner()
|
|
# Spinner must not have changed since less than 1 second has passed.
|
|
self.assertEqual(old_spinner2, current_spinner)
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
ProgressMessage(3, len('foo'), StorageUrlFromString('foo'),
|
|
start_time + 2.5))
|
|
old_spinner3 = current_spinner
|
|
current_spinner = ui_controller.manager.GetSpinner()
|
|
# Spinner must have changed since more than 1 second has passed.
|
|
self.assertNotEqual(old_spinner3, current_spinner)
|
|
PutToQueueWithTimeout(
|
|
status_queue,
|
|
FileMessage(StorageUrlFromString('foo'),
|
|
None,
|
|
start_time + 5,
|
|
finished=True,
|
|
message_type=FileMessage.FILE_UPLOAD))
|
|
old_spinner4 = current_spinner
|
|
current_spinner = ui_controller.manager.GetSpinner()
|
|
# Spinner must have changed since more than 1 second has passed.
|
|
self.assertNotEqual(old_spinner4, current_spinner)
|
|
# Moreover, since we have 4 spinner characters and were only supposed to
|
|
# change it twice, current_spinner must be different from old_spinner1 and
|
|
# old_spinner3.
|
|
self.assertNotEqual(old_spinner3, current_spinner)
|
|
self.assertNotEqual(old_spinner1, current_spinner)
|