From a5f4fd9be7b1980ae0f173f282cb3ad22afba89f Mon Sep 17 00:00:00 2001 From: larkee Date: Mon, 16 Mar 2020 12:32:47 +1100 Subject: [PATCH 01/14] add backup samples --- spanner/cloud-client/backup_sample.py | 265 +++++++++++++++++++++ spanner/cloud-client/backup_sample_test.py | 93 ++++++++ 2 files changed, 358 insertions(+) create mode 100644 spanner/cloud-client/backup_sample.py create mode 100644 spanner/cloud-client/backup_sample_test.py diff --git a/spanner/cloud-client/backup_sample.py b/spanner/cloud-client/backup_sample.py new file mode 100644 index 00000000000..6a492f35b1d --- /dev/null +++ b/spanner/cloud-client/backup_sample.py @@ -0,0 +1,265 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This application demonstrates how to create and restore from backups +using Cloud Spanner. + +For more information, see the README.rst under /spanner. +""" + +import argparse +import concurrent.futures +from datetime import ( + datetime, + timedelta +) +import time + +from google.cloud import spanner + + +# [START spanner_create_backup] +def create_backup(instance_id, database_id, backup_id): + """Creates a backup for a database.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + # Create a backup + expire_time = datetime.utcnow() + timedelta(days=14) + backup = instance.backup( + backup_id, database=database, expire_time=expire_time) + operation = backup.create() + + # Wait for backup operation to complete. + operation.result() + + # Verify that the backup is ready. + backup.reload() + assert backup.is_ready() == True + + # Get the name, create time and backup size. + backup.reload() + print("Backup {} of size {} bytes was created at {}".format( + backup.name, backup.size_bytes, backup.create_time)) +# [END spanner_create_backup] + +# [START spanner_restore_database] +def restore_database(instance_id, new_database_id, backup_id): + """Restores a database from a backup.""" + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + # Create a backup on database_id. + + # Start restoring backup to a new database. + backup = instance.backup(backup_id) + new_database = instance.database(new_database_id) + operation = new_database.restore(backup) + + # Wait for restore operation to complete. + operation.result() + + # Newly created database has restore information. + new_database.reload() + restore_info = new_database.restore_info + print("Database {} restored to {} from backup {}.".format( + restore_info.backup_info.source_database, + new_database_id, + restore_info.backup_info.backup)) +# [END spanner_restore_database] + +# [START spanner_cancel_backup] +def cancel_backup(instance_id, database_id, backup_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + database = instance.database(database_id) + + expire_time = datetime.utcnow() + timedelta(days=30) + + # Create a backup. + backup = instance.backup( + backup_id, database=database, expire_time=expire_time) + operation = backup.create() + + # Cancel backup creation. + operation.cancel() + + # Cancel operations are best effort so either it will complete or + # be cancelled. + while not operation.done(): + time.sleep(300) # 5 mins + + # Deal with resource if the operation succeeded. + if backup.exists(): + print("Backup was created before the cancel completed.") + backup.delete() + print("Backup deleted.") + else: + print("Backup creation was successfully cancelled.") +# [END spanner_cancel_backup] + +# [START spanner_list_backup_operations] +def list_backup_operations(instance_id, database_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + # List the CreateBackup operations. + filter_ = ( + "(metadata.database:{}) AND " + "(metadata.@type:type.googleapis.com/" + "google.spanner.admin.database.v1.CreateBackupMetadata)" + ).format(database_id) + operations = instance.list_backup_operations(filter_=filter_) + for op in operations: + metadata = op.metadata + # List the pending backups on the instance. + if metadata.progress.progress_percent < 100: + print("Backup {} on database {} pending: {}% complete.".format( + metadata.name, metadata.database, + metadata.progress.progress_percent)) +# [END spanner_list_backup_operations] + +# [START spanner_list_database_operations] +def list_database_operations(instance_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + # List the progress of restore + filter_ = ( + "(metadata.@type:type.googleapis.com/" + "google.spanner.admin.database.v1.OptimizeRestoredDatabaseMetadata)" + ) + operations = instance.list_database_operations(filter_=filter_) + for op in operations: + print("Database {} restored from backup is {}% optimized.".format( + op.metadata.name, op.metadata.progress.progress_percent)) +# [END spanner_list_database_operations] + +# [START spanner_list_backups] +def list_backups(instance_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + + # List all backups. + print("All backups:") + for backup in instance.list_backups(): + print(backup.name) + + # List all backups that contain a name. + print("All backups with backup name containing \"users\":") + for backup in instance.list_backups(filter_="name:users"): + print(backup.name) + + # List all backups for a database that contains a name. + print("All backups with database name containing \"bank\":") + for backup in instance.list_backups(filter_="database:bank"): + print(backup.name) + + # List all backups that expire before a timestamp. + print("All backups with expire_time before \"2019-10-18T02:56:53Z\":") + for backup in instance.list_backups( + filter_="expire_time < \"2019-10-18T02:56:53Z\""): + print(backup.name) + + # List all backups with a size greater than some bytes. + print("All backups with backup size more than 1000 bytes:") + for backup in instance.list_backups(filter_="size_bytes > 1000"): + print(backup.name) + + # List backups that were created after a timestamp that are also ready. + print("All backups created after \"2019-10-18T02:56:53Z\" and are READY:") + for backup in instance.list_backups(filter_=( + "create_time >= \"2019-10-18T02:56:53Z\" AND " + "state:READY")): + print(backup.name) +# [END spanner_list_backups] + +# [START spanner_delete_backup] +def delete_backup(instance_id, backup_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + backup = instance.backup(backup_id) + backup.reload() + + # Wait for databases that reference this backup to finish optimizing + while backup.referencing_databases: + time.sleep(30) + backup.reload() + + # Delete the backup. + backup.delete() + + # Verify that the backup is deleted. + assert backup.exists() == False + print("Backup {} has been deleted.".format(backup.name)) +# [END spanner_delete_backup] + +# [START spanner_update_backup] +def update_backup(instance_id, backup_id): + spanner_client = spanner.Client() + instance = spanner_client.instance(instance_id) + backup = instance.backup(backup_id) + backup.reload() + + # Expire time must be within 366 days of the create time of the backup. + old_expire_time = backup.expire_time + new_expire_time = old_expire_time + timedelta(days=30) + backup.update_expire_time(new_expire_time) + print("Backup {} expire time was updated from {} to {}.".format( + backup.name, old_expire_time, new_expire_time)) +# [END spanner_update_backup] + +if __name__ == '__main__': # noqa: C901 + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'instance_id', help='Your Cloud Spanner instance ID.') + parser.add_argument( + '--database-id', help='Your Cloud Spanner database ID.', + default='example_db') + parser.add_argument( + '--backup-id', help='Your Cloud Spanner backup ID.', + default='example_backup') + + subparsers = parser.add_subparsers(dest='command') + subparsers.add_parser('create_backup', help=create_backup.__doc__) + subparsers.add_parser('cancel_backup', help=cancel_backup.__doc__) + subparsers.add_parser('update_backup', help=update_backup.__doc__) + subparsers.add_parser('restore_database', help=restore_database.__doc__) + subparsers.add_parser('list_backups', help=list_backups.__doc__) + subparsers.add_parser('list_backup_operations', help=list_backup_operations.__doc__) + subparsers.add_parser('list_database_operations', help=list_database_operations.__doc__) + subparsers.add_parser('delete_backup', help=delete_backup.__doc__) + + args = parser.parse_args() + + if args.command == 'create_backup': + create_backup(args.instance_id, args.database_id, args.backup_id) + elif args.command == 'cancel_backup': + cancel_backup(args.instance_id, args.database_id, args.backup_id) + elif args.command == 'update_backup': + update_backup(args.instance_id, args.backup_id) + elif args.command == 'restore_database': + restore_database(args.instance_id, args.database_id, args.backup_id) + elif args.command == 'list_backups': + list_backups(args.instance_id) + elif args.command == 'list_backup_operations': + list_backup_operations(args.instance_id, args.database_id) + elif args.command == 'list_database_operations': + list_database_operations(args.instance_id) + elif args.command == 'delete_backup': + delete_backup(args.instance_id, args.backup_id) + else: + print("Command {} did not match expected commands.".format(args.command)) diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py new file mode 100644 index 00000000000..37487c4223e --- /dev/null +++ b/spanner/cloud-client/backup_sample_test.py @@ -0,0 +1,93 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import spanner +import mock +import pytest +import random +import string +import time + +import backup_sample +import snippets + +def unique_database_id(): + """ Creates a unique id for the database. """ + return 'test-db-{}'.format(''.join(random.choice( + string.ascii_lowercase + string.digits) for _ in range(5))) + +def unique_backup_id(): + """ Creates a unique id for the backup. """ + return 'test-backup-{}'.format(''.join(random.choice( + string.ascii_lowercase + string.digits) for _ in range(5))) + +INSTANCE_ID = os.environ['SPANNER_INSTANCE'] +DATABASE_ID = unique_database_id() +RESTORE_DB_ID = unique_database_id() +BACKUP_ID = unique_backup_id() + +@pytest.fixture(scope='module') +def spanner_instance(): + spanner_client = spanner.Client() + return spanner_client.instance(INSTANCE_ID) + + +@pytest.fixture(scope='module') +def database(spanner_instance): + """ Creates a temporary database that is removed after testing. """ + db = spanner_instance.database(DATABASE_ID) + db.create() + yield db + db.drop() + + +def test_create_backup(capsys, database): + backup_sample.create_backup(INSTANCE_ID, DATABASE_ID, BACKUP_ID) + out, _ = capsys.readouterr() + expected_output = "Backup {} of size {} bytes was created at {}" + assert "Backup " in out + assert (BACKUP_ID + " of size 0 bytes was created at ") in out + +def test_restore_database(capsys): + backup_sample.restore_database(INSTANCE_ID, RESTORE_DB_ID, BACKUP_ID) + out, _ = capsys.readouterr() + assert "Database " in out + assert (DATABASE_ID + " restored to ") in out + assert (RESTORE_DB_ID + " from backup ") in out + assert (BACKUP_ID + ".") in out + +def test_update_backup(capsys): + backup_sample.update_backup(INSTANCE_ID, BACKUP_ID) + out, _ = capsys.readouterr() + assert "Backup " in out + assert (BACKUP_ID + " expire time was updated from ") in out + assert " to " in out + +def test_delete_backup(capsys, spanner_instance): + backup_sample.delete_backup(INSTANCE_ID, BACKUP_ID) + out, _ = capsys.readouterr() + assert "Backup " in out + assert (BACKUP_ID + " has been deleted.") in out + +def test_cancel_backup(capsys): + backup_sample.cancel_backup(INSTANCE_ID, DATABASE_ID, BACKUP_ID) + out, _ = capsys.readouterr() + cancel_success = "Backup creation was successfully cancelled." in out + cancel_failure = ( + ("Backup was created before the cancel completed." in out) and + ("Backup deleted." in out) + ) + assert cancel_success or cancel_failure From e32a5f925ec77fb6fecd6b3d0963b414465cd144 Mon Sep 17 00:00:00 2001 From: larkee Date: Wed, 18 Mar 2020 08:27:03 +1100 Subject: [PATCH 02/14] update required spanner version --- spanner/cloud-client/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spanner/cloud-client/requirements.txt b/spanner/cloud-client/requirements.txt index c423bfa654b..b122372d96e 100644 --- a/spanner/cloud-client/requirements.txt +++ b/spanner/cloud-client/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-spanner==1.13.0 +google-cloud-spanner==1.15.0 futures==3.3.0; python_version < "3" From 762b8c6f93f83149665b1eccf042ea5f26a42928 Mon Sep 17 00:00:00 2001 From: larkee Date: Wed, 18 Mar 2020 09:16:46 +1100 Subject: [PATCH 03/14] fix lint errors --- spanner/cloud-client/backup_sample.py | 26 ++++++++++++++-------- spanner/cloud-client/backup_sample_test.py | 12 ++++++---- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/spanner/cloud-client/backup_sample.py b/spanner/cloud-client/backup_sample.py index 6a492f35b1d..c3d3f4dd0d1 100644 --- a/spanner/cloud-client/backup_sample.py +++ b/spanner/cloud-client/backup_sample.py @@ -19,7 +19,6 @@ """ import argparse -import concurrent.futures from datetime import ( datetime, timedelta @@ -47,7 +46,7 @@ def create_backup(instance_id, database_id, backup_id): # Verify that the backup is ready. backup.reload() - assert backup.is_ready() == True + assert backup.is_ready() is True # Get the name, create time and backup size. backup.reload() @@ -55,6 +54,7 @@ def create_backup(instance_id, database_id, backup_id): backup.name, backup.size_bytes, backup.create_time)) # [END spanner_create_backup] + # [START spanner_restore_database] def restore_database(instance_id, new_database_id, backup_id): """Restores a database from a backup.""" @@ -79,6 +79,7 @@ def restore_database(instance_id, new_database_id, backup_id): restore_info.backup_info.backup)) # [END spanner_restore_database] + # [START spanner_cancel_backup] def cancel_backup(instance_id, database_id, backup_id): spanner_client = spanner.Client() @@ -109,6 +110,7 @@ def cancel_backup(instance_id, database_id, backup_id): print("Backup creation was successfully cancelled.") # [END spanner_cancel_backup] + # [START spanner_list_backup_operations] def list_backup_operations(instance_id, database_id): spanner_client = spanner.Client() @@ -130,6 +132,7 @@ def list_backup_operations(instance_id, database_id): metadata.progress.progress_percent)) # [END spanner_list_backup_operations] + # [START spanner_list_database_operations] def list_database_operations(instance_id): spanner_client = spanner.Client() @@ -146,6 +149,7 @@ def list_database_operations(instance_id): op.metadata.name, op.metadata.progress.progress_percent)) # [END spanner_list_database_operations] + # [START spanner_list_backups] def list_backups(instance_id): spanner_client = spanner.Client() @@ -169,7 +173,7 @@ def list_backups(instance_id): # List all backups that expire before a timestamp. print("All backups with expire_time before \"2019-10-18T02:56:53Z\":") for backup in instance.list_backups( - filter_="expire_time < \"2019-10-18T02:56:53Z\""): + filter_="expire_time < \"2019-10-18T02:56:53Z\""): print(backup.name) # List all backups with a size greater than some bytes. @@ -180,11 +184,12 @@ def list_backups(instance_id): # List backups that were created after a timestamp that are also ready. print("All backups created after \"2019-10-18T02:56:53Z\" and are READY:") for backup in instance.list_backups(filter_=( - "create_time >= \"2019-10-18T02:56:53Z\" AND " - "state:READY")): + "create_time >= \"2019-10-18T02:56:53Z\" AND " + "state:READY")): print(backup.name) # [END spanner_list_backups] + # [START spanner_delete_backup] def delete_backup(instance_id, backup_id): spanner_client = spanner.Client() @@ -194,17 +199,18 @@ def delete_backup(instance_id, backup_id): # Wait for databases that reference this backup to finish optimizing while backup.referencing_databases: - time.sleep(30) - backup.reload() + time.sleep(30) + backup.reload() # Delete the backup. backup.delete() # Verify that the backup is deleted. - assert backup.exists() == False + assert backup.exists() is False print("Backup {} has been deleted.".format(backup.name)) # [END spanner_delete_backup] + # [START spanner_update_backup] def update_backup(instance_id, backup_id): spanner_client = spanner.Client() @@ -220,6 +226,7 @@ def update_backup(instance_id, backup_id): backup.name, old_expire_time, new_expire_time)) # [END spanner_update_backup] + if __name__ == '__main__': # noqa: C901 parser = argparse.ArgumentParser( description=__doc__, @@ -240,7 +247,8 @@ def update_backup(instance_id, backup_id): subparsers.add_parser('restore_database', help=restore_database.__doc__) subparsers.add_parser('list_backups', help=list_backups.__doc__) subparsers.add_parser('list_backup_operations', help=list_backup_operations.__doc__) - subparsers.add_parser('list_database_operations', help=list_database_operations.__doc__) + subparsers.add_parser('list_database_operations', + help=list_database_operations.__doc__) subparsers.add_parser('delete_backup', help=delete_backup.__doc__) args = parser.parse_args() diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 37487c4223e..0c5d9ff6a0f 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -15,30 +15,31 @@ import os from google.cloud import spanner -import mock import pytest import random import string -import time import backup_sample -import snippets + def unique_database_id(): """ Creates a unique id for the database. """ return 'test-db-{}'.format(''.join(random.choice( string.ascii_lowercase + string.digits) for _ in range(5))) + def unique_backup_id(): """ Creates a unique id for the backup. """ return 'test-backup-{}'.format(''.join(random.choice( string.ascii_lowercase + string.digits) for _ in range(5))) + INSTANCE_ID = os.environ['SPANNER_INSTANCE'] DATABASE_ID = unique_database_id() RESTORE_DB_ID = unique_database_id() BACKUP_ID = unique_backup_id() + @pytest.fixture(scope='module') def spanner_instance(): spanner_client = spanner.Client() @@ -57,10 +58,10 @@ def database(spanner_instance): def test_create_backup(capsys, database): backup_sample.create_backup(INSTANCE_ID, DATABASE_ID, BACKUP_ID) out, _ = capsys.readouterr() - expected_output = "Backup {} of size {} bytes was created at {}" assert "Backup " in out assert (BACKUP_ID + " of size 0 bytes was created at ") in out + def test_restore_database(capsys): backup_sample.restore_database(INSTANCE_ID, RESTORE_DB_ID, BACKUP_ID) out, _ = capsys.readouterr() @@ -69,6 +70,7 @@ def test_restore_database(capsys): assert (RESTORE_DB_ID + " from backup ") in out assert (BACKUP_ID + ".") in out + def test_update_backup(capsys): backup_sample.update_backup(INSTANCE_ID, BACKUP_ID) out, _ = capsys.readouterr() @@ -76,12 +78,14 @@ def test_update_backup(capsys): assert (BACKUP_ID + " expire time was updated from ") in out assert " to " in out + def test_delete_backup(capsys, spanner_instance): backup_sample.delete_backup(INSTANCE_ID, BACKUP_ID) out, _ = capsys.readouterr() assert "Backup " in out assert (BACKUP_ID + " has been deleted.") in out + def test_cancel_backup(capsys): backup_sample.cancel_backup(INSTANCE_ID, DATABASE_ID, BACKUP_ID) out, _ = capsys.readouterr() From c0f5ac45694c5ac018ced020d58cec839bed4179 Mon Sep 17 00:00:00 2001 From: larkee Date: Wed, 18 Mar 2020 12:02:03 +1100 Subject: [PATCH 04/14] run backup samples tests against a new instance --- spanner/cloud-client/backup_sample_test.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 0c5d9ff6a0f..055d7430fad 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -22,6 +22,12 @@ import backup_sample +def unique_instance_id(): + """ Creates a unique id for the database. """ + return 'test-instance-{}'.format(''.join(random.choice( + string.ascii_lowercase + string.digits) for _ in range(5))) + + def unique_database_id(): """ Creates a unique id for the database. """ return 'test-db-{}'.format(''.join(random.choice( @@ -34,7 +40,7 @@ def unique_backup_id(): string.ascii_lowercase + string.digits) for _ in range(5))) -INSTANCE_ID = os.environ['SPANNER_INSTANCE'] +INSTANCE_ID = unique_instance_id() DATABASE_ID = unique_database_id() RESTORE_DB_ID = unique_database_id() BACKUP_ID = unique_backup_id() @@ -43,7 +49,12 @@ def unique_backup_id(): @pytest.fixture(scope='module') def spanner_instance(): spanner_client = spanner.Client() - return spanner_client.instance(INSTANCE_ID) + instance_config = '{}/instanceConfigs/{}'.format( + spanner_client.project_name, 'regional-us-central1') + instance = spanner_client.instance(INSTANCE_ID, instance_config) + instance.create() + yield instance + instance.delete() @pytest.fixture(scope='module') From 71c3af08ce78a966386cb2410ecd81388977338c Mon Sep 17 00:00:00 2001 From: larkee Date: Wed, 18 Mar 2020 12:10:03 +1100 Subject: [PATCH 05/14] fix lint --- spanner/cloud-client/backup_sample_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 055d7430fad..7d7e449d3b2 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - from google.cloud import spanner import pytest import random From f3a3d747cac37d0164e6d186f7fc9866829c06b0 Mon Sep 17 00:00:00 2001 From: larkee Date: Wed, 18 Mar 2020 12:47:12 +1100 Subject: [PATCH 06/14] wait for instance creation to complete --- spanner/cloud-client/backup_sample_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 7d7e449d3b2..2b5271e75ed 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -50,7 +50,8 @@ def spanner_instance(): instance_config = '{}/instanceConfigs/{}'.format( spanner_client.project_name, 'regional-us-central1') instance = spanner_client.instance(INSTANCE_ID, instance_config) - instance.create() + op = instance.create() + op.result(30) # block until completion yield instance instance.delete() From ee448ab2a85bfe437a659126bbb0a0c80b8bf929 Mon Sep 17 00:00:00 2001 From: larkee <31196561+larkee@users.noreply.github.com> Date: Wed, 18 Mar 2020 16:06:45 +1100 Subject: [PATCH 07/14] Apply suggestions from code review Co-Authored-By: skuruppu --- spanner/cloud-client/backup_sample.py | 6 +++--- spanner/cloud-client/backup_sample_test.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spanner/cloud-client/backup_sample.py b/spanner/cloud-client/backup_sample.py index c3d3f4dd0d1..a8af4bdc1aa 100644 --- a/spanner/cloud-client/backup_sample.py +++ b/spanner/cloud-client/backup_sample.py @@ -1,4 +1,4 @@ -# Copyright 2019 Google Inc. All Rights Reserved. +# Copyright 2020 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -138,7 +138,7 @@ def list_database_operations(instance_id): spanner_client = spanner.Client() instance = spanner_client.instance(instance_id) - # List the progress of restore + # List the progress of restore. filter_ = ( "(metadata.@type:type.googleapis.com/" "google.spanner.admin.database.v1.OptimizeRestoredDatabaseMetadata)" @@ -197,7 +197,7 @@ def delete_backup(instance_id, backup_id): backup = instance.backup(backup_id) backup.reload() - # Wait for databases that reference this backup to finish optimizing + # Wait for databases that reference this backup to finish optimizing. while backup.referencing_databases: time.sleep(30) backup.reload() diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 2b5271e75ed..7dc37a22c5d 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -1,4 +1,4 @@ -# Copyright 2019 Google Inc. All Rights Reserved. +# Copyright 2020 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From a4a6ba98687f6c8cc74d5419e2c7f55e4ee4ab6a Mon Sep 17 00:00:00 2001 From: larkee Date: Thu, 19 Mar 2020 12:41:02 +1100 Subject: [PATCH 08/14] add list_backups test --- spanner/cloud-client/backup_sample_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 7dc37a22c5d..0f3b2f0ff19 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -89,6 +89,21 @@ def test_update_backup(capsys): assert " to " in out +def test_list_backups(capsys, spanner_instance): + backup_sample.list_backups(INSTANCE_ID) + out, _ = capsys.readouterr() + backup_name = spanner_instance.name + "/backups/" + BACKUP_ID + assert "All backups:\n" + backup_name in out + assert ( + "All backups with backup name containing \"users\":\n" + "All backups with database name containing \"bank\":\n" + "All backups with expire_time before \"2019-10-18T02:56:53Z\":\n" + "All backups with backup size more than 1000 bytes:\n" + "All backups created after \"2019-10-18T02:56:53Z\" and are READY:\n" + ) in out + assert "All backups created after \"2019-10-18T02:56:53Z\" and are READY:" + backup_name in out + + def test_delete_backup(capsys, spanner_instance): backup_sample.delete_backup(INSTANCE_ID, BACKUP_ID) out, _ = capsys.readouterr() From 01bcbdba1c6158f0bc3b94257eede0832cbcd7b9 Mon Sep 17 00:00:00 2001 From: larkee Date: Thu, 19 Mar 2020 12:46:37 +1100 Subject: [PATCH 09/14] fix lint --- spanner/cloud-client/backup_sample_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 0f3b2f0ff19..82e4cd4e355 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -101,7 +101,10 @@ def test_list_backups(capsys, spanner_instance): "All backups with backup size more than 1000 bytes:\n" "All backups created after \"2019-10-18T02:56:53Z\" and are READY:\n" ) in out - assert "All backups created after \"2019-10-18T02:56:53Z\" and are READY:" + backup_name in out + assert ( + "All backups created after \"2019-10-18T02:56:53Z\" and are READY:" + + backup_name + ) in out def test_delete_backup(capsys, spanner_instance): From 9eab00fa87ab638a8b61fd8412afe6c258dbd89d Mon Sep 17 00:00:00 2001 From: larkee Date: Fri, 20 Mar 2020 08:02:59 +1100 Subject: [PATCH 10/14] add missing newline character in assert --- spanner/cloud-client/backup_sample_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 82e4cd4e355..89f1603c1a1 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -102,7 +102,7 @@ def test_list_backups(capsys, spanner_instance): "All backups created after \"2019-10-18T02:56:53Z\" and are READY:\n" ) in out assert ( - "All backups created after \"2019-10-18T02:56:53Z\" and are READY:" + + "All backups created after \"2019-10-18T02:56:53Z\" and are READY:\n" + backup_name ) in out From 9d429b96899d2590d43b1ae8ab9858a54ba54e0b Mon Sep 17 00:00:00 2001 From: larkee Date: Fri, 27 Mar 2020 13:04:49 +1100 Subject: [PATCH 11/14] update samples to be consistent with other languages --- spanner/cloud-client/backup_sample.py | 37 +++++++++++----------- spanner/cloud-client/backup_sample_test.py | 37 ++++++++-------------- 2 files changed, 32 insertions(+), 42 deletions(-) diff --git a/spanner/cloud-client/backup_sample.py b/spanner/cloud-client/backup_sample.py index a8af4bdc1aa..8ae8d1b3f0a 100644 --- a/spanner/cloud-client/backup_sample.py +++ b/spanner/cloud-client/backup_sample.py @@ -125,11 +125,9 @@ def list_backup_operations(instance_id, database_id): operations = instance.list_backup_operations(filter_=filter_) for op in operations: metadata = op.metadata - # List the pending backups on the instance. - if metadata.progress.progress_percent < 100: - print("Backup {} on database {} pending: {}% complete.".format( - metadata.name, metadata.database, - metadata.progress.progress_percent)) + print("Backup {} on database {}: {}% complete.".format( + metadata.name, metadata.database, + metadata.progress.progress_percent)) # [END spanner_list_backup_operations] @@ -151,7 +149,7 @@ def list_database_operations(instance_id): # [START spanner_list_backups] -def list_backups(instance_id): +def list_backups(instance_id, database_id, backup_id): spanner_client = spanner.Client() instance = spanner_client.instance(instance_id) @@ -161,31 +159,32 @@ def list_backups(instance_id): print(backup.name) # List all backups that contain a name. - print("All backups with backup name containing \"users\":") - for backup in instance.list_backups(filter_="name:users"): + print("All backups with backup name containing \"{}\":".format(backup_id)) + for backup in instance.list_backups(filter_="name:{}".format(backup_id)): print(backup.name) # List all backups for a database that contains a name. - print("All backups with database name containing \"bank\":") - for backup in instance.list_backups(filter_="database:bank"): + print("All backups with database name containing \"{}\":".format(database_id)) + for backup in instance.list_backups(filter_="database:{}".format(database_id)): print(backup.name) # List all backups that expire before a timestamp. - print("All backups with expire_time before \"2019-10-18T02:56:53Z\":") + expire_time = datetime.utcnow().replace(microsecond=0) + timedelta(days=30) + print("All backups with expire_time before \"{}-{}-{}T{}:{}:{}Z\":".format(*expire_time.timetuple())) for backup in instance.list_backups( - filter_="expire_time < \"2019-10-18T02:56:53Z\""): + filter_="expire_time < \"{}-{}-{}T{}:{}:{}Z\"".format(*expire_time.timetuple())): print(backup.name) # List all backups with a size greater than some bytes. - print("All backups with backup size more than 1000 bytes:") - for backup in instance.list_backups(filter_="size_bytes > 1000"): + print("All backups with backup size more than 100 bytes:") + for backup in instance.list_backups(filter_="size_bytes > 100"): print(backup.name) # List backups that were created after a timestamp that are also ready. - print("All backups created after \"2019-10-18T02:56:53Z\" and are READY:") - for backup in instance.list_backups(filter_=( - "create_time >= \"2019-10-18T02:56:53Z\" AND " - "state:READY")): + create_time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) + print("All backups created after \"{}-{}-{}T{}:{}:{}Z\" and are READY:".format(*create_time.timetuple())) + for backup in instance.list_backups( + filter_="create_time >= \"{}-{}-{}T{}:{}:{}Z\" AND state:READY".format(*create_time.timetuple())): print(backup.name) # [END spanner_list_backups] @@ -262,7 +261,7 @@ def update_backup(instance_id, backup_id): elif args.command == 'restore_database': restore_database(args.instance_id, args.database_id, args.backup_id) elif args.command == 'list_backups': - list_backups(args.instance_id) + list_backups(args.instance_id, args.database_id, args.backup_id) elif args.command == 'list_backup_operations': list_backup_operations(args.instance_id, args.database_id) elif args.command == 'list_database_operations': diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 89f1603c1a1..3dce74d8c8d 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -68,50 +68,41 @@ def database(spanner_instance): def test_create_backup(capsys, database): backup_sample.create_backup(INSTANCE_ID, DATABASE_ID, BACKUP_ID) out, _ = capsys.readouterr() - assert "Backup " in out - assert (BACKUP_ID + " of size 0 bytes was created at ") in out + assert BACKUP_ID in out def test_restore_database(capsys): backup_sample.restore_database(INSTANCE_ID, RESTORE_DB_ID, BACKUP_ID) out, _ = capsys.readouterr() - assert "Database " in out assert (DATABASE_ID + " restored to ") in out assert (RESTORE_DB_ID + " from backup ") in out - assert (BACKUP_ID + ".") in out + assert BACKUP_ID in out def test_update_backup(capsys): backup_sample.update_backup(INSTANCE_ID, BACKUP_ID) out, _ = capsys.readouterr() - assert "Backup " in out - assert (BACKUP_ID + " expire time was updated from ") in out - assert " to " in out + assert BACKUP_ID in out + + +def test_list_backup_operations(capsys, spanner_instance): + backup_sample.list_backup_operations(INSTANCE_ID, DATABASE_ID) + out, _ = capsys.readouterr() + assert BACKUP_ID in out + assert DATABASE_ID in out def test_list_backups(capsys, spanner_instance): - backup_sample.list_backups(INSTANCE_ID) + backup_sample.list_backups(INSTANCE_ID, DATABASE_ID, BACKUP_ID) out, _ = capsys.readouterr() - backup_name = spanner_instance.name + "/backups/" + BACKUP_ID - assert "All backups:\n" + backup_name in out - assert ( - "All backups with backup name containing \"users\":\n" - "All backups with database name containing \"bank\":\n" - "All backups with expire_time before \"2019-10-18T02:56:53Z\":\n" - "All backups with backup size more than 1000 bytes:\n" - "All backups created after \"2019-10-18T02:56:53Z\" and are READY:\n" - ) in out - assert ( - "All backups created after \"2019-10-18T02:56:53Z\" and are READY:\n" + - backup_name - ) in out + id_count = out.count(BACKUP_ID) + assert id_count is 6 def test_delete_backup(capsys, spanner_instance): backup_sample.delete_backup(INSTANCE_ID, BACKUP_ID) out, _ = capsys.readouterr() - assert "Backup " in out - assert (BACKUP_ID + " has been deleted.") in out + assert BACKUP_ID in out def test_cancel_backup(capsys): From 447ad6b1caedcdacbfccd6d10b841a419d0283a0 Mon Sep 17 00:00:00 2001 From: larkee Date: Fri, 27 Mar 2020 13:09:57 +1100 Subject: [PATCH 12/14] lint fix --- spanner/cloud-client/backup_sample.py | 12 ++++++++---- spanner/cloud-client/backup_sample_test.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/spanner/cloud-client/backup_sample.py b/spanner/cloud-client/backup_sample.py index 8ae8d1b3f0a..d9ad3f71492 100644 --- a/spanner/cloud-client/backup_sample.py +++ b/spanner/cloud-client/backup_sample.py @@ -170,9 +170,11 @@ def list_backups(instance_id, database_id, backup_id): # List all backups that expire before a timestamp. expire_time = datetime.utcnow().replace(microsecond=0) + timedelta(days=30) - print("All backups with expire_time before \"{}-{}-{}T{}:{}:{}Z\":".format(*expire_time.timetuple())) + print("All backups with expire_time before \"{}-{}-{}T{}:{}:{}Z\":".format( + *expire_time.timetuple())) for backup in instance.list_backups( - filter_="expire_time < \"{}-{}-{}T{}:{}:{}Z\"".format(*expire_time.timetuple())): + filter_="expire_time < \"{}-{}-{}T{}:{}:{}Z\"".format( + *expire_time.timetuple())): print(backup.name) # List all backups with a size greater than some bytes. @@ -182,9 +184,11 @@ def list_backups(instance_id, database_id, backup_id): # List backups that were created after a timestamp that are also ready. create_time = datetime.utcnow().replace(microsecond=0) - timedelta(days=1) - print("All backups created after \"{}-{}-{}T{}:{}:{}Z\" and are READY:".format(*create_time.timetuple())) + print("All backups created after \"{}-{}-{}T{}:{}:{}Z\" and are READY:".format( + *create_time.timetuple())) for backup in instance.list_backups( - filter_="create_time >= \"{}-{}-{}T{}:{}:{}Z\" AND state:READY".format(*create_time.timetuple())): + filter_="create_time >= \"{}-{}-{}T{}:{}:{}Z\" AND state:READY".format( + *create_time.timetuple())): print(backup.name) # [END spanner_list_backups] diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 3dce74d8c8d..5af1fb3122a 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -96,7 +96,7 @@ def test_list_backups(capsys, spanner_instance): backup_sample.list_backups(INSTANCE_ID, DATABASE_ID, BACKUP_ID) out, _ = capsys.readouterr() id_count = out.count(BACKUP_ID) - assert id_count is 6 + assert id_count == 6 def test_delete_backup(capsys, spanner_instance): From 39ee4d92cb47066a98a7b97c1a10b9bb2d3004ed Mon Sep 17 00:00:00 2001 From: larkee Date: Fri, 27 Mar 2020 13:49:27 +1100 Subject: [PATCH 13/14] add pagination sample --- spanner/cloud-client/backup_sample.py | 5 +++++ spanner/cloud-client/backup_sample_test.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/spanner/cloud-client/backup_sample.py b/spanner/cloud-client/backup_sample.py index d9ad3f71492..c8bcce82b83 100644 --- a/spanner/cloud-client/backup_sample.py +++ b/spanner/cloud-client/backup_sample.py @@ -190,6 +190,11 @@ def list_backups(instance_id, database_id, backup_id): filter_="create_time >= \"{}-{}-{}T{}:{}:{}Z\" AND state:READY".format( *create_time.timetuple())): print(backup.name) + + print("All backups with pagination") + for page in instance.list_backups(page_size=2).pages: + for backup in page: + print(backup.name) # [END spanner_list_backups] diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index 5af1fb3122a..e2276a79296 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -96,7 +96,7 @@ def test_list_backups(capsys, spanner_instance): backup_sample.list_backups(INSTANCE_ID, DATABASE_ID, BACKUP_ID) out, _ = capsys.readouterr() id_count = out.count(BACKUP_ID) - assert id_count == 6 + assert id_count == 7 def test_delete_backup(capsys, spanner_instance): From a1be3a47dcaaea6b6536f96b14ba0f9f89bb24fc Mon Sep 17 00:00:00 2001 From: larkee Date: Mon, 30 Mar 2020 10:43:17 +1100 Subject: [PATCH 14/14] reorder tests --- spanner/cloud-client/backup_sample_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/spanner/cloud-client/backup_sample_test.py b/spanner/cloud-client/backup_sample_test.py index e2276a79296..8ed6e8f823c 100644 --- a/spanner/cloud-client/backup_sample_test.py +++ b/spanner/cloud-client/backup_sample_test.py @@ -79,12 +79,6 @@ def test_restore_database(capsys): assert BACKUP_ID in out -def test_update_backup(capsys): - backup_sample.update_backup(INSTANCE_ID, BACKUP_ID) - out, _ = capsys.readouterr() - assert BACKUP_ID in out - - def test_list_backup_operations(capsys, spanner_instance): backup_sample.list_backup_operations(INSTANCE_ID, DATABASE_ID) out, _ = capsys.readouterr() @@ -99,6 +93,12 @@ def test_list_backups(capsys, spanner_instance): assert id_count == 7 +def test_update_backup(capsys): + backup_sample.update_backup(INSTANCE_ID, BACKUP_ID) + out, _ = capsys.readouterr() + assert BACKUP_ID in out + + def test_delete_backup(capsys, spanner_instance): backup_sample.delete_backup(INSTANCE_ID, BACKUP_ID) out, _ = capsys.readouterr()