diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 0954585f2..cb06536da 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:df50e8d462f86d6bcb42f27ecad55bb12c404f1c65de9c6fe4c4d25120080bd6 + digest: sha256:5ff7446edeaede81c3ed58b23a4e76a5403fba1350ce28478045657303b6479d diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml new file mode 100644 index 000000000..d4756ac8f --- /dev/null +++ b/.github/sync-repo-settings.yaml @@ -0,0 +1,11 @@ +# https://github.com/googleapis/repo-automation-bots/tree/master/packages/sync-repo-settings +# Rules for master branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `master` +- pattern: master + requiredStatusCheckContexts: + - 'Kokoro' + - 'cla/google' + - 'Kokoro system-2.7' + - 'Kokoro system-3.8' diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile index 412b0b56a..4e1b1fb8b 100644 --- a/.kokoro/docker/docs/Dockerfile +++ b/.kokoro/docker/docs/Dockerfile @@ -40,6 +40,7 @@ RUN apt-get update \ libssl-dev \ libsqlite3-dev \ portaudio19-dev \ + python3-distutils \ redis-server \ software-properties-common \ ssh \ @@ -59,40 +60,8 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -f /var/cache/apt/archives/*.deb - -COPY fetch_gpg_keys.sh /tmp -# Install the desired versions of Python. -RUN set -ex \ - && export GNUPGHOME="$(mktemp -d)" \ - && echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \ - && /tmp/fetch_gpg_keys.sh \ - && for PYTHON_VERSION in 3.7.8 3.8.5; do \ - wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ - && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ - && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \ - && rm -r python-${PYTHON_VERSION}.tar.xz.asc \ - && mkdir -p /usr/src/python-${PYTHON_VERSION} \ - && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \ - && rm python-${PYTHON_VERSION}.tar.xz \ - && cd /usr/src/python-${PYTHON_VERSION} \ - && ./configure \ - --enable-shared \ - # This works only on Python 2.7 and throws a warning on every other - # version, but seems otherwise harmless. - --enable-unicode=ucs4 \ - --with-system-ffi \ - --without-ensurepip \ - && make -j$(nproc) \ - && make install \ - && ldconfig \ - ; done \ - && rm -rf "${GNUPGHOME}" \ - && rm -rf /usr/src/python* \ - && rm -rf ~/.cache/ - RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ - && python3.7 /tmp/get-pip.py \ && python3.8 /tmp/get-pip.py \ && rm /tmp/get-pip.py -CMD ["python3.7"] +CMD ["python3.8"] diff --git a/.kokoro/presubmit/presubmit.cfg b/.kokoro/presubmit/presubmit.cfg index 8f43917d9..b158096f0 100644 --- a/.kokoro/presubmit/presubmit.cfg +++ b/.kokoro/presubmit/presubmit.cfg @@ -1 +1,7 @@ -# Format: //devtools/kokoro/config/proto/build.proto \ No newline at end of file +# Format: //devtools/kokoro/config/proto/build.proto + +# Disable system tests. +env_vars: { + key: "RUN_SYSTEM_TESTS" + value: "false" +} diff --git a/.kokoro/presubmit/system-2.7.cfg b/.kokoro/presubmit/system-2.7.cfg new file mode 100644 index 000000000..3b6523a19 --- /dev/null +++ b/.kokoro/presubmit/system-2.7.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "system-2.7" +} \ No newline at end of file diff --git a/.kokoro/presubmit/system-3.8.cfg b/.kokoro/presubmit/system-3.8.cfg new file mode 100644 index 000000000..f4bcee3db --- /dev/null +++ b/.kokoro/presubmit/system-3.8.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "system-3.8" +} \ No newline at end of file diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index cf5de74c1..311a8d54b 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -20,9 +20,9 @@ set -eo pipefail # Enables `**` to include files nested inside sub-folders shopt -s globstar -# Exit early if samples directory doesn't exist -if [ ! -d "./samples" ]; then - echo "No tests run. `./samples` not found" +# Exit early if samples don't exist +if ! find samples -name 'requirements.txt' | grep -q .; then + echo "No tests run. './samples/**/requirements.txt' not found" exit 0 fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 742c769ef..e39599b72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +### [1.41.1](https://www.github.com/googleapis/python-storage/compare/v1.41.0...v1.41.1) (2021-07-20) + + +### Bug Fixes + +* **deps:** pin `{api,cloud}-core`, `auth` to allow 2.x versions on Python 3 ([#512](https://www.github.com/googleapis/python-storage/issues/512)) ([4d7500e](https://www.github.com/googleapis/python-storage/commit/4d7500e39c51efd817b8363b69c88be040f3edb8)) +* remove trailing commas from error message constants ([#505](https://www.github.com/googleapis/python-storage/issues/505)) ([d4a86ce](https://www.github.com/googleapis/python-storage/commit/d4a86ceb7a7c5e00ba7bae37c7078d52478040ff)), closes [#501](https://www.github.com/googleapis/python-storage/issues/501) + + +### Documentation + +* replace usage of deprecated function `download_as_string` in docs ([#508](https://www.github.com/googleapis/python-storage/issues/508)) ([8dfa4d4](https://www.github.com/googleapis/python-storage/commit/8dfa4d429dce94b671dc3e3755e52ab82733f61a)) + ## [1.41.0](https://www.github.com/googleapis/python-storage/compare/v1.40.0...v1.41.0) (2021-07-13) diff --git a/README.rst b/README.rst index bb15ee569..4b64e1023 100644 --- a/README.rst +++ b/README.rst @@ -98,7 +98,7 @@ how to create a bucket. bucket = client.get_bucket('bucket-id-here') # Then do other things... blob = bucket.get_blob('remote/path/to/file.txt') - print(blob.download_as_string()) + print(blob.download_as_bytes()) blob.upload_from_string('New contents!') blob2 = bucket.blob('remote/path/storage.txt') blob2.upload_from_filename(filename='/local/path.txt') diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index e6b7e835f..81e7e103f 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -119,7 +119,7 @@ _COMPOSE_IF_GENERATION_LIST_DEPRECATED = ( "'if_generation_match: type list' is deprecated and supported for " "backwards-compatability reasons only. Use 'if_source_generation_match' " - "instead' to match source objects' generations.", + "instead' to match source objects' generations." ) _COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR = ( "Use 'if_generation_match' to match the generation of the destination " @@ -130,14 +130,14 @@ "'if_metageneration_match: type list' is deprecated and supported for " "backwards-compatability reasons only. Note that the metageneration to " "be matched is that of the destination blob. Please pass in a single " - "value (type long).", + "value (type long)." ) _COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR = ( "'if_source_generation_match' length must be the same as 'sources' length" ) _DOWNLOAD_AS_STRING_DEPRECATED = ( "Blob.download_as_string() is deprecated and will be removed in future. " - "Use Blob.download_as_bytes() instead.", + "Use Blob.download_as_bytes() instead." ) diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py index 9a46be9ad..8e52416aa 100644 --- a/google/cloud/storage/version.py +++ b/google/cloud/storage/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.41.0" +__version__ = "1.41.1" diff --git a/owlbot.py b/owlbot.py index 0e23239ec..be172d47a 100644 --- a/owlbot.py +++ b/owlbot.py @@ -26,6 +26,7 @@ # ---------------------------------------------------------------------------- templated_files = common.py_library( cov_level=100, + split_system_tests=True, system_test_external_dependencies=[ "google-cloud-iam", "google-cloud-pubsub < 2.0.0", @@ -38,7 +39,11 @@ ) s.move( - templated_files, excludes=["docs/multiprocessing.rst", "noxfile.py", "CONTRIBUTING.rst"], + templated_files, excludes=[ + "docs/multiprocessing.rst", + "noxfile.py", + "CONTRIBUTING.rst", + ], ) s.shell.run(["nox", "-s", "blacken"], hide_output=False) diff --git a/setup.py b/setup.py index 2a54a2951..c00257025 100644 --- a/setup.py +++ b/setup.py @@ -28,9 +28,12 @@ # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 5 - Production/Stable" dependencies = [ - "google-auth >= 1.24.0, < 2.0dev", - "google-cloud-core >= 1.6.0, < 2.0dev", - "google-resumable-media >= 1.3.0, < 2.0dev", + "google-auth >= 1.24.0, < 2.0dev; python_version<'3.0'", + "google-auth >= 1.24.0, < 3.0dev; python_version>='3.6'", + "google-cloud-core >= 1.6.0, < 2.0dev; python_version<'3.0'", + "google-cloud-core >= 1.6.0, < 3.0dev; python_version>='3.6'", + "google-resumable-media >= 1.3.0, < 2.0dev; python_version<'3.0'", + "google-resumable-media >= 1.3.0, < 3.0dev; python_version>='3.6'", "requests >= 2.18.0, < 3.0.0dev", "googleapis-common-protos < 1.53.0; python_version<'3.0'", ] diff --git a/testing/constraints-2.7.txt b/testing/constraints-2.7.txt index e69de29bb..81996b797 100644 --- a/testing/constraints-2.7.txt +++ b/testing/constraints-2.7.txt @@ -0,0 +1,2 @@ +google-cloud-testutils < 1.0dev + diff --git a/tests/system/_helpers.py b/tests/system/_helpers.py index 033450da3..05e4f8ba6 100644 --- a/tests/system/_helpers.py +++ b/tests/system/_helpers.py @@ -54,8 +54,13 @@ def _no_event_based_hold(blob): return not blob.event_based_hold +def _has_kms_key_name(blob): + return blob.kms_key_name is not None + + retry_bad_copy = RetryErrors(exceptions.BadRequest, error_predicate=_bad_copy) retry_no_event_based_hold = RetryInstanceState(_no_event_based_hold) +retry_has_kms_key_name = RetryInstanceState(_has_kms_key_name) def unique_name(prefix): diff --git a/tests/system/test__signing.py b/tests/system/test__signing.py index 72e392bde..04c3687a4 100644 --- a/tests/system/test__signing.py +++ b/tests/system/test__signing.py @@ -48,13 +48,15 @@ def _create_signed_list_blobs_url_helper( assert response.status_code == 200 -def test_create_signed_list_blobs_url_v2(storage_client, signing_bucket): +def test_create_signed_list_blobs_url_v2(storage_client, signing_bucket, no_mtls): _create_signed_list_blobs_url_helper( storage_client, signing_bucket, version="v2", ) -def test_create_signed_list_blobs_url_v2_w_expiration(storage_client, signing_bucket): +def test_create_signed_list_blobs_url_v2_w_expiration( + storage_client, signing_bucket, no_mtls +): now = datetime.datetime.utcnow() delta = datetime.timedelta(seconds=10) @@ -63,13 +65,15 @@ def test_create_signed_list_blobs_url_v2_w_expiration(storage_client, signing_bu ) -def test_create_signed_list_blobs_url_v4(storage_client, signing_bucket): +def test_create_signed_list_blobs_url_v4(storage_client, signing_bucket, no_mtls): _create_signed_list_blobs_url_helper( storage_client, signing_bucket, version="v4", ) -def test_create_signed_list_blobs_url_v4_w_expiration(storage_client, signing_bucket): +def test_create_signed_list_blobs_url_v4_w_expiration( + storage_client, signing_bucket, no_mtls +): now = datetime.datetime.utcnow() delta = datetime.timedelta(seconds=10) _create_signed_list_blobs_url_helper( @@ -125,17 +129,19 @@ def _create_signed_read_url_helper( assert response.content == _helpers.signing_blob_content -def test_create_signed_read_url_v2(storage_client, signing_bucket): +def test_create_signed_read_url_v2(storage_client, signing_bucket, no_mtls): _create_signed_read_url_helper(storage_client, signing_bucket) -def test_create_signed_read_url_v4(storage_client, signing_bucket): +def test_create_signed_read_url_v4(storage_client, signing_bucket, no_mtls): _create_signed_read_url_helper( storage_client, signing_bucket, version="v4", ) -def test_create_signed_read_url_v2_w_expiration(storage_client, signing_bucket): +def test_create_signed_read_url_v2_w_expiration( + storage_client, signing_bucket, no_mtls +): now = datetime.datetime.utcnow() delta = datetime.timedelta(seconds=10) @@ -144,7 +150,9 @@ def test_create_signed_read_url_v2_w_expiration(storage_client, signing_bucket): ) -def test_create_signed_read_url_v4_w_expiration(storage_client, signing_bucket): +def test_create_signed_read_url_v4_w_expiration( + storage_client, signing_bucket, no_mtls +): now = datetime.datetime.utcnow() delta = datetime.timedelta(seconds=10) _create_signed_read_url_helper( @@ -152,17 +160,23 @@ def test_create_signed_read_url_v4_w_expiration(storage_client, signing_bucket): ) -def test_create_signed_read_url_v2_lowercase_method(storage_client, signing_bucket): +def test_create_signed_read_url_v2_lowercase_method( + storage_client, signing_bucket, no_mtls +): _create_signed_read_url_helper(storage_client, signing_bucket, method="get") -def test_create_signed_read_url_v4_lowercase_method(storage_client, signing_bucket): +def test_create_signed_read_url_v4_lowercase_method( + storage_client, signing_bucket, no_mtls +): _create_signed_read_url_helper( storage_client, signing_bucket, method="get", version="v4" ) -def test_create_signed_read_url_v2_w_non_ascii_name(storage_client, signing_bucket): +def test_create_signed_read_url_v2_w_non_ascii_name( + storage_client, signing_bucket, no_mtls +): _create_signed_read_url_helper( storage_client, signing_bucket, @@ -171,7 +185,9 @@ def test_create_signed_read_url_v2_w_non_ascii_name(storage_client, signing_buck ) -def test_create_signed_read_url_v4_w_non_ascii_name(storage_client, signing_bucket): +def test_create_signed_read_url_v4_w_non_ascii_name( + storage_client, signing_bucket, no_mtls +): _create_signed_read_url_helper( storage_client, signing_bucket, @@ -181,7 +197,7 @@ def test_create_signed_read_url_v4_w_non_ascii_name(storage_client, signing_buck ) -def test_create_signed_read_url_v2_w_csek(storage_client, signing_bucket): +def test_create_signed_read_url_v2_w_csek(storage_client, signing_bucket, no_mtls): encryption_key = os.urandom(32) _create_signed_read_url_helper( storage_client, @@ -192,7 +208,7 @@ def test_create_signed_read_url_v2_w_csek(storage_client, signing_bucket): ) -def test_create_signed_read_url_v4_w_csek(storage_client, signing_bucket): +def test_create_signed_read_url_v4_w_csek(storage_client, signing_bucket, no_mtls): encryption_key = os.urandom(32) _create_signed_read_url_helper( storage_client, @@ -205,7 +221,7 @@ def test_create_signed_read_url_v4_w_csek(storage_client, signing_bucket): def test_create_signed_read_url_v2_w_access_token( - storage_client, signing_bucket, service_account, + storage_client, signing_bucket, service_account, no_mtls ): client = iam_credentials_v1.IAMCredentialsClient() service_account_email = service_account.service_account_email @@ -229,7 +245,7 @@ def test_create_signed_read_url_v2_w_access_token( def test_create_signed_read_url_v4_w_access_token( - storage_client, signing_bucket, service_account, + storage_client, signing_bucket, service_account, no_mtls ): client = iam_credentials_v1.IAMCredentialsClient() service_account_email = service_account.service_account_email @@ -270,11 +286,11 @@ def _create_signed_delete_url_helper(client, bucket, version="v2", expiration=No assert not blob.exists() -def test_create_signed_delete_url_v2(storage_client, signing_bucket): +def test_create_signed_delete_url_v2(storage_client, signing_bucket, no_mtls): _create_signed_delete_url_helper(storage_client, signing_bucket) -def test_create_signed_delete_url_v4(storage_client, signing_bucket): +def test_create_signed_delete_url_v4(storage_client, signing_bucket, no_mtls): _create_signed_delete_url_helper(storage_client, signing_bucket, version="v4") @@ -318,20 +334,20 @@ def _create_signed_resumable_upload_url_helper( assert delete_response.status_code == 204 -def test_create_signed_resumable_upload_url_v2(storage_client, signing_bucket): +def test_create_signed_resumable_upload_url_v2(storage_client, signing_bucket, no_mtls): _create_signed_resumable_upload_url_helper( storage_client, signing_bucket, version="v2", ) -def test_create_signed_resumable_upload_url_v4(storage_client, signing_bucket): +def test_create_signed_resumable_upload_url_v4(storage_client, signing_bucket, no_mtls): _create_signed_resumable_upload_url_helper( storage_client, signing_bucket, version="v4", ) def test_generate_signed_post_policy_v4( - storage_client, buckets_to_delete, blobs_to_delete, service_account, + storage_client, buckets_to_delete, blobs_to_delete, service_account, no_mtls ): bucket_name = _helpers.unique_name("post_policy") bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket_name) @@ -364,7 +380,7 @@ def test_generate_signed_post_policy_v4( def test_generate_signed_post_policy_v4_invalid_field( - storage_client, buckets_to_delete, blobs_to_delete, service_account, + storage_client, buckets_to_delete, blobs_to_delete, service_account, no_mtls ): bucket_name = _helpers.unique_name("post_policy-invalid") bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket_name) diff --git a/tests/system/test_kms_integration.py b/tests/system/test_kms_integration.py index 123658a4e..a2df6848a 100644 --- a/tests/system/test_kms_integration.py +++ b/tests/system/test_kms_integration.py @@ -229,7 +229,7 @@ def test_blob_upload_w_bucket_cmek_enabled( blob.upload_from_string(payload) blobs_to_delete.append(blob) - _helpers.retry_429_harder(blob.reload)() + _helpers.retry_429_harder(_helpers.retry_has_kms_key_name(blob.reload))() # We don't know the current version of the key. assert blob.kms_key_name.startswith(kms_key_name)