From 79b669bbede1cd4f06f1d697b71c7f9f2442fb80 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 18 Nov 2021 18:51:49 +0100 Subject: [PATCH 01/13] chore(deps): update dependency google-cloud-storage to v1.43.0 (#654) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index addda1960..240aa5070 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-pubsub==2.9.0 -google-cloud-storage==1.42.3 +google-cloud-storage==1.43.0 pandas==1.3.4; python_version > '3.6' pandas==1.1.5; python_version < '3.7' From 0459cb4e866696c46385a5ad72e2a85db810a36b Mon Sep 17 00:00:00 2001 From: Bonnie Chan <52431539+cbonnie@users.noreply.github.com> Date: Fri, 10 Dec 2021 13:08:30 -0500 Subject: [PATCH 02/13] docs: Describe code sample more specifically (#660) docs: This is just a simple PR to better describe what the code is doing in the comments. --- samples/snippets/storage_create_bucket_class_location.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/samples/snippets/storage_create_bucket_class_location.py b/samples/snippets/storage_create_bucket_class_location.py index 64c2652d7..51fa86440 100644 --- a/samples/snippets/storage_create_bucket_class_location.py +++ b/samples/snippets/storage_create_bucket_class_location.py @@ -21,7 +21,10 @@ def create_bucket_class_location(bucket_name): - """Create a new bucket in specific location with storage class""" + """ + Create a new bucket in the US region with the coldline storage + class + """ # bucket_name = "your-new-bucket-name" storage_client = storage.Client() From b841482570528b5f4cd1d5b4a838d43290c8dea2 Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Fri, 10 Dec 2021 14:36:52 -0800 Subject: [PATCH 03/13] samples: added upload from/download into memory samples (#664) * samples: added upload from/download into memory samples * linted files: * responded to PR comments * renamed blob variable * responded to comments and updated readme * updated copyright * fixed test --- samples/README.md | 20 +++++++ samples/snippets/snippets_test.py | 20 +++++++ .../snippets/storage_download_into_memory.py | 55 +++++++++++++++++++ .../snippets/storage_upload_from_memory.py | 52 ++++++++++++++++++ 4 files changed, 147 insertions(+) create mode 100644 samples/snippets/storage_download_into_memory.py create mode 100644 samples/snippets/storage_upload_from_memory.py diff --git a/samples/README.md b/samples/README.md index bb48adc9b..31e72cc51 100644 --- a/samples/README.md +++ b/samples/README.md @@ -82,6 +82,7 @@ for instructions on setting up credentials for applications. * [Download Encrypted File](#download-encrypted-file) * [Download File](#download-file) * [Download File Requester Pays](#download-file-requester-pays) +* [Download Into Memory](#download-into-memory) * [Download Public File](#download-public-file) * [Enable Bucket Lifecycle Management](#enable-bucket-lifecycle-management) * [Enable Default Event Based Hold](#enable-default-event-based-hold) @@ -144,6 +145,7 @@ for instructions on setting up credentials for applications. * [Set Temporary Hold](#set-temporary-hold) * [Upload Encrypted File](#upload-encrypted-file) * [Upload File](#upload-file) +* [Upload From Memory](#upload-from-memory) * [Upload With KMS Key](#upload-with-kms-key) * [View Bucket IAM Members](#view-bucket-iam-members) @@ -456,6 +458,15 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_download_file_requester_pays.py ` +----- +### Download Into Memory +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_download_into_memory.py,samples/README.md) + +View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_download_into_memory.py). To run this sample: + + +`python storage_download_into_memory.py ` + ----- ### Download Public File [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_download_public_file.py,samples/README.md) @@ -1014,6 +1025,15 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_upload_file.py ` +----- +### Upload From Memory +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_upload_from_memory.py,samples/README.md) + +View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_upload_from_memory.py). To run this sample: + + +`python storage_upload_from_memory.py ` + ----- ### Upload With KMS Key [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_upload_with_kms_key.py,samples/README.md) diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index dd8e6aeaf..7dc27ae1b 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -38,6 +38,7 @@ import storage_disable_bucket_lifecycle_management import storage_disable_versioning import storage_download_file +import storage_download_into_memory import storage_download_public_file import storage_enable_bucket_lifecycle_management import storage_enable_versioning @@ -62,6 +63,7 @@ import storage_set_bucket_default_kms_key import storage_set_metadata import storage_upload_file +import storage_upload_from_memory import storage_upload_with_kms_key KMS_KEY = os.environ["CLOUD_KMS_KEY"] @@ -189,6 +191,15 @@ def test_upload_blob(test_bucket): ) +def test_upload_blob_from_memory(test_bucket, capsys): + storage_upload_from_memory.upload_blob_from_memory( + test_bucket.name, "Hello, is it me you're looking for?", "test_upload_blob" + ) + out, _ = capsys.readouterr() + + assert "Hello, is it me you're looking for?" in out + + def test_upload_blob_with_kms(test_bucket): with tempfile.NamedTemporaryFile() as source_file: source_file.write(b"test") @@ -209,6 +220,15 @@ def test_download_blob(test_blob): assert dest_file.read() +def test_download_blob_into_memory(test_blob, capsys): + storage_download_into_memory.download_blob_into_memory( + test_blob.bucket.name, test_blob.name + ) + out, _ = capsys.readouterr() + + assert "Hello, is it me you're looking for?" in out + + def test_blob_metadata(test_blob, capsys): storage_get_metadata.blob_metadata(test_blob.bucket.name, test_blob.name) out, _ = capsys.readouterr() diff --git a/samples/snippets/storage_download_into_memory.py b/samples/snippets/storage_download_into_memory.py new file mode 100644 index 000000000..dd91302aa --- /dev/null +++ b/samples/snippets/storage_download_into_memory.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_download_file] +from google.cloud import storage + + +def download_blob_into_memory(bucket_name, blob_name): + """Downloads a blob into memory.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your GCS object + # blob_name = "storage-object-name" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Construct a client side representation of a blob. + # Note `Bucket.blob` differs from `Bucket.get_blob` as it doesn't retrieve + # any content from Google Cloud Storage. As we don't need additional data, + # using `Bucket.blob` is preferred here. + blob = bucket.blob(blob_name) + contents = blob.download_as_string() + + print( + "Downloaded storage object {} from bucket {} as the following string: {}.".format( + blob_name, bucket_name, contents + ) + ) + + +# [END storage_download_file] + +if __name__ == "__main__": + download_blob_into_memory( + bucket_name=sys.argv[1], + blob_name=sys.argv[2], + ) diff --git a/samples/snippets/storage_upload_from_memory.py b/samples/snippets/storage_upload_from_memory.py new file mode 100644 index 000000000..e5f61ff93 --- /dev/null +++ b/samples/snippets/storage_upload_from_memory.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_file_upload_from_memory] +from google.cloud import storage + + +def upload_blob_from_memory(bucket_name, contents, destination_blob_name): + """Uploads a file to the bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The contents to upload to the file + # contents = "these are my contents" + # The ID of your GCS object + # destination_blob_name = "storage-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + + blob.upload_from_string(contents) + + print( + "{} with contents {} uploaded to {}.".format( + destination_blob_name, contents, destination_blob_name + ) + ) + + +# [END storage_file_upload_from_memory] + +if __name__ == "__main__": + upload_blob_from_memory( + bucket_name=sys.argv[1], + contents=sys.argv[2], + destination_blob_name=sys.argv[3], + ) From ee41b3f883632026e5d0460847122bd1c37b9b2d Mon Sep 17 00:00:00 2001 From: cojenco Date: Fri, 10 Dec 2021 15:33:14 -0800 Subject: [PATCH 04/13] tests: speed up retry conformance tests (#655) * tests: speed up retry conf tests running in parallel * update conftest readme * update noxfile and comments * check test session args * lint --- noxfile.py | 26 ++++++++++++------- ...retry_conformance_testing.md => README.md} | 8 ++++++ 2 files changed, 24 insertions(+), 10 deletions(-) rename tests/conformance/{retry_conformance_testing.md => README.md} (84%) diff --git a/noxfile.py b/noxfile.py index 67bfa4eeb..3bb910dc2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -154,24 +154,30 @@ def system(session): @nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS) def conftest_retry(session): """Run the retry conformance test suite.""" - conformance_test_path = os.path.join("tests", "conformance.py") conformance_test_folder_path = os.path.join("tests", "conformance") - conformance_test_exists = os.path.exists(conformance_test_path) conformance_test_folder_exists = os.path.exists(conformance_test_folder_path) # Environment check: only run tests if found. - if not conformance_test_exists and not conformance_test_folder_exists: + if not conformance_test_folder_exists: session.skip("Conformance tests were not found") - session.install("pytest",) + # Install all test dependencies and pytest plugin to run tests in parallel. + # Then install this package in-place. + session.install("pytest", "pytest-xdist") session.install("-e", ".") + # Run #CPU processes in parallel if no test session arguments are passed in. + if session.posargs: + test_cmd = [ + "py.test", + "--quiet", + conformance_test_folder_path, + *session.posargs, + ] + else: + test_cmd = ["py.test", "-n", "auto", "--quiet", conformance_test_folder_path] + # Run py.test against the conformance tests. - if conformance_test_exists: - session.run("py.test", "--quiet", conformance_test_path, *session.posargs) - if conformance_test_folder_exists: - session.run( - "py.test", "--quiet", conformance_test_folder_path, *session.posargs - ) + session.run(*test_cmd) @nox.session(python=DEFAULT_PYTHON_VERSION) diff --git a/tests/conformance/retry_conformance_testing.md b/tests/conformance/README.md similarity index 84% rename from tests/conformance/retry_conformance_testing.md rename to tests/conformance/README.md index 22b7c1952..3b5fa8884 100644 --- a/tests/conformance/retry_conformance_testing.md +++ b/tests/conformance/README.md @@ -27,3 +27,11 @@ To run the test suite locally: ```bash nox -s conftest_retry-3.8 ``` + +To run a single test locally: + +Single test names are displayed as "test-S{scenario_id}-{method}-{client-library-method-name}-{instructions index}", such as `test-S1-storage.buckets.get-bucket_reload-1` + +```bash +nox -re conftest_retry-3.8 -- -k +``` From 10cdad630739a324ae0b16a3d14a67ca4c8a23c2 Mon Sep 17 00:00:00 2001 From: cojenco Date: Tue, 14 Dec 2021 16:56:58 -0800 Subject: [PATCH 05/13] feat: add raw_download kwarg to BlobReader (#668) * feat: add raw_download kwarg to BlobReader * update docstrings --- google/cloud/storage/blob.py | 6 +++++- google/cloud/storage/fileio.py | 1 + tests/unit/test_fileio.py | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 2d7eccf25..75b22861e 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -3850,6 +3850,10 @@ def open( - ``timeout`` - ``retry`` + For downloads only, the following additional arguments are supported: + + - ``raw_download`` + For uploads only, the following additional arguments are supported: - ``content_type`` @@ -3877,7 +3881,7 @@ def open( >>> client = storage.Client() >>> bucket = client.bucket("bucket-name") - >>> blob = bucket.get_blob("blob-name.txt") + >>> blob = bucket.blob("blob-name.txt") >>> with blob.open("rt") as f: >>> print(f.read()) diff --git a/google/cloud/storage/fileio.py b/google/cloud/storage/fileio.py index 54d1577f4..95bb12b1f 100644 --- a/google/cloud/storage/fileio.py +++ b/google/cloud/storage/fileio.py @@ -35,6 +35,7 @@ "if_metageneration_not_match", "timeout", "retry", + "raw_download", } # Valid keyword arguments for upload methods. diff --git a/tests/unit/test_fileio.py b/tests/unit/test_fileio.py index bf017e44f..d71103707 100644 --- a/tests/unit/test_fileio.py +++ b/tests/unit/test_fileio.py @@ -109,6 +109,24 @@ def read_from_fake_data(start=0, end=None, **_): reader.close() + def test_read_with_raw_download(self): + blob = mock.Mock() + + def read_from_fake_data(start=0, end=None, **_): + return TEST_BINARY_DATA[start:end] + + blob.download_as_bytes = mock.Mock(side_effect=read_from_fake_data) + download_kwargs = {"raw_download": True} + reader = self._make_blob_reader(blob, chunk_size=8, **download_kwargs) + + # Read and trigger the first download of chunk_size. + self.assertEqual(reader.read(1), TEST_BINARY_DATA[0:1]) + blob.download_as_bytes.assert_called_once_with( + start=0, end=8, checksum=None, retry=DEFAULT_RETRY, raw_download=True + ) + + reader.close() + def test_retry_passed_through(self): blob = mock.Mock() From ceb931403a755f2a0bdc20144287dbc4700c3360 Mon Sep 17 00:00:00 2001 From: cojenco Date: Wed, 15 Dec 2021 17:22:34 -0800 Subject: [PATCH 06/13] docs: refresh readme instructions (#667) Update instructions and links in quick start/ samples readme per @cbonnie friction log :) --- README.rst | 43 ++++++++++++++++++++++++++++--------------- samples/README.md | 13 +++++-------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/README.rst b/README.rst index 358a28f29..0796bb05d 100644 --- a/README.rst +++ b/README.rst @@ -34,20 +34,22 @@ In order to use this library, you first need to go through the following steps: .. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project .. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project .. _Enable the Google Cloud Storage API.: https://cloud.google.com/storage -.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html +.. _Setup Authentication.: https://cloud.google.com/storage/docs/reference/libraries#setting_up_authentication Installation ~~~~~~~~~~~~ -Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to -create isolated Python environments. The basic problem it addresses is one of -dependencies and versions, and indirectly permissions. +`Set up a Python development environment`_ and install this library in a `venv`. +`venv`_ is a tool to create isolated Python environments. The basic problem it +addresses is one of dependencies and versions, and indirectly permissions. -With `virtualenv`_, it's possible to install this library without needing system +Make sure you're using Python 3.3 or later, which includes `venv`_ by default. +With `venv`, it's possible to install this library without needing system install permissions, and without clashing with the installed system dependencies. -.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ +.. _Set up a Python development environment: https://cloud.google.com/python/docs/setup +.. _`venv`: https://docs.python.org/3/library/venv.html Supported Python Versions @@ -68,10 +70,9 @@ Mac/Linux .. code-block:: console - pip install virtualenv - virtualenv - source /bin/activate - /bin/pip install google-cloud-storage + python -m venv env + source env/bin/activate + pip install google-cloud-storage Windows @@ -79,10 +80,9 @@ Windows .. code-block:: console - pip install virtualenv - virtualenv - \Scripts\activate - \Scripts\pip.exe install google-cloud-storage + py -m venv env + .\env\Scripts\activate + pip install google-cloud-storage Example Usage @@ -90,8 +90,13 @@ Example Usage .. code:: python + # Imports the Google Cloud client library from google.cloud import storage + + # Instantiates a client client = storage.Client() + + # Creates a new bucket and uploads an object new_bucket = client.create_bucket('new-bucket-id') new_blob = new_bucket.blob('remote/path/storage.txt') new_blob.upload_from_filename(filename='/local/path.txt') @@ -103,4 +108,12 @@ Example Usage blob = bucket.get_blob('remote/path/to/file.txt') print(blob.download_as_bytes()) blob.upload_from_string('New contents!') - + + +What's Next +~~~~~~~~~~~ + +Now that you've set up your Python client for Cloud Storage, +you can get started running `Storage samples.`_ + +.. _Storage samples.: https://github.com/googleapis/python-storage/tree/main/samples diff --git a/samples/README.md b/samples/README.md index 31e72cc51..c343fca9e 100644 --- a/samples/README.md +++ b/samples/README.md @@ -21,8 +21,8 @@ Before running the samples, make sure you've followed the steps outlined in [Quick Start](https://github.com/googleapis/python-storage#quick-start). ### Authentication -This sample requires you to have authentication setup. Refer to the [Authentication Getting Started Guide](https://cloud.google.com/docs/authentication/getting-started) -for instructions on setting up credentials for applications. +Refer to the [Authentication Set Up Guide](https://cloud.google.com/storage/docs/reference/libraries#setting_up_authentication) +for more detailed instructions. ### Install Dependencies 1. Clone this repository and change to the sample directory you want to use. @@ -30,15 +30,12 @@ for instructions on setting up credentials for applications. git clone https://github.com/googleapis/python-storage.git ``` -2. Install [pip](https://pip.pypa.io/) and [virtualenv](https://virtualenv.pypa.io) if you do not already have them. You may want to refer to the [Python Development Environment Setup Guide](https://cloud.google.com/python/setup) for Google Cloud Platform for instructions. - -3. Create a virtualenv. Samples are compatible with Python 3.6+. +2. Activate a venv if you have not already from the [Quick Start](https://github.com/googleapis/python-storage#quick-start). ``` - virtualenv env - source env/bin/activate + source /bin/activate ``` -4. Install the dependencies needed to run the samples. +3. Install the dependencies needed to run the samples. ``` cd samples/snippets pip install -r requirements.txt From 0e1a0ffa0a014fe8a8f00f10ddd94a89efd5f55b Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Thu, 16 Dec 2021 10:40:37 -0800 Subject: [PATCH 07/13] samples: delete unspecified sample (#671) * samples: delete unspecified sample * fixed lint issue * updated readme --- samples/README.md | 10 ----- .../snippets/public_access_prevention_test.py | 11 ----- ...et_public_access_prevention_unspecified.py | 43 ------------------- 3 files changed, 64 deletions(-) delete mode 100644 samples/snippets/storage_set_public_access_prevention_unspecified.py diff --git a/samples/README.md b/samples/README.md index c343fca9e..10a9571cc 100644 --- a/samples/README.md +++ b/samples/README.md @@ -137,7 +137,6 @@ for more detailed instructions. * [Set Metadata](#set-metadata) * [Set Public Access Prevention Enforced](#set-public-access-prevention-enforced) * [Set Public Access Prevention Inherited](#set-public-access-prevention-inherited) -* [Set Public Access Prevention Unspecified](#set-public-access-prevention-unspecified) * [Set Retention Policy](#set-retention-policy) * [Set Temporary Hold](#set-temporary-hold) * [Upload Encrypted File](#upload-encrypted-file) @@ -977,15 +976,6 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_set_public_access_prevention_inherited.py ` ------ -### Set Public Access Prevention Unspecified -[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_set_public_access_prevention_unspecified.py,samples/README.md) - -View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_set_public_access_prevention_unspecified.py). To run this sample: - - -`python storage_set_public_access_prevention_unspecified.py ` - ----- ### Set Retention Policy [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_set_retention_policy.py,samples/README.md) diff --git a/samples/snippets/public_access_prevention_test.py b/samples/snippets/public_access_prevention_test.py index 40d3924b2..558a4ef15 100644 --- a/samples/snippets/public_access_prevention_test.py +++ b/samples/snippets/public_access_prevention_test.py @@ -12,15 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest import storage_get_public_access_prevention import storage_set_public_access_prevention_enforced import storage_set_public_access_prevention_inherited -import storage_set_public_access_prevention_unspecified -@pytest.mark.skip(reason="Inconsistent due to unspecified->inherited change") def test_get_public_access_prevention(bucket, capsys): short_name = storage_get_public_access_prevention short_name.get_public_access_prevention(bucket.name) @@ -35,14 +32,6 @@ def test_set_public_access_prevention_enforced(bucket, capsys): assert f"Public access prevention is set to enforced for {bucket.name}." in out -@pytest.mark.skip(reason="Inconsistent due to unspecified->inherited change") -def test_set_public_access_prevention_unspecified(bucket, capsys): - short_name = storage_set_public_access_prevention_unspecified - short_name.set_public_access_prevention_unspecified(bucket.name) - out, _ = capsys.readouterr() - assert f"Public access prevention is 'unspecified' for {bucket.name}." in out - - def test_set_public_access_prevention_inherited(bucket, capsys): short_name = storage_set_public_access_prevention_inherited short_name.set_public_access_prevention_inherited(bucket.name) diff --git a/samples/snippets/storage_set_public_access_prevention_unspecified.py b/samples/snippets/storage_set_public_access_prevention_unspecified.py deleted file mode 100644 index ae2c4701c..000000000 --- a/samples/snippets/storage_set_public_access_prevention_unspecified.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -# [START storage_set_public_access_prevention_unspecified] -from google.cloud import storage -from google.cloud.storage.constants import PUBLIC_ACCESS_PREVENTION_UNSPECIFIED - - -def set_public_access_prevention_unspecified(bucket_name): - """Sets the public access prevention status to unspecified, so that the bucket inherits its setting from its parent project.""" - # The ID of your GCS bucket - # bucket_name = "my-bucket" - - storage_client = storage.Client() - bucket = storage_client.get_bucket(bucket_name) - - bucket.iam_configuration.public_access_prevention = ( - PUBLIC_ACCESS_PREVENTION_UNSPECIFIED - ) - bucket.patch() - - print(f"Public access prevention is 'unspecified' for {bucket.name}.") - - -# [END storage_set_public_access_prevention_unspecified] - -if __name__ == "__main__": - set_public_access_prevention_unspecified(bucket_name=sys.argv[1]) From 82206f400912fb48eaf65f427ca822f2fea00502 Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Mon, 20 Dec 2021 11:02:52 -0800 Subject: [PATCH 08/13] samples: storage_download_byte_range (#674) * samples: storage_download_byte_range * added spacing * updated readme * fixed test --- samples/README.md | 10 +++ samples/snippets/snippets_test.py | 9 +++ .../snippets/storage_download_byte_range.py | 69 +++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 samples/snippets/storage_download_byte_range.py diff --git a/samples/README.md b/samples/README.md index 10a9571cc..4b1501df5 100644 --- a/samples/README.md +++ b/samples/README.md @@ -76,6 +76,7 @@ for more detailed instructions. * [Disable Requester Pays](#disable-requester-pays) * [Disable Uniform Bucket Level Access](#disable-uniform-bucket-level-access) * [Disable Versioning](#disable-versioning) +* [Download Byte Range](#download-byte-range) * [Download Encrypted File](#download-encrypted-file) * [Download File](#download-file) * [Download File Requester Pays](#download-file-requester-pays) @@ -427,6 +428,15 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_disable_versioning.py ` +----- +### Download Byte Range +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_download_byte_range.py,samples/README.md) + +View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_download_byte_range.py). To run this sample: + + +`python storage_download_byte_range.py <>BASE64_ENCRYPTION_KEY` + ----- ### Download Encrypted File [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_download_encrypted_file.py,samples/README.md) diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index 7dc27ae1b..9691388d5 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -37,6 +37,7 @@ import storage_delete_file_archived_generation import storage_disable_bucket_lifecycle_management import storage_disable_versioning +import storage_download_byte_range import storage_download_file import storage_download_into_memory import storage_download_public_file @@ -211,6 +212,14 @@ def test_upload_blob_with_kms(test_bucket): assert kms_blob.kms_key_name.startswith(KMS_KEY) +def test_download_byte_range(test_blob): + with tempfile.NamedTemporaryFile() as dest_file: + storage_download_byte_range.download_byte_range( + test_blob.bucket.name, test_blob.name, 0, 4, dest_file.name + ) + assert dest_file.read() == b'Hello' + + def test_download_blob(test_blob): with tempfile.NamedTemporaryFile() as dest_file: storage_download_file.download_blob( diff --git a/samples/snippets/storage_download_byte_range.py b/samples/snippets/storage_download_byte_range.py new file mode 100644 index 000000000..e6143a04f --- /dev/null +++ b/samples/snippets/storage_download_byte_range.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_download_byte_range] +from google.cloud import storage + + +def download_byte_range( + bucket_name, source_blob_name, start_byte, end_byte, destination_file_name +): + """Downloads a blob from the bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + + # The ID of your GCS object + # source_blob_name = "storage-object-name" + + # The starting byte at which to begin the download + # start_byte = 0 + + # The ending byte at which to end the download + # end_byte = 20 + + # The path to which the file should be downloaded + # destination_file_name = "local/path/to/file" + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Construct a client side representation of a blob. + # Note `Bucket.blob` differs from `Bucket.get_blob` as it doesn't retrieve + # any content from Google Cloud Storage. As we don't need additional data, + # using `Bucket.blob` is preferred here. + blob = bucket.blob(source_blob_name) + blob.download_to_filename(destination_file_name, start=start_byte, end=end_byte) + + print( + "Downloaded bytes {} to {} of object {} from bucket {} to local file {}.".format( + start_byte, end_byte, source_blob_name, bucket_name, destination_file_name + ) + ) + + +# [END storage_download_byte_range] + +if __name__ == "__main__": + download_byte_range( + bucket_name=sys.argv[1], + source_blob_name=sys.argv[2], + start_byte=sys.argv[3], + end_byte=sys.argv[4], + destination_file_name=sys.argv[5], + ) From 7d051b72b967cc0cc369917016afcfe603934117 Mon Sep 17 00:00:00 2001 From: Sameena Shaffeeullah Date: Wed, 22 Dec 2021 10:54:31 -0800 Subject: [PATCH 09/13] samples: update region tags (#675) --- samples/snippets/storage_download_into_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/storage_download_into_memory.py b/samples/snippets/storage_download_into_memory.py index dd91302aa..453a13e21 100644 --- a/samples/snippets/storage_download_into_memory.py +++ b/samples/snippets/storage_download_into_memory.py @@ -16,7 +16,7 @@ import sys -# [START storage_download_file] +# [START storage_file_download_into_memory] from google.cloud import storage @@ -46,7 +46,7 @@ def download_blob_into_memory(bucket_name, blob_name): ) -# [END storage_download_file] +# [END storage_file_download_into_memory] if __name__ == "__main__": download_blob_into_memory( From 96092d4be36be478f9671e8940de4fd09cc6f7f0 Mon Sep 17 00:00:00 2001 From: Micah Smith Date: Thu, 23 Dec 2021 14:58:54 -0500 Subject: [PATCH 10/13] docs: use writeable streamin example for 'download_blob_to_file' (#676) Co-authored-by: Tres Seaver --- google/cloud/storage/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index bef05ea91..9d1d49af8 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -1065,7 +1065,7 @@ def download_blob_to_file( >>> bucket = client.get_bucket('my-bucket-name') >>> blob = storage.Blob('path/to/blob', bucket) - >>> with open('file-to-download-to') as file_obj: + >>> with open('file-to-download-to', 'w') as file_obj: >>> client.download_blob_to_file(blob, file_obj) # API request. @@ -1074,7 +1074,7 @@ def download_blob_to_file( >>> from google.cloud import storage >>> client = storage.Client() - >>> with open('file-to-download-to') as file_obj: + >>> with open('file-to-download-to', 'w') as file_obj: >>> client.download_blob_to_file( >>> 'gs://bucket_name/path/to/blob', file_obj) From 5e49e54adbf1fd1457d01df1ae2df63025ac7ae4 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 28 Dec 2021 13:12:20 -0500 Subject: [PATCH 11/13] chore: update .repo-metadata.json (#677) --- .repo-metadata.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.repo-metadata.json b/.repo-metadata.json index 62797084a..2cd2642fe 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -2,9 +2,9 @@ "name": "storage", "name_pretty": "Google Cloud Storage", "product_documentation": "https://cloud.google.com/storage", - "client_documentation": "https://googleapis.dev/python/storage/latest", + "client_documentation": "https://cloud.google.com/python/docs/reference/storage/latest", "issue_tracker": "https://issuetracker.google.com/savedsearches/559782", - "release_level": "ga", + "release_level": "stable", "language": "python", "library_type": "GAPIC_MANUAL", "repo": "googleapis/python-storage", @@ -12,5 +12,6 @@ "api_id": "storage.googleapis.com", "requires_billing": true, "default_version": "", - "codeowner_team": "@googleapis/cloud-storage-dpe" + "codeowner_team": "@googleapis/cloud-storage-dpe", + "api_shortname": "storage" } From 30efbdba00adf4e5eb638d45953eda0470497f56 Mon Sep 17 00:00:00 2001 From: cojenco Date: Wed, 5 Jan 2022 10:19:05 -0800 Subject: [PATCH 12/13] samples: add batch request sample and test (#656) * samples: add batch request sample and test * samples: update readme with new sample * add clarifying comment --- samples/README.md | 10 ++++ samples/snippets/snippets_test.py | 15 ++++++ samples/snippets/storage_batch_request.py | 60 +++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 samples/snippets/storage_batch_request.py diff --git a/samples/README.md b/samples/README.md index 4b1501df5..2751bf722 100644 --- a/samples/README.md +++ b/samples/README.md @@ -46,6 +46,7 @@ for more detailed instructions. List of Samples * [Activate HMAC Key](#activate-hmac-key) +* [Batch Request](#batch-request) * [Add Bucket Conditional IAM Binding](#add-bucket-conditional-iam-binding) * [Add Bucket Default Owner](#add-bucket-default-owner) * [Add Bucket IAM Member](#add-bucket-iam-member) @@ -157,6 +158,15 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa `python storage_activate_hmac_key.py ` +----- +### Batch Request +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/python-storage&page=editor&open_in_editor=samples/snippets/storage_batch_request.py,samples/README.md) + +View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_batch_request.py). To run this sample: + + +`python storage_batch_request.py ` + ----- ### Add Bucket Conditional IAM Binding diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index 9691388d5..7c0a5b91d 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -23,6 +23,7 @@ import requests import storage_add_bucket_label +import storage_batch_request import storage_bucket_delete_default_kms_key import storage_change_default_storage_class import storage_change_file_storage_class @@ -538,3 +539,17 @@ def test_storage_configure_retries(test_blob, capsys): assert "The following library method is customized to be retried" in out assert "_should_retry" in out assert "initial=1.5, maximum=45.0, multiplier=1.2, deadline=500.0" in out + + +def test_batch_request(test_bucket): + blob1 = test_bucket.blob("b/1.txt") + blob2 = test_bucket.blob("b/2.txt") + blob1.upload_from_string("hello world") + blob2.upload_from_string("hello world") + + storage_batch_request.batch_request(test_bucket.name, "b/") + blob1.reload() + blob2.reload() + + assert blob1.metadata.get("your-metadata-key") == "your-metadata-value" + assert blob2.metadata.get("your-metadata-key") == "your-metadata-value" diff --git a/samples/snippets/storage_batch_request.py b/samples/snippets/storage_batch_request.py new file mode 100644 index 000000000..863fc09cd --- /dev/null +++ b/samples/snippets/storage_batch_request.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that uses a batch request. +This sample is used on this page: + https://cloud.google.com/storage/docs/batch +For more information, see README.md. +""" + +# [START storage_batch_request] + +from google.cloud import storage + + +def batch_request(bucket_name, prefix=None): + """Use a batch request to patch a list of objects with the given prefix in a bucket.""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + # The prefix of the object paths + # prefix = "directory-prefix/" + + client = storage.Client() + bucket = client.bucket(bucket_name) + + # Accumulate in a list the objects with a given prefix. + blobs_to_patch = [blob for blob in bucket.list_blobs(prefix=prefix)] + + # Use a batch context manager to edit metadata in the list of blobs. + # The batch request is sent out when the context manager closes. + # No more than 100 calls should be included in a single batch request. + with client.batch(): + for blob in blobs_to_patch: + metadata = {"your-metadata-key": "your-metadata-value"} + blob.metadata = metadata + blob.patch() + + print( + f"Batch request edited metadata for all objects with the given prefix in {bucket.name}." + ) + + +# [END storage_batch_request] + +if __name__ == "__main__": + batch_request(bucket_name=sys.argv[1], prefix=sys.argv[2]) From 9cc38240a92eaba3205be992085dceb225a23d15 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 11:33:02 -0800 Subject: [PATCH 13/13] chore: release 1.44.0 (#669) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 15 +++++++++++++++ google/cloud/storage/version.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc7d6da38..d5839af78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +## [1.44.0](https://www.github.com/googleapis/python-storage/compare/v1.43.0...v1.44.0) (2022-01-05) + + +### Features + +* add raw_download kwarg to BlobReader ([#668](https://www.github.com/googleapis/python-storage/issues/668)) ([10cdad6](https://www.github.com/googleapis/python-storage/commit/10cdad630739a324ae0b16a3d14a67ca4c8a23c2)) + + +### Documentation + +* Describe code sample more specifically ([#660](https://www.github.com/googleapis/python-storage/issues/660)) ([0459cb4](https://www.github.com/googleapis/python-storage/commit/0459cb4e866696c46385a5ad72e2a85db810a36b)) +* refresh readme instructions ([#667](https://www.github.com/googleapis/python-storage/issues/667)) ([ceb9314](https://www.github.com/googleapis/python-storage/commit/ceb931403a755f2a0bdc20144287dbc4700c3360)) +* This is just a simple PR to better describe what the code is doing in the comments. ([0459cb4](https://www.github.com/googleapis/python-storage/commit/0459cb4e866696c46385a5ad72e2a85db810a36b)) +* use writeable streamin example for 'download_blob_to_file' ([#676](https://www.github.com/googleapis/python-storage/issues/676)) ([96092d4](https://www.github.com/googleapis/python-storage/commit/96092d4be36be478f9671e8940de4fd09cc6f7f0)) + ## [1.43.0](https://www.github.com/googleapis/python-storage/compare/v1.42.3...v1.43.0) (2021-11-15) diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py index 602d66b15..ccad14b45 100644 --- a/google/cloud/storage/version.py +++ b/google/cloud/storage/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.43.0" +__version__ = "1.44.0"