Skip to content

Commit 0a6dc31

Browse files
dhermesdhermes
authored
Merge pull request #3138 from dhermes/ds-move-lookup-to-GAPIC
Using GAPIC datastore object (and an HTTP equivalent) for lookup.
2 parents d9f2dda + 39bbf25 commit 0a6dc31

File tree

9 files changed

+274
-563
lines changed

9 files changed

+274
-563
lines changed

‎datastore/google/cloud/datastore/_gax.py‎

Lines changed: 0 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,13 @@
1919
import sys
2020

2121
from google.cloud.gapic.datastore.v1 import datastore_client
22-
from google.cloud.proto.datastore.v1 import datastore_pb2_grpc
2322
from google.gax.errors import GaxError
2423
from google.gax.grpc import exc_to_code
2524
from google.gax.utils import metrics
2625
from grpc import StatusCode
2726
import six
2827

29-
from google.cloud._helpers import make_insecure_stub
3028
from google.cloud._helpers import make_secure_channel
31-
from google.cloud._helpers import make_secure_stub
3229
from google.cloud._http import DEFAULT_USER_AGENT
3330
from google.cloud import exceptions
3431

@@ -92,50 +89,6 @@ def _grpc_catch_rendezvous():
9289
six.reraise(error_class, new_exc, sys.exc_info()[2])
9390

9491

95-
class _DatastoreAPIOverGRPC(object):
96-
"""Helper mapping datastore API methods.
97-
98-
Makes requests to send / receive protobuf content over gRPC.
99-
100-
Methods make bare API requests without any helpers for constructing
101-
the requests or parsing the responses.
102-
103-
:type connection: :class:`Connection`
104-
:param connection: A connection object that contains helpful
105-
information for making requests.
106-
"""
107-
108-
def __init__(self, connection):
109-
parse_result = six.moves.urllib_parse.urlparse(
110-
connection.api_base_url)
111-
host = parse_result.hostname
112-
if parse_result.scheme == 'https':
113-
self._stub = make_secure_stub(
114-
connection.credentials, DEFAULT_USER_AGENT,
115-
datastore_pb2_grpc.DatastoreStub, host,
116-
extra_options=_GRPC_EXTRA_OPTIONS)
117-
else:
118-
self._stub = make_insecure_stub(
119-
datastore_pb2_grpc.DatastoreStub, host)
120-
121-
def lookup(self, project, request_pb):
122-
"""Perform a ``lookup`` request.
123-
124-
:type project: str
125-
:param project: The project to connect to. This is
126-
usually your project name in the cloud console.
127-
128-
:type request_pb: :class:`.datastore_pb2.LookupRequest`
129-
:param request_pb: The request protobuf object.
130-
131-
:rtype: :class:`.datastore_pb2.LookupResponse`
132-
:returns: The returned protobuf response object.
133-
"""
134-
request_pb.project_id = project
135-
with _grpc_catch_rendezvous():
136-
return self._stub.Lookup(request_pb)
137-
138-
13992
class GAPICDatastoreAPI(datastore_client.DatastoreClient):
14093
"""An API object that sends proto-over-gRPC requests.
14194

‎datastore/google/cloud/datastore/_http.py‎

Lines changed: 21 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,13 @@
1414

1515
"""Connections to Google Cloud Datastore API servers."""
1616

17-
import os
18-
1917
from google.rpc import status_pb2
2018

2119
from google.cloud import _http as connection_module
22-
from google.cloud.environment_vars import DISABLE_GRPC
2320
from google.cloud import exceptions
2421
from google.cloud.proto.datastore.v1 import datastore_pb2 as _datastore_pb2
2522

2623
from google.cloud.datastore import __version__
27-
try:
28-
from google.cloud.datastore._gax import _DatastoreAPIOverGRPC
29-
_HAVE_GRPC = True
30-
except ImportError: # pragma: NO COVER
31-
_DatastoreAPIOverGRPC = None
32-
_HAVE_GRPC = False
3324

3425

3526
DATASTORE_API_HOST = 'datastore.googleapis.com'
@@ -42,8 +33,6 @@
4233
'/{project}:{method}')
4334
"""A template for the URL of a particular API call."""
4435

45-
_DISABLE_GRPC = os.getenv(DISABLE_GRPC, False)
46-
_USE_GRPC = _HAVE_GRPC and not _DISABLE_GRPC
4736
_CLIENT_INFO = connection_module.CLIENT_INFO_TEMPLATE.format(__version__)
4837

4938

@@ -148,121 +137,45 @@ def build_api_url(project, method, base_url):
148137
project=project, method=method)
149138

150139

151-
class _DatastoreAPIOverHttp(object):
152-
"""Helper mapping datastore API methods.
153-
154-
Makes requests to send / receive protobuf content over HTTP/1.1.
140+
class HTTPDatastoreAPI(object):
141+
"""An API object that sends proto-over-HTTP requests.
155142
156-
Methods make bare API requests without any helpers for constructing
157-
the requests or parsing the responses.
143+
Intended to provide the same methods as the GAPIC ``DatastoreClient``.
158144
159-
:type connection: :class:`Connection`
160-
:param connection: A connection object that contains helpful
161-
information for making requests.
145+
:type client: :class:`~google.cloud.datastore.client.Client`
146+
:param client: The client that provides configuration.
162147
"""
163148

164-
def __init__(self, connection):
165-
self.connection = connection
149+
def __init__(self, client):
150+
self.client = client
166151

167-
def lookup(self, project, request_pb):
152+
def lookup(self, project, read_options, key_pbs):
168153
"""Perform a ``lookup`` request.
169154
170155
:type project: str
171156
:param project: The project to connect to. This is
172157
usually your project name in the cloud console.
173158
174-
:type request_pb: :class:`.datastore_pb2.LookupRequest`
175-
:param request_pb: The request protobuf object.
176-
177-
:rtype: :class:`.datastore_pb2.LookupResponse`
178-
:returns: The returned protobuf response object.
179-
"""
180-
return _rpc(self.connection.http, project, 'lookup',
181-
self.connection.api_base_url,
182-
request_pb, _datastore_pb2.LookupResponse)
183-
184-
185-
class Connection(connection_module.Connection):
186-
"""A connection to the Google Cloud Datastore via the Protobuf API.
187-
188-
This class should understand only the basic types (and protobufs)
189-
in method arguments, however it should be capable of returning advanced
190-
types.
191-
192-
:type client: :class:`~google.cloud.datastore.client.Client`
193-
:param client: The client that owns the current connection.
194-
"""
195-
196-
def __init__(self, client):
197-
super(Connection, self).__init__(client)
198-
self.api_base_url = client._base_url
199-
if _USE_GRPC:
200-
self._datastore_api = _DatastoreAPIOverGRPC(self)
201-
else:
202-
self._datastore_api = _DatastoreAPIOverHttp(self)
203-
204-
def lookup(self, project, key_pbs,
205-
eventual=False, transaction_id=None):
206-
"""Lookup keys from a project in the Cloud Datastore.
207-
208-
Maps the ``DatastoreService.Lookup`` protobuf RPC.
209-
210-
This uses mostly protobufs
211-
(:class:`.entity_pb2.Key` as input and :class:`.entity_pb2.Entity`
212-
as output). It is used under the hood in
213-
:meth:`Client.get() <.datastore.client.Client.get>`:
214-
215-
.. code-block:: python
216-
217-
>>> from google.cloud import datastore
218-
>>> client = datastore.Client(project='project')
219-
>>> key = client.key('MyKind', 1234)
220-
>>> client.get(key)
221-
[<Entity object>]
222-
223-
Using a :class:`Connection` directly:
224-
225-
.. code-block:: python
226-
227-
>>> connection.lookup('project', [key.to_protobuf()])
228-
[<Entity protobuf>]
229-
230-
:type project: str
231-
:param project: The project to look up the keys in.
159+
:type read_options: :class:`.datastore_pb2.ReadOptions`
160+
:param read_options: The options for this lookup. Contains a
161+
either the transaction for the read or
162+
``STRONG`` or ``EVENTUAL`` read consistency.
232163
233164
:type key_pbs: list of
234165
:class:`.entity_pb2.Key`
235166
:param key_pbs: The keys to retrieve from the datastore.
236167
237-
:type eventual: bool
238-
:param eventual: If False (the default), request ``STRONG`` read
239-
consistency. If True, request ``EVENTUAL`` read
240-
consistency.
241-
242-
:type transaction_id: str
243-
:param transaction_id: If passed, make the request in the scope of
244-
the given transaction. Incompatible with
245-
``eventual==True``.
246-
247168
:rtype: :class:`.datastore_pb2.LookupResponse`
248-
:returns: The returned protobuf for the lookup request.
169+
:returns: The returned protobuf response object.
249170
"""
250-
lookup_request = _datastore_pb2.LookupRequest(keys=key_pbs)
251-
_set_read_options(lookup_request, eventual, transaction_id)
252-
return self._datastore_api.lookup(project, lookup_request)
253-
254-
255-
class HTTPDatastoreAPI(object):
256-
"""An API object that sends proto-over-HTTP requests.
257-
258-
Intended to provide the same methods as the GAPIC ``DatastoreClient``.
259-
260-
:type client: :class:`~google.cloud.datastore.client.Client`
261-
:param client: The client that provides configuration.
262-
"""
263-
264-
def __init__(self, client):
265-
self.client = client
171+
request_pb = _datastore_pb2.LookupRequest(
172+
project_id=project,
173+
read_options=read_options,
174+
keys=key_pbs,
175+
)
176+
return _rpc(self.client._http, project, 'lookup',
177+
self.client._base_url,
178+
request_pb, _datastore_pb2.LookupResponse)
266179

267180
def run_query(self, project, partition_id, read_options,
268181
query=None, gql_query=None):
@@ -390,21 +303,3 @@ def allocate_ids(self, project, key_pbs):
390303
return _rpc(self.client._http, project, 'allocateIds',
391304
self.client._base_url,
392305
request_pb, _datastore_pb2.AllocateIdsResponse)
393-
394-
395-
def _set_read_options(request, eventual, transaction_id):
396-
"""Validate rules for read options, and assign to the request.
397-
398-
Helper method for ``lookup()`` and ``run_query``.
399-
400-
:raises: :class:`ValueError` if ``eventual`` is ``True`` and the
401-
``transaction_id`` is not ``None``.
402-
"""
403-
if eventual and (transaction_id is not None):
404-
raise ValueError('eventual must be False when in a transaction')
405-
406-
opts = request.read_options
407-
if eventual:
408-
opts.read_consistency = _datastore_pb2.ReadOptions.EVENTUAL
409-
elif transaction_id:
410-
opts.transaction = transaction_id

‎datastore/google/cloud/datastore/batch.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ def _commit(self):
249249
self.project, mode, self._mutations, transaction=self._id)
250250
_, updated_keys = _parse_commit_response(commit_response_pb)
251251
# If the back-end returns without error, we are guaranteed that
252-
# :meth:`Connection.commit` will return keys that match (length and
252+
# ``commit`` will return keys that match (length and
253253
# order) directly ``_partial_key_entities``.
254254
for new_key_pb, entity in zip(updated_keys,
255255
self._partial_key_entities):

‎datastore/google/cloud/datastore/client.py‎

Lines changed: 43 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515

1616
import os
1717

18+
from google.cloud.proto.datastore.v1 import datastore_pb2 as _datastore_pb2
19+
1820
from google.cloud._helpers import _LocalStack
1921
from google.cloud._helpers import (
2022
_determine_default_project as _base_default_project)
@@ -23,7 +25,6 @@
2325
from google.cloud.environment_vars import GCD_DATASET
2426
from google.cloud.environment_vars import GCD_HOST
2527

26-
from google.cloud.datastore._http import Connection
2728
from google.cloud.datastore._http import HTTPDatastoreAPI
2829
from google.cloud.datastore import helpers
2930
from google.cloud.datastore.batch import Batch
@@ -78,15 +79,18 @@ def _determine_default_project(project=None):
7879
return project
7980

8081

81-
def _extended_lookup(connection, project, key_pbs,
82+
def _extended_lookup(datastore_api, project, key_pbs,
8283
missing=None, deferred=None,
8384
eventual=False, transaction_id=None):
8485
"""Repeat lookup until all keys found (unless stop requested).
8586
8687
Helper function for :meth:`Client.get_multi`.
8788
88-
:type connection: :class:`google.cloud.datastore._http.Connection`
89-
:param connection: The connection used to connect to datastore.
89+
:type datastore_api:
90+
:class:`google.cloud.datastore._http.HTTPDatastoreAPI`
91+
or :class:`google.cloud.datastore._gax.GAPICDatastoreAPI`
92+
:param datastore_api: The datastore API object used to connect
93+
to datastore.
9094
9195
:type project: str
9296
:param project: The project to make the request for.
@@ -127,15 +131,11 @@ def _extended_lookup(connection, project, key_pbs,
127131
results = []
128132

129133
loop_num = 0
134+
read_options = _get_read_options(eventual, transaction_id)
130135
while loop_num < _MAX_LOOPS: # loop against possible deferred.
131136
loop_num += 1
132-
133-
lookup_response = connection.lookup(
134-
project=project,
135-
key_pbs=key_pbs,
136-
eventual=eventual,
137-
transaction_id=transaction_id,
138-
)
137+
lookup_response = datastore_api.lookup(
138+
project, read_options, key_pbs)
139139

140140
# Accumulate the new results.
141141
results.extend(result.entity for result in lookup_response.found)
@@ -210,9 +210,6 @@ def __init__(self, project=None, namespace=None,
210210
self._base_url = 'http://' + host
211211
except KeyError:
212212
self._base_url = _DATASTORE_BASE_URL
213-
# NOTE: Make sure all properties are set before passing to
214-
# ``Connection`` (e.g. ``_base_url``).
215-
self._connection = Connection(self)
216213

217214
@staticmethod
218215
def _determine_default(project):
@@ -347,7 +344,7 @@ def get_multi(self, keys, missing=None, deferred=None, transaction=None):
347344
transaction = self.current_transaction
348345

349346
entity_pbs = _extended_lookup(
350-
connection=self._connection,
347+
datastore_api=self._datastore_api,
351348
project=self.project,
352349
key_pbs=[k.to_protobuf() for k in keys],
353350
missing=missing,
@@ -569,3 +566,34 @@ def do_something(entity):
569566
if 'namespace' not in kwargs:
570567
kwargs['namespace'] = self.namespace
571568
return Query(self, **kwargs)
569+
570+
571+
def _get_read_options(eventual, transaction_id):
572+
"""Validate rules for read options, and assign to the request.
573+
574+
Helper method for ``lookup()`` and ``run_query``.
575+
576+
:type eventual: bool
577+
:param eventual: Flag indicating if ``EVENTUAL`` or ``STRONG``
578+
consistency should be used.
579+
580+
:type transaction_id: bytes
581+
:param transaction_id: A transaction identifier (may be null).
582+
583+
:rtype: :class:`.datastore_pb2.ReadOptions`
584+
:returns: The read options corresponding to the inputs.
585+
:raises: :class:`ValueError` if ``eventual`` is ``True`` and the
586+
``transaction_id`` is not ``None``.
587+
"""
588+
if transaction_id is None:
589+
if eventual:
590+
return _datastore_pb2.ReadOptions(
591+
read_consistency=_datastore_pb2.ReadOptions.EVENTUAL)
592+
else:
593+
return _datastore_pb2.ReadOptions()
594+
else:
595+
if eventual:
596+
raise ValueError('eventual must be False when in a transaction')
597+
else:
598+
return _datastore_pb2.ReadOptions(
599+
transaction=transaction_id)

‎datastore/google/cloud/datastore/query.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -362,8 +362,6 @@ def fetch(self, limit=None, offset=0, start_cursor=None, end_cursor=None,
362362
363363
:rtype: :class:`Iterator`
364364
:returns: The iterator for the query.
365-
:raises: ValueError if ``connection`` is not passed and no implicit
366-
default has been set.
367365
"""
368366
if client is None:
369367
client = self._client

0 commit comments

Comments
 (0)