-
Notifications
You must be signed in to change notification settings - Fork 426
Description
Note: #835 was also for this test, but it was closed more than 10 days ago. So, I didn't mark it flaky.
commit: ba11c3d
buildURL: Build Status, Sponge
status: failed
Test output
self =
shared_state = {'bucket': , 'resources': [}
@pytest.fixture(scope="class", autouse=True)
def teardown(self, shared_state: Dict[str, Any]):
"""Delete every Vertex AI resource created during test"""
yield
# Bring all Endpoints to the front of the list
# Ensures Models are undeployed first before we attempt deletion
shared_state["resources"].sort(
key=lambda r: 1 if isinstance(r, aiplatform.Endpoint) else 2
)
for resource in shared_state["resources"]:
try:
if isinstance(resource, (aiplatform.Endpoint, aiplatform.Featurestore)):
# For endpoint, undeploy model then delete endpoint
# For featurestore, force delete its entity_types and features with the featurestore
resource.delete(force=True)
tests/system/aiplatform/e2e_base.py:110:
google/cloud/aiplatform/models.py:1325: in delete
self.undeploy_all(sync=sync)
google/cloud/aiplatform/models.py:1297: in undeploy_all
self._sync_gca_resource()
google/cloud/aiplatform/base.py:601: in _sync_gca_resource
self._gca_resource = self._get_gca_resource(resource_name=self.resource_name)
google/cloud/aiplatform/base.py:612: in resource_name
self._assert_gca_resource_is_available()
google/cloud/aiplatform/models.py:164: in _assert_gca_resource_is_available
super()._assert_gca_resource_is_available()
self = <google.cloud.aiplatform.models.Endpoint object at 0x7f7277285940> failed with Training failed with:
code: 13
message: "Internal error occurred. Please retry in a few minutes. If you still experience errors, contact Vertex AI."
def _assert_gca_resource_is_available(self) -> None:
"""Helper method to raise when accessing properties that do not exist.
Overrides VertexAiResourceNoun to provide a more informative exception if
resource creation has failed asynchronously.
Raises:
RuntimeError: When resource has not been created.
"""
if not getattr(self._gca_resource, "name", None):
raise RuntimeError(
f"{self.__class__.__name__} resource has not been created."
+ (
f" Resource failed with: {self._exception}"
if self._exception
else ""
)
)
E RuntimeError: Endpoint resource has not been created. Resource failed with: Training failed with:
E code: 13
E message: "Internal error occurred. Please retry in a few minutes. If you still experience errors, contact Vertex AI."
google/cloud/aiplatform/base.py:1202: RuntimeError