altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

fix: ignore exceptions from blobstore ops (#8565)

Browse source 
* fix: ignore exceptions from to blobstore ops

* fix: log repr(err) instead of just err
This commit is contained in:
Jennifer Richards 2025-02-20 13:12:43 -04:00 committed by GitHub
parent dda9e4e32e
commit 2f8b9c3cfa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 49 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
ietf/doc/storage_utils.py
View file

@ -8,6 +8,8 @@ from django.conf import settings
from django.core.files.base import ContentFile, File
from django.core.files.storage import storages
from ietf.utils.log import log
# TODO-BLOBSTORE (Future, maybe after leaving 3.9) : add a return type
def _get_storage(kind: str):
@ -22,16 +24,22 @@ def _get_storage(kind: str):
def exists_in_storage(kind: str, name: str) -> bool:
if settings.ENABLE_BLOBSTORAGE:
store = _get_storage(kind)
return store.exists_in_storage(kind, name)
try:
store = _get_storage(kind)
return store.exists_in_storage(kind, name)
except Exception as err:
log(f"Blobstore Error: Failed to test existence of {kind}:{name}: {repr(err)}")
else:
return False
def remove_from_storage(kind: str, name: str, warn_if_missing: bool = True) -> None:
if settings.ENABLE_BLOBSTORAGE:
store = _get_storage(kind)
store.remove_from_storage(kind, name, warn_if_missing)
try:
store = _get_storage(kind)
store.remove_from_storage(kind, name, warn_if_missing)
except Exception as err:
log(f"Blobstore Error: Failed to remove {kind}:{name}: {repr(err)}")
return None
@ -46,8 +54,11 @@ def store_file(
) -> None:
# debug.show('f"asked to store {name} into {kind}"')
if settings.ENABLE_BLOBSTORAGE:
store = _get_storage(kind)
store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev)
try:
store = _get_storage(kind)
store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev)
except Exception as err:
log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}")
return None
@ -60,7 +71,11 @@ def store_bytes(
doc_rev: Optional[str] = None,
) -> None:
if settings.ENABLE_BLOBSTORAGE:
store_file(kind, name, ContentFile(content), allow_overwrite)
try:
store_file(kind, name, ContentFile(content), allow_overwrite)
except Exception as err:
# n.b., not likely to get an exception here because store_file or store_bytes will catch it
log(f"Blobstore Error: Failed to store bytes to {kind}:{name}: {repr(err)}")
return None
@ -73,8 +88,12 @@ def store_str(
doc_rev: Optional[str] = None,
) -> None:
if settings.ENABLE_BLOBSTORAGE:
content_bytes = content.encode("utf-8")
store_bytes(kind, name, content_bytes, allow_overwrite)
try:
content_bytes = content.encode("utf-8")
store_bytes(kind, name, content_bytes, allow_overwrite)
except Exception as err:
# n.b., not likely to get an exception here because store_file or store_bytes will catch it
log(f"Blobstore Error: Failed to store string to {kind}:{name}: {repr(err)}")
return None
@ -82,22 +101,28 @@ def retrieve_bytes(kind: str, name: str) -> bytes:
from ietf.doc.storage_backends import maybe_log_timing
content = b""
if settings.ENABLE_BLOBSTORAGE:
store = _get_storage(kind)
with store.open(name) as f:
with maybe_log_timing(
hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing,
"read",
bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "",
name=name,
):
content = f.read()
try:
store = _get_storage(kind)
with store.open(name) as f:
with maybe_log_timing(
hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing,
"read",
bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "",
name=name,
):
content = f.read()
except Exception as err:
log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}")
return content
def retrieve_str(kind: str, name: str) -> str:
content = ""
if settings.ENABLE_BLOBSTORAGE:
content_bytes = retrieve_bytes(kind, name)
# TODO-BLOBSTORE: try to decode all the different ways doc.text() does
content = content_bytes.decode("utf-8")
try:
content_bytes = retrieve_bytes(kind, name)
# TODO-BLOBSTORE: try to decode all the different ways doc.text() does
content = content_bytes.decode("utf-8")
except Exception as err:
log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}")
return content

6
ietf/utils/storage.py
View file

@ -41,13 +41,13 @@ class BlobShadowFileSystemStorage(NoLocationMigrationFileSystemStorage):
saved_name = super().save(name, content, max_length)
if settings.ENABLE_BLOBSTORAGE:
# Retrieve the content and write to the blob store
blob_name = Path(saved_name).name # strips path
try:
# Retrieve the content and write to the blob store
blob_name = Path(saved_name).name # strips path
with self.open(saved_name, "rb") as f:
store_file(self.kind, blob_name, f, allow_overwrite=True)
except Exception as err:
log(f"Failed to shadow {saved_name} at {self.kind}:{blob_name}: {err}")
log(f"Blobstore Error: Failed to shadow {saved_name} at {self.kind}:{blob_name}: {repr(err)}")
return saved_name # includes the path!
def deconstruct(self):