diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index 7b5ce962e..ae87bf1af 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -2,6 +2,8 @@ # Copyright The IETF Trust 2024, All Rights Reserved import boto3 +import botocore.config +import botocore.exceptions import os import sys @@ -16,13 +18,19 @@ def init_blobstore(): aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"), aws_session_token=None, config=botocore.config.Config(signature_version="s3v4"), - verify=False, ) for bucketname in MORE_STORAGE_NAMES: - blobstore.create_bucket( - Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip() - ) - + try: + blobstore.create_bucket( + Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip() + ) + except botocore.exceptions.ClientError as err: + if err.response["Error"]["Code"] == "BucketAlreadyExists": + print(f"Bucket {bucketname} already exists") + else: + print(f"Error creating {bucketname}: {err.response['Error']['Code']}") + else: + print(f"Bucket {bucketname} created") if __name__ == "__main__": sys.exit(init_blobstore()) diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 4f0516339..012efc907 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -8,6 +8,8 @@ from django.conf import settings from django.core.files.base import ContentFile, File from django.core.files.storage import storages +from ietf.utils.log import log + # TODO-BLOBSTORE (Future, maybe after leaving 3.9) : add a return type def _get_storage(kind: str): @@ -22,16 +24,21 @@ def _get_storage(kind: str): def exists_in_storage(kind: str, name: str) -> bool: if settings.ENABLE_BLOBSTORAGE: - store = _get_storage(kind) - return store.exists_in_storage(kind, name) - else: - return False + try: + store = _get_storage(kind) + return store.exists_in_storage(kind, name) + except Exception as err: + log(f"Blobstore Error: Failed to test existence of {kind}:{name}: {repr(err)}") + return False def remove_from_storage(kind: str, name: str, warn_if_missing: bool = True) -> None: if settings.ENABLE_BLOBSTORAGE: - store = _get_storage(kind) - store.remove_from_storage(kind, name, warn_if_missing) + try: + store = _get_storage(kind) + store.remove_from_storage(kind, name, warn_if_missing) + except Exception as err: + log(f"Blobstore Error: Failed to remove {kind}:{name}: {repr(err)}") return None @@ -46,8 +53,11 @@ def store_file( ) -> None: # debug.show('f"asked to store {name} into {kind}"') if settings.ENABLE_BLOBSTORAGE: - store = _get_storage(kind) - store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev) + try: + store = _get_storage(kind) + store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev) + except Exception as err: + log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}") return None @@ -60,7 +70,11 @@ def store_bytes( doc_rev: Optional[str] = None, ) -> None: if settings.ENABLE_BLOBSTORAGE: - store_file(kind, name, ContentFile(content), allow_overwrite) + try: + store_file(kind, name, ContentFile(content), allow_overwrite) + except Exception as err: + # n.b., not likely to get an exception here because store_file or store_bytes will catch it + log(f"Blobstore Error: Failed to store bytes to {kind}:{name}: {repr(err)}") return None @@ -73,8 +87,12 @@ def store_str( doc_rev: Optional[str] = None, ) -> None: if settings.ENABLE_BLOBSTORAGE: - content_bytes = content.encode("utf-8") - store_bytes(kind, name, content_bytes, allow_overwrite) + try: + content_bytes = content.encode("utf-8") + store_bytes(kind, name, content_bytes, allow_overwrite) + except Exception as err: + # n.b., not likely to get an exception here because store_file or store_bytes will catch it + log(f"Blobstore Error: Failed to store string to {kind}:{name}: {repr(err)}") return None @@ -82,22 +100,28 @@ def retrieve_bytes(kind: str, name: str) -> bytes: from ietf.doc.storage_backends import maybe_log_timing content = b"" if settings.ENABLE_BLOBSTORAGE: - store = _get_storage(kind) - with store.open(name) as f: - with maybe_log_timing( - hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, - "read", - bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", - name=name, - ): - content = f.read() + try: + store = _get_storage(kind) + with store.open(name) as f: + with maybe_log_timing( + hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, + "read", + bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", + name=name, + ): + content = f.read() + except Exception as err: + log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") return content def retrieve_str(kind: str, name: str) -> str: content = "" if settings.ENABLE_BLOBSTORAGE: - content_bytes = retrieve_bytes(kind, name) - # TODO-BLOBSTORE: try to decode all the different ways doc.text() does - content = content_bytes.decode("utf-8") + try: + content_bytes = retrieve_bytes(kind, name) + # TODO-BLOBSTORE: try to decode all the different ways doc.text() does + content = content_bytes.decode("utf-8") + except Exception as err: + log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") return content diff --git a/ietf/doc/tests_status_change.py b/ietf/doc/tests_status_change.py index cbdc1a049..da1a4f190 100644 --- a/ietf/doc/tests_status_change.py +++ b/ietf/doc/tests_status_change.py @@ -564,8 +564,9 @@ class StatusChangeSubmitTests(TestCase): ftp_filepath = Path(settings.FTP_DIR) / "status-changes" / basename self.assertFalse(filepath.exists()) self.assertFalse(ftp_filepath.exists()) - with self.assertRaises(FileNotFoundError): - retrieve_str("statchg",basename) + # TODO-BLOBSTORE: next assert is disabled because we currently suppress all exceptions + # with self.assertRaises(FileNotFoundError): + # retrieve_str("statchg",basename) r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1")) self.assertEqual(r.status_code,302) doc = Document.objects.get(name='status-change-imaginary-mid-review') diff --git a/ietf/utils/storage.py b/ietf/utils/storage.py index 9f41f3d50..42fcf884a 100644 --- a/ietf/utils/storage.py +++ b/ietf/utils/storage.py @@ -41,13 +41,13 @@ class BlobShadowFileSystemStorage(NoLocationMigrationFileSystemStorage): saved_name = super().save(name, content, max_length) if settings.ENABLE_BLOBSTORAGE: - # Retrieve the content and write to the blob store - blob_name = Path(saved_name).name # strips path try: + # Retrieve the content and write to the blob store + blob_name = Path(saved_name).name # strips path with self.open(saved_name, "rb") as f: store_file(self.kind, blob_name, f, allow_overwrite=True) except Exception as err: - log(f"Failed to shadow {saved_name} at {self.kind}:{blob_name}: {err}") + log(f"Blobstore Error: Failed to shadow {saved_name} at {self.kind}:{blob_name}: {repr(err)}") return saved_name # includes the path! def deconstruct(self):