ci: merge main to release (#8569)

docker/scripts/app-configure-blobstore.py

@@ -2,6 +2,8 @@
 # Copyright The IETF Trust 2024, All Rights Reserved
 
 import boto3
+import botocore.config
+import botocore.exceptions
 import os
 import sys
 
@@ -16,13 +18,19 @@ def init_blobstore():
         aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"),
         aws_session_token=None,
         config=botocore.config.Config(signature_version="s3v4"),
-        verify=False,
     )
     for bucketname in MORE_STORAGE_NAMES:
-        blobstore.create_bucket(
+        try:
-            Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip()
+            blobstore.create_bucket(
-        )
+                Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip()
-
+            )
+        except botocore.exceptions.ClientError as err:
+            if err.response["Error"]["Code"] == "BucketAlreadyExists":
+                print(f"Bucket {bucketname} already exists")
+            else:
+                print(f"Error creating {bucketname}: {err.response['Error']['Code']}")
+        else:
+            print(f"Bucket {bucketname} created")
 
 if __name__ == "__main__":
     sys.exit(init_blobstore())

ietf/doc/storage_utils.py

@@ -8,6 +8,8 @@ from django.conf import settings
 from django.core.files.base import ContentFile, File
 from django.core.files.storage import storages
 
+from ietf.utils.log import log
+
 
 # TODO-BLOBSTORE (Future, maybe after leaving 3.9) : add a return type
 def _get_storage(kind: str):
@@ -22,16 +24,21 @@ def _get_storage(kind: str):
 
 def exists_in_storage(kind: str, name: str) -> bool:
     if settings.ENABLE_BLOBSTORAGE:
-        store = _get_storage(kind)
+        try:
-        return store.exists_in_storage(kind, name)
+            store = _get_storage(kind)
-    else:
+            return store.exists_in_storage(kind, name)
-        return False
+        except Exception as err:
+            log(f"Blobstore Error: Failed to test existence of {kind}:{name}: {repr(err)}")
+    return False
 
 
 def remove_from_storage(kind: str, name: str, warn_if_missing: bool = True) -> None:
     if settings.ENABLE_BLOBSTORAGE:
-        store = _get_storage(kind)
+        try:
-        store.remove_from_storage(kind, name, warn_if_missing)
+            store = _get_storage(kind)
+            store.remove_from_storage(kind, name, warn_if_missing)
+        except Exception as err:
+            log(f"Blobstore Error: Failed to remove {kind}:{name}: {repr(err)}")
     return None
 
 
@@ -46,8 +53,11 @@ def store_file(
 ) -> None:
     # debug.show('f"asked to store {name} into {kind}"')
     if settings.ENABLE_BLOBSTORAGE:
-        store = _get_storage(kind)
+        try:
-        store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev)
+            store = _get_storage(kind)
+            store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev)
+        except Exception as err:
+            log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}")
     return None
 
 
@@ -60,7 +70,11 @@ def store_bytes(
     doc_rev: Optional[str] = None,
 ) -> None:
     if settings.ENABLE_BLOBSTORAGE:
-        store_file(kind, name, ContentFile(content), allow_overwrite)
+        try:
+            store_file(kind, name, ContentFile(content), allow_overwrite)
+        except Exception as err:
+            # n.b., not likely to get an exception here because store_file or store_bytes will catch it
+            log(f"Blobstore Error: Failed to store bytes to {kind}:{name}: {repr(err)}")
     return None
 
 
@@ -73,8 +87,12 @@ def store_str(
     doc_rev: Optional[str] = None,
 ) -> None:
     if settings.ENABLE_BLOBSTORAGE:
-        content_bytes = content.encode("utf-8")
+        try:
-        store_bytes(kind, name, content_bytes, allow_overwrite)
+            content_bytes = content.encode("utf-8")
+            store_bytes(kind, name, content_bytes, allow_overwrite)
+        except Exception as err:
+            # n.b., not likely to get an exception here because store_file or store_bytes will catch it
+            log(f"Blobstore Error: Failed to store string to {kind}:{name}: {repr(err)}")
     return None
 
 
@@ -82,22 +100,28 @@ def retrieve_bytes(kind: str, name: str) -> bytes:
     from ietf.doc.storage_backends import maybe_log_timing
     content = b""
     if settings.ENABLE_BLOBSTORAGE:
-        store = _get_storage(kind)
+        try:
-        with store.open(name) as f:
+            store = _get_storage(kind)
-            with maybe_log_timing(
+            with store.open(name) as f:
-                hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing,
+                with maybe_log_timing(
-                "read",
+                    hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing,
-                bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "",
+                    "read",
-                name=name,
+                    bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "",
-            ):
+                    name=name,
-                content = f.read()
+                ):
+                    content = f.read()
+        except Exception as err:
+            log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}")
     return content
 
 
 def retrieve_str(kind: str, name: str) -> str:
     content = ""
     if settings.ENABLE_BLOBSTORAGE:
-        content_bytes = retrieve_bytes(kind, name)
+        try:
-        # TODO-BLOBSTORE: try to decode all the different ways doc.text() does
+            content_bytes = retrieve_bytes(kind, name)
-        content = content_bytes.decode("utf-8")
+            # TODO-BLOBSTORE: try to decode all the different ways doc.text() does
+            content = content_bytes.decode("utf-8")
+        except Exception as err:
+            log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}")
     return content

ietf/doc/tests_status_change.py

@@ -564,8 +564,9 @@ class StatusChangeSubmitTests(TestCase):
         ftp_filepath = Path(settings.FTP_DIR) / "status-changes" / basename
         self.assertFalse(filepath.exists())
         self.assertFalse(ftp_filepath.exists())
-        with self.assertRaises(FileNotFoundError):
+        # TODO-BLOBSTORE: next assert is disabled because we currently suppress all exceptions 
-            retrieve_str("statchg",basename)
+        # with self.assertRaises(FileNotFoundError):
+        #     retrieve_str("statchg",basename)
         r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1"))
         self.assertEqual(r.status_code,302)
         doc = Document.objects.get(name='status-change-imaginary-mid-review')

ietf/utils/storage.py

@@ -41,13 +41,13 @@ class BlobShadowFileSystemStorage(NoLocationMigrationFileSystemStorage):
         saved_name = super().save(name, content, max_length)
 
         if settings.ENABLE_BLOBSTORAGE:
-            # Retrieve the content and write to the blob store
-            blob_name = Path(saved_name).name  # strips path
             try:
+                # Retrieve the content and write to the blob store
+                blob_name = Path(saved_name).name  # strips path
                 with self.open(saved_name, "rb") as f:
                     store_file(self.kind, blob_name, f, allow_overwrite=True)
             except Exception as err:
-                log(f"Failed to shadow {saved_name} at {self.kind}:{blob_name}: {err}")
+                log(f"Blobstore Error: Failed to shadow {saved_name} at {self.kind}:{blob_name}: {repr(err)}")
         return saved_name  # includes the path!
 
     def deconstruct(self):