back to sync of swift

load_testing/test_files.py

@@ -1,5 +1,5 @@
 from locust import HttpUser,task,between
-from locust.stats import global_stats
+
 import random
 manifest_ids = ["69429/g05t3fx73x7t"]
 class AppUser(HttpUser):
@@ -9,7 +9,7 @@ def on_start(self) -> None:
         """
         Clear statistics at the start of the test.
         """
-        global_stats.reset_all()
+
 
     @task
     def get_files(self):

swift_config/swift_config.py

@@ -0,0 +1,31 @@
+import os
+from dotenv import load_dotenv
+import swiftclient
+import logging
+from swiftclient.exceptions import ClientException
+from fastapi import HTTPException
+
+# load .env file
+load_dotenv()
+# Swift configuration
+SWIFT_AUTH_URL = os.getenv("SWIFT_AUTH_URL")
+SWIFT_USER = os.getenv("SWIFT_USER")
+SWIFT_KEY = os.getenv("SWIFT_KEY")
+SWIFT_PREAUTH_URL = os.getenv("SWIFT_PREAUTH_URL")
+CONTAINER_NAME = os.getenv("CONTAINER_NAME ")
+
+#config logger
+logging.basicConfig(level=logging.INFO,handlers=[logging.StreamHandler()])
+logger = logging.getLogger(__name__)
+
+def get_swift_connection():
+    try:
+        return swiftclient.Connection(
+            user=SWIFT_USER,
+            key=SWIFT_KEY,
+            authurl=SWIFT_AUTH_URL,
+            preauthurl=SWIFT_PREAUTH_URL
+    )
+    except (ClientException) as conn_error:
+        logger.error(f"Failed to connect to Swift: {conn_error}")
+        raise HTTPException(status_code=500, detail=f"Failed to connect to Swift: {conn_error}")

utils/get_manifest_conn.py

@@ -6,6 +6,7 @@
 import pickle
 from dotenv import load_dotenv
 from fastapi.responses import JSONResponse
+from swift_config.swift_config import get_swift_connection
 
 # Load .env file
 load_dotenv()
@@ -15,18 +16,21 @@
 logging.basicConfig(level=logging.INFO,handlers=[logging.StreamHandler()])
 logger = logging.getLogger(__name__)
 
+# Connect to Swift
+conn = get_swift_connection()
 
 async def get_manifest_conn(slug:str,request: Request):
     """
     Retrieve file content from the specified Swift container by filename, and format JSON data.
-    """
+    
     #Access swift_session, swift_token, and swift_storage_url,redis from app state
     swift_session = request.app.state.swift_session
     swift_token = request.app.state.swift_token
     swift_storage_url = request.app.state.swift_storage_url
     
     if not swift_token or not swift_storage_url:
         raise HTTPException(status_code=401, detail="Authentication is not complete or failed.")
+    """
     try:
         #Access Swift and Redis objects from the app's state
         redis = request.app.state.redis
@@ -35,20 +39,23 @@ async def get_manifest_conn(slug:str,request: Request):
         #Check Redis cache,if exists return from redis
         if (cached_profile := await redis.get(f"manifest_{slug}")) is not None:
             return pickle.loads(cached_profile)
-
+        """
         # retrieve from the container
         file_url = f"{swift_storage_url}/{container_name}/{manifest_name}"
         headers = {
             "X-Auth-Token": swift_token
         }
+        
         async with swift_session.get(file_url,headers=headers,ssl=False) as resp:
             if resp.status == 200:    
                 manifest = await resp.read()
-                manifest_data = json.loads(manifest)
+                manifest_data = json.loads(manifest)     
+        """
+        _,manifest = conn.get_object(container_name, manifest_name)
+        manifest_data = json.loads(manifest)
         # Cache the manifest in Redis 
         logger.info(f"Caching manifest_{slug} in Redis.")
         await redis.set(f"manifest_{slug}", pickle.dumps(manifest_data))
-
         return JSONResponse(content=manifest_data, status_code=200)
 
     except Exception as e:

utils/lifespan_handler.py

@@ -34,18 +34,18 @@ async def lifespan(app) -> AsyncGenerator[None,None]:
     """
     global swift_token,swift_storage_url,swift_session
     swift_session = aiohttp.ClientSession()
-
+    
     try:
         #load OPENID config
         await initialize_openid_config()
-
+        """
         #swift authentication
         swift_token, swift_storage_url = await initialize_swift()
         # Store swift_session, swift_token, and swift_storage_url in app state for later use
         app.state.swift_session = swift_session
         app.state.swift_token = swift_token
         app.state.swift_storage_url = swift_storage_url
-
+        """
         # Initialize Redis connection
         app.state.redis = aioredis.from_url(
             redis_url,
@@ -102,15 +102,15 @@ async def  initialize_swift():
 async def close_session(app):
     """
     Close the aiohttp session and Redis connection when the application shuts down.
-    """ 
+    
     global swift_session
     try:
         if swift_session:
             await swift_session.close()
             logger.info("Closed aiohttp session.")
     except Exception as e:
         logger.error(f"Error closing aiohttp session: {e}")
-
+    """
     try:
         if hasattr(app.state, 'redis') and app.state.redis:
             await app.state.redis.close()

utils/upload_manifest.py

@@ -10,14 +10,18 @@
 import logging
 from urllib.parse import urlparse
 import botocore
+from swift_config.swift_config import get_swift_connection
 
 # Load .env file
 load_dotenv()
 container_name = os.getenv("CONTAINER_NAME")
+
 #config logger
 logging.basicConfig(level=logging.INFO,handlers=[logging.StreamHandler()])
 logger = logging.getLogger(__name__)
 
+# Connect to Swift
+conn = get_swift_connection()
 
 # Create an async context manager for acquiring and releasing a Redis lock
 @asynccontextmanager
@@ -36,18 +40,18 @@ async def upload_manifest_backend(
     db: AsyncSession
 ):
     # Access swift_session, swift_token, and swift_storage_url from app state
-
+    """
     swift_session = request.app.state.swift_session
     swift_token = request.app.state.swift_token
     swift_storage_url = request.app.state.swift_storage_url
-
+    """
     redis = request.app.state.redis
 
 
-
+    """
     if not swift_token:
         raise HTTPException(status_code=401, detail="Swift authentication token not found.")
-
+    """
     try:
         try:
             content = await file.read()
@@ -126,9 +130,17 @@ async def upload_manifest_backend(
                 canvas_item['items'][0]['items'][0]['id'] = annotation_id
                 new_manifest_items.append(canvas_item)
                 #updated_canvas_content = json.dumps(canvas_item)
+
             # Upload manifest to Swift
             manifest['items'] = new_manifest_items
             updated_manifest = json.dumps(manifest)
+            conn.put_object(
+                container_name,
+                manifest_name,
+                contents=updated_manifest,
+                content_type='application/json'
+            )
+            """
             upload_url = f"{swift_storage_url}/{container_name}/{manifest_name}"
             headers = {
             "X-Auth-Token": swift_token,
@@ -139,7 +151,7 @@ async def upload_manifest_backend(
                     text = await resp.text() 
                     logger.info(f"File upload failed: {text}")       
                     raise HTTPException(status_code=resp.status, detail=f"File upload failed")         
-
+            """
             # Check cache and delete if it exists
             redis_key = f"manifest_{slug}"
             if (await redis.get(redis_key)) is not None: