🚧 working on getting the tests to work within the CI

Author: m-misiura

.github/workflows/python-tests.yaml

@@ -2,7 +2,7 @@ name: Python Tests
 
 on:
   push:
-    branches: [main]
+    branches: [main, download_and_upload_endpoints]
   pull_request:
     branches: [main]
 

tests/endpoints/test_upload_endpoint.py

@@ -6,6 +6,7 @@
 import sys
 import tempfile
 import uuid
+import asyncio
 
 import h5py
 import numpy as np
@@ -24,7 +25,13 @@
 )
 from src.service.data.storage import get_storage_interface
 
-
+@pytest.fixture(autouse=True)
+def reset_storage():
+    """Reset storage interface for each test."""
+    import src.service.data.storage
+    src.service.data.storage._storage_interface = None
+    yield
+    
 def pytest_sessionfinish(session, exitstatus):
     """Clean up the temporary directory after all tests are done."""
     if os.path.exists(TEMP_DIR):
@@ -129,46 +136,43 @@ def generate_mismatched_shape_no_unique_name_multi_input_payload(n_rows, n_input
 
 def get_data_from_storage(model_name, suffix):
     """Get data from storage file."""
-    storage = get_storage_interface()
-    filename = storage._get_filename(model_name + suffix)
-    if not os.path.exists(filename):
-        return None
-    with h5py.File(filename, "r") as f:
-        if model_name + suffix in f:
-            data = f[model_name + suffix][:]
-            column_names = f[model_name + suffix].attrs.get("column_names", [])
+    async def _get_data():
+        storage = get_storage_interface()
+        try:
+            data, column_names = await storage.read_data(model_name + suffix)
             return {"data": data, "column_names": column_names}
+        except Exception as e:
+            print(f"Error reading {model_name + suffix}: {e}")
+            return None
+    
+    try:
+        return asyncio.run(_get_data())
+    except Exception as e:
+        print(f"Async error for {model_name + suffix}: {e}")
+        return None
 
 
 def get_metadata_ids(model_name):
     """Extract actual IDs from metadata storage."""
-    storage = get_storage_interface()
-    filename = storage._get_filename(model_name + METADATA_SUFFIX)
-    if not os.path.exists(filename):
-        return []
-    ids = []
-    with h5py.File(filename, "r") as f:
-        if model_name + METADATA_SUFFIX in f:
-            metadata = f[model_name + METADATA_SUFFIX][:]
-            column_names = f[model_name + METADATA_SUFFIX].attrs.get("column_names", [])
+    async def _get_ids():
+        storage = get_storage_interface()
+        try:
+            metadata, column_names = await storage.read_data(model_name + METADATA_SUFFIX)
             id_idx = next((i for i, name in enumerate(column_names) if name.lower() == "id"), None)
-            if id_idx is not None:
+            if id_idx is not None and metadata is not None:
+                ids = []
                 for row in metadata:
-                    try:
-                        if hasattr(row, "__getitem__") and len(row) > id_idx:
-                            id_val = row[id_idx]
-                        else:
-                            row_data = pickle.loads(row.tobytes())
-                            id_val = row_data[id_idx]
-                        if isinstance(id_val, np.ndarray):
-                            ids.append(str(id_val))
-                        else:
-                            ids.append(str(id_val))
-                    except Exception as e:
-                        print(f"Error processing ID from row {len(ids)}: {e}")
-                        continue
-    print(f"Successfully extracted {len(ids)} IDs: {ids}")
-    return ids
+                    if hasattr(row, "__len__") and len(row) > id_idx:
+                        ids.append(str(row[id_idx]))
+                return ids
+        except Exception as e:
+            print(f"Error getting metadata: {e}")
+        return []
+    
+    try:
+        return asyncio.run(_get_ids())
+    except Exception:
+        return []
 
 
 def get_metadata_from_storage(model_name):