test: add more

Author: Harry Le

engine/e2e-test/api/engines/test_api_get_default_engine.py

@@ -33,6 +33,7 @@ def test_api_get_default_engine_successfully(self):
             post_install_url, json=data
         )
         assert_equal(response.status_code,200)
+        log_response(response.json(), "test_api_get_default_engine_successfully")
 
         get_list_url = f"http://localhost:3928/v1/engines/{engine}"
         get_default_url = f"http://localhost:3928/v1/engines/{engine}/default"
@@ -66,11 +67,7 @@ def get_request(url):
         # Validate response schema
         jsonschema.validate(instance=json_data, schema=schema)
 
-        assert_equal(json_data["engine"], engine)
-        assert_equal(json_data["version"], version)
-        assert_equal(json_data["variant"], name)
-        
-    def test_api_get_default_engine_successfully(self):
+    def test_api_get_default_engine_failed_invalid_engine(self):
         # Data test
         engine= "invalid"
 
@@ -79,7 +76,7 @@ def test_api_get_default_engine_successfully(self):
         response_default_engine = requests.get(get_default_url)
         json_data_get_default = response_default_engine.json()
 
-        log_response(json_data_get_default, "test_api_get_default_engine_successfully")
+        log_response(json_data_get_default, "test_api_get_default_engine_failed_invalid_engine")
         assert_equal(response_default_engine.status_code, 400)
 
         assert_equal(json_data_get_default["message"], f"Engine {engine} is not supported yet!")

engine/e2e-test/api/engines/test_api_get_engine_release.py

@@ -0,0 +1,76 @@
+import pytest
+import requests
+from utils.test_runner import start_server, stop_server
+import jsonschema
+from tenacity import retry, wait_exponential, stop_after_attempt
+from utils.logger import log_response
+from utils.assertion import assert_equal, assert_contains
+
+
+class TestApiEngineRelease:
+
+    @pytest.fixture(autouse=True)
+    def setup_and_teardown(self):
+        # Setup
+        success = start_server()
+        if not success:
+            raise Exception("Failed to start server")
+
+        yield
+
+        # Teardown
+        stop_server()
+
+    def test_api_get_engine_release_successfully(self):
+        # Data test
+        engine= "llama-cpp"
+        get_release_url = f"http://localhost:3928/v1/engines/{engine}/releases"
+
+        @retry(
+            wait=wait_exponential(multiplier=2, min=2, max=30), 
+            stop=stop_after_attempt(5) 
+        )
+        def get_request(url):
+            response = requests.get(url)
+            assert len(response.json()) > 0
+
+        get_request(get_release_url)
+        
+        response_engine_release = requests.get(get_release_url)
+        json_data = response_engine_release.json()
+
+        log_response(json_data, "test_api_get_engine_release_successfully")
+        assert_equal(response_engine_release.status_code, 200)
+
+        schema = {
+            "$schema": "http://json-schema.org/draft-07/schema#",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                "draft": { "type": "boolean" },
+                "name": { "type": "string" },
+                "prerelease": { "type": "boolean" },
+                "published_at": { "type": "string", "format": "date-time" },
+                "url": { "type": "string", "format": "uri" }
+                },
+                "required": ["draft", "name", "prerelease", "published_at", "url"]
+            }
+        }
+
+        # Validate response schema
+        jsonschema.validate(instance=json_data, schema=schema)
+        
+    def test_api_ge_engine_release_failed_invalid_engine(self):
+        # Data test
+        engine= "invalid"
+    
+        get_default_url = f"http://localhost:3928/v1/engines/{engine}/releases"
+
+        response_default_engine = requests.get(get_default_url)
+        json_data_get_default = response_default_engine.json()
+
+        log_response(json_data_get_default, "test_api_ge_engine_release_failed_invalid_engine")
+        assert_equal(response_default_engine.status_code, 400)
+
+        assert_contains(json_data_get_default["message"], "Not Found")

engine/e2e-test/api/engines/test_api_get_engine_release_latest.py

@@ -0,0 +1,73 @@
+import pytest
+import requests
+from utils.test_runner import start_server, stop_server
+import jsonschema
+from tenacity import retry, wait_exponential, stop_after_attempt
+from utils.logger import log_response
+from utils.assertion import assert_equal, assert_contains
+
+
+class TestApiEngineReleaseLatest:
+
+    @pytest.fixture(autouse=True)
+    def setup_and_teardown(self):
+        # Setup
+        success = start_server()
+        if not success:
+            raise Exception("Failed to start server")
+
+        yield
+
+        # Teardown
+        stop_server()
+
+    def test_api_get_engine_release_latest_successfully(self):
+        # Data test
+        engine= "llama-cpp"
+        get_release_url = f"http://localhost:3928/v1/engines/{engine}/releases/latest"
+
+        @retry(
+            wait=wait_exponential(multiplier=2, min=2, max=30), 
+            stop=stop_after_attempt(5) 
+        )
+        def get_request(url):
+            response = requests.get(url)
+            assert len(response.json()) > 0
+
+        get_request(get_release_url)
+        
+        response_engine_release = requests.get(get_release_url)
+        json_data = response_engine_release.json()
+
+        log_response(json_data, "test_api_get_engine_release_latest_successfully")
+        assert_equal(response_engine_release.status_code, 200)
+
+        schema = {
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                "created_at": {
+                    "type": "string",
+                    "format": "date-time"
+                },
+                "download_count": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "name": {
+                    "type": "string"
+                },
+                "size": {
+                    "type": "integer",
+                    "minimum": 0
+                }
+                },
+                "required": ["created_at", "download_count", "name", "size"]
+            }
+        }
+
+
+        # Validate response schema
+        jsonschema.validate(instance=json_data, schema=schema)

engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py

@@ -18,6 +18,9 @@
 ### e2e tests are expensive, have to keep engines tests in order
 from api.engines.test_api_get_list_engine import TestApiEngineList
 from api.engines.test_api_engine_install_nightly import TestApiEngineInstall
+from api.engines.test_api_get_default_engine import TestApiDefaultEngine
+from api.engines.test_api_get_engine_release import TestApiEngineRelease
+from api.engines.test_api_get_engine_release_latest import TestApiEngineReleaseLatest
 from api.model.test_api_model import TestApiModel
 from api.model.test_api_model_import import TestApiModelImport
 

engine/e2e-test/runner/main.py

@@ -18,6 +18,9 @@
 ### e2e tests are expensive, have to keep engines tests in order
 from api.engines.test_api_get_list_engine import TestApiEngineList
 from api.engines.test_api_engine import TestApiEngine
+from api.engines.test_api_get_default_engine import TestApiDefaultEngine
+from api.engines.test_api_get_engine_release import TestApiEngineRelease
+from api.engines.test_api_get_engine_release_latest import TestApiEngineReleaseLatest
 from api.model.test_api_model import TestApiModel
 from api.model.test_api_model_import import TestApiModelImport
 

engine/e2e-test/utils/assertion.py

@@ -1,3 +1,7 @@
 def assert_equal(actual, expected):
     """Custom assertion to compare actual and expected values."""
-    assert actual == expected, f"Assertion failed: Expected {expected}, but got {actual}"
+    assert actual == expected, f"Assertion failed: Expected '{expected}', but got '{actual}'"
+    
+def assert_contains(main_string, sub_string):
+    """Custom assertion to compare actual and expected values."""
+    assert sub_string in main_string, f"Assertion failed: Expected '{main_string}' has '{sub_string}'"