|
| 1 | +"""Integration test: register_model() with real model objects → verify generated code executes. |
| 2 | +
|
| 3 | +This is the critical test that verifies the SDK model abstraction works end-to-end: |
| 4 | + 1. Pass a real model object to register_model() |
| 5 | + 2. SDK auto-detects framework, serializes model, generates source code |
| 6 | + 3. The generated source code is POSTed to the API |
| 7 | + 4. We extract that source code and exec() it with MockModelContext |
| 8 | + 5. Verify train(ctx) logs metrics and infer(ctx) produces predictions |
| 9 | +""" |
| 10 | + |
| 11 | +import json |
| 12 | +import pytest |
| 13 | +import responses |
| 14 | + |
| 15 | +from conftest import TEST_API_URL, TEST_PROJECT_ID, MockModelContext |
| 16 | + |
| 17 | + |
| 18 | +REGISTER_RESPONSE = { |
| 19 | + "model_id": "model-roundtrip-001", |
| 20 | + "name": "roundtrip-test", |
| 21 | + "version": 1, |
| 22 | +} |
| 23 | + |
| 24 | + |
| 25 | +class TestRegisterModelSklearnRoundtrip: |
| 26 | + """register_model(model=sklearn_obj) → extract source → exec train/infer.""" |
| 27 | + |
| 28 | + def test_sklearn_register_and_train(self, client, mock_api, sklearn_model): |
| 29 | + """Full pipeline: register sklearn model → exec generated train().""" |
| 30 | + mock_api.add( |
| 31 | + responses.POST, |
| 32 | + f"{TEST_API_URL}/sdk/register-model", |
| 33 | + json=REGISTER_RESPONSE, |
| 34 | + status=200, |
| 35 | + ) |
| 36 | + client.register_model("test-sklearn", model=sklearn_model) |
| 37 | + |
| 38 | + # Extract the source code that was POSTed |
| 39 | + body = json.loads(mock_api.calls[0].request.body) |
| 40 | + source_code = body["source_code"] |
| 41 | + assert body["framework"] == "sklearn" |
| 42 | + |
| 43 | + # Execute the generated train() with MockModelContext |
| 44 | + ctx = MockModelContext(hyperparameters={"n_samples": 50, "n_features": 4}) |
| 45 | + ns = {} |
| 46 | + exec(source_code, ns) |
| 47 | + ns["train"](ctx) |
| 48 | + |
| 49 | + metric_names = [m[0] for m in ctx._logged_metrics] |
| 50 | + assert "accuracy" in metric_names |
| 51 | + assert "loss" in metric_names |
| 52 | + assert "progress" in metric_names |
| 53 | + # Progress should reach 100 |
| 54 | + progress_values = [m[1] for m in ctx._logged_metrics if m[0] == "progress"] |
| 55 | + assert 100 in progress_values |
| 56 | + |
| 57 | + def test_sklearn_register_and_infer(self, client, mock_api, sklearn_model): |
| 58 | + """Full pipeline: register sklearn model → exec generated infer().""" |
| 59 | + mock_api.add( |
| 60 | + responses.POST, |
| 61 | + f"{TEST_API_URL}/sdk/register-model", |
| 62 | + json=REGISTER_RESPONSE, |
| 63 | + status=200, |
| 64 | + ) |
| 65 | + client.register_model("test-sklearn", model=sklearn_model) |
| 66 | + |
| 67 | + body = json.loads(mock_api.calls[0].request.body) |
| 68 | + source_code = body["source_code"] |
| 69 | + |
| 70 | + ctx = MockModelContext( |
| 71 | + hyperparameters={"input_data": {"features": [[1.0, 2.0, 3.0, 4.0]]}} |
| 72 | + ) |
| 73 | + ns = {} |
| 74 | + exec(source_code, ns) |
| 75 | + ns["infer"](ctx) |
| 76 | + |
| 77 | + assert ctx._output is not None |
| 78 | + assert "predictions" in ctx._output |
| 79 | + assert isinstance(ctx._output["predictions"], list) |
| 80 | + |
| 81 | + |
| 82 | +class TestRegisterModelPytorchRoundtrip: |
| 83 | + """register_model(model=pytorch_obj) → extract source → exec train/infer.""" |
| 84 | + |
| 85 | + pytestmark = pytest.mark.skipif( |
| 86 | + not pytest.importorskip("torch", reason="torch not installed"), |
| 87 | + reason="torch not installed", |
| 88 | + ) |
| 89 | + |
| 90 | + def test_pytorch_register_and_train(self, client, mock_api, pytorch_model): |
| 91 | + """Full pipeline: register PyTorch model → exec generated train().""" |
| 92 | + mock_api.add( |
| 93 | + responses.POST, |
| 94 | + f"{TEST_API_URL}/sdk/register-model", |
| 95 | + json=REGISTER_RESPONSE, |
| 96 | + status=200, |
| 97 | + ) |
| 98 | + client.register_model("test-pytorch", model=pytorch_model) |
| 99 | + |
| 100 | + body = json.loads(mock_api.calls[0].request.body) |
| 101 | + source_code = body["source_code"] |
| 102 | + assert body["framework"] == "pytorch" |
| 103 | + |
| 104 | + ctx = MockModelContext(hyperparameters={"epochs": 2, "batch_size": 4}) |
| 105 | + ns = {} |
| 106 | + exec(source_code, ns) |
| 107 | + ns["train"](ctx) |
| 108 | + |
| 109 | + metric_names = [m[0] for m in ctx._logged_metrics] |
| 110 | + assert "loss" in metric_names |
| 111 | + assert "accuracy" in metric_names |
| 112 | + assert "progress" in metric_names |
| 113 | + |
| 114 | + def test_pytorch_register_and_infer(self, client, mock_api, pytorch_model): |
| 115 | + """Full pipeline: register PyTorch model → exec generated infer().""" |
| 116 | + mock_api.add( |
| 117 | + responses.POST, |
| 118 | + f"{TEST_API_URL}/sdk/register-model", |
| 119 | + json=REGISTER_RESPONSE, |
| 120 | + status=200, |
| 121 | + ) |
| 122 | + client.register_model("test-pytorch", model=pytorch_model) |
| 123 | + |
| 124 | + body = json.loads(mock_api.calls[0].request.body) |
| 125 | + source_code = body["source_code"] |
| 126 | + |
| 127 | + # SimpleNet has input_size=4 |
| 128 | + ctx = MockModelContext( |
| 129 | + hyperparameters={"input_data": {"features": [[1.0, 2.0, 3.0, 4.0]]}} |
| 130 | + ) |
| 131 | + ns = {} |
| 132 | + exec(source_code, ns) |
| 133 | + ns["infer"](ctx) |
| 134 | + |
| 135 | + assert ctx._output is not None |
| 136 | + assert "predictions" in ctx._output |
| 137 | + preds = ctx._output["predictions"] |
| 138 | + assert isinstance(preds, list) |
| 139 | + assert len(preds) == 1 # one sample |
| 140 | + |
| 141 | + |
| 142 | +class TestRegisterModelTensorflowRoundtrip: |
| 143 | + """register_model(model=keras_obj) → extract source → exec train/infer.""" |
| 144 | + |
| 145 | + def test_tensorflow_register_and_train(self, client, mock_api, tf_model): |
| 146 | + """Full pipeline: register TF/Keras model → exec generated train().""" |
| 147 | + mock_api.add( |
| 148 | + responses.POST, |
| 149 | + f"{TEST_API_URL}/sdk/register-model", |
| 150 | + json=REGISTER_RESPONSE, |
| 151 | + status=200, |
| 152 | + ) |
| 153 | + client.register_model("test-keras", model=tf_model) |
| 154 | + |
| 155 | + body = json.loads(mock_api.calls[0].request.body) |
| 156 | + source_code = body["source_code"] |
| 157 | + assert body["framework"] == "tensorflow" |
| 158 | + |
| 159 | + ctx = MockModelContext(hyperparameters={"epochs": 1, "n_samples": 20, "batch_size": 8}) |
| 160 | + ns = {} |
| 161 | + exec(source_code, ns) |
| 162 | + ns["train"](ctx) |
| 163 | + |
| 164 | + metric_names = [m[0] for m in ctx._logged_metrics] |
| 165 | + assert "loss" in metric_names |
| 166 | + assert "progress" in metric_names |
| 167 | + |
| 168 | + def test_tensorflow_register_and_infer(self, client, mock_api, tf_model): |
| 169 | + """Full pipeline: register TF/Keras model → exec generated infer().""" |
| 170 | + mock_api.add( |
| 171 | + responses.POST, |
| 172 | + f"{TEST_API_URL}/sdk/register-model", |
| 173 | + json=REGISTER_RESPONSE, |
| 174 | + status=200, |
| 175 | + ) |
| 176 | + client.register_model("test-keras", model=tf_model) |
| 177 | + |
| 178 | + body = json.loads(mock_api.calls[0].request.body) |
| 179 | + source_code = body["source_code"] |
| 180 | + |
| 181 | + # tf_model has input_shape=(4,) |
| 182 | + ctx = MockModelContext( |
| 183 | + hyperparameters={"input_data": {"features": [[1.0, 2.0, 3.0, 4.0]]}} |
| 184 | + ) |
| 185 | + ns = {} |
| 186 | + exec(source_code, ns) |
| 187 | + ns["infer"](ctx) |
| 188 | + |
| 189 | + assert ctx._output is not None |
| 190 | + assert "predictions" in ctx._output |
0 commit comments