Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,14 +159,15 @@ Note: Test id should be started from letter "T"
You can use environment variable to control certain features of testomat.io

#### Basic configuration
| Env variable | What it does | Examples |
|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------|
| TESTOMATIO | Provides token for pytestomatio to access and push data to testomat.io. Required for **sync** and **report** commands | TESTOMATIO=tstmt_***** pytest --testomatio sync |
| Env variable | What it does | Examples |
|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------|
| TESTOMATIO | Provides token for pytestomatio to access and push data to testomat.io. Required for **sync** and **report** commands | TESTOMATIO=tstmt_***** pytest --testomatio sync |
| TESTOMATIO_SYNC_LABELS | Assign labels to a test case when you synchronise test from code with testomat.io. Labels must exist in project and their scope must be enabled for tests | TESTOMATIO_SYNC_LABELS="number:1,list:one,standalone" pytest --testomatio report |
| TESTOMATIO_CODE_STYLE | Code parsing style for test synchronization. If you are not sure, don't set this variable. Default value is 'default' | TESTOMATIO_CODE_STYLE=pep8 pytest --testomatio sync |
| TESTOMATIO_CI_DOWNSTREAM | If set, pytestomatio will not set or update build url for a test run. This is useful in scenarios where build url is already set in the test run by Testomat.io for test runs that a created directly on Testomat.io. | TESTOMATIO_CI_DOWNSTREAM=true pytest --testomatio report |
| TESTOMATIO_URL | Customize testomat.io url | TESTOMATIO_URL=https://custom.com/ pytest --testomatio report |
| BUILD_URL | Overrides build url run tests | BUILD_URL=http://custom.com/ pytest --testomatio report |
| TESTOMATIO_CODE_STYLE | Code parsing style for test synchronization. If you are not sure, don't set this variable. Default value is 'default' | TESTOMATIO_CODE_STYLE=pep8 pytest --testomatio sync |
| TESTOMATIO_CI_DOWNSTREAM | If set, pytestomatio will not set or update build url for a test run. This is useful in scenarios where build url is already set in the test run by Testomat.io for test runs that a created directly on Testomat.io. | TESTOMATIO_CI_DOWNSTREAM=true pytest --testomatio report |
| TESTOMATIO_URL | Customize testomat.io url | TESTOMATIO_URL=https://custom.com/ pytest --testomatio report |
| BUILD_URL | Overrides build url run tests | BUILD_URL=http://custom.com/ pytest --testomatio report |
| TESTOMATIO_NO_TIMESTAMP | Disable automatic timestamp generation for test results. Use this option if you run tests in parallel on different machines where time is not synchronized | TESTOMATIO_NO_TIMESTAMP=True pytest --testomatio report |


#### Test Run configuration
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ version_provider = "pep621"
update_changelog_on_bump = false
[project]
name = "pytestomatio"
version = "2.10.1"
version = "2.10.2b8"


dependencies = [
Expand Down
2 changes: 2 additions & 0 deletions pytestomatio/connect/connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ def update_test_status(self, run_id: str,
test_id: str,
message: str,
stack: str,
timestamp: float,
run_time: float,
artifacts: list[str],
steps: str,
Expand All @@ -200,6 +201,7 @@ def update_test_status(self, run_id: str,
"test_id": test_id,
"message": message,
"stack": stack,
"timestamp": timestamp,
"run_time": run_time,
"example": example,
"artifacts": artifacts,
Expand Down
4 changes: 4 additions & 0 deletions pytestomatio/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ def pytest_runtest_makereport(item: Item, call: CallInfo):
'artifacts': test_item.artifacts,
'steps': None,
'code': None,
'timestamp': None
}

# TODO: refactor it and use TestItem setter to upate those attributes
Expand All @@ -178,6 +179,9 @@ def pytest_runtest_makereport(item: Item, call: CallInfo):
if hasattr(item, 'callspec'):
request['example'] = test_item.safe_params(item.callspec.params)

if not pytest.testomatio.test_run_config.disable_timestamp:
request['timestamp'] = time.time()

if item.nodeid not in pytest.testomatio.test_run_config.status_request:
pytest.testomatio.test_run_config.status_request[item.nodeid] = request
else:
Expand Down
2 changes: 2 additions & 0 deletions pytestomatio/testomatio/testRunConfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@ def __init__(self):
run_id = os.environ.get('TESTOMATIO_RUN_ID') or os.environ.get('TESTOMATIO_RUN')
title = os.environ.get('TESTOMATIO_TITLE') if os.environ.get('TESTOMATIO_TITLE') else 'test run at ' + dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
shared_run = os.environ.get('TESTOMATIO_SHARED_RUN') in ['True', 'true', '1']
disable_timestamp = os.environ.get('TESTOMATIO_NO_TIMESTAMP') in ['True', 'true', '1']
self.test_run_id = run_id
self.title = title
self.environment = safe_string_list(os.environ.get('TESTOMATIO_ENV'))
self.disable_timestamp = disable_timestamp
self.label = safe_string_list(os.environ.get('TESTOMATIO_LABEL'))
self.group_title = os.environ.get('TESTOMATIO_RUNGROUP_TITLE')
# This allows to report tests to the test run by it's id. https://docs.testomat.io/getting-started/running-automated-tests/#reporting-parallel-tests
Expand Down
2 changes: 2 additions & 0 deletions tests/test_connect/test_connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ def test_update_test_status_success(self, mock_post, connector):
test_id="test_789",
message=None,
stack=None,
timestamp=123222.21,
run_time=1.5,
artifacts=["screenshot.png"],
steps="Step 1\nStep 2",
Expand All @@ -269,6 +270,7 @@ def test_update_test_status_success(self, mock_post, connector):
assert payload['title'] == 'Test Login'
assert payload['run_time'] == 1.5
assert payload['artifacts'] == ["screenshot.png"]
assert payload['timestamp'] == 123222.21
assert 'message' not in payload

@patch('requests.Session.put')
Expand Down
66 changes: 66 additions & 0 deletions tests/test_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,72 @@ def test_creates_test_request_structure(self, mock_call, single_test_item):
assert request['suite_title'] == item.path.name
assert request['test_id'] == '12345678'

def test_adds_timestamp_to_request(self, mock_call, single_test_item):
"""Test timestamp added to request by default"""
item = single_test_item.copy()[0]
item.config.option.testomatio = 'report'

mock_call.duration = 1.5
mock_call.when = 'call'
mock_call.excinfo = None

pytest.testomatio = Mock()
pytest.testomatio.test_run_config = Mock()
pytest.testomatio.test_run_config.disable_timestamp = False
pytest.testomatio.test_run_config.test_run_id = 'run_123'
pytest.testomatio.test_run_config.status_request = {}

main.pytest_runtest_makereport(item, mock_call)

assert item.nodeid in pytest.testomatio.test_run_config.status_request
request = pytest.testomatio.test_run_config.status_request[item.nodeid]

expected_keys = [
'status', 'title', 'run_time', 'suite_title', 'suite_id', 'timestamp',
'test_id', 'message', 'stack', 'example', 'artifacts', 'steps', 'code'
]
for key in expected_keys:
assert key in request

assert request['title'] == 'Addition'
assert request['run_time'] == 1.5
assert request['suite_title'] == item.path.name
assert request['test_id'] == '12345678'
assert request.get('timestamp') is not None

def test_timestamp_not_updated_when_option_enabled(self, mock_call, single_test_item):
"""Test timestamp is None if TESTOMATIO_NO_TIMESTAMP env set"""
item = single_test_item.copy()[0]
item.config.option.testomatio = 'report'

mock_call.duration = 1.5
mock_call.when = 'call'
mock_call.excinfo = None

pytest.testomatio = Mock()
pytest.testomatio.test_run_config = Mock()
pytest.testomatio.test_run_config.disable_timestamp = True
pytest.testomatio.test_run_config.test_run_id = 'run_123'
pytest.testomatio.test_run_config.status_request = {}

main.pytest_runtest_makereport(item, mock_call)

assert item.nodeid in pytest.testomatio.test_run_config.status_request
request = pytest.testomatio.test_run_config.status_request[item.nodeid]

expected_keys = [
'status', 'title', 'run_time', 'suite_title', 'suite_id',
'test_id', 'message', 'stack', 'example', 'artifacts', 'steps', 'code'
]
for key in expected_keys:
assert key in request

assert request['title'] == 'Addition'
assert request['run_time'] == 1.5
assert request['suite_title'] == item.path.name
assert request['test_id'] == '12345678'
assert request.get('timestamp') is None


@pytest.mark.smoke
class TestPytestUnconfigure:
Expand Down
21 changes: 20 additions & 1 deletion tests/test_testomatio/test_testRunConfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ def test_init_default_values(self):
assert config.test_run_id is None
assert config.title == "test run at 2024-01-15 10:30:45"
assert config.environment is None
assert config.disable_timestamp is False
assert config.label is None
assert config.group_title is None
assert config.parallel is True
Expand All @@ -32,7 +33,8 @@ def test_init_with_env_variables(self):
'TESTOMATIO_TITLE': 'Custom Test Run',
'TESTOMATIO_ENV': 'linux,chrome,1920x1080',
'TESTOMATIO_LABEL': 'smoke,regression',
'TESTOMATIO_RUNGROUP_TITLE': 'Release 2.0'
'TESTOMATIO_RUNGROUP_TITLE': 'Release 2.0',
'TESTOMATIO_NO_TIMESTAMP': '1'
}

with patch.dict(os.environ, env_vars, clear=True):
Expand All @@ -41,6 +43,7 @@ def test_init_with_env_variables(self):
assert config.test_run_id == 'run_12345'
assert config.title == 'Custom Test Run'
assert config.environment == 'linux,chrome,1920x1080'
assert config.disable_timestamp is True
assert config.label == 'smoke,regression'
assert config.group_title == 'Release 2.0'
assert config.parallel is True
Expand All @@ -64,6 +67,22 @@ def test_init_shared_run_false_variations(self, value):
assert config.shared_run is False
assert config.parallel is True

@pytest.mark.parametrize('value', ['True', 'true', '1'])
def test_init_disable_timestamp_true_variations(self, value):
"""Test different true values for TESTOMATIO_NO_TIMESTAMP"""
with patch.dict(os.environ, {'TESTOMATIO_NO_TIMESTAMP': value}, clear=True):
config = TestRunConfig()

assert config.disable_timestamp is True

@pytest.mark.parametrize('value', ['False', 'false', '0', 'anything'])
def test_init_disable_timestamp_false_variations(self, value):
"""Test different false values TESTOMATIO_NO_TIMESTAMP"""
with patch.dict(os.environ, {'TESTOMATIO_NO_TIMESTAMP': value}, clear=True):
config = TestRunConfig()

assert config.disable_timestamp is False

def test_to_dict_full_data(self):
"""Test to_dict with full data"""
env_vars = {
Expand Down
Loading