Skip to content

Update for SQLAlchemy 2.0+ and RobotFramework 4.0+ #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
laurensmiers wants to merge 10 commits into
base: master
Choose a base branch
from
Open

Update for SQLAlchemy 2.0+ and RobotFramework 4.0+ #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7d78c3c
fix(robot_results_parser): datetime deprecation notice
laurensmiers Mar 30, 2025
d2a5f6d
fix(database_write): Store elapsed time as datetime.timedelta
laurensmiers Mar 30, 2025
3d3bfea
fix(robot_results_parser): Convert Path-object to string
laurensmiers Mar 30, 2025
907adaf
fix(robot_results_parser): store timestamp directly
laurensmiers Mar 30, 2025
464a25f
refactor(robot_results_parser): Replace critical with skipped tests
laurensmiers Mar 30, 2025
924a841
refactor(robot_results_parser): Update suite passed/failed statistics…
laurensmiers Mar 30, 2025
46d25a6
fix(robot_results_parser): Don't parse non-keywords like Message, ...
laurensmiers Mar 30, 2025
6d0dc2e
fix(robot_results_parser): Update to sqlalchemy v2 API
laurensmiers Mar 30, 2025
ba0ffe3
fix(robot_results_parser): Update to 'keywords' RF v4 API
laurensmiers Mar 30, 2025
ff13012
fix(database_writer): Remove unique columns for keywords table
laurensmiers Mar 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions dbbot/reader/database_writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sqlalchemy import create_engine, Column, DateTime, ForeignKey, Integer, MetaData, Sequence, String, Table, Text, \
from sqlalchemy import create_engine, Column, DateTime, ForeignKey, Interval, Integer, MetaData, Sequence, String, Table, Text, \
UniqueConstraint
from sqlalchemy.sql import and_, select
from sqlalchemy.exc import IntegrityError
Expand Down Expand Up @@ -57,7 +57,7 @@ def _create_table_test_run_status(self):
return self._create_table('test_run_status', (
Column('test_run_id', Integer, ForeignKey('test_runs.id'), nullable=False),
Column('name', String(256), nullable=False),
Column('elapsed', Integer),
Column('elapsed', Interval),
Column('failed', Integer, nullable=False),
Column('passed', Integer, nullable=False)
), ('test_run_id', 'name'))
Expand All @@ -76,7 +76,7 @@ def _create_table_tag_status(self):
Column('test_run_id', Integer, ForeignKey('test_runs.id'), nullable=False),
Column('name', String(256), nullable=False),
Column('critical', Integer, nullable=False),
Column('elapsed', Integer),
Column('elapsed', Interval),
Column('failed', Integer, nullable=False),
Column('passed', Integer, nullable=False)
), ('test_run_id', 'name'))
Expand All @@ -94,7 +94,7 @@ def _create_table_suite_status(self):
return self._create_table('suite_status', (
Column('test_run_id', Integer, ForeignKey('test_runs.id'), nullable=False),
Column('suite_id', Integer, ForeignKey('suites.id'), nullable=False),
Column('elapsed', Integer, nullable=False),
Column('elapsed', Interval, nullable=False),
Column('failed', Integer, nullable=False),
Column('passed', Integer, nullable=False),
Column('status', String(4), nullable=False)
Expand All @@ -114,7 +114,7 @@ def _create_table_test_status(self):
Column('test_run_id', Integer, ForeignKey('test_runs.id'), nullable=False),
Column('test_id', Integer, ForeignKey('tests.id'), nullable=False),
Column('status', String(4), nullable=False),
Column('elapsed', Integer, nullable=False)
Column('elapsed', Interval, nullable=False),
), ('test_run_id', 'test_id'))

def _create_table_keywords(self):
Expand All @@ -126,14 +126,14 @@ def _create_table_keywords(self):
Column('type', String(64), nullable=False),
Column('timeout', String(4)),
Column('doc', Text)
), ('name', 'type'))
))

def _create_table_keyword_status(self):
return self._create_table('keyword_status', (
Column('test_run_id', Integer, ForeignKey('test_runs.id'), nullable=False),
Column('keyword_id', Integer, ForeignKey('keywords.id'), nullable=False),
Column('status', String(4), nullable=False),
Column('elapsed', Integer, nullable=False)
Column('elapsed', Interval, nullable=False),
))

def _create_table_messages(self):
Expand Down Expand Up @@ -167,7 +167,7 @@ def _create_table(self, table_name, columns, unique_columns=()):

def fetch_id(self, table_name, criteria):
table = getattr(self, table_name)
sql_statement = select([table.c.id]).where(
sql_statement = select(table.c.id).where(
and_(*(getattr(table.c, key) == value for key, value in criteria.items()))
)
result = self._connection.execute(sql_statement).first()
Expand All @@ -178,7 +178,7 @@ def fetch_id(self, table_name, criteria):

def insert(self, table_name, criteria):
sql_statement = getattr(self, table_name).insert()
result = self._connection.execute(sql_statement, **criteria)
result = self._connection.execute(sql_statement, criteria)
return result.inserted_primary_key[0]

def insert_or_ignore(self, table_name, criteria):
Expand All @@ -190,4 +190,5 @@ def insert_or_ignore(self, table_name, criteria):

def close(self):
self._verbose('- Closing database connection')
self._connection.commit()
self._connection.close()
38 changes: 20 additions & 18 deletions dbbot/reader/robot_results_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import with_statement
from datetime import datetime
import datetime

from hashlib import sha1
from robot.api import ExecutionResult
from sqlalchemy.exc import IntegrityError
Expand All @@ -35,14 +36,14 @@ def xml_to_db(self, xml_file):
try:
test_run_id = self._db.insert('test_runs', {
'hash': hash_string,
'imported_at': datetime.utcnow(),
'source_file': test_run.source,
'source_file': str(test_run.source),
'imported_at': datetime.datetime.now(datetime.UTC),
'started_at': self._format_robot_timestamp(test_run.suite.starttime),
'finished_at': self._format_robot_timestamp(test_run.suite.endtime)
})
except IntegrityError:
test_run_id = self._db.fetch_id('test_runs', {
'source_file': test_run.source,
'source_file': str(test_run.source),
'started_at': self._format_robot_timestamp(test_run.suite.starttime),
'finished_at': self._format_robot_timestamp(test_run.suite.endtime)
})
Expand All @@ -65,7 +66,7 @@ def _parse_errors(self, errors, test_run_id):
for error in errors:
self._db.insert_or_ignore('test_run_errors', {
'test_run_id': test_run_id, 'level': error.level,
'timestamp': self._format_robot_timestamp(error.timestamp),
'timestamp': error.timestamp,
'content': error.message,
'content_hash': self._string_hash(error.message)
})
Expand All @@ -86,7 +87,7 @@ def _parse_tag_stats(self, stat, test_run_id):
self._db.insert_or_ignore('tag_status', {
'test_run_id': test_run_id,
'name': stat.name,
'critical': int(stat.critical),
'skipped': int(stat.skipped),
'elapsed': getattr(stat, 'elapsed', None),
'failed': stat.failed,
'passed': stat.passed
Expand All @@ -108,26 +109,25 @@ def _parse_suite(self, suite, test_run_id, parent_suite_id=None):
'suite_id': parent_suite_id,
'xml_id': suite.id,
'name': suite.name,
'source': suite.source,
'source': str(suite.source),
'doc': suite.doc
})
except IntegrityError:
suite_id = self._db.fetch_id('suites', {
'name': suite.name,
'source': suite.source
'source': str(suite.source)
})
self._parse_suite_status(test_run_id, suite_id, suite)
self._parse_suites(suite, test_run_id, suite_id)
self._parse_tests(suite.tests, test_run_id, suite_id)
self._parse_keywords(suite.keywords, test_run_id, suite_id, None)

def _parse_suite_status(self, test_run_id, suite_id, suite):
self._db.insert_or_ignore('suite_status', {
'test_run_id': test_run_id,
'suite_id': suite_id,
'passed': suite.statistics.all.passed,
'failed': suite.statistics.all.failed,
'elapsed': suite.elapsedtime,
'passed': suite.statistics.passed,
'failed': suite.statistics.failed,
'elapsed': suite.elapsed_time,
'status': suite.status
})

Expand All @@ -154,14 +154,14 @@ def _parse_test(self, test, test_run_id, suite_id):
})
self._parse_test_status(test_run_id, test_id, test)
self._parse_tags(test.tags, test_id)
self._parse_keywords(test.keywords, test_run_id, None, test_id)
self._parse_keywords(test.body, test_run_id, None, test_id)

def _parse_test_status(self, test_run_id, test_id, test):
self._db.insert_or_ignore('test_status', {
'test_run_id': test_run_id,
'test_id': test_id,
'status': test.status,
'elapsed': test.elapsedtime
'elapsed': test.elapsed_time
})

def _parse_tags(self, tags, test_id):
Expand All @@ -173,6 +173,8 @@ def _parse_keywords(self, keywords, test_run_id, suite_id, test_id, keyword_id=N
[self._parse_keyword(keyword, test_run_id, suite_id, test_id, keyword_id) for keyword in keywords]

def _parse_keyword(self, keyword, test_run_id, suite_id, test_id, keyword_id):
if keyword.type != "KEYWORD":
return
try:
keyword_id = self._db.insert('keywords', {
'suite_id': suite_id,
Expand All @@ -191,21 +193,21 @@ def _parse_keyword(self, keyword, test_run_id, suite_id, test_id, keyword_id):
self._parse_keyword_status(test_run_id, keyword_id, keyword)
self._parse_messages(keyword.messages, keyword_id)
self._parse_arguments(keyword.args, keyword_id)
self._parse_keywords(keyword.keywords, test_run_id, None, None, keyword_id)
self._parse_keywords(keyword.body, test_run_id, None, None, keyword_id)

def _parse_keyword_status(self, test_run_id, keyword_id, keyword):
self._db.insert_or_ignore('keyword_status', {
'test_run_id': test_run_id,
'keyword_id': keyword_id,
'status': keyword.status,
'elapsed': keyword.elapsedtime
'elapsed': keyword.elapsed_time
})

def _parse_messages(self, messages, keyword_id):
for message in messages:
self._db.insert_or_ignore('messages', {
'keyword_id': keyword_id, 'level': message.level,
'timestamp': self._format_robot_timestamp(message.timestamp),
'timestamp': message.timestamp,
'content': message.message,
'content_hash': self._string_hash(message.message)
})
Expand All @@ -220,7 +222,7 @@ def _parse_arguments(self, args, keyword_id):

@staticmethod
def _format_robot_timestamp(timestamp):
return datetime.strptime(timestamp, '%Y%m%d %H:%M:%S.%f') if timestamp else None
return datetime.datetime.strptime(timestamp, '%Y%m%d %H:%M:%S.%f') if timestamp else None

@staticmethod
def _string_hash(string):
Expand Down