Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 107 additions & 73 deletions src/DatabaseLibrary/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,9 +328,7 @@ def execute_sql_script(
else:
statements_to_execute = self.split_sql_script(script_path, external_parser=external_parser)
for statement in statements_to_execute:
proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?")
line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$")
omit_semicolon = not line_ends_with_proc_end.search(statement.lower())
omit_semicolon = self._omit_semicolon_needed(statement)
self._execute_sql(cur, statement, omit_semicolon, replace_robot_variables=replace_robot_variables)
self._commit_if_needed(db_connection, no_transaction)
except Exception as e:
Expand All @@ -349,73 +347,83 @@ def split_sql_script(
Set ``external_parser`` to _True_ to use the external library [https://pypi.org/project/sqlparse/|sqlparse].
"""
with open(script_path, encoding="UTF-8") as sql_file:
logger.info("Splitting script file into statements...")
statements_to_execute = []
if external_parser:
split_statements = sqlparse.split(sql_file.read())
for statement in split_statements:
statement_without_comments = sqlparse.format(statement, strip_comments=True)
if statement_without_comments:
statements_to_execute.append(statement_without_comments)
else:
current_statement = ""
inside_statements_group = False
proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?")
proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?")
for line in sql_file:
line = line.strip()
if line.startswith("#") or line.startswith("--") or line == "/":
return self.split_sql_string(sql_file.read(), external_parser=external_parser)

def split_sql_string(self, sql_string: str, external_parser=False):
"""
Splits the content of the ``sql_string`` into individual SQL commands
and returns them as a list of strings.
SQL commands are expected to be delimited by a semicolon (';').

Set ``external_parser`` to _True_ to use the external library [https://pypi.org/project/sqlparse/|sqlparse].
"""
logger.info(f"Splitting SQL into statements. Using external parser: {external_parser}")
statements_to_execute = []
if external_parser:
split_statements = sqlparse.split(sql_string)
for statement in split_statements:
statement_without_comments = sqlparse.format(statement, strip_comments=True)
if statement_without_comments:
statements_to_execute.append(statement_without_comments)
else:
current_statement = ""
inside_statements_group = False
proc_start_pattern = re.compile("create( or replace)? (procedure|function){1}( )?")
proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?")
for line in sql_string.splitlines():
line = line.strip()
if line.startswith("#") or line.startswith("--") or line == "/":
continue

# check if the line matches the creating procedure regexp pattern
if proc_start_pattern.match(line.lower()):
inside_statements_group = True
elif line.lower().startswith("begin"):
inside_statements_group = True

# semicolons inside the line? use them to separate statements
# ... but not if they are inside a begin/end block (aka. statements group)
sqlFragments = line.split(";")
# no semicolons
if len(sqlFragments) == 1:
current_statement += line + " "
continue
quotes = 0
# "select * from person;" -> ["select..", ""]
for sqlFragment in sqlFragments:
if len(sqlFragment.strip()) == 0:
continue

# check if the line matches the creating procedure regexp pattern
if proc_start_pattern.match(line.lower()):
if inside_statements_group:
# if statements inside a begin/end block have semicolns,
# they must persist - even with oracle
sqlFragment += "; "

if proc_end_pattern.match(sqlFragment.lower()):
inside_statements_group = False
elif proc_start_pattern.match(sqlFragment.lower()):
inside_statements_group = True
elif line.lower().startswith("begin"):
elif sqlFragment.lower().startswith("begin"):
inside_statements_group = True

# semicolons inside the line? use them to separate statements
# ... but not if they are inside a begin/end block (aka. statements group)
sqlFragments = line.split(";")
# no semicolons
if len(sqlFragments) == 1:
current_statement += line + " "
continue
quotes = 0
# "select * from person;" -> ["select..", ""]
for sqlFragment in sqlFragments:
if len(sqlFragment.strip()) == 0:
continue

if inside_statements_group:
# if statements inside a begin/end block have semicolns,
# they must persist - even with oracle
sqlFragment += "; "

if proc_end_pattern.match(sqlFragment.lower()):
inside_statements_group = False
elif proc_start_pattern.match(sqlFragment.lower()):
inside_statements_group = True
elif sqlFragment.lower().startswith("begin"):
inside_statements_group = True

# check if the semicolon is a part of the value (quoted string)
quotes += sqlFragment.count("'")
quotes -= sqlFragment.count("\\'")
inside_quoted_string = quotes % 2 != 0
if inside_quoted_string:
sqlFragment += ";" # restore the semicolon

current_statement += sqlFragment
if not inside_statements_group and not inside_quoted_string:
statements_to_execute.append(current_statement.strip())
current_statement = ""
quotes = 0

current_statement = current_statement.strip()
if len(current_statement) != 0:
statements_to_execute.append(current_statement)

return statements_to_execute
# check if the semicolon is a part of the value (quoted string)
quotes += sqlFragment.count("'")
quotes -= sqlFragment.count("\\'")
inside_quoted_string = quotes % 2 != 0
if inside_quoted_string:
sqlFragment += ";" # restore the semicolon

current_statement += sqlFragment
if not inside_statements_group and not inside_quoted_string:
statements_to_execute.append(current_statement.strip())
current_statement = ""
quotes = 0

current_statement = current_statement.strip()
if len(current_statement) != 0:
statements_to_execute.append(current_statement)

return statements_to_execute

@renamed_args(
mapping={
Expand All @@ -433,12 +441,20 @@ def execute_sql_string(
omit_trailing_semicolon: Optional[bool] = None,
*,
replace_robot_variables=False,
split: bool = False,
external_parser: bool = False,
sqlString: Optional[str] = None,
sansTran: Optional[bool] = None,
omitTrailingSemicolon: Optional[bool] = None,
):
"""
Executes the ``sql_string`` as a single SQL command.
Executes the ``sql_string`` - as a single SQL command (default) or as separate statements.

Set ``split`` to _True_ to enable dividing the string into SQL commands similar to the `Execute SQL Script`
keyword. The commands are expected to be delimited by a semicolon (';') in this case -
they will be split and executed separately.

Set ``external_parser`` to _True_ to use the external library [https://pypi.org/project/sqlparse/|sqlparse] for splitting the script.

Set ``no_transaction`` to _True_ to run command without explicit transaction commit
or rollback in case of error.
Expand Down Expand Up @@ -473,13 +489,20 @@ def execute_sql_string(
cur = db_connection.client.cursor()
if omit_trailing_semicolon is None:
omit_trailing_semicolon = db_connection.omit_trailing_semicolon
self._execute_sql(
cur,
sql_string,
omit_trailing_semicolon=omit_trailing_semicolon,
parameters=parameters,
replace_robot_variables=replace_robot_variables,
)
if not split:
self._execute_sql(
cur,
sql_string,
omit_trailing_semicolon=omit_trailing_semicolon,
parameters=parameters,
replace_robot_variables=replace_robot_variables,
)
else:
statements_to_execute = self.split_sql_string(sql_string, external_parser=external_parser)
for statement in statements_to_execute:
omit_semicolon = self._omit_semicolon_needed(statement)
self._execute_sql(cur, statement, omit_semicolon, replace_robot_variables=replace_robot_variables)

self._commit_if_needed(db_connection, no_transaction)
except Exception as e:
self._rollback_and_raise(db_connection, no_transaction, e)
Expand Down Expand Up @@ -808,6 +831,17 @@ def set_logging_query_results(self, enabled: Optional[bool] = None, log_head: Op
raise ValueError(f"Wrong log head value provided: {log_head}. The value can't be negative!")
self.LOG_QUERY_RESULTS_HEAD = log_head

def _omit_semicolon_needed(self, statement: str) -> bool:
"""
Checks if the `statement` ends with a procedure ending keyword - so that semicolon should be omitted -
and returns the result.
The function is used when running multiple SQL statements from a script or an SQL string.
"""
proc_end_pattern = re.compile("end(?!( if;| loop;| case;| while;| repeat;)).*;()?")
line_ends_with_proc_end = re.compile(r"(\s|;)" + proc_end_pattern.pattern + "$")
omit_semicolon = not line_ends_with_proc_end.search(statement.lower())
return omit_semicolon

def _execute_sql(
self,
cur,
Expand Down
6 changes: 3 additions & 3 deletions test/resources/common.resource
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ ${DB_NAME} db
${DB_PASS} pass
${DB_PORT} 5432
${DB_USER} db_user
${Script files dir} ${CURDIR}/script_file_tests
${Script files dir} ${CURDIR}/script_files

# used for MySQL via PyODBC only
${DB_DRIVER} ODBC Driver 18 for SQL Server
Expand Down Expand Up @@ -96,9 +96,9 @@ Create Person Table And Insert Data
Insert Data In Person Table Using SQL Script
[Arguments] ${alias}=${None}
IF $alias is None
${output}= Execute SQL Script ${CURDIR}/insert_data_in_person_table.sql
${output}= Execute SQL Script ${Script files dir}/insert_data_in_person_table.sql
ELSE
${output}= Execute SQL Script ${CURDIR}/insert_data_in_person_table.sql alias=${alias}
${output}= Execute SQL Script ${Script files dir}/insert_data_in_person_table.sql alias=${alias}
END
RETURN ${output}

Expand Down
2 changes: 1 addition & 1 deletion test/tests/common_tests/basic_tests.robot
Original file line number Diff line number Diff line change
Expand Up @@ -161,4 +161,4 @@ Verify Query - Row Count foobar table 0 row
Query Returns Zero Results
[Documentation] Tests that nothing crashes when there are zero results
${results}= Query SELECT * FROM person WHERE id < 0
Should Be Empty ${results}
Should Be Empty ${results}
2 changes: 1 addition & 1 deletion test/tests/common_tests/encoding.robot
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ Read SQL Script Files As UTF8
... Pytho might have an issue opening this file on Windows, as it doesn't use UTF8 by default.
... In this case you the library should excplicitely set the UTF8 encoding when opening the script file.
... https://dev.to/methane/python-use-utf-8-mode-on-windows-212i
Execute Sql Script ${CURDIR}/../../resources/insert_data_in_person_table_utf8.sql
Execute Sql Script ${Script files dir}/insert_data_in_person_table_utf8.sql
${results}= Query
... SELECT LAST_NAME FROM person WHERE FIRST_NAME='Jürgen'
Should Be Equal ${results}[0][0] Gernegroß
35 changes: 0 additions & 35 deletions test/tests/common_tests/script_files.robot
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ Suite Teardown Disconnect From Database
Test Setup Create Person Table
Test Teardown Drop Tables Person And Foobar


*** Test Cases ***
Semicolons As Statement Separators In One Line
Run SQL Script File statements_in_one_line
Expand Down Expand Up @@ -35,40 +34,6 @@ Semicolons And Quotes In Values
Should Be Equal As Strings ${results}[0] (5, 'Miles', "O'Brian")
Should Be Equal As Strings ${results}[1] (6, 'Keiko', "O'Brian")

Split Script Into Statements - Internal Parser
Insert Data In Person Table Using SQL Script
@{Expected commands}= Create List
... SELECT * FROM person
... SELECT * FROM person WHERE id=1
${extracted commands}= Split Sql Script ${Script files dir}/split_commands.sql
Lists Should Be Equal ${Expected commands} ${extracted commands}
FOR ${command} IN @{extracted commands}
${results}= Query ${command}
END

Split Script Into Statements - External Parser
Insert Data In Person Table Using SQL Script
@{Expected commands}= Create List
... SELECT * FROM person;
... SELECT * FROM person WHERE id=1;
${extracted commands}= Split Sql Script ${Script files dir}/split_commands.sql external_parser=True
Lists Should Be Equal ${Expected commands} ${extracted commands}
FOR ${command} IN @{extracted commands}
${results}= Query ${command}
END

Split Script Into Statements - External Parser - Comments Are Removed
Insert Data In Person Table Using SQL Script
@{Expected commands}= Create List
... SELECT * FROM person;
... SELECT * FROM person WHERE id=1;
${extracted commands}= Split Sql Script ${Script files dir}/split_commands_comments.sql external_parser=True
Lists Should Be Equal ${Expected commands} ${extracted commands}
FOR ${command} IN @{extracted commands}
${results}= Query ${command}
END


*** Keywords ***
Run SQL Script File
[Arguments] ${File Name}
Expand Down
Loading