Skip to content

[DBMON-6299] Fix table name metadata extraction #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eric-weaver merged 1 commit into from
Mar 19, 2026
Merged

[DBMON-6299] Fix table name metadata extraction #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions normalizer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ func (n *Normalizer) normalizeToken(lexer *Lexer, normalizedSQLBuilder *strings.
var headState headState
var colonCtx colonContext
var ctes map[string]bool // Lazily initialized when first CTE is encountered
var inTableList bool

var lastValueToken *LastValueToken

Expand All @@ -180,7 +181,7 @@ func (n *Normalizer) normalizeToken(lexer *Lexer, normalizedSQLBuilder *strings.
preProcessToken(token, lastValueToken)
}
if n.shouldCollectMetadata() {
n.collectMetadata(token, lastValueToken, meta, statementMetadata, &ctes)
n.collectMetadata(token, lastValueToken, meta, statementMetadata, &ctes, &inTableList)
}
n.normalizeSQL(token, lastValueToken, normalizedSQLBuilder, &groupablePlaceholder, &headState, &colonCtx, lexerOpts...)
if token.Type == EOF {
Expand Down Expand Up @@ -232,15 +233,18 @@ func (n *Normalizer) shouldCollectMetadata() bool {
return n.config.CollectTables || n.config.CollectCommands || n.config.CollectComments || n.config.CollectProcedure
}

func (n *Normalizer) collectMetadata(token *Token, lastValueToken *LastValueToken, meta *metadataSet, statementMetadata *StatementMetadata, ctes *map[string]bool) {
func (n *Normalizer) collectMetadata(token *Token, lastValueToken *LastValueToken, meta *metadataSet, statementMetadata *StatementMetadata, ctes *map[string]bool, inTableList *bool) {
if n.config.CollectComments && (token.Type == COMMENT || token.Type == MULTILINE_COMMENT) {
comment := token.Value
meta.addMetadata(comment, meta.commentsSet, &statementMetadata.Comments)
} else if token.Type == COMMAND {
if n.config.CollectCommands {
} else if token.Type == COMMAND || token.Type == KEYWORD {
*inTableList = false
if n.config.CollectCommands && token.Type == COMMAND {
command := strings.ToUpper(token.Value)
meta.addMetadata(command, meta.commandsSet, &statementMetadata.Commands)
}
} else if token.Type == PUNCTUATION && (token.Value == "(" || token.Value == ")") {
*inTableList = false
} else if token.Type == IDENT || token.Type == QUOTED_IDENT || token.Type == FUNCTION {
tokenVal := token.Value
if token.Type == QUOTED_IDENT {
Expand All @@ -261,7 +265,12 @@ func (n *Normalizer) collectMetadata(token *Token, lastValueToken *LastValueToke
}
(*ctes)[tokenVal] = true
} else if n.config.CollectTables && lastValueToken.isTableIndicator {
// Collect table names, excluding any CTEs
*inTableList = true
isCTE := *ctes != nil && (*ctes)[tokenVal]
if !isCTE {
meta.addMetadata(tokenVal, meta.tablesSet, &statementMetadata.Tables)
}
} else if n.config.CollectTables && *inTableList && lastValueToken.Type == PUNCTUATION && lastValueToken.Value == "," {
isCTE := *ctes != nil && (*ctes)[tokenVal]
if !isCTE {
meta.addMetadata(tokenVal, meta.tablesSet, &statementMetadata.Tables)
Expand Down
4 changes: 2 additions & 2 deletions normalizer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,11 +306,11 @@ multiline comment */
input: "SELECT d.id, d.uuid, d.org_id, d.creator_id, d.updater_id, d.monitor_id, d.parent_id, d.original_parent_id, d.scope, d.start_dt, d.end_dt, d.canceled_dt, d.active, d.disabled, d.created, d.modified, d.message, d.monitor_tags, d.recurrence, d.mute_first_recovery_notification, d.scope_v2_query, d.scope_v2 FROM monitor_downtime d, org o WHERE o.id = d.org_id AND d.modified >= ? AND o.partition_num = ANY (?, ?, ?)",
expected: "SELECT d.id, d.uuid, d.org_id, d.creator_id, d.updater_id, d.monitor_id, d.parent_id, d.original_parent_id, d.scope, d.start_dt, d.end_dt, d.canceled_dt, d.active, d.disabled, d.created, d.modified, d.message, d.monitor_tags, d.recurrence, d.mute_first_recovery_notification, d.scope_v2_query, d.scope_v2 FROM monitor_downtime d, org o WHERE o.id = d.org_id AND d.modified >= ? AND o.partition_num = ANY ( ? )",
statementMetadata: StatementMetadata{
Tables: []string{"monitor_downtime"},
Tables: []string{"monitor_downtime", "org"},
Comments: []string{},
Commands: []string{"SELECT"},
Procedures: []string{},
Size: 22,
Size: 25,
},
},
{
Expand Down
11 changes: 11 additions & 0 deletions obfuscate_and_normalize_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -632,6 +632,17 @@ multiline comment */
WithDBMS(DBMSOracle),
},
},
{
input: "SELECT events.target_id, events.created_at, source_tickets.title FROM events, tickets AS source_tickets WHERE events.org_id = 123 AND source_tickets.status = 'open' LIMIT 100",
expected: "SELECT events.target_id, events.created_at, source_tickets.title FROM events, tickets WHERE events.org_id = ? AND source_tickets.status = ? LIMIT ?",
statementMetadata: StatementMetadata{
Tables: []string{"events", "tickets"},
Comments: []string{},
Commands: []string{"SELECT"},
Procedures: []string{},
Size: 19,
},
},
}

obfuscator := NewObfuscator(
Expand Down
1 change: 1 addition & 0 deletions sqllexer_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ var keywords = []string{
"INTO",
"IS",
"KEY",
"LATERAL",
Copy link
Collaborator

@lu-zhengda lu-zhengda Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what does the LATERAL keyword do with the fix?

Copy link
Contributor Author

@eric-weaver eric-weaver Mar 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It fixes test failures for lateral-join

"LEFT",
"LIKE",
"LIMIT",
Expand Down
4 changes: 2 additions & 2 deletions testdata/oracle/select/multiple-hints.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
{
"expected": "SELECT e.employee_id, e.first_name, d.department_name FROM employees e, departments d WHERE e.department_id = d.department_id;",
"statement_metadata": {
"size": 44,
"tables": ["employees"],
"size": 55,
"tables": ["employees", "departments"],
"commands": ["SELECT"],
"comments": ["/*+ LEADING(e) USE_HASH(d) */"],
"procedures": []
Expand Down
4 changes: 2 additions & 2 deletions testdata/oracle/select/select-with-oracle-specific-joins.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
{
"expected": "SELECT e.employee_id, e.last_name, d.department_name FROM employees e, departments d WHERE e.department_id = d.department_id ( + )",
"statement_metadata": {
"size": 15,
"tables": ["employees"],
"size": 26,
"tables": ["employees", "departments"],
"commands": ["SELECT"],
"comments": [],
"procedures": []
Expand Down
4 changes: 2 additions & 2 deletions testdata/oracle/select/use-nl-hint.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
{
"expected": "SELECT e.employee_id, e.first_name, d.department_name FROM employees e, departments d WHERE e.department_id = d.department_id;",
"statement_metadata": {
"size": 33,
"tables": ["employees"],
"size": 44,
"tables": ["employees", "departments"],
"commands": ["SELECT"],
"comments": ["/*+ USE_NL(e d) */"],
"procedures": []
Expand Down
Loading