fix: properly strip line comments and convert tabs to spaces in sqlparse.format

sqlparse/filters/others.py

@@ -74,6 +74,37 @@ def _get_insert_token(token):
             # "SQL-Hint" and has to be skipped
             tidx, token = get_next_comment(idx=tidx)
 
+        # Process line comments (-- comments)
+        for i, token in enumerate(tlist.tokens[:]):
+            if token.ttype in T.Comment.Single:
+                # Check if this is a SQL hint
+                if token.ttype in sql_hints:
+                    continue
+
+                # Find the position of this token in the current token list
+                try:
+                    tidx = tlist.tokens.index(token)
+                except ValueError:
+                    continue  # Token was already removed
+
+                pidx, prev_ = tlist.token_prev(tidx, skip_ws=False)
+                nidx, next_ = tlist.token_next(tidx, skip_ws=False)
+
+                # Replace by whitespace if prev and next exist and if they're not
+                # whitespaces. This doesn't apply if prev or next is a parenthesis.
+                if (
+                    prev_ is None or next_ is None
+                    or prev_.is_whitespace or prev_.match(T.Punctuation, '(')
+                    or next_.is_whitespace or next_.match(T.Punctuation, ')')
+                ):
+                    # Insert a whitespace to ensure the following SQL produces
+                    # a valid SQL (see #425).
+                    if prev_ is not None and not prev_.match(T.Punctuation, '('):
+                        tlist.tokens.insert(tidx, _get_insert_token(token))
+                    tlist.tokens.remove(token)
+                else:
+                    tlist.tokens[tidx] = _get_insert_token(token)
+
     def process(self, stmt):
         [self.process(sgroup) for sgroup in stmt.get_sublists()]
         StripCommentsFilter._process(stmt)

sqlparse/filters/reindent.py

@@ -26,6 +26,21 @@ def __init__(self, width=2, char=' ', wrap_after=0, n='\n',
         self._last_stmt = None
         self._last_func = None
 
+    def _normalize_tabs(self, tlist):
+        """Convert tabs to spaces in all tokens."""
+        for token in tlist.tokens:
+            if hasattr(token, 'tokens') and token.tokens:
+                self._normalize_tabs(token)
+            elif token.is_whitespace and '\t' in token.value:
+                # Convert tabs to spaces, preserving newlines
+                lines = token.value.split('\n')
+                normalized_lines = []
+                for line in lines:
+                    # Replace tabs with appropriate number of spaces
+                    normalized_line = line.expandtabs(self.width)
+                    normalized_lines.append(normalized_line)
+                token.value = '\n'.join(normalized_lines)
+
     def _flatten_up_to_token(self, token):
         """Yields all tokens up to token but excluding current."""
         if token.is_group:
@@ -237,6 +252,10 @@ def _process_default(self, tlist, stmts=True):
 
     def process(self, stmt):
         self._curr_stmt = stmt
+
+        # Normalize tabs to spaces first
+        self._normalize_tabs(stmt)
+
         self._process(stmt)
 
         if self._last_stmt is not None: