fix: properly strip -- comments and convert tabs to spaces in sqlparse.format

sqlparse/filters/others.py

@@ -74,6 +74,39 @@ def _get_insert_token(token):
             # "SQL-Hint" and has to be skipped
             tidx, token = get_next_comment(idx=tidx)
 
+        # Handle single-line comments (-- comments) that might not be wrapped in Comment groups
+        tidx = 0
+        while tidx < len(tlist.tokens):
+            token = tlist.tokens[tidx]
+            if token.ttype in (T.Comment.Single, T.Comment.Single.Hint):
+                # Skip SQL hints
+                if token.ttype == T.Comment.Single.Hint:
+                    tidx += 1
+                    continue
+
+                pidx, prev_ = tlist.token_prev(tidx, skip_ws=False)
+                nidx, next_ = tlist.token_next(tidx, skip_ws=False)
+
+                # Check if we need to preserve line breaks
+                m = re.search(r'([\r\n]+) *$', token.value)
+                if m is not None:
+                    # Replace with newline token to preserve line structure
+                    tlist.tokens[tidx] = sql.Token(T.Whitespace.Newline, m.groups()[0])
+                else:
+                    # Remove the comment token entirely
+                    if (
+                        prev_ is None or next_ is None
+                        or prev_.is_whitespace or prev_.match(T.Punctuation, '(')
+                        or next_.is_whitespace or next_.match(T.Punctuation, ')')
+                    ):
+                        # Just remove the token
+                        tlist.tokens.remove(token)
+                        tidx -= 1
+                    else:
+                        # Replace with a space to maintain valid SQL
+                        tlist.tokens[tidx] = sql.Token(T.Whitespace, ' ')
+            tidx += 1
+
     def process(self, stmt):
         [self.process(sgroup) for sgroup in stmt.get_sublists()]
         StripCommentsFilter._process(stmt)

sqlparse/filters/reindent.py

@@ -96,7 +96,23 @@ def _split_statements(self, tlist):
                 tidx += 1
             tidx, token = tlist.token_next_by(t=ttypes, idx=tidx)
 
+    def _normalize_whitespace(self, tlist):
+        """Convert tabs to spaces in whitespace tokens."""
+        for token in tlist.tokens:
+            if token.is_whitespace and '\t' in token.value:
+                # Replace tabs with spaces, preserving newlines
+                lines = token.value.split('\n')
+                normalized_lines = []
+                for line in lines:
+                    # Convert tabs to spaces (assuming tab width of 4 for conversion)
+                    normalized_line = line.replace('\t', self.char * 4)
+                    normalized_lines.append(normalized_line)
+                token.value = '\n'.join(normalized_lines)
+
     def _process(self, tlist):
+        # Normalize whitespace first to convert tabs to spaces
+        self._normalize_whitespace(tlist)
+
         func_name = f'_process_{type(tlist).__name__}'
         func = getattr(self, func_name.lower(), self._process_default)
         func(tlist)