refactor sql.lock and validation

Author: Schultzer

.github/workflows/ci.yml

@@ -4,7 +4,7 @@ on:
 jobs:
   test:
     runs-on: ubuntu-22.04
-    name: OTP 27 / Elixir 1.18
+    name: OTP 27 / Elixir 1.19
     services:
       postgres:
         image: postgres:latest
@@ -17,7 +17,7 @@ jobs:
       - uses: erlef/setup-beam@v1
         with:
           otp-version: 27
-          elixir-version: 1.18
+          elixir-version: main
       - uses: actions/cache@v4
         with:
           path: |

.github/workflows/conformance.yml

@@ -4,7 +4,7 @@ on:
 jobs:
   test:
     runs-on: ubuntu-22.04
-    name: OTP 27 / Elixir 1.18
+    name: OTP 28 / Elixir main
     services:
       postgres:
         image: postgres:latest
@@ -22,8 +22,8 @@ jobs:
             repository: elliotchance/sqltest
       - uses: erlef/setup-beam@v1
         with:
-          otp-version: 27
-          elixir-version: 1.18
+          otp-version: 28
+          elixir-version: main
       - uses: actions/cache@v4
         with:
           path: |

README.md

@@ -61,6 +61,35 @@ iex(6)> inspect(sql)
   [%User{id: 1, email: "john@example.com"}, %User{id: 2, email: "jane@example.com"}]
 ```
 
+## Compile time errors
+run `mix sql.get` to generate your `sql.lock` file which is used for error reporting.
+
+```elixir
+  ==> myapp
+  Compiling 1 file (.ex)
+  warning:
+    the relation OPS does not exist
+    the relation email is mentioned 2 times but does not exist
+    the relation users does not exist
+    ~SQL"""
+    select
+      email,
+      1 + "OPS"
+    from
+      users
+    where
+      email = 'john@example.com'
+    """
+    lib/myapp.ex:18: Myapp.list_users/0
+    (sql 0.4.0) lib/sql.ex:225: SQL.__inspect__/3
+    (sql 0.4.0) lib/sql.ex:115: SQL.build/4
+    (elixir 1.20.0-dev) src/elixir_dispatch.erl:263: :elixir_dispatch.expand_macro_fun/7
+    (elixir 1.20.0-dev) src/elixir_dispatch.erl:122: :elixir_dispatch.dispatch_import/6
+    (elixir 1.20.0-dev) src/elixir_clauses.erl:192: :elixir_clauses.def/3
+    (elixir 1.20.0-dev) src/elixir_def.erl:218: :elixir_def."-store_definition/10-lc$^0/1-0-"/3
+    (elixir 1.20.0-dev) src/elixir_def.erl:219: :elixir_def.store_definition/10
+```
+
 ## Benchmark
 You can find benchmark results [here](https://github.com/elixir-dbvisor/sql/benchmarks) or run `mix sql.bench`
 

lib/format.ex

@@ -9,7 +9,7 @@ defmodule SQL.Format do
 
   @doc false
   @doc since: "0.4.0"
-  def to_iodata(tokens, context), do: newline(to_iodata(tokens, context.binding, context.case, context.errors, 0, []), 0)
+  def to_iodata(tokens, context, indent \\ 0), do: newline(to_iodata(tokens, context.binding, context.case, context.errors, indent, []), indent)
 
   defp indention(acc, [{:preset, {_,0}},_,{:offset, {_,0,_,_}}|_], 0), do: acc
   defp indention(acc, [_,{:offset, {_,0}}|_], 0), do: acc
@@ -27,18 +27,18 @@ defmodule SQL.Format do
   {reserved, non_reserved, operators} = SQL.BNF.get_rules()
   for atom <- Enum.uniq(Enum.map(reserved++non_reserved++operators,&elem(&1, 0))), atom not in newline do
     defp to_iodata(unquote(atom), _binding ,:lower, _errors, _indent, acc) do
-      [unquote("#{atom}")|acc]
+      ["\e[35m", unquote("#{atom}"), "\e[0m"|acc]
     end
     defp to_iodata(unquote(atom), _binding, :upper, _errors, _indent, acc) do
-     [unquote(String.upcase("#{atom}"))|acc]
+     ["\e[35m", unquote(String.upcase("#{atom}")), "\e[0m"|acc]
     end
   end
   for atom <- newline do
     defp to_iodata({unquote(atom), m, values}, binding, :lower=case, errors, indent, acc) do
-      newline(indention([unquote("#{atom}")|newline(to_iodata(values, binding, case, errors, indent+1, acc), indent+1)], m, indent), indent)
+      newline(indention(["\e[35m", unquote("#{atom}"), "\e[0m"|newline(to_iodata(values, binding, case, errors, indent+1, acc), indent+1)], m, indent), indent)
     end
     defp to_iodata({unquote(atom), m, values}, binding, :upper=case, errors, indent, acc) do
-      newline(indention([unquote(String.upcase("#{atom}"))|newline(to_iodata(values, binding, case, errors, indent+1, acc), indent+1)], m, indent), indent)
+      newline(indention(["\e[35m", unquote(String.upcase("#{atom}")), "\e[0m"|newline(to_iodata(values, binding, case, errors, indent+1, acc), indent+1)], m, indent), indent)
     end
   end
   defp to_iodata(:comma, _binding, _case, _errors, 0, acc) do
@@ -66,10 +66,10 @@ defmodule SQL.Format do
     [?\\,?*,value,?*,?\\|acc]
   end
   defp to_iodata({:quote, m, value}, _binding, _case, _errors, indent, acc) do
-    indention([?',value,?'|acc], m, indent)
+    indention([?',"\e[32m" , value, "\e[0m",?'|acc], m, indent)
   end
   defp to_iodata({:backtick, m, value}, _binding, _case, _errors, indent, acc) do
-    indention([?`,value,?`|acc], m, indent)
+    indention([?`,"\e[32m" , value, "\e[0m",?`|acc], m, indent)
   end
   defp to_iodata({tag, m, []}, binding, case, errors, indent, acc) do
     indention(to_iodata(tag, binding, case, errors, indent, acc), m, indent)
@@ -101,13 +101,13 @@ defmodule SQL.Format do
   defp to_iodata({tag, m, value}=node, _binding, _case, errors, indent, acc) when tag in ~w[ident numeric special]a do
     case node in errors do
       true -> indention(["\e[31m", value, "\e[0m"|acc], m, indent)
-      false -> indention([value|acc], m, indent)
+      false -> indention(["\e[33m", value, "\e[0m"|acc], m, indent)
     end
   end
   defp to_iodata({:double_quote, m, value}=node, _binding, _case, errors, indent, acc) do
     case node in errors do
       true -> indention([?","\e[31m",value, "\e[0m", ?"|acc], m, indent)
-      false -> indention([?", value, ?"|acc], m, indent)
+      false -> indention([?", "\e[32m" , value, "\e[0m", ?"|acc], m, indent)
     end
   end
   defp to_iodata({tag, m, values}, binding, case, errors, indent, acc) do

lib/mix/tasks/sql.get.ex

@@ -7,49 +7,43 @@ defmodule Mix.Tasks.Sql.Get do
   import Mix.Generator
   @moduledoc since: "0.3.0"
 
-  defguard is_postgres(value) when value in [Ecto.Adapters.Postgres, Postgrex]
-  defguard is_mysql(value) when value in [Ecto.Adapters.MyXQL, MyXQL]
-  defguard is_tds(value) when value in [Ecto.Adapters.Tds, Tds]
-  defguard is_sqlite(value) when value in [Ecto.Adapters.SQLite3, Exqlite]
-
   @shortdoc "Generates a sql.lock"
   def run(args) do
     app = Mix.Project.config()[:app]
     Application.load(app)
     repos = Application.get_env(app, :ecto_repos)
     Mix.Task.run("app.config", args)
     Application.ensure_all_started(:ecto_sql, :permanent)
-    lock = Enum.reduce(repos, %{}, fn repo, acc ->
+    lock = Enum.reduce(repos, [], fn repo, acc ->
       repo.__adapter__().ensure_all_started(repo.config(), [])
       repo.start_link(repo.config())
-      Map.merge(acc, to_lock(repo, repo.__adapter__()), fn _, l, r -> Enum.uniq(:lists.flatten(l, r)) end)
+      get(repo)++acc
     end)
     create_file("sql.lock", lock_template(lock: lock), force: true)
   end
 
-  def get(repo, sql) do
+  defp get(repo) do
+    sql = to_query(repo.__adapter__())
     {:ok, %{columns: columns, rows: rows}} = repo.query(to_string(sql), [])
     columns = Enum.map(columns, &String.to_atom(String.downcase(&1)))
     Enum.map(rows, &Map.new(Enum.zip(columns, &1)))
   end
 
+  defp to_query(value) when value in [Ecto.Adapters.Postgres, Postgrex], do: ~SQL"select * from information_schema.columns where table_schema not in ('information_schema', 'pg_catalog')"
+  defp to_query(value) when value in [Ecto.Adapters.Tds, Tds], do: ~SQL"select * from information_schema.columns where table_schema not in ('information_schema', 'pg_catalog')"
+  defp to_query(value) when value in [Ecto.Adapters.MyXQL, MyXQL], do: ~SQL"select * from information_schema.columns where table_schema not in ('mysql', 'performance_schema', 'sys')"
+  defp to_query(value) when value in [Ecto.Adapters.SQLite3, Exqlite], do: ~SQL"select * from sqlite_master join pragma_table_info (sqlite_master.name)"
+
   embed_template(:lock, """
-  <%= inspect @lock, pretty: true, limit: :infinity %>
+    %{
+      columns: <%= inspect @lock, pretty: true, limit: :infinity %>,
+      validate: fn
+      <%= for %{table_name: table, column_name: column} <- @lock do %>
+        <%= inspect String.to_charlist(table) %>, nil -> true
+        <%= inspect String.to_charlist(table) %>, <%= inspect String.to_charlist(column) %> -> true
+      <% end %>
+      _, _ -> false
+      end
+    }
   """)
-
-  def to_lock(repo, value) when not is_sqlite(value) do
-    %{columns: get(repo, to_query(value, :columns)), tables: get(repo, to_query(value, :tables))}
-  end
-  def to_lock(repo, value) when is_sqlite(value) do
-    tables = get(repo, to_query(value, :tables))
-    %{columns: tables, tables: tables}
-  end
-
-  def to_query(value, :tables) when is_postgres(value), do: ~SQL"select * from information_schema.tables where table_schema not in ('information_schema', 'pg_catalog')"
-  def to_query(value, :columns) when is_postgres(value), do: ~SQL"select * from information_schema.columns where table_schema not in ('information_schema', 'pg_catalog')"
-  def to_query(value, :tables) when is_tds(value), do: ~SQL"select * from information_schema.tables where table_schema not in ('information_schema', 'pg_catalog')"
-  def to_query(value, :columns) when is_tds(value), do: ~SQL"select * from information_schema.columns where table_schema not in ('information_schema', 'pg_catalog')"
-  def to_query(value, :tables) when is_mysql(value), do: ~SQL"select * from information_schema.tables where table_schema not in ('mysql', 'performance_schema', 'sys')"
-  def to_query(value, :columns) when is_mysql(value), do: ~SQL"select * from information_schema.columns where table_schema not in ('mysql', 'performance_schema', 'sys')"
-  def to_query(value, :tables) when is_sqlite(value), do: ~SQL"select * from sqlite_master join pragma_table_info (sqlite_master.name)"
 end

lib/parser.ex

@@ -266,11 +266,9 @@ defmodule SQL.Parser do
       e -> e++errors
     end
   end
-  defp validate({tag, _, _}, %{validate: nil}, errors) when tag in ~w[from join]a, do: errors
   defp validate({tag, _, values}, %{validate: fun} = context, errors) when tag in ~w[from join]a do
     values
     |> Enum.reduce([], fn
-      {:paren, _, _}, acc -> acc
       {:on, _, _} = node, acc -> validate(node, context, acc)
       {tag, _, _}=node, acc when tag in ~w[ident double_quote]a -> validate_table(fun, node, acc)
       {:as, _, [{tag, _, _}=node, _]}, acc when tag in ~w[ident double_quote]a  -> validate_table(fun, node, acc)
@@ -279,7 +277,8 @@ defmodule SQL.Parser do
       {:comma, _, [{:dot, _, [_, {:bracket, _, [{:ident, _, _}=node]}]}]}, acc -> validate_table(fun, node, acc)
       {:comma, _, [{:dot, _, [_, {tag, _, _}=node]}]}, acc when tag in ~w[ident double_quote]a -> validate_table(fun, node, acc)
       {:comma, _, [{:as, _, [{tag, _, _}=node, _]}]}, acc when tag in ~w[ident double_quote]a  -> validate_table(fun, node, acc)
-      {:comma, _, [{tag, _, _}=node]}, acc when tag in ~w[ident double_quote]a -> validate_table(fun, node, acc)
+      {:comma, _, [{tag, _, _}=node|_]}, acc when tag in ~w[ident double_quote]a -> validate_table(fun, node, acc)
+      _, acc -> acc
     end)
     |> case do
       [] -> errors
@@ -290,18 +289,19 @@ defmodule SQL.Parser do
   defp validate({tag, _, _} = node, _, errors) when tag in ~w[offset limit]a, do: [node|errors]
 
   defp validate_table(fun, {_, _, value}=node, acc)  do
-    case fun.(value) do
+    case fun.(value, nil) do
       true -> acc
       false -> [node|acc]
     end
   end
 
   defp validate_columns(fun, {tag, _, value}=node, acc) when tag in ~w[ident double_quote]a do
-    case fun.(value) do
+    case fun.(nil, value) do
       true -> acc
       false -> [node|acc]
     end
   end
+  defp validate_columns(fun, {_, _, values}, acc), do: validate_columns(fun, values, acc)
   defp validate_columns(fun, [node|values], acc) do
     validate_columns(fun, values, validate_columns(fun, node, acc))
   end

lib/sql.ex

@@ -13,14 +13,16 @@ defmodule SQL do
     quote bind_quoted: [opts: opts] do
       @doc false
       import SQL
-      if File.exists?(Path.relative_to_cwd("sql.lock")) do
-        Module.put_attribute(__MODULE__, :sql_config, %{case: opts[:case] || :lower, adapter: opts[:adapter] || Application.compile_env(:sql, :adapter, ANSI), validate: &__MODULE__.sql_validate/1})
-        def sql_validate, do: unquote(build_lock())
-      else
-        # IO.warn("Could not find a sql.lock, please run mix sql.get")
-        Module.put_attribute(__MODULE__, :sql_config, %{case: opts[:case] || :lower, adapter: opts[:adapter] || Application.compile_env(:sql, :adapter, ANSI), validate: nil})
-        def sql_validate, do: fn _ -> true end
-      end
+      config = Map.new(Keyword.merge([case: :lower, adapter: Application.compile_env(:sql, :adapter, ANSI), validate: fn _, _ -> true end],  opts))
+      @external_resource Path.relative_to_cwd("sql.lock")
+      config = with true <- File.exists?(Path.relative_to_cwd("sql.lock")),
+                    %{validate: validate} <- elem(Code.eval_file("sql.lock", File.cwd!()), 0) do
+                    %{config | validate: validate}
+               else
+                _ -> config
+               end
+      Module.put_attribute(__MODULE__, :sql_config, config)
+      def sql_repo, do: unquote(opts[:repo] || config.adapter)
     end
   end
 
@@ -93,9 +95,9 @@ defmodule SQL do
 
   @doc false
   def build(left, {:<<>>, _, _} = right, _modifiers, env) do
-    config = %{case: :lower, adapter: Application.get_env(:sql, :adapter, ANSI), validate: nil}
+    config = %{case: :lower, adapter: Application.get_env(:sql, :adapter, ANSI), validate: fn _, _ -> true end}
     config = if env.module, do: Module.get_attribute(env.module, :sql_config, config), else: config
-    sql = struct(SQL, module: config.adapter)
+    sql = struct(SQL, module: env.module)
     stack = if env.function do
               {env.module, elem(env.function, 0), elem(env.function, 1), [file: Path.relative_to_cwd(env.file), line: env.line]}
             else
@@ -105,6 +107,7 @@ defmodule SQL do
       {:static, data} ->
         id = id(data)
         {:ok, context, tokens} = SQL.Lexer.lex(data, env.file)
+        %{context|validate: config.validate, module: config.adapter, case: config.case}
         {:ok, context, tokens} = SQL.Parser.parse(tokens, %{context|validate: config.validate, module: config.adapter, case: config.case})
         sql = %{sql | idx: context.idx, tokens: tokens, string: IO.iodata_to_binary(context.module.to_iodata(tokens, context)), inspect: __inspect__(tokens, context, stack), id: id}
         case context.binding do
@@ -117,7 +120,7 @@ defmodule SQL do
 
       {:dynamic, data} ->
         sql = %{sql | id: id(data)}
-        quote bind_quoted: [left: Macro.unpipe(left), right: right, file: env.file, data: data, sql: Macro.escape(sql), env: Macro.escape(env), config: Macro.escape(config), stack: Macro.escape(stack)] do
+        quote bind_quoted: [left: Macro.unpipe(left), right: right, file: env.file, data: data, sql: Macro.escape(sql), env: Macro.escape(env), config: Macro.escape(%{config| validate: nil}), stack: Macro.escape(stack)] do
           {t,p,idx} = Enum.reduce(left, {[], [], 0}, fn
             {[], 0}, acc        -> acc
             {v, 0}, {t, p, idx} -> {t++v.tokens, p++v.params, idx+v.idx}
@@ -210,26 +213,9 @@ defmodule SQL do
     end
   end
 
-  @doc false
-  def build_lock() do
-    case elem(Code.eval_file("sql.lock", File.cwd!()), 0) do
-      %{tables: tables, columns: columns} ->
-      tables = Enum.map(tables, fn %{table_name: value} -> String.downcase(value) end)
-      columns = Enum.map(columns, fn %{table_name: table_name, column_name: column_name} -> {String.downcase(table_name), String.downcase(column_name)} end)
-      quote bind_quoted: [tables: tables, columns: columns] do
-        fn
-          {table, column} -> {String.downcase(table), String.downcase(column)} in columns
-          table -> String.downcase(table) in tables
-        end
-      end
-      _data ->
-        nil
-    end
-  end
-
   @doc false
   def __inspect__(tokens, context, stack) do
-    inspect = IO.iodata_to_binary(["~SQL\"\"\""|[SQL.Format.to_iodata(tokens, context)|~c"\n\"\"\""]])
+    inspect = IO.iodata_to_binary(["\e[0m", "  ~SQL\"\"\""|[SQL.Format.to_iodata(tokens, context, 1)|~c"\n  \"\"\""]])
     case context.errors do
       [] -> inspect
       errors ->
@@ -241,10 +227,10 @@ defmodule SQL do
 
   @doc false
   def format_error(errors), do: Enum.group_by(errors, &elem(&1, 2)) |> Enum.reduce([], fn
-    {k, [{:special, _, _}]}, acc -> [acc|["the operator \e[31m",k,"\e[0m is invalid, did you mean any of #{__suggest__(k)}\n"]]
-    {k, [{:special, _, _}|_]=v}, acc -> [acc|["the operator \e[31m",k,"\e[0m is mentioned #{length(v)} times but is invalid, did you mean any of #{__suggest__(k)}\n"]]
-    {k, [_]}, acc -> [acc|["the relation \e[31m",k,"\e[0m does not exist\n"]]
-    {k, v}, acc -> [acc|["the relation \e[31m",k,"\e[0m is mentioned #{length(v)} times but does not exist\n"]]
+    {k, [{:special, _, _}]}, acc -> [acc|["  the operator \e[31m",k,"\e[0m is invalid, did you mean any of #{__suggest__(k)}\n"]]
+    {k, [{:special, _, _}|_]=v}, acc -> [acc|["  the operator \e[31m",k,"\e[0m is mentioned #{length(v)} times but is invalid, did you mean any of #{__suggest__(k)}\n"]]
+    {k, [_]}, acc -> [acc|["  the relation \e[31m",k,"\e[0m does not exist\n"]]
+    {k, v}, acc -> [acc|["  the relation \e[31m",k,"\e[0m is mentioned #{length(v)} times but does not exist\n"]]
   end)
 
   @doc false
@@ -257,8 +243,13 @@ defmodule SQL do
     def member?(_enumerable, _element) do
       {:error, __MODULE__}
     end
+    def reduce(%SQL{fn: nil} = enumerable, acc, fun) do
+      repo = enumerable.module.sql_repo()
+      %{rows: rows, columns: columns} = repo.query!(enumerable.string, enumerable.params)
+      Enumerable.reduce(rows, acc, fn row, acc -> fun.(Map.new(Enum.zip(columns, row)), acc) end)
+    end
     def reduce(%SQL{} = enumerable, acc, fun) do
-      repo = enumerable.module.sql_config()[:repo]
+      repo = enumerable.module.sql_repo()
       %{rows: rows, columns: columns} = repo.query!(enumerable.string, enumerable.params)
       fun = case Function.info(enumerable.fn, :arity) do
               {:arity, 1} -> fn row, acc -> fun.(enumerable.fn.(row), acc) end

mix.exs

@@ -10,7 +10,7 @@ defmodule SQL.MixProject do
     [
       app: :sql,
       version: @version,
-      elixir: "~> 1.16",
+      elixir: "~> 1.19",
       deps: deps(),
       description: "Brings an extensible SQL parser and sigil to Elixir, confidently write SQL with automatic parameterized queries.",
       name: "SQL",

test/formatter_test.exs

@@ -9,6 +9,6 @@ defmodule SQL.FormatterTest do
   end
 
   test "format/2 preserve interpolation" do
-    assert "\nwith recursive temp(n, fact) as (\n  select\n    0,\n    1\n  union all\n  select\n    n + {{one}},\n    (n + {{one}}) * fact\n  from\n    temp\n  where\n    n < 9\n)" == SQL.MixFormatter.format("with recursive temp(n, fact) as (select 0, 1 union all select n + {{one}}, (n + {{one}}) * fact from temp where n < 9)", [])
+    assert "\n\e[35mwith\e[0m \e[35mrecursive\e[0m \e[33mtemp\e[0m(\e[33mn\e[0m, \e[33mfact\e[0m) \e[35mas\e[0m (\n  \e[35mselect\e[0m\n    \e[33m0\e[0m,\n    \e[33m1\e[0m\n  \e[35munion\e[0m \e[35mall\e[0m\n  \e[35mselect\e[0m\n    \e[33mn\e[0m \e[35m+\e[0m {{one}},\n    (\e[33mn\e[0m \e[35m+\e[0m {{one}}) \e[35m*\e[0m \e[33mfact\e[0m\n  \e[35mfrom\e[0m\n    \e[33mtemp\e[0m\n  \e[35mwhere\e[0m\n    \e[33mn\e[0m \e[35m<\e[0m \e[33m9\e[0m\n)" == SQL.MixFormatter.format("with recursive temp(n, fact) as (select 0, 1 union all select n + {{one}}, (n + {{one}}) * fact from temp where n < 9)", [])
   end
 end

test/parser_test.exs

@@ -25,8 +25,8 @@ defmodule SQL.ParserTest do
   describe "error" do
     test "missing table with sql.lock and with tables in it" do
       {:ok, context, tokens} = SQL.Lexer.lex(@query)
-      {:ok, %{errors: errors}, _tokens} = SQL.Parser.parse(tokens, SQLTest.Helpers.set_sql_lock(context))
-      assert length(errors) == 20
+      {:ok, %{errors: errors}, _tokens} = SQL.Parser.parse(tokens, SQLTest.Helpers.set_validate(context))
+      assert length(errors) == 30
     end
 
     test "missing table with sql.lock and without tables in it" do