Ruby/Rust/QL: simplify generation of overlay-related tables/predicates

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -53,7 +53,7 @@ jobs:
       - name: Create database
         run: |
           "${CODEQL}" database create \
-          --search-path "${{ github.workspace }}"
+          --search-path "${{ github.workspace }}" \
           --threads 4 \
             --language ql --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"

ql/extractor/src/generator.rs

@@ -36,5 +36,5 @@ pub fn run(options: Options) -> std::io::Result<()> {
         },
     ];
 
-    generate(languages, options.dbscheme, options.library, false)
+    generate(languages, options.dbscheme, options.library)
 }

ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll

@@ -5,6 +5,10 @@
 
 import codeql.Locations as L
 
+/** Holds if the database is an overlay. */
+overlay[local]
+private predicate isOverlay() { databaseMetadata("isOverlay", "true") }
+
 module QL {
   /** The base class for all AST nodes */
   class AstNode extends @ql_ast_node {
@@ -48,6 +52,30 @@
     final override string getAPrimaryQlClass() { result = "ReservedWord" }
   }
 
+  /** Gets the file containing the given `node`. */
+  overlay[local]
+  private @file getNodeFile(@ql_ast_node node) {
+    exists(@location_default loc | ql_ast_node_location(node, loc) |
+      locations_default(loc, result, _, _, _, _)
+    )
+  }
+
+  /** Holds if `file` was extracted as part of the overlay database. */
+  overlay[local]
+  private predicate discardFile(@file file) { isOverlay() and file = getNodeFile(_) }
+
+  /** Holds if `node` is in the `file` and is part of the overlay base database. */
+  overlay[local]
+  private predicate discardableAstNode(@file file, @ql_ast_node node) {
+    not isOverlay() and file = getNodeFile(node)
+  }
+
+  /** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
+  overlay[discard_entity]
+  private predicate discardAstNode(@ql_ast_node node) {
+    exists(@file file | discardableAstNode(file, node) and discardFile(file))
+  }
+
   /** A class representing `add_expr` nodes. */
   class AddExpr extends @ql_add_expr, AstNode {
     /** Gets the name of the primary QL class for this element. */
@@ -1318,6 +1346,30 @@
     final override string getAPrimaryQlClass() { result = "ReservedWord" }
   }
 
+  /** Gets the file containing the given `node`. */
+  overlay[local]
+  private @file getNodeFile(@dbscheme_ast_node node) {
+    exists(@location_default loc | dbscheme_ast_node_location(node, loc) |
+      locations_default(loc, result, _, _, _, _)
+    )
+  }
+
+  /** Holds if `file` was extracted as part of the overlay database. */
+  overlay[local]
+  private predicate discardFile(@file file) { isOverlay() and file = getNodeFile(_) }
+
+  /** Holds if `node` is in the `file` and is part of the overlay base database. */
+  overlay[local]
+  private predicate discardableAstNode(@file file, @dbscheme_ast_node node) {
+    not isOverlay() and file = getNodeFile(node)
+  }
+
+  /** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
+  overlay[discard_entity]
+  private predicate discardAstNode(@dbscheme_ast_node node) {
+    exists(@file file | discardableAstNode(file, node) and discardFile(file))
+  }
+
   /** A class representing `annotName` tokens. */
   class AnnotName extends @dbscheme_token_annot_name, Token {
     /** Gets the name of the primary QL class for this element. */
@@ -1654,6 +1706,30 @@
     final override string getAPrimaryQlClass() { result = "ReservedWord" }
   }
 
+  /** Gets the file containing the given `node`. */
+  overlay[local]
+  private @file getNodeFile(@blame_ast_node node) {
+    exists(@location_default loc | blame_ast_node_location(node, loc) |
+      locations_default(loc, result, _, _, _, _)
+    )
+  }
+
+  /** Holds if `file` was extracted as part of the overlay database. */
+  overlay[local]
+  private predicate discardFile(@file file) { isOverlay() and file = getNodeFile(_) }
+
+  /** Holds if `node` is in the `file` and is part of the overlay base database. */
+  overlay[local]
+  private predicate discardableAstNode(@file file, @blame_ast_node node) {
+    not isOverlay() and file = getNodeFile(node)
+  }
+
+  /** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
+  overlay[discard_entity]
+  private predicate discardAstNode(@blame_ast_node node) {
+    exists(@file file | discardableAstNode(file, node) and discardFile(file))
+  }
+
   /** A class representing `blame_entry` nodes. */
   class BlameEntry extends @blame_blame_entry, AstNode {
     /** Gets the name of the primary QL class for this element. */
@@ -1767,6 +1843,30 @@
     final override string getAPrimaryQlClass() { result = "ReservedWord" }
   }
 
+  /** Gets the file containing the given `node`. */
+  overlay[local]
+  private @file getNodeFile(@json_ast_node node) {
+    exists(@location_default loc | json_ast_node_location(node, loc) |
+      locations_default(loc, result, _, _, _, _)
+    )
+  }
+
+  /** Holds if `file` was extracted as part of the overlay database. */
+  overlay[local]
+  private predicate discardFile(@file file) { isOverlay() and file = getNodeFile(_) }
+
+  /** Holds if `node` is in the `file` and is part of the overlay base database. */
+  overlay[local]
+  private predicate discardableAstNode(@file file, @json_ast_node node) {
+    not isOverlay() and file = getNodeFile(node)
+  }
+
+  /** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
+  overlay[discard_entity]
+  private predicate discardAstNode(@json_ast_node node) {
+    exists(@file file | discardableAstNode(file, node) and discardFile(file))
+  }
+
   class UnderscoreValue extends @json_underscore_value, AstNode { }
 
   /** A class representing `array` nodes. */

ql/ql/src/ql.dbscheme

@@ -108,6 +108,12 @@ yaml_locations(unique int locatable: @yaml_locatable ref,
 
 @yaml_locatable = @yaml_node | @yaml_error;
 
+/*- Database metadata -*/
+databaseMetadata(
+  string metadataKey: string ref,
+  string value: string ref
+);
+
 /*- QL dbscheme -*/
 @ql_add_expr_left_type = @ql_add_expr | @ql_aggregate | @ql_call_or_unqual_agg_expr | @ql_comp_term | @ql_conjunction | @ql_disjunction | @ql_expr_annotation | @ql_if_term | @ql_implication | @ql_in_expr | @ql_instance_of | @ql_literal | @ql_mul_expr | @ql_negation | @ql_par_expr | @ql_prefix_cast | @ql_qualified_expr | @ql_quantified | @ql_range | @ql_set_literal | @ql_special_call | @ql_super_ref | @ql_unary_expr | @ql_variable
 

ql/ql/src/ql.dbscheme.stats

@@ -22436,5 +22436,41 @@
                 </dep>
             </dependencies>
         </relation>
+        <relation>
+            <name>databaseMetadata</name>
+            <cardinality>1</cardinality>
+            <columnsizes>
+                <e>
+                    <k>metadataKey</k>
+                    <v>1</v>
+                </e>
+                <e>
+                    <k>value</k>
+                    <v>1</v>
+                </e>
+            </columnsizes>
+            <dependencies>
+                <dep>
+                    <src>metadataKey</src>
+                    <trg>value</trg>
+                    <val>
+                        <hist>
+                            <budget>12</budget>
+                            <bs/>
+                        </hist>
+                    </val>
+                </dep>
+                <dep>
+                    <src>value</src>
+                    <trg>metadataKey</trg>
+                    <val>
+                        <hist>
+                            <budget>12</budget>
+                            <bs/>
+                        </hist>
+                    </val>
+                </dep>
+            </dependencies>
+        </relation>
         </stats>
 </dbstats>

ruby/extractor/src/generator.rs

@@ -28,5 +28,5 @@ pub fn run(options: Options) -> std::io::Result<()> {
         },
     ];
 
-    generate(languages, options.dbscheme, options.library, true)
+    generate(languages, options.dbscheme, options.library)
 }