Rust: expand attribute macros

rust/ast-generator/BUILD.bazel

@@ -46,8 +46,7 @@ codeql_rust_binary(
     ) + [":codegen"],
     aliases = aliases(),
     args = ["$(rlocationpath :rust.ungram)"],
-    compile_data = glob(["src/templates/*.mustache"]),
-    data = [":rust.ungram"],
+    data = [":rust.ungram"] + glob(["templates/*.mustache"]),
     proc_macro_deps = all_crate_deps(
         proc_macro = True,
     ),

rust/ast-generator/src/main.rs

@@ -142,6 +142,7 @@ fn fix_blank_lines(s: &str) -> String {
 fn write_schema(
     grammar: &AstSrc,
     super_types: BTreeMap<String, BTreeSet<String>>,
+    mustache_ctx: &mustache::Context,
 ) -> mustache::Result<String> {
     let mut schema = Schema::default();
     schema.classes.extend(
@@ -156,7 +157,7 @@ fn write_schema(
             .iter()
             .map(|node| node_src_to_schema_class(node, &super_types)),
     );
-    let template = mustache::compile_str(include_str!("templates/schema.mustache"))?;
+    let template = mustache_ctx.compile_path("schema")?;
     let res = template.render_to_string(&schema)?;
     Ok(fix_blank_lines(&res))
 }
@@ -541,7 +542,7 @@ fn node_to_extractor_info(node: &AstNodeSrc) -> ExtractorNodeInfo {
     }
 }
 
-fn write_extractor(grammar: &AstSrc) -> mustache::Result<String> {
+fn write_extractor(grammar: &AstSrc, mustache_ctx: &mustache::Context) -> mustache::Result<String> {
     let extractor_info = ExtractorInfo {
         enums: grammar
             .enums
@@ -550,7 +551,7 @@ fn write_extractor(grammar: &AstSrc) -> mustache::Result<String> {
             .collect(),
         nodes: grammar.nodes.iter().map(node_to_extractor_info).collect(),
     };
-    let template = mustache::compile_str(include_str!("templates/extractor.mustache"))?;
+    let template = mustache_ctx.compile_path("extractor")?;
     let res = template.render_to_string(&extractor_info)?;
     Ok(fix_blank_lines(&res))
 }
@@ -578,16 +579,21 @@ fn main() -> anyhow::Result<()> {
         let super_class_y = super_types.get(&y.name).into_iter().flatten().max();
         super_class_x.cmp(&super_class_y).then(x.name.cmp(&y.name))
     });
-    let schema = write_schema(&grammar, super_types)?;
-    let schema_path = project_root().join("schema/ast.py");
+    let root = project_root();
+    let mustache_ctx = mustache::Context {
+        template_path: root.join("ast-generator").join("templates"),
+        template_extension: "mustache".to_string(),
+    };
+    let schema = write_schema(&grammar, super_types, &mustache_ctx)?;
+    let schema_path = root.join("schema/ast.py");
     codegen::ensure_file_contents(
         crate::flags::CodegenType::Grammar,
         &schema_path,
         &schema,
         false,
     );
 
-    let extractor = write_extractor(&grammar)?;
+    let extractor = write_extractor(&grammar, &mustache_ctx)?;
     let extractor_path = project_root().join("extractor/src/translate/generated.rs");
     codegen::ensure_file_contents(
         crate::flags::CodegenType::Grammar,

rust/ast-generator/src/templates/extractor.mustache → rust/ast-generator/templates/extractor.mustache

@@ -1,7 +1,5 @@
 //! Generated by `ast-generator`, do not edit by hand.
-¶{{! <- denotes empty line that should be kept, all blank lines are removed otherwise}}
-#![cfg_attr(any(), rustfmt::skip)]
-¶
+
 use super::base::Translator;
 use super::mappings::TextValue;
 use crate::emit_detached;
@@ -11,30 +9,33 @@ use ra_ap_syntax::ast::{
     HasArgList, HasAttrs, HasGenericArgs, HasGenericParams, HasLoopBody, HasModuleItem, HasName,
     HasTypeBounds, HasVisibility, RangeItem,
 };
-use ra_ap_syntax::{ast, AstNode};
-¶
+#[rustfmt::skip]
+use ra_ap_syntax::{AstNode, ast};
+
 impl Translator<'_> {
-    fn emit_else_branch(&mut self, node: ast::ElseBranch) -> Option<Label<generated::Expr>> {
+    fn emit_else_branch(&mut self, node: &ast::ElseBranch) -> Option<Label<generated::Expr>> {
         match node {
             ast::ElseBranch::IfExpr(inner) => self.emit_if_expr(inner).map(Into::into),
             ast::ElseBranch::Block(inner) => self.emit_block_expr(inner).map(Into::into),
         }
     }
     {{#enums}}
-¶
-    pub(crate) fn emit_{{snake_case_name}}(&mut self, node: ast::{{ast_name}}) -> Option<Label<generated::{{name}}>> {
-        match node {
+
+    pub(crate) fn emit_{{snake_case_name}}(&mut self, node: &ast::{{ast_name}}) -> Option<Label<generated::{{name}}>> {
+        let label = match node {
             {{#variants}}
             ast::{{ast_name}}::{{variant_ast_name}}(inner) => self.emit_{{snake_case_name}}(inner).map(Into::into),
             {{/variants}}
-        }
+        }?;
+        emit_detached!({{name}}, self, node, label);
+        Some(label)
     }
     {{/enums}}
     {{#nodes}}
-¶
-    pub(crate) fn emit_{{snake_case_name}}(&mut self, node: ast::{{ast_name}}) -> Option<Label<generated::{{name}}>> {
+
+    pub(crate) fn emit_{{snake_case_name}}(&mut self, node: &ast::{{ast_name}}) -> Option<Label<generated::{{name}}>> {
         {{#has_attrs}}
-        if self.should_be_excluded(&node) { return None; }
+        if self.should_be_excluded(node) { return None; }
         {{/has_attrs}}
         {{#fields}}
         {{#predicate}}
@@ -44,10 +45,10 @@ impl Translator<'_> {
         let {{name}} = node.try_get_text();
         {{/string}}
         {{#list}}
-        let {{name}} = node.{{method}}().filter_map(|x| self.emit_{{snake_case_ty}}(x)).collect();
+        let {{name}} = node.{{method}}().filter_map(|x| self.emit_{{snake_case_ty}}(&x)).collect();
         {{/list}}
         {{#optional}}
-        let {{name}} = node.{{method}}().and_then(|x| self.emit_{{snake_case_ty}}(x));
+        let {{name}} = node.{{method}}().and_then(|x| self.emit_{{snake_case_ty}}(&x));
         {{/optional}}
         {{/fields}}
         let label = self.trap.emit(generated::{{name}} {
@@ -56,9 +57,9 @@ impl Translator<'_> {
             {{name}},
             {{/fields}}
         });
-        self.emit_location(label, &node);
+        self.emit_location(label, node);
         emit_detached!({{name}}, self, node, label);
-        self.emit_tokens(&node, label.into(), node.syntax().children_with_tokens());
+        self.emit_tokens(node, label.into(), node.syntax().children_with_tokens());
         Some(label)
     }
     {{/nodes}}

rust/codegen/BUILD.bazel

@@ -7,6 +7,7 @@ _args = [
     "//rust/ast-generator:Cargo.toml",
     "//misc/codegen",
     "//rust:codegen-conf",
+    "@rules_rust//tools/rustfmt:upstream_rustfmt",
 ]
 
 sh_binary(

rust/codegen/codegen.sh

@@ -9,7 +9,9 @@ grammar_file="$(rlocation "$2")"
 ast_generator_manifest="$(rlocation "$3")"
 codegen="$(rlocation "$4")"
 codegen_conf="$(rlocation "$5")"
-shift 5
+rustfmt="$(rlocation "$6")"
+shift 6
 
 CARGO_MANIFEST_DIR="$(dirname "$ast_generator_manifest")" "$ast_generator" "$grammar_file"
+"$rustfmt" "$(dirname "$ast_generator_manifest")/../extractor/src/translate/generated.rs"
 "$codegen" --configuration-file="$codegen_conf" "$@"

rust/downgrades/f78cb8f2ab3bafb0d116cd8128f0315db24aea33/downgrade.ql

@@ -0,0 +1,7 @@
+class Element extends @element {
+  string toString() { none() }
+}
+
+query predicate new_macro_call_expandeds(Element id, Element expanded) {
+  item_expandeds(id, expanded) and macro_calls(id)
+}