Implement PEP 492: async/await support

eng/scripts/generate_exceptions.py

@@ -85,6 +85,7 @@ def MakeNewException(self):
         ExceptionInfo('Exception', 'IronPython.Runtime.Exceptions.PythonException', None, (), (
                 ExceptionInfo('StopIteration', 'IronPython.Runtime.Exceptions.StopIterationException', None, ('value',), ()),
                 ExceptionInfo('StopAsyncIteration', 'IronPython.Runtime.Exceptions.StopAsyncIterationException', None, ('value',), ()),
+                ExceptionInfo('CancelledError', 'System.OperationCanceledException', None, (), ()),
                 ExceptionInfo('ArithmeticError', 'System.ArithmeticException', None, (), (
                         ExceptionInfo('FloatingPointError', 'IronPython.Runtime.Exceptions.FloatingPointException', None, (), ()),
                         ExceptionInfo('OverflowError', 'System.OverflowException', None, (), ()),
@@ -261,16 +262,26 @@ def gen_topython_helper(cw):
     cw.exit_block()
 
 
+_clr_name_overrides = {
+    'CancelledError': 'OperationCanceledException',
+}
+
 def get_clr_name(e):
+    if e in _clr_name_overrides:
+        return _clr_name_overrides[e]
     return e.replace('Error', '') + 'Exception'
 
 FACTORY = """
 internal static Exception %(name)s(string message) => new %(clrname)s(message);
 public static Exception %(name)s(string format, params object?[] args) => new %(clrname)s(string.Format(format, args));
 """.rstrip()
 
+# Exceptions that map to existing CLR types (no generated CLR class needed),
+# but still need factory methods in PythonOps.
+_factory_only_exceptions = ['CancelledError']
+
 def factory_gen(cw):
-    for e in pythonExcs:
+    for e in pythonExcs + _factory_only_exceptions:
         cw.write(FACTORY, name=e, clrname=get_clr_name(e))
 
 CLASS1 = """\

eng/scripts/generate_ops.py

@@ -10,7 +10,7 @@
 kwlist = [
     'and', 'assert', 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except',
     'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or', 'pass',
-    'raise', 'return', 'try', 'while', 'yield', 'as', 'with', 'async', 'nonlocal'
+    'raise', 'return', 'try', 'while', 'yield', 'as', 'with', 'async', 'nonlocal', 'await'
 ]
 
 class Symbol:

src/core/IronPython.StdLib/lib/test/exception_hierarchy.txt

@@ -5,6 +5,7 @@ BaseException
  +-- Exception
       +-- StopIteration
       +-- StopAsyncIteration
+      +-- CancelledError
       +-- ArithmeticError
       |    +-- FloatingPointError
       |    +-- OverflowError

src/core/IronPython/Compiler/Ast/AstMethods.cs

@@ -79,6 +79,7 @@ internal static class AstMethods {
         public static readonly MethodInfo PushFrame = GetMethod((Func<CodeContext, FunctionCode, List<FunctionStack>>)PythonOps.PushFrame);
         public static readonly MethodInfo FormatString = GetMethod((Func<CodeContext, string, object, string>)PythonOps.FormatString);
         public static readonly MethodInfo GeneratorCheckThrowableAndReturnSendValue = GetMethod((Func<object, object>)PythonOps.GeneratorCheckThrowableAndReturnSendValue);
+        public static readonly MethodInfo MakeCoroutine = GetMethod((Func<PythonFunction, MutableTuple, object, PythonCoroutine>)PythonOps.MakeCoroutine);
 
         // builtins
         public static readonly MethodInfo Format = GetMethod((Func<CodeContext, object, string, string>)PythonOps.Format);

src/core/IronPython/Compiler/Ast/AsyncForStatement.cs

@@ -0,0 +1,146 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+using System.Threading;
+
+using Microsoft.Scripting;
+using MSAst = System.Linq.Expressions;
+
+namespace IronPython.Compiler.Ast {
+
+    /// <summary>
+    /// Represents an async for statement.
+    /// Desugared to Python AST that uses __aiter__ and await __anext__().
+    /// </summary>
+    public class AsyncForStatement : Statement, ILoopStatement {
+        private static int _counter;
+        private Statement? _desugared;
+
+        public AsyncForStatement(Expression left, Expression list, Statement body, Statement? @else) {
+            Left = left;
+            List = list;
+            Body = body;
+            Else = @else;
+        }
+
+        public int HeaderIndex { private get; set; }
+
+        public Expression Left { get; }
+
+        public Expression List { get; set; }
+
+        public Statement Body { get; set; }
+
+        public Statement? Else { get; }
+
+        MSAst.LabelTarget ILoopStatement.BreakLabel { get; set; } = null!;
+
+        MSAst.LabelTarget ILoopStatement.ContinueLabel { get; set; } = null!;
+
+        /// <summary>
+        /// Build the desugared tree. Called during Walk when Parent and IndexSpan are available.
+        /// </summary>
+        private Statement BuildDesugared() {
+            var parent = Parent;
+            var span = IndexSpan;
+            var id = Interlocked.Increment(ref _counter);
+
+            // async for TARGET in ITER:
+            //     BLOCK
+            // else:
+            //     ELSE_BLOCK
+            //
+            // desugars to:
+            //
+            // __aiter = ITER.__aiter__()
+            // __running = True
+            // while __running:
+            //     try:
+            //         TARGET = await __aiter.__anext__()
+            //     except StopAsyncIteration:
+            //         __running = False
+            //     else:
+            //         BLOCK
+            // else:
+            //     ELSE_BLOCK
+
+            var iterName = $"__asyncfor_iter{id}";
+            var runningName = $"__asyncfor_running{id}";
+
+            // Helper to create nodes with proper parent and span
+            NameExpression MakeName(string name) {
+                var n = new NameExpression(name) { Parent = parent };
+                n.IndexSpan = span;
+                return n;
+            }
+
+            T WithSpan<T>(T node) where T : Node {
+                node.IndexSpan = span;
+                return node;
+            }
+
+            // _iter = ITER.__aiter__()
+            var aiterAttr = WithSpan(new MemberExpression(List, "__aiter__") { Parent = parent });
+            var aiterCall = WithSpan(new CallExpression(aiterAttr, null, null) { Parent = parent });
+            var assignIter = WithSpan(new AssignmentStatement(new Expression[] { MakeName(iterName) }, aiterCall) { Parent = parent });
+
+            // running = True
+            var trueConst = new ConstantExpression(true) { Parent = parent }; trueConst.IndexSpan = span;
+            var assignRunning = WithSpan(new AssignmentStatement(new Expression[] { MakeName(runningName) }, trueConst) { Parent = parent });
+
+            // TARGET = await __aiter.__anext__()
+            var anextAttr = WithSpan(new MemberExpression(MakeName(iterName), "__anext__") { Parent = parent });
+            var anextCall = WithSpan(new CallExpression(anextAttr, null, null) { Parent = parent });
+            var awaitNext = new AwaitExpression(anextCall);
+            var assignTarget = WithSpan(new AssignmentStatement(new Expression[] { Left }, awaitNext) { Parent = parent });
+
+            // except StopAsyncIteration: __running = False
+            var falseConst = new ConstantExpression(false) { Parent = parent }; falseConst.IndexSpan = span;
+            var stopRunning = WithSpan(new AssignmentStatement(
+                new Expression[] { MakeName(runningName) }, falseConst) { Parent = parent });
+            var handler = WithSpan(new TryStatementHandler(
+                MakeName("StopAsyncIteration"),
+                null!,
+                WithSpan(new SuiteStatement(new Statement[] { stopRunning }) { Parent = parent })
+            ) { Parent = parent });
+            handler.HeaderIndex = span.End;
+
+            // try/except/else block
+            var tryExcept = WithSpan(new TryStatement(
+                assignTarget,
+                new[] { handler },
+                WithSpan(new SuiteStatement(new Statement[] { Body }) { Parent = parent }),
+                null!
+            ) { Parent = parent });
+            tryExcept.HeaderIndex = span.End;
+
+            // while __running: try/except/else
+            var whileStmt = new WhileStatement(MakeName(runningName), tryExcept, Else);
+            whileStmt.SetLoc(GlobalParent, span.Start, span.End, span.End);
+            whileStmt.Parent = parent;
+
+            var suite = WithSpan(new SuiteStatement(new Statement[] { assignIter, assignRunning, whileStmt }) { Parent = parent });
+            return suite;
+        }
+
+        public override MSAst.Expression Reduce() {
+            return _desugared!.Reduce();
+        }
+
+        public override void Walk(PythonWalker walker) {
+            if (walker.Walk(this)) {
+                // Build the desugared tree on first walk (when Parent and IndexSpan are set)
+                if (_desugared == null) {
+                    _desugared = BuildDesugared();
+                }
+                _desugared.Walk(walker);
+            }
+            walker.PostWalk(this);
+        }
+
+        internal override bool CanThrow => true;
+    }
+}

src/core/IronPython/Compiler/Ast/AsyncWithStatement.cs

@@ -0,0 +1,123 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+using Microsoft.Scripting;
+using MSAst = System.Linq.Expressions;
+
+using AstUtils = Microsoft.Scripting.Ast.Utils;
+
+namespace IronPython.Compiler.Ast {
+    using Ast = MSAst.Expression;
+
+    /// <summary>
+    /// Represents an async with statement.
+    /// Desugared to Python AST that uses await on __aenter__ and __aexit__.
+    /// </summary>
+    public class AsyncWithStatement : Statement {
+        private Statement? _desugared;
+
+        public AsyncWithStatement(Expression contextManager, Expression? var, Statement body) {
+            ContextManager = contextManager;
+            Variable = var;
+            Body = body;
+        }
+
+        public int HeaderIndex { private get; set; }
+
+        public Expression ContextManager { get; }
+
+        public new Expression? Variable { get; }
+
+        public Statement Body { get; }
+
+        /// <summary>
+        /// Build the desugared tree. Called during Walk when Parent and IndexSpan are available.
+        /// </summary>
+        private Statement BuildDesugared() {
+            var parent = Parent;
+            var span = IndexSpan;
+
+            // async with EXPR as VAR:
+            //     BLOCK
+            //
+            // desugars to:
+            //
+            // mgr = EXPR
+            // try:
+            //     VAR = await mgr.__aenter__()  (or just await mgr.__aenter__())
+            //     BLOCK
+            // finally:
+            //     await mgr.__aexit__(None, None, None)
+
+            // Helper to create nodes with proper parent and span
+            NameExpression MakeName(string name) {
+                var n = new NameExpression(name) { Parent = parent };
+                n.IndexSpan = span;
+                return n;
+            }
+
+            // mgr = EXPR
+            var assignMgr = new AssignmentStatement(new Expression[] { MakeName("__asyncwith_mgr") }, ContextManager) { Parent = parent };
+            assignMgr.IndexSpan = span;
+
+            // await mgr.__aenter__()
+            var aenterAttr = new MemberExpression(MakeName("__asyncwith_mgr"), "__aenter__") { Parent = parent };
+            aenterAttr.IndexSpan = span;
+            var aenterCall = new CallExpression(aenterAttr, null, null) { Parent = parent };
+            aenterCall.IndexSpan = span;
+            var awaitEnter = new AwaitExpression(aenterCall);
+
+            Statement bodyStmt;
+            if (Variable != null) {
+                // VAR = await value; BLOCK
+                var assignVar = new AssignmentStatement(new Expression[] { Variable }, awaitEnter) { Parent = parent };
+                assignVar.IndexSpan = span;
+                bodyStmt = new SuiteStatement(new Statement[] { assignVar, Body }) { Parent = parent };
+            } else {
+                var exprStmt = new ExpressionStatement(awaitEnter) { Parent = parent };
+                exprStmt.IndexSpan = span;
+                bodyStmt = new SuiteStatement(new Statement[] { exprStmt, Body }) { Parent = parent };
+            }
+
+            // await mgr.__aexit__(None, None, None)
+            var aexitAttr = new MemberExpression(MakeName("__asyncwith_mgr"), "__aexit__") { Parent = parent };
+            aexitAttr.IndexSpan = span;
+            var none1 = new ConstantExpression(null) { Parent = parent }; none1.IndexSpan = span;
+            var none2 = new ConstantExpression(null) { Parent = parent }; none2.IndexSpan = span;
+            var none3 = new ConstantExpression(null) { Parent = parent }; none3.IndexSpan = span;
+            var aexitCallNormal = new CallExpression(aexitAttr,
+                new Expression[] { none1, none2, none3 }, null) { Parent = parent };
+            aexitCallNormal.IndexSpan = span;
+            var awaitExitNormal = new AwaitExpression(aexitCallNormal);
+
+            // try/finally: await __aexit__ on normal exit
+            var finallyExprStmt = new ExpressionStatement(awaitExitNormal) { Parent = parent };
+            finallyExprStmt.IndexSpan = span;
+            var tryFinally = new TryStatement(bodyStmt, null, null, finallyExprStmt) { Parent = parent };
+            tryFinally.IndexSpan = span;
+            tryFinally.HeaderIndex = span.End;
+
+            var suite = new SuiteStatement(new Statement[] { assignMgr, tryFinally }) { Parent = parent };
+            suite.IndexSpan = span;
+            return suite;
+        }
+
+        public override MSAst.Expression Reduce() {
+            return _desugared!.Reduce();
+        }
+
+        public override void Walk(PythonWalker walker) {
+            if (walker.Walk(this)) {
+                // Build the desugared tree on first walk (when Parent and IndexSpan are set)
+                if (_desugared == null) {
+                    _desugared = BuildDesugared();
+                }
+                _desugared.Walk(walker);
+            }
+            walker.PostWalk(this);
+        }
+    }
+}

src/core/IronPython/Compiler/Ast/AwaitExpression.cs

@@ -0,0 +1,66 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+using MSAst = System.Linq.Expressions;
+
+using AstUtils = Microsoft.Scripting.Ast.Utils;
+
+namespace IronPython.Compiler.Ast {
+    using Ast = MSAst.Expression;
+
+    /// <summary>
+    /// Represents an await expression. Implemented as yield from expr.__await__().
+    /// </summary>
+    public class AwaitExpression : Expression {
+        private readonly Statement _statement;
+        private readonly NameExpression _result;
+
+        public AwaitExpression(Expression expression) {
+            Expression = expression;
+
+            // await expr is equivalent to yield from expr.__await__()
+            // We build: __awaitprefix_EXPR = expr; yield from __awaitprefix_EXPR.__await__(); __awaitprefix_r = __yieldfromprefix_r
+            var parent = expression.Parent;
+
+            var awaitableExpr = new NameExpression("__awaitprefix_EXPR") { Parent = parent };
+            var getAwait = new MemberExpression(awaitableExpr, "__await__") { Parent = parent };
+            var callAwait = new CallExpression(getAwait, null, null) { Parent = parent };
+            var yieldFrom = new YieldFromExpression(callAwait);
+
+            Statement s1 = new AssignmentStatement(new Expression[] { new NameExpression("__awaitprefix_EXPR") { Parent = parent } }, expression) { Parent = parent };
+            Statement s2 = new ExpressionStatement(yieldFrom) { Parent = parent };
+            Statement s3 = new AssignmentStatement(
+                new Expression[] { new NameExpression("__awaitprefix_r") { Parent = parent } },
+                new NameExpression("__yieldfromprefix_r") { Parent = parent }
+            ) { Parent = parent };
+
+            _statement = new SuiteStatement(new Statement[] { s1, s2, s3 }) { Parent = parent };
+
+            _result = new NameExpression("__awaitprefix_r") { Parent = parent };
+        }
+
+        public Expression Expression { get; }
+
+        public override MSAst.Expression Reduce() {
+            return Ast.Block(
+                typeof(object),
+                _statement,
+                AstUtils.Convert(_result, typeof(object))
+            ).Reduce();
+        }
+
+        public override void Walk(PythonWalker walker) {
+            if (walker.Walk(this)) {
+                Expression?.Walk(walker);
+                _statement.Walk(walker);
+                _result.Walk(walker);
+            }
+            walker.PostWalk(this);
+        }
+
+        public override string NodeName => "await expression";
+    }
+}