[rush] cobuilds: yield priority to other tasks in the queue

common/changes/@microsoft/rush/sennyeya-fix-cobuild-ordering_2026-04-02-12-35.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "comment": "When cobuilds are remote executing, check them after locally executable tasks.",
+      "type": "none",
+      "packageName": "@microsoft/rush"
+    }
+  ],
+  "packageName": "@microsoft/rush",
+  "email": "aramissennyeydd@users.noreply.github.com"
+}

libraries/rush-lib/src/logic/operations/AsyncOperationQueue.ts

@@ -86,6 +86,19 @@ export class AsyncOperationQueue
     }
   }
 
+  /**
+   * Moves an operation to the back of the queue (lowest priority).
+   * Used when a cobuild lock acquisition fails, so that locally-executable operations are
+   * assigned before this operation is retried.
+   */
+  public yieldPriority(record: OperationExecutionRecord): void {
+    const index: number = this._queue.indexOf(record);
+    if (index > 0) {
+      this._queue.splice(index, 1);
+      this._queue.unshift(record);
+    }
+  }
+
   /**
    * Routes ready operations with 0 dependencies to waiting iterators. Normally invoked as part of `next()`, but
    * if the caller does not update operation dependencies prior to calling `next()`, may need to be invoked manually.

libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts

@@ -411,6 +411,7 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
               periodicCallback.start();
             } else {
               setTimeout(() => {
+                record.yieldPriority();
                 record.status = OperationStatus.Ready;
               }, 500);
               return OperationStatus.Executing;

libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts

@@ -148,6 +148,9 @@ export class OperationExecutionManager {
     const executionRecordContext: IOperationExecutionRecordContext = {
       streamCollator: this._streamCollator,
       onOperationStatusChanged: this._onOperationStatusChanged,
+      yieldPriority: (record: OperationExecutionRecord) => {
+        this._executionQueue.yieldPriority(record);
+      },
       createEnvironment: this._createEnvironmentForOperation,
       inputsSnapshot,
       debugMode,

libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts

@@ -42,6 +42,7 @@ import {
 export interface IOperationExecutionRecordContext {
   streamCollator: StreamCollator;
   onOperationStatusChanged?: (record: OperationExecutionRecord) => void;
+  yieldPriority?: (record: OperationExecutionRecord) => void;
   createEnvironment?: (record: OperationExecutionRecord) => IEnvironment;
   inputsSnapshot: IInputsSnapshot | undefined;
 
@@ -234,6 +235,14 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera
     return !this.operation.enabled || this.runner.silent;
   }
 
+  /**
+   * Moves this operation to the back of the execution queue so that other operations
+   * are assigned first.
+   */
+  public yieldPriority(): void {
+    this._context.yieldPriority?.(this);
+  }
+
   public getStateHash(): string {
     if (this._stateHash === undefined) {
       const components: readonly string[] = this.getStateHashComponents();

libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts

@@ -168,4 +168,77 @@ describe(AsyncOperationQueue.name, () => {
     const result: IteratorResult<OperationExecutionRecord> = await iterator.next();
     expect(result.done).toEqual(true);
   });
+
+  it('yieldPriority moves an operation to the back of the queue', async () => {
+    // Three independent operations: A, B, C (all Ready).
+    // A is assigned first. Calling yieldPriority + setting Ready should move it to the back,
+    // so B and C are assigned before A on the next pass.
+    const opA = createRecord('a');
+    const opB = createRecord('b');
+    const opC = createRecord('c');
+
+    const queue: AsyncOperationQueue = new AsyncOperationQueue([opA, opB, opC], nullSort);
+
+    // Assign one operation
+    const r1: IteratorResult<OperationExecutionRecord> = await queue.next();
+    const firstAssigned: OperationExecutionRecord = r1.value;
+
+    // Simulate cobuild retry: yield priority then re-ready the operation
+    queue.yieldPriority(firstAssigned);
+    firstAssigned.status = OperationStatus.Ready;
+
+    // Now assign the remaining — untried operations should come before the retry
+    const results: OperationExecutionRecord[] = [];
+    const r2: IteratorResult<OperationExecutionRecord> = await queue.next();
+    results.push(r2.value);
+    const r3: IteratorResult<OperationExecutionRecord> = await queue.next();
+    results.push(r3.value);
+    const r4: IteratorResult<OperationExecutionRecord> = await queue.next();
+    results.push(r4.value);
+
+    // The yielded operation should be last
+    expect(results[2]).toBe(firstAssigned);
+  });
+
+  it('yieldPriority assigns freshly unblocked operations first', async () => {
+    // A (no deps), B (depends on C), C (no deps)
+    // A is assigned and yields priority (cobuild retry).
+    // C completes, unblocking B. B should be assigned before A.
+    const opA = createRecord('a');
+    const opB = createRecord('b');
+    const opC = createRecord('c');
+
+    addDependency(opB, opC);
+
+    const queue: AsyncOperationQueue = new AsyncOperationQueue([opA, opB, opC], nullSort);
+
+    // Pull both initially ready operations (A and C)
+    const r1: IteratorResult<OperationExecutionRecord> = await queue.next();
+    const r2: IteratorResult<OperationExecutionRecord> = await queue.next();
+    expect(new Set([r1.value, r2.value])).toEqual(new Set([opA, opC]));
+
+    // Simulate: A fails cobuild lock
+    queue.yieldPriority(opA);
+    opA.status = OperationStatus.Ready;
+
+    // C succeeds, which unblocks B
+    opC.status = OperationStatus.Success;
+    queue.complete(opC);
+
+    // B is freshly unblocked, A yielded priority — B should be assigned first
+    const r3: IteratorResult<OperationExecutionRecord> = await queue.next();
+    expect(r3.value).toBe(opB);
+
+    const r4: IteratorResult<OperationExecutionRecord> = await queue.next();
+    expect(r4.value).toBe(opA);
+
+    // Complete remaining
+    opA.status = OperationStatus.Success;
+    queue.complete(opA);
+    opB.status = OperationStatus.Success;
+    queue.complete(opB);
+
+    const rEnd: IteratorResult<OperationExecutionRecord> = await queue.next();
+    expect(rEnd.done).toBe(true);
+  });
 });