GH-48623: [CI][Archery][Dev] Add missing headers to email reports

.github/workflows/cpp_extra.yml

@@ -39,6 +39,7 @@ on:
       - 'ci/scripts/util_*'
       - 'cpp/**'
       - 'compose.yaml'
+      - 'dev/archery/archery/**'
       - 'format/Flight.proto'
       - 'testing'
     tags:
@@ -61,6 +62,7 @@ on:
       - 'ci/scripts/util_*'
       - 'cpp/**'
       - 'compose.yaml'
+      - 'dev/archery/archery/**'
       - 'format/Flight.proto'
       - 'testing'
     types:

.github/workflows/package_linux.yml

@@ -29,6 +29,7 @@ on:
       - '.github/workflows/report_ci.yml'
       - 'cpp/**'
       - 'c_glib/**'
+      - 'dev/archery/archery/**'
       - 'dev/release/binary-task.rb'
       - 'dev/release/verify-apt.sh'
       - 'dev/release/verify-yum.sh'
@@ -43,6 +44,7 @@ on:
       - '.github/workflows/report_ci.yml'
       - 'cpp/**'
       - 'c_glib/**'
+      - 'dev/archery/archery/**'
       - 'dev/release/binary-task.rb'
       - 'dev/release/verify-apt.sh'
       - 'dev/release/verify-yum.sh'

.github/workflows/r_extra.yml

@@ -27,15 +27,16 @@ on:
       - '.github/workflows/check_labels.yml'
       - '.github/workflows/r_extra.yml'
       - '.github/workflows/report_ci.yml'
-      - "ci/docker/**"
-      - "ci/etc/rprofile"
-      - "ci/scripts/PKGBUILD"
-      - "ci/scripts/cpp_*.sh"
-      - "ci/scripts/install_minio.sh"
-      - "ci/scripts/r_*.sh"
-      - "cpp/**"
-      - "compose.yaml"
-      - "r/**"
+      - 'ci/docker/**'
+      - 'ci/etc/rprofile'
+      - 'ci/scripts/PKGBUILD'
+      - 'ci/scripts/cpp_*.sh'
+      - 'ci/scripts/install_minio.sh'
+      - 'ci/scripts/r_*.sh'
+      - 'cpp/**'
+      - 'compose.yaml'
+      - 'dev/archery/archery/**'
+      - 'r/**'
     tags:
       - '**'
   pull_request:
@@ -44,15 +45,16 @@ on:
       - '.github/workflows/check_labels.yml'
       - '.github/workflows/r_extra.yml'
       - '.github/workflows/report_ci.yml'
-      - "ci/docker/**"
-      - "ci/etc/rprofile"
-      - "ci/scripts/PKGBUILD"
-      - "ci/scripts/cpp_*.sh"
-      - "ci/scripts/install_minio.sh"
-      - "ci/scripts/r_*.sh"
-      - "cpp/**"
-      - "compose.yaml"
-      - "r/**"
+      - 'ci/docker/**'
+      - 'ci/etc/rprofile'
+      - 'ci/scripts/PKGBUILD'
+      - 'ci/scripts/cpp_*.sh'
+      - 'ci/scripts/install_minio.sh'
+      - 'ci/scripts/r_*.sh'
+      - 'cpp/**'
+      - 'compose.yaml'
+      - 'dev/archery/archery/**'
+      - 'r/**'
     types:
       - labeled
       - opened

dev/archery/archery/ci/cli.py

@@ -73,6 +73,22 @@ def report_chat(obj, workflow_id, send, repository, ignore, webhook,
         output.write(report_chat.render("workflow_report"))
 
 
+class WorkflowEmailReport(EmailReport):
+    def __init__(self, **kwargs):
+        super().__init__('workflow_report', **kwargs)
+
+    def date(self):
+        return self.report.datetime
+
+    def subject(self):
+        workflow = self.report
+        date = self.date().strftime('%Y-%m-%d')
+        return (
+            f'[{date}] Arrow Build Report for Job {workflow.name}: '
+            f'{len(workflow.failed_jobs())} failed'
+        )
+
+
 @ci.command()
 @click.argument('workflow_id', required=True)
 @click.option('--sender-name', '-n',
@@ -105,9 +121,10 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email,
     """
     output = obj['output']
 
-    email_report = EmailReport(
-        report=Workflow(workflow_id, repository,
-                        ignore_job=ignore, gh_token=obj['github_token']),
+    workflow = Workflow(workflow_id, repository,
+                        ignore_job=ignore, gh_token=obj['github_token'])
+    email_report = WorkflowEmailReport(
+        report=workflow,
         sender_name=sender_name,
         sender_email=sender_email,
         recipient_email=recipient_email
@@ -119,8 +136,7 @@ def report_email(obj, workflow_id, sender_name, sender_email, recipient_email,
             smtp_password=smtp_password,
             smtp_server=smtp_server,
             smtp_port=smtp_port,
-            recipient_email=recipient_email,
-            message=email_report.render("workflow_report")
+            report=email_report
         )
     else:
-        output.write(email_report.render("workflow_report"))
+        output.write(str(email_report.render()))

dev/archery/archery/crossbow/cli.py

@@ -343,6 +343,22 @@ def latest_prefix(obj, prefix, fetch):
     click.echo(latest.branch)
 
 
+class NightlyEmailReport(EmailReport):
+    def __init__(self, **kwargs):
+        super().__init__('nightly_report', **kwargs)
+
+    def subject(self):
+        report = self.report
+        n_errors = len(report.tasks_by_state['error'])
+        n_failures = len(report.tasks_by_state['failure'])
+        n_pendings = len(report.tasks_by_state['pending'])
+        return (
+            f'[NIGHTLY] Arrow Build Report for Job {report.job.branch}: '
+            f'{n_errors + n_failures} failed, '
+            f'{n_pendings} pending'
+        )
+
+
 @crossbow.command()
 @click.argument('job-name', required=True)
 @click.option('--sender-name', '-n',
@@ -382,8 +398,9 @@ def report(obj, job_name, sender_name, sender_email, recipient_email,
         queue.fetch()
 
     job = queue.get(job_name)
-    email_report = EmailReport(
-        report=Report(job),
+    report = Report(job)
+    email_report = NightlyEmailReport(
+        report=report,
         sender_name=sender_name,
         sender_email=sender_email,
         recipient_email=recipient_email
@@ -401,11 +418,10 @@ def report(obj, job_name, sender_name, sender_email, recipient_email,
             smtp_password=smtp_password,
             smtp_server=smtp_server,
             smtp_port=smtp_port,
-            recipient_email=recipient_email,
-            message=email_report.render("nightly_report")
+            report=email_report
         )
     else:
-        output.write(email_report.render("nightly_report"))
+        output.write(str(email_report.render()))
 
 
 @crossbow.command()
@@ -601,6 +617,17 @@ def batch_gen(iterable, step):
             print(batch)
 
 
+class TokenExpirationEmailReport(EmailReport):
+    def __init__(self, **kwargs):
+        super().__init__('token_expiration', **kwargs)
+
+    def subject(self):
+        token_expiration_date = self.report.token_expiration_date
+        return (
+            f'[CI] Arrow Crossbow Token Expiration in {token_expiration_date}'
+        )
+
+
 @crossbow.command()
 @click.option('--days', default=30,
               help='Notification will be sent if expiration date is '
@@ -645,23 +672,18 @@ def __init__(self, token_expiration_date, days_left):
             self.token_expiration_date = token_expiration_date
             self.days_left = days_left
 
-    email_report = EmailReport(
-        report=TokenExpirationReport(
-            token_expiration_date or "ALREADY_EXPIRED", days_left),
-        sender_name=sender_name,
-        sender_email=sender_email,
-        recipient_email=recipient_email
-    )
+    if not token_expiration_date:
+        token_expiration_date = 'ALREADY_EXPIRED'
+    report = TokenExpirationReport(token_expiration_date, days_left)
+    email_report = TokenExpirationEmailReport(report)
 
-    message = email_report.render("token_expiration").strip()
     if send:
         ReportUtils.send_email(
             smtp_user=smtp_user,
             smtp_password=smtp_password,
             smtp_server=smtp_server,
             smtp_port=smtp_port,
-            recipient_email=recipient_email,
-            message=message
+            report=email_report
         )
     else:
-        output.write(message)
+        output.write(str(email_report.render()))

dev/archery/archery/crossbow/reports.py

@@ -17,6 +17,10 @@
 
 import collections
 import csv
+import datetime
+import email.headerregistry
+import email.message
+import email.utils
 import operator
 import fnmatch
 import functools
@@ -246,7 +250,7 @@ def send_message(cls, webhook, message):
 
     @classmethod
     def send_email(cls, smtp_user, smtp_password, smtp_server, smtp_port,
-                   recipient_email, message):
+                   report):
         from smtplib import SMTP, SMTP_SSL
 
         if smtp_port == 465:
@@ -259,7 +263,8 @@ def send_email(cls, smtp_user, smtp_password, smtp_server, smtp_port,
             else:
                 smtp.starttls()
             smtp.login(smtp_user, smtp_password)
-            smtp.sendmail(smtp_user, recipient_email, message)
+            message = report.render()
+            smtp.send_message(smtp_user, report.recipient_email, message)
 
     @classmethod
     def write_csv(cls, report, add_headers=True):
@@ -271,18 +276,42 @@ def write_csv(cls, report, add_headers=True):
 
 
 class EmailReport(JinjaReport):
-    templates = {
-        'nightly_report': 'email_nightly_report.txt.j2',
-        'token_expiration': 'email_token_expiration.txt.j2',
-        'workflow_report': 'email_workflow_report.txt.j2',
-    }
     fields = [
         'report',
         'sender_name',
         'sender_email',
         'recipient_email',
     ]
 
+    def __init__(self, template_name, **kwargs):
+        self._template_name = template_name
+        super().__init__(**kwargs)
+
+    @property
+    def templates(self):
+        return {
+            self._template_name: f'email_{self._template_name}.txt.j2',
+        }
+
+    def date(self):
+        return None
+
+    def render(self):
+        message = email.message.EmailMessage()
+        message.set_charset('utf-8')
+        message['Message-Id'] = email.utils.make_msgid()
+        date = self.date()
+        if isinstance(date, datetime.datetime):
+            message['Date'] = date
+        else:
+            message['Date'] = email.utils.formatdate(date)
+        message['From'] = email.headerregistry.Address(
+            self.sender_name, addr_spec=self.sender_email)
+        message['To'] = email.headerregistry.Address(addr_spec=self.recipient_email)
+        message['Subject'] = self.subject()
+        message.set_content(super().render(self._template_name))
+        return message
+
 
 class CommentReport(Report):
 

dev/archery/archery/crossbow/tests/fixtures/email-report.txt → dev/archery/archery/crossbow/tests/fixtures/nightly-email-report.txt

@@ -1,6 +1,11 @@
+MIME-Version: 1.0
+Message-Id: <message-id>
+Date: date
 From: Sender Reporter <sender@arrow.com>
 To: recipient@arrow.com
 Subject: [NIGHTLY] Arrow Build Report for Job ursabot-1: 2 failed, 1 pending
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 7bit
 
 Arrow Build Report for Job ursabot-1
 

dev/archery/archery/crossbow/tests/fixtures/token-expiration-email-report.txt

@@ -0,0 +1,14 @@
+MIME-Version: 1.0
+Message-Id: <message-id>
+Date: date
+From: Sender Reporter <sender@arrow.com>
+To: recipient@arrow.com
+Subject: [CI] Arrow Crossbow Token Expiration in 2026-01-17
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 7bit
+
+The Arrow Crossbow Token will expire in 7 days.
+
+Please generate a new Token. Send it to Apache INFRA to update the
+CROSSBOW_GITHUB_TOKEN. Update it on the crossbow repository and in
+the Azure pipelines.