Refactor/update message sendin

composer.json

@@ -79,7 +79,11 @@
         "ext-imap": "*",
         "tatevikgr/rss-feed": "dev-main",
         "ext-pdo": "*",
-        "ezyang/htmlpurifier": "^4.19"
+        "ezyang/htmlpurifier": "^4.19",
+        "ext-libxml": "*",
+        "ext-gd": "*",
+        "ext-curl": "*",
+        "ext-fileinfo": "*"
     },
     "require-dev": {
         "phpunit/phpunit": "^9.5",

config/PHPMD/rules.xml

@@ -7,7 +7,7 @@
     <exclude-pattern>*/Migrations/*</exclude-pattern>
 
     <!-- rules from the "clean code" rule set -->
-    <rule ref="rulesets/cleancode.xml/BooleanArgumentFlag"/>
+<!--    <rule ref="rulesets/cleancode.xml/BooleanArgumentFlag"/>-->
     <rule ref="rulesets/codesize.xml/CyclomaticComplexity"/>
     <rule ref="rulesets/codesize.xml/NPathComplexity"/>
     <rule ref="rulesets/codesize.xml/ExcessiveMethodLength"/>
@@ -41,7 +41,7 @@
     <!-- rules from the "naming" rule set -->
     <rule ref="rulesets/naming.xml/ShortVariable">
         <properties>
-            <property name="exceptions" value="id,ip,cc,io"/>
+            <property name="exceptions" value="id,ip,cc,io,to"/>
         </properties>
     </rule>
     <rule ref="rulesets/naming.xml/LongVariable">

config/PhpCodeSniffer/ruleset.xml

@@ -54,6 +54,11 @@
     <!-- Functions -->
     <rule ref="Squiz.Functions.FunctionDuplicateArgument"/>
     <rule ref="Squiz.Functions.GlobalFunction"/>
+    <rule ref="Squiz.WhiteSpace.OperatorSpacing">
+        <properties>
+            <property name="ignoreNewlines" value="true"/>
+        </properties>
+    </rule>
 
     <!-- Metrics -->
     <rule ref="Generic.Metrics.CyclomaticComplexity"/>

config/parameters.yml.dist

@@ -25,12 +25,18 @@ parameters:
     env(DATABASE_PREFIX): 'phplist_'
     list_table_prefix: '%%env(LIST_TABLE_PREFIX)%%'
     env(LIST_TABLE_PREFIX): 'listattr_'
+    app.dev_version: '%%env(APP_DEV_VERSION)%%'
+    env(APP_DEV_VERSION): '0'
+    app.dev_email: '%%env(APP_DEV_EMAIL)%%'
+    env(APP_DEV_EMAIL): 'dev@dev.com'
+    app.powered_by_phplist: '%%env(APP_POWERED_BY_PHPLIST)%%'
+    env(APP_POWERED_BY_PHPLIST): 0
 
     # Email configuration
     app.mailer_from: '%%env(MAILER_FROM)%%'
     env(MAILER_FROM): 'noreply@phplist.com'
     app.mailer_dsn: '%%env(MAILER_DSN)%%'
-    env(MAILER_DSN): 'null://null'
+    env(MAILER_DSN): 'null://null' # set local_domain on transport
     app.confirmation_url: '%%env(CONFIRMATION_URL)%%'
     env(CONFIRMATION_URL): 'https://example.com/subscriber/confirm/'
     app.subscription_confirmation_url: '%%env(SUBSCRIPTION_CONFIRMATION_URL)%%'
@@ -89,3 +95,30 @@ parameters:
     env(MESSAGING_MAX_PROCESS_TIME): '600'
     messaging.max_mail_size: '%%env(MAX_MAILSIZE)%%'
     env(MAX_MAILSIZE): '209715200'
+    messaging.default_message_age: '%%env(DEFAULT_MESSAGEAGE)%%'
+    env(DEFAULT_MESSAGEAGE): '691200'
+    messaging.use_manual_text_part: '%%env(USE_MANUAL_TEXT_PART)%%'
+    env(USE_MANUAL_TEXT_PART): 0
+    messaging.blacklist_grace_time: '%%env(MESSAGING_BLACKLIST_GRACE_TIME)%%'
+    env(MESSAGING_BLACKLIST_GRACE_TIME): '600'
+    messaging.google_sender_id: '%%env(GOOGLE_SENDERID)%%'
+    env(GOOGLE_SENDERID): ''
+    messaging.use_amazon_ses: '%%env(USE_AMAZONSES)%%'
+    env(USE_AMAZONSES): 0
+    messaging.embed_external_images: '%%env(EMBEDEXTERNALIMAGES)%%'
+    env(EMBEDEXTERNALIMAGES): 0
+    messaging.embed_uploaded_images: '%%env(EMBEDUPLOADIMAGES)%%'
+    env(EMBEDUPLOADIMAGES): 0
+    messaging.external_image_max_age: '%%env(EXTERNALIMAGE_MAXAGE)%%'
+    env(EXTERNALIMAGE_MAXAGE): 0
+    messaging.external_image_timeout: '%%env(EXTERNALIMAGE_TIMEOUT)%%'
+    env(EXTERNALIMAGE_TIMEOUT): 30
+    messaging.external_image_max_size: '%%env(EXTERNALIMAGE_MAXSIZE)%%'
+    env(EXTERNALIMAGE_MAXSIZE): 204800
+
+    phplist.upload_images_dir: '%%env(PHPLIST_UPLOADIMAGES_DIR)%%'
+    env(PHPLIST_UPLOADIMAGES_DIR): 'images'
+    phplist.editor_images_dir: '%%env(FCKIMAGES_DIR)%%'
+    env(FCKIMAGES_DIR): 'uploadimages'
+    phplist.public_schema: '%%env(PUBLIC_SCHEMA)%%'
+    env(PUBLIC_SCHEMA): 'https'

resources/translations/messages.en.xlf

@@ -738,6 +738,14 @@ Thank you.</target>
         <source>Value must be an AttributeTypeEnum or string.</source>
         <target>__Value must be an AttributeTypeEnum or string.</target>
       </trans-unit>
+      <trans-unit id="1cRuI31" resname="Campaign started">
+        <source>Campaign started</source>
+        <target>__Campaign started</target>
+      </trans-unit>
+      <trans-unit id="Y4qXXak" resname="phplist has started sending the campaign with subject %s">
+        <source>phplist has started sending the campaign with subject %s</source>
+        <target>__phplist has started sending the campaign with subject %s</target>
+      </trans-unit>
     </body>
   </file>
 </xliff>

src/Bounce/Service/LockService.php

@@ -34,9 +34,6 @@ public function __construct(
         $this->maxWaitCycles = $maxWaitCycles;
     }
 
-    /**
-     * @SuppressWarnings("BooleanArgumentFlag")
-     */
     public function acquirePageLock(
         string $page,
         bool $force = false,

src/Domain/Analytics/Service/LinkTrackService.php

@@ -8,8 +8,7 @@
 use PhpList\Core\Domain\Analytics\Exception\MissingMessageIdException;
 use PhpList\Core\Domain\Analytics\Model\LinkTrack;
 use PhpList\Core\Domain\Analytics\Repository\LinkTrackRepository;
-use PhpList\Core\Domain\Messaging\Model\Message;
-use PhpList\Core\Domain\Messaging\Model\Message\MessageContent;
+use PhpList\Core\Domain\Messaging\Model\Dto\MessagePrecacheDto;
 
 class LinkTrackService
 {
@@ -39,7 +38,7 @@ public function isExtractAndSaveLinksApplicable(): bool
      * @return LinkTrack[] The saved LinkTrack entities
      * @throws MissingMessageIdException
      */
-    public function extractAndSaveLinks(MessageContent $content, int $userId, ?int $messageId = null): array
+    public function extractAndSaveLinks(MessagePrecacheDto $content, int $userId, ?int $messageId = null): array
     {
         if (!$this->isExtractAndSaveLinksApplicable()) {
             return [];
@@ -49,10 +48,10 @@ public function extractAndSaveLinks(MessageContent $content, int $userId, ?int $
             throw new MissingMessageIdException();
         }
 
-        $links = $this->extractLinksFromHtml($content->getText() ?? '');
+        $links = $this->extractLinksFromHtml($content->content ?? '');
 
-        if ($content->getFooter() !== null) {
-            $links = array_merge($links, $this->extractLinksFromHtml($content->getFooter()));
+        if ($content->htmlFooter) {
+            $links = array_merge($links, $this->extractLinksFromHtml($content->htmlFooter));
         }
 
         $links = array_unique($links);

src/Domain/Common/ExternalImageService.php

@@ -0,0 +1,229 @@
+<?php
+
+declare(strict_types=1);
+
+namespace PhpList\Core\Domain\Common;
+
+use PhpList\Core\Domain\Configuration\Model\ConfigOption;
+use PhpList\Core\Domain\Configuration\Service\Provider\ConfigProvider;
+use Psr\Log\LoggerInterface;
+use Throwable;
+
+class ExternalImageService
+{
+    private string $externalCacheDir;
+
+    public function __construct(
+        private readonly ConfigProvider $configProvider,
+        private readonly LoggerInterface  $logger,
+        private readonly string $tempDir,
+        private readonly int $externalImageMaxAge,
+        private readonly int $externalImageMaxSize,
+        private readonly ?int $externalImageTimeout = 30,
+    ) {
+        $this->externalCacheDir = $this->tempDir . '/external_cache';
+    }
+
+    public function getFromCache(string $filename, int $messageId): ?string
+    {
+        $cacheFile = $this->generateLocalFileName($filename, $messageId);
+
+        if (!is_file($cacheFile) || filesize($cacheFile) <= 64) {
+            return null;
+        }
+
+        $content = file_get_contents($cacheFile);
+        if ($content === false) {
+            return null;
+        }
+
+        return base64_encode($content);
+    }
+
+    public function cache($filename, $messageId): bool
+    {
+        if (!$this->isCacheableUrl($filename)) {
+            return false;
+        }
+
+        if (!$this->ensureCacheDirectory()) {
+            return false;
+        }
+
+        $this->removeOldFilesInCache();
+
+        $cacheFileName = $this->generateLocalFileName($filename, $messageId);
+
+        if (!file_exists($cacheFileName)) {
+            $cacheFileContent = null;
+
+            if (function_exists('curl_init')) {
+                $cacheFileContent = $this->downloadUsingCurl($filename);
+            }
+
+            if ($cacheFileContent === null) {
+                $cacheFileContent = $this->downloadUsingFileGetContent($filename);
+            }
+
+            if ($this->externalImageMaxSize && (strlen($cacheFileContent) > $this->externalImageMaxSize)) {
+                $cacheFileContent = 'MAX_SIZE';
+            }
+
+            $this->writeCacheFile($cacheFileName, $cacheFileContent);
+        }
+
+        return $this->isValidCacheFile($cacheFileName);
+    }
+
+    private function removeOldFilesInCache(): void
+    {
+        // phpcs:ignore Generic.PHP.NoSilencedErrors
+        $extCacheDirHandle = @opendir($this->externalCacheDir);
+        if (!$this->externalImageMaxAge || !$extCacheDirHandle) {
+            return;
+        }
+
+        while (true) {
+            // phpcs:ignore Generic.PHP.NoSilencedErrors
+            $cacheFile = @readdir($extCacheDirHandle);
+
+            if ($cacheFile === false) {
+                break;
+            }
+            // todo: make sure that this is what we need
+            if (!str_starts_with($cacheFile, '.')) {
+                // phpcs:ignore Generic.PHP.NoSilencedErrors
+                $cfmt = @filemtime($this->externalCacheDir . '/' . $cacheFile);
+
+                if (is_numeric($cfmt) && ($cfmt > 0) && ((time() - $cfmt) > $this->externalImageMaxAge)) {
+                    // phpcs:ignore Generic.PHP.NoSilencedErrors
+                    @unlink($this->externalCacheDir . '/' . $cacheFile);
+                }
+            }
+        }
+        // phpcs:ignore Generic.PHP.NoSilencedErrors
+        @closedir($extCacheDirHandle);
+    }
+
+    private function generateLocalFileName(string $filename, int $messageId): string
+    {
+        return $this->externalCacheDir
+            . '/'
+            . $messageId
+            . '_'
+            . preg_replace([ '~[\.][\.]+~Ui', '~[^\w\.]~Ui',], ['', '_'], $filename);
+    }
+
+    private function downloadUsingCurl(string $filename): ?string
+    {
+        $cURLHandle = curl_init($filename);
+
+        if ($cURLHandle !== false) {
+            curl_setopt($cURLHandle, CURLOPT_HTTPGET, true);
+            curl_setopt($cURLHandle, CURLOPT_HEADER, 0);
+            curl_setopt($cURLHandle, CURLOPT_RETURNTRANSFER, true);
+            curl_setopt($cURLHandle, CURLOPT_TIMEOUT, $this->externalImageTimeout);
+            curl_setopt($cURLHandle, CURLOPT_FOLLOWLOCATION, true);
+            curl_setopt($cURLHandle, CURLOPT_MAXREDIRS, 10);
+            curl_setopt($cURLHandle, CURLOPT_SSL_VERIFYPEER, false);
+            curl_setopt($cURLHandle, CURLOPT_FAILONERROR, true);
+
+            $cacheFileContent = curl_exec($cURLHandle);
+
+            $cURLErrNo = curl_errno($cURLHandle);
+            $cURLInfo = curl_getinfo($cURLHandle);
+
+            curl_close($cURLHandle);
+
+            if ($cURLErrNo != 0) {
+                $cacheFileContent = 'CURL_ERROR_' . $cURLErrNo;
+            }
+            if ($cURLInfo['http_code'] >= 400) {
+                $cacheFileContent = 'HTTP_CODE_' . $cURLInfo['http_code'];
+            }
+        }
+
+        return $cacheFileContent ?? null;
+    }
+
+    private function downloadUsingFileGetContent(string $filename): string
+    {
+        $remoteURLContext = stream_context_create([
+            'http' => [
+                'method' => 'GET',
+                'timeout' => $this->externalImageTimeout,
+                'max_redirects' => '10',
+            ]
+        ]);
+
+        $cacheFileContent = file_get_contents($filename, false, $remoteURLContext);
+        if ($cacheFileContent === false) {
+            $cacheFileContent = 'FGC_ERROR';
+        }
+
+        return $cacheFileContent;
+    }
+
+    private function isCacheableUrl($filename): bool
+    {
+        if (!(str_starts_with($filename, 'http'))
+            || str_contains($filename, '://' . $this->configProvider->getValue(ConfigOption::Website) . '/')
+        ) {
+            return false;
+        }
+
+        return true;
+    }
+
+    private function ensureCacheDirectory(): bool
+    {
+
+        if (!file_exists($this->externalCacheDir)) {
+            // phpcs:ignore Generic.PHP.NoSilencedErrors
+            @mkdir($this->externalCacheDir);
+        }
+
+        if (!file_exists($this->externalCacheDir) || !is_writable($this->externalCacheDir)) {
+            return false;
+        }
+
+        return true;
+    }
+
+    private function isValidCacheFile(string $cacheFileName): bool
+    {
+        // phpcs:ignore Generic.PHP.NoSilencedErrors
+        if (file_exists($cacheFileName) && (@filesize($cacheFileName) > 64)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    private function writeCacheFile(string $cacheFileName, $content): void
+    {
+        try {
+            $handle = fopen($cacheFileName, 'wb');
+
+            if ($handle === false) {
+                $this->logger->error('Cannot open cache file', [
+                    'file' => $cacheFileName,
+                ]);
+                return;
+            }
+
+            if (flock($handle, LOCK_EX)) {
+                fwrite($handle, $content);
+                fflush($handle);
+                flock($handle, LOCK_UN);
+            }
+
+            fclose($handle);
+        } catch (Throwable $e) {
+            $this->logger->error('Cache file write failed', [
+                'file' => $cacheFileName,
+                'error' => $e->getMessage(),
+            ]);
+        }
+    }
+}