From d8af80122b8b49ff618a95a96cfda7c39e13b239 Mon Sep 17 00:00:00 2001 From: Kevin von Spiczak Date: Wed, 3 Feb 2021 12:15:38 +0100 Subject: [PATCH 1/3] [FEATURE] add DeleteOldMailsCommand --- Classes/Command/DeleteOldMailsCommand.php | 151 +++++++++++++++++++ Classes/Domain/Repository/MailRepository.php | 25 +++ Configuration/Commands.php | 3 + Configuration/Services.yaml | 5 + 4 files changed, 184 insertions(+) create mode 100644 Classes/Command/DeleteOldMailsCommand.php diff --git a/Classes/Command/DeleteOldMailsCommand.php b/Classes/Command/DeleteOldMailsCommand.php new file mode 100644 index 0000000..61dc9c3 --- /dev/null +++ b/Classes/Command/DeleteOldMailsCommand.php @@ -0,0 +1,151 @@ +setDescription( + 'Automatically deletes records from `tx_sgmail_domain_model_mail`, which are older than the specified age. Attachments sent as part of the email are deleted as well.' + )->addArgument( + 'maxAge', InputArgument::REQUIRED, + 'The maximum age (in days), mails older than this will be deleted.' + ); + } + + /** + * Executes the command for the deletion of old mails + * + * @param InputInterface $input + * @param OutputInterface $output + * @return int|void|null + * @throws Exception + */ + protected function execute(InputInterface $input, OutputInterface $output) { + $io = new SymfonyStyle($input, $output); + $io->title($this->getDescription()); + $maxAgeArg = $input->getArgument('maxAge'); + $maxAgeInDays = (int) $maxAgeArg; + + if ($maxAgeInDays <= 0) { + $io->error('Please enter a maximum age (in days) for the mail\'s to be deleted.'); + return; + } + + if (version_compare(VersionNumberUtility::getCurrentTypo3Version(), '10.4.0', '<')) { + $objectManager = GeneralUtility::makeInstance(ObjectManager::class); + /** @var MailRepository $mailRepository */ + $mailRepository = $objectManager->get(MailRepository::class); + } else { + $mailRepository = GeneralUtility::makeInstance(MailRepository::class); + } + + $mailUidsForDeletion = $mailRepository->findMailsForDeletion($maxAgeInDays); + if (!$mailUidsForDeletion) { + $io->success("No matching mails found for deletion criteria: older than $maxAgeInDays days"); + return; + } + + $this->markRelatedAttachmentsAsDeleted($mailUidsForDeletion); + $this->markOldMailsAsDeleted($mailUidsForDeletion); + + if ($this->amountOfDeletedRecords > 0) { + $io->success('Successfully deleted ' . $this->amountOfDeletedRecords . ' records.'); + } else { + $io->note('No records deleted'); + } + + return 0; + } + + /** + * @param array $mailUidsForDeletion + */ + private function markOldMailsAsDeleted(array $mailUidsForDeletion): void { + $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME); + $queryBuilder + ->update(self::TABLE_NAME) + ->set('deleted', 1) + ->where( + $queryBuilder->expr()->in( + 'uid', + $queryBuilder->createNamedParameter($mailUidsForDeletion, Connection::PARAM_INT_ARRAY) + ) + ); + + $result = $queryBuilder->execute(); + $this->amountOfDeletedRecords += $result; + } + + private function markRelatedAttachmentsAsDeleted(array $mailUidsForDeletion): void { + $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME); + $queryBuilder + ->update('sys_file_reference') + ->set('deleted', 1) + ->where( + $queryBuilder->expr()->in( + 'uid_foreign', + $queryBuilder->createNamedParameter($mailUidsForDeletion, Connection::PARAM_INT_ARRAY) + ), + $queryBuilder->expr()->eq( + 'tablenames', $queryBuilder->createNamedParameter(self::TABLE_NAME, Connection::PARAM_STR) + ), + $queryBuilder->expr()->eq( + 'fieldname', $queryBuilder->createNamedParameter('attachments', Connection::PARAM_STR) + ), + ); + + $result = $queryBuilder->execute(); + $this->amountOfDeletedRecords += $result; + } +} diff --git a/Classes/Domain/Repository/MailRepository.php b/Classes/Domain/Repository/MailRepository.php index 14cc20d..0b87e4b 100644 --- a/Classes/Domain/Repository/MailRepository.php +++ b/Classes/Domain/Repository/MailRepository.php @@ -304,4 +304,29 @@ class MailRepository extends AbstractRepository { return $queryBuilder->orderBy('tstamp', 'desc'); } + + /** + * Returns an array with uids of all mails that are older than $maxAgeInDays + * + * @param int $maxAgeInDays + * @return array + */ + public function findMailsForDeletion(int $maxAgeInDays = 0): array { + $query = $this->createQuery(); + $query->statement( + 'SELECT uid FROM tx_sgmail_domain_model_mail WHERE FROM_UNIXTIME(crdate) < now() - interval ? day', + [$maxAgeInDays] + ); + + $mailUidsForDeletion = $query->execute(TRUE); + + // "flatten" the array, so that we end up with one numerical array with just the uids + foreach ($mailUidsForDeletion as $key => $value) { + if (is_array($value) && array_key_exists('uid', $value)) { + $mailUidsForDeletion[$key] = $value['uid']; + } + } + + return $mailUidsForDeletion; + } } diff --git a/Configuration/Commands.php b/Configuration/Commands.php index 0d05fb4..cb665d0 100644 --- a/Configuration/Commands.php +++ b/Configuration/Commands.php @@ -2,5 +2,8 @@ return [ 'sg_mail:sendMail' => [ 'class' => \SGalinski\SgMail\Command\SendMailCommandController::class + ], + 'sg_mail:deleteOldMails' => [ + 'class' => \SGalinski\SgMail\Command\DeleteOldMailsCommand::class ] ]; diff --git a/Configuration/Services.yaml b/Configuration/Services.yaml index fda2956..2c0b758 100644 --- a/Configuration/Services.yaml +++ b/Configuration/Services.yaml @@ -7,6 +7,11 @@ services: SGalinski\SgMail\: resource: '../Classes/*' + SGalinski\SgMail\Command\DeleteOldMailsCommand: + tags: + - name: 'console.command' + command: 'sg_mail:deleteOldMails' + SGalinski\SgMail\Domain\Repository\FrontendUserGroupRepository: public: true -- GitLab From 9d5e95f91c690c4b2c6158b64328bf9704b0ad32 Mon Sep 17 00:00:00 2001 From: Kevin von Spiczak Date: Wed, 3 Feb 2021 13:39:17 +0100 Subject: [PATCH 2/3] [TASK] disable "deleted" feature in TCA https://docs.typo3.org/m/typo3/reference-tca/master/en-us/Ctrl/Properties/Delete.html --- Configuration/TCA/tx_sgmail_domain_model_mail.php | 1 - 1 file changed, 1 deletion(-) diff --git a/Configuration/TCA/tx_sgmail_domain_model_mail.php b/Configuration/TCA/tx_sgmail_domain_model_mail.php index 12fe86e..a95b1d0 100644 --- a/Configuration/TCA/tx_sgmail_domain_model_mail.php +++ b/Configuration/TCA/tx_sgmail_domain_model_mail.php @@ -34,7 +34,6 @@ $columns = [ 'cruser_id' => 'cruser_id', 'searchFields' => 'blacklisted, mail_subject, mail_body, to_address, from_address, from_name, bcc_addresses, cc_addresses, extension_key, template_name, sending_time, last_sending_time, language', 'dividers2tabs' => TRUE, - 'delete' => 'deleted', 'enablecolumns' => [ 'disabled' => 'hidden', ], -- GitLab From bf3c79af335a08d86c7d5db1c7665e2b21ef9465 Mon Sep 17 00:00:00 2001 From: Kevin von Spiczak Date: Mon, 8 Feb 2021 10:48:46 +0100 Subject: [PATCH 3/3] [TASK] add DeleteOldMailsCommand --- Classes/Command/DeleteOldMailsCommand.php | 149 +++++++++++++++++----- 1 file changed, 120 insertions(+), 29 deletions(-) diff --git a/Classes/Command/DeleteOldMailsCommand.php b/Classes/Command/DeleteOldMailsCommand.php index 61dc9c3..e683212 100644 --- a/Classes/Command/DeleteOldMailsCommand.php +++ b/Classes/Command/DeleteOldMailsCommand.php @@ -34,6 +34,8 @@ use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; use TYPO3\CMS\Core\Database\Connection; use TYPO3\CMS\Core\Database\ConnectionPool; +use TYPO3\CMS\Core\Resource\Exception\ResourceDoesNotExistException; +use TYPO3\CMS\Core\Resource\ResourceFactory; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Core\Utility\VersionNumberUtility; use TYPO3\CMS\Extbase\Object\Exception; @@ -45,7 +47,11 @@ use TYPO3\CMS\Extbase\Object\ObjectManager; * @package SGalinski\SgMail\Command */ class DeleteOldMailsCommand extends Command { - private const TABLE_NAME = 'tx_sgmail_domain_model_mail'; + private const TABLE_NAME_MAIL = 'tx_sgmail_domain_model_mail'; + private const TABLE_NAME_FILE_REFERENCE = 'sys_file_reference'; + private const TABLE_NAME_FILE = 'sys_file'; + private const TABLE_NAME_REFINDEX = 'sys_refindex'; + /** * @var int */ @@ -79,7 +85,7 @@ class DeleteOldMailsCommand extends Command { if ($maxAgeInDays <= 0) { $io->error('Please enter a maximum age (in days) for the mail\'s to be deleted.'); - return; + return 0; } if (version_compare(VersionNumberUtility::getCurrentTypo3Version(), '10.4.0', '<')) { @@ -93,11 +99,11 @@ class DeleteOldMailsCommand extends Command { $mailUidsForDeletion = $mailRepository->findMailsForDeletion($maxAgeInDays); if (!$mailUidsForDeletion) { $io->success("No matching mails found for deletion criteria: older than $maxAgeInDays days"); - return; + return 0; } - $this->markRelatedAttachmentsAsDeleted($mailUidsForDeletion); - $this->markOldMailsAsDeleted($mailUidsForDeletion); + $this->deleteAttachments($mailUidsForDeletion); + $this->deleteOldMails($mailUidsForDeletion); if ($this->amountOfDeletedRecords > 0) { $io->success('Successfully deleted ' . $this->amountOfDeletedRecords . ' records.'); @@ -109,43 +115,128 @@ class DeleteOldMailsCommand extends Command { } /** + * Deletes all tx_sgmail_domain_model_mail records, where the uid matches one of the uids in $mailUidsForDeletion + * * @param array $mailUidsForDeletion */ - private function markOldMailsAsDeleted(array $mailUidsForDeletion): void { - $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME); - $queryBuilder - ->update(self::TABLE_NAME) - ->set('deleted', 1) + private function deleteOldMails(array $mailUidsForDeletion): void { + $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable( + self::TABLE_NAME_MAIL + ); + $this->amountOfDeletedRecords += $queryBuilder + ->delete(self::TABLE_NAME_MAIL) ->where( $queryBuilder->expr()->in( 'uid', $queryBuilder->createNamedParameter($mailUidsForDeletion, Connection::PARAM_INT_ARRAY) ) - ); - - $result = $queryBuilder->execute(); - $this->amountOfDeletedRecords += $result; + )->execute(); } - private function markRelatedAttachmentsAsDeleted(array $mailUidsForDeletion): void { - $queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE_NAME); - $queryBuilder - ->update('sys_file_reference') - ->set('deleted', 1) + /** + * Deletes the attachments (sys_file_reference / sys_file records), referenced in the mails to be deleted. + * For every file reference found in a mail record, we check if it is the only reference, + * or if we can find more records, referencing the same sys_file. + * + * The sys_file_reference found in the mail record is always deleted. + * If the referenced file has no more references, besides the one we just deleted, both the sys_file record and + * the actual file in the file system is deleted as well. + * + * @param array $mailUidsForDeletion + */ + private function deleteAttachments(array $mailUidsForDeletion): void { + $fileReferenceQueryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable( + self::TABLE_NAME_FILE_REFERENCE + ); + // Fetch all sys_file_reference records, with a relation to the mails we want to delete. + $sysFileReferencesToDelete = $fileReferenceQueryBuilder + ->select('uid', 'uid_local') + ->from(self::TABLE_NAME_FILE_REFERENCE) ->where( - $queryBuilder->expr()->in( + $fileReferenceQueryBuilder->expr()->in( 'uid_foreign', - $queryBuilder->createNamedParameter($mailUidsForDeletion, Connection::PARAM_INT_ARRAY) + $fileReferenceQueryBuilder->createNamedParameter( + $mailUidsForDeletion, Connection::PARAM_INT_ARRAY + ) ), - $queryBuilder->expr()->eq( - 'tablenames', $queryBuilder->createNamedParameter(self::TABLE_NAME, Connection::PARAM_STR) + $fileReferenceQueryBuilder->expr()->eq( + 'tablenames', + $fileReferenceQueryBuilder->createNamedParameter(self::TABLE_NAME_MAIL, Connection::PARAM_STR) ), - $queryBuilder->expr()->eq( - 'fieldname', $queryBuilder->createNamedParameter('attachments', Connection::PARAM_STR) + $fileReferenceQueryBuilder->expr()->eq( + 'fieldname', + $fileReferenceQueryBuilder->createNamedParameter('attachments', Connection::PARAM_STR) ), - ); - - $result = $queryBuilder->execute(); - $this->amountOfDeletedRecords += $result; + )->execute()->fetchAllAssociative(); + + foreach ($sysFileReferencesToDelete as $sysFileReferenceToDelete) { + $sysFileReferenceUid = (int) $sysFileReferenceToDelete['uid']; + $sysFileUid = (int) $sysFileReferenceToDelete['uid_local']; + + $resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class); + try { + $sysFileReference = $resourceFactory->getFileReferenceObject($sysFileReferenceUid); + } catch (ResourceDoesNotExistException $e) { + $sysFileReference = NULL; + } + + if ($sysFileReference === NULL) { + return; + } + + $referencedSysFile = $sysFileReference->getOriginalFile(); + + $refIndexQueryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable( + self::TABLE_NAME_REFINDEX + )->createQueryBuilder(); + + // Check if there are more rows in sys_refindex, + // referencing the same file as the current $sysFileReferenceToDelete. + // If we do not find any rows here, we delete + // * the sys_file_reference record, + // * the sys_file record, + // * the actual file in the filesystem + // If we find more rows here, only the sys_file_reference record is deleted. + $countRefIndexEntries = $refIndexQueryBuilder + ->count('*') + ->from(self::TABLE_NAME_REFINDEX) + ->where( + $refIndexQueryBuilder->expr()->eq( + 'ref_table', + $refIndexQueryBuilder->createNamedParameter(self::TABLE_NAME_FILE, Connection::PARAM_STR) + ), + $refIndexQueryBuilder->expr()->eq( + 'ref_uid', + $refIndexQueryBuilder->createNamedParameter($sysFileUid, Connection::PARAM_INT) + ), + // exclude current sys_file_reference record + $refIndexQueryBuilder->expr()->neq( + 'recuid', + $refIndexQueryBuilder->createNamedParameter($sysFileReferenceUid, Connection::PARAM_INT) + ) + )->execute()->fetchOne(); + + if ($countRefIndexEntries <= 0) { + $referencedSysFile->delete(); + } + + $deleteFileReferenceQueryBuilder = GeneralUtility::makeInstance( + ConnectionPool::class + )->getConnectionForTable( + self::TABLE_NAME_FILE_REFERENCE + )->createQueryBuilder(); + + // delete the sys_file_reference + $this->amountOfDeletedRecords += $deleteFileReferenceQueryBuilder + ->delete(self::TABLE_NAME_FILE_REFERENCE) + ->where( + $deleteFileReferenceQueryBuilder->expr()->eq( + 'uid', + $deleteFileReferenceQueryBuilder->createNamedParameter( + $sysFileReferenceUid, Connection::PARAM_INT + ) + ) + )->execute(); + } } } -- GitLab