Merge branch 'feature_news_migration' into 'master'

Feature news migration

See merge request !6

Classes/Command/MigrateNewsCommandController.php

+<?php
+
+namespace SGalinski\SgNews\Command;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) sgalinski Internet Services (https://www.sgalinski.de)
+ *
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use SGalinski\SgNews\Domain\Model\News;
+use SGalinski\SgNews\Domain\Model\Tag;
+use TYPO3\CMS\Backend\FrontendBackendUserAuthentication;
+use TYPO3\CMS\Core\Database\DatabaseConnection;
+use TYPO3\CMS\Core\DataHandling\DataHandler;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Extbase\Domain\Model\FileReference;
+use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
+use TYPO3\CMS\Extbase\Persistence\Exception\IllegalObjectTypeException;
+use TYPO3\CMS\Extbase\Persistence\Exception\UnknownObjectException;
+use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController;
+
+/**
+ * Command controller, that migrates data from tx_news to sg_news
+ */
+class MigrateNewsCommandController extends CommandController {
+	/**
+	 * @var bool
+	 */
+	protected $requestAdminPermissions = TRUE;
+
+	/**
+	 * @var \TYPO3\CMS\Extbase\Persistence\Generic\PersistenceManager
+	 * @inject
+	 */
+	protected $persistenceManager;
+
+	/**
+	 * @var \SGalinski\SgNews\Domain\Repository\NewsRepository
+	 * @inject
+	 */
+	protected $newsRepository;
+
+	/**
+	 * @var \SGalinski\SgNews\Domain\Repository\TagRepository
+	 * @inject
+	 */
+	private $tagRepository;
+
+	/**
+	 * @var \SGalinski\SgNews\Domain\Repository\FileReferenceRepository
+	 * @inject
+	 */
+	private $fileReferenceRepository;
+
+	/**
+	 * this array maps new pages to their original entry in the tx_news table
+	 *
+	 * @var array $newsPagesMap
+	 */
+	protected $newsPagesMap = [];
+
+	/**
+	 * this array contains all pages that should be migrated to sg_news
+	 *
+	 * @var array $pagesToMigrate
+	 */
+	protected $pagesToMigrate = [];
+
+	/**
+	 * @var array $languageMap
+	 */
+	protected $languageMap = [];
+
+	/**
+	 * @var array $languageMap
+	 */
+	protected $categoryMap = [];
+
+	/**
+	 * @param string $copyPageId the page id of the template that should be copied
+	 * @param int $categoryPid the page id of the category page
+	 * @param int $year only news from that year will be migrated
+	 * @param string $languageMapAsJson a json, mapping language ids (old => new). this is needed if the sys_language_uids have changed
+	 * @param string $categoryMapAsJson a json, mapping sys_category ids (old => new). t
+	 * @param int $pId only news from that pid will be migrated
+	 *
+	 * @throws IllegalObjectTypeException
+	 * @throws UnknownObjectException
+	 * @throws \Exception
+	 */
+	public function runMigrateNewsCommand(
+		$copyPageId, $categoryPid, $year = 2015,
+		$languageMapAsJson = '{"3":1,"1":0,"2":2,"0":3}',
+		$categoryMapAsJson = '{"2":10,"3":11,"4":12,"5":13,"6":14,"7":15,"8":16,"9":17}',
+		$pId = 52
+	) {
+		// fix repair translation bug where tsfe is missing from command controller, can be removed when v1.5 is released
+		if (!$GLOBALS['TSFE']) {
+			/** @var TypoScriptFrontendController $typoScriptController */
+			$GLOBALS['TSFE'] = $typoScriptController = $this->objectManager->get(
+				TypoScriptFrontendController::class, $GLOBALS['TYPO3_CONF_VARS'], 0, 0
+			);
+		}
+
+		$this->languageMap = json_decode($languageMapAsJson, TRUE);
+		$this->categoryMap = json_decode($categoryMapAsJson, TRUE);
+
+		/** @var DatabaseConnection $db */
+		$db = $GLOBALS['TYPO3_DB'];
+		$where = 'hidden = 0 and deleted = 0 and endtime = 0 and pid = ' . (int) $pId;
+
+		/** @var \mysqli_result $result */
+		$result = $db->exec_SELECTquery('*', 'tx_news_domain_model_news', $where);
+
+		$localDataHandler = GeneralUtility::makeInstance(DataHandler::class);
+		$beUser = $this->simulateBackendUser();
+		$localCommandMap = [
+			'pages' => [
+				$copyPageId => [
+					'copy' => (int) $categoryPid
+				]
+			]
+		];
+
+		$resultArray = [];
+		while ($row = $result->fetch_assoc()) {
+			// ignore the entry if its not within the given year
+			if ((int) date('Y', $row['datetime']) !== $year) {
+				continue;
+			}
+			$resultArray[] = $row;
+
+			// if no l10n_parent exists, create a copy of the page
+			if ((int) $row['l10n_parent'] === 0) {
+				$localDataHandler->start([], $localCommandMap, $beUser);
+				$localDataHandler->bypassAccessCheckForRecords = TRUE;
+				$localDataHandler->checkModifyAccessList('pages');
+				$localDataHandler->process_cmdmap();
+
+				// get the id of the new object
+				$newPageId = $localDataHandler->copyMappingArray['pages'][$copyPageId];
+
+				// make entry in news map, so we can fetch it later
+				$this->newsPagesMap[$row['uid']] = $newPageId;
+
+				/** @var News $newsPage */
+				$newsPage = $this->newsRepository->findByUidIgnoreEnableFields($newPageId);
+
+				if ($newsPage !== NULL) {
+					$title = date('Y-m-d', $row['datetime']) . ' - ' . $row['title'];
+					$newsPage->setTitle($title);
+					$newsPage->setSubtitle($row['title']);
+
+					$date = new \DateTime('@' . $row['datetime']);
+					$newsPage->setLastUpdated($date);
+
+					$matchingTag = $this->getMatchingTag($row);
+					if ($matchingTag && $matchingTag instanceof Tag) {
+						$newsPage->addTag($matchingTag);
+					}
+
+					/** @var FileReference $image */
+					$image = $this->getMatchingImage($row);
+					if ($image !== NULL) {
+						$newsPage->addTeaser1Image($image);
+
+						$originalResource = $image->getOriginalResource();
+						if ($originalResource !== NULL) {
+							$originalFile = $originalResource->getOriginalFile();
+							if ($originalFile !== NULL) {
+								$teaserImage2 = $this->fileReferenceRepository->addFileReferenceFromFile(
+									$originalFile, $this->newsPagesMap[$row['uid']],
+									$this->newsPagesMap[$row['uid']], 'pages', 'tx_sgnews_teaser2_image'
+								);
+								$newsPage->addTeaser2Image($teaserImage2);
+							}
+						}
+					}
+
+					$this->newsRepository->update($newsPage);
+					$this->persistenceManager->persistAll();
+
+					// update content element from the new page
+					$where = 'pid = ' . $newPageId . ' AND sys_language_uid = ' . $this->languageMap[(int) $row['sys_language_uid']];
+					$db->exec_UPDATEquery('tt_content', $where, ['bodytext' => $row['bodytext']]);
+				}
+			} else { // this row is a translation and should simply update the content accordingly
+				$this->updateTranslation($row);
+			}
+		}
+	}
+
+	/**
+	 * Get the image file matching the news
+	 *
+	 * @param array $row
+	 * @return null|FileReference
+	 */
+	private function getMatchingImage(array $row) {
+		/** @var DatabaseConnection $db */
+		$db = $GLOBALS['TYPO3_DB'];
+		$where = 'tablenames = "tx_news_domain_model_news" AND fieldname = "fal_media" AND uid_foreign = ' . $row['uid'];
+
+		/** @var \mysqli_result $result */
+		$result = $db->exec_SELECTgetSingleRow('uid, uid_local', 'sys_file_reference', $where);
+		if (!$result) {
+			return NULL;
+		}
+
+		$where = 'uid = ' . $result['uid'];
+		$db->exec_UPDATEquery(
+			'sys_file_reference',
+			$where,
+			['pid' => $this->newsPagesMap[$row['uid']], 'uid_foreign' => $this->newsPagesMap[$row['uid']], 'tablenames' => 'pages', 'fieldname' => 'tx_sgnews_teaser1_image']
+		);
+
+		$result = $this->fileReferenceRepository->findByUid((int) $result['uid']);
+		if ($result instanceof FileReference) {
+			$result->setPid($this->newsPagesMap[$row['uid']]);
+			return $result;
+		}
+
+		return NULL;
+	}
+
+	/**
+	 * Get the tag / category, matching the news
+	 *
+	 * @param array $row
+	 * @return Object $tag
+	 */
+	private function getMatchingTag(array $row) {
+		// look up the correct category id, if they have changed
+		if (isset($this->categoryMap[(int) $row['categories']])) {
+			$categoryId = $this->categoryMap[(int) $row['categories']];
+		} else {
+			$categoryId = (int) $row['categories'];
+		}
+
+		return $this->tagRepository->findByUid($categoryId);
+	}
+
+	/**
+	 * Updates content element of a news translationuid
+	 *
+	 * @param array $row
+	 */
+	private function updateTranslation(array $row) {
+		/** @var DatabaseConnection $db */
+		$db = $GLOBALS['TYPO3_DB'];
+
+		// get entry in news map
+		$parentId = $this->newsPagesMap[$row['l10n_parent']];
+
+		// get content element from the original page
+		$where = 'pid = ' . (int) $parentId;
+
+		/** @var \mysqli_result $result */
+		$result = $db->exec_SELECTquery('l18n_parent', 'tt_content', $where);
+		$originalContentElement = $result->fetch_row();
+
+		// if its the new default, there is no l18n_parent
+		if ((int) $this->languageMap[(int) $row['sys_language_uid'] === 0]) {
+			$where = 'uid = ' . (int) $originalContentElement[0];
+		} else {
+			$where = 'l18n_parent = ' . (int) $originalContentElement[0];
+		}
+
+		// look up the correct language id, if they have changed
+		if (isset($this->languageMap[(int) $row['sys_language_uid']])) {
+			$where .= ' AND sys_language_uid = ' . $this->languageMap[(int) $row['sys_language_uid']];
+		} else {
+			$where .= ' AND sys_language_uid = ' . (int) $row['sys_language_uid'];
+		}
+
+		$db->exec_UPDATEquery('tt_content', $where, ['bodytext' => $row['bodytext']]);
+
+		// possibly the default language needs to be overwritten and the old default translation needs to be preserved
+		if (isset($this->languageMap[(int) $row['sys_language_uid']]) && $this->languageMap[(int) $row['sys_language_uid']] === 0) {
+
+			$where = 'uid = ' . (int) $parentId;
+			/** @var \mysqli_result $result */
+			$result = $db->exec_SELECTgetSingleRow('title, subtitle', 'pages', $where);
+			if ($result) {
+				$where = 'pid = ' . (int) $parentId . ' AND sys_language_uid = ' . $this->languageMap[0];
+				$db->exec_UPDATEquery(
+					'pages_language_overlay', $where,
+					['title' => $result['title'], 'subtitle' => $result['subtitle']]
+				);
+			}
+
+			$where = 'uid = ' . (int) $parentId;
+			$db->exec_UPDATEquery(
+				'pages', $where, ['title' => date(
+						'Y-m-d', $row['datetime']
+					) . ' - ' . $row['title'], 'subtitle' => $row['title'], 'lastUpdated' => $row['datetime']]
+			);
+		} else {
+			// finally translate the page title if necessary
+			/** @noinspection NotOptimalIfConditionsInspection */
+			if (isset($this->languageMap[(int) $row['sys_language_uid']]) && $this->languageMap[(int) $row['sys_language_uid']] > 0) {
+				$where = 'pid = ' . (int) $parentId . ' AND sys_language_uid = ' . $this->languageMap[(int) $row['sys_language_uid']];
+			} else {
+				$where = 'pid = ' . (int) $parentId . ' AND sys_language_uid = ' . (int) $row['sys_language_uid'];
+			}
+		}
+
+		$db->exec_UPDATEquery(
+			'pages_language_overlay', $where, ['title' => date('Y-m-d', $row['datetime']) . ' - ' . $row['title']]
+		);
+	}
+
+	/**
+	 * Simulate Backend User for DataHandler
+	 *
+	 * @return FrontendBackendUserAuthentication
+	 */
+	private function simulateBackendUser() {
+		/** @var \TYPO3\CMS\Backend\FrontendBackendUserAuthentication $BE_USER */
+		$BE_USER = GeneralUtility::makeInstance(FrontendBackendUserAuthentication::class);
+		$BE_USER->setBeUserByName('admin');
+		if ($BE_USER->user['uid']) {
+			$BE_USER->fetchGroupData();
+		}
+		$BE_USER->uc_default['copyLevels'] = '9999';
+		$BE_USER->uc = $BE_USER->uc_default;
+		$GLOBALS['PAGES_TYPES'][254]['allowedTables'] = '*';
+		return $BE_USER;
+	}
+}

Classes/Domain/Model/FileReference.php

+<?php
+
+namespace SGalinski\SgNews\Domain\Model;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) sgalinski Internet Services (https://www.sgalinski.de)
+ *
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+/**
+ * FileReference
+ */
+class FileReference extends \TYPO3\CMS\Extbase\Domain\Model\FileReference {
+
+}

Classes/Domain/Repository/FileReferenceRepository.php

+<?php
+
+namespace SGalinski\SgNews\Domain\Repository;
+
+/***************************************************************
+ *  Copyright notice
+ *
+ *  (c) sgalinski Internet Services (https://www.sgalinski.de)
+ *
+ *  All rights reserved
+ *
+ *  This script is part of the TYPO3 project. The TYPO3 project is
+ *  free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  The GNU General Public License can be found at
+ *  http://www.gnu.org/copyleft/gpl.html.
+ *
+ *  This script is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  This copyright notice MUST APPEAR in all copies of the script!
+ ***************************************************************/
+
+use SGalinski\SgNews\Domain\Model\FileReference;
+use TYPO3\CMS\Core\Database\DatabaseConnection;
+use TYPO3\CMS\Core\Resource\File;
+
+/**
+ * FileReference Repository
+ */
+class FileReferenceRepository extends AbstractRepository {
+
+	/**
+	 * Method creates a file reference entry in the database. This step is necessary because the
+	 * extbase can not handle the default TCA for FAL records. Without this method the FAL records will miss
+	 * the uid_foreign, field name and table names.
+	 *
+	 * @param File $file
+	 * @param int $uid
+	 * @param int $pid
+	 * @param string $tablename
+	 * @param string $fieldname
+	 * @return FileReference
+	 * @throws \Exception
+	 */
+	public function addFileReferenceFromFile(File $file, $uid, $pid, $tablename, $fieldname) {
+		$fileReferenceId = $this->addFileReferenceFromFileId(
+			$file->getUid(),
+			['uid' => $uid, 'pid' => $pid],
+			$tablename,
+			$fieldname
+		);
+
+		$fileReference = NULL;
+		if ($fileReferenceId > 0) {
+			/** @var FileReference $fileReference */
+			$fileReference = $this->findByUid($fileReferenceId);
+		}
+
+		return $fileReference;
+	}
+
+	/**
+	 * Method creates a file reference entry in the database. This step is necessary because the
+	 * extbase can not handle the default TCA for FAL records. Without this method the FAL records will miss
+	 * the uid_foreign, field name and table names.
+	 *
+	 * @param $fileId
+	 * @param array $reference
+	 * @param string $tablename
+	 * @param string $fieldname
+	 * @return int
+	 * @throws \Exception
+	 */
+	public function addFileReferenceFromFileId($fileId, array $reference, $tablename, $fieldname) {
+		/** @var DatabaseConnection $db */
+		$db = $GLOBALS['TYPO3_DB'];
+		$arguments = [
+			'crdate' => $GLOBALS['EXEC_TIME'],
+			'tstamp' => $GLOBALS['EXEC_TIME'],
+			'pid' => (int) $reference['pid'],
+			'table_local' => 'sys_file',
+			'uid_local' => $fileId,
+			'uid_foreign' => (int) $reference['uid'],
+			'tablenames' => $tablename,
+			'fieldname' => $fieldname,
+		];
+
+		if (!$db->exec_INSERTquery('sys_file_reference', $arguments)) {
+			throw new \Exception('An error occurred while adding a file reference record.', 1452590219);
+		}
+
+		return (int) $db->sql_insert_id();
+	}
+}

Classes/Domain/Repository/NewsRepository.php

@@ -215,9 +215,11 @@ class NewsRepository extends AbstractRepository {
 		$onlyHighlighted = FALSE, array $categoryIds = NULL, $hideNeverHighlightedNews = FALSE, $sortBy = 'date',
 		array $tagIds = NULL, $startTime = 0, $endTime = 0
 	): int {
-		return $this->getCount($this->getQueryForLastUpdatedOrHighlightedNewsByCategories(
-			0, $onlyHighlighted, $categoryIds, 0, $hideNeverHighlightedNews, $sortBy, $tagIds, $startTime, $endTime
-		));
+		return $this->getCount(
+			$this->getQueryForLastUpdatedOrHighlightedNewsByCategories(
+				0, $onlyHighlighted, $categoryIds, 0, $hideNeverHighlightedNews, $sortBy, $tagIds, $startTime, $endTime
+			)
+		);
 	}
 
 	/**
@@ -452,4 +454,22 @@ class NewsRepository extends AbstractRepository {
 
 		return $this->getCount($query);
 	}
+
+	/**
+	 * Find news record by uid, but with ignoring enable fields
+	 *
+	 * @param int $uid
+	 * @return object
+	 */
+	public function findByUidIgnoreEnableFields($uid) {
+		$query = $this->createQuery();
+		/** @var $querySettings \TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings */
+		$querySettings = $query->getQuerySettings();
+		$querySettings->setIgnoreEnableFields(TRUE);
+		$querySettings->setRespectStoragePage(FALSE);
+		$query->setQuerySettings($querySettings);
+
+		$query->matching($query->equals('uid', $uid));
+		return $query->execute()->getFirst();
+	}
 }

Configuration/TypoScript/Common/setup.txt

@@ -35,6 +35,12 @@ config.tx_extbase {
 					tableName = sys_category
 				}
 			}
+
+			SGalinski\SgNews\Domain\Model\FileReference {
+				mapping {
+					tableName = sys_file_reference
+				}
+			}
 		}
 	}
 }

README.md

@@ -313,4 +313,16 @@ You can also handle the localisation of news pages for all the languages in your
 - **full-opacity flag icon** : Edit the existing localisation record
 - **half-transparent flag icon with <img height="14px" width="14px" src="https://camo.githubusercontent.com/b10f4d50dc21b152cde15cef5983c139e77b1c34/68747470733a2f2f7261776769742e636f6d2f5459504f332f5459504f332e49636f6e732f6d61737465722f646973742f6f7665726c61792f6f7665726c61792d6e65772e737667"/> overlay** : Create new localisation record
 
-***Note*** : The extension also adds a News Module link in the Page Module of News/Category pages
\ No newline at end of file
+***Note*** : The extension also adds a News Module link in the Page Module of News/Category pages
+
+
+## Migrating tx_news to sg_news
+
+This extension comes with the command controller task **** to migirate tx_news entries to sg_news. The task comes with the following parameters:
+
+- **copyPageId** : You will need a template site, which will be the basis of each new news record. Create your content elements accordingly on this site.
+- **categoryPid** : This needs to be the category page id, you want your news to be children of.
+- **year** : Only news of this year will be migrated
+- **languageMapAsJson** : a json string, mapping language ids (old => new). this is needed if the sys_language_uids have changed
+- **categoryMapAsJson** : a json string, mapping sys_category ids (old => new).
+- **pId** : only news from that pid will be migrated

ext_localconf.php

@@ -98,4 +98,9 @@ if(!is_array($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['cms/layout/db_layout.php
 	$GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['cms/layout/db_layout.php']['drawHeaderHook'] = [];
 }
 $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['cms/layout/db_layout.php']['drawHeaderHook'][] =
-	\SGalinski\SgNews\Hooks\PageLayoutController::class . '->addNewsModuleLink';
\ No newline at end of file
+	\SGalinski\SgNews\Hooks\PageLayoutController::class . '->addNewsModuleLink';
+
+
+// register command controllers
+$GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['extbase']['commandControllers'][] =
+	'SGalinski\SgNews\Command\MigrateNewsCommandController';