Improved MessageList handling

plugins/kolab/KolabAddressBook.php

@@ -295,9 +295,6 @@ class KolabAddressBook implements \RainLoop\Providers\AddressBook\AddressBookInt
 				$oParams->sSearch = 'from='.$sSearch;
 			}
 			$oParams->sSort = 'FROM';
-			$oParams->bUseSortIfSupported = !!\RainLoop\Api::Actions()->Config()->Get('labs', 'use_imap_sort', true);
-//			$oParams->iPrevUidNext = $this->GetActionParam('UidNext', 0);
-//			$oParams->bUseThreads = false;
 
 			$oMessageList = $this->MailClient()->MessageList($oParams);
 			foreach ($oMessageList as $oMessage) {

plugins/kolab/index.php

@@ -4,8 +4,8 @@ class KolabPlugin extends \RainLoop\Plugins\AbstractPlugin
 {
 	const
 		NAME = 'Kolab',
-		VERSION = '2.2',
+		VERSION = '2.3',
-		RELEASE  = '2022-12-16',
+		RELEASE  = '2022-12-30',
 		CATEGORY = 'Contacts',
 		DESCRIPTION = 'Use an Address Book of Kolab.',
 		REQUIRED = '2.23.0';

snappymail/v/0.0.0/app/libraries/MailSo/Mail/MailClient.php

@@ -476,7 +476,7 @@ class MailClient
 	 * @throws \MailSo\Net\Exceptions\*
 	 * @throws \MailSo\Imap\Exceptions\*
 	 */
-	protected function MessageListByRequestIndexOrUids(MessageCollection $oMessageCollection, SequenceSet $oRange) : void
+	protected function MessageListByRequestIndexOrUids(MessageCollection $oMessageCollection, SequenceSet $oRange, array &$aAllThreads = []) : void
 	{
 		if (\count($oRange)) {
 			$aFetchResponse = $this->oImapClient->Fetch(array(
@@ -494,7 +494,21 @@ class MailClient
 					$id = $oRange->UID
 						? $oFetchResponseItem->GetFetchValue(FetchType::UID)
 						: $oFetchResponseItem->oImapResponse->ResponseList[1];
-					$aCollection[$id] = Message::fromFetchResponse($oMessageCollection->FolderName, $oFetchResponseItem);
+					$oMessage = Message::fromFetchResponse($oMessageCollection->FolderName, $oFetchResponseItem);
+					if ($oMessage) {
+						if ($aAllThreads) {
+							$iUid = $oMessage->Uid;
+							// Find thread and set it.
+							// Used by GUI to delete/move the whole thread or other features
+							foreach ($aAllThreads as $aMap) {
+								if (\in_array($iUid, $aMap)) {
+									$oMessage->SetThreads($aMap);
+									break;
+								}
+							}
+						}
+						$aCollection[$id] = $oMessage;
+					}
 				}
 				$oMessageCollection->exchangeArray(\array_values(\array_filter($aCollection)));
 			}
@@ -508,20 +522,13 @@ class MailClient
 	 * @throws \MailSo\Imap\Exceptions\*
 	 */
 	private function GetUids(MessageListParams $oParams, string $sSearch,
-		string $sFolderHash, bool $bUseSortIfSupported = false) : array
+		string $sFolderHash, bool $bUseSort = false) : array
 	{
 		$oCacher = $oParams->oCacher;
 		$sFolderName = $oParams->sFolderName;
 
-		$aResultUids = false;
+		$bUseSort = $bUseSort && $this->oImapClient->hasCapability('SORT');
-		$bUidsFromCacher = false;
+		$sSort = $bUseSort ? $oParams->sSort : '';
-		$bUseCacheAfterSearch = $oCacher && $oCacher->IsInited();
-
-		$sSerializedHash = '';
-		$sSerializedLog = '';
-
-		$bUseSortIfSupported = $bUseSortIfSupported && !\strlen($sSearch) && $this->oImapClient->hasCapability('SORT');
-		$sSort = $bUseSortIfSupported ? $oParams->sSort : '';
 		/* TODO: Validate $sSort
 			ARRIVAL
 				Internal date and time of the message.  This differs from the
@@ -563,7 +570,10 @@ class MailClient
 				DISPLAYFROM, DISPLAYTO
 		 */
 
+		$bUseCacheAfterSearch = $oCacher && $oCacher->IsInited();
 		$sSearchCriterias = \MailSo\Imap\SearchCriterias::fromString($this->oImapClient, $sFolderName, $sSearch, $oParams->bHideDeleted, $bUseCacheAfterSearch);
+		// Disable? as there are many cases that change the result
+//		$bUseCacheAfterSearch = false;
 
 		$bReturnUid = true;
 		if ($oParams->oSequenceSet) {
@@ -571,11 +581,11 @@ class MailClient
 			$sSearchCriterias = $oParams->oSequenceSet . ' ' . $sSearchCriterias;
 		}
 
-		// Disabled for now as there are many cases that change the result
+		$sSerializedHash = '';
-		$bUseCacheAfterSearch = false;
+		$sSerializedLog = '';
 		if ($bUseCacheAfterSearch) {
 			$sSerializedHash = 'GetUids/'.
-				($bUseSortIfSupported ? 'S' . $sSort : 'N').'/'.
+				($bUseSort ? 'S' . $sSort : 'N').'/'.
 				$this->oImapClient->Hash().'/'.
 				$sFolderName.'/'.$sSearchCriterias;
 			$sSerializedLog = '"'.$sFolderName.'" / '.$sSearchCriterias.'';
@@ -590,45 +600,44 @@ class MailClient
 					if ($this->oLogger) {
 						$this->oLogger->Write('Get Serialized UIDS from cache ('.$sSerializedLog.') [count:'.\count($aSerialized['Uids']).']');
 					}
-
+					if (\is_array($aSerialized['Uids'])) {
-					$aResultUids = $aSerialized['Uids'];
+						return $aSerialized['Uids'];
-					$bUidsFromCacher = true;
+					}
 				}
 			}
 		}
 
-		if (!$bUidsFromCacher) {
+		$aResultUids = [];
-			if ($bUseSortIfSupported) {
+		if ($bUseSort) {
-				$aSortTypes = [];
+			$aSortTypes = [];
-				if ($sSort) {
+			if ($sSort) {
-					$aSortTypes[] = $sSort;
+				$aSortTypes[] = $sSort;
-				}
-				if (false === \strpos($sSort, 'DATE')) {
-					// Always also sort DATE descending when DATE is not defined
-					$aSortTypes[] = 'REVERSE DATE';
-				}
-//				$this->oImapClient->hasCapability('ESORT')
-//				$aResultUids = $this->oImapClient->MessageSimpleESort($aSortTypes, $sSearchCriterias)['ALL'];
-				$aResultUids = $this->oImapClient->MessageSimpleSort($aSortTypes, $sSearchCriterias, $bReturnUid);
-			} else {
-//				$this->oImapClient->hasCapability('ESEARCH')
-//				$aResultUids = $this->oImapClient->MessageSimpleESearch($sSearchCriterias, null, $bReturnUid, \MailSo\Base\Utils::IsAscii($sSearchCriterias) ? '' : 'UTF-8')
-				$aResultUids = $this->oImapClient->MessageSimpleSearch($sSearchCriterias,        $bReturnUid, \MailSo\Base\Utils::IsAscii($sSearchCriterias) ? '' : 'UTF-8');
 			}
+			if (false === \strpos($sSort, 'DATE')) {
+				// Always also sort DATE descending when DATE is not defined
+				$aSortTypes[] = 'REVERSE DATE';
+			}
+//			$this->oImapClient->hasCapability('ESORT')
+//			$aResultUids = $this->oImapClient->MessageSimpleESort($aSortTypes, $sSearchCriterias)['ALL'];
+			$aResultUids = $this->oImapClient->MessageSimpleSort($aSortTypes, $sSearchCriterias, $bReturnUid);
+		} else {
+//			$this->oImapClient->hasCapability('ESEARCH')
+//			$aResultUids = $this->oImapClient->MessageSimpleESearch($sSearchCriterias, null, $bReturnUid, \MailSo\Base\Utils::IsAscii($sSearchCriterias) ? '' : 'UTF-8')
+			$aResultUids = $this->oImapClient->MessageSimpleSearch($sSearchCriterias,        $bReturnUid, \MailSo\Base\Utils::IsAscii($sSearchCriterias) ? '' : 'UTF-8');
+		}
 
-			if ($bUseCacheAfterSearch) {
+		if ($bUseCacheAfterSearch) {
-				$oCacher->Set($sSerializedHash, \json_encode(array(
+			$oCacher->Set($sSerializedHash, \json_encode(array(
-					'FolderHash' => $sFolderHash,
+				'FolderHash' => $sFolderHash,
-					'Uids' => $aResultUids
+				'Uids' => $aResultUids
-				)));
+			)));
 
-				if ($this->oLogger) {
+			if ($this->oLogger) {
-					$this->oLogger->Write('Save Serialized UIDS to cache ('.$sSerializedLog.') [count:'.\count($aResultUids).']');
+				$this->oLogger->Write('Save Serialized UIDS to cache ('.$sSerializedLog.') [count:'.\count($aResultUids).']');
-				}
 			}
 		}
 
-		return \is_array($aResultUids) ? $aResultUids : array();
+		return $aResultUids;
 	}
 
 	/**
@@ -656,8 +665,7 @@ class MailClient
 
 		$oInfo = $this->oImapClient->FolderStatusAndSelect($oParams->sFolderName);
 		$oMessageCollection->FolderInfo = $oInfo;
-
+		$oMessageCollection->totalEmails = $oInfo->MESSAGES;
-		$aAllThreads = [];
 
 		$bUseThreads = $oParams->bUseThreads
 			&& ($this->oImapClient->hasCapability('THREAD=REFS') || $this->oImapClient->hasCapability('THREAD=REFERENCES') || $this->oImapClient->hasCapability('THREAD=ORDEREDSUBJECT'));
@@ -674,52 +682,50 @@ class MailClient
 		}
 
 		if ($oInfo->MESSAGES) {
+			$bUseSort = $oParams->bUseSort || $oParams->sSort;
+			$aAllThreads = [];
+			$aUids = [];
+
 			$message_list_limit = $this->oImapClient->Settings->message_list_limit;
 			if (0 < $message_list_limit && $message_list_limit < $oInfo->MESSAGES) {
-				// Don't use SORT nor THREAD
+				// Don't use THREAD for speed
 				$oMessageCollection->Limited = true;
 				if ($this->oLogger) {
-					$this->oLogger->Write('List optimization (count: '.$oInfo->MESSAGES.
+					$this->oLogger->Write('List optimization (count: '.$oInfo->MESSAGES.', limit:'.$message_list_limit.')');
-						', limit:'.$message_list_limit.')');
 				}
 				if (\strlen($sSearch)) {
-					$aUids = $this->GetUids($oParams, $sSearch, $oMessageCollection->FolderHash);
+					// Don't use SORT for speed
-					$oMessageCollection->totalEmails = \count($aUids);
+					$aUids = $this->GetUids($oParams, $sSearch, $oMessageCollection->FolderHash/*, $bUseSort*/);
-					if ($oMessageCollection->totalEmails) {
-						$this->MessageListByRequestIndexOrUids(
-							$oMessageCollection,
-							new SequenceSet(\array_slice($aUids, $oParams->iOffset, $oParams->iLimit))
-						);
-					}
 				} else {
-					$oMessageCollection->totalEmails = $oInfo->MESSAGES;
+					$bUseSort = $this->oImapClient->hasCapability('SORT');
-					if (1 < $oInfo->MESSAGES) {
+					if (2 > $oInfo->MESSAGES) {
+						$aRequestIndexes = \array_slice([1], $oParams->iOffset, 1);
+					} else if ($bUseSort) {
 						// Attempt to sort REVERSE DATE with a bigger range then $oParams->iLimit
 						$end = \min($oInfo->MESSAGES, \max(1, $oInfo->MESSAGES - $oParams->iOffset + $oParams->iLimit));
 						$start = \max(1, $end - ($oParams->iLimit * 3) + 1);
 						$oParams->oSequenceSet = new SequenceSet(\range($end, $start), false);
-						$aRequestIndexes = $this->GetUids($oParams, '', $oMessageCollection->FolderHash, true);
+						$aRequestIndexes = $this->GetUids($oParams, '', $oMessageCollection->FolderHash, $bUseSort);
 						// Attempt to get the correct $oParams->iLimit slice
 						$aRequestIndexes = \array_slice($aRequestIndexes, $oParams->iOffset ? $oParams->iLimit : 0, $oParams->iLimit);
-/*
+					} else {
+						// Fetch ID's from high to low
 						$end = \max(1, $oInfo->MESSAGES - $oParams->iOffset);
 						$start = \max(1, $end - $oParams->iLimit + 1);
-						// Attempt to sort REVERSE DATE
 						$aRequestIndexes = \range($end, $start);
-*/
-					} else {
-						$aRequestIndexes = \array_slice([1], $oParams->iOffset, 1);
 					}
 					$this->MessageListByRequestIndexOrUids($oMessageCollection, new SequenceSet($aRequestIndexes, false));
 				}
 			} else {
-				$aUids = [];
+				$aUids = ($bUseThreads && $oParams->iThreadUid)
+					? [$oParams->iThreadUid]
+					: $this->GetUids($oParams, '', $oMessageCollection->FolderHash, $bUseSort);
+
 				if ($bUseThreads) {
 					$aAllThreads = $this->MessageListThreadsMap($oMessageCollection, $oParams->oCacher);
 					$oMessageCollection->totalThreads = \count($aAllThreads);
 //					$iThreadLimit = $this->oImapClient->Settings->thread_limit;
 					if ($oParams->iThreadUid) {
-						$aUids = [$oParams->iThreadUid];
 						// Only show the selected thread messages
 						foreach ($aAllThreads as $aMap) {
 							if (\in_array($oParams->iThreadUid, $aMap)) {
@@ -727,8 +733,10 @@ class MailClient
 								break;
 							}
 						}
+						$aAllThreads = [];
+						// This only speeds up the search when not cached
+//						$oParams->oSequenceSet = new SequenceSet($aUids);
 					} else {
-						$aUids = $this->GetUids($oParams, '', $oMessageCollection->FolderHash, $oParams->bUseSortIfSupported);
 						// Remove all threaded UID's except the most recent of each thread
 						$threadedUids = [];
 						foreach ($aAllThreads as $aMap) {
@@ -737,12 +745,10 @@ class MailClient
 						}
 						$aUids = \array_diff($aUids, $threadedUids);
 					}
-				} else {
-					$aUids = $this->GetUids($oParams, '', $oMessageCollection->FolderHash, $oParams->bUseSortIfSupported);
 				}
 
 				if ($aUids && \strlen($sSearch)) {
-					$aSearchedUids = $this->GetUids($oParams, $sSearch, $oMessageCollection->FolderHash);
+					$aSearchedUids = $this->GetUids($oParams, $sSearch, $oMessageCollection->FolderHash/*, $bUseSort*/);
 					if ($bUseThreads && !$oParams->iThreadUid) {
 						$matchingThreadUids = [];
 						foreach ($aAllThreads as $aMap) {
@@ -759,34 +765,17 @@ class MailClient
 						});
 					}
 				}
+			}
 
+			if (\count($aUids)) {
 				$oMessageCollection->totalEmails = \count($aUids);
-
+				$aUids = \array_slice($aUids, $oParams->iOffset, $oParams->iLimit);
-				if (\count($aUids)) {
+				$this->MessageListByRequestIndexOrUids($oMessageCollection, new SequenceSet($aUids), $aAllThreads);
-					$this->MessageListByRequestIndexOrUids(
-						$oMessageCollection,
-						new SequenceSet(\array_slice($aUids, $oParams->iOffset, $oParams->iLimit))
-					);
-				}
 			}
 		} else if ($this->oLogger) {
 			$this->oLogger->Write('No messages in '.$oMessageCollection->FolderName);
 		}
 
-		if ($aAllThreads && !$oParams->iThreadUid) {
-			foreach ($oMessageCollection as $oMessage) {
-				$iUid = $oMessage->Uid;
-				// Find thread and set it.
-				// Used by GUI to delete/move the whole thread or other features
-				foreach ($aAllThreads as $aMap) {
-					if (\in_array($iUid, $aMap)) {
-						$oMessage->SetThreads($aMap);
-						break;
-					}
-				}
-			}
-		}
-
 		return $oMessageCollection;
 	}
 

snappymail/v/0.0.0/app/libraries/MailSo/Mail/MessageListParams.php

@@ -22,7 +22,7 @@ class MessageListParams
 		$oCacher = null;
 
 	public bool
-		$bUseSortIfSupported = false,
+		$bUseSort = true,
 		$bUseThreads = false,
 		$bHideDeleted = true;
 

snappymail/v/0.0.0/app/libraries/RainLoop/Actions/Messages.php

@@ -67,7 +67,7 @@ trait Messages
 			}
 
 			$oParams->oCacher = $this->cacherForUids();
-			$oParams->bUseSortIfSupported = true;
+			$oParams->bUseSort = true;
 
 			$oSettingsLocal = $this->SettingsProvider(true)->Load($oAccount);
 			if ($oSettingsLocal instanceof \RainLoop\Settings) {