Src/DirScan.cpp

   1 /**
   2  *  @file DirScan.cpp
   3  *
   4  *  @brief Implementation of DirScan (q.v.) and helper functions
   5  */
   6
   7 #include "pch.h"
   8 #include "DirScan.h"
   9 #include <cassert>
  10 #include <memory>
  11 #define POCO_NO_UNWINDOWS 1
  12 #include <Poco/Semaphore.h>
  13 #include <Poco/Notification.h>
  14 #include <Poco/NotificationQueue.h>
  15 #include <Poco/Environment.h>
  16 #include <Poco/ThreadPool.h>
  17 #include <Poco/Thread.h>
  18 #include <Poco/Runnable.h>
  19 #include <Poco/Mutex.h>
  20 #include <Poco/AutoPtr.h>
  21 #include <Poco/Stopwatch.h>
  22 #include <Poco/Format.h>
  23 #include "DiffThread.h"
  24 #include "UnicodeString.h"
  25 #include "DiffWrapper.h"
  26 #include "CompareStats.h"
  27 #include "FolderCmp.h"
  28 #include "FileFilterHelper.h"
  29 #include "IAbortable.h"
  30 #include "DirItem.h"
  31 #include "DirTravel.h"
  32 #include "paths.h"
  33 #include "Plugins.h"
  34 #include "MergeApp.h"
  35 #include "OptionsDef.h"
  36 #include "OptionsMgr.h"
  37 #include "PathContext.h"
  38 #include "DebugNew.h"
  39
  40 using Poco::NotificationQueue;
  41 using Poco::Notification;
  42 using Poco::AutoPtr;
  43 using Poco::Thread;
  44 using Poco::ThreadPool;
  45 using Poco::Runnable;
  46 using Poco::Environment;
  47 using Poco::Stopwatch;
  48
  49 // Static functions (ie, functions only used locally)
  50 static void CompareDiffItem(FolderCmp &fc, DIFFITEM &di);
  51 static DIFFITEM *AddToList(const String &sLeftDir, const String &sRightDir, const DirItem *lent, const DirItem *rent,
  52         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent);
  53 static DIFFITEM *AddToList(const String &sDir1, const String &sDir2, const String &sDir3, const DirItem *ent1, const DirItem *ent2, const DirItem *ent3,
  54         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent, int nItems = 3);
  55 static void UpdateDiffItem(DIFFITEM &di, bool &bExists, CDiffContext *pCtxt);
  56 static int CompareItems(NotificationQueue &queue, DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos);
  57
  58 class WorkNotification: public Poco::Notification
  59 {
  60 public:
  61         WorkNotification(DIFFITEM& di, NotificationQueue& queueResult): m_di(di), m_queueResult(queueResult) {}
  62         DIFFITEM& data() const { return m_di; }
  63         NotificationQueue& queueResult() const { return m_queueResult; }
  64 private:
  65         DIFFITEM& m_di;
  66         NotificationQueue& m_queueResult;
  67 };
  68
  69 class WorkCompletedNotification: public Poco::Notification
  70 {
  71 public:
  72         explicit WorkCompletedNotification(DIFFITEM& di): m_di(di) {}
  73         DIFFITEM& data() const { return m_di; }
  74 private:
  75         DIFFITEM& m_di;
  76 };
  77
  78 class DiffWorker: public Runnable
  79 {
  80 public:
  81         DiffWorker(NotificationQueue& queue, CDiffContext *pCtxt, int id):
  82           m_queue(queue), m_pCtxt(pCtxt), m_id(id) {}
  83
  84         void run()
  85         {
  86                 FolderCmp fc(m_pCtxt);
  87                 // keep the scripts alive during the Rescan
  88                 // when we exit the thread, we delete this and release the scripts
  89                 CAssureScriptsForThread scriptsForRescan;
  90
  91                 AutoPtr<Notification> pNf(m_queue.waitDequeueNotification());
  92                 while (pNf.get() != nullptr)
  93                 {
  94                         WorkNotification* pWorkNf = dynamic_cast<WorkNotification*>(pNf.get());
  95                         if (pWorkNf != nullptr) {
  96                                 m_pCtxt->m_pCompareStats->BeginCompare(&pWorkNf->data(), m_id);
  97                                 if (!m_pCtxt->ShouldAbort())
  98                                         CompareDiffItem(fc, pWorkNf->data());
  99                                 pWorkNf->queueResult().enqueueNotification(new WorkCompletedNotification(pWorkNf->data()));
 100                         }
 101                         pNf = m_queue.waitDequeueNotification();
 102                 }
 103         }
 104
 105 private:
 106         NotificationQueue& m_queue;
 107         CDiffContext *m_pCtxt;
 108         int m_id;
 109 };
 110
 111 typedef std::shared_ptr<DiffWorker> DiffWorkerPtr;
 112
 113 /**
 114  * @brief Collect file- and folder-names to list.
 115  * This function walks given folders and adds found subfolders and files into
 116  * lists. There are two modes, determined by the @p depth:
 117  * - in non-recursive mode we walk only given folders, and add files
 118  *   contained. Subfolders are added as folder items, not walked into.
 119  * - in recursive mode we walk all subfolders and add the files they
 120  *   contain into list.
 121  *
 122  * Items are tested against file filters in this function.
 123  *
 124  * @param [in] paths Root paths of compare
 125  * @param [in] leftsubdir Left side subdirectory under root path
 126  * @param [in] bLeftUniq Is left-side folder unique folder?
 127  * @param [in] rightsubdir Right side subdirectory under root path
 128  * @param [in] bRightUniq Is right-side folder unique folder?
 129  * @param [in] myStruct Compare-related data, like context etc.
 130  * @param [in] casesensitive Is filename compare case sensitive?
 131  * @param [in] depth Levels of subdirectories to scan, -1 scans all
 132  * @param [in] parent Folder diff item to be scanned
 133  * @param [in] bUniques If true, walk into unique folders.
 134  * @return 1 normally, -1 if compare was aborted
 135  */
 136 int DirScan_GetItems(const PathContext &paths, const String subdir[],
 137                 DiffFuncStruct *myStruct,
 138                 bool casesensitive, int depth, DIFFITEM *parent,
 139                 bool bUniques)
 140 {
 141         static const TCHAR backslash[] = _T("\\");
 142         int nDirs = paths.GetSize();
 143         CDiffContext *pCtxt = myStruct->context;
 144         String sDir[3];
 145         String subprefix[3];
 146
 147         std::copy(paths.begin(), paths.end(), sDir);
 148
 149         if (!subdir[0].empty())
 150         {
 151                 for (int nIndex = 0; nIndex < paths.GetSize(); nIndex++)
 152                 {
 153                         sDir[nIndex] = paths::ConcatPath(sDir[nIndex], subdir[nIndex]);
 154                         subprefix[nIndex] = subdir[nIndex] + backslash;
 155                 }
 156         }
 157
 158         DirItemArray dirs[3], aFiles[3];
 159         for (int nIndex = 0; nIndex < nDirs; nIndex++)
 160                 LoadAndSortFiles(sDir[nIndex], &dirs[nIndex], &aFiles[nIndex], casesensitive);
 161
 162         // Allow user to abort scanning
 163         if (pCtxt->ShouldAbort())
 164                 return -1;
 165
 166         // Handle directories
 167         // i points to current directory in left list (leftDirs)
 168         // j points to current directory in right list (rightDirs)
 169
 170         {
 171                 int nIndex;
 172                 for (nIndex = 0; nIndex < nDirs; nIndex++)
 173                         if (dirs[nIndex].size() != 0 || aFiles[nIndex].size() != 0) break;
 174                 if (nIndex == nDirs)
 175                         return 0;
 176         }
 177
 178         DirItemArray::size_type i=0, j=0, k=0;
 179         while (true)
 180         {
 181                 if (pCtxt->ShouldAbort())
 182                         return -1;
 183
 184                 if (i >= dirs[0].size() && j >= dirs[1].size() && (nDirs < 3 || k >= dirs[2].size()))
 185                         break;
 186
 187                 unsigned nDiffCode = DIFFCODE::DIR;
 188                 // Comparing directories leftDirs[i].name to rightDirs[j].name
 189                 if (i<dirs[0].size() && (j==dirs[1].size() || collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive)<0)
 190                         && (nDirs < 3 ||      (k==dirs[2].size() || collstr(dirs[0][i].filename, dirs[2][k].filename, casesensitive)<0) ))
 191                 {
 192                         nDiffCode |= DIFFCODE::FIRST;
 193                 }
 194                 else if (j<dirs[1].size() && (i==dirs[0].size() || collstr(dirs[1][j].filename, dirs[0][i].filename, casesensitive)<0)
 195                         && (nDirs < 3 ||      (k==dirs[2].size() || collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive)<0) ))
 196                 {
 197                         nDiffCode |= DIFFCODE::SECOND;
 198                 }
 199                 else if (nDirs < 3)
 200                 {
 201                         nDiffCode |= DIFFCODE::BOTH;
 202                 }
 203                 else
 204                 {
 205                         if (k<dirs[2].size() && (i==dirs[0].size() || collstr(dirs[2][k].filename, dirs[0][i].filename, casesensitive)<0)
 206                                 &&                     (j==dirs[1].size() || collstr(dirs[2][k].filename, dirs[1][j].filename, casesensitive)<0) )
 207                         {
 208                                 nDiffCode |= DIFFCODE::THIRD;
 209                         }
 210                         else if ((i<dirs[0].size() && j<dirs[1].size() && collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive) == 0)
 211                                 && (k==dirs[2].size() || collstr(dirs[2][k].filename, dirs[0][i].filename, casesensitive) != 0))
 212                         {
 213                                 nDiffCode |= DIFFCODE::FIRST | DIFFCODE::SECOND;
 214                         }
 215                         else if ((i<dirs[0].size() && k<dirs[2].size() && collstr(dirs[0][i].filename, dirs[2][k].filename, casesensitive) == 0)
 216                                 && (j==dirs[1].size() || collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive) != 0))
 217                         {
 218                                 nDiffCode |= DIFFCODE::FIRST | DIFFCODE::THIRD;
 219                         }
 220                         else if ((j<dirs[1].size() && k<dirs[2].size() && collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive) == 0)
 221                                 && (i==dirs[0].size() || collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive) != 0))
 222                         {
 223                                 nDiffCode |= DIFFCODE::SECOND | DIFFCODE::THIRD;
 224                         }
 225                         else
 226                         {
 227                                 nDiffCode |= DIFFCODE::ALL;
 228                         }
 229                 }
 230
 231                 String leftnewsub;
 232                 String rightnewsub;
 233                 String middlenewsub;
 234                 if (nDirs < 3)
 235                 {
 236                         leftnewsub  = (nDiffCode & DIFFCODE::FIRST)  ? subprefix[0] + dirs[0][i].filename.get() : subprefix[0] + dirs[1][j].filename.get();
 237                         rightnewsub = (nDiffCode & DIFFCODE::SECOND) ? subprefix[1] + dirs[1][j].filename.get() : subprefix[1] + dirs[0][i].filename.get();
 238
 239                         // Test against filter so we don't include contents of filtered out directories
 240                         // Also this is only place we can test for both-sides directories in recursive compare
 241                         if ((pCtxt->m_piFilterGlobal!=nullptr && !pCtxt->m_piFilterGlobal->includeDir(leftnewsub, rightnewsub)) ||
 242                                 (pCtxt->m_bIgnoreReparsePoints && (
 243                                 (nDiffCode & DIFFCODE::FIRST) && (dirs[0][i].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 244                                         (nDiffCode & DIFFCODE::SECOND) && (dirs[1][j].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT))
 245                                         )
 246                                 )
 247                                 nDiffCode |= DIFFCODE::SKIPPED;
 248                 }
 249                 else
 250                 {
 251                         leftnewsub   = subprefix[0];
 252                         if (nDiffCode & DIFFCODE::FIRST)       leftnewsub += dirs[0][i].filename;
 253                         else if (nDiffCode & DIFFCODE::SECOND) leftnewsub += dirs[1][j].filename;
 254                         else if (nDiffCode & DIFFCODE::THIRD)  leftnewsub += dirs[2][k].filename;
 255                         middlenewsub = subprefix[1];
 256                         if (nDiffCode & DIFFCODE::SECOND)      middlenewsub += dirs[1][j].filename;
 257                         else if (nDiffCode & DIFFCODE::FIRST)  middlenewsub += dirs[0][i].filename;
 258                         else if (nDiffCode & DIFFCODE::THIRD)  middlenewsub += dirs[2][k].filename;
 259                         rightnewsub  = subprefix[2];
 260                         if (nDiffCode & DIFFCODE::THIRD)       rightnewsub += dirs[2][k].filename;
 261                         else if (nDiffCode & DIFFCODE::FIRST)  rightnewsub += dirs[0][i].filename;
 262                         else if (nDiffCode & DIFFCODE::SECOND) rightnewsub += dirs[1][j].filename;
 263
 264                         // Test against filter so we don't include contents of filtered out directories
 265                         // Also this is only place we can test for both-sides directories in recursive compare
 266                         if ((pCtxt->m_piFilterGlobal!=nullptr && !pCtxt->m_piFilterGlobal->includeDir(leftnewsub, middlenewsub, rightnewsub)) ||
 267                                 (pCtxt->m_bIgnoreReparsePoints && (
 268                                   (nDiffCode & DIFFCODE::FIRST)  && (dirs[0][i].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 269                                   (nDiffCode & DIFFCODE::SECOND) && (dirs[1][j].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 270                                   (nDiffCode & DIFFCODE::THIRD)  && (dirs[2][k].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT))
 271                                 )
 272                            )
 273                                 nDiffCode |= DIFFCODE::SKIPPED;
 274                 }
 275
 276                 // add to list
 277                 if (!depth)
 278                 {
 279                         if (nDirs < 3)
 280                                 AddToList(subdir[0], subdir[1],
 281                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 282                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 283                                         nDiffCode, myStruct, parent);
 284                         else
 285                                 AddToList(subdir[0], subdir[1], subdir[2],
 286                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 287                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 288                                         (nDiffCode & DIFFCODE::THIRD ) ? &dirs[2][k] : nullptr,
 289                                         nDiffCode, myStruct, parent);
 290                 }
 291                 else
 292                 {
 293                         // Recursive compare
 294                         if (nDirs < 3)
 295                         {
 296                                 DIFFITEM *me = AddToList(subdir[0], subdir[1],
 297                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 298                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 299                                         nDiffCode, myStruct, parent);
 300                                 if ((nDiffCode & DIFFCODE::SKIPPED) == 0 && ((nDiffCode & DIFFCODE::SIDEFLAGS) == DIFFCODE::BOTH || bUniques))
 301                                 {
 302                                         // Scan recursively all subdirectories too, we are not adding folders
 303                                         String newsubdir[3] = {leftnewsub, rightnewsub};
 304                                         int result = DirScan_GetItems(paths, newsubdir, myStruct, casesensitive,
 305                                                         depth - 1, me, bUniques);
 306                                         if (result == -1)
 307                                                 return -1;
 308                                 }
 309                         }
 310                         else
 311                         {
 312                                 DIFFITEM *me = AddToList(subdir[0], subdir[1], subdir[2],
 313                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 314                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 315                                         (nDiffCode & DIFFCODE::THIRD ) ? &dirs[2][k] : nullptr,
 316                                         nDiffCode, myStruct, parent);
 317                                 if ((nDiffCode & DIFFCODE::SKIPPED) == 0 && ((nDiffCode & DIFFCODE::SIDEFLAGS) == DIFFCODE::ALL || bUniques))
 318                                 {
 319                                         // Scan recursively all subdirectories too, we are not adding folders
 320                                         String newsubdir[3] = {leftnewsub, middlenewsub, rightnewsub};
 321                                         int result = DirScan_GetItems(paths, newsubdir, myStruct, casesensitive,
 322                                                         depth - 1, me, bUniques);
 323                                         if (result == -1)
 324                                                 return -1;
 325                                 }
 326                         }
 327                 }
 328                 if (nDiffCode & DIFFCODE::FIRST)
 329                         i++;
 330                 if (nDiffCode & DIFFCODE::SECOND)
 331                         j++;
 332                 if (nDiffCode & DIFFCODE::THIRD)
 333                         k++;
 334         }
 335         // Handle files
 336         // i points to current file in left list (aFiles[0])
 337         // j points to current file in right list (aFiles[1])
 338         i=0, j=0, k=0;
 339         while (true)
 340         {
 341                 if (pCtxt->ShouldAbort())
 342                         return -1;
 343
 344
 345                 // Comparing file aFiles[0][i].name to aFiles[1][j].name
 346                 if (i<aFiles[0].size() && (j==aFiles[1].size() ||
 347                                 collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) < 0)
 348                         && (nDirs < 3 ||
 349                                 (k==aFiles[2].size() || collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive)<0) ))
 350                 {
 351                         if (nDirs < 3)
 352                         {
 353                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::FILE;
 354                                 AddToList(subdir[0], subdir[1], &aFiles[0][i], nullptr, nDiffCode, myStruct, parent);
 355                         }
 356                         else
 357                         {
 358                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::FILE;
 359                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], nullptr, nullptr, nDiffCode, myStruct, parent);
 360                         }
 361                         // Advance left pointer over left-only entry, and then retest with new pointers
 362                         ++i;
 363                         continue;
 364                 }
 365                 if (j<aFiles[1].size() && (i==aFiles[0].size() ||
 366                                 collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) > 0)
 367                         && (nDirs < 3 ||
 368                                 (k==aFiles[2].size() || collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive)<0) ))
 369                 {
 370                         const unsigned nDiffCode = DIFFCODE::SECOND | DIFFCODE::FILE;
 371                         if (nDirs < 3)
 372                                 AddToList(subdir[0], subdir[1], nullptr, &aFiles[1][j], nDiffCode, myStruct, parent);
 373                         else
 374                                 AddToList(subdir[0], subdir[1], subdir[2], nullptr, &aFiles[1][j], nullptr, nDiffCode, myStruct, parent);
 375                         // Advance right pointer over right-only entry, and then retest with new pointers
 376                         ++j;
 377                         continue;
 378                 }
 379                 if (nDirs == 3)
 380                 {
 381                         if (k<aFiles[2].size() && (i==aFiles[0].size() ||
 382                                         collstr(aFiles[2][k].filename, aFiles[0][i].filename, casesensitive)<0)
 383                                 && (j==aFiles[1].size() || collstr(aFiles[2][k].filename, aFiles[1][j].filename, casesensitive)<0) )
 384                         {
 385                                 const unsigned nDiffCode = DIFFCODE::THIRD | DIFFCODE::FILE;
 386                                 AddToList(subdir[0], subdir[1], subdir[2], nullptr, nullptr, &aFiles[2][k], nDiffCode, myStruct, parent);
 387                                 ++k;
 388                                 // Advance right pointer over right-only entry, and then retest with new pointers
 389                                 continue;
 390                         }
 391
 392                         if ((i<aFiles[0].size() && j<aFiles[1].size() && collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) == 0)
 393                             && (k==aFiles[2].size() || collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive) != 0))
 394                         {
 395                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::SECOND | DIFFCODE::FILE;
 396                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], &aFiles[1][j], nullptr, nDiffCode, myStruct, parent);
 397                                 ++i;
 398                                 ++j;
 399                                 continue;
 400                         }
 401                         else if ((i<aFiles[0].size() && k<aFiles[2].size() && collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive) == 0)
 402                             && (j==aFiles[1].size() || collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive) != 0))
 403                         {
 404                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::THIRD | DIFFCODE::FILE;
 405                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], nullptr, &aFiles[2][k], nDiffCode, myStruct, parent);
 406                                 ++i;
 407                                 ++k;
 408                                 continue;
 409                         }
 410                         else if ((j<aFiles[1].size() && k<aFiles[2].size() && collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive) == 0)
 411                             && (i==aFiles[0].size() || collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) != 0))
 412                         {
 413                                 const unsigned nDiffCode = DIFFCODE::SECOND | DIFFCODE::THIRD | DIFFCODE::FILE;
 414                                 AddToList(subdir[0], subdir[1], subdir[2], nullptr, &aFiles[1][j], &aFiles[2][k], nDiffCode, myStruct, parent);
 415                                 ++j;
 416                                 ++k;
 417                                 continue;
 418                         }
 419                 }
 420                 if (i<aFiles[0].size())
 421                 {
 422                         if (nDirs < 3)
 423                         {
 424                                 assert(j<aFiles[1].size());
 425                                 const unsigned nDiffCode = DIFFCODE::BOTH | DIFFCODE::FILE;
 426                                 AddToList(subdir[0], subdir[1], &aFiles[0][i], &aFiles[1][j], nDiffCode, myStruct, parent);
 427                                 ++i;
 428                                 ++j;
 429                                 continue;
 430                         }
 431                         else
 432                         {
 433                                 assert(j<aFiles[1].size());
 434                                 assert(k<aFiles[2].size());
 435                                 const unsigned nDiffCode = DIFFCODE::ALL | DIFFCODE::FILE;
 436                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], &aFiles[1][j], &aFiles[2][k], nDiffCode, myStruct, parent);
 437                                 ++i;
 438                                 ++j;
 439                                 ++k;
 440                                 continue;
 441                         }
 442                 }
 443                 break;
 444         }
 445
 446         if (parent != nullptr)
 447         {
 448                 for (int nIndex = 0; nIndex < nDirs; ++nIndex)
 449                         if (parent->diffcode.exists(nIndex) && parent->diffFileInfo[nIndex].size == DirItem::FILE_SIZE_NONE)
 450                                 parent->diffFileInfo[nIndex].size = 0;
 451
 452                 DIFFITEM *dic = parent->GetFirstChild();
 453                 while (dic)
 454                 {
 455                         for (int nIndex = 0; nIndex < nDirs; ++nIndex)
 456                         {
 457                                 if (dic->diffFileInfo[nIndex].size != DirItem::FILE_SIZE_NONE)
 458                                         parent->diffFileInfo[nIndex].size += dic->diffFileInfo[nIndex].size;
 459                         }
 460                         dic = dic->GetFwdSiblingLink();
 461                 }
 462         }
 463
 464         return 1;
 465 }
 466
 467 /**
 468  * @brief Compare DiffItems in list and add results to compare context.
 469  *
 470  * @param myStruct [in] A structure containing compare-related data.
 471  * @param parentdiffpos [in] Position of parent diff item
 472  * @return >= 0 number of diff items, -1 if compare was aborted
 473  */
 474 int DirScan_CompareItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 475 {
 476         const int compareMethod = myStruct->context->GetCompareMethod();
 477         int nworkers = 1;
 478
 479         if (compareMethod == CMP_CONTENT || compareMethod == CMP_QUICK_CONTENT)
 480         {
 481                 nworkers = GetOptionsMgr()->GetInt(OPT_CMP_COMPARE_THREADS);
 482                 if (nworkers <= 0)
 483                         nworkers += Environment::processorCount();
 484                 nworkers = std::clamp(nworkers, 1, static_cast<int>(Environment::processorCount()));
 485         }
 486
 487         ThreadPool threadPool(nworkers, nworkers);
 488         std::vector<DiffWorkerPtr> workers;
 489         NotificationQueue queue;
 490         myStruct->context->m_pCompareStats->SetCompareThreadCount(nworkers);
 491         for (int i = 0; i < nworkers; ++i)
 492         {
 493                 workers.push_back(DiffWorkerPtr(new DiffWorker(queue, myStruct->context, i)));
 494                 threadPool.start(*workers[i]);
 495         }
 496
 497         int res = CompareItems(queue, myStruct, parentdiffpos);
 498
 499         Thread::sleep(100);
 500         queue.wakeUpAll();
 501         threadPool.joinAll();
 502
 503         return res;
 504 }
 505
 506 static int CompareItems(NotificationQueue& queue, DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 507 {
 508         NotificationQueue queueResult;
 509         Stopwatch stopwatch;
 510         CDiffContext *pCtxt = myStruct->context;
 511         int res = 0;
 512         int count = 0;
 513         bool bCompareFailure = false;
 514         if (parentdiffpos == nullptr)
 515                 myStruct->pSemaphore->wait();
 516         stopwatch.start();
 517         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 518         while (pos != nullptr)
 519         {
 520                 if (pCtxt->ShouldAbort())
 521                         break;
 522
 523                 if (stopwatch.elapsed() > 2000000)
 524                 {
 525                         int event = CDiffThread::EVENT_COMPARE_PROGRESSED;
 526                         myStruct->m_listeners.notify(myStruct, event);
 527                         stopwatch.restart();
 528                 }
 529                 myStruct->pSemaphore->wait();
 530                 DIFFITEM *curpos = pos;
 531                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 532                 bool existsalldirs = di.diffcode.existAll();
 533                 if (di.diffcode.isDirectory() && pCtxt->m_bRecursive)
 534                 {
 535                         if ((di.diffcode.diffcode & DIFFCODE::CMPERR) != DIFFCODE::CMPERR)
 536                         {       // Only clear DIFF|SAME flags if not CMPERR (eg. both flags together)
 537                                 di.diffcode.diffcode &= ~(DIFFCODE::DIFF | DIFFCODE::SAME);
 538                         }
 539                         int ndiff = CompareItems(queue, myStruct, curpos);
 540                         // Propogate sub-directory status to this directory
 541                         if (ndiff > 0)
 542                         {       // There were differences in the sub-directories
 543                                 if (existsalldirs)
 544                                         di.diffcode.diffcode |= DIFFCODE::DIFF;
 545                                 res += ndiff;
 546                         }
 547                         else
 548                         if (ndiff == 0)
 549                         {       // Sub-directories were identical
 550                                 if (existsalldirs)
 551                                         di.diffcode.diffcode |= DIFFCODE::SAME;
 552                         }
 553                         else
 554                         if (ndiff == -1)
 555                         {       // There were file IO-errors during sub-directory comparison.
 556                                 di.diffcode.diffcode |= DIFFCODE::CMPERR;
 557                                 bCompareFailure = true;
 558                         }
 559                 }
 560                 if (existsalldirs)
 561                         queue.enqueueUrgentNotification(new WorkNotification(di, queueResult));
 562                 else
 563                         queue.enqueueNotification(new WorkNotification(di, queueResult));
 564                 ++count;
 565                 pos = curpos;
 566                 pCtxt->GetNextSiblingDiffRefPosition(pos);
 567         }
 568
 569         while (count > 0)
 570         {
 571                 AutoPtr<Notification> pNf(queueResult.waitDequeueNotification());
 572                 if (pNf.get() == nullptr)
 573                         break;
 574                 WorkCompletedNotification* pWorkCompletedNf = dynamic_cast<WorkCompletedNotification*>(pNf.get());
 575                 if (pWorkCompletedNf != nullptr) {
 576                         DIFFITEM &di = pWorkCompletedNf->data();
 577                         if (di.diffcode.isResultError()) {
 578                                 DIFFITEM *diParent = di.GetParentLink();
 579                                 assert(diParent != nullptr);
 580                                 if (diParent != nullptr)
 581                                 {
 582                                         diParent->diffcode.diffcode |= DIFFCODE::CMPERR;
 583                                         bCompareFailure = true;
 584                                 }
 585                         }
 586
 587                         if (di.diffcode.isResultDiff() ||
 588                                 (!di.diffcode.existAll() && !di.diffcode.isResultFiltered()))
 589                                 res++;
 590                 }
 591                 --count;
 592         }
 593
 594         return bCompareFailure || pCtxt->ShouldAbort() ? -1 : res;
 595 }
 596
 597 /**
 598  * @brief Compare DiffItems in context marked for rescan.
 599  *
 600  * @param myStruct [in,out] A structure containing compare-related data.
 601  * @param parentdiffpos [in] Position of parent diff item
 602  * @return >= 0 number of diff items, -1 if compare was aborted
 603  */
 604 static int CompareRequestedItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 605 {
 606         CDiffContext *pCtxt = myStruct->context;
 607         FolderCmp fc(pCtxt);
 608         int res = 0;
 609         bool bCompareFailure = false;
 610         if (parentdiffpos == nullptr)
 611                 myStruct->pSemaphore->wait();
 612         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 613         while (pos != nullptr)
 614         {
 615                 if (pCtxt->ShouldAbort())
 616                 {
 617                         res = -1;
 618                         break;
 619                 }
 620
 621                 DIFFITEM *curpos = pos;
 622                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 623                 bool existsalldirs = di.diffcode.existAll();
 624                 if (di.diffcode.isDirectory())
 625                 {
 626                         if (pCtxt->m_bRecursive)
 627                         {
 628                                 di.diffcode.diffcode &= ~(DIFFCODE::DIFF | DIFFCODE::SAME);
 629                                 int ndiff = CompareRequestedItems(myStruct, curpos);
 630                                 if (ndiff > 0)
 631                                 {
 632                                         if (existsalldirs)
 633                                                 di.diffcode.diffcode |= DIFFCODE::DIFF;
 634                                         res += ndiff;
 635                                 }
 636                                 else
 637                                 if (ndiff == 0)
 638                                 {
 639                                         if (existsalldirs)
 640                                                 di.diffcode.diffcode |= DIFFCODE::SAME;
 641                                 }
 642                                 else
 643                                 if (ndiff == -1)
 644                                 {       // There were file IO-errors during sub-directory comparison.
 645                                         di.diffcode.diffcode |= DIFFCODE::CMPERR;
 646                                         bCompareFailure = true;
 647                                 }
 648                         }
 649                 }
 650                 else
 651                 {
 652                         if (di.diffcode.isScanNeeded())
 653                         {
 654                                 CompareDiffItem(fc, di);
 655                                 if (di.diffcode.isResultError()) {
 656                                         DIFFITEM *diParent = di.GetParentLink();
 657                                         assert(diParent != nullptr);
 658                                         if (diParent != nullptr)
 659                                         {
 660                                                 diParent->diffcode.diffcode |= DIFFCODE::CMPERR;
 661                                                 bCompareFailure = true;
 662                                         }
 663                         }
 664
 665                         }
 666                 }
 667                 if (di.diffcode.isResultDiff() ||
 668                         (!existsalldirs && !di.diffcode.isResultFiltered()))
 669                         res++;
 670         }
 671         return bCompareFailure ? -1 : res;;
 672 }
 673
 674 int DirScan_CompareRequestedItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 675 {
 676         return CompareRequestedItems(myStruct, parentdiffpos);
 677 }
 678
 679 static int markChildrenForRescan(CDiffContext *pCtxt, DIFFITEM *parentdiffpos)
 680 {
 681         int ncount = 0;
 682         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 683         while (pos != nullptr)
 684         {
 685                 DIFFITEM *curpos = pos;
 686                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 687                 if (di.diffcode.isDirectory())
 688                         ncount += markChildrenForRescan(pCtxt, curpos);
 689                 else
 690                 {
 691                         di.diffcode.diffcode |= DIFFCODE::NEEDSCAN;
 692                         ++ncount;
 693                 }
 694         }
 695         return ncount;
 696 }
 697
 698 int DirScan_UpdateMarkedItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 699 {
 700         CDiffContext *pCtxt = myStruct->context;
 701         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 702         int ncount = 0;
 703
 704         while (pos != nullptr)
 705         {
 706                 if (pCtxt->ShouldAbort())
 707                         break;
 708                 DIFFITEM *curpos = pos;
 709                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 710                 if (di.diffcode.isScanNeeded())
 711                 {
 712                         bool bItemsExist = true;
 713                         UpdateDiffItem(di, bItemsExist, pCtxt);
 714                         if (!bItemsExist)
 715                         {
 716                                 di.DelinkFromSiblings();        // delink from list of Siblings
 717                                 delete &di;                                     // Also delete all Children items
 718                                 continue;                                       // (... because `di` is now invalid)
 719                         }
 720                         if (!di.diffcode.isDirectory())
 721                                 ++ncount;
 722                 }
 723                 if (di.diffcode.isDirectory() && pCtxt->m_bRecursive)
 724                 {
 725                         for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 726                                 if (di.diffcode.exists(i))
 727                                         di.diffFileInfo[i].size = 0;
 728                         if (di.diffcode.isScanNeeded() && !di.diffcode.isResultFiltered())
 729                         {
 730                                 di.RemoveChildren();
 731                                 di.diffcode.diffcode &= ~DIFFCODE::NEEDSCAN;
 732
 733                                 bool casesensitive = false;
 734                                 int depth = myStruct->context->m_bRecursive ? -1 : 0;
 735                                 String subdir[3];
 736                                 PathContext paths = myStruct->context->GetNormalizedPaths();
 737                                 for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 738                                         subdir[i] = di.diffFileInfo[i].GetFile();
 739                                 DirScan_GetItems(paths, subdir, myStruct,
 740                                         casesensitive, depth, &di, myStruct->context->m_bWalkUniques);
 741                                 ncount += markChildrenForRescan(myStruct->context, curpos);
 742                         }
 743                         else
 744                         {
 745                                 ncount += DirScan_UpdateMarkedItems(myStruct, curpos);
 746                         }
 747                 }
 748                 if (parentdiffpos != nullptr && pCtxt->m_bRecursive)
 749                 {
 750                         for (int nIndex = 0; nIndex < pCtxt->GetCompareDirs(); ++nIndex)
 751                                 if (curpos->diffFileInfo[nIndex].size != DirItem::FILE_SIZE_NONE)
 752                                         parentdiffpos->diffFileInfo[nIndex].size += curpos->diffFileInfo[nIndex].size;
 753                 }
 754         }
 755         return ncount;
 756 }
 757 /**
 758  * @brief Update diffitem file/dir infos.
 759  *
 760  * Re-tests dirs/files if sides still exists, and updates infos for
 761  * existing sides. This assumes filenames, or paths are not changed.
 762  * Since in normal situations (I can think of) they cannot change
 763  * after first compare.
 764  *
 765  * @param [in,out] di DiffItem to update.
 766  * @param [out] bExists Set to
 767  *  - true if one of items exists so diffitem is valid
 768  *  - false if items were deleted, so diffitem is not valid
 769  * @param [in] pCtxt Compare context
 770  */
 771 static void UpdateDiffItem(DIFFITEM &di, bool & bExists, CDiffContext *pCtxt)
 772 {
 773         bExists = false;
 774         di.diffcode.setSideNone();
 775         for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 776         {
 777                 di.diffFileInfo[i].ClearPartial();
 778                 if (pCtxt->UpdateInfoFromDiskHalf(di, i))
 779                 {
 780                         di.diffcode.diffcode |= DIFFCODE::FIRST << i;
 781                         bExists = true;
 782                 }
 783         }
 784 }
 785
 786 /**
 787  * @brief Compare two diffitems and add results to difflist in context.
 788  *
 789  * This function does the actual compare for previously gathered list of
 790  * items. Basically we:
 791  * - ignore items matching file filters
 792  * - add non-ignored directories (no compare for directory items)
 793  * - add unique files
 794  * - compare files
 795  *
 796  * @param [in] di DiffItem to compare
 797  * @param [in,out] pCtxt Compare context: contains difflist, encoding info etc.
 798  * @todo For date compare, maybe we should use creation date if modification
 799  * date is missing?
 800  */
 801 static void CompareDiffItem(FolderCmp &fc, DIFFITEM &di)
 802 {
 803         CDiffContext *const pCtxt = fc.m_pCtxt;
 804         int nDirs = pCtxt->GetCompareDirs();
 805         // Clear rescan-request flag (not set by all codepaths)
 806         di.diffcode.diffcode &= ~DIFFCODE::NEEDSCAN;
 807         // Is it a directory?
 808         if (di.diffcode.isDirectory())
 809         {
 810                 // We don't actually 'compare' directories, just add non-ignored
 811                 // directories to list.
 812         }
 813         else
 814         {
 815                 // 1. Test against filters
 816                 if (pCtxt->m_piFilterGlobal==nullptr ||
 817                         (nDirs == 2 && pCtxt->m_piFilterGlobal->includeFile(di.diffFileInfo[0].filename, di.diffFileInfo[1].filename)) ||
 818                         (nDirs == 3 && pCtxt->m_piFilterGlobal->includeFile(di.diffFileInfo[0].filename, di.diffFileInfo[1].filename, di.diffFileInfo[2].filename))
 819                         )
 820                 {
 821                         di.diffcode.diffcode |= DIFFCODE::INCLUDED;
 822                         di.diffcode.diffcode |= fc.prepAndCompareFiles(di);
 823                         di.nsdiffs = fc.m_ndiffs;
 824                         di.nidiffs = fc.m_ntrivialdiffs;
 825
 826                         for (int i = 0; i < nDirs; ++i)
 827                         {
 828                                 // Set text statistics
 829                                 if (di.diffcode.exists(i))
 830                                 {
 831                                         di.diffFileInfo[i].m_textStats = fc.m_diffFileData.m_textStats[i];
 832                                         di.diffFileInfo[i].encoding = fc.m_diffFileData.m_FileLocation[i].encoding;
 833                                 }
 834                         }
 835                 }
 836                 else
 837                 {
 838                         di.diffcode.diffcode |= DIFFCODE::SKIPPED;
 839                 }
 840         }
 841         pCtxt->m_pCompareStats->AddItem(di.diffcode.diffcode);
 842 }
 843
 844 /**
 845  * @brief Add one compare item to list.
 846  * @param [in] sLeftDir Left subdirectory.
 847  * @param [in] sRightDir Right subdirectory.
 848  * @param [in] lent Left item data to add.
 849  * @param [in] rent Right item data to add.
 850  * @param [in] pCtxt Compare context.
 851  * @param [in] parent Parent of item to be added
 852  */
 853 static DIFFITEM *AddToList(const String& sLeftDir, const String& sRightDir,
 854         const DirItem * lent, const DirItem * rent,
 855         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent)
 856 {
 857         return AddToList(sLeftDir, sRightDir, sLeftDir, lent, rent, nullptr, code, myStruct, parent, 2);
 858 }
 859
 860 /**
 861  * @brief Add one compare item to list.
 862  */
 863 static DIFFITEM *AddToList(const String& sDir1, const String& sDir2, const String& sDir3,
 864         const DirItem *ent1, const DirItem *ent2, const DirItem *ent3,
 865         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent, int nItems /*= 3*/)
 866 {
 867         // We must store both paths - we cannot get paths later
 868         // and we need unique item paths for example when items
 869         // change to identical
 870         DIFFITEM *di = myStruct->context->AddNewDiff(parent);
 871
 872         di->diffFileInfo[0].path = sDir1;
 873         di->diffFileInfo[1].path = sDir2;
 874         di->diffFileInfo[2].path = sDir3;
 875
 876         if (ent1 != nullptr)
 877         {
 878                 di->diffFileInfo[0].filename = ent1->filename;
 879                 di->diffFileInfo[0].mtime = ent1->mtime;
 880                 di->diffFileInfo[0].ctime = ent1->ctime;
 881                 di->diffFileInfo[0].size = ent1->size;
 882                 di->diffFileInfo[0].flags.attributes = ent1->flags.attributes;
 883         }
 884         else
 885         {
 886                 // Don't break CDirView::DoCopyRightToLeft()
 887                 if (ent3 != nullptr)
 888                         di->diffFileInfo[0].filename = ent3->filename;
 889                 else if (ent2 != nullptr)
 890                         di->diffFileInfo[0].filename = ent2->filename;
 891         }
 892
 893         if (ent2 != nullptr)
 894         {
 895                 di->diffFileInfo[1].filename = ent2->filename;
 896                 di->diffFileInfo[1].mtime = ent2->mtime;
 897                 di->diffFileInfo[1].ctime = ent2->ctime;
 898                 di->diffFileInfo[1].size = ent2->size;
 899                 di->diffFileInfo[1].flags.attributes = ent2->flags.attributes;
 900         }
 901         else
 902         {
 903                 // Don't break CDirView::DoCopyLeftToRight()
 904                 if (ent1 != nullptr)
 905                         di->diffFileInfo[1].filename = ent1->filename;
 906                 else if (ent3 != nullptr)
 907                         di->diffFileInfo[1].filename = ent3->filename;
 908         }
 909
 910         if (ent3 != nullptr)
 911         {
 912                 di->diffFileInfo[2].filename = ent3->filename;
 913                 di->diffFileInfo[2].mtime = ent3->mtime;
 914                 di->diffFileInfo[2].ctime = ent3->ctime;
 915                 di->diffFileInfo[2].size = ent3->size;
 916                 di->diffFileInfo[2].flags.attributes = ent3->flags.attributes;
 917         }
 918         else
 919         {
 920                 // Don't break CDirView::DoCopyLeftToRight()
 921                 if (ent1 != nullptr)
 922                         di->diffFileInfo[2].filename = ent1->filename;
 923                 else if (ent2 != nullptr)
 924                         di->diffFileInfo[2].filename = ent2->filename;
 925         }
 926
 927         if (nItems == 2)
 928                 di->diffcode.diffcode = code;
 929         else
 930                 di->diffcode.diffcode = code | DIFFCODE::THREEWAY;
 931
 932         if (myStruct->m_fncCollect)
 933         {
 934                 myStruct->context->m_pCompareStats->IncreaseTotalItems();
 935                 myStruct->pSemaphore->set();
 936         }
 937         return di;
 938 }