Src/DirScan.cpp

   1 /**
   2  *  @file DirScan.cpp
   3  *
   4  *  @brief Implementation of DirScan (q.v.) and helper functions
   5  */
   6
   7 #include "pch.h"
   8 #include "DirScan.h"
   9 #include <cassert>
  10 #include <memory>
  11 #define POCO_NO_UNWINDOWS 1
  12 #include <Poco/Semaphore.h>
  13 #include <Poco/Notification.h>
  14 #include <Poco/NotificationQueue.h>
  15 #include <Poco/Environment.h>
  16 #include <Poco/ThreadPool.h>
  17 #include <Poco/Thread.h>
  18 #include <Poco/Runnable.h>
  19 #include <Poco/Mutex.h>
  20 #include <Poco/AutoPtr.h>
  21 #include <Poco/Stopwatch.h>
  22 #include <Poco/Format.h>
  23 #include "DiffThread.h"
  24 #include "UnicodeString.h"
  25 #include "DiffWrapper.h"
  26 #include "CompareStats.h"
  27 #include "FolderCmp.h"
  28 #include "FileFilterHelper.h"
  29 #include "IAbortable.h"
  30 #include "DirItem.h"
  31 #include "DirTravel.h"
  32 #include "paths.h"
  33 #include "Plugins.h"
  34 #include "MergeApp.h"
  35 #include "OptionsDef.h"
  36 #include "OptionsMgr.h"
  37 #include "PathContext.h"
  38 #include "DebugNew.h"
  39
  40 using Poco::NotificationQueue;
  41 using Poco::Notification;
  42 using Poco::AutoPtr;
  43 using Poco::Thread;
  44 using Poco::ThreadPool;
  45 using Poco::Runnable;
  46 using Poco::Environment;
  47 using Poco::Stopwatch;
  48
  49 // Static functions (ie, functions only used locally)
  50 static void CompareDiffItem(FolderCmp &fc, DIFFITEM &di);
  51 static DIFFITEM *AddToList(const String &sLeftDir, const String &sRightDir, const DirItem *lent, const DirItem *rent,
  52         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent);
  53 static DIFFITEM *AddToList(const String &sDir1, const String &sDir2, const String &sDir3, const DirItem *ent1, const DirItem *ent2, const DirItem *ent3,
  54         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent, int nItems = 3);
  55 static void UpdateDiffItem(DIFFITEM &di, bool &bExists, CDiffContext *pCtxt);
  56 static int CompareItems(NotificationQueue &queue, DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos);
  57
  58 class WorkNotification: public Poco::Notification
  59 {
  60 public:
  61         WorkNotification(DIFFITEM& di, NotificationQueue& queueResult): m_di(di), m_queueResult(queueResult) {}
  62         DIFFITEM& data() const { return m_di; }
  63         NotificationQueue& queueResult() const { return m_queueResult; }
  64 private:
  65         DIFFITEM& m_di;
  66         NotificationQueue& m_queueResult;
  67 };
  68
  69 class WorkCompletedNotification: public Poco::Notification
  70 {
  71 public:
  72         explicit WorkCompletedNotification(DIFFITEM& di): m_di(di) {}
  73         DIFFITEM& data() const { return m_di; }
  74 private:
  75         DIFFITEM& m_di;
  76 };
  77
  78 class DiffWorker: public Runnable
  79 {
  80 public:
  81         DiffWorker(NotificationQueue& queue, CDiffContext *pCtxt, int id):
  82           m_queue(queue), m_pCtxt(pCtxt), m_id(id) {}
  83
  84         void run()
  85         {
  86                 FolderCmp fc(m_pCtxt);
  87                 // keep the scripts alive during the Rescan
  88                 // when we exit the thread, we delete this and release the scripts
  89                 CAssureScriptsForThread scriptsForRescan;
  90
  91                 AutoPtr<Notification> pNf(m_queue.waitDequeueNotification());
  92                 while (pNf.get() != nullptr)
  93                 {
  94                         WorkNotification* pWorkNf = dynamic_cast<WorkNotification*>(pNf.get());
  95                         if (pWorkNf != nullptr) {
  96                                 m_pCtxt->m_pCompareStats->BeginCompare(&pWorkNf->data(), m_id);
  97                                 if (!m_pCtxt->ShouldAbort())
  98                                         CompareDiffItem(fc, pWorkNf->data());
  99                                 pWorkNf->queueResult().enqueueNotification(new WorkCompletedNotification(pWorkNf->data()));
 100                         }
 101                         pNf = m_queue.waitDequeueNotification();
 102                 }
 103         }
 104
 105 private:
 106         NotificationQueue& m_queue;
 107         CDiffContext *m_pCtxt;
 108         int m_id;
 109 };
 110
 111 typedef std::shared_ptr<DiffWorker> DiffWorkerPtr;
 112
 113 /**
 114  * @brief Collect file- and folder-names to list.
 115  * This function walks given folders and adds found subfolders and files into
 116  * lists. There are two modes, determined by the @p depth:
 117  * - in non-recursive mode we walk only given folders, and add files
 118  *   contained. Subfolders are added as folder items, not walked into.
 119  * - in recursive mode we walk all subfolders and add the files they
 120  *   contain into list.
 121  *
 122  * Items are tested against file filters in this function.
 123  *
 124  * @param [in] paths Root paths of compare
 125  * @param [in] leftsubdir Left side subdirectory under root path
 126  * @param [in] bLeftUniq Is left-side folder unique folder?
 127  * @param [in] rightsubdir Right side subdirectory under root path
 128  * @param [in] bRightUniq Is right-side folder unique folder?
 129  * @param [in] myStruct Compare-related data, like context etc.
 130  * @param [in] casesensitive Is filename compare case sensitive?
 131  * @param [in] depth Levels of subdirectories to scan, -1 scans all
 132  * @param [in] parent Folder diff item to be scanned
 133  * @param [in] bUniques If true, walk into unique folders.
 134  * @return 1 normally, -1 if compare was aborted
 135  */
 136 int DirScan_GetItems(const PathContext &paths, const String subdir[],
 137                 DiffFuncStruct *myStruct,
 138                 bool casesensitive, int depth, DIFFITEM *parent,
 139                 bool bUniques)
 140 {
 141         static const TCHAR backslash[] = _T("\\");
 142         int nDirs = paths.GetSize();
 143         CDiffContext *pCtxt = myStruct->context;
 144         String sDir[3];
 145         String subprefix[3];
 146
 147         std::copy(paths.begin(), paths.end(), sDir);
 148
 149         if (!subdir[0].empty())
 150         {
 151                 for (int nIndex = 0; nIndex < paths.GetSize(); nIndex++)
 152                 {
 153                         sDir[nIndex] = paths::ConcatPath(sDir[nIndex], subdir[nIndex]);
 154                         subprefix[nIndex] = subdir[nIndex] + backslash;
 155                 }
 156         }
 157
 158         DirItemArray dirs[3], aFiles[3];
 159         for (int nIndex = 0; nIndex < nDirs; nIndex++)
 160                 LoadAndSortFiles(sDir[nIndex], &dirs[nIndex], &aFiles[nIndex], casesensitive);
 161
 162         // Allow user to abort scanning
 163         if (pCtxt->ShouldAbort())
 164                 return -1;
 165
 166         // Handle directories
 167         // i points to current directory in left list (leftDirs)
 168         // j points to current directory in right list (rightDirs)
 169
 170         {
 171                 int nIndex;
 172                 for (nIndex = 0; nIndex < nDirs; nIndex++)
 173                         if (dirs[nIndex].size() != 0 || aFiles[nIndex].size() != 0) break;
 174                 if (nIndex == nDirs)
 175                         return 0;
 176         }
 177
 178         DirItemArray::size_type i=0, j=0, k=0;
 179         while (true)
 180         {
 181                 if (pCtxt->ShouldAbort())
 182                         return -1;
 183
 184                 if (i >= dirs[0].size() && j >= dirs[1].size() && (nDirs < 3 || k >= dirs[2].size()))
 185                         break;
 186
 187                 unsigned nDiffCode = DIFFCODE::DIR;
 188                 // Comparing directories leftDirs[i].name to rightDirs[j].name
 189                 if (i<dirs[0].size() && (j==dirs[1].size() || collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive)<0)
 190                         && (nDirs < 3 ||      (k==dirs[2].size() || collstr(dirs[0][i].filename, dirs[2][k].filename, casesensitive)<0) ))
 191                 {
 192                         nDiffCode |= DIFFCODE::FIRST;
 193                 }
 194                 else if (j<dirs[1].size() && (i==dirs[0].size() || collstr(dirs[1][j].filename, dirs[0][i].filename, casesensitive)<0)
 195                         && (nDirs < 3 ||      (k==dirs[2].size() || collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive)<0) ))
 196                 {
 197                         nDiffCode |= DIFFCODE::SECOND;
 198                 }
 199                 else if (nDirs < 3)
 200                 {
 201                         nDiffCode |= DIFFCODE::BOTH;
 202                 }
 203                 else
 204                 {
 205                         if (k<dirs[2].size() && (i==dirs[0].size() || collstr(dirs[2][k].filename, dirs[0][i].filename, casesensitive)<0)
 206                                 &&                     (j==dirs[1].size() || collstr(dirs[2][k].filename, dirs[1][j].filename, casesensitive)<0) )
 207                         {
 208                                 nDiffCode |= DIFFCODE::THIRD;
 209                         }
 210                         else if ((i<dirs[0].size() && j<dirs[1].size() && collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive) == 0)
 211                                 && (k==dirs[2].size() || collstr(dirs[2][k].filename, dirs[0][i].filename, casesensitive) != 0))
 212                         {
 213                                 nDiffCode |= DIFFCODE::FIRST | DIFFCODE::SECOND;
 214                         }
 215                         else if ((i<dirs[0].size() && k<dirs[2].size() && collstr(dirs[0][i].filename, dirs[2][k].filename, casesensitive) == 0)
 216                                 && (j==dirs[1].size() || collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive) != 0))
 217                         {
 218                                 nDiffCode |= DIFFCODE::FIRST | DIFFCODE::THIRD;
 219                         }
 220                         else if ((j<dirs[1].size() && k<dirs[2].size() && collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive) == 0)
 221                                 && (i==dirs[0].size() || collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive) != 0))
 222                         {
 223                                 nDiffCode |= DIFFCODE::SECOND | DIFFCODE::THIRD;
 224                         }
 225                         else
 226                         {
 227                                 nDiffCode |= DIFFCODE::ALL;
 228                         }
 229                 }
 230
 231                 String leftnewsub;
 232                 String rightnewsub;
 233                 String middlenewsub;
 234                 if (nDirs < 3)
 235                 {
 236                         leftnewsub  = (nDiffCode & DIFFCODE::FIRST)  ? subprefix[0] + dirs[0][i].filename.get() : subprefix[0] + dirs[1][j].filename.get();
 237                         rightnewsub = (nDiffCode & DIFFCODE::SECOND) ? subprefix[1] + dirs[1][j].filename.get() : subprefix[1] + dirs[0][i].filename.get();
 238
 239                         // Test against filter so we don't include contents of filtered out directories
 240                         // Also this is only place we can test for both-sides directories in recursive compare
 241                         if ((pCtxt->m_piFilterGlobal!=nullptr && !pCtxt->m_piFilterGlobal->includeDir(leftnewsub, rightnewsub)) ||
 242                                 (pCtxt->m_bIgnoreReparsePoints && (
 243                                 (nDiffCode & DIFFCODE::FIRST) && (dirs[0][i].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 244                                         (nDiffCode & DIFFCODE::SECOND) && (dirs[1][j].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT))
 245                                         )
 246                                 )
 247                                 nDiffCode |= DIFFCODE::SKIPPED;
 248                 }
 249                 else
 250                 {
 251                         leftnewsub   = subprefix[0];
 252                         if (nDiffCode & DIFFCODE::FIRST)       leftnewsub += dirs[0][i].filename;
 253                         else if (nDiffCode & DIFFCODE::SECOND) leftnewsub += dirs[1][j].filename;
 254                         else if (nDiffCode & DIFFCODE::THIRD)  leftnewsub += dirs[2][k].filename;
 255                         middlenewsub = subprefix[1];
 256                         if (nDiffCode & DIFFCODE::SECOND)      middlenewsub += dirs[1][j].filename;
 257                         else if (nDiffCode & DIFFCODE::FIRST)  middlenewsub += dirs[0][i].filename;
 258                         else if (nDiffCode & DIFFCODE::THIRD)  middlenewsub += dirs[2][k].filename;
 259                         rightnewsub  = subprefix[2];
 260                         if (nDiffCode & DIFFCODE::THIRD)       rightnewsub += dirs[2][k].filename;
 261                         else if (nDiffCode & DIFFCODE::FIRST)  rightnewsub += dirs[0][i].filename;
 262                         else if (nDiffCode & DIFFCODE::SECOND) rightnewsub += dirs[1][j].filename;
 263
 264                         // Test against filter so we don't include contents of filtered out directories
 265                         // Also this is only place we can test for both-sides directories in recursive compare
 266                         if ((pCtxt->m_piFilterGlobal!=nullptr && !pCtxt->m_piFilterGlobal->includeDir(leftnewsub, middlenewsub, rightnewsub)) ||
 267                                 (pCtxt->m_bIgnoreReparsePoints && (
 268                                   (nDiffCode & DIFFCODE::FIRST)  && (dirs[0][i].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 269                                   (nDiffCode & DIFFCODE::SECOND) && (dirs[1][j].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 270                                   (nDiffCode & DIFFCODE::THIRD)  && (dirs[2][k].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT))
 271                                 )
 272                            )
 273                                 nDiffCode |= DIFFCODE::SKIPPED;
 274                 }
 275
 276                 // add to list
 277                 if (!depth)
 278                 {
 279                         if (nDirs < 3)
 280                                 AddToList(subdir[0], subdir[1],
 281                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 282                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 283                                         nDiffCode, myStruct, parent);
 284                         else
 285                                 AddToList(subdir[0], subdir[1], subdir[2],
 286                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 287                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 288                                         (nDiffCode & DIFFCODE::THIRD ) ? &dirs[2][k] : nullptr,
 289                                         nDiffCode, myStruct, parent);
 290                 }
 291                 else
 292                 {
 293                         // Recursive compare
 294                         if (nDirs < 3)
 295                         {
 296                                 DIFFITEM *me = AddToList(subdir[0], subdir[1],
 297                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 298                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 299                                         nDiffCode, myStruct, parent);
 300                                 if ((nDiffCode & DIFFCODE::SKIPPED) == 0 && ((nDiffCode & DIFFCODE::SIDEFLAGS) == DIFFCODE::BOTH || bUniques))
 301                                 {
 302                                         // Scan recursively all subdirectories too, we are not adding folders
 303                                         String newsubdir[3] = {leftnewsub, rightnewsub};
 304                                         int result = DirScan_GetItems(paths, newsubdir, myStruct, casesensitive,
 305                                                         depth - 1, me, bUniques);
 306                                         if (result == -1)
 307                                                 return -1;
 308                                 }
 309                         }
 310                         else
 311                         {
 312                                 DIFFITEM *me = AddToList(subdir[0], subdir[1], subdir[2],
 313                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 314                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 315                                         (nDiffCode & DIFFCODE::THIRD ) ? &dirs[2][k] : nullptr,
 316                                         nDiffCode, myStruct, parent);
 317                                 if ((nDiffCode & DIFFCODE::SKIPPED) == 0 && ((nDiffCode & DIFFCODE::SIDEFLAGS) == DIFFCODE::ALL || bUniques))
 318                                 {
 319                                         // Scan recursively all subdirectories too, we are not adding folders
 320                                         String newsubdir[3] = {leftnewsub, middlenewsub, rightnewsub};
 321                                         int result = DirScan_GetItems(paths, newsubdir, myStruct, casesensitive,
 322                                                         depth - 1, me, bUniques);
 323                                         if (result == -1)
 324                                                 return -1;
 325                                 }
 326                         }
 327                 }
 328                 if (nDiffCode & DIFFCODE::FIRST)
 329                         i++;
 330                 if (nDiffCode & DIFFCODE::SECOND)
 331                         j++;
 332                 if (nDiffCode & DIFFCODE::THIRD)
 333                         k++;
 334         }
 335         // Handle files
 336         // i points to current file in left list (aFiles[0])
 337         // j points to current file in right list (aFiles[1])
 338         i=0, j=0, k=0;
 339         while (true)
 340         {
 341                 if (pCtxt->ShouldAbort())
 342                         return -1;
 343
 344
 345                 // Comparing file aFiles[0][i].name to aFiles[1][j].name
 346                 if (i<aFiles[0].size() && (j==aFiles[1].size() ||
 347                                 collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) < 0)
 348                         && (nDirs < 3 ||
 349                                 (k==aFiles[2].size() || collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive)<0) ))
 350                 {
 351                         if (nDirs < 3)
 352                         {
 353                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::FILE;
 354                                 AddToList(subdir[0], subdir[1], &aFiles[0][i], nullptr, nDiffCode, myStruct, parent);
 355                         }
 356                         else
 357                         {
 358                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::FILE;
 359                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], nullptr, nullptr, nDiffCode, myStruct, parent);
 360                         }
 361                         // Advance left pointer over left-only entry, and then retest with new pointers
 362                         ++i;
 363                         continue;
 364                 }
 365                 if (j<aFiles[1].size() && (i==aFiles[0].size() ||
 366                                 collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) > 0)
 367                         && (nDirs < 3 ||
 368                                 (k==aFiles[2].size() || collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive)<0) ))
 369                 {
 370                         const unsigned nDiffCode = DIFFCODE::SECOND | DIFFCODE::FILE;
 371                         if (nDirs < 3)
 372                                 AddToList(subdir[0], subdir[1], nullptr, &aFiles[1][j], nDiffCode, myStruct, parent);
 373                         else
 374                                 AddToList(subdir[0], subdir[1], subdir[2], nullptr, &aFiles[1][j], nullptr, nDiffCode, myStruct, parent);
 375                         // Advance right pointer over right-only entry, and then retest with new pointers
 376                         ++j;
 377                         continue;
 378                 }
 379                 if (nDirs == 3)
 380                 {
 381                         if (k<aFiles[2].size() && (i==aFiles[0].size() ||
 382                                         collstr(aFiles[2][k].filename, aFiles[0][i].filename, casesensitive)<0)
 383                                 && (j==aFiles[1].size() || collstr(aFiles[2][k].filename, aFiles[1][j].filename, casesensitive)<0) )
 384                         {
 385                                 const unsigned nDiffCode = DIFFCODE::THIRD | DIFFCODE::FILE;
 386                                 AddToList(subdir[0], subdir[1], subdir[2], nullptr, nullptr, &aFiles[2][k], nDiffCode, myStruct, parent);
 387                                 ++k;
 388                                 // Advance right pointer over right-only entry, and then retest with new pointers
 389                                 continue;
 390                         }
 391
 392                         if ((i<aFiles[0].size() && j<aFiles[1].size() && collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) == 0)
 393                             && (k==aFiles[2].size() || collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive) != 0))
 394                         {
 395                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::SECOND | DIFFCODE::FILE;
 396                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], &aFiles[1][j], nullptr, nDiffCode, myStruct, parent);
 397                                 ++i;
 398                                 ++j;
 399                                 continue;
 400                         }
 401                         else if ((i<aFiles[0].size() && k<aFiles[2].size() && collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive) == 0)
 402                             && (j==aFiles[1].size() || collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive) != 0))
 403                         {
 404                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::THIRD | DIFFCODE::FILE;
 405                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], nullptr, &aFiles[2][k], nDiffCode, myStruct, parent);
 406                                 ++i;
 407                                 ++k;
 408                                 continue;
 409                         }
 410                         else if ((j<aFiles[1].size() && k<aFiles[2].size() && collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive) == 0)
 411                             && (i==aFiles[0].size() || collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) != 0))
 412                         {
 413                                 const unsigned nDiffCode = DIFFCODE::SECOND | DIFFCODE::THIRD | DIFFCODE::FILE;
 414                                 AddToList(subdir[0], subdir[1], subdir[2], nullptr, &aFiles[1][j], &aFiles[2][k], nDiffCode, myStruct, parent);
 415                                 ++j;
 416                                 ++k;
 417                                 continue;
 418                         }
 419                 }
 420                 if (i<aFiles[0].size())
 421                 {
 422                         if (nDirs < 3)
 423                         {
 424                                 assert(j<aFiles[1].size());
 425                                 const unsigned nDiffCode = DIFFCODE::BOTH | DIFFCODE::FILE;
 426                                 AddToList(subdir[0], subdir[1], &aFiles[0][i], &aFiles[1][j], nDiffCode, myStruct, parent);
 427                                 ++i;
 428                                 ++j;
 429                                 continue;
 430                         }
 431                         else
 432                         {
 433                                 assert(j<aFiles[1].size());
 434                                 assert(k<aFiles[2].size());
 435                                 const unsigned nDiffCode = DIFFCODE::ALL | DIFFCODE::FILE;
 436                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], &aFiles[1][j], &aFiles[2][k], nDiffCode, myStruct, parent);
 437                                 ++i;
 438                                 ++j;
 439                                 ++k;
 440                                 continue;
 441                         }
 442                 }
 443                 break;
 444         }
 445
 446         if (parent != nullptr)
 447         {
 448                 for (int nIndex = 0; nIndex < nDirs; ++nIndex)
 449                         if (parent->diffcode.exists(nIndex) && parent->diffFileInfo[nIndex].size == DirItem::FILE_SIZE_NONE)
 450                                 parent->diffFileInfo[nIndex].size = 0;
 451
 452                 DIFFITEM *dic = parent->GetFirstChild();
 453                 while (dic)
 454                 {
 455                         for (int nIndex = 0; nIndex < nDirs; ++nIndex)
 456                         {
 457                                 if (dic->diffFileInfo[nIndex].size != DirItem::FILE_SIZE_NONE)
 458                                         parent->diffFileInfo[nIndex].size += dic->diffFileInfo[nIndex].size;
 459                         }
 460                         dic = dic->GetFwdSiblingLink();
 461                 }
 462         }
 463
 464         return 1;
 465 }
 466
 467 /**
 468  * @brief Compare DiffItems in list and add results to compare context.
 469  *
 470  * @param myStruct [in] A structure containing compare-related data.
 471  * @param parentdiffpos [in] Position of parent diff item
 472  * @return >= 0 number of diff items, -1 if compare was aborted
 473  */
 474 int DirScan_CompareItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 475 {
 476         const int compareMethod = myStruct->context->GetCompareMethod();
 477         int nworkers = 1;
 478
 479         if (compareMethod == CMP_CONTENT || compareMethod == CMP_QUICK_CONTENT)
 480         {
 481                 nworkers = GetOptionsMgr()->GetInt(OPT_CMP_COMPARE_THREADS);
 482                 if (nworkers <= 0)
 483                 {
 484                         nworkers += Environment::processorCount();
 485                         if (nworkers <= 0)
 486                                 nworkers = 1;
 487                 }
 488         }
 489
 490         ThreadPool threadPool(nworkers, nworkers);
 491         std::vector<DiffWorkerPtr> workers;
 492         NotificationQueue queue;
 493         myStruct->context->m_pCompareStats->SetCompareThreadCount(nworkers);
 494         for (int i = 0; i < nworkers; ++i)
 495         {
 496                 workers.push_back(DiffWorkerPtr(new DiffWorker(queue, myStruct->context, i)));
 497                 threadPool.start(*workers[i]);
 498         }
 499
 500         int res = CompareItems(queue, myStruct, parentdiffpos);
 501
 502         Thread::sleep(100);
 503         queue.wakeUpAll();
 504         threadPool.joinAll();
 505
 506         return res;
 507 }
 508
 509 static int CompareItems(NotificationQueue& queue, DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 510 {
 511         NotificationQueue queueResult;
 512         Stopwatch stopwatch;
 513         CDiffContext *pCtxt = myStruct->context;
 514         int res = 0;
 515         int count = 0;
 516         bool bCompareFailure = false;
 517         if (parentdiffpos == nullptr)
 518                 myStruct->pSemaphore->wait();
 519         stopwatch.start();
 520         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 521         while (pos != nullptr)
 522         {
 523                 if (pCtxt->ShouldAbort())
 524                         break;
 525
 526                 if (stopwatch.elapsed() > 2000000)
 527                 {
 528                         int event = CDiffThread::EVENT_COMPARE_PROGRESSED;
 529                         myStruct->m_listeners.notify(myStruct, event);
 530                         stopwatch.restart();
 531                 }
 532                 myStruct->pSemaphore->wait();
 533                 DIFFITEM *curpos = pos;
 534                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 535                 bool existsalldirs = di.diffcode.existAll();
 536                 if (di.diffcode.isDirectory() && pCtxt->m_bRecursive)
 537                 {
 538                         if ((di.diffcode.diffcode & DIFFCODE::CMPERR) != DIFFCODE::CMPERR)
 539                         {       // Only clear DIFF|SAME flags if not CMPERR (eg. both flags together)
 540                                 di.diffcode.diffcode &= ~(DIFFCODE::DIFF | DIFFCODE::SAME);
 541                         }
 542                         int ndiff = CompareItems(queue, myStruct, curpos);
 543                         // Propogate sub-directory status to this directory
 544                         if (ndiff > 0)
 545                         {       // There were differences in the sub-directories
 546                                 if (existsalldirs)
 547                                         di.diffcode.diffcode |= DIFFCODE::DIFF;
 548                                 res += ndiff;
 549                         }
 550                         else
 551                         if (ndiff == 0)
 552                         {       // Sub-directories were identical
 553                                 if (existsalldirs)
 554                                         di.diffcode.diffcode |= DIFFCODE::SAME;
 555                         }
 556                         else
 557                         if (ndiff == -1)
 558                         {       // There were file IO-errors during sub-directory comparison.
 559                                 di.diffcode.diffcode |= DIFFCODE::CMPERR;
 560                                 bCompareFailure = true;
 561                         }
 562                 }
 563                 if (existsalldirs)
 564                         queue.enqueueUrgentNotification(new WorkNotification(di, queueResult));
 565                 else
 566                         queue.enqueueNotification(new WorkNotification(di, queueResult));
 567                 ++count;
 568                 pos = curpos;
 569                 pCtxt->GetNextSiblingDiffRefPosition(pos);
 570         }
 571
 572         while (count > 0)
 573         {
 574                 AutoPtr<Notification> pNf(queueResult.waitDequeueNotification());
 575                 if (pNf.get() == nullptr)
 576                         break;
 577                 WorkCompletedNotification* pWorkCompletedNf = dynamic_cast<WorkCompletedNotification*>(pNf.get());
 578                 if (pWorkCompletedNf != nullptr) {
 579                         DIFFITEM &di = pWorkCompletedNf->data();
 580                         if (di.diffcode.isResultError()) {
 581                                 DIFFITEM *diParent = di.GetParentLink();
 582                                 assert(diParent != nullptr);
 583                                 if (diParent != nullptr)
 584                                 {
 585                                         diParent->diffcode.diffcode |= DIFFCODE::CMPERR;
 586                                         bCompareFailure = true;
 587                                 }
 588                         }
 589
 590                         if (di.diffcode.isResultDiff() ||
 591                                 (!di.diffcode.existAll() && !di.diffcode.isResultFiltered()))
 592                                 res++;
 593                 }
 594                 --count;
 595         }
 596
 597         return bCompareFailure || pCtxt->ShouldAbort() ? -1 : res;
 598 }
 599
 600 /**
 601  * @brief Compare DiffItems in context marked for rescan.
 602  *
 603  * @param myStruct [in,out] A structure containing compare-related data.
 604  * @param parentdiffpos [in] Position of parent diff item
 605  * @return >= 0 number of diff items, -1 if compare was aborted
 606  */
 607 static int CompareRequestedItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 608 {
 609         CDiffContext *pCtxt = myStruct->context;
 610         FolderCmp fc(pCtxt);
 611         int res = 0;
 612         bool bCompareFailure = false;
 613         if (parentdiffpos == nullptr)
 614                 myStruct->pSemaphore->wait();
 615         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 616         while (pos != nullptr)
 617         {
 618                 if (pCtxt->ShouldAbort())
 619                 {
 620                         res = -1;
 621                         break;
 622                 }
 623
 624                 DIFFITEM *curpos = pos;
 625                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 626                 bool existsalldirs = di.diffcode.existAll();
 627                 if (di.diffcode.isDirectory())
 628                 {
 629                         if (pCtxt->m_bRecursive)
 630                         {
 631                                 di.diffcode.diffcode &= ~(DIFFCODE::DIFF | DIFFCODE::SAME);
 632                                 int ndiff = CompareRequestedItems(myStruct, curpos);
 633                                 if (ndiff > 0)
 634                                 {
 635                                         if (existsalldirs)
 636                                                 di.diffcode.diffcode |= DIFFCODE::DIFF;
 637                                         res += ndiff;
 638                                 }
 639                                 else
 640                                 if (ndiff == 0)
 641                                 {
 642                                         if (existsalldirs)
 643                                                 di.diffcode.diffcode |= DIFFCODE::SAME;
 644                                 }
 645                                 else
 646                                 if (ndiff == -1)
 647                                 {       // There were file IO-errors during sub-directory comparison.
 648                                         di.diffcode.diffcode |= DIFFCODE::CMPERR;
 649                                         bCompareFailure = true;
 650                                 }
 651                         }
 652                 }
 653                 else
 654                 {
 655                         if (di.diffcode.isScanNeeded())
 656                         {
 657                                 CompareDiffItem(fc, di);
 658                                 if (di.diffcode.isResultError()) {
 659                                         DIFFITEM *diParent = di.GetParentLink();
 660                                         assert(diParent != nullptr);
 661                                         if (diParent != nullptr)
 662                                         {
 663                                                 diParent->diffcode.diffcode |= DIFFCODE::CMPERR;
 664                                                 bCompareFailure = true;
 665                                         }
 666                         }
 667
 668                         }
 669                 }
 670                 if (di.diffcode.isResultDiff() ||
 671                         (!existsalldirs && !di.diffcode.isResultFiltered()))
 672                         res++;
 673         }
 674         return bCompareFailure ? -1 : res;;
 675 }
 676
 677 int DirScan_CompareRequestedItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 678 {
 679         return CompareRequestedItems(myStruct, parentdiffpos);
 680 }
 681
 682 static int markChildrenForRescan(CDiffContext *pCtxt, DIFFITEM *parentdiffpos)
 683 {
 684         int ncount = 0;
 685         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 686         while (pos != nullptr)
 687         {
 688                 DIFFITEM *curpos = pos;
 689                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 690                 if (di.diffcode.isDirectory())
 691                         ncount += markChildrenForRescan(pCtxt, curpos);
 692                 else
 693                 {
 694                         di.diffcode.diffcode |= DIFFCODE::NEEDSCAN;
 695                         ++ncount;
 696                 }
 697         }
 698         return ncount;
 699 }
 700
 701 int DirScan_UpdateMarkedItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 702 {
 703         CDiffContext *pCtxt = myStruct->context;
 704         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 705         int ncount = 0;
 706
 707         while (pos != nullptr)
 708         {
 709                 if (pCtxt->ShouldAbort())
 710                         break;
 711                 DIFFITEM *curpos = pos;
 712                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 713                 if (di.diffcode.isScanNeeded())
 714                 {
 715                         bool bItemsExist = true;
 716                         UpdateDiffItem(di, bItemsExist, pCtxt);
 717                         if (!bItemsExist)
 718                         {
 719                                 di.DelinkFromSiblings();        // delink from list of Siblings
 720                                 delete &di;                                     // Also delete all Children items
 721                                 continue;                                       // (... because `di` is now invalid)
 722                         }
 723                         if (!di.diffcode.isDirectory())
 724                                 ++ncount;
 725                 }
 726                 if (di.diffcode.isDirectory() && pCtxt->m_bRecursive)
 727                 {
 728                         for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 729                                 if (di.diffcode.exists(i))
 730                                         di.diffFileInfo[i].size = 0;
 731                         if (di.diffcode.isScanNeeded() && !di.diffcode.isResultFiltered())
 732                         {
 733                                 di.RemoveChildren();
 734                                 di.diffcode.diffcode &= ~DIFFCODE::NEEDSCAN;
 735
 736                                 bool casesensitive = false;
 737                                 int depth = myStruct->context->m_bRecursive ? -1 : 0;
 738                                 String subdir[3];
 739                                 PathContext paths = myStruct->context->GetNormalizedPaths();
 740                                 for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 741                                         subdir[i] = di.diffFileInfo[i].GetFile();
 742                                 DirScan_GetItems(paths, subdir, myStruct,
 743                                         casesensitive, depth, &di, myStruct->context->m_bWalkUniques);
 744                                 ncount += markChildrenForRescan(myStruct->context, curpos);
 745                         }
 746                         else
 747                         {
 748                                 ncount += DirScan_UpdateMarkedItems(myStruct, curpos);
 749                         }
 750                 }
 751                 if (parentdiffpos != nullptr && pCtxt->m_bRecursive)
 752                 {
 753                         for (int nIndex = 0; nIndex < pCtxt->GetCompareDirs(); ++nIndex)
 754                                 if (curpos->diffFileInfo[nIndex].size != DirItem::FILE_SIZE_NONE)
 755                                         parentdiffpos->diffFileInfo[nIndex].size += curpos->diffFileInfo[nIndex].size;
 756                 }
 757         }
 758         return ncount;
 759 }
 760 /**
 761  * @brief Update diffitem file/dir infos.
 762  *
 763  * Re-tests dirs/files if sides still exists, and updates infos for
 764  * existing sides. This assumes filenames, or paths are not changed.
 765  * Since in normal situations (I can think of) they cannot change
 766  * after first compare.
 767  *
 768  * @param [in,out] di DiffItem to update.
 769  * @param [out] bExists Set to
 770  *  - true if one of items exists so diffitem is valid
 771  *  - false if items were deleted, so diffitem is not valid
 772  * @param [in] pCtxt Compare context
 773  */
 774 static void UpdateDiffItem(DIFFITEM &di, bool & bExists, CDiffContext *pCtxt)
 775 {
 776         bExists = false;
 777         di.diffcode.setSideNone();
 778         for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 779         {
 780                 di.diffFileInfo[i].ClearPartial();
 781                 if (pCtxt->UpdateInfoFromDiskHalf(di, i))
 782                 {
 783                         di.diffcode.diffcode |= DIFFCODE::FIRST << i;
 784                         bExists = true;
 785                 }
 786         }
 787 }
 788
 789 /**
 790  * @brief Compare two diffitems and add results to difflist in context.
 791  *
 792  * This function does the actual compare for previously gathered list of
 793  * items. Basically we:
 794  * - ignore items matching file filters
 795  * - add non-ignored directories (no compare for directory items)
 796  * - add unique files
 797  * - compare files
 798  *
 799  * @param [in] di DiffItem to compare
 800  * @param [in,out] pCtxt Compare context: contains difflist, encoding info etc.
 801  * @todo For date compare, maybe we should use creation date if modification
 802  * date is missing?
 803  */
 804 static void CompareDiffItem(FolderCmp &fc, DIFFITEM &di)
 805 {
 806         CDiffContext *const pCtxt = fc.m_pCtxt;
 807         int nDirs = pCtxt->GetCompareDirs();
 808         // Clear rescan-request flag (not set by all codepaths)
 809         di.diffcode.diffcode &= ~DIFFCODE::NEEDSCAN;
 810         // Is it a directory?
 811         if (di.diffcode.isDirectory())
 812         {
 813                 // We don't actually 'compare' directories, just add non-ignored
 814                 // directories to list.
 815         }
 816         else
 817         {
 818                 // 1. Test against filters
 819                 if (pCtxt->m_piFilterGlobal==nullptr ||
 820                         (nDirs == 2 && pCtxt->m_piFilterGlobal->includeFile(di.diffFileInfo[0].filename, di.diffFileInfo[1].filename)) ||
 821                         (nDirs == 3 && pCtxt->m_piFilterGlobal->includeFile(di.diffFileInfo[0].filename, di.diffFileInfo[1].filename, di.diffFileInfo[2].filename))
 822                         )
 823                 {
 824                         di.diffcode.diffcode |= DIFFCODE::INCLUDED;
 825                         di.diffcode.diffcode |= fc.prepAndCompareFiles(di);
 826                         di.nsdiffs = fc.m_ndiffs;
 827                         di.nidiffs = fc.m_ntrivialdiffs;
 828
 829                         for (int i = 0; i < nDirs; ++i)
 830                         {
 831                                 // Set text statistics
 832                                 if (di.diffcode.exists(i))
 833                                 {
 834                                         di.diffFileInfo[i].m_textStats = fc.m_diffFileData.m_textStats[i];
 835                                         di.diffFileInfo[i].encoding = fc.m_diffFileData.m_FileLocation[i].encoding;
 836                                 }
 837                         }
 838                 }
 839                 else
 840                 {
 841                         di.diffcode.diffcode |= DIFFCODE::SKIPPED;
 842                 }
 843         }
 844         pCtxt->m_pCompareStats->AddItem(di.diffcode.diffcode);
 845 }
 846
 847 /**
 848  * @brief Add one compare item to list.
 849  * @param [in] sLeftDir Left subdirectory.
 850  * @param [in] sRightDir Right subdirectory.
 851  * @param [in] lent Left item data to add.
 852  * @param [in] rent Right item data to add.
 853  * @param [in] pCtxt Compare context.
 854  * @param [in] parent Parent of item to be added
 855  */
 856 static DIFFITEM *AddToList(const String& sLeftDir, const String& sRightDir,
 857         const DirItem * lent, const DirItem * rent,
 858         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent)
 859 {
 860         return AddToList(sLeftDir, sRightDir, sLeftDir, lent, rent, nullptr, code, myStruct, parent, 2);
 861 }
 862
 863 /**
 864  * @brief Add one compare item to list.
 865  */
 866 static DIFFITEM *AddToList(const String& sDir1, const String& sDir2, const String& sDir3,
 867         const DirItem *ent1, const DirItem *ent2, const DirItem *ent3,
 868         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent, int nItems /*= 3*/)
 869 {
 870         // We must store both paths - we cannot get paths later
 871         // and we need unique item paths for example when items
 872         // change to identical
 873         DIFFITEM *di = myStruct->context->AddNewDiff(parent);
 874
 875         di->diffFileInfo[0].path = sDir1;
 876         di->diffFileInfo[1].path = sDir2;
 877         di->diffFileInfo[2].path = sDir3;
 878
 879         if (ent1 != nullptr)
 880         {
 881                 di->diffFileInfo[0].filename = ent1->filename;
 882                 di->diffFileInfo[0].mtime = ent1->mtime;
 883                 di->diffFileInfo[0].ctime = ent1->ctime;
 884                 di->diffFileInfo[0].size = ent1->size;
 885                 di->diffFileInfo[0].flags.attributes = ent1->flags.attributes;
 886         }
 887         else
 888         {
 889                 // Don't break CDirView::DoCopyRightToLeft()
 890                 if (ent3 != nullptr)
 891                         di->diffFileInfo[0].filename = ent3->filename;
 892                 else if (ent2 != nullptr)
 893                         di->diffFileInfo[0].filename = ent2->filename;
 894         }
 895
 896         if (ent2 != nullptr)
 897         {
 898                 di->diffFileInfo[1].filename = ent2->filename;
 899                 di->diffFileInfo[1].mtime = ent2->mtime;
 900                 di->diffFileInfo[1].ctime = ent2->ctime;
 901                 di->diffFileInfo[1].size = ent2->size;
 902                 di->diffFileInfo[1].flags.attributes = ent2->flags.attributes;
 903         }
 904         else
 905         {
 906                 // Don't break CDirView::DoCopyLeftToRight()
 907                 if (ent1 != nullptr)
 908                         di->diffFileInfo[1].filename = ent1->filename;
 909                 else if (ent3 != nullptr)
 910                         di->diffFileInfo[1].filename = ent3->filename;
 911         }
 912
 913         if (ent3 != nullptr)
 914         {
 915                 di->diffFileInfo[2].filename = ent3->filename;
 916                 di->diffFileInfo[2].mtime = ent3->mtime;
 917                 di->diffFileInfo[2].ctime = ent3->ctime;
 918                 di->diffFileInfo[2].size = ent3->size;
 919                 di->diffFileInfo[2].flags.attributes = ent3->flags.attributes;
 920         }
 921         else
 922         {
 923                 // Don't break CDirView::DoCopyLeftToRight()
 924                 if (ent1 != nullptr)
 925                         di->diffFileInfo[2].filename = ent1->filename;
 926                 else if (ent2 != nullptr)
 927                         di->diffFileInfo[2].filename = ent2->filename;
 928         }
 929
 930         if (nItems == 2)
 931                 di->diffcode.diffcode = code;
 932         else
 933                 di->diffcode.diffcode = code | DIFFCODE::THREEWAY;
 934
 935         if (myStruct->m_fncCollect)
 936         {
 937                 myStruct->context->m_pCompareStats->IncreaseTotalItems();
 938                 myStruct->pSemaphore->set();
 939         }
 940         return di;
 941 }