Src/DirScan.cpp

   1 /**
   2  *  @file DirScan.cpp
   3  *
   4  *  @brief Implementation of DirScan (q.v.) and helper functions
   5  */
   6
   7 #include "pch.h"
   8 #include "DirScan.h"
   9 #include <cassert>
  10 #include <memory>
  11 #define POCO_NO_UNWINDOWS 1
  12 #include <Poco/Semaphore.h>
  13 #include <Poco/Notification.h>
  14 #include <Poco/NotificationQueue.h>
  15 #include <Poco/Environment.h>
  16 #include <Poco/ThreadPool.h>
  17 #include <Poco/Thread.h>
  18 #include <Poco/Runnable.h>
  19 #include <Poco/Mutex.h>
  20 #include <Poco/AutoPtr.h>
  21 #include <Poco/Stopwatch.h>
  22 #include <Poco/Format.h>
  23 #include "DiffThread.h"
  24 #include "UnicodeString.h"
  25 #include "DiffWrapper.h"
  26 #include "CompareStats.h"
  27 #include "FolderCmp.h"
  28 #include "FileFilterHelper.h"
  29 #include "IAbortable.h"
  30 #include "FolderCmp.h"
  31 #include "DirItem.h"
  32 #include "DirTravel.h"
  33 #include "paths.h"
  34 #include "Plugins.h"
  35 #include "MergeApp.h"
  36 #include "OptionsDef.h"
  37 #include "OptionsMgr.h"
  38 #include "DebugNew.h"
  39
  40 using Poco::NotificationQueue;
  41 using Poco::Notification;
  42 using Poco::AutoPtr;
  43 using Poco::Thread;
  44 using Poco::ThreadPool;
  45 using Poco::Runnable;
  46 using Poco::Environment;
  47 using Poco::Stopwatch;
  48
  49 // Static functions (ie, functions only used locally)
  50 void CompareDiffItem(DIFFITEM &di, CDiffContext *pCtxt);
  51 static void StoreDiffData(DIFFITEM &di, CDiffContext *pCtxt,
  52                 const FolderCmp *pCmpData);
  53 static DIFFITEM *AddToList(const String &sLeftDir, const String &sRightDir, const DirItem *lent, const DirItem *rent,
  54         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent);
  55 static DIFFITEM *AddToList(const String &sDir1, const String &sDir2, const String &sDir3, const DirItem *ent1, const DirItem *ent2, const DirItem *ent3,
  56         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent, int nItems = 3);
  57 static void UpdateDiffItem(DIFFITEM &di, bool &bExists, CDiffContext *pCtxt);
  58 static int CompareItems(NotificationQueue &queue, DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos);
  59
  60 class WorkNotification: public Poco::Notification
  61 {
  62 public:
  63         WorkNotification(DIFFITEM& di, NotificationQueue& queueResult): m_di(di), m_queueResult(queueResult) {}
  64         DIFFITEM& data() const { return m_di; }
  65         NotificationQueue& queueResult() const { return m_queueResult; }
  66 private:
  67         DIFFITEM& m_di;
  68         NotificationQueue& m_queueResult;
  69 };
  70
  71 class WorkCompletedNotification: public Poco::Notification
  72 {
  73 public:
  74         explicit WorkCompletedNotification(DIFFITEM& di): m_di(di) {}
  75         DIFFITEM& data() const { return m_di; }
  76 private:
  77         DIFFITEM& m_di;
  78 };
  79
  80 class DiffWorker: public Runnable
  81 {
  82 public:
  83         DiffWorker(NotificationQueue& queue, CDiffContext *pCtxt, int id):
  84           m_queue(queue), m_pCtxt(pCtxt), m_id(id) {}
  85
  86         void run()
  87         {
  88                 // keep the scripts alive during the Rescan
  89                 // when we exit the thread, we delete this and release the scripts
  90                 CAssureScriptsForThread scriptsForRescan;
  91
  92                 AutoPtr<Notification> pNf(m_queue.waitDequeueNotification());
  93                 while (pNf.get() != nullptr)
  94                 {
  95                         WorkNotification* pWorkNf = dynamic_cast<WorkNotification*>(pNf.get());
  96                         if (pWorkNf != nullptr) {
  97                                 m_pCtxt->m_pCompareStats->BeginCompare(&pWorkNf->data(), m_id);
  98                                 if (!m_pCtxt->ShouldAbort())
  99                                         CompareDiffItem(pWorkNf->data(), m_pCtxt);
 100                                 pWorkNf->queueResult().enqueueNotification(new WorkCompletedNotification(pWorkNf->data()));
 101                         }
 102                         pNf = m_queue.waitDequeueNotification();
 103                 }
 104         }
 105
 106 private:
 107         NotificationQueue& m_queue;
 108         CDiffContext *m_pCtxt;
 109         int m_id;
 110 };
 111
 112 typedef std::shared_ptr<DiffWorker> DiffWorkerPtr;
 113
 114 /**
 115  * @brief Collect file- and folder-names to list.
 116  * This function walks given folders and adds found subfolders and files into
 117  * lists. There are two modes, determined by the @p depth:
 118  * - in non-recursive mode we walk only given folders, and add files
 119  *   contained. Subfolders are added as folder items, not walked into.
 120  * - in recursive mode we walk all subfolders and add the files they
 121  *   contain into list.
 122  *
 123  * Items are tested against file filters in this function.
 124  *
 125  * @param [in] paths Root paths of compare
 126  * @param [in] leftsubdir Left side subdirectory under root path
 127  * @param [in] bLeftUniq Is left-side folder unique folder?
 128  * @param [in] rightsubdir Right side subdirectory under root path
 129  * @param [in] bRightUniq Is right-side folder unique folder?
 130  * @param [in] myStruct Compare-related data, like context etc.
 131  * @param [in] casesensitive Is filename compare case sensitive?
 132  * @param [in] depth Levels of subdirectories to scan, -1 scans all
 133  * @param [in] parent Folder diff item to be scanned
 134  * @param [in] bUniques If true, walk into unique folders.
 135  * @return 1 normally, -1 if compare was aborted
 136  */
 137 int DirScan_GetItems(const PathContext &paths, const String subdir[],
 138                 DiffFuncStruct *myStruct,
 139                 bool casesensitive, int depth, DIFFITEM *parent,
 140                 bool bUniques)
 141 {
 142         static const TCHAR backslash[] = _T("\\");
 143         int nDirs = paths.GetSize();
 144         CDiffContext *pCtxt = myStruct->context;
 145         String sDir[3];
 146         String subprefix[3];
 147
 148         int nIndex;
 149         std::copy(paths.begin(), paths.end(), sDir);
 150
 151         if (!subdir[0].empty())
 152         {
 153                 for (nIndex = 0; nIndex < paths.GetSize(); nIndex++)
 154                 {
 155                         sDir[nIndex] = paths::ConcatPath(sDir[nIndex], subdir[nIndex]);
 156                         subprefix[nIndex] = subdir[nIndex] + backslash;
 157                 }
 158         }
 159
 160         DirItemArray dirs[3], aFiles[3];
 161         for (nIndex = 0; nIndex < nDirs; nIndex++)
 162                 LoadAndSortFiles(sDir[nIndex], &dirs[nIndex], &aFiles[nIndex], casesensitive);
 163
 164         // Allow user to abort scanning
 165         if (pCtxt->ShouldAbort())
 166                 return -1;
 167
 168         // Handle directories
 169         // i points to current directory in left list (leftDirs)
 170         // j points to current directory in right list (rightDirs)
 171
 172         for (nIndex = 0; nIndex < nDirs; nIndex++)
 173                 if (dirs[nIndex].size() != 0 || aFiles[nIndex].size() != 0) break;
 174         if (nIndex == nDirs)
 175                 return 0;
 176
 177         DirItemArray::size_type i=0, j=0, k=0;
 178         while (true)
 179         {
 180                 if (pCtxt->ShouldAbort())
 181                         return -1;
 182
 183                 if (i >= dirs[0].size() && j >= dirs[1].size() && (nDirs < 3 || k >= dirs[2].size()))
 184                         break;
 185
 186                 unsigned nDiffCode = DIFFCODE::DIR;
 187                 // Comparing directories leftDirs[i].name to rightDirs[j].name
 188                 if (i<dirs[0].size() && (j==dirs[1].size() || collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive)<0)
 189                         && (nDirs < 3 ||      (k==dirs[2].size() || collstr(dirs[0][i].filename, dirs[2][k].filename, casesensitive)<0) ))
 190                 {
 191                         nDiffCode |= DIFFCODE::FIRST;
 192                 }
 193                 else if (j<dirs[1].size() && (i==dirs[0].size() || collstr(dirs[1][j].filename, dirs[0][i].filename, casesensitive)<0)
 194                         && (nDirs < 3 ||      (k==dirs[2].size() || collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive)<0) ))
 195                 {
 196                         nDiffCode |= DIFFCODE::SECOND;
 197                 }
 198                 else if (nDirs < 3)
 199                 {
 200                         nDiffCode |= DIFFCODE::BOTH;
 201                 }
 202                 else
 203                 {
 204                         if (k<dirs[2].size() && (i==dirs[0].size() || collstr(dirs[2][k].filename, dirs[0][i].filename, casesensitive)<0)
 205                                 &&                     (j==dirs[1].size() || collstr(dirs[2][k].filename, dirs[1][j].filename, casesensitive)<0) )
 206                         {
 207                                 nDiffCode |= DIFFCODE::THIRD;
 208                         }
 209                         else if ((i<dirs[0].size() && j<dirs[1].size() && collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive) == 0)
 210                                 && (k==dirs[2].size() || collstr(dirs[2][k].filename, dirs[0][i].filename, casesensitive) != 0))
 211                         {
 212                                 nDiffCode |= DIFFCODE::FIRST | DIFFCODE::SECOND;
 213                         }
 214                         else if ((i<dirs[0].size() && k<dirs[2].size() && collstr(dirs[0][i].filename, dirs[2][k].filename, casesensitive) == 0)
 215                                 && (j==dirs[1].size() || collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive) != 0))
 216                         {
 217                                 nDiffCode |= DIFFCODE::FIRST | DIFFCODE::THIRD;
 218                         }
 219                         else if ((j<dirs[1].size() && k<dirs[2].size() && collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive) == 0)
 220                                 && (i==dirs[0].size() || collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive) != 0))
 221                         {
 222                                 nDiffCode |= DIFFCODE::SECOND | DIFFCODE::THIRD;
 223                         }
 224                         else
 225                         {
 226                                 nDiffCode |= DIFFCODE::ALL;
 227                         }
 228                 }
 229
 230                 String leftnewsub;
 231                 String rightnewsub;
 232                 String middlenewsub;
 233                 if (nDirs < 3)
 234                 {
 235                         leftnewsub  = (nDiffCode & DIFFCODE::FIRST)  ? subprefix[0] + dirs[0][i].filename.get() : subprefix[0] + dirs[1][j].filename.get();
 236                         rightnewsub = (nDiffCode & DIFFCODE::SECOND) ? subprefix[1] + dirs[1][j].filename.get() : subprefix[1] + dirs[0][i].filename.get();
 237
 238                         // Test against filter so we don't include contents of filtered out directories
 239                         // Also this is only place we can test for both-sides directories in recursive compare
 240                         if ((pCtxt->m_piFilterGlobal!=nullptr && !pCtxt->m_piFilterGlobal->includeDir(leftnewsub, rightnewsub)) ||
 241                                 (pCtxt->m_bIgnoreReparsePoints && (
 242                                 (nDiffCode & DIFFCODE::FIRST) && (dirs[0][i].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 243                                         (nDiffCode & DIFFCODE::SECOND) && (dirs[1][j].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT))
 244                                         )
 245                                 )
 246                                 nDiffCode |= DIFFCODE::SKIPPED;
 247                 }
 248                 else
 249                 {
 250                         leftnewsub   = subprefix[0];
 251                         if (nDiffCode & DIFFCODE::FIRST)       leftnewsub += dirs[0][i].filename;
 252                         else if (nDiffCode & DIFFCODE::SECOND) leftnewsub += dirs[1][j].filename;
 253                         else if (nDiffCode & DIFFCODE::THIRD)  leftnewsub += dirs[2][k].filename;
 254                         middlenewsub = subprefix[1];
 255                         if (nDiffCode & DIFFCODE::SECOND)      middlenewsub += dirs[1][j].filename;
 256                         else if (nDiffCode & DIFFCODE::FIRST)  middlenewsub += dirs[0][i].filename;
 257                         else if (nDiffCode & DIFFCODE::THIRD)  middlenewsub += dirs[2][k].filename;
 258                         rightnewsub  = subprefix[2];
 259                         if (nDiffCode & DIFFCODE::THIRD)       rightnewsub += dirs[2][k].filename;
 260                         else if (nDiffCode & DIFFCODE::FIRST)  rightnewsub += dirs[0][i].filename;
 261                         else if (nDiffCode & DIFFCODE::SECOND) rightnewsub += dirs[1][j].filename;
 262
 263                         // Test against filter so we don't include contents of filtered out directories
 264                         // Also this is only place we can test for both-sides directories in recursive compare
 265                         if ((pCtxt->m_piFilterGlobal!=nullptr && !pCtxt->m_piFilterGlobal->includeDir(leftnewsub, middlenewsub, rightnewsub)) ||
 266                                 (pCtxt->m_bIgnoreReparsePoints && (
 267                                   (nDiffCode & DIFFCODE::FIRST)  && (dirs[0][i].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 268                                   (nDiffCode & DIFFCODE::SECOND) && (dirs[1][j].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 269                                   (nDiffCode & DIFFCODE::THIRD)  && (dirs[2][k].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT))
 270                                 )
 271                            )
 272                                 nDiffCode |= DIFFCODE::SKIPPED;
 273                 }
 274
 275                 // add to list
 276                 if (!depth)
 277                 {
 278                         if (nDirs < 3)
 279                                 AddToList(subdir[0], subdir[1],
 280                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 281                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 282                                         nDiffCode, myStruct, parent);
 283                         else
 284                                 AddToList(subdir[0], subdir[1], subdir[2],
 285                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 286                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 287                                         (nDiffCode & DIFFCODE::THIRD ) ? &dirs[2][k] : nullptr,
 288                                         nDiffCode, myStruct, parent);
 289                 }
 290                 else
 291                 {
 292                         // Recursive compare
 293                         if (nDirs < 3)
 294                         {
 295                                 DIFFITEM *me = AddToList(subdir[0], subdir[1],
 296                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 297                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 298                                         nDiffCode, myStruct, parent);
 299                                 if ((nDiffCode & DIFFCODE::SKIPPED) == 0 && ((nDiffCode & DIFFCODE::SIDEFLAGS) == DIFFCODE::BOTH || bUniques))
 300                                 {
 301                                         // Scan recursively all subdirectories too, we are not adding folders
 302                                         String newsubdir[3] = {leftnewsub, rightnewsub};
 303                                         int result = DirScan_GetItems(paths, newsubdir, myStruct, casesensitive,
 304                                                         depth - 1, me, bUniques);
 305                                         if (result == -1)
 306                                                 return -1;
 307                                 }
 308                         }
 309                         else
 310                         {
 311                                 DIFFITEM *me = AddToList(subdir[0], subdir[1], subdir[2],
 312                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : nullptr,
 313                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : nullptr,
 314                                         (nDiffCode & DIFFCODE::THIRD ) ? &dirs[2][k] : nullptr,
 315                                         nDiffCode, myStruct, parent);
 316                                 if ((nDiffCode & DIFFCODE::SKIPPED) == 0 && ((nDiffCode & DIFFCODE::SIDEFLAGS) == DIFFCODE::ALL || bUniques))
 317                                 {
 318                                         // Scan recursively all subdirectories too, we are not adding folders
 319                                         String newsubdir[3] = {leftnewsub, middlenewsub, rightnewsub};
 320                                         int result = DirScan_GetItems(paths, newsubdir, myStruct, casesensitive,
 321                                                         depth - 1, me, bUniques);
 322                                         if (result == -1)
 323                                                 return -1;
 324                                 }
 325                         }
 326                 }
 327                 if (nDiffCode & DIFFCODE::FIRST)
 328                         i++;
 329                 if (nDiffCode & DIFFCODE::SECOND)
 330                         j++;
 331                 if (nDiffCode & DIFFCODE::THIRD)
 332                         k++;
 333         }
 334         // Handle files
 335         // i points to current file in left list (aFiles[0])
 336         // j points to current file in right list (aFiles[1])
 337         i=0, j=0, k=0;
 338         while (true)
 339         {
 340                 if (pCtxt->ShouldAbort())
 341                         return -1;
 342
 343
 344                 // Comparing file aFiles[0][i].name to aFiles[1][j].name
 345                 if (i<aFiles[0].size() && (j==aFiles[1].size() ||
 346                                 collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) < 0)
 347                         && (nDirs < 3 ||
 348                                 (k==aFiles[2].size() || collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive)<0) ))
 349                 {
 350                         if (nDirs < 3)
 351                         {
 352                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::FILE;
 353                                 AddToList(subdir[0], subdir[1], &aFiles[0][i], nullptr, nDiffCode, myStruct, parent);
 354                         }
 355                         else
 356                         {
 357                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::FILE;
 358                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], nullptr, nullptr, nDiffCode, myStruct, parent);
 359                         }
 360                         // Advance left pointer over left-only entry, and then retest with new pointers
 361                         ++i;
 362                         continue;
 363                 }
 364                 if (j<aFiles[1].size() && (i==aFiles[0].size() ||
 365                                 collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) > 0)
 366                         && (nDirs < 3 ||
 367                                 (k==aFiles[2].size() || collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive)<0) ))
 368                 {
 369                         const unsigned nDiffCode = DIFFCODE::SECOND | DIFFCODE::FILE;
 370                         if (nDirs < 3)
 371                                 AddToList(subdir[0], subdir[1], nullptr, &aFiles[1][j], nDiffCode, myStruct, parent);
 372                         else
 373                                 AddToList(subdir[0], subdir[1], subdir[2], nullptr, &aFiles[1][j], nullptr, nDiffCode, myStruct, parent);
 374                         // Advance right pointer over right-only entry, and then retest with new pointers
 375                         ++j;
 376                         continue;
 377                 }
 378                 if (nDirs == 3)
 379                 {
 380                         if (k<aFiles[2].size() && (i==aFiles[0].size() ||
 381                                         collstr(aFiles[2][k].filename, aFiles[0][i].filename, casesensitive)<0)
 382                                 && (j==aFiles[1].size() || collstr(aFiles[2][k].filename, aFiles[1][j].filename, casesensitive)<0) )
 383                         {
 384                                 const unsigned nDiffCode = DIFFCODE::THIRD | DIFFCODE::FILE;
 385                                 AddToList(subdir[0], subdir[1], subdir[2], nullptr, nullptr, &aFiles[2][k], nDiffCode, myStruct, parent);
 386                                 ++k;
 387                                 // Advance right pointer over right-only entry, and then retest with new pointers
 388                                 continue;
 389                         }
 390
 391                         if ((i<aFiles[0].size() && j<aFiles[1].size() && collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) == 0)
 392                             && (k==aFiles[2].size() || collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive) != 0))
 393                         {
 394                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::SECOND | DIFFCODE::FILE;
 395                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], &aFiles[1][j], nullptr, nDiffCode, myStruct, parent);
 396                                 ++i;
 397                                 ++j;
 398                                 continue;
 399                         }
 400                         else if ((i<aFiles[0].size() && k<aFiles[2].size() && collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive) == 0)
 401                             && (j==aFiles[1].size() || collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive) != 0))
 402                         {
 403                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::THIRD | DIFFCODE::FILE;
 404                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], nullptr, &aFiles[2][k], nDiffCode, myStruct, parent);
 405                                 ++i;
 406                                 ++k;
 407                                 continue;
 408                         }
 409                         else if ((j<aFiles[1].size() && k<aFiles[2].size() && collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive) == 0)
 410                             && (i==aFiles[0].size() || collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) != 0))
 411                         {
 412                                 const unsigned nDiffCode = DIFFCODE::SECOND | DIFFCODE::THIRD | DIFFCODE::FILE;
 413                                 AddToList(subdir[0], subdir[1], subdir[2], nullptr, &aFiles[1][j], &aFiles[2][k], nDiffCode, myStruct, parent);
 414                                 ++j;
 415                                 ++k;
 416                                 continue;
 417                         }
 418                 }
 419                 if (i<aFiles[0].size())
 420                 {
 421                         if (nDirs < 3)
 422                         {
 423                                 assert(j<aFiles[1].size());
 424                                 const unsigned nDiffCode = DIFFCODE::BOTH | DIFFCODE::FILE;
 425                                 AddToList(subdir[0], subdir[1], &aFiles[0][i], &aFiles[1][j], nDiffCode, myStruct, parent);
 426                                 ++i;
 427                                 ++j;
 428                                 continue;
 429                         }
 430                         else
 431                         {
 432                                 assert(j<aFiles[1].size());
 433                                 assert(k<aFiles[2].size());
 434                                 const unsigned nDiffCode = DIFFCODE::ALL | DIFFCODE::FILE;
 435                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], &aFiles[1][j], &aFiles[2][k], nDiffCode, myStruct, parent);
 436                                 ++i;
 437                                 ++j;
 438                                 ++k;
 439                                 continue;
 440                         }
 441                 }
 442                 break;
 443         }
 444         return 1;
 445 }
 446
 447 /**
 448  * @brief Compare DiffItems in list and add results to compare context.
 449  *
 450  * @param myStruct [in] A structure containing compare-related data.
 451  * @param parentdiffpos [in] Position of parent diff item
 452  * @return >= 0 number of diff items, -1 if compare was aborted
 453  */
 454 int DirScan_CompareItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 455 {
 456         const int compareMethod = myStruct->context->GetCompareMethod();
 457         int nworkers = 1;
 458
 459         if (compareMethod == CMP_CONTENT || compareMethod == CMP_QUICK_CONTENT)
 460         {
 461                 nworkers = GetOptionsMgr()->GetInt(OPT_CMP_COMPARE_THREADS);
 462                 if (nworkers <= 0)
 463                 {
 464                         nworkers += Environment::processorCount();
 465                         if (nworkers <= 0)
 466                                 nworkers = 1;
 467                 }
 468         }
 469
 470         ThreadPool threadPool(nworkers, nworkers);
 471         std::vector<DiffWorkerPtr> workers;
 472         NotificationQueue queue;
 473         myStruct->context->m_pCompareStats->SetCompareThreadCount(nworkers);
 474         for (int i = 0; i < nworkers; ++i)
 475         {
 476                 workers.push_back(DiffWorkerPtr(new DiffWorker(queue, myStruct->context, i)));
 477                 threadPool.start(*workers[i]);
 478         }
 479
 480         int res = CompareItems(queue, myStruct, parentdiffpos);
 481
 482         Thread::sleep(100);
 483         queue.wakeUpAll();
 484         threadPool.joinAll();
 485
 486         return res;
 487 }
 488
 489 static int CompareItems(NotificationQueue& queue, DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 490 {
 491         NotificationQueue queueResult;
 492         Stopwatch stopwatch;
 493         CDiffContext *pCtxt = myStruct->context;
 494         int res = 0;
 495         int count = 0;
 496         bool bCompareFailure = false;
 497         if (parentdiffpos == nullptr)
 498                 myStruct->pSemaphore->wait();
 499         stopwatch.start();
 500         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 501         while (pos != nullptr)
 502         {
 503                 if (pCtxt->ShouldAbort())
 504                         break;
 505
 506                 if (stopwatch.elapsed() > 2000000)
 507                 {
 508                         int event = CDiffThread::EVENT_COMPARE_PROGRESSED;
 509                         myStruct->m_listeners.notify(myStruct, event);
 510                         stopwatch.restart();
 511                 }
 512                 myStruct->pSemaphore->wait();
 513                 DIFFITEM *curpos = pos;
 514                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 515                 bool existsalldirs = di.diffcode.existAll();
 516                 if (di.diffcode.isDirectory() && pCtxt->m_bRecursive)
 517                 {
 518                         if ((di.diffcode.diffcode & DIFFCODE::CMPERR) != DIFFCODE::CMPERR)
 519                         {       // Only clear DIFF|SAME flags if not CMPERR (eg. both flags together)
 520                                 di.diffcode.diffcode &= ~(DIFFCODE::DIFF | DIFFCODE::SAME);
 521                         }
 522                         int ndiff = CompareItems(queue, myStruct, curpos);
 523                         // Propogate sub-directory status to this directory
 524                         if (ndiff > 0)
 525                         {       // There were differences in the sub-directories
 526                                 if (existsalldirs)
 527                                         di.diffcode.diffcode |= DIFFCODE::DIFF;
 528                                 res += ndiff;
 529                         }
 530                         else
 531                         if (ndiff == 0)
 532                         {       // Sub-directories were identical
 533                                 if (existsalldirs)
 534                                         di.diffcode.diffcode |= DIFFCODE::SAME;
 535                         }
 536                         else
 537                         if (ndiff == -1)
 538                         {       // There were file IO-errors during sub-directory comparison.
 539                                 di.diffcode.diffcode |= DIFFCODE::CMPERR;
 540                                 bCompareFailure = true;
 541                         }
 542                 }
 543                 if (existsalldirs)
 544                         queue.enqueueUrgentNotification(new WorkNotification(di, queueResult));
 545                 else
 546                         queue.enqueueNotification(new WorkNotification(di, queueResult));
 547                 ++count;
 548                 pos = curpos;
 549                 pCtxt->GetNextSiblingDiffRefPosition(pos);
 550         }
 551
 552         while (count > 0)
 553         {
 554                 AutoPtr<Notification> pNf(queueResult.waitDequeueNotification());
 555                 if (pNf.get() == nullptr)
 556                         break;
 557                 WorkCompletedNotification* pWorkCompletedNf = dynamic_cast<WorkCompletedNotification*>(pNf.get());
 558                 if (pWorkCompletedNf != nullptr) {
 559                         DIFFITEM &di = pWorkCompletedNf->data();
 560                         if (di.diffcode.isResultError()) {
 561                                 DIFFITEM *diParent = di.GetParentLink();
 562                                 assert(diParent != nullptr);
 563                                 if (diParent != nullptr)
 564                                 {
 565                                         diParent->diffcode.diffcode |= DIFFCODE::CMPERR;
 566                                         bCompareFailure = true;
 567                                 }
 568                         }
 569
 570                         if (di.diffcode.isResultDiff() ||
 571                                 (!di.diffcode.existAll() && !di.diffcode.isResultFiltered()))
 572                                 res++;
 573                 }
 574                 --count;
 575         }
 576
 577         return bCompareFailure || pCtxt->ShouldAbort() ? -1 : res;
 578 }
 579
 580 /**
 581  * @brief Compare DiffItems in context marked for rescan.
 582  *
 583  * @param myStruct [in,out] A structure containing compare-related data.
 584  * @param parentdiffpos [in] Position of parent diff item
 585  * @return >= 0 number of diff items, -1 if compare was aborted
 586  */
 587 static int CompareRequestedItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 588 {
 589         CDiffContext *pCtxt = myStruct->context;
 590         int res = 0;
 591         bool bCompareFailure = false;
 592         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 593
 594         while (pos != nullptr)
 595         {
 596                 if (pCtxt->ShouldAbort())
 597                 {
 598                         res = -1;
 599                         break;
 600                 }
 601
 602                 DIFFITEM *curpos = pos;
 603                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 604                 bool existsalldirs = di.diffcode.existAll();
 605                 if (di.diffcode.isDirectory())
 606                 {
 607                         if (pCtxt->m_bRecursive)
 608                         {
 609                                 di.diffcode.diffcode &= ~(DIFFCODE::DIFF | DIFFCODE::SAME);
 610                                 int ndiff = CompareRequestedItems(myStruct, curpos);
 611                                 if (ndiff > 0)
 612                                 {
 613                                         if (existsalldirs)
 614                                                 di.diffcode.diffcode |= DIFFCODE::DIFF;
 615                                         res += ndiff;
 616                                 }
 617                                 else
 618                                 if (ndiff == 0)
 619                                 {
 620                                         if (existsalldirs)
 621                                                 di.diffcode.diffcode |= DIFFCODE::SAME;
 622                                 }
 623                                 else
 624                                 if (ndiff == -1)
 625                                 {       // There were file IO-errors during sub-directory comparison.
 626                                         di.diffcode.diffcode |= DIFFCODE::CMPERR;
 627                                         bCompareFailure = true;
 628                                 }
 629                         }
 630                 }
 631                 else
 632                 {
 633                         if (di.diffcode.isScanNeeded())
 634                         {
 635                                 CompareDiffItem(di, pCtxt);
 636                                 if (di.diffcode.isResultError()) {
 637                                         DIFFITEM *diParent = di.GetParentLink();
 638                                         assert(diParent != nullptr);
 639                                         if (diParent != nullptr)
 640                                         {
 641                                                 diParent->diffcode.diffcode |= DIFFCODE::CMPERR;
 642                                                 bCompareFailure = true;
 643                                         }
 644                         }
 645
 646                         }
 647                 }
 648                 if (di.diffcode.isResultDiff() ||
 649                         (!existsalldirs && !di.diffcode.isResultFiltered()))
 650                         res++;
 651         }
 652         return bCompareFailure ? -1 : res;;
 653 }
 654
 655 int DirScan_CompareRequestedItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 656 {
 657         CAssureScriptsForThread scriptsForRescan;
 658         return CompareRequestedItems(myStruct, parentdiffpos);
 659 }
 660
 661 static int markChildrenForRescan(CDiffContext *pCtxt, DIFFITEM *parentdiffpos)
 662 {
 663         int ncount = 0;
 664         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 665         while (pos != nullptr)
 666         {
 667                 DIFFITEM *curpos = pos;
 668                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 669                 if (di.diffcode.isDirectory())
 670                         ncount += markChildrenForRescan(pCtxt, curpos);
 671                 else
 672                 {
 673                         di.diffcode.diffcode |= DIFFCODE::NEEDSCAN;
 674                         ++ncount;
 675                 }
 676         }
 677         return ncount;
 678 }
 679
 680 int DirScan_UpdateMarkedItems(DiffFuncStruct *myStruct, DIFFITEM *parentdiffpos)
 681 {
 682         CDiffContext *pCtxt = myStruct->context;
 683         DIFFITEM *pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 684         int ncount = 0;
 685
 686         while (pos != nullptr)
 687         {
 688                 if (pCtxt->ShouldAbort())
 689                         break;
 690                 DIFFITEM *curpos = pos;
 691                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 692                 if (di.diffcode.isScanNeeded())
 693                 {
 694                         bool bItemsExist = true;
 695                         UpdateDiffItem(di, bItemsExist, pCtxt);
 696                         if (!bItemsExist)
 697                         {
 698                                 di.DelinkFromSiblings();        // delink from list of Siblings
 699                                 delete &di;                                     // Also delete all Children items
 700                                 continue;                                       // (... because `di` is now invalid)
 701                         }
 702                         if (!di.diffcode.isDirectory())
 703                                 ++ncount;
 704                 }
 705                 if (di.diffcode.isDirectory() && pCtxt->m_bRecursive)
 706                 {
 707                         if (di.diffcode.isScanNeeded() && !di.diffcode.isResultFiltered())
 708                         {
 709                                 di.RemoveChildren();
 710                                 di.diffcode.diffcode &= ~DIFFCODE::NEEDSCAN;
 711
 712                                 bool casesensitive = false;
 713                                 int depth = myStruct->context->m_bRecursive ? -1 : 0;
 714                                 String subdir[3];
 715                                 PathContext paths = myStruct->context->GetNormalizedPaths();
 716                                 for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 717                                         subdir[i] = di.diffFileInfo[i].GetFile();
 718                                 DirScan_GetItems(paths, subdir, myStruct,
 719                                         casesensitive, depth, &di, myStruct->context->m_bWalkUniques);
 720                                 ncount += markChildrenForRescan(myStruct->context, curpos);
 721                         }
 722                         else
 723                         {
 724                                 ncount += DirScan_UpdateMarkedItems(myStruct, curpos);
 725                         }
 726                 }
 727         }
 728         return ncount;
 729 }
 730 /**
 731  * @brief Update diffitem file/dir infos.
 732  *
 733  * Re-tests dirs/files if sides still exists, and updates infos for
 734  * existing sides. This assumes filenames, or paths are not changed.
 735  * Since in normal situations (I can think of) they cannot change
 736  * after first compare.
 737  *
 738  * @param [in,out] di DiffItem to update.
 739  * @param [out] bExists Set to
 740  *  - true if one of items exists so diffitem is valid
 741  *  - false if items were deleted, so diffitem is not valid
 742  * @param [in] pCtxt Compare context
 743  */
 744 static void UpdateDiffItem(DIFFITEM &di, bool & bExists, CDiffContext *pCtxt)
 745 {
 746         bExists = false;
 747         di.diffcode.setSideNone();
 748         for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 749         {
 750                 di.diffFileInfo[i].ClearPartial();
 751                 if (pCtxt->UpdateInfoFromDiskHalf(di, i))
 752                 {
 753                         di.diffcode.diffcode |= DIFFCODE::FIRST << i;
 754                         bExists = true;
 755                 }
 756         }
 757 }
 758
 759 /**
 760  * @brief Compare two diffitems and add results to difflist in context.
 761  *
 762  * This function does the actual compare for previously gathered list of
 763  * items. Basically we:
 764  * - ignore items matching file filters
 765  * - add non-ignored directories (no compare for directory items)
 766  * - add unique files
 767  * - compare files
 768  *
 769  * @param [in] di DiffItem to compare
 770  * @param [in,out] pCtxt Compare context: contains difflist, encoding info etc.
 771  * @todo For date compare, maybe we should use creation date if modification
 772  * date is missing?
 773  */
 774 void CompareDiffItem(DIFFITEM &di, CDiffContext * pCtxt)
 775 {
 776         int nDirs = pCtxt->GetCompareDirs();
 777         // Clear rescan-request flag (not set by all codepaths)
 778         di.diffcode.diffcode &= ~DIFFCODE::NEEDSCAN;
 779         // Is it a directory?
 780         if (di.diffcode.isDirectory())
 781         {
 782                 // We don't actually 'compare' directories, just add non-ignored
 783                 // directories to list.
 784                 StoreDiffData(di, pCtxt, nullptr);
 785         }
 786         else
 787         {
 788                 // 1. Test against filters
 789                 if (pCtxt->m_piFilterGlobal==nullptr ||
 790                         (nDirs == 2 && pCtxt->m_piFilterGlobal->includeFile(di.diffFileInfo[0].filename, di.diffFileInfo[1].filename)) ||
 791                         (nDirs == 3 && pCtxt->m_piFilterGlobal->includeFile(di.diffFileInfo[0].filename, di.diffFileInfo[1].filename, di.diffFileInfo[2].filename))
 792                         )
 793                 {
 794                         di.diffcode.diffcode |= DIFFCODE::INCLUDED;
 795                         FolderCmp folderCmp;
 796                         di.diffcode.diffcode |= folderCmp.prepAndCompareFiles(pCtxt, di);
 797                         StoreDiffData(di, pCtxt, &folderCmp);
 798                 }
 799                 else
 800                 {
 801                         di.diffcode.diffcode |= DIFFCODE::SKIPPED;
 802                         StoreDiffData(di, pCtxt, nullptr);
 803                 }
 804         }
 805 }
 806
 807 /**
 808  * @brief Send one file or directory result back through the diff context.
 809  * @param [in] di Data to store.
 810  * @param [in] pCtxt Compare context.
 811  * @param [in] pCmpData Folder compare data.
 812  */
 813 static void StoreDiffData(DIFFITEM &di, CDiffContext * pCtxt,
 814                 const FolderCmp * pCmpData)
 815 {
 816         if (pCmpData != nullptr)
 817         {
 818                 di.nsdiffs = pCmpData->m_ndiffs;
 819                 di.nidiffs = pCmpData->m_ntrivialdiffs;
 820
 821                 for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 822                 {
 823                         // Set text statistics
 824                         if (di.diffcode.exists(i))
 825                         {
 826                                 di.diffFileInfo[i].m_textStats = pCmpData->m_diffFileData.m_textStats[i];
 827                                 di.diffFileInfo[i].encoding = pCmpData->m_diffFileData.m_FileLocation[i].encoding;
 828                         }
 829                 }
 830         }
 831
 832         pCtxt->m_pCompareStats->AddItem(di.diffcode.diffcode);
 833         //pCtxt->AddDiff(di);
 834 }
 835
 836 /**
 837  * @brief Add one compare item to list.
 838  * @param [in] sLeftDir Left subdirectory.
 839  * @param [in] sRightDir Right subdirectory.
 840  * @param [in] lent Left item data to add.
 841  * @param [in] rent Right item data to add.
 842  * @param [in] pCtxt Compare context.
 843  * @param [in] parent Parent of item to be added
 844  */
 845 static DIFFITEM *AddToList(const String& sLeftDir, const String& sRightDir,
 846         const DirItem * lent, const DirItem * rent,
 847         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent)
 848 {
 849         return AddToList(sLeftDir, sRightDir, sLeftDir, lent, rent, nullptr, code, myStruct, parent, 2);
 850 }
 851
 852 /**
 853  * @brief Add one compare item to list.
 854  */
 855 static DIFFITEM *AddToList(const String& sDir1, const String& sDir2, const String& sDir3,
 856         const DirItem *ent1, const DirItem *ent2, const DirItem *ent3,
 857         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent, int nItems /*= 3*/)
 858 {
 859         // We must store both paths - we cannot get paths later
 860         // and we need unique item paths for example when items
 861         // change to identical
 862         DIFFITEM *di = myStruct->context->AddNewDiff(parent);
 863
 864         di->diffFileInfo[0].path = sDir1;
 865         di->diffFileInfo[1].path = sDir2;
 866         di->diffFileInfo[2].path = sDir3;
 867
 868         if (ent1 != nullptr)
 869         {
 870                 di->diffFileInfo[0].filename = ent1->filename;
 871                 di->diffFileInfo[0].mtime = ent1->mtime;
 872                 di->diffFileInfo[0].ctime = ent1->ctime;
 873                 di->diffFileInfo[0].size = ent1->size;
 874                 di->diffFileInfo[0].flags.attributes = ent1->flags.attributes;
 875         }
 876         else
 877         {
 878                 // Don't break CDirView::DoCopyRightToLeft()
 879                 if (ent3 != nullptr)
 880                         di->diffFileInfo[0].filename = ent3->filename;
 881                 else if (ent2 != nullptr)
 882                         di->diffFileInfo[0].filename = ent2->filename;
 883         }
 884
 885         if (ent2 != nullptr)
 886         {
 887                 di->diffFileInfo[1].filename = ent2->filename;
 888                 di->diffFileInfo[1].mtime = ent2->mtime;
 889                 di->diffFileInfo[1].ctime = ent2->ctime;
 890                 di->diffFileInfo[1].size = ent2->size;
 891                 di->diffFileInfo[1].flags.attributes = ent2->flags.attributes;
 892         }
 893         else
 894         {
 895                 // Don't break CDirView::DoCopyLeftToRight()
 896                 if (ent1 != nullptr)
 897                         di->diffFileInfo[1].filename = ent1->filename;
 898                 else if (ent3 != nullptr)
 899                         di->diffFileInfo[1].filename = ent3->filename;
 900         }
 901
 902         if (ent3 != nullptr)
 903         {
 904                 di->diffFileInfo[2].filename = ent3->filename;
 905                 di->diffFileInfo[2].mtime = ent3->mtime;
 906                 di->diffFileInfo[2].ctime = ent3->ctime;
 907                 di->diffFileInfo[2].size = ent3->size;
 908                 di->diffFileInfo[2].flags.attributes = ent3->flags.attributes;
 909         }
 910         else
 911         {
 912                 // Don't break CDirView::DoCopyLeftToRight()
 913                 if (ent1 != nullptr)
 914                         di->diffFileInfo[2].filename = ent1->filename;
 915                 else if (ent2 != nullptr)
 916                         di->diffFileInfo[2].filename = ent2->filename;
 917         }
 918
 919         if (nItems == 2)
 920                 di->diffcode.diffcode = code;
 921         else
 922                 di->diffcode.diffcode = code | DIFFCODE::THREEWAY;
 923
 924         myStruct->context->m_pCompareStats->IncreaseTotalItems();
 925         myStruct->pSemaphore->set();
 926         return di;
 927 }