Src/DirScan.cpp

   1 /**
   2  *  @file DirScan.cpp
   3  *
   4  *  @brief Implementation of DirScan (q.v.) and helper functions
   5  */
   6
   7 #include "DirScan.h"
   8 #include <cassert>
   9 #include <memory>
  10 #define POCO_NO_UNWINDOWS 1
  11 #include <Poco/Semaphore.h>
  12 #include <Poco/Notification.h>
  13 #include <Poco/NotificationQueue.h>
  14 #include <Poco/Environment.h>
  15 #include <Poco/ThreadPool.h>
  16 #include <Poco/Thread.h>
  17 #include <Poco/Runnable.h>
  18 #include <Poco/Mutex.h>
  19 #include <Poco/AutoPtr.h>
  20 #include <Poco/Stopwatch.h>
  21 #include <Poco/Format.h>
  22 #include "DiffThread.h"
  23 #include "UnicodeString.h"
  24 #include "DiffWrapper.h"
  25 #include "CompareStats.h"
  26 #include "FolderCmp.h"
  27 #include "FileFilterHelper.h"
  28 #include "IAbortable.h"
  29 #include "FolderCmp.h"
  30 #include "DirItem.h"
  31 #include "DirTravel.h"
  32 #include "paths.h"
  33 #include "Plugins.h"
  34 #include "MergeApp.h"
  35 #include "OptionsDef.h"
  36 #include "OptionsMgr.h"
  37
  38 using Poco::NotificationQueue;
  39 using Poco::Notification;
  40 using Poco::AutoPtr;
  41 using Poco::Thread;
  42 using Poco::ThreadPool;
  43 using Poco::Runnable;
  44 using Poco::Environment;
  45 using Poco::Stopwatch;
  46
  47 // Static functions (ie, functions only used locally)
  48 void CompareDiffItem(DIFFITEM &di, CDiffContext * pCtxt);
  49 static void StoreDiffData(DIFFITEM &di, CDiffContext * pCtxt,
  50                 const FolderCmp * pCmpData);
  51 static DIFFITEM *AddToList(const String& sLeftDir, const String& sRightDir, const DirItem * lent, const DirItem * rent,
  52         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent);
  53 static DIFFITEM *AddToList(const String& sDir1, const String& sDir2, const String& sDir3, const DirItem * ent1, const DirItem * ent2, const DirItem * ent3,
  54         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent, int nItems = 3);
  55 static void UpdateDiffItem(DIFFITEM & di, bool & bExists, CDiffContext *pCtxt);
  56 static int CompareItems(NotificationQueue& queue, DiffFuncStruct *myStruct, uintptr_t parentdiffpos);
  57
  58 class WorkNotification: public Poco::Notification
  59 {
  60 public:
  61         WorkNotification(DIFFITEM& di, NotificationQueue& queueResult): m_di(di), m_queueResult(queueResult) {}
  62         DIFFITEM& data() const { return m_di; }
  63         NotificationQueue& queueResult() const { return m_queueResult; }
  64 private:
  65         DIFFITEM& m_di;
  66         NotificationQueue& m_queueResult;
  67 };
  68
  69 class WorkCompletedNotification: public Poco::Notification
  70 {
  71 public:
  72         explicit WorkCompletedNotification(DIFFITEM& di): m_di(di) {}
  73         DIFFITEM& data() const { return m_di; }
  74 private:
  75         DIFFITEM& m_di;
  76 };
  77
  78 class DiffWorker: public Runnable
  79 {
  80 public:
  81         DiffWorker(NotificationQueue& queue, CDiffContext *pCtxt, int id):
  82           m_queue(queue), m_pCtxt(pCtxt), m_id(id) {}
  83
  84         void run()
  85         {
  86                 // keep the scripts alive during the Rescan
  87                 // when we exit the thread, we delete this and release the scripts
  88                 CAssureScriptsForThread scriptsForRescan;
  89
  90                 AutoPtr<Notification> pNf(m_queue.waitDequeueNotification());
  91                 while (pNf)
  92                 {
  93                         WorkNotification* pWorkNf = dynamic_cast<WorkNotification*>(pNf.get());
  94                         if (pWorkNf) {
  95                                 m_pCtxt->m_pCompareStats->BeginCompare(&pWorkNf->data(), m_id);
  96                                 if (!m_pCtxt->ShouldAbort())
  97                                         CompareDiffItem(pWorkNf->data(), m_pCtxt);
  98                                 pWorkNf->queueResult().enqueueNotification(new WorkCompletedNotification(pWorkNf->data()));
  99                         }
 100                         pNf = m_queue.waitDequeueNotification();
 101                 }
 102         }
 103
 104 private:
 105         NotificationQueue& m_queue;
 106         CDiffContext *m_pCtxt;
 107         int m_id;
 108 };
 109
 110 typedef std::shared_ptr<DiffWorker> DiffWorkerPtr;
 111
 112 /**
 113  * @brief Collect file- and folder-names to list.
 114  * This function walks given folders and adds found subfolders and files into
 115  * lists. There are two modes, determined by the @p depth:
 116  * - in non-recursive mode we walk only given folders, and add files.
 117  *   contained. Subfolders are added as folder items, not walked into.
 118  * - in recursive mode we walk all subfolders and add the files they
 119  *   contain into list.
 120  *
 121  * Items are tested against file filters in this function.
 122  *
 123  * @param [in] paths Root paths of compare
 124  * @param [in] leftsubdir Left side subdirectory under root path
 125  * @param [in] bLeftUniq Is left-side folder unique folder?
 126  * @param [in] rightsubdir Right side subdirectory under root path
 127  * @param [in] bRightUniq Is right-side folder unique folder?
 128  * @param [in] myStruct Compare-related data, like context etc.
 129  * @param [in] casesensitive Is filename compare casesensitive?
 130  * @param [in] depth Levels of subdirectories to scan, -1 scans all
 131  * @param [in] parent Folder diff item to be scanned
 132  * @param [in] bUniques If true, walk into unique folders.
 133  * @return 1 normally, -1 if compare was aborted
 134  */
 135 int DirScan_GetItems(const PathContext &paths, const String subdir[],
 136                 DiffFuncStruct *myStruct,
 137                 bool casesensitive, int depth, DIFFITEM *parent,
 138                 bool bUniques)
 139 {
 140         static const TCHAR backslash[] = _T("\\");
 141         int nDirs = paths.GetSize();
 142         CDiffContext *pCtxt = myStruct->context;
 143         String sDir[3];
 144         String subprefix[3];
 145
 146         int nIndex;
 147         std::copy(paths.begin(), paths.end(), sDir);
 148
 149         if (!subdir[0].empty())
 150         {
 151                 for (nIndex = 0; nIndex < paths.GetSize(); nIndex++)
 152                 {
 153                         sDir[nIndex] = paths::ConcatPath(sDir[nIndex], subdir[nIndex]);
 154                         subprefix[nIndex] = subdir[nIndex] + backslash;
 155                 }
 156         }
 157
 158         DirItemArray dirs[3], aFiles[3];
 159         for (nIndex = 0; nIndex < nDirs; nIndex++)
 160                 LoadAndSortFiles(sDir[nIndex], &dirs[nIndex], &aFiles[nIndex], casesensitive);
 161
 162         // Allow user to abort scanning
 163         if (pCtxt->ShouldAbort())
 164                 return -1;
 165
 166         // Handle directories
 167         // i points to current directory in left list (leftDirs)
 168         // j points to current directory in right list (rightDirs)
 169
 170         for (nIndex = 0; nIndex < nDirs; nIndex++)
 171                 if (dirs[nIndex].size() != 0 || aFiles[nIndex].size() != 0) break;
 172         if (nIndex == nDirs)
 173                 return 0;
 174
 175         DirItemArray::size_type i=0, j=0, k=0;
 176         while (1)
 177         {
 178                 if (pCtxt->ShouldAbort())
 179                         return -1;
 180
 181                 if (i >= dirs[0].size() && j >= dirs[1].size() && (nDirs < 3 || k >= dirs[2].size()))
 182                         break;
 183
 184                 unsigned nDiffCode = DIFFCODE::DIR;
 185                 // Comparing directories leftDirs[i].name to rightDirs[j].name
 186                 if (i<dirs[0].size() && (j==dirs[1].size() || collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive)<0)
 187                         && (nDirs < 3 ||      (k==dirs[2].size() || collstr(dirs[0][i].filename, dirs[2][k].filename, casesensitive)<0) ))
 188                 {
 189                         nDiffCode |= DIFFCODE::FIRST;
 190                 }
 191                 else if (j<dirs[1].size() && (i==dirs[0].size() || collstr(dirs[1][j].filename, dirs[0][i].filename, casesensitive)<0)
 192                         && (nDirs < 3 ||      (k==dirs[2].size() || collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive)<0) ))
 193                 {
 194                         nDiffCode |= DIFFCODE::SECOND;
 195                 }
 196                 else if (nDirs < 3)
 197                 {
 198                         nDiffCode |= DIFFCODE::BOTH;
 199                 }
 200                 else
 201                 {
 202                         if (k<dirs[2].size() && (i==dirs[0].size() || collstr(dirs[2][k].filename, dirs[0][i].filename, casesensitive)<0)
 203                                 &&                     (j==dirs[1].size() || collstr(dirs[2][k].filename, dirs[1][j].filename, casesensitive)<0) )
 204                         {
 205                                 nDiffCode |= DIFFCODE::THIRD;
 206                         }
 207                         else if ((i<dirs[0].size() && j<dirs[1].size() && collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive) == 0)
 208                                 && (k==dirs[2].size() || collstr(dirs[2][k].filename, dirs[0][i].filename, casesensitive) != 0))
 209                         {
 210                                 nDiffCode |= DIFFCODE::FIRST | DIFFCODE::SECOND;
 211                         }
 212                         else if ((i<dirs[0].size() && k<dirs[2].size() && collstr(dirs[0][i].filename, dirs[2][k].filename, casesensitive) == 0)
 213                                 && (j==dirs[1].size() || collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive) != 0))
 214                         {
 215                                 nDiffCode |= DIFFCODE::FIRST | DIFFCODE::THIRD;
 216                         }
 217                         else if ((j<dirs[1].size() && k<dirs[2].size() && collstr(dirs[1][j].filename, dirs[2][k].filename, casesensitive) == 0)
 218                                 && (i==dirs[0].size() || collstr(dirs[0][i].filename, dirs[1][j].filename, casesensitive) != 0))
 219                         {
 220                                 nDiffCode |= DIFFCODE::SECOND | DIFFCODE::THIRD;
 221                         }
 222                         else
 223                         {
 224                                 nDiffCode |= DIFFCODE::ALL;
 225                         }
 226                 }
 227
 228                 String leftnewsub;
 229                 String rightnewsub;
 230                 String middlenewsub;
 231                 if (nDirs < 3)
 232                 {
 233                         leftnewsub  = (nDiffCode & DIFFCODE::FIRST)  ? subprefix[0] + dirs[0][i].filename.get() : subprefix[0] + dirs[1][j].filename.get();
 234                         rightnewsub = (nDiffCode & DIFFCODE::SECOND) ? subprefix[1] + dirs[1][j].filename.get() : subprefix[1] + dirs[0][i].filename.get();
 235
 236                         // Test against filter so we don't include contents of filtered out directories
 237                         // Also this is only place we can test for both-sides directories in recursive compare
 238                         if ((pCtxt->m_piFilterGlobal && !pCtxt->m_piFilterGlobal->includeDir(leftnewsub, rightnewsub)) ||
 239                                 (pCtxt->m_bIgnoreReparsePoints && (
 240                                 (nDiffCode & DIFFCODE::FIRST) && (dirs[0][i].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 241                                         (nDiffCode & DIFFCODE::SECOND) && (dirs[1][j].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT))
 242                                         )
 243                                 )
 244                                 nDiffCode |= DIFFCODE::SKIPPED;
 245                 }
 246                 else
 247                 {
 248                         leftnewsub   = subprefix[0];
 249                         if (nDiffCode & DIFFCODE::FIRST)       leftnewsub += dirs[0][i].filename;
 250                         else if (nDiffCode & DIFFCODE::SECOND) leftnewsub += dirs[1][j].filename;
 251                         else if (nDiffCode & DIFFCODE::THIRD)  leftnewsub += dirs[2][k].filename;
 252                         middlenewsub = subprefix[1];
 253                         if (nDiffCode & DIFFCODE::SECOND)      middlenewsub += dirs[1][j].filename;
 254                         else if (nDiffCode & DIFFCODE::FIRST)  middlenewsub += dirs[0][i].filename;
 255                         else if (nDiffCode & DIFFCODE::THIRD)  middlenewsub += dirs[2][k].filename;
 256                         rightnewsub  = subprefix[2];
 257                         if (nDiffCode & DIFFCODE::THIRD)       rightnewsub += dirs[2][k].filename;
 258                         else if (nDiffCode & DIFFCODE::FIRST)  rightnewsub += dirs[0][i].filename;
 259                         else if (nDiffCode & DIFFCODE::SECOND) rightnewsub += dirs[1][j].filename;
 260
 261                         // Test against filter so we don't include contents of filtered out directories
 262                         // Also this is only place we can test for both-sides directories in recursive compare
 263                         if ((pCtxt->m_piFilterGlobal && !pCtxt->m_piFilterGlobal->includeDir(leftnewsub, middlenewsub, rightnewsub)) ||
 264                                 (pCtxt->m_bIgnoreReparsePoints && (
 265                                   (nDiffCode & DIFFCODE::FIRST)  && (dirs[0][i].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 266                                   (nDiffCode & DIFFCODE::SECOND) && (dirs[1][j].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT) ||
 267                                   (nDiffCode & DIFFCODE::THIRD)  && (dirs[2][k].flags.attributes & FILE_ATTRIBUTE_REPARSE_POINT))
 268                                 )
 269                            )
 270                                 nDiffCode |= DIFFCODE::SKIPPED;
 271                 }
 272
 273                 // add to list
 274                 if (!depth)
 275                 {
 276                         if (nDirs < 3)
 277                                 AddToList(subdir[0], subdir[1],
 278                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : NULL,
 279                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : NULL,
 280                                         nDiffCode, myStruct, parent);
 281                         else
 282                                 AddToList(subdir[0], subdir[1], subdir[2],
 283                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : NULL,
 284                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : NULL,
 285                                         (nDiffCode & DIFFCODE::THIRD ) ? &dirs[2][k] : NULL,
 286                                         nDiffCode, myStruct, parent);
 287                 }
 288                 else
 289                 {
 290                         // Recursive compare
 291                         if (nDirs < 3)
 292                         {
 293                                 DIFFITEM *me = AddToList(subdir[0], subdir[1],
 294                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : NULL,
 295                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : NULL,
 296                                         nDiffCode, myStruct, parent);
 297                                 if ((nDiffCode & DIFFCODE::SKIPPED) == 0 && ((nDiffCode & DIFFCODE::SIDEFLAGS) == DIFFCODE::BOTH || bUniques))
 298                                 {
 299                                         // Scan recursively all subdirectories too, we are not adding folders
 300                                         String newsubdir[3] = {leftnewsub, rightnewsub};
 301                                         int result = DirScan_GetItems(paths, newsubdir, myStruct, casesensitive,
 302                                                         depth - 1, me, bUniques);
 303                                         if (result == -1)
 304                                                 return -1;
 305                                 }
 306                         }
 307                         else
 308                         {
 309                                 DIFFITEM *me = AddToList(subdir[0], subdir[1], subdir[2],
 310                                         (nDiffCode & DIFFCODE::FIRST ) ? &dirs[0][i] : NULL,
 311                                         (nDiffCode & DIFFCODE::SECOND) ? &dirs[1][j] : NULL,
 312                                         (nDiffCode & DIFFCODE::THIRD ) ? &dirs[2][k] : NULL,
 313                                         nDiffCode, myStruct, parent);
 314                                 if ((nDiffCode & DIFFCODE::SKIPPED) == 0 && ((nDiffCode & DIFFCODE::SIDEFLAGS) == DIFFCODE::ALL || bUniques))
 315                                 {
 316                                         // Scan recursively all subdirectories too, we are not adding folders
 317                                         String newsubdir[3] = {leftnewsub, middlenewsub, rightnewsub};
 318                                         int result = DirScan_GetItems(paths, newsubdir, myStruct, casesensitive,
 319                                                         depth - 1, me, bUniques);
 320                                         if (result == -1)
 321                                                 return -1;
 322                                 }
 323                         }
 324                 }
 325                 if (nDiffCode & DIFFCODE::FIRST)
 326                         i++;
 327                 if (nDiffCode & DIFFCODE::SECOND)
 328                         j++;
 329                 if (nDiffCode & DIFFCODE::THIRD)
 330                         k++;
 331         }
 332         // Handle files
 333         // i points to current file in left list (aFiles[0])
 334         // j points to current file in right list (aFiles[1])
 335         i=0, j=0, k=0;
 336         while (1)
 337         {
 338                 if (pCtxt->ShouldAbort())
 339                         return -1;
 340
 341
 342                 // Comparing file aFiles[0][i].name to aFiles[1][j].name
 343                 if (i<aFiles[0].size() && (j==aFiles[1].size() ||
 344                                 collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) < 0)
 345                         && (nDirs < 3 ||
 346                                 (k==aFiles[2].size() || collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive)<0) ))
 347                 {
 348                         if (nDirs < 3)
 349                         {
 350                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::FILE;
 351                                 AddToList(subdir[0], subdir[1], &aFiles[0][i], 0, nDiffCode, myStruct, parent);
 352                         }
 353                         else
 354                         {
 355                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::FILE;
 356                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], 0, 0, nDiffCode, myStruct, parent);
 357                         }
 358                         // Advance left pointer over left-only entry, and then retest with new pointers
 359                         ++i;
 360                         continue;
 361                 }
 362                 if (j<aFiles[1].size() && (i==aFiles[0].size() ||
 363                                 collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) > 0)
 364                         && (nDirs < 3 ||
 365                                 (k==aFiles[2].size() || collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive)<0) ))
 366                 {
 367                         const unsigned nDiffCode = DIFFCODE::SECOND | DIFFCODE::FILE;
 368                         if (nDirs < 3)
 369                                 AddToList(subdir[0], subdir[1], 0, &aFiles[1][j], nDiffCode, myStruct, parent);
 370                         else
 371                                 AddToList(subdir[0], subdir[1], subdir[2], 0, &aFiles[1][j], 0, nDiffCode, myStruct, parent);
 372                         // Advance right pointer over right-only entry, and then retest with new pointers
 373                         ++j;
 374                         continue;
 375                 }
 376                 if (nDirs == 3)
 377                 {
 378                         if (k<aFiles[2].size() && (i==aFiles[0].size() ||
 379                                         collstr(aFiles[2][k].filename, aFiles[0][i].filename, casesensitive)<0)
 380                                 && (j==aFiles[1].size() || collstr(aFiles[2][k].filename, aFiles[1][j].filename, casesensitive)<0) )
 381                         {
 382                                 const unsigned nDiffCode = DIFFCODE::THIRD | DIFFCODE::FILE;
 383                                 AddToList(subdir[0], subdir[1], subdir[2], 0, 0, &aFiles[2][k], nDiffCode, myStruct, parent);
 384                                 ++k;
 385                                 // Advance right pointer over right-only entry, and then retest with new pointers
 386                                 continue;
 387                         }
 388
 389                         if ((i<aFiles[0].size() && j<aFiles[1].size() && collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) == 0)
 390                             && (k==aFiles[2].size() || collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive) != 0))
 391                         {
 392                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::SECOND | DIFFCODE::FILE;
 393                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], &aFiles[1][j], 0, nDiffCode, myStruct, parent);
 394                                 ++i;
 395                                 ++j;
 396                                 continue;
 397                         }
 398                         else if ((i<aFiles[0].size() && k<aFiles[2].size() && collstr(aFiles[0][i].filename, aFiles[2][k].filename, casesensitive) == 0)
 399                             && (j==aFiles[1].size() || collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive) != 0))
 400                         {
 401                                 const unsigned nDiffCode = DIFFCODE::FIRST | DIFFCODE::THIRD | DIFFCODE::FILE;
 402                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], 0, &aFiles[2][k], nDiffCode, myStruct, parent);
 403                                 ++i;
 404                                 ++k;
 405                                 continue;
 406                         }
 407                         else if ((j<aFiles[1].size() && k<aFiles[2].size() && collstr(aFiles[1][j].filename, aFiles[2][k].filename, casesensitive) == 0)
 408                             && (i==aFiles[0].size() || collstr(aFiles[0][i].filename, aFiles[1][j].filename, casesensitive) != 0))
 409                         {
 410                                 const unsigned nDiffCode = DIFFCODE::SECOND | DIFFCODE::THIRD | DIFFCODE::FILE;
 411                                 AddToList(subdir[0], subdir[1], subdir[2], 0, &aFiles[1][j], &aFiles[2][k], nDiffCode, myStruct, parent);
 412                                 ++j;
 413                                 ++k;
 414                                 continue;
 415                         }
 416                 }
 417                 if (i<aFiles[0].size())
 418                 {
 419                         if (nDirs < 3)
 420                         {
 421                                 assert(j<aFiles[1].size());
 422                                 const unsigned nDiffCode = DIFFCODE::BOTH | DIFFCODE::FILE;
 423                                 AddToList(subdir[0], subdir[1], &aFiles[0][i], &aFiles[1][j], nDiffCode, myStruct, parent);
 424                                 ++i;
 425                                 ++j;
 426                                 continue;
 427                         }
 428                         else
 429                         {
 430                                 assert(j<aFiles[1].size());
 431                                 assert(k<aFiles[2].size());
 432                                 const unsigned nDiffCode = DIFFCODE::ALL | DIFFCODE::FILE;
 433                                 AddToList(subdir[0], subdir[1], subdir[2], &aFiles[0][i], &aFiles[1][j], &aFiles[2][k], nDiffCode, myStruct, parent);
 434                                 ++i;
 435                                 ++j;
 436                                 ++k;
 437                                 continue;
 438                         }
 439                 }
 440                 break;
 441         }
 442         return 1;
 443 }
 444
 445 /**
 446  * @brief Compare DiffItems in list and add results to compare context.
 447  *
 448  * @param myStruct [in] A structure containing compare-related data.
 449  * @param parentdiffpos [in] Position of parent diff item
 450  * @return >= 0 number of diff items, -1 if compare was aborted
 451  */
 452 int DirScan_CompareItems(DiffFuncStruct *myStruct, uintptr_t parentdiffpos)
 453 {
 454         const int compareMethod = myStruct->context->GetCompareMethod();
 455         int nworkers = 1;
 456
 457         if (compareMethod == CMP_CONTENT || compareMethod == CMP_QUICK_CONTENT)
 458         {
 459                 nworkers = GetOptionsMgr()->GetInt(OPT_CMP_COMPARE_THREADS);
 460                 if (nworkers <= 0)
 461                 {
 462                         nworkers += Environment::processorCount();
 463                         if (nworkers <= 0)
 464                                 nworkers = 1;
 465                 }
 466         }
 467
 468         ThreadPool threadPool(nworkers, nworkers);
 469         std::vector<DiffWorkerPtr> workers;
 470         NotificationQueue queue;
 471         myStruct->context->m_pCompareStats->SetCompareThreadCount(nworkers);
 472         for (int i = 0; i < nworkers; ++i)
 473         {
 474                 workers.push_back(DiffWorkerPtr(new DiffWorker(queue, myStruct->context, i)));
 475                 threadPool.start(*workers[i]);
 476         }
 477
 478         int res = CompareItems(queue, myStruct, parentdiffpos);
 479
 480         Thread::sleep(100);
 481         queue.wakeUpAll();
 482         threadPool.joinAll();
 483
 484         return res;
 485 }
 486
 487 static int CompareItems(NotificationQueue& queue, DiffFuncStruct *myStruct, uintptr_t parentdiffpos)
 488 {
 489         NotificationQueue queueResult;
 490         Stopwatch stopwatch;
 491         CDiffContext *pCtxt = myStruct->context;
 492         int res = 0;
 493         int count = 0;
 494         if (!parentdiffpos)
 495                 myStruct->pSemaphore->wait();
 496         stopwatch.start();
 497         uintptr_t pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 498         while (pos)
 499         {
 500                 if (pCtxt->ShouldAbort())
 501                         break;
 502
 503                 if (stopwatch.elapsed() > 2000000)
 504                 {
 505                         int event = CDiffThread::EVENT_COMPARE_PROGRESSED;
 506                         myStruct->m_listeners.notify(myStruct, event);
 507                         stopwatch.restart();
 508                 }
 509                 myStruct->pSemaphore->wait();
 510                 uintptr_t curpos = pos;
 511                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 512                 bool existsalldirs = di.diffcode.existAll();
 513                 if (di.diffcode.isDirectory() && pCtxt->m_bRecursive)
 514                 {
 515                         di.diffcode.diffcode &= ~(DIFFCODE::DIFF | DIFFCODE::SAME);
 516                         int ndiff = CompareItems(queue, myStruct, curpos);
 517                         if (ndiff > 0)
 518                         {
 519                                 if (existsalldirs)
 520                                         di.diffcode.diffcode |= DIFFCODE::DIFF;
 521                                 res += ndiff;
 522                         }
 523                         else if (ndiff == 0)
 524                         {
 525                                 if (existsalldirs)
 526                                         di.diffcode.diffcode |= DIFFCODE::SAME;
 527                         }
 528                 }
 529                 if (existsalldirs)
 530                         queue.enqueueUrgentNotification(new WorkNotification(di, queueResult));
 531                 else
 532                         queue.enqueueNotification(new WorkNotification(di, queueResult));
 533                 ++count;
 534                 pos = curpos;
 535                 pCtxt->GetNextSiblingDiffRefPosition(pos);
 536         }
 537
 538         while (count > 0)
 539         {
 540                 AutoPtr<Notification> pNf(queueResult.waitDequeueNotification());
 541                 if (!pNf)
 542                         break;
 543                 WorkCompletedNotification* pWorkCompletedNf = dynamic_cast<WorkCompletedNotification*>(pNf.get());
 544                 if (pWorkCompletedNf) {
 545                         DIFFITEM &di = pWorkCompletedNf->data();
 546                         if (di.diffcode.isResultDiff() ||
 547                                 (!di.diffcode.existAll() && !di.diffcode.isResultFiltered()))
 548                                 res++;
 549                 }
 550                 --count;
 551         }
 552
 553         return pCtxt->ShouldAbort() ? -1 : res;
 554 }
 555
 556 /**
 557  * @brief Compare DiffItems in context marked for rescan.
 558  *
 559  * @param myStruct [in,out] A structure containing compare-related data.
 560  * @param parentdiffpos [in] Position of parent diff item
 561  * @return >= 0 number of diff items, -1 if compare was aborted
 562  */
 563 static int CompareRequestedItems(DiffFuncStruct *myStruct, uintptr_t parentdiffpos)
 564 {
 565         CDiffContext *pCtxt = myStruct->context;
 566         int res = 0;
 567         uintptr_t pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 568
 569         while (pos != NULL)
 570         {
 571                 if (pCtxt->ShouldAbort())
 572                 {
 573                         res = -1;
 574                         break;
 575                 }
 576
 577                 uintptr_t curpos = pos;
 578                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 579                 bool existsalldirs = di.diffcode.existAll();
 580                 if (di.diffcode.isDirectory())
 581                 {
 582                         if (pCtxt->m_bRecursive)
 583                         {
 584                                 di.diffcode.diffcode &= ~(DIFFCODE::DIFF | DIFFCODE::SAME);
 585                                 int ndiff = CompareRequestedItems(myStruct, curpos);
 586                                 if (ndiff > 0)
 587                                 {
 588                                         if (existsalldirs)
 589                                                 di.diffcode.diffcode |= DIFFCODE::DIFF;
 590                                         res += ndiff;
 591                                 }
 592                                 else if (ndiff == 0)
 593                                 {
 594                                         if (existsalldirs)
 595                                                 di.diffcode.diffcode |= DIFFCODE::SAME;
 596                                 }
 597                         }
 598                 }
 599                 else
 600                 {
 601                         if (di.diffcode.isScanNeeded())
 602                                 CompareDiffItem(di, pCtxt);
 603                 }
 604                 if (di.diffcode.isResultDiff() ||
 605                         (!existsalldirs && !di.diffcode.isResultFiltered()))
 606                         res++;
 607         }
 608         return res;
 609 }
 610
 611 int DirScan_CompareRequestedItems(DiffFuncStruct *myStruct, uintptr_t parentdiffpos)
 612 {
 613         CAssureScriptsForThread scriptsForRescan;
 614         return CompareRequestedItems(myStruct, parentdiffpos);
 615 }
 616
 617 static int markChildrenForRescan(CDiffContext *pCtxt, uintptr_t parentdiffpos)
 618 {
 619         int ncount = 0;
 620         uintptr_t pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 621         while (pos != NULL)
 622         {
 623                 uintptr_t curpos = pos;
 624                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 625                 if (di.diffcode.isDirectory())
 626                         ncount += markChildrenForRescan(pCtxt, curpos);
 627                 else
 628                 {
 629                         di.diffcode.diffcode |= DIFFCODE::NEEDSCAN;
 630                         ++ncount;
 631                 }
 632         }
 633         return ncount;
 634 }
 635
 636 int DirScan_UpdateMarkedItems(DiffFuncStruct *myStruct, uintptr_t parentdiffpos)
 637 {
 638         CDiffContext *pCtxt = myStruct->context;
 639         uintptr_t pos = pCtxt->GetFirstChildDiffPosition(parentdiffpos);
 640         int ncount = 0;
 641
 642         while (pos != NULL)
 643         {
 644                 if (pCtxt->ShouldAbort())
 645                         break;
 646                 uintptr_t curpos = pos;
 647                 DIFFITEM &di = pCtxt->GetNextSiblingDiffRefPosition(pos);
 648                 bool bItemsExist = true;
 649                 if (di.diffcode.isScanNeeded())
 650                 {
 651                         UpdateDiffItem(di, bItemsExist, pCtxt);
 652                         if (!bItemsExist)
 653                                 di.RemoveSelf();
 654                         else if (!di.diffcode.isDirectory())
 655                                 ++ncount;
 656                 }
 657                 if (bItemsExist && di.diffcode.isDirectory() && pCtxt->m_bRecursive)
 658                 {
 659                         if (di.diffcode.isScanNeeded() && !di.diffcode.isResultFiltered())
 660                         {
 661                                 di.RemoveChildren();
 662                                 di.diffcode.diffcode &= ~DIFFCODE::NEEDSCAN;
 663
 664                                 bool casesensitive = false;
 665                                 int depth = myStruct->context->m_bRecursive ? -1 : 0;
 666                                 String subdir[3];
 667                                 PathContext paths = myStruct->context->GetNormalizedPaths();
 668                                 for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 669                                         subdir[i] = di.diffFileInfo[i].GetFile();
 670                                 DirScan_GetItems(paths, subdir, myStruct,
 671                                         casesensitive, depth, &di, myStruct->context->m_bWalkUniques);
 672                                 ncount += markChildrenForRescan(myStruct->context, curpos);
 673                         }
 674                         else
 675                         {
 676                                 ncount += DirScan_UpdateMarkedItems(myStruct, curpos);
 677                         }
 678                 }
 679         }
 680         return ncount;
 681 }
 682 /**
 683  * @brief Update diffitem file/dir infos.
 684  *
 685  * Re-tests dirs/files if sides still exists, and updates infos for
 686  * existing sides. This assumes filenames, or paths are not changed.
 687  * Since in normal situations (I can think of) they cannot change
 688  * after first compare.
 689  *
 690  * @param [in,out] di DiffItem to update.
 691  * @param [out] bExists Set to
 692  *  - true if one of items exists so diffitem is valid
 693  *  - false if items were deleted, so diffitem is not valid
 694  * @param [in] pCtxt Compare context
 695  */
 696 static void UpdateDiffItem(DIFFITEM & di, bool & bExists, CDiffContext *pCtxt)
 697 {
 698         bExists = false;
 699         di.diffcode.setSideNone();
 700         for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 701         {
 702                 di.diffFileInfo[i].ClearPartial();
 703                 if (pCtxt->UpdateInfoFromDiskHalf(di, i))
 704                 {
 705                         di.diffcode.diffcode |= DIFFCODE::FIRST << i;
 706                         bExists = true;
 707                 }
 708         }
 709 }
 710
 711 /**
 712  * @brief Compare two diffitems and add results to difflist in context.
 713  *
 714  * This function does the actual compare for previously gathered list of
 715  * items. Basically we:
 716  * - ignore items matching filefilters
 717  * - add non-ignored directories (no compare for directory items)
 718  * - add  unique files
 719  * - compare files
 720  *
 721  * @param [in] di DiffItem to compare
 722  * @param [in,out] pCtxt Compare context: contains difflist, encoding info etc.
 723  * @todo For date compare, maybe we should use creation date if modification
 724  * date is missing?
 725  */
 726 void CompareDiffItem(DIFFITEM &di, CDiffContext * pCtxt)
 727 {
 728         int nDirs = pCtxt->GetCompareDirs();
 729         // Clear rescan-request flag (not set by all codepaths)
 730         di.diffcode.diffcode &= ~DIFFCODE::NEEDSCAN;
 731         // Is it a directory?
 732         if (di.diffcode.isDirectory())
 733         {
 734                 // We don't actually 'compare' directories, just add non-ignored
 735                 // directories to list.
 736                 StoreDiffData(di, pCtxt, NULL);
 737         }
 738         else
 739         {
 740                 // 1. Test against filters
 741                 if (!pCtxt->m_piFilterGlobal ||
 742                         (nDirs == 2 && pCtxt->m_piFilterGlobal->includeFile(di.diffFileInfo[0].filename, di.diffFileInfo[1].filename)) ||
 743                         (nDirs == 3 && pCtxt->m_piFilterGlobal->includeFile(di.diffFileInfo[0].filename, di.diffFileInfo[1].filename, di.diffFileInfo[2].filename))
 744                         )
 745                 {
 746                         di.diffcode.diffcode |= DIFFCODE::INCLUDED;
 747                         FolderCmp folderCmp;
 748                         di.diffcode.diffcode |= folderCmp.prepAndCompareFiles(pCtxt, di);
 749                         StoreDiffData(di, pCtxt, &folderCmp);
 750                 }
 751                 else
 752                 {
 753                         di.diffcode.diffcode |= DIFFCODE::SKIPPED;
 754                         StoreDiffData(di, pCtxt, NULL);
 755                 }
 756         }
 757 }
 758
 759 /**
 760  * @brief Send one file or directory result back through the diff context.
 761  * @param [in] di Data to store.
 762  * @param [in] pCtxt Compare context.
 763  * @param [in] pCmpData Folder compare data.
 764  */
 765 static void StoreDiffData(DIFFITEM &di, CDiffContext * pCtxt,
 766                 const FolderCmp * pCmpData)
 767 {
 768         if (pCmpData)
 769         {
 770                 di.nsdiffs = pCmpData->m_ndiffs;
 771                 di.nidiffs = pCmpData->m_ntrivialdiffs;
 772
 773                 for (int i = 0; i < pCtxt->GetCompareDirs(); ++i)
 774                 {
 775                         // Set text statistics
 776                         if (di.diffcode.exists(i))
 777                         {
 778                                 di.diffFileInfo[i].m_textStats = pCmpData->m_diffFileData.m_textStats[i];
 779                                 di.diffFileInfo[i].encoding = pCmpData->m_diffFileData.m_FileLocation[i].encoding;
 780                         }
 781                 }
 782         }
 783
 784         pCtxt->m_pCompareStats->AddItem(di.diffcode.diffcode);
 785         //pCtxt->AddDiff(di);
 786 }
 787
 788 /**
 789  * @brief Add one compare item to list.
 790  * @param [in] sLeftDir Left subdirectory.
 791  * @param [in] sRightDir Right subdirectory.
 792  * @param [in] lent Left item data to add.
 793  * @param [in] rent Right item data to add.
 794  * @param [in] pCtxt Compare context.
 795  * @param [in] parent Parent of item to be added
 796  */
 797 static DIFFITEM *AddToList(const String& sLeftDir, const String& sRightDir,
 798         const DirItem * lent, const DirItem * rent,
 799         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent)
 800 {
 801         return AddToList(sLeftDir, sRightDir, sLeftDir, lent, rent, 0, code, myStruct, parent, 2);
 802 }
 803
 804 /**
 805  * @brief Add one compare item to list.
 806  */
 807 static DIFFITEM *AddToList(const String& sDir1, const String& sDir2, const String& sDir3,
 808         const DirItem * ent1, const DirItem * ent2, const DirItem * ent3,
 809         unsigned code, DiffFuncStruct *myStruct, DIFFITEM *parent, int nItems)
 810 {
 811         // We must store both paths - we cannot get paths later
 812         // and we need unique item paths for example when items
 813         // change to identical
 814         DIFFITEM *di = myStruct->context->AddDiff(parent);
 815
 816         di->parent = parent;
 817         di->diffFileInfo[0].path = sDir1;
 818         di->diffFileInfo[1].path = sDir2;
 819         di->diffFileInfo[2].path = sDir3;
 820
 821         if (ent1)
 822         {
 823                 di->diffFileInfo[0].filename = ent1->filename;
 824                 di->diffFileInfo[0].mtime = ent1->mtime;
 825                 di->diffFileInfo[0].ctime = ent1->ctime;
 826                 di->diffFileInfo[0].size = ent1->size;
 827                 di->diffFileInfo[0].flags.attributes = ent1->flags.attributes;
 828         }
 829         else
 830         {
 831                 // Don't break CDirView::DoCopyRightToLeft()
 832                 if (ent3)
 833                         di->diffFileInfo[0].filename = ent3->filename;
 834                 else if (ent2)
 835                         di->diffFileInfo[0].filename = ent2->filename;
 836         }
 837
 838         if (ent2)
 839         {
 840                 di->diffFileInfo[1].filename = ent2->filename;
 841                 di->diffFileInfo[1].mtime = ent2->mtime;
 842                 di->diffFileInfo[1].ctime = ent2->ctime;
 843                 di->diffFileInfo[1].size = ent2->size;
 844                 di->diffFileInfo[1].flags.attributes = ent2->flags.attributes;
 845         }
 846         else
 847         {
 848                 // Don't break CDirView::DoCopyLeftToRight()
 849                 if (ent1)
 850                         di->diffFileInfo[1].filename = ent1->filename;
 851                 else if (ent3)
 852                         di->diffFileInfo[1].filename = ent3->filename;
 853         }
 854
 855         if (ent3)
 856         {
 857                 di->diffFileInfo[2].filename = ent3->filename;
 858                 di->diffFileInfo[2].mtime = ent3->mtime;
 859                 di->diffFileInfo[2].ctime = ent3->ctime;
 860                 di->diffFileInfo[2].size = ent3->size;
 861                 di->diffFileInfo[2].flags.attributes = ent3->flags.attributes;
 862         }
 863         else
 864         {
 865                 // Don't break CDirView::DoCopyLeftToRight()
 866                 if (ent1)
 867                         di->diffFileInfo[2].filename = ent1->filename;
 868                 else if (ent2)
 869                         di->diffFileInfo[2].filename = ent2->filename;
 870         }
 871
 872         if (nItems == 2)
 873                 di->diffcode.diffcode = code;
 874         else
 875                 di->diffcode.diffcode = code | DIFFCODE::THREEWAY;
 876
 877         myStruct->context->m_pCompareStats->IncreaseTotalItems();
 878         myStruct->pSemaphore->set();
 879         return di;
 880 }