1 /* ----------------------------------------------------------------------------------------------- */
2 /* MHash-384 - Generate tables utility program */
3 /* Copyright(c) 2016-2017 LoRd_MuldeR <mulder2@gmx.de> */
5 /* Permission is hereby granted, free of charge, to any person obtaining a copy of this software */
6 /* and associated documentation files(the "Software"), to deal in the Software without */
7 /* restriction, including without limitation the rights to use, copy, modify, merge, publish, */
8 /* distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the */
9 /* Software is furnished to do so, subject to the following conditions: */
11 /* The above copyright notice and this permission notice shall be included in all copies or */
12 /* substantial portions of the Software. */
14 /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING */
15 /* BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND */
16 /* NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, */
17 /* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, */
18 /* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
19 /* ----------------------------------------------------------------------------------------------- */
22 #include "thread_utils.h"
24 #include "boxmuller.h"
35 //-----------------------------------------------------------------------------
37 //-----------------------------------------------------------------------------
40 #define DISTANCE_MIN 192U
42 #define THREAD_COUNT 8U
46 #define ROW_NUM (UINT8_MAX+2) /*total number of rows*/
47 #define ROW_LEN (HASH_LEN / CHAR_BIT) /*number of bits per row*/
49 #define TARGET_PROCESSING_TIME 5400.0
51 #define __DISTANCE_STR(X) #X
52 #define _DISTANCE_STR(X) __DISTANCE_STR(X)
53 #define DISTANCE_STR _DISTANCE_STR(DISTANCE_MIN)
55 #define MAGIC_NUMBER 0x837EAF3A7F3A4DFDui64
56 #define THREAD_ID (pthread_getw32threadid_np(pthread_self()))
58 //-----------------------------------------------------------------------------
60 //-----------------------------------------------------------------------------
62 static const char SPINNER[4] = { '/', '-', '\\', '|' };
63 static const double SQRT2 = 1.41421356237309504880168872420969807856967187537694;
65 static size_t g_spinpos = 0;
67 //-----------------------------------------------------------------------------
69 //-----------------------------------------------------------------------------
72 #define TRACE(X, ...) printf("[%04X] " X "\n", THREAD_ID, __VA_ARGS__)
74 #define TRACE(X, ...) __noop()
77 static inline void print_row(uint8_t *const row_buffer)
79 for (size_t w = 0; w < ROW_LEN; ++w)
81 printf("%02X", row_buffer[w]);
86 static inline void flip_bit_at(uint8_t *const row_buffer, const size_t pos)
88 row_buffer[pos >> 3] ^= ((uint8_t)(1U << (pos & 0x7)));
91 static inline void flip_rand_n(uint8_t *const row_buffer, msws_t rand, const uint32_t n)
94 memset(&taken, 0, sizeof(bool) * HASH_LEN);
95 for (uint_fast32_t i = 0; i < n; ++i)
100 next = msws_uint32_max(rand, HASH_LEN);
103 flip_bit_at(row_buffer, next);
108 static inline uint_fast32_t get_distance_rows(const uint8_t table[ROW_NUM][ROW_LEN], const uint_fast16_t index_1, const uint_fast16_t index_2)
110 return hamming_distance(table[index_1], table[index_2], ROW_LEN);
113 #define ERROR_ENC(MAX,ACC) (((MAX) << 22U) | (ACC))
114 #define ERROR_DEC_ACC(VAL) ((VAL) & 0x3FFFFF)
115 #define ERROR_DEC_MAX(VAL) (((VAL) >> 22U) & 0x3FF)
116 static inline uint_fast32_t get_error_table(const uint8_t table[ROW_NUM][ROW_LEN], const uint_fast32_t limit)
118 uint_fast32_t error_max = 0U, error_acc = 0U;
119 const uint_fast32_t limit_acc = ERROR_DEC_ACC(limit), limit_max = ERROR_DEC_MAX(limit);
120 for (uint_fast16_t i = 0; i < ROW_NUM; i++)
122 for (uint_fast16_t j = i + 1U; j < ROW_NUM; j++)
124 const uint_fast32_t dist = get_distance_rows(table, i, j);
125 if (dist < DISTANCE_MIN)
127 const uint_fast32_t current = DISTANCE_MIN - dist;
128 if (current > error_max)
130 if ((error_max = current) > limit_max)
135 if (((error_acc += current) >= limit_acc) && (error_max == limit_max))
142 return ERROR_ENC(error_max, error_acc);
145 static inline void copy_table(uint8_t dst[ROW_NUM][ROW_LEN], const uint8_t src[ROW_NUM][ROW_LEN])
147 for (uint_fast16_t i = 0; i < ROW_NUM; i++)
149 memcpy(&dst[i][0], &src[i][0], sizeof(uint8_t) * ROW_LEN);
153 //static void dump_table(FILE *out)
155 // fputs("uint8_t MHASH_384_TABLE_XOR[UINT8_MAX+2][MHASH_384_LEN] =\n{\n", out);
156 // for (size_t i = 0; i < ROW_NUM; i++)
158 // fputs("\t{ ", out);
159 // for (size_t j = 0; j < ROW_LEN; j++)
165 // fprintf(out, "0x%02X", g_table[i][j]);
167 // fprintf(out, " }%s /*%02X*/\n", (i != (ROW_NUM - 1)) ? "," : " ", (uint32_t)(i % 0x100));
169 // fputs("};\n", out);
172 //-----------------------------------------------------------------------------
174 //-----------------------------------------------------------------------------
178 uint8_t table[ROW_NUM][ROW_LEN];
179 uint_fast32_t threshold;
180 uint_fast16_t row_offset;
182 pthread_mutex_t *mutex;
186 #define CHECK_SUCCESS(ERR_CURR,ERR_INIT,THRESHLD) \
187 (((ERR_CURR) < (ERR_INIT)) && (((ERR_INIT) - (ERR_CURR) >= (THRESHLD)) || (!(ERR_CURR))))
189 #define CHECK_TERMINATION() do \
191 if (SEM_TRYWAIT(data->stop)) \
198 static const int MAX_ROUNDS = (THREAD_COUNT > 1) ? (ROW_NUM / (THREAD_COUNT / 2U)) : ROW_NUM;
200 static void* thread_main(void *const param)
202 thread_data_t *const data = (thread_data_t*)param;
205 uint8_t backup[ROW_LEN];
206 const uint_fast32_t error_initial = get_error_table(data->table, UINT_FAST32_MAX);
207 uint_fast32_t error = error_initial;
208 uint_fast16_t row_index = data->row_offset;
209 int16_t row_iter = 0;
210 bool reduceFlag = false;
211 TRACE("Initial error: %08X [row offset: %03u, threshold: %u]", error_initial, data->row_offset, data->threshold);
214 msws_init(rand, make_seed());
215 gaussian_noise_init(&bxmller);
216 //--------------------//
217 //--- RAND REPLACE ---//
218 //--------------------//
219 for (row_index = data->row_offset, row_iter = 0; row_iter < MAX_ROUNDS; row_index = (row_index + 1U) % ROW_NUM, ++row_iter)
221 TRACE("Rand-replace round %d of %d", row_iter + 1, MAX_ROUNDS);
222 memcpy(backup, data->table[row_index], sizeof(uint8_t) * ROW_LEN);
223 for (uint_fast16_t rand_loop = 0; rand_loop < 9973U; ++rand_loop)
225 msws_bytes(rand, data->table[row_index], ROW_LEN);
226 const uint_fast32_t error_next = get_error_table(data->table, error);
227 if (error_next < error)
229 TRACE("Improved by rand-replace (%08X -> %08X) [row: %03u]", error, error_next, row_index);
230 memcpy(backup, data->table[row_index], sizeof(uint8_t) * ROW_LEN);
235 memcpy(data->table[row_index], backup, sizeof(uint8_t) * ROW_LEN);
236 if (CHECK_SUCCESS(error, error_initial, data->threshold))
238 TRACE("Success by rand-replace <<<---");
243 //-------------------//
244 //---- XCHG BYTE ----//
245 //-------------------//
246 for (row_index = data->row_offset, row_iter = 0; row_iter < MAX_ROUNDS; row_index = (row_index + 1U) % ROW_NUM, ++row_iter)
248 TRACE("Xchg-byte round %d of %d", row_iter + 1, MAX_ROUNDS);
249 for (uint_fast16_t xchg_pos = 0U; xchg_pos < ROW_LEN; ++xchg_pos)
251 uint8_t value_backup = data->table[row_index][xchg_pos];
252 for (uint_fast16_t value_next = 0U; value_next <= UINT8_MAX; ++value_next)
254 if ((uint8_t)value_next != value_backup)
256 data->table[row_index][xchg_pos] = (uint8_t)value_next;
257 const uint_fast32_t error_next = get_error_table(data->table, error);
258 if (error_next < error)
260 TRACE("Improved by xchg-byte (%08X -> %08X) [row: %03u]", error, error_next, row_index);
261 value_backup = data->table[row_index][xchg_pos];
267 data->table[row_index][xchg_pos] = value_backup;
269 if (CHECK_SUCCESS(error, error_initial, data->threshold))
271 TRACE("Success by xchg-byte <<<---");
276 //--------------------//
277 //------ FLIP-2 ------//
278 //--------------------//
279 for (row_index = data->row_offset, row_iter = 0; row_iter < MAX_ROUNDS; row_index = (row_index + 1U) % ROW_NUM, ++row_iter)
281 TRACE("Flip-2 round %d of %d", row_iter + 1, MAX_ROUNDS);
282 for (uint_fast16_t flip_pos_x = 0U; flip_pos_x < HASH_LEN; ++flip_pos_x)
284 flip_bit_at(data->table[row_index], flip_pos_x);
285 bool revert_x = true;
286 for (uint_fast16_t flip_pos_y = flip_pos_x + 1U; flip_pos_y < HASH_LEN; ++flip_pos_y)
288 flip_bit_at(data->table[row_index], flip_pos_y);
289 const uint_fast32_t error_next = get_error_table(data->table, error);
290 if (error_next < error)
292 TRACE("Improved by flip-2 (%08X -> %08X) [row: %03u]", error, error_next, row_index);
299 flip_bit_at(data->table[row_index], flip_pos_y);
304 flip_bit_at(data->table[row_index], flip_pos_x);
307 if (CHECK_SUCCESS(error, error_initial, data->threshold))
309 TRACE("Success by flip-2 <<<---");
314 //--------------------//
315 //------ FLIP-N ------//
316 //--------------------//
317 for (row_index = data->row_offset, row_iter = 0; row_iter < MAX_ROUNDS; row_index = (row_index + 1U) % ROW_NUM, ++row_iter)
319 TRACE("Flip-N round %d of %d", row_iter + 1, MAX_ROUNDS);
320 memcpy(backup, data->table[row_index], sizeof(uint8_t) * ROW_LEN);
321 for (uint_fast16_t refine_attempt = 0; refine_attempt < 11U; ++refine_attempt)
323 const uint32_t flip_count = gaussian_noise_next(rand, &bxmller, 1.4142, 3U, HASH_LEN);
324 for (uint_fast16_t refine_step = 0; refine_step < 997U; ++refine_step)
326 flip_rand_n(data->table[row_index], rand, flip_count);
327 const uint_fast32_t error_next = get_error_table(data->table, error);
328 if (error_next < error)
330 TRACE("Improved by flip-%u (%08X -> %08X)", flip_count, error, error_next);
331 memcpy(backup, data->table[row_index], sizeof(uint8_t) * ROW_LEN);
337 memcpy(data->table[row_index], backup, sizeof(uint8_t) * ROW_LEN);
341 if (CHECK_SUCCESS(error, error_initial, data->threshold))
343 TRACE("Success by flip-n <<<---");
348 //-------------------//
349 //----- RESTART -----//
350 //-------------------//
351 if ((reduceFlag) && (data->threshold > 1U))
353 data->threshold = data->threshold / 2U;
355 TRACE("Threshold reduced to: %u", data->threshold);
356 if (CHECK_SUCCESS(error, error_initial, data->threshold))
358 TRACE("Success by lower threshold :-P <<<---");
366 TRACE("Restarting!");
370 if (get_error_table(data->table, UINT_FAST32_MAX) >= error_initial)
372 fprintf(stderr, "ERROR MISCOMPARE!\n");
376 MUTEX_LOCK(data->mutex);
377 if (SEM_TRYWAIT(data->stop))
379 MUTEX_UNLOCK(data->mutex);
383 SEM_POST(data->stop, THREAD_COUNT);
384 MUTEX_UNLOCK(data->mutex);
385 return data->table; /*success*/
388 static void* thread_spin(void *const param)
390 unsigned long delay = 1U;
391 sem_t *const stop = ((sem_t*)param);
394 if (SEM_TRYWAIT(stop))
402 printf("\b\b\b[%c]", SPINNER[g_spinpos]);
403 g_spinpos = (g_spinpos + 1) % 4;
412 //-----------------------------------------------------------------------------
414 //-----------------------------------------------------------------------------
416 static bool save_table_data(const uint8_t table[ROW_NUM][ROW_LEN], const uint_fast32_t threshold_in, const wchar_t *const filename)
418 wchar_t filename_temp[_MAX_PATH];
419 swprintf_s(filename_temp, _MAX_PATH, L"%s~%X", filename, make_seed());
420 FILE *const file = _wfopen(filename_temp, L"wb");
424 const uint64_t magic_number = MAGIC_NUMBER;
425 const uint32_t hash_len = HASH_LEN, distance_min = DISTANCE_MIN, threshold = threshold_in;
426 fwrite(&magic_number, sizeof(uint64_t), 1, file);
427 fwrite(&hash_len, sizeof(uint32_t), 1, file);
428 fwrite(&distance_min, sizeof(uint32_t), 1, file);
429 fwrite(&threshold, sizeof(uint32_t), 1, file);
430 for (uint32_t i = 0; i < ROW_NUM; ++i)
432 const uint32_t checksum = adler32(&table[i][0], ROW_LEN);
433 fwrite(&checksum, sizeof(uint32_t), 1, file);
434 fwrite(&table[i][0], sizeof(uint8_t), ROW_LEN, file);
438 printf("ERROR: Failed to write table data!\n");
444 for (size_t i = 0; i < 42; ++i)
446 if (_wremove(filename))
450 printf("ERROR: Failed to delete existing file!\n");
457 if (_wrename(filename_temp, filename))
459 printf("ERROR: Failed to rename temp file!\n");
465 _wremove(filename_temp);
471 printf("ERROR: Failed to open table file for writing!\n");
476 static bool load_table_data(uint8_t table[ROW_NUM][ROW_LEN], uint_fast32_t *const threshold_out, const wchar_t *const filename)
478 FILE *const file = _wfopen(filename, L"rb");
481 uint64_t magic_number;
482 uint32_t hash_len, distance_min, threshold;
483 fread(&magic_number, sizeof(uint64_t), 1, file);
484 fread(&hash_len, sizeof(uint32_t), 1, file);
485 fread(&distance_min, sizeof(uint32_t), 1, file);
486 fread(&threshold, sizeof(uint32_t), 1, file);
487 if (ferror(file) || feof(file))
489 printf("ERROR: Failed to read the table header!\n");
492 if (magic_number != MAGIC_NUMBER)
494 printf("ERROR: Table file format could not be recognized!\n");
497 if ((hash_len != HASH_LEN) || (distance_min != DISTANCE_MIN))
499 printf("ERROR: Table properties are incompatibe with this instance!\n");
502 for (uint_fast16_t i = 0; i < ROW_NUM; ++i)
504 uint32_t checksum_expected;
505 if ((fread(&checksum_expected, sizeof(uint32_t), 1, file) != 1) || (fread(&table[i][0], sizeof(uint8_t), ROW_LEN, file) != ROW_LEN))
507 printf("ERROR: Failed to read table data from file!\n");
510 if (adler32(&table[i][0], ROW_LEN) != checksum_expected)
512 printf("ERROR: Table checksum does *not* match table contents!\n");
517 *threshold_out = threshold;
525 printf("ERROR: Failed to open table file for reading!\n");
530 //-----------------------------------------------------------------------------
532 //-----------------------------------------------------------------------------
534 static uint8_t g_table[ROW_NUM][ROW_LEN];
535 static thread_data_t g_thread_data[THREAD_COUNT];
536 static pthread_t g_thread_id[THREAD_COUNT + 1];
538 int wmain(int argc, wchar_t *argv[])
541 pthread_mutex_t stop_mutex;
542 uint_fast32_t error = UINT_FAST32_MAX, threshold = 256U;
543 FILE *file_out = NULL;
545 printf("MHash GenTableXOR V2 [%s]\n\n", __DATE__);
546 printf("HashLen: %d, Distance Min: %d, Threads: %d, MSVC: %u\n\n", HASH_LEN, DISTANCE_MIN, THREAD_COUNT, _MSC_FULL_VER);
548 if ((HASH_LEN % (8 * sizeof(uint32_t))) != 0)
550 crit_exit("FATAL: Hash length must be a multiple of 32 bits!");
555 printf("Table file not specified!\n\n");
557 printf(" GenTables_XOR.exe <table_file>\n\n");
561 for (size_t i = 0; i < ROW_NUM; i++)
563 memset(&g_table[i][0], 0, sizeof(uint8_t) * ROW_LEN);
566 memset(&g_thread_id, 0, sizeof(pthread_t) * (THREAD_COUNT + 1));
567 memset(&g_thread_data, 0, sizeof(thread_data_t) * THREAD_COUNT);
569 SEM_INIT(&stop_flag);
570 MUTEX_INIT(&stop_mutex);
572 if (_waccess(argv[1], 4) == 0)
574 printf("Loading existing table data and proceeding...\n");
575 if (!load_table_data(g_table, &threshold, argv[1]))
579 error = get_error_table(g_table, UINT_FAST32_MAX);
584 printf("Generating new initial random table...\n");
585 msws_init(rand, make_seed());
586 for (int_fast32_t round = 0; round < 999983; ++round)
588 for (uint_fast16_t i = 0; i < ROW_NUM; i++)
590 msws_bytes(rand, g_thread_data[0].table[i], ROW_LEN);
592 const uint_fast32_t error_next = get_error_table(g_thread_data[0].table, error);
593 if (error_next < error)
595 TRACE("Improved by rand-init (%08X -> %08X)", error, error_next);
596 copy_table(g_table, g_thread_data[0].table);
604 char time_string[64];
605 const clock_t ref_clock = clock();
606 printf("\aRemaining error: %u (total: %u) [%c]", ERROR_DEC_MAX(error), ERROR_DEC_ACC(error), SPINNER[g_spinpos]);
607 g_spinpos = (g_spinpos + 1) % 4;
609 PTHREAD_CREATE(&g_thread_id[THREAD_COUNT], NULL, thread_spin, &stop_flag);
610 for (uint_fast16_t t = 0; t < THREAD_COUNT; t++)
612 g_thread_data[t].stop = &stop_flag;
613 g_thread_data[t].mutex = &stop_mutex;
614 g_thread_data[t].row_offset = t * ((ROW_NUM) / THREAD_COUNT);
615 g_thread_data[t].threshold = threshold;
616 copy_table(g_thread_data[t].table, g_table);
617 PTHREAD_CREATE(&g_thread_id[t], NULL, thread_main, &g_thread_data[t]);
618 PTHREAD_SET_PRIORITY(g_thread_id[t], -15);
621 for (uint_fast16_t t = 0; t < THREAD_COUNT; t++)
623 void *return_value = NULL;
624 PTHREAD_JOIN(g_thread_id[t], &return_value);
627 const uint_fast32_t error_thread = get_error_table(g_thread_data[t].table, error);
628 if (error_thread < error)
630 copy_table(g_table, g_thread_data[t].table);
631 error = error_thread;
636 PTHREAD_JOIN(g_thread_id[THREAD_COUNT], NULL);
637 threshold = (uint_fast32_t) ceil(((double)threshold) * (TARGET_PROCESSING_TIME / (((double)(clock() - ref_clock)) / ((double)CLOCKS_PER_SEC))));
639 get_time_str(time_string, 64);
640 printf("\b\b\b[#] - %s\n", time_string);
642 if (!save_table_data(g_table, threshold, argv[1]))
644 getchar(); /*failed to save current table data*/
648 printf("\n-----\n\n");
650 //dump_table(stdout);
651 if (fopen_s(&file_out, "table_XOR." DISTANCE_STR ".txt", "w") == 0)
653 //dump_table(file_out);
657 printf("\n-----\n\n");
659 for (uint_fast16_t i = 0; i < ROW_NUM; i++)
661 for (uint_fast16_t j = 0; j < ROW_NUM; j++)
667 if (get_distance_rows(g_table, i, j) < DISTANCE_MIN)
669 crit_exit("FATAL: Table verification has failed!");
674 SEM_DESTROY(&stop_flag);
675 MUTEX_DESTROY(&stop_mutex);
677 printf("COMPLETED.\n\n");
678 system("shutdown /s /t 180");