1 /* ----------------------------------------------------------------------------------------------- */
2 /* MHash-384 - Generate tables utility program */
3 /* Copyright(c) 2016-2017 LoRd_MuldeR <mulder2@gmx.de> */
5 /* Permission is hereby granted, free of charge, to any person obtaining a copy of this software */
6 /* and associated documentation files(the "Software"), to deal in the Software without */
7 /* restriction, including without limitation the rights to use, copy, modify, merge, publish, */
8 /* distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the */
9 /* Software is furnished to do so, subject to the following conditions: */
11 /* The above copyright notice and this permission notice shall be included in all copies or */
12 /* substantial portions of the Software. */
14 /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING */
15 /* BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND */
16 /* NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, */
17 /* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, */
18 /* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
19 /* ----------------------------------------------------------------------------------------------- */
22 #include "thread_utils.h"
24 #include "boxmuller.h"
36 //-----------------------------------------------------------------------------
38 //-----------------------------------------------------------------------------
42 #define DISTANCE_MIN 182U
43 #define DISTANCE_MAX DISTANCE_MIN + 16U
45 #define THREAD_COUNT 8U
50 #define ROW_NUM (UINT8_MAX+2) /*total number of rows*/
51 #define ROW_LEN (HASH_LEN / CHAR_BIT) /*number of bits per row*/
53 #define __DISTANCE_STR(X) #X
54 #define _DISTANCE_STR(X) __DISTANCE_STR(X)
55 #define DISTANCE_STR _DISTANCE_STR(DISTANCE_MIN)
57 #define MAGIC_NUMBER 0x3C6058A7C1132CB2ui64
58 #define THREAD_ID (pthread_getw32threadid_np(pthread_self()))
60 //-----------------------------------------------------------------------------
62 //-----------------------------------------------------------------------------
64 static uint8_t g_table[ROW_NUM][ROW_LEN];
65 static uint8_t g_thread_buffer[THREAD_COUNT][ROW_LEN];
67 static size_t g_spinpos = 0;
68 static char SPINNER[4] = { '/', '-', '\\', '|' };
71 static volatile long long g_stat_too_hi = 0i64;
72 static volatile long long g_stat_too_lo = 0i64;
75 //-----------------------------------------------------------------------------
77 //-----------------------------------------------------------------------------
80 #define TRACE(X, ...) printf("[%04X] " X "\n", THREAD_ID, __VA_ARGS__)
82 #define TRACE(X, ...) __noop()
85 static inline void print_row(uint8_t *const row_buffer)
87 for (size_t w = 0; w < ROW_LEN; ++w)
89 printf("%02X", row_buffer[w]);
94 static inline void flip_bit_at(uint8_t *const row_buffer, const size_t pos)
96 row_buffer[pos >> 3] ^= ((uint8_t)(1U << (pos & 0x7)));
99 static inline void flip_rand_n(uint8_t *const row_buffer, twister_t *const rand, const uint32_t n)
101 bool taken[HASH_LEN];
102 memset(&taken, 0, sizeof(bool) * HASH_LEN);
103 for (uint32_t i = 0; i < n; ++i)
108 next = next_rand_range(rand, HASH_LEN);
111 flip_bit_at(row_buffer, next);
116 static inline bool check_distance_rows(const size_t index_1, const size_t index_2)
118 const uint32_t dist = hamming_distance(&g_table[index_1][0], &g_table[index_2][0], ROW_LEN);
119 return (dist <= DISTANCE_MAX) && (dist >= DISTANCE_MIN);
122 static inline uint32_t check_distance_buff(const size_t index, const uint8_t *const row_buffer, const uint32_t limit)
127 #endif //ENABLE_STATS
128 for (size_t k = 0; k < index; k++)
131 const uint32_t dist = hamming_distance(&g_table[k][0], row_buffer, ROW_LEN);
132 if (dist > DISTANCE_MAX)
134 const uint32_t current = dist - DISTANCE_MAX;
139 #endif //ENABLE_STATS
140 if ((error = current) >= limit)
142 break; /*early termination*/
146 else if (dist < DISTANCE_MIN)
148 const uint32_t current = DISTANCE_MIN - dist;
153 #endif //ENABLE_STATS
154 if ((error = current) >= limit)
156 break; /*early termination*/
162 _InterlockedIncrement64(reason ? &g_stat_too_lo : &g_stat_too_hi);
163 #endif //ENABLE_STATS
167 static inline bool update_global_minimum(volatile long *const global_minimum, const long value)
169 long old_minimum = LONG_MAX;
172 if (value > old_minimum)
176 const long new_minimum = _InterlockedCompareExchange(global_minimum, value, old_minimum);
177 if (new_minimum == old_minimum)
181 old_minimum = new_minimum;
185 static dump_table(FILE *out)
187 fputs("uint8_t MHASH_384_TABLE_XOR[UINT8_MAX+2][MHASH_384_LEN] =\n{\n", out);
188 for (size_t i = 0; i < ROW_NUM; i++)
191 for (size_t j = 0; j < ROW_LEN; j++)
197 fprintf(out, "0x%02X", g_table[i][j]);
199 fprintf(out, " }%s /*%02X*/\n", (i != (ROW_NUM - 1)) ? "," : " ", (uint32_t)(i % 0x100));
204 //-----------------------------------------------------------------------------
206 //-----------------------------------------------------------------------------
213 pthread_mutex_t *mutex;
214 volatile long *minimum;
218 static void* thread_main(void *const param)
220 const thread_data_t *const data = ((const thread_data_t*)param);
223 uint8_t temp[ROW_LEN];
226 rand_init(&rand, make_seed());
227 gaussian_noise_init(&bxmller);
228 rand_next_bytes(&rand, data->row_buffer, ROW_LEN);
229 uint32_t error = check_distance_buff(data->index, data->row_buffer, HASH_LEN);
234 for (int32_t round = 0; round < 4999; ++round)
236 if (!(round & 0x3FF))
238 if (SEM_TRYWAIT(data->stop))
243 for (size_t rand_loop = 0; rand_loop < 99991U; ++rand_loop)
245 rand_next_bytes(&rand, temp, ROW_LEN);
246 const uint32_t next_error = check_distance_buff(data->index, temp, error);
247 if (next_error < error)
250 memcpy(data->row_buffer, temp, sizeof(uint8_t) * ROW_LEN);
251 if (!((error = next_error) > 0U))
253 TRACE("Success by rand-init");
259 for (int32_t round = 0; round < 743; ++round)
261 TRACE("Optimizer round %u of 743", round);
264 for (size_t xchg_pos = 0U; xchg_pos < ROW_LEN; ++xchg_pos)
266 uint8_t original = data->row_buffer[xchg_pos];
267 for (uint16_t xchg_val = 0U; xchg_val <= UINT8_MAX; ++xchg_val)
269 data->row_buffer[xchg_pos] = (uint8_t)xchg_val;
270 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
271 if (next_error < error)
273 TRACE("Improved by xchg-byte");
274 original = (uint8_t)xchg_val;
276 if (!((error = next_error) > 0U))
278 TRACE("Success by xchg-byte");
283 data->row_buffer[xchg_pos] = original;
285 for (size_t flip_pos_w = 0U; flip_pos_w < HASH_LEN; ++flip_pos_w)
287 if (SEM_TRYWAIT(data->stop))
291 flip_bit_at(data->row_buffer, flip_pos_w);
292 bool revert_w = true;
293 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
294 if (next_error < error)
296 TRACE("Improved by flip-1");
299 if (!((error = next_error) > 0U))
301 TRACE("Sucess by flip-1");
305 for (size_t flip_pos_x = flip_pos_w + 1U; flip_pos_x < HASH_LEN; ++flip_pos_x)
307 flip_bit_at(data->row_buffer, flip_pos_x);
308 bool revert_x = true;
309 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
310 if (next_error < error)
312 TRACE("Improved by flip-2");
316 if (!((error = next_error) > 0U))
318 TRACE("Sucess by flip-2");
322 for (size_t flip_pos_y = flip_pos_x + 1U; flip_pos_y < HASH_LEN; ++flip_pos_y)
324 flip_bit_at(data->row_buffer, flip_pos_y);
325 bool revert_y = true;
326 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
327 if (next_error < error)
329 TRACE("Improved by flip-3");
334 if (!((error = next_error) > 0U))
336 TRACE("Sucess by flip-3");
340 for (size_t flip_pos_z = flip_pos_y + 1U; flip_pos_z < HASH_LEN; ++flip_pos_z)
342 flip_bit_at(data->row_buffer, flip_pos_z);
343 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
344 if (next_error < error)
346 TRACE("Improved by flip-4");
351 if (!((error = next_error) > 0U))
353 TRACE("Sucess by flip-4");
359 flip_bit_at(data->row_buffer, flip_pos_z);
364 flip_bit_at(data->row_buffer, flip_pos_y);
369 flip_bit_at(data->row_buffer, flip_pos_x);
374 flip_bit_at(data->row_buffer, flip_pos_w);
378 for (size_t rand_mode = 0; rand_mode < 2U; ++rand_mode)
380 memcpy(temp, &data->row_buffer, sizeof(uint8_t) * ROW_LEN);
381 for (size_t rand_loop = 0; rand_loop < 99991U; ++rand_loop)
383 rand_next_bytes(&rand, &temp[rand_mode ? (ROW_LEN / 2U) : 0U], ROW_LEN / 2U);
384 const uint32_t next_error = check_distance_buff(data->index, temp, error);
385 if (next_error < error)
387 TRACE("Improved by rand-replace #%zu", rand_mode);
389 memcpy(data->row_buffer, temp, sizeof(uint8_t) * ROW_LEN);
390 if (!((error = next_error) > 0U))
392 TRACE("Success by rand-replace");
398 for (size_t refine_loop = 0; refine_loop < 9973U; ++refine_loop)
400 if (!(refine_loop & 0x3FF))
402 if (SEM_TRYWAIT(data->stop))
407 const uint32_t flip_count = gaussian_noise_next(&rand, &bxmller, 3.14159, 5U, HASH_LEN);
408 for (size_t refine_step = 0; refine_step < 997U; ++refine_step)
410 memcpy(temp, data->row_buffer, sizeof(uint8_t) * ROW_LEN);
411 flip_rand_n(temp, &rand, flip_count);
412 const uint32_t next_error = check_distance_buff(data->index, temp, error);
413 if (next_error < error)
415 TRACE("Improved by rand-flip (%u)", flip_count);
417 memcpy(data->row_buffer, temp, sizeof(uint8_t) * ROW_LEN);
418 if (!((error = next_error) > 0U))
420 TRACE("Success by rand-flip (%u)", flip_count);
428 while (update_global_minimum(data->minimum, (long)error));
438 if (check_distance_buff(data->index, data->row_buffer, HASH_LEN))
440 fprintf(stderr, "ERROR MISCOMPARE!\n");
444 MUTEX_LOCK(data->mutex);
445 if (SEM_TRYWAIT(data->stop))
447 MUTEX_UNLOCK(data->mutex);
451 SEM_POST(data->stop, THREAD_COUNT);
452 MUTEX_UNLOCK(data->mutex);
453 return data->row_buffer; /*success*/
456 static void* thread_spin(void *const param)
458 unsigned long delay = 1U;
459 sem_t *const stop = ((sem_t*)param);
462 if (SEM_TRYWAIT(stop))
471 const long long s_too_lo = g_stat_too_lo, s_too_hi = g_stat_too_hi;
472 const long long s_too_ac = s_too_lo + s_too_hi;
473 printf("Too low: %lld (%.2f), too high: %lld (%.2f)\n", s_too_lo, s_too_lo / ((double)s_too_ac), s_too_hi, s_too_hi / ((double)s_too_ac));
474 #endif //ENABLE_STATS
475 printf("\b\b\b[%c]", SPINNER[g_spinpos]);
476 g_spinpos = (g_spinpos + 1) % 4;
485 //-----------------------------------------------------------------------------
487 //-----------------------------------------------------------------------------
489 static bool save_table_data(const wchar_t *const filename, const size_t rows_completed_in)
491 FILE *const file = _wfopen(filename, L"wb");
495 const uint64_t magic_number = MAGIC_NUMBER;
496 const uint32_t hash_len = HASH_LEN, distance_min = DISTANCE_MIN, distance_max = DISTANCE_MAX, rows_completed = (uint32_t)rows_completed_in;
497 fwrite(&magic_number, sizeof(uint64_t), 1, file);
498 fwrite(&hash_len, sizeof(uint32_t), 1, file);
499 fwrite(&distance_min, sizeof(uint32_t), 1, file);
500 fwrite(&distance_max, sizeof(uint32_t), 1, file);
501 fwrite(&rows_completed, sizeof(uint32_t), 1, file);
502 for (uint32_t i = 0; i < rows_completed; ++i)
504 const uint32_t checksum = adler32(&g_table[i][0], ROW_LEN);
505 fwrite(&checksum, sizeof(uint32_t), 1, file);
506 fwrite(&g_table[i][0], sizeof(uint8_t), ROW_LEN, file);
510 printf("ERROR: Failed to write table data!\n");
518 printf("ERROR: Failed to open table file for writing!\n");
523 static bool load_table_data(const wchar_t *const filename, size_t *const rows_completed_out)
525 FILE *const file = _wfopen(filename, L"rb");
529 uint64_t magic_number;
530 uint32_t hash_len, distance_min, distance_max, rows_completed;
531 fread(&magic_number, sizeof(uint64_t), 1, file);
532 fread(&hash_len, sizeof(uint32_t), 1, file);
533 fread(&distance_min, sizeof(uint32_t), 1, file);
534 fread(&distance_max, sizeof(uint32_t), 1, file);
535 fread(&rows_completed, sizeof(uint32_t), 1, file);
536 if (ferror(file) || feof(file))
538 printf("ERROR: Failed to read the table header!\n");
542 if (magic_number != MAGIC_NUMBER)
544 printf("ERROR: Table file format could not be recognized!\n");
548 if ((hash_len != HASH_LEN) || (distance_min != DISTANCE_MIN) || (distance_max != DISTANCE_MAX) || (rows_completed > ROW_NUM))
550 printf("ERROR: Table properties are incompatibe with this instance!\n");
554 for (size_t i = 0; i < rows_completed; ++i)
556 uint32_t checksum_expected;
557 if ((fread(&checksum_expected, sizeof(uint32_t), 1, file) != 1) || (fread(&g_table[i][0], sizeof(uint8_t), ROW_LEN, file) != ROW_LEN))
559 printf("ERROR: Failed to read table data from file!\n");
563 if (adler32(&g_table[i][0], ROW_LEN) != checksum_expected)
565 printf("ERROR: Table checksum does *not* match table contents!\n");
569 for (size_t j = 0; j < i; j++)
571 if (!check_distance_rows(i, j))
573 printf("ERROR: Table distance verification has failed!\n");
581 if (success && rows_completed_out)
583 *rows_completed_out = (size_t)rows_completed;
589 printf("ERROR: Failed to open table file for reading!\n");
594 //-----------------------------------------------------------------------------
596 //-----------------------------------------------------------------------------
598 int wmain(int argc, wchar_t *argv[])
600 pthread_t thread_id[THREAD_COUNT + 1];
601 thread_data_t thread_data[THREAD_COUNT];
603 pthread_mutex_t stop_mutex;
604 FILE *file_out = NULL;
605 size_t initial_row_index = 0;
606 volatile long minimum;
608 printf("MHash GenTableXOR [%s]\n\n", __DATE__);
609 printf("HashLen: %d, Distance Min/Max: %d/%d, Threads: %d\n\n", HASH_LEN, DISTANCE_MIN, DISTANCE_MAX, THREAD_COUNT);
611 if ((HASH_LEN % (8 * sizeof(uint32_t))) != 0)
613 crit_exit("FATAL: Hash length must be a multiple of 32 bits!");
618 printf("Table file not specified!\n\n");
620 printf(" GenTables_XOR.exe <table_file>\n\n");
624 for (size_t i = 0; i < ROW_NUM; i++)
626 memset(&g_table[i][0], 0, sizeof(uint8_t) * ROW_LEN);
629 memset(&thread_id, 0, sizeof(pthread_t) * (THREAD_COUNT + 1));
630 memset(&thread_data, 0, sizeof(thread_data_t) * THREAD_COUNT);
631 for (size_t t = 0; t < THREAD_COUNT; t++)
633 memset(&g_thread_buffer[t][0], 0, sizeof(uint8_t) * ROW_LEN);
636 SEM_INIT(&stop_flag);
637 MUTEX_INIT(&stop_mutex);
639 if (_waccess(argv[1], 4) == 0)
641 printf("Loading existing table data and proceeding...\n");
642 if (!load_table_data(argv[1], &initial_row_index))
648 for (size_t i = initial_row_index; i < ROW_NUM; i++)
650 char time_string[64];
651 printf("\aRow %03u of %03u [%c]", (uint32_t)(i+1U), ROW_NUM, SPINNER[g_spinpos]);
653 g_spinpos = (g_spinpos + 1) % 4;
655 g_stat_too_hi = g_stat_too_lo = 0i64;
656 #endif //INCREMENT_STAT
658 PTHREAD_CREATE(&thread_id[THREAD_COUNT], NULL, thread_spin, &stop_flag);
659 for (size_t t = 0; t < THREAD_COUNT; t++)
661 thread_data[t].index = i;
662 thread_data[t].row_buffer = &g_thread_buffer[t][0];
663 thread_data[t].stop = &stop_flag;
664 thread_data[t].mutex = &stop_mutex;
665 thread_data[t].minimum = &minimum;
666 PTHREAD_CREATE(&thread_id[t], NULL, thread_main, &thread_data[t]);
667 PTHREAD_SET_PRIORITY(thread_id[t], -15);
670 for (size_t t = 0; t < THREAD_COUNT; t++)
672 void *return_value = NULL;
673 PTHREAD_JOIN(thread_id[t], &return_value);
676 memcpy(&g_table[i][0], return_value, sizeof(uint8_t) * ROW_LEN);
680 PTHREAD_JOIN(thread_id[THREAD_COUNT], NULL);
681 get_time_str(time_string, 64);
682 printf("\b\b\b[#] - %s\n", time_string);
684 if (!save_table_data(argv[1], i + 1U))
686 return 1; /*failed to save current table data*/
690 printf("\n-----\n\n");
693 if (fopen_s(&file_out, "table_XOR." DISTANCE_STR ".txt", "w") == 0)
695 dump_table(file_out);
699 printf("\n-----\n\n");
701 for (size_t i = 0; i < ROW_NUM; i++)
703 for (size_t j = 0; j < ROW_NUM; j++)
709 if (!check_distance_rows(i, j))
711 crit_exit("FATAL: Table verification has failed!");
716 SEM_DESTROY(&stop_flag);
717 MUTEX_DESTROY(&stop_mutex);
719 printf("COMPLETED.\n\n");
720 system("shutdown /s /t 180");