1 /* ----------------------------------------------------------------------------------------------- */
2 /* MHash-384 - Generate tables utility program */
3 /* Copyright(c) 2016-2017 LoRd_MuldeR <mulder2@gmx.de> */
5 /* Permission is hereby granted, free of charge, to any person obtaining a copy of this software */
6 /* and associated documentation files(the "Software"), to deal in the Software without */
7 /* restriction, including without limitation the rights to use, copy, modify, merge, publish, */
8 /* distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the */
9 /* Software is furnished to do so, subject to the following conditions: */
11 /* The above copyright notice and this permission notice shall be included in all copies or */
12 /* substantial portions of the Software. */
14 /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING */
15 /* BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND */
16 /* NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, */
17 /* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, */
18 /* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
19 /* ----------------------------------------------------------------------------------------------- */
22 #include "thread_utils.h"
24 #include "boxmuller.h"
35 //-----------------------------------------------------------------------------
37 //-----------------------------------------------------------------------------
41 #define DISTANCE_MIN 182U
42 #define DISTANCE_MAX DISTANCE_MIN + 16U
44 #define THREAD_COUNT 8U
47 #define ROW_NUM (UINT8_MAX+2) /*total number of rows*/
48 #define ROW_LEN (HASH_LEN / CHAR_BIT) /*number of bits per row*/
50 #define __DISTANCE_STR(X) #X
51 #define _DISTANCE_STR(X) __DISTANCE_STR(X)
52 #define DISTANCE_STR _DISTANCE_STR(DISTANCE_MIN)
54 #define MAGIC_NUMBER 0x3C6058A7C1132CB2ui64
60 //-----------------------------------------------------------------------------
62 //-----------------------------------------------------------------------------
64 static uint8_t g_table[ROW_NUM][ROW_LEN];
65 static uint8_t g_thread_buffer[THREAD_COUNT][ROW_LEN];
67 static size_t g_spinpos = 0;
68 static char SPINNER[4] = { '/', '-', '\\', '|' };
71 static volatile long long g_stat_too_hi = 0i64;
72 static volatile long long g_stat_too_lo = 0i64;
75 //-----------------------------------------------------------------------------
77 //-----------------------------------------------------------------------------
79 static inline void print_row(uint8_t *const row_buffer)
81 for (size_t w = 0; w < ROW_LEN; ++w)
83 printf("%02X", row_buffer[w]);
88 static inline void flip_bit_at(uint8_t *const row_buffer, const size_t pos)
90 row_buffer[(pos / CHAR_BIT)] ^= ((uint8_t)(1U << (pos % CHAR_BIT)));
93 static inline void flip_all_bits(uint8_t *const row_buffer, const size_t *const pos_list, const size_t n)
95 for (size_t i = 0; i < n; ++i)
97 flip_bit_at(row_buffer, pos_list[i]);
101 static inline void rand_indices_n(size_t *const indices_out, twister_t *const rand, const uint32_t n)
103 bool taken[HASH_LEN];
104 memset(&taken, 0, sizeof(bool) * HASH_LEN);
105 for (uint32_t i = 0; i < n; ++i)
110 next = next_rand_range(rand, HASH_LEN);
113 taken[indices_out[i] = next] = true;
117 static inline bool check_distance_rows(const size_t index_1, const size_t index_2)
119 const uint32_t dist = hamming_distance(&g_table[index_1][0], &g_table[index_2][0], ROW_LEN);
120 return (dist <= DISTANCE_MAX) && (dist >= DISTANCE_MIN);
123 static inline uint32_t check_distance_buff(const size_t index, const uint8_t *const row_buffer, const uint32_t limit)
128 #endif //ENABLE_STATS
129 for (size_t k = 0; k < index; k++)
132 const uint32_t dist = hamming_distance(&g_table[k][0], row_buffer, ROW_LEN);
133 if (dist > DISTANCE_MAX)
135 const uint32_t current = dist - DISTANCE_MAX;
140 #endif //ENABLE_STATS
141 if ((error = current) >= limit)
143 break; /*early termination*/
147 else if (dist < DISTANCE_MIN)
149 const uint32_t current = DISTANCE_MIN - dist;
154 #endif //ENABLE_STATS
155 if ((error = current) >= limit)
157 break; /*early termination*/
163 _InterlockedIncrement64(reason ? &g_stat_too_lo : &g_stat_too_hi);
164 #endif //ENABLE_STATS
168 static dump_table(FILE *out)
170 fputs("uint8_t MHASH_384_TABLE_XOR[UINT8_MAX+2][MHASH_384_LEN] =\n{\n", out);
171 for (size_t i = 0; i < ROW_NUM; i++)
174 for (size_t j = 0; j < ROW_LEN; j++)
180 fprintf(out, "0x%02X", g_table[i][j]);
182 fprintf(out, " }%s /*%02X*/\n", (i != (ROW_NUM - 1)) ? "," : " ", (uint32_t)(i % 0x100));
187 //-----------------------------------------------------------------------------
189 //-----------------------------------------------------------------------------
196 pthread_mutex_t *mutex;
201 static void* thread_main(void *const param)
203 const thread_data_t *const data = ((const thread_data_t*)param);
205 uint8_t temp[ROW_LEN];
206 gaussian_noise_init(&bxmller);
209 rand_next_bytes(data->rand, data->row_buffer, ROW_LEN);
210 uint32_t error = check_distance_buff(data->index, data->row_buffer, HASH_LEN);
213 for (size_t round = 0; round < 997; ++round)
215 bool improved = false;
216 if (SEM_TRYWAIT(data->stop))
220 for (size_t rand_loop = 0; rand_loop < 99991U; ++rand_loop)
222 rand_next_bytes(data->rand, temp, ROW_LEN);
223 const uint32_t next_error = check_distance_buff(data->index, temp, error);
224 if (next_error < error)
227 memcpy(data->row_buffer, temp, sizeof(uint8_t) * ROW_LEN);
228 if (!((error = next_error) > 0U))
236 break; /*early termination*/
239 for (size_t round = 0; round < 997U; ++round)
241 bool improved = false;
242 for (size_t flip_pos_x = 0U; flip_pos_x < HASH_LEN; ++flip_pos_x)
244 if (!(flip_pos_x % 31))
246 if (SEM_TRYWAIT(data->stop))
251 flip_bit_at(data->row_buffer, flip_pos_x);
252 bool revert_x = true;
253 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
254 if (next_error < error)
258 if (!((error = next_error) > 0U))
263 for (size_t flip_pos_y = flip_pos_x + 1U; flip_pos_y < HASH_LEN; ++flip_pos_y)
265 flip_bit_at(data->row_buffer, flip_pos_y);
266 bool revert_y = true;
267 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
268 if (next_error < error)
270 revert_x = revert_y = false;
272 if (!((error = next_error) > 0U))
277 for (size_t flip_pos_z = flip_pos_y + 1U; flip_pos_z < HASH_LEN; ++flip_pos_z)
279 flip_bit_at(data->row_buffer, flip_pos_z);
280 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
281 if (next_error < error)
283 revert_x = revert_y = false;
285 if (!((error = next_error) > 0U))
292 flip_bit_at(data->row_buffer, flip_pos_z);
297 flip_bit_at(data->row_buffer, flip_pos_y);
302 flip_bit_at(data->row_buffer, flip_pos_x);
305 for (size_t refine_loop = 0; refine_loop < 9973U; ++refine_loop)
307 size_t flip_indices[HASH_LEN];
308 if (!(refine_loop % 97))
310 if (SEM_TRYWAIT(data->stop))
315 const uint32_t flip_count = gaussian_noise_next(data->rand, &bxmller, 11.0, 4U, HASH_LEN);
316 for (size_t refine_step = 0; refine_step < 997U; ++refine_step)
318 rand_indices_n(flip_indices, data->rand, flip_count);
319 flip_all_bits(data->row_buffer, flip_indices, flip_count);
320 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
321 if (next_error >= error)
323 flip_all_bits(data->row_buffer, flip_indices, flip_count); /*revert*/
329 if (!((error = next_error) > 0U))
338 break; /*early termination*/
349 MUTEX_LOCK(data->mutex);
350 if (SEM_TRYWAIT(data->stop))
352 MUTEX_UNLOCK(data->mutex);
356 SEM_POST(data->stop, THREAD_COUNT);
357 MUTEX_UNLOCK(data->mutex);
358 return data->row_buffer; /*success*/
361 static void* thread_spin(void *const param)
363 unsigned long delay = 1U;
364 sem_t *const stop = ((sem_t*)param);
367 if (SEM_TRYWAIT(stop))
376 const long long s_too_lo = g_stat_too_lo, s_too_hi = g_stat_too_hi;
377 const long long s_too_ac = s_too_lo + s_too_hi;
378 printf("Too low: %lld (%.2f), too high: %lld (%.2f)\n", s_too_lo, s_too_lo / ((double)s_too_ac), s_too_hi, s_too_hi / ((double)s_too_ac));
379 #endif //ENABLE_STATS
380 printf("\b\b\b[%c]", SPINNER[g_spinpos]);
381 g_spinpos = (g_spinpos + 1) % 4;
390 //-----------------------------------------------------------------------------
392 //-----------------------------------------------------------------------------
394 static bool save_table_data(const wchar_t *const filename, const size_t rows_completed_in)
396 FILE *const file = _wfopen(filename, L"wb");
400 const uint64_t magic_number = MAGIC_NUMBER;
401 const uint32_t hash_len = HASH_LEN, distance_min = DISTANCE_MIN, distance_max = DISTANCE_MAX, rows_completed = (uint32_t)rows_completed_in;
402 fwrite(&magic_number, sizeof(uint64_t), 1, file);
403 fwrite(&hash_len, sizeof(uint32_t), 1, file);
404 fwrite(&distance_min, sizeof(uint32_t), 1, file);
405 fwrite(&distance_max, sizeof(uint32_t), 1, file);
406 fwrite(&rows_completed, sizeof(uint32_t), 1, file);
407 for (uint32_t i = 0; i < rows_completed; ++i)
409 const uint32_t checksum = adler32(&g_table[i][0], ROW_LEN);
410 fwrite(&checksum, sizeof(uint32_t), 1, file);
411 fwrite(&g_table[i][0], sizeof(uint8_t), ROW_LEN, file);
415 printf("ERROR: Failed to write table data!\n");
423 printf("ERROR: Failed to open table file for writing!\n");
428 static bool load_table_data(const wchar_t *const filename, size_t *const rows_completed_out)
430 FILE *const file = _wfopen(filename, L"rb");
434 uint64_t magic_number;
435 uint32_t hash_len, distance_min, distance_max, rows_completed;
436 fread(&magic_number, sizeof(uint64_t), 1, file);
437 fread(&hash_len, sizeof(uint32_t), 1, file);
438 fread(&distance_min, sizeof(uint32_t), 1, file);
439 fread(&distance_max, sizeof(uint32_t), 1, file);
440 fread(&rows_completed, sizeof(uint32_t), 1, file);
441 if (ferror(file) || feof(file))
443 printf("ERROR: Failed to read the table header!\n");
447 if (magic_number != MAGIC_NUMBER)
449 printf("ERROR: Table file format could not be recognized!\n");
453 if ((hash_len != HASH_LEN) || (distance_min != DISTANCE_MIN) || (distance_max != DISTANCE_MAX) || (rows_completed > ROW_NUM))
455 printf("ERROR: Table properties are incompatibe with this instance!\n");
459 for (size_t i = 0; i < rows_completed; ++i)
461 uint32_t checksum_expected;
462 if ((fread(&checksum_expected, sizeof(uint32_t), 1, file) != 1) || (fread(&g_table[i][0], sizeof(uint8_t), ROW_LEN, file) != ROW_LEN))
464 printf("ERROR: Failed to read table data from file!\n");
468 if (adler32(&g_table[i][0], ROW_LEN) != checksum_expected)
470 printf("ERROR: Table checksum does *not* match table contents!\n");
474 for (size_t j = 0; j < i; j++)
476 if (!check_distance_rows(i, j))
478 printf("ERROR: Table distance verification has failed!\n");
486 if (success && rows_completed_out)
488 *rows_completed_out = (size_t)rows_completed;
494 printf("ERROR: Failed to open table file for reading!\n");
499 //-----------------------------------------------------------------------------
501 //-----------------------------------------------------------------------------
503 int wmain(int argc, wchar_t *argv[])
505 pthread_t thread_id[THREAD_COUNT + 1];
506 thread_data_t thread_data[THREAD_COUNT];
507 twister_t thread_rand[THREAD_COUNT];
509 pthread_mutex_t stop_mutex;
510 FILE *file_out = NULL;
511 size_t initial_row_index = 0;
513 printf("MHash GenTableXOR [%s]\n\n", __DATE__);
514 printf("HashLen: %d, Distance Min/Max: %d/%d, Threads: %d\n\n", HASH_LEN, DISTANCE_MIN, DISTANCE_MAX, THREAD_COUNT);
516 if ((HASH_LEN % (8 * sizeof(uint32_t))) != 0)
518 crit_exit("FATAL: Hash length must be a multiple of 32 bits!");
523 printf("Table file not specified!\n\n");
525 printf(" GenTables_XOR.exe <table_file>\n\n");
529 for (size_t i = 0; i < ROW_NUM; i++)
531 memset(&g_table[i][0], 0, sizeof(uint8_t) * ROW_LEN);
534 memset(&thread_id, 0, sizeof(pthread_t) * (THREAD_COUNT + 1));
535 memset(&thread_data, 0, sizeof(thread_data_t) * THREAD_COUNT);
536 for (size_t t = 0; t < THREAD_COUNT; t++)
538 memset(&g_thread_buffer[t][0], 0, sizeof(uint8_t) * ROW_LEN);
539 rand_init(&thread_rand[t], make_seed());
542 SEM_INIT(&stop_flag);
543 MUTEX_INIT(&stop_mutex);
545 if (_waccess(argv[1], 4) == 0)
547 printf("Loading existing table data and proceeding...\n");
548 if (!load_table_data(argv[1], &initial_row_index))
554 for (size_t i = initial_row_index; i < ROW_NUM; i++)
556 char time_string[64];
557 printf("Row %03u of %03u [%c]", (uint32_t)(i+1U), ROW_NUM, SPINNER[g_spinpos]);
558 g_spinpos = (g_spinpos + 1) % 4;
560 g_stat_too_hi = g_stat_too_lo = 0i64;
561 #endif //INCREMENT_STAT
562 PTHREAD_CREATE(&thread_id[THREAD_COUNT], NULL, thread_spin, &stop_flag);
563 for (size_t t = 0; t < THREAD_COUNT; t++)
565 thread_data[t].index = i;
566 thread_data[t].row_buffer = &g_thread_buffer[t][0];
567 thread_data[t].stop = &stop_flag;
568 thread_data[t].mutex = &stop_mutex;
569 thread_data[t].rand = &thread_rand[t];
570 PTHREAD_CREATE(&thread_id[t], NULL, thread_main, &thread_data[t]);
573 for (size_t t = 0; t < THREAD_COUNT; t++)
575 void *return_value = NULL;
576 PTHREAD_JOIN(thread_id[t], &return_value);
579 memcpy(&g_table[i][0], return_value, sizeof(uint8_t) * ROW_LEN);
583 PTHREAD_JOIN(thread_id[THREAD_COUNT], NULL);
584 get_time_str(time_string, 64);
585 printf("\b\b\b[#] - %s\n", time_string);
587 if (!save_table_data(argv[1], i + 1U))
589 return 1; /*failed to save current table data*/
593 printf("\n-----\n\n");
596 if (fopen_s(&file_out, "table_XOR." DISTANCE_STR ".txt", "w") == 0)
598 dump_table(file_out);
602 printf("\n-----\n\n");
604 for (size_t i = 0; i < ROW_NUM; i++)
606 for (size_t j = 0; j < ROW_NUM; j++)
612 if (!check_distance_rows(i, j))
614 crit_exit("FATAL: Table verification has failed!");
619 SEM_DESTROY(&stop_flag);
620 MUTEX_DESTROY(&stop_mutex);
622 printf("COMPLETED.\n\n");
623 system("shutdown /s /t 180");