OSDN Git Service

Some more tweaks to generate tables tool.
[mhash384/mhash384.git] / tools / GenTables / src / gen_table_xor.c
1 /* ----------------------------------------------------------------------------------------------- */
2 /* MHash-384 - Generate tables utility program                                                     */
3 /* Copyright(c) 2016-2017 LoRd_MuldeR <mulder2@gmx.de>                                             */
4 /*                                                                                                 */
5 /* Permission is hereby granted, free of charge, to any person obtaining a copy of this software   */
6 /* and associated documentation files(the "Software"), to deal in the Software without             */
7 /* restriction, including without limitation the rights to use, copy, modify, merge, publish,      */
8 /* distribute, sublicense, and / or sell copies of the Software, and to permit persons to whom the */
9 /* Software is furnished to do so, subject to the following conditions:                            */
10 /*                                                                                                 */
11 /* The above copyright notice and this permission notice shall be included in all copies or        */
12 /* substantial portions of the Software.                                                           */
13 /*                                                                                                 */
14 /* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING   */
15 /* BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND      */
16 /* NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,     */
17 /* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,  */
18 /* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.         */
19 /* ----------------------------------------------------------------------------------------------- */
20
21 #include "common.h"
22 #include "thread_utils.h"
23 #include "twister.h"
24 #include "boxmuller.h"
25
26 #include <stdio.h>
27 #include <stdint.h>
28 #include <string.h>
29 #include <time.h>
30 #include <stdbool.h>
31 #include <io.h>
32 #include <float.h>
33 #include <math.h>
34
35 //-----------------------------------------------------------------------------
36 // Const
37 //-----------------------------------------------------------------------------
38
39 #define HASH_LEN 384U
40
41 #define DISTANCE_MIN 182U
42 #define DISTANCE_MAX DISTANCE_MIN + 16U
43
44 #define THREAD_COUNT 8U
45 #undef ENABLE_STATS
46
47 #define ROW_NUM (UINT8_MAX+2)           /*total number of rows*/
48 #define ROW_LEN (HASH_LEN / CHAR_BIT)   /*number of bits per row*/
49
50 #define __DISTANCE_STR(X) #X
51 #define _DISTANCE_STR(X) __DISTANCE_STR(X)
52 #define DISTANCE_STR _DISTANCE_STR(DISTANCE_MIN)
53
54 #define MAGIC_NUMBER 0x3C6058A7C1132CB2ui64
55
56 #ifdef ENABLE_STATS
57 #include <intrin.h>
58 #endif
59
60 //-----------------------------------------------------------------------------
61 // Globals
62 //-----------------------------------------------------------------------------
63
64 static uint8_t g_table[ROW_NUM][ROW_LEN];
65 static uint8_t g_thread_buffer[THREAD_COUNT][ROW_LEN];
66
67 static size_t g_spinpos = 0;
68 static char SPINNER[4] = { '/', '-', '\\', '|' };
69
70 #ifdef ENABLE_STATS
71 static volatile long long g_stat_too_hi = 0i64;
72 static volatile long long g_stat_too_lo = 0i64;
73 #endif //ENABLE_STATS
74
75 //-----------------------------------------------------------------------------
76 // Utility Functions
77 //-----------------------------------------------------------------------------
78
79 static inline void print_row(uint8_t *const row_buffer)
80 {
81         for (size_t w = 0; w < ROW_LEN; ++w)
82         {
83                 printf("%02X", row_buffer[w]);
84         }
85         puts("");
86 }
87
88 static inline void flip_bit_at(uint8_t *const row_buffer, const size_t pos)
89 {
90         row_buffer[(pos / CHAR_BIT)] ^= ((uint8_t)(1U << (pos % CHAR_BIT)));
91 }
92
93 static inline void flip_all_bits(uint8_t *const row_buffer, const size_t *const pos_list, const size_t n)
94 {
95         for (size_t i = 0; i < n; ++i)
96         {
97                 flip_bit_at(row_buffer, pos_list[i]);
98         }
99 }
100
101 static inline void rand_indices_n(size_t *const indices_out, twister_t *const rand, const uint32_t n)
102 {
103         bool taken[HASH_LEN];
104         memset(&taken, 0, sizeof(bool) * HASH_LEN);
105         for (uint32_t i = 0; i < n; ++i)
106         {
107                 size_t next;
108                 do
109                 {
110                         next = next_rand_range(rand, HASH_LEN);
111                 }
112                 while (taken[next]);
113                 taken[indices_out[i] = next] = true;
114         }
115 }
116
117 static inline bool check_distance_rows(const size_t index_1, const size_t index_2)
118 {
119         const uint32_t dist = hamming_distance(&g_table[index_1][0], &g_table[index_2][0], ROW_LEN);
120         return (dist <= DISTANCE_MAX) && (dist >= DISTANCE_MIN);
121 }
122
123 static inline uint32_t check_distance_buff(const size_t index, const uint8_t *const row_buffer, const uint32_t limit)
124 {
125         uint32_t error = 0U;
126 #ifdef ENABLE_STATS
127         bool reason;
128 #endif //ENABLE_STATS
129         for (size_t k = 0; k < index; k++)
130         {
131
132                 const uint32_t dist = hamming_distance(&g_table[k][0], row_buffer, ROW_LEN);
133                 if (dist > DISTANCE_MAX)
134                 {
135                         const uint32_t current = dist - DISTANCE_MAX;
136                         if (current > error)
137                         {
138 #ifdef ENABLE_STATS
139                                 reason = false;
140 #endif //ENABLE_STATS
141                                 if ((error = current) >= limit)
142                                 {
143                                         break; /*early termination*/
144                                 }
145                         }
146                 }
147                 else if (dist < DISTANCE_MIN)
148                 {
149                         const uint32_t current = DISTANCE_MIN - dist;
150                         if (current > error)
151                         {
152 #ifdef ENABLE_STATS
153                                 reason = true;
154 #endif //ENABLE_STATS
155                                 if ((error = current) >= limit)
156                                 {
157                                         break; /*early termination*/
158                                 }
159                         }
160                 }
161         }
162 #ifdef ENABLE_STATS
163         _InterlockedIncrement64(reason ? &g_stat_too_lo : &g_stat_too_hi);
164 #endif //ENABLE_STATS
165         return error;
166 }
167
168 static dump_table(FILE *out)
169 {
170         fputs("uint8_t MHASH_384_TABLE_XOR[UINT8_MAX+2][MHASH_384_LEN] =\n{\n", out);
171         for (size_t i = 0; i < ROW_NUM; i++)
172         {
173                 fputs("\t{ ", out);
174                 for (size_t j = 0; j < ROW_LEN; j++)
175                 {
176                         if (j > 0)
177                         {
178                                 fputc(',', out);
179                         }
180                         fprintf(out, "0x%02X", g_table[i][j]);
181                 }
182                 fprintf(out, " }%s /*%02X*/\n", (i != (ROW_NUM - 1)) ? "," : " ", (uint32_t)(i % 0x100));
183         }
184         fputs("};\n", out);
185 }
186
187 //-----------------------------------------------------------------------------
188 // Thread function
189 //-----------------------------------------------------------------------------
190
191 typedef struct
192 {
193         size_t index;
194         uint8_t *row_buffer;
195         sem_t *stop;
196         pthread_mutex_t *mutex;
197         twister_t *rand;
198 }
199 thread_data_t;
200
201 static void* thread_main(void *const param)
202 {
203         const thread_data_t *const data = ((const thread_data_t*)param);
204         bxmller_t bxmller;
205         uint8_t temp[ROW_LEN];
206         gaussian_noise_init(&bxmller);
207         for(;;)
208         {
209                 rand_next_bytes(data->rand, data->row_buffer, ROW_LEN);
210                 uint32_t error = check_distance_buff(data->index, data->row_buffer, HASH_LEN);
211                 if(error > 0U)
212                 {
213                         for (size_t round = 0; round < 997; ++round)
214                         {
215                                 bool improved = false;
216                                 if (SEM_TRYWAIT(data->stop))
217                                 {
218                                         return NULL;
219                                 }
220                                 for (size_t rand_loop = 0; rand_loop < 99991U; ++rand_loop)
221                                 {
222                                         rand_next_bytes(data->rand, temp, ROW_LEN);
223                                         const uint32_t next_error = check_distance_buff(data->index, temp, error);
224                                         if (next_error < error)
225                                         {
226                                                 improved = true;
227                                                 memcpy(data->row_buffer, temp, sizeof(uint8_t) * ROW_LEN);
228                                                 if (!((error = next_error) > 0U))
229                                                 {
230                                                         goto success;
231                                                 }
232                                         }
233                                 }
234                                 if (!improved)
235                                 {
236                                         break; /*early termination*/
237                                 }
238                         }
239                         for (size_t round = 0; round < 997U; ++round)
240                         {
241                                 bool improved = false;
242                                 for (size_t flip_pos_x = 0U; flip_pos_x < HASH_LEN; ++flip_pos_x)
243                                 {
244                                         if (!(flip_pos_x % 31))
245                                         {
246                                                 if (SEM_TRYWAIT(data->stop))
247                                                 {
248                                                         return NULL;
249                                                 }
250                                         }
251                                         flip_bit_at(data->row_buffer, flip_pos_x);
252                                         bool revert_x = true;
253                                         const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
254                                         if (next_error < error)
255                                         {
256                                                 revert_x = false;
257                                                 improved = true;
258                                                 if (!((error = next_error) > 0U))
259                                                 {
260                                                         goto success;
261                                                 }
262                                         }
263                                         for (size_t flip_pos_y = flip_pos_x + 1U; flip_pos_y < HASH_LEN; ++flip_pos_y)
264                                         {
265                                                 flip_bit_at(data->row_buffer, flip_pos_y);
266                                                 bool revert_y = true;
267                                                 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
268                                                 if (next_error < error)
269                                                 {
270                                                         revert_x = revert_y = false;
271                                                         improved = true;
272                                                         if (!((error = next_error) > 0U))
273                                                         {
274                                                                 goto success;
275                                                         }
276                                                 }
277                                                 for (size_t flip_pos_z = flip_pos_y + 1U; flip_pos_z < HASH_LEN; ++flip_pos_z)
278                                                 {
279                                                         flip_bit_at(data->row_buffer, flip_pos_z);
280                                                         const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
281                                                         if (next_error < error)
282                                                         {
283                                                                 revert_x = revert_y = false;
284                                                                 improved = true;
285                                                                 if (!((error = next_error) > 0U))
286                                                                 {
287                                                                         goto success;
288                                                                 }
289                                                         }
290                                                         else
291                                                         {
292                                                                 flip_bit_at(data->row_buffer, flip_pos_z);
293                                                         }
294                                                 }
295                                                 if (revert_y)
296                                                 {
297                                                         flip_bit_at(data->row_buffer, flip_pos_y);
298                                                 }
299                                         }
300                                         if (revert_x)
301                                         {
302                                                 flip_bit_at(data->row_buffer, flip_pos_x);
303                                         }
304                                 }
305                                 for (size_t refine_loop = 0; refine_loop < 9973U; ++refine_loop)
306                                 {
307                                         size_t flip_indices[HASH_LEN];
308                                         if (!(refine_loop % 97))
309                                         {
310                                                 if (SEM_TRYWAIT(data->stop))
311                                                 {
312                                                         return NULL;
313                                                 }
314                                         }
315                                         const uint32_t flip_count = gaussian_noise_next(data->rand, &bxmller, 11.0, 4U, HASH_LEN);
316                                         for (size_t refine_step = 0; refine_step < 997U; ++refine_step)
317                                         {
318                                                 rand_indices_n(flip_indices, data->rand, flip_count);
319                                                 flip_all_bits(data->row_buffer, flip_indices, flip_count);
320                                                 const uint32_t next_error = check_distance_buff(data->index, data->row_buffer, error);
321                                                 if (next_error >= error)
322                                                 {
323                                                         flip_all_bits(data->row_buffer, flip_indices, flip_count); /*revert*/
324                                                         continue;
325                                                 }
326                                                 else
327                                                 {
328                                                         improved = true;
329                                                         if (!((error = next_error) > 0U))
330                                                         {
331                                                                 goto success;
332                                                         }
333                                                 }
334                                         }
335                                 }
336                                 if (!improved)
337                                 {
338                                         break; /*early termination*/
339                                 }
340                         }
341                 }
342                 else
343                 {
344                         break; /*success*/
345                 }
346         }
347
348 success:
349         MUTEX_LOCK(data->mutex);
350         if (SEM_TRYWAIT(data->stop))
351         {
352                 MUTEX_UNLOCK(data->mutex);
353                 return NULL;
354         }
355
356         SEM_POST(data->stop, THREAD_COUNT);
357         MUTEX_UNLOCK(data->mutex);
358         return data->row_buffer; /*success*/
359 }
360
361 static void* thread_spin(void *const param)
362 {
363         unsigned long delay = 1U;
364         sem_t *const stop = ((sem_t*)param);
365         for (;;)
366         {
367                 if (SEM_TRYWAIT(stop))
368                 {
369                         printf("\b\b\b[!]");
370                         return NULL;
371                 }
372                 _sleep(delay);
373                 if (delay >= 500)
374                 {
375 #ifdef ENABLE_STATS
376                         const long long s_too_lo = g_stat_too_lo, s_too_hi = g_stat_too_hi;
377                         const long long s_too_ac = s_too_lo + s_too_hi;
378                         printf("Too low: %lld (%.2f), too high: %lld (%.2f)\n", s_too_lo, s_too_lo / ((double)s_too_ac), s_too_hi, s_too_hi / ((double)s_too_ac));
379 #endif //ENABLE_STATS
380                         printf("\b\b\b[%c]", SPINNER[g_spinpos]);
381                         g_spinpos = (g_spinpos + 1) % 4;
382                 }
383                 else
384                 {
385                         delay *= 2U;
386                 }
387         }
388 }
389
390 //-----------------------------------------------------------------------------
391 // Save / Load
392 //-----------------------------------------------------------------------------
393
394 static bool save_table_data(const wchar_t *const filename, const size_t rows_completed_in)
395 {
396         FILE *const file = _wfopen(filename, L"wb");
397         if (file)
398         {
399                 bool success = true;
400                 const uint64_t magic_number = MAGIC_NUMBER;
401                 const uint32_t hash_len = HASH_LEN, distance_min = DISTANCE_MIN, distance_max = DISTANCE_MAX, rows_completed = (uint32_t)rows_completed_in;
402                 fwrite(&magic_number, sizeof(uint64_t), 1, file);
403                 fwrite(&hash_len, sizeof(uint32_t), 1, file);
404                 fwrite(&distance_min, sizeof(uint32_t), 1, file);
405                 fwrite(&distance_max, sizeof(uint32_t), 1, file);
406                 fwrite(&rows_completed, sizeof(uint32_t), 1, file);
407                 for (uint32_t i = 0; i < rows_completed; ++i)
408                 {
409                         const uint32_t checksum = adler32(&g_table[i][0], ROW_LEN);
410                         fwrite(&checksum, sizeof(uint32_t), 1, file);
411                         fwrite(&g_table[i][0], sizeof(uint8_t), ROW_LEN, file);
412                 }
413                 if (ferror(file))
414                 {
415                         printf("ERROR: Failed to write table data!\n");
416                         success = false;
417                 }
418                 fclose(file);
419                 return success;
420         }
421         else
422         {
423                 printf("ERROR: Failed to open table file for writing!\n");
424                 return false;
425         }
426 }
427
428 static bool load_table_data(const wchar_t *const filename, size_t *const rows_completed_out)
429 {
430         FILE *const file = _wfopen(filename, L"rb");
431         if (file)
432         {
433                 bool success = true;
434                 uint64_t magic_number;
435                 uint32_t hash_len, distance_min, distance_max, rows_completed;
436                 fread(&magic_number, sizeof(uint64_t), 1, file);
437                 fread(&hash_len, sizeof(uint32_t), 1, file);
438                 fread(&distance_min, sizeof(uint32_t), 1, file);
439                 fread(&distance_max, sizeof(uint32_t), 1, file);
440                 fread(&rows_completed, sizeof(uint32_t), 1, file);
441                 if (ferror(file) || feof(file))
442                 {
443                         printf("ERROR: Failed to read the table header!\n");
444                         success = false;
445                         goto failed;
446                 }
447                 if (magic_number != MAGIC_NUMBER)
448                 {
449                         printf("ERROR: Table file format could not be recognized!\n");
450                         success = false;
451                         goto failed;
452                 }
453                 if ((hash_len != HASH_LEN) || (distance_min != DISTANCE_MIN) || (distance_max != DISTANCE_MAX) || (rows_completed > ROW_NUM))
454                 {
455                         printf("ERROR: Table properties are incompatibe with this instance!\n");
456                         success = false;
457                         goto failed;
458                 }
459                 for (size_t i = 0; i < rows_completed; ++i)
460                 {
461                         uint32_t checksum_expected;
462                         if ((fread(&checksum_expected, sizeof(uint32_t), 1, file) != 1) || (fread(&g_table[i][0], sizeof(uint8_t), ROW_LEN, file) != ROW_LEN))
463                         {
464                                 printf("ERROR: Failed to read table data from file!\n");
465                                 success = false;
466                                 goto failed;
467                         }
468                         if (adler32(&g_table[i][0], ROW_LEN) != checksum_expected)
469                         {
470                                 printf("ERROR: Table checksum does *not* match table contents!\n");
471                                 success = false;
472                                 goto failed;
473                         }
474                         for (size_t j = 0; j < i; j++)
475                         {
476                                 if (!check_distance_rows(i, j))
477                                 {
478                                         printf("ERROR: Table distance verification has failed!\n");
479                                         success = false;
480                                         goto failed;
481                                 }
482                         }
483                 }
484         failed:
485                 fclose(file);
486                 if (success && rows_completed_out)
487                 {
488                         *rows_completed_out = (size_t)rows_completed;
489                 }
490                 return success;
491         }
492         else
493         {
494                 printf("ERROR: Failed to open table file for reading!\n");
495                 return false;
496         }
497 }
498
499 //-----------------------------------------------------------------------------
500 // MAIN
501 //-----------------------------------------------------------------------------
502
503 int wmain(int argc, wchar_t *argv[])
504 {
505         pthread_t thread_id[THREAD_COUNT + 1];
506         thread_data_t thread_data[THREAD_COUNT];
507         twister_t thread_rand[THREAD_COUNT];
508         sem_t stop_flag;
509         pthread_mutex_t stop_mutex;
510         FILE *file_out = NULL;
511         size_t initial_row_index = 0;
512
513         printf("MHash GenTableXOR [%s]\n\n", __DATE__);
514         printf("HashLen: %d, Distance Min/Max: %d/%d, Threads: %d\n\n", HASH_LEN, DISTANCE_MIN, DISTANCE_MAX, THREAD_COUNT);
515
516         if ((HASH_LEN % (8 * sizeof(uint32_t))) != 0)
517         {
518                 crit_exit("FATAL: Hash length must be a multiple of 32 bits!");
519         }
520
521         if (argc < 2)
522         {
523                 printf("Table file not specified!\n\n");
524                 printf("Usage:\n");
525                 printf("   GenTables_XOR.exe <table_file>\n\n");
526                 return 1;
527         }
528
529         for (size_t i = 0; i < ROW_NUM; i++)
530         {
531                 memset(&g_table[i][0], 0, sizeof(uint8_t) * ROW_LEN);
532         }
533
534         memset(&thread_id, 0, sizeof(pthread_t) * (THREAD_COUNT + 1));
535         memset(&thread_data, 0, sizeof(thread_data_t) * THREAD_COUNT);
536         for (size_t t = 0; t < THREAD_COUNT; t++)
537         {
538                 memset(&g_thread_buffer[t][0], 0, sizeof(uint8_t) * ROW_LEN);
539                 rand_init(&thread_rand[t], make_seed());
540         }
541
542         SEM_INIT(&stop_flag);
543         MUTEX_INIT(&stop_mutex);
544
545         if (_waccess(argv[1], 4) == 0)
546         {
547                 printf("Loading existing table data and proceeding...\n");
548                 if (!load_table_data(argv[1], &initial_row_index))
549                 {
550                         return 1;
551                 }
552         }
553
554         for (size_t i = initial_row_index; i < ROW_NUM; i++)
555         {
556                 char time_string[64];
557                 printf("Row %03u of %03u [%c]", (uint32_t)(i+1U), ROW_NUM, SPINNER[g_spinpos]);
558                 g_spinpos = (g_spinpos + 1) % 4;
559 #ifdef ENABLE_STATS
560                 g_stat_too_hi = g_stat_too_lo = 0i64;
561 #endif //INCREMENT_STAT
562                 PTHREAD_CREATE(&thread_id[THREAD_COUNT], NULL, thread_spin, &stop_flag);
563                 for (size_t t = 0; t < THREAD_COUNT; t++)
564                 {
565                         thread_data[t].index = i;
566                         thread_data[t].row_buffer = &g_thread_buffer[t][0];
567                         thread_data[t].stop = &stop_flag;
568                         thread_data[t].mutex = &stop_mutex;
569                         thread_data[t].rand = &thread_rand[t];
570                         PTHREAD_CREATE(&thread_id[t], NULL, thread_main, &thread_data[t]);
571                 }
572
573                 for (size_t t = 0; t < THREAD_COUNT; t++)
574                 {
575                         void *return_value = NULL;
576                         PTHREAD_JOIN(thread_id[t], &return_value);
577                         if (return_value)
578                         {
579                                 memcpy(&g_table[i][0], return_value, sizeof(uint8_t) * ROW_LEN);
580                         }
581                 }
582
583                 PTHREAD_JOIN(thread_id[THREAD_COUNT], NULL);
584                 get_time_str(time_string, 64);
585                 printf("\b\b\b[#] - %s\n", time_string);
586
587                 if (!save_table_data(argv[1], i + 1U))
588                 {
589                         return 1; /*failed to save current table data*/
590                 }
591         }
592
593         printf("\n-----\n\n");
594
595         dump_table(stdout);
596         if (fopen_s(&file_out, "table_XOR." DISTANCE_STR ".txt", "w") == 0)
597         {
598                 dump_table(file_out);
599                 fclose(file_out);
600         }
601
602         printf("\n-----\n\n");
603
604         for (size_t i = 0; i < ROW_NUM; i++)
605         {
606                 for (size_t j = 0; j < ROW_NUM; j++)
607                 {
608                         if (i == j)
609                         {
610                                 continue;
611                         }
612                         if (!check_distance_rows(i, j))
613                         {
614                                 crit_exit("FATAL: Table verification has failed!");
615                         }
616                 }
617         }
618
619         SEM_DESTROY(&stop_flag);
620         MUTEX_DESTROY(&stop_mutex);
621
622         printf("COMPLETED.\n\n");
623         system("shutdown /s /t 180");
624         return getchar();
625 }