ShardProcessor/ProcessingMethods.cs

   1 using System;
   2 using System.Diagnostics;
   3 using System.IO;
   4 using System.Linq;
   5 using System.Text.RegularExpressions;
   6
   7 using Automap;
   8
   9 using Hjg.Pngcs;
  10 using Hjg.Pngcs.Chunks;
  11
  12 using ProtoBuf;
  13 using Vintagestory.API.Common;
  14 using Vintagestory.API.MathTools;
  15 using Vintagestory.GameContent;
  16
  17 namespace ShardProcessor
  18 {
  19         public partial class MainClass
  20         {
  21                 internal const int chunkDefaultSize = 32;
  22
  23                 private static void Process_ShardData( )
  24                 {
  25                 var shardsDir = new DirectoryInfo(Path.Combine(mapPath, _chunkPath));
  26
  27                 var shardFiles = shardsDir.GetFiles(chunkFile_filter);
  28
  29                 if (shardFiles.Length > 0) {
  30                 #if DEBUG
  31                 //Logger.VerboseDebug("Metadata reloading from {0} shards", shardFiles.Length);
  32                 #endif
  33
  34                 foreach (var shardFile in shardFiles) {
  35
  36                 if (shardFile.Length < 1024) continue;
  37                 var result = chunkShardRegex.Match(shardFile.Name);
  38                 if (!result.Success) continue;
  39
  40                 int X_chunk_pos = int.Parse(result.Groups["X"].Value);
  41                 int Z_chunk_pos = int.Parse(result.Groups["Z"].Value);
  42
  43                 try {
  44                 using (var fileStream = shardFile.OpenRead( )) {
  45
  46                 PngReader pngRead = new PngReader(fileStream);
  47                 pngRead.ReadSkippingAllRows( );
  48                 pngRead.End( );
  49                 //Parse PNG chunks for METADATA in shard
  50                 PngMetadataChunk metadataFromPng = pngRead.GetChunksList( ).GetById1(PngMetadataChunk.ID) as PngMetadataChunk;
  51                 ColumnMeta columnData = metadataFromPng.ChunkMetadata;
  52                 //columnData.HeightMap //Should be sane Heightmap...
  53
  54
  55
  56                 }
  57
  58                 } catch (PngjException someEx) {
  59                 //Logger.Error("PNG Corruption file '{0}' - Reason: {1}", shardFile.Name, someEx);
  60                 continue;
  61                 } catch (ProtoException protoEx) {
  62                 //Logger.Error("ProtoBuf invalid! file:'{0}' - Reason: {1}", shardFile.Name, protoEx);
  63                 continue;
  64                 }
  65                 }
  66                 }
  67
  68
  69                 }
  70
  71
  72                 private static void Scan_PointsData( )
  73                 {
  74                 try {
  75                 var eoiFile = new FileInfo(Path.Combine(mapPath, @"eoi_binary"));
  76                 var poiFile = new FileInfo(Path.Combine(mapPath, @"poi_binary"));
  77                 uint entities = 0, points = 0;
  78
  79                 if (eoiFile.Exists) {
  80                 using (var eoiStream = eoiFile.OpenRead( )) {
  81
  82                 EntitiesOfInterest eoiData = Serializer.Deserialize<EntitiesOfInterest>(eoiStream);
  83
  84                 foreach (var entry in eoiData) {
  85                 Console.WriteLine("#{0}, [{1}], '{2}', {3}",
  86                 entry.EntityId,
  87                 entry.Location,
  88                   entry.Name,
  89                 entry.Timestamp.ToUniversalTime( )
  90                                         );
  91                 entities++;
  92                 }
  93                 Console.WriteLine("Entities Of Interest: {0}", entities);
  94                 }
  95
  96
  97                 if (poiFile.Exists) {
  98                 using (var poiStream = poiFile.OpenRead( )) {
  99
 100                 PointsOfInterest poiData = Serializer.Deserialize<PointsOfInterest>(poiStream);
 101                 foreach (var entry in poiData) {
 102                 Console.WriteLine("[{0}], {1}, {2}, {3}",
 103                 entry.Location,
 104                   entry.Name,
 105                 entry.Destination,
 106                   entry.Timestamp.ToUniversalTime( )
 107                 );
 108                 points++;
 109                 }
 110                 }
 111                 Console.WriteLine("Points Of Interest: {0}", points);
 112                 }
 113
 114                 }
 115                 } catch (Exception uhOh) {
 116                 Console.WriteLine(uhOh);
 117                 }
 118
 119                 }
 120
 121                 private static void Scan_ShardData( )
 122                 {
 123                 var shardsDir = new DirectoryInfo(Path.Combine(mapPath, _chunkPath));
 124                 ulong count = 0, errors = 0, flat = 0;
 125                 var shardFiles = shardsDir.GetFiles(chunkFile_filter);
 126
 127                 if (shardFiles.Length > 0) {
 128                 #if DEBUG
 129                 //Logger.VerboseDebug("Metadata reloading from {0} shards", shardFiles.Length);
 130                 #endif
 131
 132                 foreach (var shardFile in shardFiles) {
 133
 134                 if (shardFile.Length < 1024) {
 135                 Console.WriteLine("File: '{0}' too small to be valid; skipping!", shardFile.FullName);
 136                 errors++;
 137                 continue;
 138                 }
 139
 140                 var result = chunkShardRegex.Match(shardFile.Name);
 141                 if (!result.Success) continue;
 142
 143                 int X_chunk_pos = int.Parse(result.Groups["X"].Value);
 144                 int Z_chunk_pos = int.Parse(result.Groups["Z"].Value);
 145
 146                 try {
 147                 using (var fileStream = shardFile.OpenRead( )) {
 148
 149                 PngReader pngRead = new PngReader(fileStream);
 150                 pngRead.ReadSkippingAllRows( );
 151                 pngRead.End( );
 152                 //Parse PNG chunks for METADATA in shard
 153                 PngMetadataChunk metadataFromPng = pngRead.GetChunksList( ).GetById1(PngMetadataChunk.ID) as PngMetadataChunk;
 154                 ColumnMeta columnData = metadataFromPng.ChunkMetadata;
 155
 156                 Console.Write("X{0,6:D} Y{1,6:D} Age:{2:N1} ", columnData.Location.X, columnData.Location.Y, columnData.ChunkAge.TotalDays);
 157                 Console.Write("YMax:{0:D3} ChkS:{1} Air:{2,7:D} NotAir:{3,7:D} ",
 158                 columnData.YMax, columnData.ChunkSize, columnData.AirBlocks, columnData.NonAirBlocks
 159                  );
 160                 if (columnData.HeightMap != null) {
 161                 Console.Write("(Heights [{0}x{1}] ", columnData.HeightMap.GetLength(0), columnData.HeightMap.GetLength(1));
 162                 ushort lowest = ushort.MaxValue, highest = 0;
 163                 ulong sum = 0;
 164                 foreach (var hmEntry in columnData.HeightMap) {
 165                 lowest = Math.Min(lowest, hmEntry);
 166                 highest = Math.Max(highest, hmEntry);
 167                 sum += hmEntry;
 168                 }
 169                 Console.Write("Max:{0,3}, Min:{1,3}, ", highest, lowest);
 170                 if (sum > 0) Console.Write("Avg:{0:F1})", ( float )sum / (columnData.ChunkSize * columnData.ChunkSize));
 171                 Console.WriteLine( );
 172                 /*------ROCK RATIOs mini-table----------*/
 173                 if (columnData.RockRatio != null && columnData.RockRatio.Count > 0) {
 174                 Console.Write("Ratios({0,2:D})[", columnData.RockRatio.Count);
 175                 foreach (var rock in columnData.RockRatio) {
 176                 Console.Write("ID:{0,5:D} x{1,4:D}, ", rock.Key, rock.Value);
 177                 }
 178                 Console.Write(" ]\n");
 179                 }
 180
 181                 if (sum == 0 || columnData.YMax == 0) flat++;
 182                 }
 183                 else {
 184                 flat++;
 185                 }
 186
 187
 188                 }
 189
 190                 } catch (PngjException someEx) {
 191                 Console.WriteLine("PNG Corruption file '{0}' - Reason: {1}", shardFile.Name, someEx);
 192                 errors++;
 193                 continue;
 194                 } catch (ProtoException protoEx) {
 195                 Console.WriteLine("ProtoBuf invalid! file:'{0}' - Reason: {1}", shardFile.Name, protoEx);
 196                 errors++;
 197                 continue;
 198                 }
 199                 count++;
 200                 }
 201                 }
 202
 203                 Console.WriteLine("Scanned {0} files, {1} errors, {2} FLAT entries", count, errors, flat);
 204                 }
 205
 206                 private static void Scan_OneShard( )
 207                 {
 208                 //--oneshard ~/ApplicationData/vintagestory/Maps/World_1316328588/Chunks/9363_9379.png
 209                 var oneChunkFile = new FileInfo(mapPath);
 210                 if (oneChunkFile.Exists) {
 211
 212
 213                 try {
 214                 using (var fileStream = oneChunkFile.OpenRead( )) {
 215
 216                 PngReader pngRead = new PngReader(fileStream);
 217                 pngRead.ReadSkippingAllRows( );
 218
 219                 //Parse PNG chunks for METADATA in shard
 220                 PngMetadataChunk metadataFromPng = pngRead.GetChunksList( ).GetById1(PngMetadataChunk.ID) as PngMetadataChunk;
 221                 ColumnMeta columnData = metadataFromPng.ChunkMetadata;
 222                 var metadata = pngRead.GetMetadata( );
 223                 var pngWriteTime = metadata.GetTime( );
 224                 var chunkX = metadata.GetTxtForKey(@"Chunk_X");
 225                 var chunkY = metadata.GetTxtForKey(@"Chunk_Y");
 226                 var pixelSize = metadata.GetTxtForKey(@"PxSz");
 227                 var gameDate = metadata.GetTxtForKey(@"GameDY");
 228                 var dateBlob = pngWriteTime.GetYMDHMS( );
 229                 /*
 230                 return new int[] {
 231                 this.year,
 232                 this.mon,
 233                 this.day,
 234                 this.hour,
 235                 this.min,
 236                 this.sec
 237                 */
 238
 239                 Console.WriteLine($"PNG-Timestamp:  Y{dateBlob[0] - 456960} M{dateBlob[1]} D{dateBlob[2]} H{dateBlob[3]} M {dateBlob[4]} S{dateBlob[5]} Chunk: X {chunkX} Y {chunkY} PixelSize:{pixelSize} Game-Date: {gameDate}");
 240
 241                 pngRead.End( );
 242                 }
 243                 } catch (Exception darn) {
 244                 Debug.Write("Oops! File causes: {0}", darn.ToString( ));
 245                 }
 246
 247                 }
 248                 }
 249
 250                 private static void Emit_ProtoHeader( )
 251                 {
 252                 Console.WriteLine("Created Protobuf Header files.");
 253                 using (var entitiesProto = File.CreateText("Entities.proto")) {
 254                 entitiesProto.Write(Serializer.GetProto<EntitiesOfInterest>( ));
 255                 entitiesProto.Flush( );
 256                 }
 257
 258
 259                 using (var pointsProto = File.CreateText("Points.proto")) {
 260                 pointsProto.Write(Serializer.GetProto<PointsOfInterest>( ));
 261                 pointsProto.Flush( );
 262                 }
 263
 264                 using (var metadataProto = File.CreateText("ColumnMeta.proto")) {
 265                 metadataProto.Write(Serializer.GetProto<ColumnMeta>( ));
 266                 metadataProto.Flush( );
 267                 }
 268                 }
 269
 270                 private static void Dump_Minimap( )
 271                 {
 272                 //Extract MapDB -> Shard compatible PNG?
 273                 var logger = new LogAdaptor( );
 274                 WalkableMapDB minimapDatabase = new WalkableMapDB(logger);
 275                 string outmsg = string.Empty;
 276                 logger.Event("Started Logging @{0}", DateTimeOffset.UtcNow.ToString("u"));
 277                 Console.WriteLine("Starting to Dump Minimap data");
 278                 var tilesPath = Path.GetDirectoryName(mapPath);
 279                 Directory.CreateDirectory(Path.Combine(tilesPath, _minimapTilesPath));
 280
 281                 if (minimapDatabase.OpenOrCreate(mapPath, ref outmsg, false, false, false)) {
 282
 283                 foreach (var mapPiece in minimapDatabase.WalkMapTiles( )) {
 284                 logger.VerboseDebug("ScanDB Tile - X:{0} Y:{1}, Bitmap Int#{2}", mapPiece.ChunkPos.X, mapPiece.ChunkPos.Y, mapPiece.Pixels.Length);
 285                 MinimalShardWriter(mapPiece.ChunkPos, mapPiece.Pixels, tilesPath,logger);
 286                 }
 287
 288                 }
 289                 else {
 290                 logger.Error("Failed to access Minimap Database: '{0}'", outmsg);
 291                 }
 292
 293                 Console.WriteLine("DONE Dumping Minimap data!");
 294                 }
 295
 296                 private static void MinimalShardWriter(Vec2i coord, int[] pixelData, string tilesPath, ILogger logger )
 297                 {
 298                 ImageInfo imageInf = new ImageInfo(chunkDefaultSize, chunkDefaultSize, 8, false);
 299
 300                 string filename = $"{coord.X}_{coord.Y}.png";
 301                 filename = Path.Combine(tilesPath, _minimapTilesPath, filename);
 302
 303                 var PngWriter = FileHelper.CreatePngWriter(filename, imageInf, true);
 304                 PngMetadata meta = PngWriter.GetMetadata( );
 305                 meta.SetTimeYMDHMS(
 306                                 DateTime.UtcNow.Year,
 307                                 DateTime.UtcNow.Month,
 308                                 DateTime.UtcNow.Day,
 309                                 DateTime.UtcNow.Hour,
 310                                 DateTime.UtcNow.Minute,
 311                                 DateTime.UtcNow.Second);
 312                 meta.SetText("Chunk_X", coord.X.ToString("D"));
 313                 meta.SetText("Chunk_Y", coord.Y.ToString("D"));
 314                 meta.SetText("PxSz", "1");
 315                 /*
 316                 var transparencyChunk = meta.CreateTRNSChunk( );
 317                 transparencyChunk.SetRGB(0, 0, 0);//Same as Snapshots
 318                 */
 319                 var minimalMetadata = new ColumnMeta(coord);
 320
 321                 //Setup specialized meta-data PNG chunks here...
 322                 PngMetadataChunk pngChunkMeta = new PngMetadataChunk(PngWriter.ImgInfo) {
 323                         ChunkMetadata = minimalMetadata
 324                 };
 325                 PngWriter.GetChunksList( ).Queue(pngChunkMeta);
 326                 PngWriter.CompLevel = 5;// 9 is the maximum compression but thats too high for the small benefit it gives
 327                 PngWriter.CompressionStrategy = Hjg.Pngcs.Zlib.EDeflateCompressStrategy.Huffman;
 328
 329                 //pre-create PNG line slices...
 330                 ImageLine[ ] lines = Enumerable.Repeat(new object( ), chunkDefaultSize).Select(l => new ImageLine(PngWriter.ImgInfo)).ToArray( );
 331
 332                 Vec2i pixelPosn = new Vec2i();
 333                 for (int pixelIndex = 0; pixelIndex < (chunkDefaultSize * chunkDefaultSize); pixelIndex++) {
 334                 MapUtil.PosInt2d(pixelIndex, chunkDefaultSize, pixelPosn);
 335                 int red, green, blue;
 336                 red = ColorUtil.ColorB(pixelData[pixelIndex]);
 337                 green = ColorUtil.ColorG(pixelData[pixelIndex]);
 338                 blue = ColorUtil.ColorR(pixelData[pixelIndex]);
 339
 340                 ImageLineHelper.SetPixel(lines[pixelPosn.Y], pixelPosn.X, red, green, blue);
 341
 342
 343                 }
 344
 345                 for (int row = 0; row < PngWriter.ImgInfo.Rows; row++) {
 346                 PngWriter.WriteRow(lines[row], row);
 347                 }
 348                 PngWriter.End( );
 349
 350                 logger.Debug("Wrote mini map tile: {0}", coord);
 351                 }
 352         }
 353 }
 354