ShardProcessor/Program.cs

   1 using System;
   2 using System.Diagnostics;
   3 using System.IO;
   4 using System.Text.RegularExpressions;
   5
   6 using Automap;
   7
   8 using Hjg.Pngcs;
   9 using Hjg.Pngcs.Chunks;
  10
  11 using ProtoBuf;
  12
  13 namespace ShardProcessor
  14 {
  15         class MainClass
  16         {
  17                 //private ILogger Logger { get; set; }
  18                 const string chunkFile_filter = @"*_*.png";
  19                 static Regex chunkShardRegex = new Regex(@"(?<X>[\d]+)_(?<Z>[\d]+)\.png", RegexOptions.Singleline);
  20                 static string mapPath;
  21                 internal const string _chunkPath = @"Chunks";
  22
  23                 /* TODO:
  24                         -Process existing PNGs: Report/Dump contents of Chunk Metadata, as per current version
  25                         -Grayscale Heightmap extraction from P.Buf heightmap from shards
  26                         -Extract contents of game's SQLLite map DB, INTO Automap type shards...
  27                         -Other stuff? chunk fixing / validation?
  28                 */
  29                 public static void Main(string[ ] args)
  30                 {
  31                 Console.WriteLine("AUTOMAP Offline Shard processor v0.1");
  32                 //Called once - thus it can only be in a static constructor.
  33                 PngChunk.FactoryRegister(PngMetadataChunk.ID, typeof(PngMetadataChunk));
  34
  35                 ArgsDecoder(args);
  36
  37                 }
  38
  39                 private static void ArgsDecoder(string[ ] args)
  40                 {
  41                 //#1 Path to maps '~/ApplicationData/vintagestory/Map/World_1234567890
  42                 mapPath = args[1];
  43
  44                 //#0 Command: Heightmaps (Generation from existing shard data)
  45                 string command = args[0];
  46
  47
  48                 switch (command) {
  49
  50                         case @"--heightmap":
  51                                 Process_ShardData( );
  52                         break;
  53
  54                         case @"--shards":
  55                                 Scan_ShardData( );
  56                         break;
  57
  58                         case @"--points":
  59                                 Scan_PointsData( );
  60                         break;
  61
  62                         case @"--oneshard":
  63                                 Scan_OneShard( );
  64                         break;
  65
  66                 default:
  67                         Console.WriteLine("Unrecognized Command: {0}", command);
  68                         break;
  69                 }
  70
  71
  72
  73                 }
  74
  75                 private static void Process_ShardData( )
  76                 {
  77                 var shardsDir = new DirectoryInfo( Path.Combine(mapPath , _chunkPath));
  78
  79                 var shardFiles = shardsDir.GetFiles(chunkFile_filter);
  80
  81                 if (shardFiles.Length > 0) {
  82                 #if DEBUG
  83                 //Logger.VerboseDebug("Metadata reloading from {0} shards", shardFiles.Length);
  84                 #endif
  85
  86                 foreach (var shardFile in shardFiles) {
  87
  88                 if (shardFile.Length < 1024) continue;
  89                 var result = chunkShardRegex.Match(shardFile.Name);
  90                 if (!result.Success) continue;
  91
  92                 int X_chunk_pos = int.Parse(result.Groups["X"].Value);
  93                 int Z_chunk_pos = int.Parse(result.Groups["Z"].Value);
  94
  95                 try {
  96                 using (var fileStream = shardFile.OpenRead( )) {
  97
  98                 PngReader pngRead = new PngReader(fileStream);
  99                 pngRead.ReadSkippingAllRows( );
 100                 pngRead.End( );
 101                 //Parse PNG chunks for METADATA in shard
 102                 PngMetadataChunk metadataFromPng = pngRead.GetChunksList( ).GetById1(PngMetadataChunk.ID) as PngMetadataChunk;
 103                 ColumnMeta columnData = metadataFromPng.ChunkMetadata;
 104                 //columnData.HeightMap //Should be sane Heightmap...
 105
 106
 107
 108                 }
 109
 110                 } catch (PngjException someEx) {
 111                 //Logger.Error("PNG Corruption file '{0}' - Reason: {1}", shardFile.Name, someEx);
 112                 continue;
 113                 } catch (ProtoException protoEx) {
 114                 //Logger.Error("ProtoBuf invalid! file:'{0}' - Reason: {1}", shardFile.Name, protoEx);
 115                 continue;
 116                 }
 117                 }
 118                 }
 119
 120
 121                 }
 122
 123
 124                 private static void Scan_PointsData( )
 125                 {
 126                 try {
 127                 var eoiFile = new FileInfo(Path.Combine(mapPath, @"eoi_binary"));
 128                 var poiFile = new FileInfo(Path.Combine(mapPath, @"poi_binary"));
 129                 uint entities = 0, points = 0;
 130
 131                 if (eoiFile.Exists) {
 132                 using (var eoiStream = eoiFile.OpenRead( )) {
 133
 134                 EntitiesOfInterest eoiData = Serializer.Deserialize<EntitiesOfInterest>(eoiStream);
 135
 136                 foreach (var entry in eoiData) {
 137                         Console.WriteLine("#{0}, [{1}], '{2}', {3}",
 138                         entry.EntityId,
 139                         entry.Location,
 140                           entry.Name,
 141                         entry.Timestamp.ToUniversalTime( )
 142                                                 );
 143                         entities++;
 144                 }
 145                 Console.WriteLine("Entities Of Interest: {0}", entities);
 146                 }
 147
 148
 149                 if (poiFile.Exists) {
 150                 using (var poiStream = poiFile.OpenRead( )) {
 151
 152                 PointsOfInterest poiData = Serializer.Deserialize<PointsOfInterest>(poiStream);
 153                 foreach (var entry in poiData) {
 154                         Console.WriteLine("[{0}], {1}, {2}, {3}",
 155                         entry.Location,
 156                         entry.Name,
 157                         entry.Destination,
 158                         entry.Timestamp.ToUniversalTime( )
 159                         );
 160                         points++;
 161                 }
 162                 }
 163                 Console.WriteLine("Points Of Interest: {0}", points);
 164                 }
 165
 166                 }
 167                 } catch (Exception uhOh) {
 168                 Console.WriteLine(uhOh);
 169                 }
 170
 171                 }
 172
 173                 private static void Scan_ShardData( )
 174                 {
 175                 var shardsDir = new DirectoryInfo(Path.Combine(mapPath, _chunkPath));
 176                 ulong count = 0,errors = 0, flat = 0;
 177                 var shardFiles = shardsDir.GetFiles(chunkFile_filter);
 178
 179                 if (shardFiles.Length > 0) {
 180                 #if DEBUG
 181                 //Logger.VerboseDebug("Metadata reloading from {0} shards", shardFiles.Length);
 182                 #endif
 183
 184                 foreach (var shardFile in shardFiles) {
 185
 186                 if (shardFile.Length < 1024) {
 187                 Console.WriteLine("File: '{0}' too small to be valid; skipping!", shardFile.FullName);
 188                 errors++;
 189                 continue;
 190                 }
 191
 192                 var result = chunkShardRegex.Match(shardFile.Name);
 193                 if (!result.Success) continue;
 194
 195                 int X_chunk_pos = int.Parse(result.Groups["X"].Value);
 196                 int Z_chunk_pos = int.Parse(result.Groups["Z"].Value);
 197
 198                 try {
 199                 using (var fileStream = shardFile.OpenRead( )) {
 200
 201                 PngReader pngRead = new PngReader(fileStream);
 202                 pngRead.ReadSkippingAllRows( );
 203                 pngRead.End( );
 204                 //Parse PNG chunks for METADATA in shard
 205                 PngMetadataChunk metadataFromPng = pngRead.GetChunksList( ).GetById1(PngMetadataChunk.ID) as PngMetadataChunk;
 206                 ColumnMeta columnData = metadataFromPng.ChunkMetadata;
 207
 208                 Console.Write("X{0,6:D} Y{1,6:D} Age:{2:N1} ", columnData.Location.X, columnData.Location.Y, columnData.ChunkAge.TotalDays);
 209                 Console.Write("YMax:{0:D3} ChkS:{1} Air:{2,7:D} NotAir:{3,7:D} ",
 210                 columnData.YMax,columnData.ChunkSize , columnData.AirBlocks, columnData.NonAirBlocks
 211          );
 212                 if (columnData.HeightMap != null) {
 213                 Console.Write("(Heights [{0}x{1}] ", columnData.HeightMap.GetLength(0), columnData.HeightMap.GetLength(1));
 214                 ushort lowest = ushort.MaxValue, highest = 0;
 215                 ulong sum = 0;
 216                 foreach (var hmEntry in columnData.HeightMap) {
 217                 lowest = Math.Min(lowest, hmEntry);
 218                 highest = Math.Max(highest, hmEntry);
 219                 sum += hmEntry;
 220                 }
 221                 Console.Write("Max:{0,3}, Min:{1,3}, ", highest, lowest);
 222                 if (sum > 0) Console.Write("Avg:{0:F1})", ( float )sum / (columnData.ChunkSize * columnData.ChunkSize));
 223                 Console.WriteLine( );
 224                 /*------ROCK RATIOs mini-table----------*/
 225                 if (columnData.RockRatio != null && columnData.RockRatio.Count > 0) {
 226                 Console.Write("Ratios({0,2:D})[",columnData.RockRatio.Count);
 227                 foreach (var rock in columnData.RockRatio) {
 228                 Console.Write("ID:{0,5:D} x{1,4:D}, ", rock.Key, rock.Value);
 229                 }
 230                 Console.Write(" ]\n");
 231                 }
 232
 233                 if ( sum == 0 || columnData.YMax == 0) flat++;
 234                 }
 235                 else {
 236                 flat++;
 237                 }
 238
 239
 240                 }
 241
 242                 } catch (PngjException someEx) {
 243                 Console.WriteLine("PNG Corruption file '{0}' - Reason: {1}", shardFile.Name, someEx);
 244                 errors++;
 245                 continue;
 246                 } catch (ProtoException protoEx) {
 247                 Console.WriteLine("ProtoBuf invalid! file:'{0}' - Reason: {1}", shardFile.Name, protoEx);
 248                 errors++;
 249                 continue;
 250                 }
 251                 count++;
 252                 }
 253                 }
 254
 255                 Console.WriteLine("Scanned {0} files, {1} errors, {2} FLAT entries", count, errors, flat);
 256                 }
 257
 258                 private static void Scan_OneShard( )
 259                 {
 260                 //--oneshard ~/ApplicationData/vintagestory/Maps/World_1316328588/Chunks/9363_9379.png
 261                 var oneChunkFile = new FileInfo(mapPath);
 262                 if (oneChunkFile.Exists) {
 263
 264
 265                 try {
 266                 using (var fileStream = oneChunkFile.OpenRead( )) {
 267
 268                 PngReader pngRead = new PngReader(fileStream);
 269                 pngRead.ReadSkippingAllRows( );
 270
 271                 //Parse PNG chunks for METADATA in shard
 272                 PngMetadataChunk metadataFromPng = pngRead.GetChunksList( ).GetById1(PngMetadataChunk.ID) as PngMetadataChunk;
 273                 ColumnMeta columnData = metadataFromPng.ChunkMetadata;
 274                 var metadata = pngRead.GetMetadata( );
 275                 var pngWriteTime = metadata.GetTime( );
 276                 var chunkX = metadata.GetTxtForKey(@"Chunk_X");
 277                 var chunkY = metadata.GetTxtForKey(@"Chunk_Y");
 278                 var pixelSize = metadata.GetTxtForKey(@"PxSz");
 279                 var gameDate = metadata.GetTxtForKey(@"GameDY");
 280                 var dateBlob = pngWriteTime.GetYMDHMS( );
 281                 /*
 282                 return new int[] {
 283                 this.year,
 284                 this.mon,
 285                 this.day,
 286                 this.hour,
 287                 this.min,
 288                 this.sec
 289                 */
 290
 291                 Console.WriteLine($"PNG-Timestamp:  Y{dateBlob[0] - 456960} M{dateBlob[1]} D{dateBlob[2]} H{dateBlob[3]} M {dateBlob[4]} S{dateBlob[5]} Chunk: X {chunkX} Y {chunkY} PixelSize:{pixelSize} Game-Date: {gameDate}");
 292
 293                 pngRead.End( );
 294                 }
 295                 } catch (Exception darn) {
 296                 Debug.Write("Oops! File causes: {0}", darn.ToString( ));
 297                 }
 298
 299                 }
 300                 }
 301         }
 302 }