using System; using System.Diagnostics; using System.IO; using System.Linq; using System.Text.RegularExpressions; using Automap; using Hjg.Pngcs; using Hjg.Pngcs.Chunks; using ProtoBuf; using Vintagestory.API.Common; using Vintagestory.API.MathTools; using Vintagestory.GameContent; namespace ShardProcessor { public partial class MainClass { internal const int chunkDefaultSize = 32; private static void Process_ShardData( ) { var shardsDir = new DirectoryInfo(Path.Combine(mapPath, _chunkPath)); var shardFiles = shardsDir.GetFiles(chunkFile_filter); if (shardFiles.Length > 0) { #if DEBUG //Logger.VerboseDebug("Metadata reloading from {0} shards", shardFiles.Length); #endif foreach (var shardFile in shardFiles) { if (shardFile.Length < 1024) continue; var result = chunkShardRegex.Match(shardFile.Name); if (!result.Success) continue; int X_chunk_pos = int.Parse(result.Groups["X"].Value); int Z_chunk_pos = int.Parse(result.Groups["Z"].Value); try { using (var fileStream = shardFile.OpenRead( )) { PngReader pngRead = new PngReader(fileStream); pngRead.ReadSkippingAllRows( ); pngRead.End( ); //Parse PNG chunks for METADATA in shard PngMetadataChunk metadataFromPng = pngRead.GetChunksList( ).GetById1(PngMetadataChunk.ID) as PngMetadataChunk; ColumnMeta columnData = metadataFromPng.ChunkMetadata; //columnData.HeightMap //Should be sane Heightmap... } } catch (PngjException someEx) { //Logger.Error("PNG Corruption file '{0}' - Reason: {1}", shardFile.Name, someEx); continue; } catch (ProtoException protoEx) { //Logger.Error("ProtoBuf invalid! file:'{0}' - Reason: {1}", shardFile.Name, protoEx); continue; } } } } private static void Scan_PointsData( ) { try { var eoiFile = new FileInfo(Path.Combine(mapPath, @"eoi_binary")); var poiFile = new FileInfo(Path.Combine(mapPath, @"poi_binary")); uint entities = 0, points = 0; if (eoiFile.Exists) { using (var eoiStream = eoiFile.OpenRead( )) { EntitiesOfInterest eoiData = Serializer.Deserialize(eoiStream); foreach (var entry in eoiData) { Console.WriteLine("#{0}, [{1}], '{2}', {3}", entry.EntityId, entry.Location, entry.Name, entry.Timestamp.ToUniversalTime( ) ); entities++; } Console.WriteLine("Entities Of Interest: {0}", entities); } if (poiFile.Exists) { using (var poiStream = poiFile.OpenRead( )) { PointsOfInterest poiData = Serializer.Deserialize(poiStream); foreach (var entry in poiData) { Console.WriteLine("[{0}], {1}, {2}, {3}", entry.Location, entry.Name, entry.Destination, entry.Timestamp.ToUniversalTime( ) ); points++; } } Console.WriteLine("Points Of Interest: {0}", points); } } } catch (Exception uhOh) { Console.WriteLine(uhOh); } } private static void Scan_ShardData( ) { var shardsDir = new DirectoryInfo(Path.Combine(mapPath, _chunkPath)); ulong count = 0, errors = 0, flat = 0; var shardFiles = shardsDir.GetFiles(chunkFile_filter); if (shardFiles.Length > 0) { #if DEBUG //Logger.VerboseDebug("Metadata reloading from {0} shards", shardFiles.Length); #endif foreach (var shardFile in shardFiles) { if (shardFile.Length < 1024) { Console.WriteLine("File: '{0}' too small to be valid; skipping!", shardFile.FullName); errors++; continue; } var result = chunkShardRegex.Match(shardFile.Name); if (!result.Success) continue; int X_chunk_pos = int.Parse(result.Groups["X"].Value); int Z_chunk_pos = int.Parse(result.Groups["Z"].Value); try { using (var fileStream = shardFile.OpenRead( )) { PngReader pngRead = new PngReader(fileStream); pngRead.ReadSkippingAllRows( ); pngRead.End( ); //Parse PNG chunks for METADATA in shard PngMetadataChunk metadataFromPng = pngRead.GetChunksList( ).GetById1(PngMetadataChunk.ID) as PngMetadataChunk; ColumnMeta columnData = metadataFromPng.ChunkMetadata; Console.Write("X{0,6:D} Y{1,6:D} Age:{2:N1} ", columnData.Location.X, columnData.Location.Y, columnData.ChunkAge.TotalDays); Console.Write("YMax:{0:D3} ChkS:{1} Air:{2,7:D} NotAir:{3,7:D} ", columnData.YMax, columnData.ChunkSize, columnData.AirBlocks, columnData.NonAirBlocks ); if (columnData.HeightMap != null) { Console.Write("(Heights [{0}x{1}] ", columnData.HeightMap.GetLength(0), columnData.HeightMap.GetLength(1)); ushort lowest = ushort.MaxValue, highest = 0; ulong sum = 0; foreach (var hmEntry in columnData.HeightMap) { lowest = Math.Min(lowest, hmEntry); highest = Math.Max(highest, hmEntry); sum += hmEntry; } Console.Write("Max:{0,3}, Min:{1,3}, ", highest, lowest); if (sum > 0) Console.Write("Avg:{0:F1})", ( float )sum / (columnData.ChunkSize * columnData.ChunkSize)); Console.WriteLine( ); /*------ROCK RATIOs mini-table----------*/ if (columnData.RockRatio != null && columnData.RockRatio.Count > 0) { Console.Write("Ratios({0,2:D})[", columnData.RockRatio.Count); foreach (var rock in columnData.RockRatio) { Console.Write("ID:{0,5:D} x{1,4:D}, ", rock.Key, rock.Value); } Console.Write(" ]\n"); } if (sum == 0 || columnData.YMax == 0) flat++; } else { flat++; } } } catch (PngjException someEx) { Console.WriteLine("PNG Corruption file '{0}' - Reason: {1}", shardFile.Name, someEx); errors++; continue; } catch (ProtoException protoEx) { Console.WriteLine("ProtoBuf invalid! file:'{0}' - Reason: {1}", shardFile.Name, protoEx); errors++; continue; } count++; } } Console.WriteLine("Scanned {0} files, {1} errors, {2} FLAT entries", count, errors, flat); } private static void Scan_OneShard( ) { //--oneshard ~/ApplicationData/vintagestory/Maps/World_1316328588/Chunks/9363_9379.png var oneChunkFile = new FileInfo(mapPath); if (oneChunkFile.Exists) { try { using (var fileStream = oneChunkFile.OpenRead( )) { PngReader pngRead = new PngReader(fileStream); pngRead.ReadSkippingAllRows( ); //Parse PNG chunks for METADATA in shard PngMetadataChunk metadataFromPng = pngRead.GetChunksList( ).GetById1(PngMetadataChunk.ID) as PngMetadataChunk; ColumnMeta columnData = metadataFromPng.ChunkMetadata; var metadata = pngRead.GetMetadata( ); var pngWriteTime = metadata.GetTime( ); var chunkX = metadata.GetTxtForKey(@"Chunk_X"); var chunkY = metadata.GetTxtForKey(@"Chunk_Y"); var pixelSize = metadata.GetTxtForKey(@"PxSz"); var gameDate = metadata.GetTxtForKey(@"GameDY"); var dateBlob = pngWriteTime.GetYMDHMS( ); /* return new int[] { this.year, this.mon, this.day, this.hour, this.min, this.sec */ Console.WriteLine($"PNG-Timestamp: Y{dateBlob[0] - 456960} M{dateBlob[1]} D{dateBlob[2]} H{dateBlob[3]} M {dateBlob[4]} S{dateBlob[5]} Chunk: X {chunkX} Y {chunkY} PixelSize:{pixelSize} Game-Date: {gameDate}"); pngRead.End( ); } } catch (Exception darn) { Debug.Write("Oops! File causes: {0}", darn.ToString( )); } } } private static void Emit_ProtoHeader( ) { Console.WriteLine("Created Protobuf Header files."); using (var entitiesProto = File.CreateText("Entities.proto")) { entitiesProto.Write(Serializer.GetProto( )); entitiesProto.Flush( ); } using (var pointsProto = File.CreateText("Points.proto")) { pointsProto.Write(Serializer.GetProto( )); pointsProto.Flush( ); } using (var metadataProto = File.CreateText("ColumnMeta.proto")) { metadataProto.Write(Serializer.GetProto( )); metadataProto.Flush( ); } } private static void Dump_Minimap( ) { //Extract MapDB -> Shard compatible PNG? var logger = new LogAdaptor( ); WalkableMapDB minimapDatabase = new WalkableMapDB(logger); string outmsg = string.Empty; logger.Event("Started Logging @{0}", DateTimeOffset.UtcNow.ToString("u")); Console.WriteLine("Starting to Dump Minimap data"); var tilesPath = Path.GetDirectoryName(mapPath); Directory.CreateDirectory(Path.Combine(tilesPath, _minimapTilesPath)); if (minimapDatabase.OpenOrCreate(mapPath, ref outmsg, false, false, false)) { foreach (var mapPiece in minimapDatabase.WalkMapTiles( )) { logger.VerboseDebug("ScanDB Tile - X:{0} Y:{1}, Bitmap Int#{2}", mapPiece.ChunkPos.X, mapPiece.ChunkPos.Y, mapPiece.Pixels.Length); MinimalShardWriter(mapPiece.ChunkPos, mapPiece.Pixels, tilesPath,logger); } } else { logger.Error("Failed to access Minimap Database: '{0}'", outmsg); } Console.WriteLine("DONE Dumping Minimap data!"); } private static void MinimalShardWriter(Vec2i coord, int[] pixelData, string tilesPath, ILogger logger ) { ImageInfo imageInf = new ImageInfo(chunkDefaultSize, chunkDefaultSize, 8, false); string filename = $"{coord.X}_{coord.Y}.png"; filename = Path.Combine(tilesPath, _minimapTilesPath, filename); var PngWriter = FileHelper.CreatePngWriter(filename, imageInf, true); PngMetadata meta = PngWriter.GetMetadata( ); meta.SetTimeYMDHMS( DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day, DateTime.UtcNow.Hour, DateTime.UtcNow.Minute, DateTime.UtcNow.Second); meta.SetText("Chunk_X", coord.X.ToString("D")); meta.SetText("Chunk_Y", coord.Y.ToString("D")); meta.SetText("PxSz", "1"); /* var transparencyChunk = meta.CreateTRNSChunk( ); transparencyChunk.SetRGB(0, 0, 0);//Same as Snapshots */ var minimalMetadata = new ColumnMeta(coord); //Setup specialized meta-data PNG chunks here... PngMetadataChunk pngChunkMeta = new PngMetadataChunk(PngWriter.ImgInfo) { ChunkMetadata = minimalMetadata }; PngWriter.GetChunksList( ).Queue(pngChunkMeta); PngWriter.CompLevel = 5;// 9 is the maximum compression but thats too high for the small benefit it gives PngWriter.CompressionStrategy = Hjg.Pngcs.Zlib.EDeflateCompressStrategy.Huffman; //pre-create PNG line slices... ImageLine[ ] lines = Enumerable.Repeat(new object( ), chunkDefaultSize).Select(l => new ImageLine(PngWriter.ImgInfo)).ToArray( ); Vec2i pixelPosn = new Vec2i(); for (int pixelIndex = 0; pixelIndex < (chunkDefaultSize * chunkDefaultSize); pixelIndex++) { MapUtil.PosInt2d(pixelIndex, chunkDefaultSize, pixelPosn); int red, green, blue; red = ColorUtil.ColorB(pixelData[pixelIndex]); green = ColorUtil.ColorG(pixelData[pixelIndex]); blue = ColorUtil.ColorR(pixelData[pixelIndex]); ImageLineHelper.SetPixel(lines[pixelPosn.Y], pixelPosn.X, red, green, blue); } for (int row = 0; row < PngWriter.ImgInfo.Rows; row++) { PngWriter.WriteRow(lines[row], row); } PngWriter.End( ); logger.Debug("Wrote mini map tile: {0}", coord); } } }