From: The Grand Dog Date: Mon, 1 Jun 2020 23:29:46 +0000 (-0400) Subject: added WIP snapshot system X-Git-Tag: PR7x~5 X-Git-Url: http://git.osdn.net/view?p=automap%2Fautomap.git;a=commitdiff_plain;h=a3e67c0e2d5139cc74fbb7256d69eb109ce6d69e added WIP snapshot system --- diff --git a/Automap/Subsystems/AutomapSystem.cs b/Automap/Subsystems/AutomapSystem.cs index 767fa39..f61b35a 100644 --- a/Automap/Subsystems/AutomapSystem.cs +++ b/Automap/Subsystems/AutomapSystem.cs @@ -27,6 +27,8 @@ namespace Automap public class AutomapSystem { private Thread cartographer_thread; + private Thread snapshotThread; + private Snapshotter snapshot; private ICoreClientAPI ClientAPI { get; set; } private ILogger Logger { get; set; } private IChunkRenderer ChunkRenderer { get; set; } @@ -39,7 +41,7 @@ namespace Automap private const string poiFileName = @"poi_binary"; private const string eoiFileName = @"eoi_binary"; private const string pointsTsvFileName = @"points_of_interest.tsv"; - private static Regex chunkShardRegex = new Regex(@"(?[\d]+)_(?[\d]+).png", RegexOptions.Singleline); + private static Regex chunkShardRegex = new Regex(@"(?[\d]+)_(?[\d]+)\.png", RegexOptions.Singleline); private ConcurrentDictionary columnCounter = new ConcurrentDictionary(3, 150); private ColumnsMetadata chunkTopMetadata; @@ -72,13 +74,14 @@ namespace Automap ClientAPI.Event.LevelFinalize += EngageAutomap; configuration = config; - //TODO:Choose which one from GUI this.ChunkRenderer = new StandardRenderer(clientAPI, logger); //Listen on bus for commands ClientAPI.Event.RegisterEventBusListener(CommandListener, 1.0, AutomapSystem.AutomapCommandEventKey); + ClientAPI.RegisterCommand("snapshot", "", "", (id, args) => CurrentState = CommandType.Snapshot); + if (configuration.Autostart) { CurrentState = CommandType.Run; @@ -107,7 +110,6 @@ namespace Automap Prefill_POI_Designators(); startChunkColumn = new Vec2i((ClientAPI.World.Player.Entity.LocalPos.AsBlockPos.X / chunkSize), (ClientAPI.World.Player.Entity.LocalPos.AsBlockPos.Z / chunkSize)); chunkTopMetadata = new ColumnsMetadata(startChunkColumn); - Logger.Notification("AUTOMAP Start {0}", startChunkColumn); Reload_Metadata(); @@ -120,6 +122,14 @@ namespace Automap IsBackground = true }; + snapshot = new Snapshotter(path, _chunkPath, chunkTopMetadata, chunkSize); + snapshotThread = new Thread(Snap) + { + Name = "Snapshot", + Priority = ThreadPriority.Lowest, + IsBackground = true + }; + ClientAPI.Event.RegisterGameTickListener(AwakenCartographer, 6000); } @@ -135,9 +145,9 @@ namespace Automap if (CurrentState == CommandType.Run && (ClientAPI.IsGamePaused != false || ClientAPI.IsShuttingDown != true)) { - #if DEBUG +#if DEBUG Logger.VerboseDebug("Cartographer re-trigger from [{0}]", cartographer_thread.ThreadState); - #endif +#endif if (cartographer_thread.ThreadState.HasFlag(ThreadState.Unstarted)) { @@ -154,7 +164,13 @@ namespace Automap } else if (CurrentState == CommandType.Snapshot) { - //TODO: Snapshot generator second thread... + if (snapshotThread.ThreadState.HasFlag(ThreadState.Unstarted)) + { + snapshotThread.Start(); + } else if (snapshotThread.ThreadState.HasFlag(ThreadState.WaitSleepJoin)) + { + snapshotThread.Interrupt(); + } } } @@ -195,7 +211,6 @@ namespace Automap chunkMeta = chunkTopMetadata[mostActiveCol.Key]; #if DEBUG Logger.VerboseDebug("Loaded chunk {0}", mostActiveCol.Key); - //Console.WriteLine($"Load {mostActiveCol.Key}"); #endif } else @@ -203,10 +218,8 @@ namespace Automap chunkMeta = CreateColumnMetadata(mostActiveCol, mapChunk); #if DEBUG Logger.VerboseDebug("Created chunk {0}", mostActiveCol.Key); - //Console.WriteLine($"Created chunk {mostActiveCol.Key}"); #endif } - ProcessChunkBlocks(mostActiveCol.Key, mapChunk, ref chunkMeta); PngWriter pngWriter = SetupPngImage(mostActiveCol.Key, ref chunkMeta); @@ -224,11 +237,13 @@ namespace Automap else { columnCounter.TryRemove(mostActiveCol.Key, out ejectedItem); +#if DEBUG Logger.VerboseDebug("Un-painted chunk: ({0}) ", mostActiveCol.Key); +#endif } } - //Cleanup persisted Metadata... - chunkTopMetadata.ClearMetadata( ); + //Cleanup persisted Metadata... + chunkTopMetadata.ClearMetadata(); } UpdateStatus(this.updatedChunksTotal, this.nullChunkCount, updatedChunks); @@ -242,31 +257,60 @@ namespace Automap } //Then sleep until interupted again, and repeat - +#if DEBUG Logger.VerboseDebug("Thread '{0}' about to sleep indefinitely.", Thread.CurrentThread.Name); - +#endif Thread.Sleep(Timeout.Infinite); } catch (ThreadInterruptedException) { +#if DEBUG Logger.VerboseDebug("Thread '{0}' interupted [awoken]", Thread.CurrentThread.Name); +#endif goto wake; } catch (ThreadAbortException) { +#if DEBUG Logger.VerboseDebug("Thread '{0}' aborted.", Thread.CurrentThread.Name); - +#endif } finally { +#if DEBUG Logger.VerboseDebug("Thread '{0}' executing finally block.", Thread.CurrentThread.Name); +#endif PersistPointsData(); } } + private void Snap() + { + snapshotTake: +#if DEBUG + Logger.VerboseDebug("Snapshot started"); +#endif + try + { + snapshot.Take(); +#if DEBUG + Logger.VerboseDebug("Snapshot sleeping"); +#endif + CurrentState = CommandType.Run; + Thread.Sleep(Timeout.Infinite); + } + catch (ThreadInterruptedException) + { +#if DEBUG + Logger.VerboseDebug("Snapshot intertupted"); +#endif + goto snapshotTake; + } + } + private void UpdateStatus(uint totalUpdates, uint voidChunks, uint delta) { StatusData updateData = new StatusData(totalUpdates, voidChunks, delta, CommandType.Run); @@ -313,8 +357,6 @@ namespace Automap this.Entity_Designators.Add(match.Code, designator); } - - //EntityProperties props = ClientAPI.World.GetEntityType(designator.Pattern); } @@ -326,48 +368,62 @@ namespace Automap /// /// Store Points/Entity of Interest /// - private void PersistPointsData( ) + private void PersistPointsData() { - //POI and EOI raw dump files ~ WRITE em! - //var poiRawFile = File. - string poiPath = Path.Combine(path, poiFileName); - string eoiPath = Path.Combine(path, eoiFileName); - - if (this.POIs.Count > 0) { - using (var poiFile = File.OpenWrite(poiPath)) { - Serializer.Serialize(poiFile, this.POIs); - } - } + //POI and EOI raw dump files ~ WRITE em! + //var poiRawFile = File. + string poiPath = Path.Combine(path, poiFileName); + string eoiPath = Path.Combine(path, eoiFileName); - if (this.EOIs.Count > 0) { - using (var eoiFile = File.OpenWrite(eoiPath)) { - Serializer.Serialize(eoiFile, this.EOIs); - } - } + if (this.POIs.Count > 0) + { + using (var poiFile = File.OpenWrite(poiPath)) + { + Serializer.Serialize(poiFile, this.POIs); + } + } - //Create Easy to Parse TSV file for tool/human use.... - string pointsTsvPath = Path.Combine(path, pointsTsvFileName); + if (this.EOIs.Count > 0) + { + using (var eoiFile = File.OpenWrite(eoiPath)) + { + Serializer.Serialize(eoiFile, this.EOIs); + } + } - using (var tsvWriter = new StreamWriter(pointsTsvPath, false, Encoding.UTF8)) - { - tsvWriter.WriteLine("Name\tDescription\tLocation\tTime\t"); - foreach (var point in this.POIs) { + //Create Easy to Parse TSV file for tool/human use.... + string pointsTsvPath = Path.Combine(path, pointsTsvFileName); + + using (var tsvWriter = new StreamWriter(pointsTsvPath, false, Encoding.UTF8)) + { + tsvWriter.WriteLine("Name\tDescription\tLocation\tTime\t"); + foreach (var point in this.POIs) + { tsvWriter.Write(point.Name + "\t"); - tsvWriter.Write(point.Notes + "\t"); + var notes = point.Notes + .Replace("\n", "\\n") + .Replace("\t", "\\t") + .Replace("\\", "\\\\"); + tsvWriter.Write(notes + "\t"); tsvWriter.Write(point.Location.PrettyCoords(ClientAPI) + "\t"); - tsvWriter.Write(point.Timestamp.ToString("u")+"\t"); + tsvWriter.Write(point.Timestamp.ToString("u") + "\t"); tsvWriter.WriteLine(); - } - foreach (var entity in this.EOIs) { + } + foreach (var entity in this.EOIs) + { tsvWriter.Write(entity.Name + "\t"); - tsvWriter.Write(entity.Notes + "\t"); + var notes = entity.Notes + .Replace("\n", "\\n") + .Replace("\t", "\\t") + .Replace("\\", "\\\\"); + tsvWriter.Write(notes + "\t"); tsvWriter.Write(entity.Location.PrettyCoords(ClientAPI) + "\t"); tsvWriter.Write(entity.Timestamp.ToString("u") + "\t"); - tsvWriter.WriteLine( ); + tsvWriter.WriteLine(); + } + tsvWriter.WriteLine(); + tsvWriter.Flush(); } - tsvWriter.WriteLine(); - tsvWriter.Flush( ); - } } @@ -412,35 +468,33 @@ namespace Automap if (shardFile.Length < 1024) continue; var result = chunkShardRegex.Match(shardFile.Name); - if (result.Success) - { - int X_chunk_pos = int.Parse(result.Groups["X"].Value); - int Z_chunk_pos = int.Parse(result.Groups["Z"].Value); + if (!result.Success) continue; - try - { - using (var fileStream = shardFile.OpenRead()) - { - - PngReader pngRead = new PngReader(fileStream); - pngRead.ReadSkippingAllRows(); - pngRead.End(); - //Parse PNG chunks for METADATA in shard - PngMetadataChunk metadataFromPng = pngRead.GetChunksList().GetById1(PngMetadataChunk.ID) as PngMetadataChunk; - var column = metadataFromPng.ChunkMetadata; - if (column.PrettyLocation == null) - column = column.Reload(ClientAPI); - chunkTopMetadata.Add(column); - } + int X_chunk_pos = int.Parse(result.Groups["X"].Value); + int Z_chunk_pos = int.Parse(result.Groups["Z"].Value); - } - catch (PngjException someEx) + try + { + using (var fileStream = shardFile.OpenRead()) { - Logger.Error("PNG Corruption file '{0}' - Reason: {1}", shardFile.Name, someEx); - continue; + + PngReader pngRead = new PngReader(fileStream); + pngRead.ReadSkippingAllRows(); + pngRead.End(); + //Parse PNG chunks for METADATA in shard + PngMetadataChunk metadataFromPng = pngRead.GetChunksList().GetById1(PngMetadataChunk.ID) as PngMetadataChunk; + var column = metadataFromPng.ChunkMetadata; + if (column.PrettyLocation == null) + column = column.Reload(ClientAPI); + chunkTopMetadata.Add(column); } - } + } + catch (PngjException someEx) + { + Logger.Error("PNG Corruption file '{0}' - Reason: {1}", shardFile.Name, someEx); + continue; + } } } @@ -507,13 +561,14 @@ namespace Automap { WorldChunk chunkData = ClientAPI.World.BlockAccessor.GetChunk(key.X, targetChunkY, key.Y) as WorldChunk; - if (chunkData == null || chunkData.BlockEntities == null) { - #if DEBUG - Logger.VerboseDebug("Chunk null or empty X{0} Y{1} Z{2}", key.X, targetChunkY, key.Y); - #endif - nullChunkCount++; - continue; - } + if (chunkData == null || chunkData.BlockEntities == null) + { +#if DEBUG + Logger.VerboseDebug("Chunk null or empty X{0} Y{1} Z{2}", key.X, targetChunkY, key.Y); +#endif + nullChunkCount++; + continue; + } /*************** Chunk Entities Scanning *********************/ if (chunkData.BlockEntities != null && chunkData.BlockEntities.Length > 0) @@ -535,50 +590,50 @@ namespace Automap //Heightmap, Stats, block tally chunkData.Unpack(); - int X_index, Y_index, Z_index; - X_index = Y_index = Z_index = 0; + //int X_index, Y_index, Z_index; //Ensure ChunkData Metadata fields arn't null...due to being tossed out - if (chunkMeta.HeightMap == null) { chunkMeta.HeightMap = new ushort[chunkSize, chunkSize]; } - if (chunkMeta.RockRatio == null) { chunkMeta.RockRatio = new Dictionary(10); } - - do - { - do - { - do - { - /* Encode packed indicie - (y * chunksize + z) * chunksize + x - */ - var indicie = Helpers.ChunkBlockIndicie16(X_index, Y_index, Z_index); - int aBlockId = chunkData.Blocks[indicie]; - - if (aBlockId == 0) - {//Air - chunkMeta.AirBlocks++; - continue; - } - - if (RockIdCodes.ContainsKey(aBlockId)) - { - if (chunkMeta.RockRatio.ContainsKey(aBlockId)) { chunkMeta.RockRatio[aBlockId]++; } else { chunkMeta.RockRatio.Add(aBlockId, 1); } - } - - chunkMeta.NonAirBlocks++; - - //Heightmap - if (chunkMeta.HeightMap[X_index, Z_index] == 0) - { chunkMeta.HeightMap[X_index, Z_index] = (ushort) (Y_index + (targetChunkY * chunkSize)); } - - } - while (X_index++ < (chunkSize - 1)); - X_index = 0; - } - while (Z_index++ < (chunkSize - 1)); - Z_index = 0; - } - while (Y_index++ < (chunkSize - 1)); + //if (chunkMeta.HeightMap == null) { chunkMeta.HeightMap = new ushort[chunkSize, chunkSize]; } + //if (chunkMeta.RockRatio == null) { chunkMeta.RockRatio = new Dictionary(10); } + + //for (Y_index = 0; Y_index < chunkSize - 1; Y_index++) + //{ + // for (Z_index = 0; Z_index < chunkSize - 1; Z_index++) + // { + // for (X_index = 0; X_index < chunkSize - 1; X_index++) + // { + // /* Encode packed indicie + // (y * chunksize + z) * chunksize + x + // */ + // var indicie = Helpers.ChunkBlockIndicie16(X_index, Y_index, Z_index); + // int aBlockId = chunkData.Blocks[indicie]; + + // if (aBlockId == 0) + // {//Air + // chunkMeta.AirBlocks++; + // continue; + // } + + // if (RockIdCodes.ContainsKey(aBlockId)) + // { + // if (chunkMeta.RockRatio.ContainsKey(aBlockId)) + // chunkMeta.RockRatio[aBlockId]++; + // else + // chunkMeta.RockRatio.Add(aBlockId, 1); + // } + + // chunkMeta.NonAirBlocks++; + + // //Heightmap + // //if (chunkMeta.HeightMap[X_index, Z_index] == 0) + // //{ + + // // chunkMeta.HeightMap[X_index, Z_index] = (ushort) (Y_index + (targetChunkY * chunkSize)); + // //} + // } + // } + + //} } } @@ -622,48 +677,31 @@ namespace Automap private void CommandListener(string eventName, ref EnumHandling handling, IAttribute data) { - Logger.VerboseDebug("MsgBus RX: AutomapCommandMsg: {0}", data.ToJsonToken()); + //Logger.VerboseDebug("MsgBus RX: AutomapCommandMsg: {0}", data.ToJsonToken()); CommandData cmdData = data as CommandData; - - if (CurrentState != CommandType.Snapshot) + switch (cmdData.State) { - switch (cmdData.State) - { - case CommandType.Run: - case CommandType.Stop: - if (CurrentState != cmdData.State) - { - CurrentState = cmdData.State; - AwakenCartographer(0.0f); - } - break; - - case CommandType.Snapshot: - CurrentState = CommandType.Stop; - //Snapshot starts a second thread/process... - - break; + case CommandType.Run: + case CommandType.Stop: + case CommandType.Snapshot: + if (CurrentState != cmdData.State) + { + CurrentState = cmdData.State; + AwakenCartographer(0.0f); + } + break; - case CommandType.Notation: - //Add to POI list where player location - AddNote(cmdData.Notation); - break; - } + case CommandType.Notation: + //Add to POI list where player location + AddNote(cmdData.Notation); + break; } - - - #if DEBUG ClientAPI.TriggerChatMessage($"Automap commanded to: {cmdData.State} "); #endif - } - - - - #endregion } diff --git a/Automap/Subsystems/Snapshot.cs b/Automap/Subsystems/Snapshot.cs new file mode 100644 index 0000000..c427813 --- /dev/null +++ b/Automap/Subsystems/Snapshot.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Hjg.Pngcs; + +namespace Automap +{ + public class Snapshotter + { + public string path; + public string chunkPath; + public ColumnsMetadata cols; + public int chunkSize; + public int NorthEdge => cols.North_mostChunk; + public int WestEdge => cols.West_mostChunk; + public int Width => (cols.East_mostChunk - WestEdge + 1); + public int Height => (cols.South_mostChunk - NorthEdge + 1); + + public Snapshotter(string path, string chunkPath, ColumnsMetadata cols, int chunkSize) + { + this.path = Path.Combine(path, "snapshot.png"); + this.chunkPath = Path.Combine(path, chunkPath); + this.cols = cols; + this.chunkSize = chunkSize; + } + + /// + /// takes a snapshot. this should be called from an extra thread. + /// + /// path to the map dir + /// name of the chunks dir part thing + /// + /// + public async void Take() + { + var t = new Stopwatch(); + t.Start(); + Console.WriteLine("snapshot started"); + + ImageInfo info = new ImageInfo(Width * chunkSize, Height * chunkSize, 8, false); + PngWriter snapWriter = FileHelper.CreatePngWriter(path, info, true); + snapWriter.CompLevel = 5; + snapWriter.CompressionStrategy = Hjg.Pngcs.Zlib.EDeflateCompressStrategy.Huffman; + + var orderedList = + from ch in cols + group ch by ch.Location.Y into g + orderby g.Key + select g; + // that sorts things in ascending order so we can only create (chunkSize) lines at once + int row = 0; + foreach (var chunkGroup in orderedList) + { + var posY = chunkGroup.Key - NorthEdge; + var inGroup = (await ReadAllInGroup(chunkGroup)).ToArray(); + // oh god here we go... + for (int r = 0; r < chunkSize; r++) + { + var line = new byte[info.BytesPerRow]; + for (int chunk = 0; chunk < inGroup.Length; chunk++) + { + var posX = inGroup[chunk].Key * chunkSize * info.BytesPixel; + inGroup[chunk].Value?[r].CopyTo(line, posX); + } + snapWriter.WriteRowByte(line, row++); + } + } + snapWriter.ShouldCloseStream = true; + + Console.WriteLine($"snapshot finished in {t.ElapsedMilliseconds}"); + } + + private async Task> ReadAllInGroup(IGrouping group) + { + var taskGroup = new Dictionary(group.Count()); + foreach (var shardMeta in group) + { + var shardPath = Path.Combine(chunkPath, $"{shardMeta.Location.X}_{shardMeta.Location.Y}.png"); + taskGroup.Add(shardMeta.Location.X - WestEdge, await ReadNoThrow(shardPath)); + } + return taskGroup; + } + + private async Task ReadNoThrow(string readPath) + { + try + { + return await Task.Run(() => FileHelper.CreatePngReader(readPath).ReadRowsByte().ScanlinesB); + } + catch (Exception e) + { + Console.WriteLine(e.Message); + return null; + } // do nothing on error + + } + } +}