From 59e04c6f92da584b322c87072f18e6cab4de4c60 Mon Sep 17 00:00:00 2001 From: Adam Lesinski Date: Thu, 4 Feb 2016 15:59:23 -0800 Subject: [PATCH] AAPT2: Switch to protobuf for intermediate format Without needing to conform to the runtime data format, it is much easier to add new features such as debugging symbols and carrying over product data to link time. This also simplifies the runtime format parser and serializer, which will change much less frequently than the protobuf intermediate format. Change-Id: I209787bbf087db0a58a534cb8511c51d21133e00 --- tools/aapt2/Android.mk | 20 +- tools/aapt2/Format.proto | 210 +++++++++ tools/aapt2/Main.cpp | 5 +- tools/aapt2/ResourceUtils.cpp | 4 + tools/aapt2/ResourceUtils.h | 3 +- tools/aapt2/ResourceUtils_test.cpp | 2 + tools/aapt2/ResourceValues.cpp | 3 +- tools/aapt2/ResourceValues.h | 4 +- tools/aapt2/ValueVisitor.h | 18 + tools/aapt2/compile/Compile.cpp | 169 +++---- tools/aapt2/compile/IdAssigner.cpp | 10 +- tools/aapt2/dump/Dump.cpp | 128 +++++ tools/aapt2/flatten/Archive.h | 8 +- tools/aapt2/flatten/FileExportWriter.h | 67 --- tools/aapt2/flatten/FileExportWriter_test.cpp | 51 -- tools/aapt2/flatten/ResourceTypeExtensions.h | 202 -------- tools/aapt2/flatten/TableFlattener.cpp | 402 +++------------- tools/aapt2/flatten/TableFlattener.h | 15 +- tools/aapt2/flatten/TableFlattener_test.cpp | 89 +--- tools/aapt2/link/Link.cpp | 72 +-- tools/aapt2/proto/ProtoHelpers.cpp | 137 ++++++ tools/aapt2/proto/ProtoHelpers.h | 52 +++ tools/aapt2/proto/ProtoSerialize.h | 77 +++ tools/aapt2/proto/TableProtoDeserializer.cpp | 514 +++++++++++++++++++++ tools/aapt2/proto/TableProtoSerializer.cpp | 316 +++++++++++++ tools/aapt2/proto/TableProtoSerializer_test.cpp | 112 +++++ tools/aapt2/unflatten/BinaryResourceParser.cpp | 430 +---------------- tools/aapt2/unflatten/BinaryResourceParser.h | 27 -- tools/aapt2/unflatten/FileExportHeaderReader.h | 159 ------- .../unflatten/FileExportHeaderReader_test.cpp | 58 --- 30 files changed, 1827 insertions(+), 1537 deletions(-) create mode 100644 tools/aapt2/Format.proto create mode 100644 tools/aapt2/dump/Dump.cpp delete mode 100644 tools/aapt2/flatten/FileExportWriter.h delete mode 100644 tools/aapt2/flatten/FileExportWriter_test.cpp create mode 100644 tools/aapt2/proto/ProtoHelpers.cpp create mode 100644 tools/aapt2/proto/ProtoHelpers.h create mode 100644 tools/aapt2/proto/ProtoSerialize.h create mode 100644 tools/aapt2/proto/TableProtoDeserializer.cpp create mode 100644 tools/aapt2/proto/TableProtoSerializer.cpp create mode 100644 tools/aapt2/proto/TableProtoSerializer_test.cpp delete mode 100644 tools/aapt2/unflatten/FileExportHeaderReader.h delete mode 100644 tools/aapt2/unflatten/FileExportHeaderReader_test.cpp diff --git a/tools/aapt2/Android.mk b/tools/aapt2/Android.mk index f74b93abd796..88b6270fad60 100644 --- a/tools/aapt2/Android.mk +++ b/tools/aapt2/Android.mk @@ -43,6 +43,9 @@ sources := \ link/TableMerger.cpp \ link/XmlReferenceLinker.cpp \ process/SymbolTable.cpp \ + proto/ProtoHelpers.cpp \ + proto/TableProtoDeserializer.cpp \ + proto/TableProtoSerializer.cpp \ unflatten/BinaryResourceParser.cpp \ unflatten/ResChunkPullParser.cpp \ util/BigBuffer.cpp \ @@ -67,13 +70,14 @@ sources := \ xml/XmlPullParser.cpp \ xml/XmlUtil.cpp +sources += Format.proto + testSources := \ compile/IdAssigner_test.cpp \ compile/PseudolocaleGenerator_test.cpp \ compile/Pseudolocalizer_test.cpp \ compile/XmlIdCollector_test.cpp \ filter/ConfigFilter_test.cpp \ - flatten/FileExportWriter_test.cpp \ flatten/TableFlattener_test.cpp \ flatten/XmlFlattener_test.cpp \ link/AutoVersioner_test.cpp \ @@ -83,7 +87,7 @@ testSources := \ link/TableMerger_test.cpp \ link/XmlReferenceLinker_test.cpp \ process/SymbolTable_test.cpp \ - unflatten/FileExportHeaderReader_test.cpp \ + proto/TableProtoSerializer_test.cpp \ util/BigBuffer_test.cpp \ util/Maybe_test.cpp \ util/StringPiece_test.cpp \ @@ -105,6 +109,7 @@ testSources := \ toolSources := \ compile/Compile.cpp \ + dump/Dump.cpp \ link/Link.cpp hostLdLibs := @@ -119,6 +124,9 @@ hostStaticLibs := \ libpng \ libbase +hostSharedLibs := \ + libprotobuf-cpp-lite + ifneq ($(strip $(USE_MINGW)),) hostStaticLibs += libz else @@ -127,21 +135,23 @@ endif cFlags := -Wall -Werror -Wno-unused-parameter -UNDEBUG cppFlags := -std=c++11 -Wno-missing-field-initializers -fno-exceptions -fno-rtti +protoIncludes := $(call generated-sources-dir-for,STATIC_LIBRARIES,libaapt2,HOST) # ========================================================== # Build the host static library: libaapt2 # ========================================================== include $(CLEAR_VARS) +LOCAL_MODULE_CLASS := STATIC_LIBRARIES LOCAL_MODULE := libaapt2 LOCAL_SRC_FILES := $(sources) LOCAL_STATIC_LIBRARIES += $(hostStaticLibs) LOCAL_CFLAGS += $(cFlags) LOCAL_CPPFLAGS += $(cppFlags) +LOCAL_C_INCLUDES += $(protoIncludes) include $(BUILD_HOST_STATIC_LIBRARY) - # ========================================================== # Build the host tests: libaapt2_tests # ========================================================== @@ -152,9 +162,11 @@ LOCAL_MODULE_TAGS := tests LOCAL_SRC_FILES := $(testSources) LOCAL_STATIC_LIBRARIES += libaapt2 $(hostStaticLibs) +LOCAL_SHARED_LIBRARIES += $(hostSharedLibs) LOCAL_LDLIBS += $(hostLdLibs) LOCAL_CFLAGS += $(cFlags) LOCAL_CPPFLAGS += $(cppFlags) +LOCAL_C_INCLUDES += $(protoIncludes) include $(BUILD_HOST_NATIVE_TEST) @@ -167,9 +179,11 @@ LOCAL_MODULE := aapt2 LOCAL_SRC_FILES := $(main) $(toolSources) LOCAL_STATIC_LIBRARIES += libaapt2 $(hostStaticLibs) +LOCAL_SHARED_LIBRARIES += $(hostSharedLibs) LOCAL_LDLIBS += $(hostLdLibs) LOCAL_CFLAGS += $(cFlags) LOCAL_CPPFLAGS += $(cppFlags) +LOCAL_C_INCLUDES += $(protoIncludes) include $(BUILD_HOST_EXECUTABLE) diff --git a/tools/aapt2/Format.proto b/tools/aapt2/Format.proto new file mode 100644 index 000000000000..d05425c5c64d --- /dev/null +++ b/tools/aapt2/Format.proto @@ -0,0 +1,210 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +option optimize_for = LITE_RUNTIME; + +package aapt.pb; + +message ConfigDescription { + optional bytes data = 1; + optional string product = 2; +} + +message StringPool { + optional bytes data = 1; +} + +message CompiledFile { + message Symbol { + optional string resource_name = 1; + optional uint32 line_no = 2; + } + + optional string resource_name = 1; + optional ConfigDescription config = 2; + optional string source_path = 3; + repeated Symbol exported_symbols = 4; +} + +message ResourceTable { + optional StringPool string_pool = 1; + optional StringPool source_pool = 2; + optional StringPool symbol_pool = 3; + repeated Package packages = 4; +} + +message Package { + optional uint32 package_id = 1; + optional string package_name = 2; + repeated Type types = 3; +} + +message Type { + optional uint32 id = 1; + optional string name = 2; + repeated Entry entries = 3; +} + +message SymbolStatus { + enum Visibility { + Unknown = 0; + Private = 1; + Public = 2; + } + optional Visibility visibility = 1; + optional Source source = 2; + optional string comment = 3; +} + +message Entry { + optional uint32 id = 1; + optional string name = 2; + optional SymbolStatus symbol_status = 3; + repeated ConfigValue config_values = 4; +} + +message ConfigValue { + optional ConfigDescription config = 1; + optional Value value = 2; +} + +message Source { + optional uint32 path_idx = 1; + optional uint32 line_no = 2; + optional uint32 col_no = 3; +} + +message Reference { + enum Type { + Ref = 0; + Attr = 1; + } + optional Type type = 1; + optional uint32 id = 2; + optional uint32 symbol_idx = 3; + optional bool private = 4; +} + +message Id { +} + +message String { + optional uint32 idx = 1; +} + +message RawString { + optional uint32 idx = 1; +} + +message FileReference { + optional uint32 path_idx = 1; +} + +message Primitive { + optional uint32 type = 1; + optional uint32 data = 2; +} + +message Attribute { + message Symbol { + optional Source source = 1; + optional string comment = 2; + optional Reference name = 3; + optional uint32 value = 4; + } + optional uint32 format_flags = 1; + optional int32 min_int = 2; + optional int32 max_int = 3; + repeated Symbol symbols = 4; +} + +message Style { + message Entry { + optional Source source = 1; + optional string comment = 2; + optional Reference key = 3; + optional Item item = 4; + } + + optional Reference parent = 1; + optional Source parent_source = 2; + repeated Entry entries = 3; +} + +message Styleable { + message Entry { + optional Source source = 1; + optional string comment = 2; + optional Reference attr = 3; + } + repeated Entry entries = 1; +} + +message Array { + message Entry { + optional Source source = 1; + optional string comment = 2; + optional Item item = 3; + } + repeated Entry entries = 1; +} + +message Plural { + enum Arity { + Zero = 0; + One = 1; + Two = 2; + Few = 3; + Many = 4; + Other = 5; + } + + message Entry { + optional Source source = 1; + optional string comment = 2; + optional Arity arity = 3; + optional Item item = 4; + } + repeated Entry entries = 1; +} + +message Item { + optional Reference ref = 1; + optional String str = 2; + optional RawString raw_str = 3; + optional FileReference file = 4; + optional Id id = 5; + optional Primitive prim = 6; +} + +message CompoundValue { + optional Attribute attr = 1; + optional Style style = 2; + optional Styleable styleable = 3; + optional Array array = 4; + optional Plural plural = 5; +} + +message Value { + optional Source source = 1; + optional string comment = 2; + optional bool weak = 3; + + optional Item item = 4; + optional CompoundValue compound_value = 5; +} diff --git a/tools/aapt2/Main.cpp b/tools/aapt2/Main.cpp index 248e7ad73a82..a2fadd95db3f 100644 --- a/tools/aapt2/Main.cpp +++ b/tools/aapt2/Main.cpp @@ -23,6 +23,7 @@ namespace aapt { extern int compile(const std::vector& args); extern int link(const std::vector& args); +extern int dump(const std::vector& args); } // namespace aapt @@ -41,12 +42,14 @@ int main(int argc, char** argv) { return aapt::compile(args); } else if (command == "link" || command == "l") { return aapt::link(args); + } else if (command == "dump" || command == "d") { + return aapt::dump(args); } std::cerr << "unknown command '" << command << "'\n"; } else { std::cerr << "no command specified\n"; } - std::cerr << "\nusage: aapt2 [compile|link] ..." << std::endl; + std::cerr << "\nusage: aapt2 [compile|link|dump] ..." << std::endl; return 1; } diff --git a/tools/aapt2/ResourceUtils.cpp b/tools/aapt2/ResourceUtils.cpp index 07f62afe05b9..74c48b0f8426 100644 --- a/tools/aapt2/ResourceUtils.cpp +++ b/tools/aapt2/ResourceUtils.cpp @@ -51,6 +51,10 @@ bool extractResourceName(const StringPiece16& str, StringPiece16* outPackage, } bool parseResourceName(const StringPiece16& str, ResourceNameRef* outRef, bool* outPrivate) { + if (str.empty()) { + return false; + } + size_t offset = 0; bool priv = false; if (str.data()[0] == u'*') { diff --git a/tools/aapt2/ResourceUtils.h b/tools/aapt2/ResourceUtils.h index 64ca97185153..a0fbcc6e700b 100644 --- a/tools/aapt2/ResourceUtils.h +++ b/tools/aapt2/ResourceUtils.h @@ -45,7 +45,8 @@ bool extractResourceName(const StringPiece16& str, StringPiece16* outPackage, * `outResource` set to the parsed resource name and `outPrivate` set to true if a '*' prefix * was present. */ -bool parseResourceName(const StringPiece16& str, ResourceNameRef* outResource, bool* outPrivate); +bool parseResourceName(const StringPiece16& str, ResourceNameRef* outResource, + bool* outPrivate = nullptr); /* * Returns true if the string was parsed as a reference (@[+][package:]type/name), with diff --git a/tools/aapt2/ResourceUtils_test.cpp b/tools/aapt2/ResourceUtils_test.cpp index c9f93e1dd7c2..7425f97ef8de 100644 --- a/tools/aapt2/ResourceUtils_test.cpp +++ b/tools/aapt2/ResourceUtils_test.cpp @@ -58,6 +58,8 @@ TEST(ResourceUtilsTest, ParseResourceName) { EXPECT_TRUE(ResourceUtils::parseResourceName(u"*android:color/foo", &actual, &actualPriv)); EXPECT_EQ(ResourceNameRef(u"android", ResourceType::kColor, u"foo"), actual); EXPECT_TRUE(actualPriv); + + EXPECT_FALSE(ResourceUtils::parseResourceName(StringPiece16(), &actual, &actualPriv)); } TEST(ResourceUtilsTest, ParseReferenceWithNoPackage) { diff --git a/tools/aapt2/ResourceValues.cpp b/tools/aapt2/ResourceValues.cpp index b93e6d889ad0..ab9c792876b3 100644 --- a/tools/aapt2/ResourceValues.cpp +++ b/tools/aapt2/ResourceValues.cpp @@ -19,7 +19,6 @@ #include "ResourceValues.h" #include "ValueVisitor.h" #include "util/Util.h" -#include "flatten/ResourceTypeExtensions.h" #include #include @@ -47,7 +46,7 @@ RawString* RawString::clone(StringPool* newPool) const { } bool RawString::flatten(android::Res_value* outValue) const { - outValue->dataType = ExtendedTypes::TYPE_RAW_STRING; + outValue->dataType = android::Res_value::TYPE_STRING; outValue->data = util::hostToDevice32(static_cast(value.getIndex())); return true; } diff --git a/tools/aapt2/ResourceValues.h b/tools/aapt2/ResourceValues.h index 8e317dbcd1b1..dc2e28ee3abd 100644 --- a/tools/aapt2/ResourceValues.h +++ b/tools/aapt2/ResourceValues.h @@ -154,8 +154,8 @@ struct Reference : public BaseItem { bool privateReference = false; Reference(); - Reference(const ResourceNameRef& n, Type type = Type::kResource); - Reference(const ResourceId& i, Type type = Type::kResource); + explicit Reference(const ResourceNameRef& n, Type type = Type::kResource); + explicit Reference(const ResourceId& i, Type type = Type::kResource); bool flatten(android::Res_value* outValue) const override; Reference* clone(StringPool* newPool) const override; diff --git a/tools/aapt2/ValueVisitor.h b/tools/aapt2/ValueVisitor.h index 94042e3c2618..549303939351 100644 --- a/tools/aapt2/ValueVisitor.h +++ b/tools/aapt2/ValueVisitor.h @@ -18,6 +18,7 @@ #define AAPT_VALUE_VISITOR_H #include "ResourceValues.h" +#include "ResourceTable.h" namespace aapt { @@ -140,6 +141,23 @@ T* valueCast(Value* value) { return visitor.value; } + +inline void visitAllValuesInPackage(ResourceTablePackage* pkg, RawValueVisitor* visitor) { + for (auto& type : pkg->types) { + for (auto& entry : type->entries) { + for (auto& configValue : entry->values) { + configValue.value->accept(visitor); + } + } + } +} + +inline void visitAllValuesInTable(ResourceTable* table, RawValueVisitor* visitor) { + for (auto& pkg : table->packages) { + visitAllValuesInPackage(pkg.get(), visitor); + } +} + } // namespace aapt #endif // AAPT_VALUE_VISITOR_H diff --git a/tools/aapt2/compile/Compile.cpp b/tools/aapt2/compile/Compile.cpp index 689ace6e6aa1..1eefb821768e 100644 --- a/tools/aapt2/compile/Compile.cpp +++ b/tools/aapt2/compile/Compile.cpp @@ -24,15 +24,17 @@ #include "compile/PseudolocaleGenerator.h" #include "compile/XmlIdCollector.h" #include "flatten/Archive.h" -#include "flatten/FileExportWriter.h" -#include "flatten/TableFlattener.h" #include "flatten/XmlFlattener.h" +#include "proto/ProtoSerialize.h" #include "util/Files.h" #include "util/Maybe.h" #include "util/Util.h" #include "xml/XmlDom.h" #include "xml/XmlPullParser.h" +#include +#include + #include #include #include @@ -232,34 +234,95 @@ static bool compileTable(IAaptContext* context, const CompileOptions& options, } } - // Assign IDs to prepare the table for flattening. - IdAssigner idAssigner; - if (!idAssigner.consume(context, &table)) { + // Create the file/zip entry. + if (!writer->startEntry(outputPath, 0)) { + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to open"); return false; } - // Flatten the table. - BigBuffer buffer(1024); - TableFlattenerOptions tableFlattenerOptions; - tableFlattenerOptions.useExtendedChunks = true; - TableFlattener flattener(&buffer, tableFlattenerOptions); - if (!flattener.consume(context, &table)) { + std::unique_ptr pbTable = serializeTableToPb(&table); + + // Wrap our IArchiveWriter with an adaptor that implements the ZeroCopyOutputStream interface. + { + google::protobuf::io::CopyingOutputStreamAdaptor adaptor(writer); + + if (!pbTable->SerializeToZeroCopyStream(&adaptor)) { + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); + return false; + } + } + + if (!writer->finishEntry()) { + context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to finish entry"); return false; } + return true; +} +static bool writeHeaderAndBufferToWriter(const StringPiece& outputPath, const ResourceFile& file, + const BigBuffer& buffer, IArchiveWriter* writer, + IDiagnostics* diag) { + // Start the entry so we can write the header. if (!writer->startEntry(outputPath, 0)) { - context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to open"); + diag->error(DiagMessage(outputPath) << "failed to open file"); + return false; + } + + // Create the header. + std::unique_ptr pbCompiledFile = serializeCompiledFileToPb(file); + + { + // The stream must be destroyed before we finish the entry, or else + // some data won't be flushed. + // Wrap our IArchiveWriter with an adaptor that implements the ZeroCopyOutputStream + // interface. + google::protobuf::io::CopyingOutputStreamAdaptor adaptor(writer); + CompiledFileOutputStream outputStream(&adaptor, pbCompiledFile.get()); + for (const BigBuffer::Block& block : buffer) { + if (!outputStream.Write(block.buffer.get(), block.size)) { + diag->error(DiagMessage(outputPath) << "failed to write data"); + return false; + } + } + } + + if (!writer->finishEntry()) { + diag->error(DiagMessage(outputPath) << "failed to finish writing data"); return false; } + return true; +} - if (writer->writeEntry(buffer)) { - if (writer->finishEntry()) { - return true; +static bool writeHeaderAndMmapToWriter(const StringPiece& outputPath, const ResourceFile& file, + const android::FileMap& map, IArchiveWriter* writer, + IDiagnostics* diag) { + // Start the entry so we can write the header. + if (!writer->startEntry(outputPath, 0)) { + diag->error(DiagMessage(outputPath) << "failed to open file"); + return false; + } + + // Create the header. + std::unique_ptr pbCompiledFile = serializeCompiledFileToPb(file); + + { + // The stream must be destroyed before we finish the entry, or else + // some data won't be flushed. + // Wrap our IArchiveWriter with an adaptor that implements the ZeroCopyOutputStream + // interface. + google::protobuf::io::CopyingOutputStreamAdaptor adaptor(writer); + CompiledFileOutputStream outputStream(&adaptor, pbCompiledFile.get()); + if (!outputStream.Write(map.getDataPtr(), map.getDataLength())) { + diag->error(DiagMessage(outputPath) << "failed to write data"); + return false; } } - context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); - return false; + if (!writer->finishEntry()) { + diag->error(DiagMessage(outputPath) << "failed to finish writing data"); + return false; + } + return true; } static bool compileXml(IAaptContext* context, const CompileOptions& options, @@ -267,7 +330,6 @@ static bool compileXml(IAaptContext* context, const CompileOptions& options, const std::string& outputPath) { std::unique_ptr xmlRes; - { std::ifstream fin(pathData.source.path, std::ifstream::binary); if (!fin) { @@ -295,30 +357,18 @@ static bool compileXml(IAaptContext* context, const CompileOptions& options, xmlRes->file.source = pathData.source; BigBuffer buffer(1024); - ChunkWriter fileExportWriter = wrapBufferWithFileExportHeader(&buffer, &xmlRes->file); - XmlFlattenerOptions xmlFlattenerOptions; xmlFlattenerOptions.keepRawValues = true; - XmlFlattener flattener(fileExportWriter.getBuffer(), xmlFlattenerOptions); + XmlFlattener flattener(&buffer, xmlFlattenerOptions); if (!flattener.consume(context, xmlRes.get())) { return false; } - fileExportWriter.finish(); - - if (!writer->startEntry(outputPath, 0)) { - context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to open"); + if (!writeHeaderAndBufferToWriter(outputPath, xmlRes->file, buffer, writer, + context->getDiagnostics())) { return false; } - - if (writer->writeEntry(buffer)) { - if (writer->finishEntry()) { - return true; - } - } - - context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); - return false; + return true; } static bool compilePng(IAaptContext* context, const CompileOptions& options, @@ -330,8 +380,6 @@ static bool compilePng(IAaptContext* context, const CompileOptions& options, resFile.config = pathData.config; resFile.source = pathData.source; - ChunkWriter fileExportWriter = wrapBufferWithFileExportHeader(&buffer, &resFile); - { std::ifstream fin(pathData.source.path, std::ifstream::binary); if (!fin) { @@ -340,26 +388,16 @@ static bool compilePng(IAaptContext* context, const CompileOptions& options, } Png png(context->getDiagnostics()); - if (!png.process(pathData.source, &fin, fileExportWriter.getBuffer(), {})) { + if (!png.process(pathData.source, &fin, &buffer, {})) { return false; } } - fileExportWriter.finish(); - - if (!writer->startEntry(outputPath, 0)) { - context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to open"); + if (!writeHeaderAndBufferToWriter(outputPath, resFile, buffer, writer, + context->getDiagnostics())) { return false; } - - if (writer->writeEntry(buffer)) { - if (writer->finishEntry()) { - return true; - } - } - - context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); - return false; + return true; } static bool compileFile(IAaptContext* context, const CompileOptions& options, @@ -371,8 +409,6 @@ static bool compileFile(IAaptContext* context, const CompileOptions& options, resFile.config = pathData.config; resFile.source = pathData.source; - ChunkWriter fileExportWriter = wrapBufferWithFileExportHeader(&buffer, &resFile); - std::string errorStr; Maybe f = file::mmapPath(pathData.source.path, &errorStr); if (!f) { @@ -380,35 +416,10 @@ static bool compileFile(IAaptContext* context, const CompileOptions& options, return false; } - if (!writer->startEntry(outputPath, 0)) { - context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to open"); - return false; - } - - // Manually set the size and don't call finish(). This is because we are not copying from - // the buffer the entire file. - fileExportWriter.getChunkHeader()->size = - util::hostToDevice32(buffer.size() + f.value().getDataLength()); - - if (!writer->writeEntry(buffer)) { - context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); + if (!writeHeaderAndMmapToWriter(outputPath, resFile, f.value(), writer, + context->getDiagnostics())) { return false; } - - // Only write if we have something to write. This is because mmap fails with length of 0, - // but we still want to compile the file to get the resource ID. - if (f.value().getDataPtr() && f.value().getDataLength() > 0) { - if (!writer->writeEntry(f.value().getDataPtr(), f.value().getDataLength())) { - context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); - return false; - } - } - - if (!writer->finishEntry()) { - context->getDiagnostics()->error(DiagMessage(outputPath) << "failed to write"); - return false; - } - return true; } diff --git a/tools/aapt2/compile/IdAssigner.cpp b/tools/aapt2/compile/IdAssigner.cpp index 80c6bbc1abca..aa4a5803b8df 100644 --- a/tools/aapt2/compile/IdAssigner.cpp +++ b/tools/aapt2/compile/IdAssigner.cpp @@ -64,14 +64,12 @@ bool IdAssigner::consume(IAaptContext* context, ResourceTable* table) { // Mark entry ID as taken. if (!usedEntryIds.insert(entry->id.value()).second) { // This ID existed before! - ResourceNameRef nameRef = - { package->name, type->type, entry->name }; - ResourceId takenId(package->id.value(), type->id.value(), - entry->id.value()); + ResourceNameRef nameRef(package->name, type->type, entry->name); context->getDiagnostics()->error(DiagMessage() << "resource '" << nameRef << "' " - << "has duplicate ID '" - << takenId << "'"); + << "has duplicate entry ID " + << std::hex << (int) entry->id.value() + << std::dec); return false; } } diff --git a/tools/aapt2/dump/Dump.cpp b/tools/aapt2/dump/Dump.cpp new file mode 100644 index 000000000000..915fae80fcbb --- /dev/null +++ b/tools/aapt2/dump/Dump.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Debug.h" +#include "Diagnostics.h" +#include "Flags.h" +#include "process/IResourceTableConsumer.h" +#include "proto/ProtoSerialize.h" +#include "util/Files.h" +#include "util/StringPiece.h" + +#include + +namespace aapt { + +//struct DumpOptions { +// +//}; + +void dumpCompiledFile(const pb::CompiledFile& pbFile, const void* data, size_t len, + const Source& source, IAaptContext* context) { + std::unique_ptr file = deserializeCompiledFileFromPb(pbFile, source, + context->getDiagnostics()); + if (!file) { + return; + } + + std::cout << "Resource: " << file->name << "\n" + << "Config: " << file->config << "\n" + << "Source: " << file->source << "\n"; +} + +void dumpCompiledTable(const pb::ResourceTable& pbTable, const Source& source, + IAaptContext* context) { + std::unique_ptr table = deserializeTableFromPb(pbTable, source, + context->getDiagnostics()); + if (!table) { + return; + } + + Debug::printTable(table.get()); +} + +void tryDumpFile(IAaptContext* context, const std::string& filePath) { + std::string err; + Maybe file = file::mmapPath(filePath, &err); + if (!file) { + context->getDiagnostics()->error(DiagMessage(filePath) << err); + return; + } + + android::FileMap* fileMap = &file.value(); + + // Try as a compiled table. + pb::ResourceTable pbTable; + if (pbTable.ParseFromArray(fileMap->getDataPtr(), fileMap->getDataLength())) { + dumpCompiledTable(pbTable, Source(filePath), context); + return; + } + + // Try as a compiled file. + CompiledFileInputStream input(fileMap->getDataPtr(), fileMap->getDataLength()); + if (const pb::CompiledFile* pbFile = input.CompiledFile()) { + dumpCompiledFile(*pbFile, input.data(), input.size(), Source(filePath), context); + return; + } +} + +class DumpContext : public IAaptContext { +public: + IDiagnostics* getDiagnostics() override { + return &mDiagnostics; + } + + NameMangler* getNameMangler() override { + abort(); + return nullptr; + } + + StringPiece16 getCompilationPackage() override { + return {}; + } + + uint8_t getPackageId() override { + return 0; + } + + ISymbolTable* getExternalSymbols() override { + abort(); + return nullptr; + } + +private: + StdErrDiagnostics mDiagnostics; +}; + +/** + * Entry point for dump command. + */ +int dump(const std::vector& args) { + //DumpOptions options; + Flags flags = Flags(); + if (!flags.parse("aapt2 dump", args, &std::cerr)) { + return 1; + } + + DumpContext context; + + for (const std::string& arg : flags.getArgs()) { + tryDumpFile(&context, arg); + } + return 0; +} + +} // namespace aapt diff --git a/tools/aapt2/flatten/Archive.h b/tools/aapt2/flatten/Archive.h index 6da1d2ac5620..34c10ad40365 100644 --- a/tools/aapt2/flatten/Archive.h +++ b/tools/aapt2/flatten/Archive.h @@ -22,6 +22,7 @@ #include "util/Files.h" #include "util/StringPiece.h" +#include #include #include #include @@ -40,13 +41,18 @@ struct ArchiveEntry { size_t uncompressedSize; }; -struct IArchiveWriter { +struct IArchiveWriter : public google::protobuf::io::CopyingOutputStream { virtual ~IArchiveWriter() = default; virtual bool startEntry(const StringPiece& path, uint32_t flags) = 0; virtual bool writeEntry(const BigBuffer& buffer) = 0; virtual bool writeEntry(const void* data, size_t len) = 0; virtual bool finishEntry() = 0; + + // CopyingOutputStream implementations. + bool Write(const void* buffer, int size) override { + return writeEntry(buffer, size); + } }; std::unique_ptr createDirectoryArchiveWriter(IDiagnostics* diag, diff --git a/tools/aapt2/flatten/FileExportWriter.h b/tools/aapt2/flatten/FileExportWriter.h deleted file mode 100644 index 7688fa71246e..000000000000 --- a/tools/aapt2/flatten/FileExportWriter.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef AAPT_FLATTEN_FILEEXPORTWRITER_H -#define AAPT_FLATTEN_FILEEXPORTWRITER_H - -#include "StringPool.h" - -#include "flatten/ResourceTypeExtensions.h" -#include "flatten/ChunkWriter.h" -#include "process/IResourceTableConsumer.h" -#include "util/BigBuffer.h" -#include "util/Util.h" - -#include -#include - -namespace aapt { - -static ChunkWriter wrapBufferWithFileExportHeader(BigBuffer* buffer, ResourceFile* res) { - ChunkWriter fileExportWriter(buffer); - FileExport_header* fileExport = fileExportWriter.startChunk( - RES_FILE_EXPORT_TYPE); - - ExportedSymbol* symbolRefs = nullptr; - if (!res->exportedSymbols.empty()) { - symbolRefs = fileExportWriter.nextBlock( - res->exportedSymbols.size()); - } - fileExport->exportedSymbolCount = util::hostToDevice32(res->exportedSymbols.size()); - - StringPool symbolExportPool; - memcpy(fileExport->magic, "AAPT", NELEM(fileExport->magic)); - fileExport->config = res->config; - fileExport->config.swapHtoD(); - fileExport->name.index = util::hostToDevice32(symbolExportPool.makeRef(res->name.toString()) - .getIndex()); - fileExport->source.index = util::hostToDevice32(symbolExportPool.makeRef(util::utf8ToUtf16( - res->source.path)).getIndex()); - - for (const SourcedResourceName& name : res->exportedSymbols) { - symbolRefs->name.index = util::hostToDevice32(symbolExportPool.makeRef(name.name.toString()) - .getIndex()); - symbolRefs->line = util::hostToDevice32(name.line); - symbolRefs++; - } - - StringPool::flattenUtf16(fileExportWriter.getBuffer(), symbolExportPool); - return fileExportWriter; -} - -} // namespace aapt - -#endif /* AAPT_FLATTEN_FILEEXPORTWRITER_H */ diff --git a/tools/aapt2/flatten/FileExportWriter_test.cpp b/tools/aapt2/flatten/FileExportWriter_test.cpp deleted file mode 100644 index 32fc203c4dee..000000000000 --- a/tools/aapt2/flatten/FileExportWriter_test.cpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Resource.h" - -#include "flatten/FileExportWriter.h" -#include "util/BigBuffer.h" -#include "util/Util.h" - -#include "test/Common.h" - -#include - -namespace aapt { - -TEST(FileExportWriterTest, FlattenResourceFileDataWithNoExports) { - ResourceFile resFile = { - test::parseNameOrDie(u"@android:layout/main.xml"), - test::parseConfigOrDie("sw600dp-v4"), - Source{ "res/layout/main.xml" }, - }; - - BigBuffer buffer(1024); - ChunkWriter writer = wrapBufferWithFileExportHeader(&buffer, &resFile); - *writer.getBuffer()->nextBlock() = 42u; - writer.finish(); - - std::unique_ptr data = util::copy(buffer); - - // There should be more data (string pool) besides the header and our data. - ASSERT_GT(buffer.size(), sizeof(FileExport_header) + sizeof(uint32_t)); - - // Write at the end of this chunk is our data. - uint32_t* val = (uint32_t*)(data.get() + buffer.size()) - 1; - EXPECT_EQ(*val, 42u); -} - -} // namespace aapt diff --git a/tools/aapt2/flatten/ResourceTypeExtensions.h b/tools/aapt2/flatten/ResourceTypeExtensions.h index 02bff2c69362..3e20ad643eb6 100644 --- a/tools/aapt2/flatten/ResourceTypeExtensions.h +++ b/tools/aapt2/flatten/ResourceTypeExtensions.h @@ -22,208 +22,6 @@ namespace aapt { /** - * New android::ResChunk_header types defined - * for AAPT to use. - * - * TODO(adamlesinski): Consider reserving these - * enums in androidfw/ResourceTypes.h to avoid - * future collisions. - */ -enum { - /** - * A chunk that contains an entire file that - * has been compiled. - */ - RES_FILE_EXPORT_TYPE = 0x000c, - - RES_TABLE_PUBLIC_TYPE = 0x000d, - - /** - * A chunk that holds the string pool - * for source entries (path/to/source:line). - */ - RES_TABLE_SOURCE_POOL_TYPE = 0x000e, - - /** - * A chunk holding names of externally - * defined symbols and offsets to where - * they are referenced in the table. - */ - RES_TABLE_SYMBOL_TABLE_TYPE = 0x000f, -}; - -/** - * New resource types that are meant to only be used - * by AAPT and will not end up on the device. - */ -struct ExtendedTypes { - enum { - /** - * A raw string value that hasn't had its escape sequences - * processed nor whitespace removed. - */ - TYPE_RAW_STRING = 0xfe, - }; -}; - -/** - * New types for a ResTable_map. - */ -struct ExtendedResTableMapTypes { - enum { - /** - * Type that contains the source path of the next item in the map. - */ - ATTR_SOURCE_PATH = Res_MAKEINTERNAL(0xffff), - - /** - * Type that contains the source line of the next item in the map. - */ - ATTR_SOURCE_LINE = Res_MAKEINTERNAL(0xfffe), - - /** - * Type that contains the comment of the next item in the map. - */ - ATTR_COMMENT = Res_MAKEINTERNAL(0xfffd) - }; -}; - -/** - * Followed by exportedSymbolCount ExportedSymbol structs, followed by the string pool. - */ -struct FileExport_header { - android::ResChunk_header header; - - /** - * MAGIC value. Must be 'AAPT' (0x41415054) - */ - uint8_t magic[4]; - - /** - * Version of AAPT that built this file. - */ - uint32_t version; - - /** - * The resource name. - */ - android::ResStringPool_ref name; - - /** - * Configuration of this file. - */ - android::ResTable_config config; - - /** - * Original source path of this file. - */ - android::ResStringPool_ref source; - - /** - * Number of symbols exported by this file. - */ - uint32_t exportedSymbolCount; -}; - -struct ExportedSymbol { - android::ResStringPool_ref name; - uint32_t line; -}; - -struct Public_header { - android::ResChunk_header header; - - /** - * The ID of the type this structure refers to. - */ - uint8_t typeId; - - /** - * Reserved. Must be 0. - */ - uint8_t res0; - - /** - * Reserved. Must be 0. - */ - uint16_t res1; - - /** - * Number of public entries. - */ - uint32_t count; -}; - -/** - * A structure representing source data for a resource entry. - * Appears after an android::ResTable_entry or android::ResTable_map_entry. - * - * TODO(adamlesinski): This causes some issues when runtime code checks - * the size of an android::ResTable_entry. It assumes it is an - * android::ResTable_map_entry if the size is bigger than an android::ResTable_entry - * which may not be true if this structure is present. - */ -struct ResTable_entry_source { - /** - * File path reference. - */ - android::ResStringPool_ref path; - - /** - * Line number this resource was defined on. - */ - uint32_t line; - - /** - * Comment string reference. - */ - android::ResStringPool_ref comment; -}; - -struct Public_entry { - uint16_t entryId; - - enum : uint16_t { - kUndefined = 0, - kPublic = 1, - kPrivate = 2, - }; - - uint16_t state; - android::ResStringPool_ref key; - ResTable_entry_source source; -}; - -/** - * A chunk with type RES_TABLE_SYMBOL_TABLE_TYPE. - * Following the header are count number of SymbolTable_entry - * structures, followed by an android::ResStringPool_header. - */ -struct SymbolTable_header { - android::ResChunk_header header; - - /** - * Number of SymbolTable_entry structures following - * this header. - */ - uint32_t count; -}; - -struct SymbolTable_entry { - /** - * Offset from the beginning of the resource table - * where the symbol entry is referenced. - */ - uint32_t offset; - - /** - * The index into the string pool where the name of this - * symbol exists. - */ - android::ResStringPool_ref name; -}; - -/** * An alternative struct to use instead of ResTable_map_entry. This one is a standard_layout * struct. */ diff --git a/tools/aapt2/flatten/TableFlattener.cpp b/tools/aapt2/flatten/TableFlattener.cpp index 26d7c2ca055c..71ab3dbf52ca 100644 --- a/tools/aapt2/flatten/TableFlattener.cpp +++ b/tools/aapt2/flatten/TableFlattener.cpp @@ -51,157 +51,49 @@ static void strcpy16_htod(uint16_t* dst, size_t len, const StringPiece16& src) { dst[i] = 0; } +static bool cmpStyleEntries(const Style::Entry& a, const Style::Entry& b) { + if (a.key.id) { + if (b.key.id) { + return a.key.id.value() < b.key.id.value(); + } + return true; + } else if (!b.key.id) { + return a.key.name.value() < b.key.name.value(); + } + return false; +} + struct FlatEntry { ResourceEntry* entry; Value* value; // The entry string pool index to the entry's name. uint32_t entryKey; - - // The source string pool index to the source file path. - uint32_t sourcePathKey; - uint32_t sourceLine; - - // The source string pool index to the comment. - uint32_t commentKey; }; -class SymbolWriter { +class MapFlattenVisitor : public RawValueVisitor { public: - struct Entry { - StringPool::Ref name; - size_t offset; - }; - - std::vector symbols; - - explicit SymbolWriter(StringPool* pool) : mPool(pool) { - } - - void addSymbol(const Reference& ref, size_t offset) { - const ResourceName& name = ref.name.value(); - std::u16string fullName; - if (ref.privateReference) { - fullName += u"*"; - } - - if (!name.package.empty()) { - fullName += name.package + u":"; - } - fullName += toString(name.type).toString() + u"/" + name.entry; - symbols.push_back(Entry{ mPool->makeRef(fullName), offset }); - } - - void shiftAllOffsets(size_t offset) { - for (Entry& entry : symbols) { - entry.offset += offset; - } - } - -private: - StringPool* mPool; -}; - -struct MapFlattenVisitor : public RawValueVisitor { using RawValueVisitor::visit; - SymbolWriter* mSymbols; - FlatEntry* mEntry; - BigBuffer* mBuffer; - StringPool* mSourcePool; - StringPool* mCommentPool; - bool mUseExtendedChunks; - - size_t mEntryCount = 0; - const Reference* mParent = nullptr; - - MapFlattenVisitor(SymbolWriter* symbols, FlatEntry* entry, BigBuffer* buffer, - StringPool* sourcePool, StringPool* commentPool, - bool useExtendedChunks) : - mSymbols(symbols), mEntry(entry), mBuffer(buffer), mSourcePool(sourcePool), - mCommentPool(commentPool), mUseExtendedChunks(useExtendedChunks) { - } - - void flattenKey(Reference* key, ResTable_map* outEntry) { - if (!key->id || (key->privateReference && mUseExtendedChunks)) { - assert(key->name && "reference must have a name"); - - outEntry->name.ident = util::hostToDevice32(0); - mSymbols->addSymbol(*key, (mBuffer->size() - sizeof(ResTable_map)) + - offsetof(ResTable_map, name)); - } else { - outEntry->name.ident = util::hostToDevice32(key->id.value().id); - } - } - - void flattenValue(Item* value, ResTable_map* outEntry) { - bool privateRef = false; - if (Reference* ref = valueCast(value)) { - privateRef = ref->privateReference && mUseExtendedChunks; - if (!ref->id || privateRef) { - assert(ref->name && "reference must have a name"); - - mSymbols->addSymbol(*ref, (mBuffer->size() - sizeof(ResTable_map)) + - offsetof(ResTable_map, value) + offsetof(Res_value, data)); - } - } - - bool result = value->flatten(&outEntry->value); - if (privateRef) { - outEntry->value.data = 0; - } - assert(result && "flatten failed"); - } - - void flattenEntry(Reference* key, Item* value) { - ResTable_map* outEntry = mBuffer->nextBlock(); - flattenKey(key, outEntry); - flattenValue(value, outEntry); - outEntry->value.size = util::hostToDevice16(sizeof(outEntry->value)); - mEntryCount++; - } - - void flattenMetaData(Value* value) { - if (!mUseExtendedChunks) { - return; - } - - Reference key(ResourceId{ ExtendedResTableMapTypes::ATTR_SOURCE_PATH }); - StringPool::Ref sourcePathRef = mSourcePool->makeRef( - util::utf8ToUtf16(value->getSource().path)); - BinaryPrimitive val(Res_value::TYPE_INT_DEC, - static_cast(sourcePathRef.getIndex())); - flattenEntry(&key, &val); - - if (value->getSource().line) { - key.id = ResourceId(ExtendedResTableMapTypes::ATTR_SOURCE_LINE); - val.value.data = static_cast(value->getSource().line.value()); - flattenEntry(&key, &val); - } - - if (!value->getComment().empty()) { - key.id = ResourceId(ExtendedResTableMapTypes::ATTR_COMMENT); - StringPool::Ref commentRef = mCommentPool->makeRef(value->getComment()); - val.value.data = static_cast(commentRef.getIndex()); - flattenEntry(&key, &val); - } + MapFlattenVisitor(ResTable_entry_ext* outEntry, BigBuffer* buffer) : + mOutEntry(outEntry), mBuffer(buffer) { } void visit(Attribute* attr) override { { - Reference key(ResourceId{ ResTable_map::ATTR_TYPE }); + Reference key = Reference(ResTable_map::ATTR_TYPE); BinaryPrimitive val(Res_value::TYPE_INT_DEC, attr->typeMask); flattenEntry(&key, &val); } if (attr->minInt != std::numeric_limits::min()) { - Reference key(ResourceId{ ResTable_map::ATTR_MIN }); + Reference key = Reference(ResTable_map::ATTR_MIN); BinaryPrimitive val(Res_value::TYPE_INT_DEC, static_cast(attr->minInt)); flattenEntry(&key, &val); } if (attr->maxInt != std::numeric_limits::max()) { - Reference key(ResourceId{ ResTable_map::ATTR_MAX }); + Reference key = Reference(ResTable_map::ATTR_MAX); BinaryPrimitive val(Res_value::TYPE_INT_DEC, static_cast(attr->maxInt)); flattenEntry(&key, &val); } @@ -212,22 +104,11 @@ struct MapFlattenVisitor : public RawValueVisitor { } } - static bool cmpStyleEntries(const Style::Entry& a, const Style::Entry& b) { - if (a.key.id) { - if (b.key.id) { - return a.key.id.value() < b.key.id.value(); - } - return true; - } else if (!b.key.id) { - return a.key.name.value() < b.key.name.value(); - } - return false; - } - void visit(Style* style) override { if (style->parent) { - // Parents are treated a bit differently, so record the existence and move on. - mParent = &style->parent.value(); + const Reference& parentRef = style->parent.value(); + assert(parentRef.id && "parent has no ID"); + mOutEntry->parent.ident = util::hostToDevice32(parentRef.id.value().id); } // Sort the style. @@ -235,7 +116,6 @@ struct MapFlattenVisitor : public RawValueVisitor { for (Style::Entry& entry : style->entries) { flattenEntry(&entry.key, entry.value.get()); - flattenMetaData(&entry.key); } } @@ -243,8 +123,8 @@ struct MapFlattenVisitor : public RawValueVisitor { for (auto& attrRef : styleable->entries) { BinaryPrimitive val(Res_value{}); flattenEntry(&attrRef, &val); - flattenMetaData(&attrRef); } + } void visit(Array* array) override { @@ -253,7 +133,6 @@ struct MapFlattenVisitor : public RawValueVisitor { flattenValue(item.get(), outEntry); outEntry->value.size = util::hostToDevice16(sizeof(outEntry->value)); mEntryCount++; - flattenMetaData(item.get()); } } @@ -297,18 +176,45 @@ struct MapFlattenVisitor : public RawValueVisitor { Reference key(q); flattenEntry(&key, plural->values[i].get()); - flattenMetaData(plural->values[i].get()); } } + + /** + * Call this after visiting a Value. This will finish any work that + * needs to be done to prepare the entry. + */ + void finish() { + mOutEntry->count = util::hostToDevice32(mEntryCount); + } + +private: + void flattenKey(Reference* key, ResTable_map* outEntry) { + assert(key->id && "key has no ID"); + outEntry->name.ident = util::hostToDevice32(key->id.value().id); + } + + void flattenValue(Item* value, ResTable_map* outEntry) { + bool result = value->flatten(&outEntry->value); + assert(result && "flatten failed"); + } + + void flattenEntry(Reference* key, Item* value) { + ResTable_map* outEntry = mBuffer->nextBlock(); + flattenKey(key, outEntry); + flattenValue(value, outEntry); + outEntry->value.size = util::hostToDevice16(sizeof(outEntry->value)); + mEntryCount++; + } + + ResTable_entry_ext* mOutEntry; + BigBuffer* mBuffer; + size_t mEntryCount = 0; }; class PackageFlattener { public: - PackageFlattener(IDiagnostics* diag, TableFlattenerOptions options, - ResourceTablePackage* package, SymbolWriter* symbolWriter, - StringPool* sourcePool) : - mDiag(diag), mOptions(options), mPackage(package), mSymbols(symbolWriter), - mSourcePool(sourcePool) { + PackageFlattener(IDiagnostics* diag, ResourceTablePackage* package) : + mDiag(diag), mPackage(package) { } bool flattenPackage(BigBuffer* buffer) { @@ -337,9 +243,6 @@ public: pkgHeader->keyStrings = util::hostToDevice32(pkgWriter.size()); StringPool::flattenUtf16(pkgWriter.getBuffer(), mKeyPool); - // Add the ResTable_package header/type/key strings to the offset. - mSymbols->shiftAllOffsets(pkgWriter.size()); - // Append the types. buffer->appendBuffer(std::move(typeBuffer)); @@ -349,12 +252,9 @@ public: private: IDiagnostics* mDiag; - TableFlattenerOptions mOptions; ResourceTablePackage* mPackage; StringPool mTypePool; StringPool mKeyPool; - SymbolWriter* mSymbols; - StringPool* mSourcePool; template T* writeEntry(FlatEntry* entry, BigBuffer* buffer) { @@ -376,62 +276,24 @@ private: outEntry->flags |= ResTable_entry::FLAG_COMPLEX; } - outEntry->key.index = util::hostToDevice32(entry->entryKey); - outEntry->size = sizeof(T); - - if (mOptions.useExtendedChunks) { - // Write the extra source block. This will be ignored by the Android runtime. - ResTable_entry_source* sourceBlock = buffer->nextBlock(); - sourceBlock->path.index = util::hostToDevice32(entry->sourcePathKey); - sourceBlock->line = util::hostToDevice32(entry->sourceLine); - sourceBlock->comment.index = util::hostToDevice32(entry->commentKey); - outEntry->size += sizeof(*sourceBlock); - } - outEntry->flags = util::hostToDevice16(outEntry->flags); - outEntry->size = util::hostToDevice16(outEntry->size); + outEntry->key.index = util::hostToDevice32(entry->entryKey); + outEntry->size = util::hostToDevice16(sizeof(T)); return result; } bool flattenValue(FlatEntry* entry, BigBuffer* buffer) { if (Item* item = valueCast(entry->value)) { writeEntry(entry, buffer); - bool privateRef = false; - if (Reference* ref = valueCast(entry->value)) { - // If there is no ID or the reference is private and we allow extended chunks, - // write out a 0 and mark the symbol table with the name of the reference. - privateRef = (ref->privateReference && mOptions.useExtendedChunks); - if (!ref->id || privateRef) { - assert(ref->name && "reference must have at least a name"); - mSymbols->addSymbol(*ref, buffer->size() + offsetof(Res_value, data)); - } - } Res_value* outValue = buffer->nextBlock(); bool result = item->flatten(outValue); assert(result && "flatten failed"); - if (privateRef) { - // Force the value of 0 so we look up the symbol at unflatten time. - outValue->data = 0; - } outValue->size = util::hostToDevice16(sizeof(*outValue)); } else { - const size_t beforeEntry = buffer->size(); ResTable_entry_ext* outEntry = writeEntry(entry, buffer); - MapFlattenVisitor visitor(mSymbols, entry, buffer, mSourcePool, mSourcePool, - mOptions.useExtendedChunks); + MapFlattenVisitor visitor(outEntry, buffer); entry->value->accept(&visitor); - outEntry->count = util::hostToDevice32(visitor.mEntryCount); - if (visitor.mParent) { - const bool forceSymbol = visitor.mParent->privateReference && - mOptions.useExtendedChunks; - if (!visitor.mParent->id || forceSymbol) { - assert(visitor.mParent->name && "reference must have a name"); - mSymbols->addSymbol(*visitor.mParent, - beforeEntry + offsetof(ResTable_entry_ext, parent)); - } else { - outEntry->parent.ident = util::hostToDevice32(visitor.mParent->id.value().id); - } - } + visitor.finish(); } return true; } @@ -480,7 +342,7 @@ private: std::vector collectAndSortTypes() { std::vector sortedTypes; for (auto& type : mPackage->types) { - if (type->type == ResourceType::kStyleable && !mOptions.useExtendedChunks) { + if (type->type == ResourceType::kStyleable) { // Styleables aren't real Resource Types, they are represented in the R.java // file. continue; @@ -551,52 +413,6 @@ private: return true; } - bool flattenPublic(ResourceTableType* type, std::vector* sortedEntries, - BigBuffer* buffer) { - ChunkWriter publicWriter(buffer); - Public_header* publicHeader = publicWriter.startChunk(RES_TABLE_PUBLIC_TYPE); - publicHeader->typeId = type->id.value(); - - for (ResourceEntry* entry : *sortedEntries) { - if (entry->symbolStatus.state != SymbolState::kUndefined) { - // Write the public status of this entry. - Public_entry* publicEntry = publicWriter.nextBlock(); - publicEntry->entryId = util::hostToDevice32(entry->id.value()); - publicEntry->key.index = util::hostToDevice32(mKeyPool.makeRef( - entry->name).getIndex()); - publicEntry->source.path.index = util::hostToDevice32(mSourcePool->makeRef( - util::utf8ToUtf16(entry->symbolStatus.source.path)).getIndex()); - if (entry->symbolStatus.source.line) { - publicEntry->source.line = util::hostToDevice32( - entry->symbolStatus.source.line.value()); - } - publicEntry->source.comment.index = util::hostToDevice32(mSourcePool->makeRef( - entry->symbolStatus.comment).getIndex()); - - switch (entry->symbolStatus.state) { - case SymbolState::kPrivate: - publicEntry->state = Public_entry::kPrivate; - break; - - case SymbolState::kPublic: - publicEntry->state = Public_entry::kPublic; - break; - - case SymbolState::kUndefined: - publicEntry->state = Public_entry::kUndefined; - break; - } - - // Don't hostToDevice until the last step. - publicHeader->count += 1; - } - } - - publicHeader->count = util::hostToDevice32(publicHeader->count); - publicWriter.finish(); - return true; - } - bool flattenTypes(BigBuffer* buffer) { // Sort the types by their IDs. They will be inserted into the StringPool in this order. std::vector sortedTypes = collectAndSortTypes(); @@ -620,12 +436,6 @@ private: return false; } - if (mOptions.useExtendedChunks) { - if (!flattenPublic(type, &sortedEntries, buffer)) { - return false; - } - } - // The binary resource table lists resource entries for each configuration. // We store them inverted, where a resource entry lists the values for each // configuration available. Here we reverse this to match the binary table. @@ -635,26 +445,8 @@ private: // Group values by configuration. for (auto& configValue : entry->values) { - Value* value = configValue.value.get(); - - const StringPool::Ref sourceRef = mSourcePool->makeRef( - util::utf8ToUtf16(value->getSource().path)); - - uint32_t lineNumber = 0; - if (value->getSource().line) { - lineNumber = value->getSource().line.value(); - } - - const StringPool::Ref commentRef = mSourcePool->makeRef(value->getComment()); - - configToEntryListMap[configValue.config] - .push_back(FlatEntry{ - entry, - value, - keyIndex, - (uint32_t) sourceRef.getIndex(), - lineNumber, - (uint32_t) commentRef.getIndex() }); + configToEntryListMap[configValue.config].push_back(FlatEntry{ + entry, configValue.value.get(), keyIndex }); } } @@ -692,86 +484,18 @@ bool TableFlattener::consume(IAaptContext* context, ResourceTable* table) { // Flatten the values string pool. StringPool::flattenUtf8(tableWriter.getBuffer(), table->stringPool); - // If we have a reference to a symbol that doesn't exist, we don't know its resource ID. - // We encode the name of the symbol along with the offset of where to include the resource ID - // once it is found. - StringPool symbolPool; - std::vector symbolOffsets; - - // String pool holding the source paths of each value. - StringPool sourcePool; - BigBuffer packageBuffer(1024); // Flatten each package. for (auto& package : table->packages) { - const size_t beforePackageSize = packageBuffer.size(); - - // All packages will share a single global symbol pool. - SymbolWriter packageSymbolWriter(&symbolPool); - - PackageFlattener flattener(context->getDiagnostics(), mOptions, package.get(), - &packageSymbolWriter, &sourcePool); + PackageFlattener flattener(context->getDiagnostics(), package.get()); if (!flattener.flattenPackage(&packageBuffer)) { return false; } - - // The symbols are offset only from their own Package start. Offset them from the - // start of the packageBuffer. - packageSymbolWriter.shiftAllOffsets(beforePackageSize); - - // Extract all the symbols to offset - symbolOffsets.insert(symbolOffsets.end(), - std::make_move_iterator(packageSymbolWriter.symbols.begin()), - std::make_move_iterator(packageSymbolWriter.symbols.end())); } - SymbolTable_entry* symbolEntryData = nullptr; - if (mOptions.useExtendedChunks) { - if (!symbolOffsets.empty()) { - // Sort the offsets so we can scan them linearly. - std::sort(symbolOffsets.begin(), symbolOffsets.end(), - [](const SymbolWriter::Entry& a, const SymbolWriter::Entry& b) -> bool { - return a.offset < b.offset; - }); - - // Write the Symbol header. - ChunkWriter symbolWriter(tableWriter.getBuffer()); - SymbolTable_header* symbolHeader = symbolWriter.startChunk( - RES_TABLE_SYMBOL_TABLE_TYPE); - symbolHeader->count = util::hostToDevice32(symbolOffsets.size()); - - symbolEntryData = symbolWriter.nextBlock(symbolOffsets.size()); - StringPool::flattenUtf8(symbolWriter.getBuffer(), symbolPool); - symbolWriter.finish(); - } - - if (sourcePool.size() > 0) { - // Write out source pool. - ChunkWriter srcWriter(tableWriter.getBuffer()); - srcWriter.startChunk(RES_TABLE_SOURCE_POOL_TYPE); - StringPool::flattenUtf8(srcWriter.getBuffer(), sourcePool); - srcWriter.finish(); - } - } - - const size_t beforePackagesSize = tableWriter.size(); - // Finally merge all the packages into the main buffer. tableWriter.getBuffer()->appendBuffer(std::move(packageBuffer)); - - // Update the offsets to their final values. - if (symbolEntryData) { - for (SymbolWriter::Entry& entry : symbolOffsets) { - symbolEntryData->name.index = util::hostToDevice32(entry.name.getIndex()); - - // The symbols were all calculated with the packageBuffer offset. We need to - // add the beginning of the output buffer. - symbolEntryData->offset = util::hostToDevice32(entry.offset + beforePackagesSize); - symbolEntryData++; - } - } - tableWriter.finish(); return true; } diff --git a/tools/aapt2/flatten/TableFlattener.h b/tools/aapt2/flatten/TableFlattener.h index 901b129725ea..0ab01974044b 100644 --- a/tools/aapt2/flatten/TableFlattener.h +++ b/tools/aapt2/flatten/TableFlattener.h @@ -24,28 +24,15 @@ namespace aapt { class BigBuffer; class ResourceTable; -struct TableFlattenerOptions { - /** - * Specifies whether to output extended chunks, like - * source information and missing symbol entries. Default - * is false. - * - * Set this to true when emitting intermediate resource table. - */ - bool useExtendedChunks = false; -}; - class TableFlattener : public IResourceTableConsumer { public: - TableFlattener(BigBuffer* buffer, TableFlattenerOptions options) : - mBuffer(buffer), mOptions(options) { + TableFlattener(BigBuffer* buffer) : mBuffer(buffer) { } bool consume(IAaptContext* context, ResourceTable* table) override; private: BigBuffer* mBuffer; - TableFlattenerOptions mOptions; }; } // namespace aapt diff --git a/tools/aapt2/flatten/TableFlattener_test.cpp b/tools/aapt2/flatten/TableFlattener_test.cpp index 7030603e5bbd..39c4fd318508 100644 --- a/tools/aapt2/flatten/TableFlattener_test.cpp +++ b/tools/aapt2/flatten/TableFlattener_test.cpp @@ -38,9 +38,7 @@ public: ::testing::AssertionResult flatten(ResourceTable* table, ResTable* outTable) { BigBuffer buffer(1024); - TableFlattenerOptions options = {}; - options.useExtendedChunks = true; - TableFlattener flattener(&buffer, options); + TableFlattener flattener(&buffer); if (!flattener.consume(mContext.get(), table)) { return ::testing::AssertionFailure() << "failed to flatten ResourceTable"; } @@ -54,9 +52,7 @@ public: ::testing::AssertionResult flatten(ResourceTable* table, ResourceTable* outTable) { BigBuffer buffer(1024); - TableFlattenerOptions options = {}; - options.useExtendedChunks = true; - TableFlattener flattener(&buffer, options); + TableFlattener flattener(&buffer); if (!flattener.consume(mContext.get(), table)) { return ::testing::AssertionFailure() << "failed to flatten ResourceTable"; } @@ -210,58 +206,6 @@ TEST_F(TableFlattenerTest, FlattenEntriesWithGapsInIds) { Res_value::TYPE_INT_BOOLEAN, 0u, 0u)); } -TEST_F(TableFlattenerTest, FlattenUnlinkedTable) { - std::unique_ptr table = test::ResourceTableBuilder() - .setPackageId(u"com.app.test", 0x7f) - .addValue(u"@com.app.test:integer/one", ResourceId(0x7f020000), - test::buildReference(u"@android:integer/foo")) - .addValue(u"@com.app.test:style/Theme", ResourceId(0x7f030000), test::StyleBuilder() - .setParent(u"@android:style/Theme.Material") - .addItem(u"@android:attr/background", {}) - .addItem(u"@android:attr/colorAccent", - test::buildReference(u"@com.app.test:color/green")) - .build()) - .build(); - - { - // Need access to stringPool to make RawString. - Style* style = test::getValue