[prev in list] [next in list] [prev in thread] [next in thread]
List: cfe-commits
Subject: Re: r242868 - [modules] Stop performing PCM lookups for all identifiers when building with C++ modul
From: Richard Smith <richard () metafoo ! co ! uk>
Date: 2015-07-22 22:51:37
Message-ID: CAOfiQqnQiQtWJCXaEODtQVzGzFetZDeWUVy0hBxGqJ8gOaWy8Q () mail ! gmail ! com
[Download RAW message or body]
[Attachment #2 (multipart/alternative)]
On Wed, Jul 22, 2015 at 3:37 PM, Adam Nemet <anemet@apple.com> wrote:
> Hi Richard,
>
> Looks like this caused some ASan failures:
>
> http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux-fast/builds/5677
>
> Can you please take a look?
>
Thanks, should be fixed in r242960.
> Thanks,
> Adam
>
> On Jul 21, 2015, at 7:08 PM, Richard Smith <richard-llvm@metafoo.co.uk>
> wrote:
>
> Author: rsmith
> Date: Tue Jul 21 21:08:40 2015
> New Revision: 242868
>
> URL: http://llvm.org/viewvc/llvm-project?rev=242868&view=rev
> Log:
> [modules] Stop performing PCM lookups for all identifiers when building
> with C++ modules. Instead, serialize a list of interesting identifiers and
> mark those ones out of date on module import. Avoiding the identifier
> lookups here gives a 20-30% speedup in builds with large numbers of
> modules. No functionality change intended.
>
> Modified:
> cfe/trunk/include/clang/Serialization/ASTBitCodes.h
> cfe/trunk/include/clang/Serialization/Module.h
> cfe/trunk/include/clang/Serialization/ModuleManager.h
> cfe/trunk/lib/Sema/Sema.cpp
> cfe/trunk/lib/Serialization/ASTReader.cpp
> cfe/trunk/lib/Serialization/ASTWriter.cpp
> cfe/trunk/lib/Serialization/ModuleManager.cpp
>
> Modified: cfe/trunk/include/clang/Serialization/ASTBitCodes.h
> URL:
> http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ASTBitCodes.h?rev=242868&r1=242867&r2=242868&view=diff
>
> ==============================================================================
> --- cfe/trunk/include/clang/Serialization/ASTBitCodes.h (original)
> +++ cfe/trunk/include/clang/Serialization/ASTBitCodes.h Tue Jul 21
> 21:08:40 2015
> @@ -543,7 +543,10 @@ namespace clang {
> /// macro definition.
> MACRO_OFFSET = 47,
>
> - // ID 48 used to be a table of macros.
> + /// \brief A list of "interesting" identifiers. Only used in C++
> (where we
> + /// don't normally do lookups into the serialized identifier
> table). These
> + /// are eagerly deserialized.
> + INTERESTING_IDENTIFIERS = 48,
>
> /// \brief Record code for undefined but used functions and
> variables that
> /// need a definition in this TU.
>
> Modified: cfe/trunk/include/clang/Serialization/Module.h
> URL:
> http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/Module.h?rev=242868&r1=242867&r2=242868&view=diff
>
> ==============================================================================
> --- cfe/trunk/include/clang/Serialization/Module.h (original)
> +++ cfe/trunk/include/clang/Serialization/Module.h Tue Jul 21 21:08:40 2015
> @@ -270,6 +270,10 @@ public:
> /// IdentifierHashTable.
> void *IdentifierLookupTable;
>
> + /// \brief Offsets of identifiers that we're going to preload within
> + /// IdentifierTableData.
> + std::vector<unsigned> PreloadIdentifierOffsets;
> +
> // === Macros ===
>
> /// \brief The cursor to the start of the preprocessor block, which
> stores
>
> Modified: cfe/trunk/include/clang/Serialization/ModuleManager.h
> URL:
> http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ModuleManager.h?rev=242868&r1=242867&r2=242868&view=diff
>
>
> ==============================================================================
> --- cfe/trunk/include/clang/Serialization/ModuleManager.h (original)
> +++ cfe/trunk/include/clang/Serialization/ModuleManager.h Tue Jul 21
> 21:08:40 2015
> @@ -30,14 +30,19 @@ namespace serialization {
>
> /// \brief Manages the set of modules loaded by an AST reader.
> class ModuleManager {
> - /// \brief The chain of AST files. The first entry is the one named by
> the
> - /// user, the last one is the one that doesn't depend on anything
> further.
> + /// \brief The chain of AST files, in the order in which we started to
> load
> + /// them (this order isn't really useful for anything).
> SmallVector<ModuleFile *, 2> Chain;
>
> + /// \brief The chain of non-module PCH files. The first entry is the
> one named
> + /// by the user, the last one is the one that doesn't depend on anything
> + /// further.
> + SmallVector<ModuleFile *, 2> PCHChain;
> +
> // \brief The roots of the dependency DAG of AST files. This is used
> // to implement short-circuiting logic when running DFS over the
> dependencies.
> SmallVector<ModuleFile *, 2> Roots;
> -
> +
> /// \brief All loaded modules, indexed by name.
> llvm::DenseMap<const FileEntry *, ModuleFile *> Modules;
>
> @@ -138,6 +143,11 @@ public:
> ModuleReverseIterator rbegin() { return Chain.rbegin(); }
> /// \brief Reverse iterator end-point to traverse all loaded modules.
> ModuleReverseIterator rend() { return Chain.rend(); }
> +
> + /// \brief A range covering the PCH and preamble module files loaded.
> + llvm::iterator_range<ModuleConstIterator> pch_modules() const {
> + return llvm::make_range(PCHChain.begin(), PCHChain.end());
> + }
>
> /// \brief Returns the primary module associated with the manager, that
> is,
> /// the first module loaded
>
> Modified: cfe/trunk/lib/Sema/Sema.cpp
> URL:
> http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Sema/Sema.cpp?rev=242868&r1=242867&r2=242868&view=diff
>
> ==============================================================================
> --- cfe/trunk/lib/Sema/Sema.cpp (original)
> +++ cfe/trunk/lib/Sema/Sema.cpp Tue Jul 21 21:08:40 2015
> @@ -154,6 +154,9 @@ void Sema::Initialize() {
> // will not be able to merge any duplicate __va_list_tag decls correctly.
> VAListTagName = PP.getIdentifierInfo("__va_list_tag");
>
> + if (!TUScope)
> + return;
> +
> // Initialize predefined 128-bit integer types, if needed.
> if (Context.getTargetInfo().hasInt128Type()) {
> // If either of the 128-bit integer types are unavailable to name
> lookup,
>
> Modified: cfe/trunk/lib/Serialization/ASTReader.cpp
> URL:
> http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTReader.cpp?rev=242868&r1=242867&r2=242868&view=diff
>
>
> ==============================================================================
> --- cfe/trunk/lib/Serialization/ASTReader.cpp (original)
> +++ cfe/trunk/lib/Serialization/ASTReader.cpp Tue Jul 21 21:08:40 2015
> @@ -2564,6 +2564,10 @@ ASTReader::ReadASTBlock(ModuleFile &F, u
> break;
> }
>
> + case INTERESTING_IDENTIFIERS:
> + F.PreloadIdentifierOffsets.assign(Record.begin(), Record.end());
> + break;
> +
> case EAGERLY_DESERIALIZED_DECLS:
> // FIXME: Skip reading this record if our ASTConsumer doesn't care
> // about "interesting" decls (for instance, if we're building a
> module).
> @@ -3410,6 +3414,18 @@ ASTReader::ASTReadResult ASTReader::Read
> // SourceManager.
> SourceMgr.getLoadedSLocEntryByID(Index);
> }
> +
> + // Preload all the pending interesting identifiers by marking them
> out of
> + // date.
> + for (auto Offset : F.PreloadIdentifierOffsets) {
> + const unsigned char *Data = reinterpret_cast<const unsigned char *>(
> + F.IdentifierTableData + Offset);
> +
> + ASTIdentifierLookupTrait Trait(*this, F);
> + auto KeyDataLen = Trait.ReadKeyDataLength(Data);
> + auto Key = Trait.ReadKey(Data, KeyDataLen.first);
> + PP.getIdentifierTable().getOwn(Key).setOutOfDate(true);
> + }
> }
>
> // Setup the import locations and notify the module manager that we've
> @@ -3430,13 +3446,20 @@ ASTReader::ASTReadResult ASTReader::Read
> M->ImportLoc.getRawEncoding());
> }
>
> - // Mark all of the identifiers in the identifier table as being out of
> date,
> - // so that various accessors know to check the loaded modules when the
> - // identifier is used.
> - for (IdentifierTable::iterator Id = PP.getIdentifierTable().begin(),
> - IdEnd = PP.getIdentifierTable().end();
> - Id != IdEnd; ++Id)
> - Id->second->setOutOfDate(true);
> + if (!Context.getLangOpts().CPlusPlus ||
> + (Type != MK_ImplicitModule && Type != MK_ExplicitModule)) {
> + // Mark all of the identifiers in the identifier table as being out
> of date,
> + // so that various accessors know to check the loaded modules when the
> + // identifier is used.
> + //
> + // For C++ modules, we don't need information on many identifiers
> (just
> + // those that provide macros or are poisoned), so we mark all of
> + // the interesting ones via PreloadIdentifierOffsets.
> + for (IdentifierTable::iterator Id = PP.getIdentifierTable().begin(),
> + IdEnd = PP.getIdentifierTable().end();
> + Id != IdEnd; ++Id)
> + Id->second->setOutOfDate(true);
> + }
>
> // Resolve any unresolved module exports.
> for (unsigned I = 0, N = UnresolvedModuleRefs.size(); I != N; ++I) {
> @@ -6827,19 +6850,32 @@ IdentifierInfo *ASTReader::get(StringRef
> // Note that we are loading an identifier.
> Deserializing AnIdentifier(this);
>
> - // If there is a global index, look there first to determine which
> modules
> - // provably do not have any results for this identifier.
> - GlobalModuleIndex::HitSet Hits;
> - GlobalModuleIndex::HitSet *HitsPtr = nullptr;
> - if (!loadGlobalIndex()) {
> - if (GlobalIndex->lookupIdentifier(Name, Hits)) {
> - HitsPtr = &Hits;
> - }
> - }
> IdentifierLookupVisitor Visitor(Name, /*PriorGeneration=*/0,
> NumIdentifierLookups,
> NumIdentifierLookupHits);
> - ModuleMgr.visit(IdentifierLookupVisitor::visit, &Visitor, HitsPtr);
> +
> + // We don't need to do identifier table lookups in C++ modules (we
> preload
> + // all interesting declarations, and don't need to use the scope for
> name
> + // lookups). Perform the lookup in PCH files, though, since we don't
> build
> + // a complete initial identifier table if we're carrying on from a PCH.
> + if (Context.getLangOpts().CPlusPlus) {
> + for (auto F : ModuleMgr.pch_modules())
> + if (Visitor.visit(*F, &Visitor))
> + break;
> + } else {
> + // If there is a global index, look there first to determine which
> modules
> + // provably do not have any results for this identifier.
> + GlobalModuleIndex::HitSet Hits;
> + GlobalModuleIndex::HitSet *HitsPtr = nullptr;
> + if (!loadGlobalIndex()) {
> + if (GlobalIndex->lookupIdentifier(Name, Hits)) {
> + HitsPtr = &Hits;
> + }
> + }
> +
> + ModuleMgr.visit(IdentifierLookupVisitor::visit, &Visitor, HitsPtr);
> + }
> +
> IdentifierInfo *II = Visitor.getIdentifierInfo();
> markIdentifierUpToDate(II);
> return II;
>
> Modified: cfe/trunk/lib/Serialization/ASTWriter.cpp
> URL:
> http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTWriter.cpp?rev=242868&r1=242867&r2=242868&view=diff
>
>
> ==============================================================================
> --- cfe/trunk/lib/Serialization/ASTWriter.cpp (original)
> +++ cfe/trunk/lib/Serialization/ASTWriter.cpp Tue Jul 21 21:08:40 2015
> @@ -3104,6 +3104,7 @@ class ASTIdentifierTableTrait {
> IdentifierResolver &IdResolver;
> bool IsModule;
> bool NeedDecls;
> + ASTWriter::RecordData *InterestingIdentifierOffsets;
>
> /// \brief Determines whether this is an "interesting" identifier that
> needs a
> /// full IdentifierInfo structure written into the hash table. Notably,
> this
> @@ -3131,14 +3132,20 @@ public:
> typedef unsigned offset_type;
>
> ASTIdentifierTableTrait(ASTWriter &Writer, Preprocessor &PP,
> - IdentifierResolver &IdResolver, bool IsModule)
> + IdentifierResolver &IdResolver, bool IsModule,
> + ASTWriter::RecordData
> *InterestingIdentifierOffsets)
> > Writer(Writer), PP(PP), IdResolver(IdResolver), IsModule(IsModule),
> - NeedDecls(!IsModule || !Writer.getLangOpts().CPlusPlus) {}
> + NeedDecls(!IsModule || !Writer.getLangOpts().CPlusPlus),
> + InterestingIdentifierOffsets(InterestingIdentifierOffsets) {}
>
> static hash_value_type ComputeHash(const IdentifierInfo* II) {
> return llvm::HashString(II->getName());
> }
>
> + bool isInterestingIdentifier(const IdentifierInfo *II) {
> + auto MacroOffset = Writer.getMacroDirectivesOffset(II);
> + return isInterestingIdentifier(II, MacroOffset);
> + }
> bool isInterestingNonMacroIdentifier(const IdentifierInfo *II) {
> return isInterestingIdentifier(II, 0);
> }
> @@ -3178,6 +3185,12 @@ public:
> // Record the location of the key data. This is used when generating
> // the mapping from persistent IDs to strings.
> Writer.SetIdentifierOffset(II, Out.tell());
> +
> + // Emit the offset of the key/data length information to the
> interesting
> + // identifiers table if necessary.
> + if (InterestingIdentifierOffsets && isInterestingIdentifier(II))
> + InterestingIdentifierOffsets->push_back(Out.tell() - 4);
> +
> Out.write(II->getNameStart(), KeyLen);
> }
>
> @@ -3238,11 +3251,15 @@ void ASTWriter::WriteIdentifierTable(Pre
> bool IsModule) {
> using namespace llvm;
>
> + RecordData InterestingIdents;
> +
> // Create and write out the blob that contains the identifier
> // strings.
> {
> llvm::OnDiskChainedHashTableGenerator<ASTIdentifierTableTrait>
> Generator;
> - ASTIdentifierTableTrait Trait(*this, PP, IdResolver, IsModule);
> + ASTIdentifierTableTrait Trait(
> + *this, PP, IdResolver, IsModule,
> + (getLangOpts().CPlusPlus && IsModule) ? &InterestingIdents :
> nullptr);
>
> // Look for any identifiers that were named while processing the
> // headers, but are otherwise not needed. We add these to the hash
> @@ -3316,6 +3333,11 @@ void ASTWriter::WriteIdentifierTable(Pre
> Record.push_back(FirstIdentID - NUM_PREDEF_IDENT_IDS);
> Stream.EmitRecordWithBlob(IdentifierOffsetAbbrev, Record,
> bytes(IdentifierOffsets));
> +
> + // In C++, write the list of interesting identifiers (those that are
> + // defined as macros, poisoned, or similar unusual things).
> + if (!InterestingIdents.empty())
> + Stream.EmitRecord(INTERESTING_IDENTIFIERS, InterestingIdents);
> }
>
>
> //===----------------------------------------------------------------------===//
>
> Modified: cfe/trunk/lib/Serialization/ModuleManager.cpp
> URL:
> http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ModuleManager.cpp?rev=242868&r1=242867&r2=242868&view=diff
>
>
> ==============================================================================
> --- cfe/trunk/lib/Serialization/ModuleManager.cpp (original)
> +++ cfe/trunk/lib/Serialization/ModuleManager.cpp Tue Jul 21 21:08:40 2015
> @@ -95,6 +95,8 @@ ModuleManager::addModule(StringRef FileN
> New->File = Entry;
> New->ImportLoc = ImportLoc;
> Chain.push_back(New);
> + if (!New->isModule())
> + PCHChain.push_back(New);
> if (!ImportedBy)
> Roots.push_back(New);
> NewModule = true;
> @@ -159,6 +161,8 @@ ModuleManager::addModule(StringRef FileN
> Modules.erase(Entry);
> assert(Chain.back() == ModuleEntry);
> Chain.pop_back();
> + if (!ModuleEntry->isModule())
> + PCHChain.pop_back();
> if (Roots.back() == ModuleEntry)
> Roots.pop_back();
> else
> @@ -225,6 +229,15 @@ void ModuleManager::removeModules(
>
> // Remove the modules from the chain.
> Chain.erase(first, last);
> +
> + // Also remove them from the PCH chain.
> + for (auto I = first; I != last; ++I) {
> + if (!(*I)->isModule()) {
> + PCHChain.erase(std::find(PCHChain.begin(), PCHChain.end(), *I),
> + PCHChain.end());
> + break;
> + }
> + }
> }
>
> void
>
>
> _______________________________________________
> cfe-commits mailing list
> cfe-commits@cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
>
>
>
[Attachment #5 (text/html)]
<div dir="ltr"><div class="gmail_extra"><div class="gmail_quote">On Wed, Jul 22, 2015 at 3:37 \
PM, Adam Nemet <span dir="ltr"><<a href="mailto:anemet@apple.com" \
target="_blank">anemet@apple.com</a>></span> wrote:<br><blockquote class="gmail_quote" \
style="margin:0px 0px 0px \
0.8ex;border-left-width:1px;border-left-color:rgb(204,204,204);border-left-style:solid;padding-left:1ex"><div \
style="word-wrap:break-word"><div>Hi Richard,</div><div><br></div><div>Looks like this caused \
some ASan failures:</div><div><br></div><div><a \
href="https://urldefense.proofpoint.com/v2/url?u=http-3A__lab.llvm.org-3A8011_builders_sanitizer \
-2Dx86-5F64-2Dlinux-2Dfast_builds_5677&d=AwMFaQ&c=8hUWFZcy2Z-Za5rBPlktOQ&r=BSqEv9KvKMW_Ob8SyngJ7 \
0KdZISM_ASROnREeq0cCxk&m=96E3-MBcA8Hx2Rr1FY_OAxgwCQXuu49G7vPan0Ijzl4&s=m6FCA4gEDuTbDv6J0EqEpf3zedKOgVOz4XYV-gMcLdc&e=" \
target="_blank">http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux-fast/builds/5677</a></div><div><br></div><div>Can \
you please take a look?</div></div></blockquote><div><br></div><div>Thanks, should be fixed in \
r242960.</div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px \
0.8ex;border-left-width:1px;border-left-color:rgb(204,204,204);border-left-style:solid;padding-left:1ex"><div \
style="word-wrap:break-word"><div>Thanks,</div><div>Adam</div><br><div><blockquote \
type="cite"><span class=""><div>On Jul 21, 2015, at 7:08 PM, Richard Smith <<a \
href="mailto:richard-llvm@metafoo.co.uk" target="_blank">richard-llvm@metafoo.co.uk</a>> \
wrote:</div><br></span><div><span class="">Author: rsmith<br>Date: Tue Jul 21 21:08:40 \
2015<br>New Revision: 242868<br><br></span>URL: <a \
href="https://urldefense.proofpoint.com/v2/url?u=http-3A__llvm.org_viewvc_llvm-2Dproject-3Frev-3 \
D242868-26view-3Drev&d=AwMFaQ&c=8hUWFZcy2Z-Za5rBPlktOQ&r=BSqEv9KvKMW_Ob8SyngJ70KdZISM_ASROnREeq0 \
cCxk&m=96E3-MBcA8Hx2Rr1FY_OAxgwCQXuu49G7vPan0Ijzl4&s=uq5huht3cbl23LDURcYCAXM59u3cMw37ehJV_uYKGh8&e=" \
target="_blank">http://llvm.org/viewvc/llvm-project?rev=242868&view=rev</a><span \
class=""><br>Log:<br>[modules] Stop performing PCM lookups for all identifiers when building \
with C++ modules. Instead, serialize a list of interesting identifiers and mark those ones out \
of date on module import. Avoiding the identifier lookups here gives a 20-30% speedup in builds \
with large numbers of modules. No functionality change intended.<br><br>Modified:<br> \
cfe/trunk/include/clang/Serialization/ASTBitCodes.h<br> \
cfe/trunk/include/clang/Serialization/Module.h<br> \
cfe/trunk/include/clang/Serialization/ModuleManager.h<br> cfe/trunk/lib/Sema/Sema.cpp<br> \
cfe/trunk/lib/Serialization/ASTReader.cpp<br> \
cfe/trunk/lib/Serialization/ASTWriter.cpp<br> \
cfe/trunk/lib/Serialization/ModuleManager.cpp<br><br>Modified: \
cfe/trunk/include/clang/Serialization/ASTBitCodes.h<br></span>URL: <a \
href="https://urldefense.proofpoint.com/v2/url?u=http-3A__llvm.org_viewvc_llvm-2Dproject_cfe_tru \
nk_include_clang_Serialization_ASTBitCodes.h-3Frev-3D242868-26r1-3D242867-26r2-3D242868-26view-3 \
Ddiff&d=AwMFaQ&c=8hUWFZcy2Z-Za5rBPlktOQ&r=BSqEv9KvKMW_Ob8SyngJ70KdZISM_ASROnREeq0cCxk&m=96E3-MBc \
A8Hx2Rr1FY_OAxgwCQXuu49G7vPan0Ijzl4&s=KkjXG_Ae8YmE-Vvy2lQpRxNjhPe17FJqL4ygTXMWbEo&e=" \
target="_blank">http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ASTBitCodes.h?rev=242868&r1=242867&r2=242868&view=diff</a><span \
class=""><br>==============================================================================<br>--- \
cfe/trunk/include/clang/Serialization/ASTBitCodes.h (original)<br>+++ \
cfe/trunk/include/clang/Serialization/ASTBitCodes.h Tue Jul 21 21:08:40 2015<br>@@ -543,7 \
+543,10 @@ namespace clang {<br> /// macro definition.<br> MACRO_OFFSET \
= 47,<br><br>- // ID 48 used to be a table of macros.<br>+ /// \brief A \
list of "interesting" identifiers. Only used in C++ (where we<br>+ /// \
don't normally do lookups into the serialized identifier table). These<br>+ /// \
are eagerly deserialized.<br>+ INTERESTING_IDENTIFIERS = 48,<br><br> /// \
\brief Record code for undefined but used functions and variables that<br> /// need \
a definition in this TU.<br><br>Modified: \
cfe/trunk/include/clang/Serialization/Module.h<br></span>URL: <a \
href="https://urldefense.proofpoint.com/v2/url?u=http-3A__llvm.org_viewvc_llvm-2Dproject_cfe_tru \
nk_include_clang_Serialization_Module.h-3Frev-3D242868-26r1-3D242867-26r2-3D242868-26view-3Ddiff \
&d=AwMFaQ&c=8hUWFZcy2Z-Za5rBPlktOQ&r=BSqEv9KvKMW_Ob8SyngJ70KdZISM_ASROnREeq0cCxk&m=96E3-MBcA8Hx2Rr1FY_OAxgwCQXuu49G7vPan0Ijzl4&s=1PKFApCg6YE9l6icFqb6vsxAoJb7tn4BqijU7oj4ipw&e=" \
target="_blank">http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/Module.h?rev=242868&r1=242867&r2=242868&view=diff</a><span \
class=""><br>==============================================================================<br>--- \
cfe/trunk/include/clang/Serialization/Module.h (original)<br>+++ \
cfe/trunk/include/clang/Serialization/Module.h Tue Jul 21 21:08:40 2015<br>@@ -270,6 +270,10 @@ \
public:<br> /// IdentifierHashTable.<br> void *IdentifierLookupTable;<br><br>+ /// \
\brief Offsets of identifiers that we're going to preload within<br>+ /// \
IdentifierTableData.<br>+ std::vector<unsigned> PreloadIdentifierOffsets;<br>+<br> \
// === Macros ===<br><br> /// \brief The cursor to the start of the preprocessor block, \
which stores<br><br>Modified: \
cfe/trunk/include/clang/Serialization/ModuleManager.h<br></span>URL: <a \
href="https://urldefense.proofpoint.com/v2/url?u=http-3A__llvm.org_viewvc_llvm-2Dproject_cfe_tru \
nk_include_clang_Serialization_ModuleManager.h-3Frev-3D242868-26r1-3D242867-26r2-3D242868-26view \
-3Ddiff&d=AwMFaQ&c=8hUWFZcy2Z-Za5rBPlktOQ&r=BSqEv9KvKMW_Ob8SyngJ70KdZISM_ASROnREeq0cCxk&m=96E3-M \
BcA8Hx2Rr1FY_OAxgwCQXuu49G7vPan0Ijzl4&s=ofPMRJKo_Dnz2-ecDqsjF59O_yDHDc-PcxC0yRXAvxs&e=" \
target="_blank">http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ModuleManager.h?rev=242868&r1=242867&r2=242868&view=diff</a><div><div \
class="h5"><br>==============================================================================<br>--- \
cfe/trunk/include/clang/Serialization/ModuleManager.h (original)<br>+++ \
cfe/trunk/include/clang/Serialization/ModuleManager.h Tue Jul 21 21:08:40 2015<br>@@ -30,14 \
+30,19 @@ namespace serialization {<br><br> /// \brief Manages the set of modules loaded by an \
AST reader.<br> class ModuleManager {<br>- /// \brief The chain of AST files. The first entry \
is the one named by the<br>- /// user, the last one is the one that doesn't depend on \
anything further.<br>+ /// \brief The chain of AST files, in the order in which we started to \
load<br>+ /// them (this order isn't really useful for anything).<br> \
SmallVector<ModuleFile *, 2> Chain;<br><br>+ /// \brief The chain of non-module PCH \
files. The first entry is the one named<br>+ /// by the user, the last one is the one that \
doesn't depend on anything<br>+ /// further.<br>+ SmallVector<ModuleFile *, 2> \
PCHChain;<br>+<br> // \brief The roots of the dependency DAG of AST files. This is used<br> \
// to implement short-circuiting logic when running DFS over the dependencies.<br> \
SmallVector<ModuleFile *, 2> Roots;<br>- <br>+<br> /// \brief All loaded modules, \
indexed by name.<br> llvm::DenseMap<const FileEntry *, ModuleFile *> \
Modules;<br><br>@@ -138,6 +143,11 @@ public:<br> ModuleReverseIterator rbegin() { return \
Chain.rbegin(); }<br> /// \brief Reverse iterator end-point to traverse all loaded \
modules.<br> ModuleReverseIterator rend() { return Chain.rend(); }<br>+<br>+ /// \brief A \
range covering the PCH and preamble module files loaded.<br>+ \
llvm::iterator_range<ModuleConstIterator> pch_modules() const {<br>+ return \
llvm::make_range(PCHChain.begin(), PCHChain.end());<br>+ }<br><br> /// \brief Returns the \
primary module associated with the manager, that is,<br> /// the first module \
loaded<br><br>Modified: cfe/trunk/lib/Sema/Sema.cpp<br></div></div>URL: <a \
href="https://urldefense.proofpoint.com/v2/url?u=http-3A__llvm.org_viewvc_llvm-2Dproject_cfe_tru \
nk_lib_Sema_Sema.cpp-3Frev-3D242868-26r1-3D242867-26r2-3D242868-26view-3Ddiff&d=AwMFaQ&c=8hUWFZc \
y2Z-Za5rBPlktOQ&r=BSqEv9KvKMW_Ob8SyngJ70KdZISM_ASROnREeq0cCxk&m=96E3-MBcA8Hx2Rr1FY_OAxgwCQXuu49G7vPan0Ijzl4&s=OXjMVzaeYR6o-Bs7Jm-HZxl6s7ZignEtTCnnP2tLc2Q&e=" \
target="_blank">http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Sema/Sema.cpp?rev=242868&r1=242867&r2=242868&view=diff</a><span \
class=""><br>==============================================================================<br>--- \
cfe/trunk/lib/Sema/Sema.cpp (original)<br>+++ cfe/trunk/lib/Sema/Sema.cpp Tue Jul 21 21:08:40 \
2015<br>@@ -154,6 +154,9 @@ void Sema::Initialize() {<br> // will not be able to merge any \
duplicate __va_list_tag decls correctly.<br> VAListTagName = \
PP.getIdentifierInfo("__va_list_tag");<br><br>+ if (!TUScope)<br>+ \
return;<br>+<br> // Initialize predefined 128-bit integer types, if needed.<br> if \
(Context.getTargetInfo().hasInt128Type()) {<br> // If either of the 128-bit integer \
types are unavailable to name lookup,<br><br>Modified: \
cfe/trunk/lib/Serialization/ASTReader.cpp<br></span>URL: <a \
href="https://urldefense.proofpoint.com/v2/url?u=http-3A__llvm.org_viewvc_llvm-2Dproject_cfe_tru \
nk_lib_Serialization_ASTReader.cpp-3Frev-3D242868-26r1-3D242867-26r2-3D242868-26view-3Ddiff&d=Aw \
MFaQ&c=8hUWFZcy2Z-Za5rBPlktOQ&r=BSqEv9KvKMW_Ob8SyngJ70KdZISM_ASROnREeq0cCxk&m=96E3-MBcA8Hx2Rr1FY_OAxgwCQXuu49G7vPan0Ijzl4&s=1l1uYEfZYqi9WJj3C8VcaiUzZveNpiq2w7pfQEyL16M&e=" \
target="_blank">http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTReader.cpp?rev=242868&r1=242867&r2=242868&view=diff</a><div><div \
class="h5"><br>==============================================================================<br>--- \
cfe/trunk/lib/Serialization/ASTReader.cpp (original)<br>+++ \
cfe/trunk/lib/Serialization/ASTReader.cpp Tue Jul 21 21:08:40 2015<br>@@ -2564,6 +2564,10 @@ \
ASTReader::ReadASTBlock(ModuleFile &F, u<br> break;<br> }<br><br>+ \
case INTERESTING_IDENTIFIERS:<br>+ F.PreloadIdentifierOffsets.assign(Record.begin(), \
Record.end());<br>+ break;<br>+<br> case EAGERLY_DESERIALIZED_DECLS:<br> \
// FIXME: Skip reading this record if our ASTConsumer doesn't care<br> // about \
"interesting" decls (for instance, if we're building a module).<br>@@ -3410,6 \
+3414,18 @@ ASTReader::ASTReadResult ASTReader::Read<br> // SourceManager.<br> \
SourceMgr.getLoadedSLocEntryByID(Index);<br> }<br>+<br>+ // Preload all the \
pending interesting identifiers by marking them out of<br>+ // date.<br>+ for (auto \
Offset : F.PreloadIdentifierOffsets) {<br>+ const unsigned char *Data = \
reinterpret_cast<const unsigned char *>(<br>+ F.IdentifierTableData + \
Offset);<br>+<br>+ ASTIdentifierLookupTrait Trait(*this, F);<br>+ auto \
KeyDataLen = Trait.ReadKeyDataLength(Data);<br>+ auto Key = Trait.ReadKey(Data, \
KeyDataLen.first);<br>+ PP.getIdentifierTable().getOwn(Key).setOutOfDate(true);<br>+ \
}<br> }<br><br> // Setup the import locations and notify the module manager that \
we've<br>@@ -3430,13 +3446,20 @@ ASTReader::ASTReadResult ASTReader::Read<br> \
M->ImportLoc.getRawEncoding());<br> }<br><br>- // Mark all of the identifiers in the \
identifier table as being out of date,<br>- // so that various accessors know to check the \
loaded modules when the<br>- // identifier is used.<br>- for (IdentifierTable::iterator Id \
= PP.getIdentifierTable().begin(),<br>- \
IdEnd = PP.getIdentifierTable().end();<br>- Id != IdEnd; ++Id)<br>- \
Id->second->setOutOfDate(true);<br>+ if (!Context.getLangOpts().CPlusPlus ||<br>+ \
(Type != MK_ImplicitModule && Type != MK_ExplicitModule)) {<br>+ // Mark all of \
the identifiers in the identifier table as being out of date,<br>+ // so that various \
accessors know to check the loaded modules when the<br>+ // identifier is used.<br>+ \
//<br>+ // For C++ modules, we don't need information on many identifiers (just<br>+ \
// those that provide macros or are poisoned), so we mark all of<br>+ // the interesting \
ones via PreloadIdentifierOffsets.<br>+ for (IdentifierTable::iterator Id = \
PP.getIdentifierTable().begin(),<br>+ \
IdEnd = PP.getIdentifierTable().end();<br>+ Id != IdEnd; ++Id)<br>+ \
Id->second->setOutOfDate(true);<br>+ }<br><br> // Resolve any unresolved module \
exports.<br> for (unsigned I = 0, N = UnresolvedModuleRefs.size(); I != N; ++I) {<br>@@ \
-6827,19 +6850,32 @@ IdentifierInfo *ASTReader::get(StringRef<br> // Note that we are \
loading an identifier.<br> Deserializing AnIdentifier(this);<br><br>- // If there is a \
global index, look there first to determine which modules<br>- // provably do not have any \
results for this identifier.<br>- GlobalModuleIndex::HitSet Hits;<br>- \
GlobalModuleIndex::HitSet *HitsPtr = nullptr;<br>- if (!loadGlobalIndex()) {<br>- if \
(GlobalIndex->lookupIdentifier(Name, Hits)) {<br>- HitsPtr = &Hits;<br>- \
}<br>- }<br> IdentifierLookupVisitor Visitor(Name, /*PriorGeneration=*/0,<br> \
NumIdentifierLookups,<br> \
NumIdentifierLookupHits);<br>- ModuleMgr.visit(IdentifierLookupVisitor::visit, &Visitor, \
HitsPtr);<br>+<br>+ // We don't need to do identifier table lookups in C++ modules (we \
preload<br>+ // all interesting declarations, and don't need to use the scope for \
name<br>+ // lookups). Perform the lookup in PCH files, though, since we don't build<br>+ \
// a complete initial identifier table if we're carrying on from a PCH.<br>+ if \
(Context.getLangOpts().CPlusPlus) {<br>+ for (auto F : ModuleMgr.pch_modules())<br>+ \
if (Visitor.visit(*F, &Visitor))<br>+ break;<br>+ } else {<br>+ // If \
there is a global index, look there first to determine which modules<br>+ // provably do \
not have any results for this identifier.<br>+ GlobalModuleIndex::HitSet Hits;<br>+ \
GlobalModuleIndex::HitSet *HitsPtr = nullptr;<br>+ if (!loadGlobalIndex()) {<br>+ \
if (GlobalIndex->lookupIdentifier(Name, Hits)) {<br>+ HitsPtr = \
&Hits;<br>+ }<br>+ }<br>+<br>+ \
ModuleMgr.visit(IdentifierLookupVisitor::visit, &Visitor, HitsPtr);<br>+ }<br>+<br> \
IdentifierInfo *II = Visitor.getIdentifierInfo();<br> markIdentifierUpToDate(II);<br> \
return II;<br><br>Modified: cfe/trunk/lib/Serialization/ASTWriter.cpp<br></div></div>URL: <a \
href="https://urldefense.proofpoint.com/v2/url?u=http-3A__llvm.org_viewvc_llvm-2Dproject_cfe_tru \
nk_lib_Serialization_ASTWriter.cpp-3Frev-3D242868-26r1-3D242867-26r2-3D242868-26view-3Ddiff&d=Aw \
MFaQ&c=8hUWFZcy2Z-Za5rBPlktOQ&r=BSqEv9KvKMW_Ob8SyngJ70KdZISM_ASROnREeq0cCxk&m=96E3-MBcA8Hx2Rr1FY_OAxgwCQXuu49G7vPan0Ijzl4&s=ot-en0PnvvFnkn6CvNlxWxNUKY_apOL-rPptuBtMsTA&e=" \
target="_blank">http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTWriter.cpp?rev=242868&r1=242867&r2=242868&view=diff</a><div><div \
class="h5"><br>==============================================================================<br>--- \
cfe/trunk/lib/Serialization/ASTWriter.cpp (original)<br>+++ \
cfe/trunk/lib/Serialization/ASTWriter.cpp Tue Jul 21 21:08:40 2015<br>@@ -3104,6 +3104,7 @@ \
class ASTIdentifierTableTrait {<br> IdentifierResolver &IdResolver;<br> bool \
IsModule;<br> bool NeedDecls;<br>+ ASTWriter::RecordData \
*InterestingIdentifierOffsets;<br><br> /// \brief Determines whether this is an \
"interesting" identifier that needs a<br> /// full IdentifierInfo structure \
written into the hash table. Notably, this<br>@@ -3131,14 +3132,20 @@ public:<br> typedef \
unsigned offset_type;<br><br> ASTIdentifierTableTrait(ASTWriter &Writer, Preprocessor \
&PP,<br>- IdentifierResolver \
&IdResolver, bool IsModule)<br>+ \
IdentifierResolver &IdResolver, bool IsModule,<br>+ \
ASTWriter::RecordData *InterestingIdentifierOffsets)<br> : Writer(Writer), PP(PP), \
IdResolver(IdResolver), IsModule(IsModule),<br>- NeedDecls(!IsModule || \
!Writer.getLangOpts().CPlusPlus) {}<br>+ NeedDecls(!IsModule || \
!Writer.getLangOpts().CPlusPlus),<br>+ \
InterestingIdentifierOffsets(InterestingIdentifierOffsets) {}<br><br> static \
hash_value_type ComputeHash(const IdentifierInfo* II) {<br> return \
llvm::HashString(II->getName());<br> }<br><br>+ bool isInterestingIdentifier(const \
IdentifierInfo *II) {<br>+ auto MacroOffset = Writer.getMacroDirectivesOffset(II);<br>+ \
return isInterestingIdentifier(II, MacroOffset);<br>+ }<br> bool \
isInterestingNonMacroIdentifier(const IdentifierInfo *II) {<br> return \
isInterestingIdentifier(II, 0);<br> }<br>@@ -3178,6 +3185,12 @@ public:<br> // \
Record the location of the key data. This is used when generating<br> // the mapping \
from persistent IDs to strings.<br> Writer.SetIdentifierOffset(II, \
Out.tell());<br>+<br>+ // Emit the offset of the key/data length information to the \
interesting<br>+ // identifiers table if necessary.<br>+ if \
(InterestingIdentifierOffsets && isInterestingIdentifier(II))<br>+ \
InterestingIdentifierOffsets->push_back(Out.tell() - 4);<br>+<br> \
Out.write(II->getNameStart(), KeyLen);<br> }<br><br>@@ -3238,11 +3251,15 @@ void \
ASTWriter::WriteIdentifierTable(Pre<br> \
bool IsModule) {<br> using namespace llvm;<br><br>+ RecordData \
InterestingIdents;<br>+<br> // Create and write out the blob that contains the \
_______________________________________________
cfe-commits mailing list
cfe-commits@cs.uiuc.edu
http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic