head	1.1;
branch	1.1.1;
access;
symbols
	netbsd-11-0-RC4:1.1.1.10
	netbsd-11-0-RC3:1.1.1.10
	netbsd-11-0-RC2:1.1.1.10
	netbsd-11-0-RC1:1.1.1.10
	perseant-exfatfs-base-20250801:1.1.1.10
	netbsd-11:1.1.1.10.0.10
	netbsd-11-base:1.1.1.10
	netbsd-10-1-RELEASE:1.1.1.10
	perseant-exfatfs-base-20240630:1.1.1.10
	perseant-exfatfs:1.1.1.10.0.8
	perseant-exfatfs-base:1.1.1.10
	netbsd-8-3-RELEASE:1.1.1.7
	netbsd-9-4-RELEASE:1.1.1.9
	netbsd-10-0-RELEASE:1.1.1.10
	netbsd-10-0-RC6:1.1.1.10
	netbsd-10-0-RC5:1.1.1.10
	netbsd-10-0-RC4:1.1.1.10
	netbsd-10-0-RC3:1.1.1.10
	netbsd-10-0-RC2:1.1.1.10
	netbsd-10-0-RC1:1.1.1.10
	netbsd-10:1.1.1.10.0.6
	netbsd-10-base:1.1.1.10
	netbsd-9-3-RELEASE:1.1.1.9
	cjep_sun2x:1.1.1.10.0.4
	cjep_sun2x-base:1.1.1.10
	cjep_staticlib_x-base1:1.1.1.10
	netbsd-9-2-RELEASE:1.1.1.9
	cjep_staticlib_x:1.1.1.10.0.2
	cjep_staticlib_x-base:1.1.1.10
	netbsd-9-1-RELEASE:1.1.1.9
	phil-wifi-20200421:1.1.1.10
	phil-wifi-20200411:1.1.1.10
	phil-wifi-20200406:1.1.1.10
	netbsd-8-2-RELEASE:1.1.1.7
	netbsd-9-0-RELEASE:1.1.1.9
	netbsd-9-0-RC2:1.1.1.9
	netbsd-9-0-RC1:1.1.1.9
	netbsd-9:1.1.1.9.0.2
	netbsd-9-base:1.1.1.9
	phil-wifi-20190609:1.1.1.9
	netbsd-8-1-RELEASE:1.1.1.7
	netbsd-8-1-RC1:1.1.1.7
	pgoyette-compat-merge-20190127:1.1.1.8.2.1
	pgoyette-compat-20190127:1.1.1.9
	pgoyette-compat-20190118:1.1.1.9
	pgoyette-compat-1226:1.1.1.9
	pgoyette-compat-1126:1.1.1.9
	pgoyette-compat-1020:1.1.1.9
	pgoyette-compat-0930:1.1.1.9
	pgoyette-compat-0906:1.1.1.9
	netbsd-7-2-RELEASE:1.1.1.4.2.1
	pgoyette-compat-0728:1.1.1.9
	clang-337282:1.1.1.9
	netbsd-8-0-RELEASE:1.1.1.7
	phil-wifi:1.1.1.8.0.4
	phil-wifi-base:1.1.1.8
	pgoyette-compat-0625:1.1.1.8
	netbsd-8-0-RC2:1.1.1.7
	pgoyette-compat-0521:1.1.1.8
	pgoyette-compat-0502:1.1.1.8
	pgoyette-compat-0422:1.1.1.8
	netbsd-8-0-RC1:1.1.1.7
	pgoyette-compat-0415:1.1.1.8
	pgoyette-compat-0407:1.1.1.8
	pgoyette-compat-0330:1.1.1.8
	pgoyette-compat-0322:1.1.1.8
	pgoyette-compat-0315:1.1.1.8
	netbsd-7-1-2-RELEASE:1.1.1.4.2.1
	pgoyette-compat:1.1.1.8.0.2
	pgoyette-compat-base:1.1.1.8
	netbsd-7-1-1-RELEASE:1.1.1.4.2.1
	clang-319952:1.1.1.8
	matt-nb8-mediatek:1.1.1.7.0.10
	matt-nb8-mediatek-base:1.1.1.7
	clang-309604:1.1.1.8
	perseant-stdc-iso10646:1.1.1.7.0.8
	perseant-stdc-iso10646-base:1.1.1.7
	netbsd-8:1.1.1.7.0.6
	netbsd-8-base:1.1.1.7
	prg-localcount2-base3:1.1.1.7
	prg-localcount2-base2:1.1.1.7
	prg-localcount2-base1:1.1.1.7
	prg-localcount2:1.1.1.7.0.4
	prg-localcount2-base:1.1.1.7
	pgoyette-localcount-20170426:1.1.1.7
	bouyer-socketcan-base1:1.1.1.7
	pgoyette-localcount-20170320:1.1.1.7
	netbsd-7-1:1.1.1.4.2.1.0.6
	netbsd-7-1-RELEASE:1.1.1.4.2.1
	netbsd-7-1-RC2:1.1.1.4.2.1
	clang-294123:1.1.1.7
	netbsd-7-nhusb-base-20170116:1.1.1.4.2.1
	bouyer-socketcan:1.1.1.7.0.2
	bouyer-socketcan-base:1.1.1.7
	clang-291444:1.1.1.7
	pgoyette-localcount-20170107:1.1.1.6
	netbsd-7-1-RC1:1.1.1.4.2.1
	pgoyette-localcount-20161104:1.1.1.6
	netbsd-7-0-2-RELEASE:1.1.1.4.2.1
	localcount-20160914:1.1.1.6
	netbsd-7-nhusb:1.1.1.4.2.1.0.4
	netbsd-7-nhusb-base:1.1.1.4.2.1
	clang-280599:1.1.1.6
	pgoyette-localcount-20160806:1.1.1.6
	pgoyette-localcount-20160726:1.1.1.6
	pgoyette-localcount:1.1.1.6.0.2
	pgoyette-localcount-base:1.1.1.6
	netbsd-7-0-1-RELEASE:1.1.1.4.2.1
	clang-261930:1.1.1.6
	netbsd-7-0:1.1.1.4.2.1.0.2
	netbsd-7-0-RELEASE:1.1.1.4.2.1
	netbsd-7-0-RC3:1.1.1.4.2.1
	netbsd-7-0-RC2:1.1.1.4.2.1
	netbsd-7-0-RC1:1.1.1.4.2.1
	clang-237755:1.1.1.5
	clang-232565:1.1.1.5
	clang-227398:1.1.1.5
	tls-maxphys-base:1.1.1.4
	tls-maxphys:1.1.1.4.0.4
	netbsd-7:1.1.1.4.0.2
	netbsd-7-base:1.1.1.4
	clang-215315:1.1.1.4
	clang-209886:1.1.1.3
	yamt-pagecache:1.1.1.2.0.4
	yamt-pagecache-base9:1.1.1.2
	tls-earlyentropy:1.1.1.2.0.2
	tls-earlyentropy-base:1.1.1.3
	riastradh-xf86-video-intel-2-7-1-pre-2-21-15:1.1.1.2
	riastradh-drm2-base3:1.1.1.2
	clang-202566:1.1.1.2
	clang-201163:1.1.1.2
	clang-199312:1.1.1.2
	clang-198450:1.1.1.1
	clang-196603:1.1.1.1
	clang-195771:1.1.1.1
	LLVM:1.1.1;
locks; strict;
comment	@// @;


1.1
date	2013.11.28.14.14.49;	author joerg;	state Exp;
branches
	1.1.1.1;
next	;
commitid	ow8OybrawrB1f3fx;

1.1.1.1
date	2013.11.28.14.14.49;	author joerg;	state Exp;
branches;
next	1.1.1.2;
commitid	ow8OybrawrB1f3fx;

1.1.1.2
date	2014.01.15.21.26.19;	author joerg;	state Exp;
branches
	1.1.1.2.2.1
	1.1.1.2.4.1;
next	1.1.1.3;
commitid	NQXlzzA0SPkc5glx;

1.1.1.3
date	2014.05.30.18.14.44;	author joerg;	state Exp;
branches;
next	1.1.1.4;
commitid	8q0kdlBlCn09GACx;

1.1.1.4
date	2014.08.10.17.08.26;	author joerg;	state Exp;
branches
	1.1.1.4.2.1
	1.1.1.4.4.1;
next	1.1.1.5;
commitid	N85tXAN6Ex9VZPLx;

1.1.1.5
date	2015.01.29.19.57.32;	author joerg;	state Exp;
branches;
next	1.1.1.6;
commitid	mlISSizlPKvepX7y;

1.1.1.6
date	2016.02.27.22.11.51;	author joerg;	state Exp;
branches
	1.1.1.6.2.1;
next	1.1.1.7;
commitid	tIimz3oDlh1NpBWy;

1.1.1.7
date	2017.01.11.10.35.22;	author joerg;	state Exp;
branches;
next	1.1.1.8;
commitid	CNnUNfII1jgNmxBz;

1.1.1.8
date	2017.08.01.19.35.36;	author joerg;	state Exp;
branches
	1.1.1.8.2.1
	1.1.1.8.4.1;
next	1.1.1.9;
commitid	pMuDy65V0VicSx1A;

1.1.1.9
date	2018.07.17.18.31.26;	author joerg;	state Exp;
branches;
next	1.1.1.10;
commitid	wDzL46ALjrCZgwKA;

1.1.1.10
date	2019.11.13.22.19.28;	author joerg;	state dead;
branches;
next	;
commitid	QD8YATxuNG34YJKB;

1.1.1.2.2.1
date	2014.08.10.07.08.10;	author tls;	state Exp;
branches;
next	;
commitid	t01A1TLTYxkpGMLx;

1.1.1.2.4.1
date	2014.01.15.21.26.19;	author yamt;	state dead;
branches;
next	1.1.1.2.4.2;
commitid	WSrDtL5nYAUyiyBx;

1.1.1.2.4.2
date	2014.05.22.16.18.30;	author yamt;	state Exp;
branches;
next	;
commitid	WSrDtL5nYAUyiyBx;

1.1.1.4.2.1
date	2015.06.04.20.04.30;	author snj;	state Exp;
branches;
next	;
commitid	yRnjq9fueSo6n9oy;

1.1.1.4.4.1
date	2014.08.10.17.08.26;	author tls;	state dead;
branches;
next	1.1.1.4.4.2;
commitid	jTnpym9Qu0o4R1Nx;

1.1.1.4.4.2
date	2014.08.19.23.47.31;	author tls;	state Exp;
branches;
next	;
commitid	jTnpym9Qu0o4R1Nx;

1.1.1.6.2.1
date	2017.03.20.06.52.41;	author pgoyette;	state Exp;
branches;
next	;
commitid	jjw7cAwgyKq7RfKz;

1.1.1.8.2.1
date	2018.07.28.04.33.23;	author pgoyette;	state Exp;
branches;
next	;
commitid	1UP1xAIUxv1ZgRLA;

1.1.1.8.4.1
date	2019.06.10.21.45.28;	author christos;	state Exp;
branches;
next	1.1.1.8.4.2;
commitid	jtc8rnCzWiEEHGqB;

1.1.1.8.4.2
date	2020.04.13.07.46.38;	author martin;	state dead;
branches;
next	;
commitid	X01YhRUPVUDaec4C;


desc
@@


1.1
log
@Initial revision
@
text
@//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements the GlobalModuleIndex class.
//
//===----------------------------------------------------------------------===//

#include "ASTReaderInternals.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/OnDiskHashTable.h"
#include "clang/Serialization/ASTBitCodes.h"
#include "clang/Serialization/GlobalModuleIndex.h"
#include "clang/Serialization/Module.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Bitcode/BitstreamReader.h"
#include "llvm/Bitcode/BitstreamWriter.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/LockFileManager.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include <cstdio>
using namespace clang;
using namespace serialization;

//----------------------------------------------------------------------------//
// Shared constants
//----------------------------------------------------------------------------//
namespace {
  enum {
    /// \brief The block containing the index.
    GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
  };

  /// \brief Describes the record types in the index.
  enum IndexRecordTypes {
    /// \brief Contains version information and potentially other metadata,
    /// used to determine if we can read this global index file.
    INDEX_METADATA,
    /// \brief Describes a module, including its file name and dependencies.
    MODULE,
    /// \brief The index for identifiers.
    IDENTIFIER_INDEX
  };
}

/// \brief The name of the global index file.
static const char * const IndexFileName = "modules.idx";

/// \brief The global index file version.
static const unsigned CurrentVersion = 1;

//----------------------------------------------------------------------------//
// Global module index reader.
//----------------------------------------------------------------------------//

namespace {

/// \brief Trait used to read the identifier index from the on-disk hash
/// table.
class IdentifierIndexReaderTrait {
public:
  typedef StringRef external_key_type;
  typedef StringRef internal_key_type;
  typedef SmallVector<unsigned, 2> data_type;

  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
    return a == b;
  }

  static unsigned ComputeHash(const internal_key_type& a) {
    return llvm::HashString(a);
  }

  static std::pair<unsigned, unsigned>
  ReadKeyDataLength(const unsigned char*& d) {
    using namespace clang::io;
    unsigned KeyLen = ReadUnalignedLE16(d);
    unsigned DataLen = ReadUnalignedLE16(d);
    return std::make_pair(KeyLen, DataLen);
  }

  static const internal_key_type&
  GetInternalKey(const external_key_type& x) { return x; }

  static const external_key_type&
  GetExternalKey(const internal_key_type& x) { return x; }

  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
    return StringRef((const char *)d, n);
  }

  static data_type ReadData(const internal_key_type& k,
                            const unsigned char* d,
                            unsigned DataLen) {
    using namespace clang::io;

    data_type Result;
    while (DataLen > 0) {
      unsigned ID = ReadUnalignedLE32(d);
      Result.push_back(ID);
      DataLen -= 4;
    }

    return Result;
  }
};

typedef OnDiskChainedHashTable<IdentifierIndexReaderTrait> IdentifierIndexTable;

}

GlobalModuleIndex::GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
                                     llvm::BitstreamCursor Cursor)
  : Buffer(Buffer), IdentifierIndex(),
    NumIdentifierLookups(), NumIdentifierLookupHits()
{
  // Read the global index.
  bool InGlobalIndexBlock = false;
  bool Done = false;
  while (!Done) {
    llvm::BitstreamEntry Entry = Cursor.advance();

    switch (Entry.Kind) {
    case llvm::BitstreamEntry::Error:
      return;

    case llvm::BitstreamEntry::EndBlock:
      if (InGlobalIndexBlock) {
        InGlobalIndexBlock = false;
        Done = true;
        continue;
      }
      return;


    case llvm::BitstreamEntry::Record:
      // Entries in the global index block are handled below.
      if (InGlobalIndexBlock)
        break;

      return;

    case llvm::BitstreamEntry::SubBlock:
      if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
        if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
          return;

        InGlobalIndexBlock = true;
      } else if (Cursor.SkipBlock()) {
        return;
      }
      continue;
    }

    SmallVector<uint64_t, 64> Record;
    StringRef Blob;
    switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
    case INDEX_METADATA:
      // Make sure that the version matches.
      if (Record.size() < 1 || Record[0] != CurrentVersion)
        return;
      break;

    case MODULE: {
      unsigned Idx = 0;
      unsigned ID = Record[Idx++];

      // Make room for this module's information.
      if (ID == Modules.size())
        Modules.push_back(ModuleInfo());
      else
        Modules.resize(ID + 1);

      // Size/modification time for this module file at the time the
      // global index was built.
      Modules[ID].Size = Record[Idx++];
      Modules[ID].ModTime = Record[Idx++];

      // File name.
      unsigned NameLen = Record[Idx++];
      Modules[ID].FileName.assign(Record.begin() + Idx,
                                  Record.begin() + Idx + NameLen);
      Idx += NameLen;

      // Dependencies
      unsigned NumDeps = Record[Idx++];
      Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
                                      Record.begin() + Idx,
                                      Record.begin() + Idx + NumDeps);
      Idx += NumDeps;

      // Make sure we're at the end of the record.
      assert(Idx == Record.size() && "More module info?");

      // Record this module as an unresolved module.
      UnresolvedModules[llvm::sys::path::stem(Modules[ID].FileName)] = ID;
      break;
    }

    case IDENTIFIER_INDEX:
      // Wire up the identifier index.
      if (Record[0]) {
        IdentifierIndex = IdentifierIndexTable::Create(
                            (const unsigned char *)Blob.data() + Record[0],
                            (const unsigned char *)Blob.data(),
                            IdentifierIndexReaderTrait());
      }
      break;
    }
  }
}

GlobalModuleIndex::~GlobalModuleIndex() { }

std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
GlobalModuleIndex::readIndex(StringRef Path) {
  // Load the index file, if it's there.
  llvm::SmallString<128> IndexPath;
  IndexPath += Path;
  llvm::sys::path::append(IndexPath, IndexFileName);

  llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
  if (llvm::MemoryBuffer::getFile(IndexPath.c_str(), Buffer) !=
      llvm::errc::success)
    return std::make_pair((GlobalModuleIndex *)0, EC_NotFound);

  /// \brief The bitstream reader from which we'll read the AST file.
  llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(),
                               (const unsigned char *)Buffer->getBufferEnd());

  /// \brief The main bitstream cursor for the main block.
  llvm::BitstreamCursor Cursor(Reader);

  // Sniff for the signature.
  if (Cursor.Read(8) != 'B' ||
      Cursor.Read(8) != 'C' ||
      Cursor.Read(8) != 'G' ||
      Cursor.Read(8) != 'I') {
    return std::make_pair((GlobalModuleIndex *)0, EC_IOError);
  }
  
  return std::make_pair(new GlobalModuleIndex(Buffer.take(), Cursor), EC_None);
}

void
GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
  ModuleFiles.clear();
  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
    if (ModuleFile *MF = Modules[I].File)
      ModuleFiles.push_back(MF);
  }
}

void GlobalModuleIndex::getModuleDependencies(
       ModuleFile *File,
       SmallVectorImpl<ModuleFile *> &Dependencies) {
  // Look for information about this module file.
  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
    = ModulesByFile.find(File);
  if (Known == ModulesByFile.end())
    return;

  // Record dependencies.
  Dependencies.clear();
  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
    if (ModuleFile *MF = Modules[I].File)
      Dependencies.push_back(MF);
  }
}

bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
  Hits.clear();
  
  // If there's no identifier index, there is nothing we can do.
  if (!IdentifierIndex)
    return false;

  // Look into the identifier index.
  ++NumIdentifierLookups;
  IdentifierIndexTable &Table
    = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
  IdentifierIndexTable::iterator Known = Table.find(Name);
  if (Known == Table.end()) {
    return true;
  }

  SmallVector<unsigned, 2> ModuleIDs = *Known;
  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
    if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
      Hits.insert(MF);
  }

  ++NumIdentifierLookupHits;
  return true;
}

bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
  // Look for the module in the global module index based on the module name.
  StringRef Name = llvm::sys::path::stem(File->FileName);
  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
  if (Known == UnresolvedModules.end()) {
    return true;
  }

  // Rectify this module with the global module index.
  ModuleInfo &Info = Modules[Known->second];

  //  If the size and modification time match what we expected, record this
  // module file.
  bool Failed = true;
  if (File->File->getSize() == Info.Size &&
      File->File->getModificationTime() == Info.ModTime) {
    Info.File = File;
    ModulesByFile[File] = Known->second;

    Failed = false;
  }

  // One way or another, we have resolved this module file.
  UnresolvedModules.erase(Known);
  return Failed;
}

void GlobalModuleIndex::printStats() {
  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
  if (NumIdentifierLookups) {
    fprintf(stderr, "  %u / %u identifier lookups succeeded (%f%%)\n",
            NumIdentifierLookupHits, NumIdentifierLookups,
            (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
  }
  std::fprintf(stderr, "\n");
}

//----------------------------------------------------------------------------//
// Global module index writer.
//----------------------------------------------------------------------------//

namespace {
  /// \brief Provides information about a specific module file.
  struct ModuleFileInfo {
    /// \brief The numberic ID for this module file.
    unsigned ID;

    /// \brief The set of modules on which this module depends. Each entry is
    /// a module ID.
    SmallVector<unsigned, 4> Dependencies;
  };

  /// \brief Builder that generates the global module index file.
  class GlobalModuleIndexBuilder {
    FileManager &FileMgr;

    /// \brief Mapping from files to module file information.
    typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;

    /// \brief Information about each of the known module files.
    ModuleFilesMap ModuleFiles;

    /// \brief Mapping from identifiers to the list of module file IDs that
    /// consider this identifier to be interesting.
    typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;

    /// \brief A mapping from all interesting identifiers to the set of module
    /// files in which those identifiers are considered interesting.
    InterestingIdentifierMap InterestingIdentifiers;
    
    /// \brief Write the block-info block for the global module index file.
    void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);

    /// \brief Retrieve the module file information for the given file.
    ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
      llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
        = ModuleFiles.find(File);
      if (Known != ModuleFiles.end())
        return Known->second;

      unsigned NewID = ModuleFiles.size();
      ModuleFileInfo &Info = ModuleFiles[File];
      Info.ID = NewID;
      return Info;
    }

  public:
    explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){}

    /// \brief Load the contents of the given module file into the builder.
    ///
    /// \returns true if an error occurred, false otherwise.
    bool loadModuleFile(const FileEntry *File);

    /// \brief Write the index to the given bitstream.
    void writeIndex(llvm::BitstreamWriter &Stream);
  };
}

static void emitBlockID(unsigned ID, const char *Name,
                        llvm::BitstreamWriter &Stream,
                        SmallVectorImpl<uint64_t> &Record) {
  Record.clear();
  Record.push_back(ID);
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);

  // Emit the block name if present.
  if (Name == 0 || Name[0] == 0) return;
  Record.clear();
  while (*Name)
    Record.push_back(*Name++);
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
}

static void emitRecordID(unsigned ID, const char *Name,
                         llvm::BitstreamWriter &Stream,
                         SmallVectorImpl<uint64_t> &Record) {
  Record.clear();
  Record.push_back(ID);
  while (*Name)
    Record.push_back(*Name++);
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
}

void
GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
  SmallVector<uint64_t, 64> Record;
  Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);

#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
#define RECORD(X) emitRecordID(X, #X, Stream, Record)
  BLOCK(GLOBAL_INDEX_BLOCK);
  RECORD(INDEX_METADATA);
  RECORD(MODULE);
  RECORD(IDENTIFIER_INDEX);
#undef RECORD
#undef BLOCK

  Stream.ExitBlock();
}

namespace {
  class InterestingASTIdentifierLookupTrait
    : public serialization::reader::ASTIdentifierLookupTraitBase {

  public:
    /// \brief The identifier and whether it is "interesting".
    typedef std::pair<StringRef, bool> data_type;

    data_type ReadData(const internal_key_type& k,
                       const unsigned char* d,
                       unsigned DataLen) {
      // The first bit indicates whether this identifier is interesting.
      // That's all we care about.
      using namespace clang::io;
      unsigned RawID = ReadUnalignedLE32(d);
      bool IsInteresting = RawID & 0x01;
      return std::make_pair(k, IsInteresting);
    }
  };
}

bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
  // Open the module file.
  OwningPtr<llvm::MemoryBuffer> Buffer;
  std::string ErrorStr;
  Buffer.reset(FileMgr.getBufferForFile(File, &ErrorStr, /*isVolatile=*/true));
  if (!Buffer) {
    return true;
  }

  // Initialize the input stream
  llvm::BitstreamReader InStreamFile;
  llvm::BitstreamCursor InStream;
  InStreamFile.init((const unsigned char *)Buffer->getBufferStart(),
                  (const unsigned char *)Buffer->getBufferEnd());
  InStream.init(InStreamFile);

  // Sniff for the signature.
  if (InStream.Read(8) != 'C' ||
      InStream.Read(8) != 'P' ||
      InStream.Read(8) != 'C' ||
      InStream.Read(8) != 'H') {
    return true;
  }

  // Record this module file and assign it a unique ID (if it doesn't have
  // one already).
  unsigned ID = getModuleFileInfo(File).ID;

  // Search for the blocks and records we care about.
  enum { Other, ControlBlock, ASTBlock } State = Other;
  bool Done = false;
  while (!Done) {
    llvm::BitstreamEntry Entry = InStream.advance();
    switch (Entry.Kind) {
    case llvm::BitstreamEntry::Error:
      Done = true;
      continue;

    case llvm::BitstreamEntry::Record:
      // In the 'other' state, just skip the record. We don't care.
      if (State == Other) {
        InStream.skipRecord(Entry.ID);
        continue;
      }

      // Handle potentially-interesting records below.
      break;

    case llvm::BitstreamEntry::SubBlock:
      if (Entry.ID == CONTROL_BLOCK_ID) {
        if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
          return true;

        // Found the control block.
        State = ControlBlock;
        continue;
      }

      if (Entry.ID == AST_BLOCK_ID) {
        if (InStream.EnterSubBlock(AST_BLOCK_ID))
          return true;

        // Found the AST block.
        State = ASTBlock;
        continue;
      }

      if (InStream.SkipBlock())
        return true;

      continue;

    case llvm::BitstreamEntry::EndBlock:
      State = Other;
      continue;
    }

    // Read the given record.
    SmallVector<uint64_t, 64> Record;
    StringRef Blob;
    unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);

    // Handle module dependencies.
    if (State == ControlBlock && Code == IMPORTS) {
      // Load each of the imported PCH files.
      unsigned Idx = 0, N = Record.size();
      while (Idx < N) {
        // Read information about the AST file.

        // Skip the imported kind
        ++Idx;

        // Skip the import location
        ++Idx;

        // Load stored size/modification time. 
        off_t StoredSize = (off_t)Record[Idx++];
        time_t StoredModTime = (time_t)Record[Idx++];

        // Retrieve the imported file name.
        unsigned Length = Record[Idx++];
        SmallString<128> ImportedFile(Record.begin() + Idx,
                                      Record.begin() + Idx + Length);
        Idx += Length;

        // Find the imported module file.
        const FileEntry *DependsOnFile
          = FileMgr.getFile(ImportedFile, /*openFile=*/false,
                            /*cacheFailure=*/false);
        if (!DependsOnFile ||
            (StoredSize != DependsOnFile->getSize()) ||
            (StoredModTime != DependsOnFile->getModificationTime()))
          return true;

        // Record the dependency.
        unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
        getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
      }

      continue;
    }

    // Handle the identifier table
    if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
      typedef OnDiskChainedHashTable<InterestingASTIdentifierLookupTrait>
        InterestingIdentifierTable;
      llvm::OwningPtr<InterestingIdentifierTable>
        Table(InterestingIdentifierTable::Create(
                (const unsigned char *)Blob.data() + Record[0],
                (const unsigned char *)Blob.data()));
      for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
                                                     DEnd = Table->data_end();
           D != DEnd; ++D) {
        std::pair<StringRef, bool> Ident = *D;
        if (Ident.second)
          InterestingIdentifiers[Ident.first].push_back(ID);
        else
          (void)InterestingIdentifiers[Ident.first];
      }
    }

    // We don't care about this record.
  }

  return false;
}

namespace {

/// \brief Trait used to generate the identifier index as an on-disk hash
/// table.
class IdentifierIndexWriterTrait {
public:
  typedef StringRef key_type;
  typedef StringRef key_type_ref;
  typedef SmallVector<unsigned, 2> data_type;
  typedef const SmallVector<unsigned, 2> &data_type_ref;

  static unsigned ComputeHash(key_type_ref Key) {
    return llvm::HashString(Key);
  }

  std::pair<unsigned,unsigned>
  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
    unsigned KeyLen = Key.size();
    unsigned DataLen = Data.size() * 4;
    clang::io::Emit16(Out, KeyLen);
    clang::io::Emit16(Out, DataLen);
    return std::make_pair(KeyLen, DataLen);
  }
  
  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
    Out.write(Key.data(), KeyLen);
  }

  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
                unsigned DataLen) {
    for (unsigned I = 0, N = Data.size(); I != N; ++I)
      clang::io::Emit32(Out, Data[I]);
  }
};

}

void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
  using namespace llvm;
  
  // Emit the file header.
  Stream.Emit((unsigned)'B', 8);
  Stream.Emit((unsigned)'C', 8);
  Stream.Emit((unsigned)'G', 8);
  Stream.Emit((unsigned)'I', 8);

  // Write the block-info block, which describes the records in this bitcode
  // file.
  emitBlockInfoBlock(Stream);

  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);

  // Write the metadata.
  SmallVector<uint64_t, 2> Record;
  Record.push_back(CurrentVersion);
  Stream.EmitRecord(INDEX_METADATA, Record);

  // Write the set of known module files.
  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
                                MEnd = ModuleFiles.end();
       M != MEnd; ++M) {
    Record.clear();
    Record.push_back(M->second.ID);
    Record.push_back(M->first->getSize());
    Record.push_back(M->first->getModificationTime());

    // File name
    StringRef Name(M->first->getName());
    Record.push_back(Name.size());
    Record.append(Name.begin(), Name.end());

    // Dependencies
    Record.push_back(M->second.Dependencies.size());
    Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
    Stream.EmitRecord(MODULE, Record);
  }

  // Write the identifier -> module file mapping.
  {
    OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
    IdentifierIndexWriterTrait Trait;

    // Populate the hash table.
    for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
                                            IEnd = InterestingIdentifiers.end();
         I != IEnd; ++I) {
      Generator.insert(I->first(), I->second, Trait);
    }
    
    // Create the on-disk hash table in a buffer.
    SmallString<4096> IdentifierTable;
    uint32_t BucketOffset;
    {
      llvm::raw_svector_ostream Out(IdentifierTable);
      // Make sure that no bucket is at offset 0
      clang::io::Emit32(Out, 0);
      BucketOffset = Generator.Emit(Out, Trait);
    }

    // Create a blob abbreviation
    BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
    Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
    unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);

    // Write the identifier table
    Record.clear();
    Record.push_back(IDENTIFIER_INDEX);
    Record.push_back(BucketOffset);
    Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str());
  }

  Stream.ExitBlock();
}

GlobalModuleIndex::ErrorCode
GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) {
  llvm::SmallString<128> IndexPath;
  IndexPath += Path;
  llvm::sys::path::append(IndexPath, IndexFileName);

  // Coordinate building the global index file with other processes that might
  // try to do the same.
  llvm::LockFileManager Locked(IndexPath);
  switch (Locked) {
  case llvm::LockFileManager::LFS_Error:
    return EC_IOError;

  case llvm::LockFileManager::LFS_Owned:
    // We're responsible for building the index ourselves. Do so below.
    break;

  case llvm::LockFileManager::LFS_Shared:
    // Someone else is responsible for building the index. We don't care
    // when they finish, so we're done.
    return EC_Building;
  }

  // The module index builder.
  GlobalModuleIndexBuilder Builder(FileMgr);
  
  // Load each of the module files.
  llvm::error_code EC;
  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
       D != DEnd && !EC;
       D.increment(EC)) {
    // If this isn't a module file, we don't care.
    if (llvm::sys::path::extension(D->path()) != ".pcm") {
      // ... unless it's a .pcm.lock file, which indicates that someone is
      // in the process of rebuilding a module. They'll rebuild the index
      // at the end of that translation unit, so we don't have to.
      if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
        return EC_Building;

      continue;
    }

    // If we can't find the module file, skip it.
    const FileEntry *ModuleFile = FileMgr.getFile(D->path());
    if (!ModuleFile)
      continue;

    // Load this module file.
    if (Builder.loadModuleFile(ModuleFile))
      return EC_IOError;
  }

  // The output buffer, into which the global index will be written.
  SmallVector<char, 16> OutputBuffer;
  {
    llvm::BitstreamWriter OutputStream(OutputBuffer);
    Builder.writeIndex(OutputStream);
  }

  // Write the global index file to a temporary file.
  llvm::SmallString<128> IndexTmpPath;
  int TmpFD;
  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
                                      IndexTmpPath))
    return EC_IOError;

  // Open the temporary global index file for output.
  llvm::raw_fd_ostream Out(TmpFD, true);
  if (Out.has_error())
    return EC_IOError;

  // Write the index.
  Out.write(OutputBuffer.data(), OutputBuffer.size());
  Out.close();
  if (Out.has_error())
    return EC_IOError;

  // Remove the old index file. It isn't relevant any more.
  bool OldIndexExisted;
  llvm::sys::fs::remove(IndexPath.str(), OldIndexExisted);

  // Rename the newly-written index file to the proper name.
  if (llvm::sys::fs::rename(IndexTmpPath.str(), IndexPath.str())) {
    // Rename failed; just remove the 
    llvm::sys::fs::remove(IndexTmpPath.str(), OldIndexExisted);
    return EC_IOError;
  }

  // We're done.
  return EC_None;
}

namespace {
  class GlobalIndexIdentifierIterator : public IdentifierIterator {
    /// \brief The current position within the identifier lookup table.
    IdentifierIndexTable::key_iterator Current;

    /// \brief The end position within the identifier lookup table.
    IdentifierIndexTable::key_iterator End;

  public:
    explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
      Current = Idx.key_begin();
      End = Idx.key_end();
    }

    virtual StringRef Next() {
      if (Current == End)
        return StringRef();

      StringRef Result = *Current;
      ++Current;
      return Result;
    }
  };
}

IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
  IdentifierIndexTable &Table =
    *static_cast<IdentifierIndexTable *>(IdentifierIndex);
  return new GlobalIndexIdentifierIterator(Table);
}
@


1.1.1.1
log
@Import Clang 3.4rc1 r195771.
@
text
@@


1.1.1.2
log
@Import Clang 3.5svn r199312
@
text
@d810 2
a811 1
  llvm::sys::fs::remove(IndexPath.str());
d816 1
a816 1
    llvm::sys::fs::remove(IndexTmpPath.str());
@


1.1.1.2.2.1
log
@Rebase.
@
text
@d16 1
a16 1
#include "clang/Lex/HeaderSearch.h"
a28 1
#include "llvm/Support/OnDiskHashTable.h"
a73 2
  typedef unsigned hash_value_type;
  typedef unsigned offset_type;
d79 1
a79 1
  static hash_value_type ComputeHash(const internal_key_type& a) {
d85 3
a87 3
    using namespace llvm::support;
    unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
    unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
d104 1
a104 1
    using namespace llvm::support;
d108 1
a108 1
      unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
d117 1
a117 2
typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
    IdentifierIndexTable;
d205 1
a205 6
      // FIXME: this doesn't work correctly for module names containing path
      // separators.
      StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
      // Remove the -<hash of ModuleMapPath>
      ModuleName = ModuleName.rsplit('-').first;
      UnresolvedModules[ModuleName] = ID;
d213 3
a215 3
            (const unsigned char *)Blob.data() + Record[0],
            (const unsigned char *)Blob.data() + sizeof(uint32_t),
            (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
d222 1
a222 3
GlobalModuleIndex::~GlobalModuleIndex() {
  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
}
d231 1
a231 1
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
d234 1
a234 1
    return std::make_pair(nullptr, EC_NotFound);
d248 1
a248 1
    return std::make_pair(nullptr, EC_IOError);
d250 2
a251 3

  return std::make_pair(new GlobalModuleIndex(Buffer.release(), Cursor),
                        EC_None);
d309 1
a309 1
  StringRef Name = File->ModuleName;
a343 13
void GlobalModuleIndex::dump() {
  llvm::errs() << "*** Global Module Index Dump:\n";
  llvm::errs() << "Module files:\n";
  for (auto &MI : Modules) {
    llvm::errs() << "** " << MI.FileName << "\n";
    if (MI.File)
      MI.File->dump();
    else
      llvm::errs() << "\n";
  }
  llvm::errs() << "\n";
}

d414 1
a414 1
  if (!Name || Name[0] == 0) return;
d461 2
a462 2
      using namespace llvm::support;
      unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
d471 1
a471 1
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
d593 6
a598 7
      typedef llvm::OnDiskIterableChainedHashTable<
          InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
      std::unique_ptr<InterestingIdentifierTable> Table(
          InterestingIdentifierTable::Create(
              (const unsigned char *)Blob.data() + Record[0],
              (const unsigned char *)Blob.data() + sizeof(uint32_t),
              (const unsigned char *)Blob.data()));
a625 2
  typedef unsigned hash_value_type;
  typedef unsigned offset_type;
d627 1
a627 1
  static hash_value_type ComputeHash(key_type_ref Key) {
a632 2
    using namespace llvm::support;
    endian::Writer<little> LE(Out);
d635 2
a636 2
    LE.write<uint16_t>(KeyLen);
    LE.write<uint16_t>(DataLen);
a645 1
    using namespace llvm::support;
d647 1
a647 1
      endian::Writer<little>(Out).write<uint32_t>(Data[I]);
d695 1
a695 1
    llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
a708 1
      using namespace llvm::support;
d711 1
a711 1
      endian::Writer<little>(Out).write<uint32_t>(0);
d837 1
a837 1
    StringRef Next() override {
@


1.1.1.3
log
@Import Clang 3.5svn r209886.
@
text
@d16 1
a16 1
#include "clang/Lex/HeaderSearch.h"
a28 1
#include "llvm/Support/OnDiskHashTable.h"
a73 2
  typedef unsigned hash_value_type;
  typedef unsigned offset_type;
d79 1
a79 1
  static hash_value_type ComputeHash(const internal_key_type& a) {
d85 3
a87 3
    using namespace llvm::support;
    unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
    unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
d104 1
a104 1
    using namespace llvm::support;
d108 1
a108 1
      unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
d117 1
a117 2
typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
    IdentifierIndexTable;
d205 1
a205 6
      // FIXME: this doesn't work correctly for module names containing path
      // separators.
      StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
      // Remove the -<hash of ModuleMapPath>
      ModuleName = ModuleName.rsplit('-').first;
      UnresolvedModules[ModuleName] = ID;
d213 3
a215 3
            (const unsigned char *)Blob.data() + Record[0],
            (const unsigned char *)Blob.data() + sizeof(uint32_t),
            (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
d222 1
a222 3
GlobalModuleIndex::~GlobalModuleIndex() {
  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
}
d231 1
a231 1
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
d234 1
a234 1
    return std::make_pair(nullptr, EC_NotFound);
d248 1
a248 1
    return std::make_pair(nullptr, EC_IOError);
d250 2
a251 3

  return std::make_pair(new GlobalModuleIndex(Buffer.release(), Cursor),
                        EC_None);
d309 1
a309 1
  StringRef Name = File->ModuleName;
a343 13
void GlobalModuleIndex::dump() {
  llvm::errs() << "*** Global Module Index Dump:\n";
  llvm::errs() << "Module files:\n";
  for (auto &MI : Modules) {
    llvm::errs() << "** " << MI.FileName << "\n";
    if (MI.File)
      MI.File->dump();
    else
      llvm::errs() << "\n";
  }
  llvm::errs() << "\n";
}

d414 1
a414 1
  if (!Name || Name[0] == 0) return;
d461 2
a462 2
      using namespace llvm::support;
      unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
d471 1
a471 1
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
d593 6
a598 7
      typedef llvm::OnDiskIterableChainedHashTable<
          InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
      std::unique_ptr<InterestingIdentifierTable> Table(
          InterestingIdentifierTable::Create(
              (const unsigned char *)Blob.data() + Record[0],
              (const unsigned char *)Blob.data() + sizeof(uint32_t),
              (const unsigned char *)Blob.data()));
a625 2
  typedef unsigned hash_value_type;
  typedef unsigned offset_type;
d627 1
a627 1
  static hash_value_type ComputeHash(key_type_ref Key) {
a632 2
    using namespace llvm::support;
    endian::Writer<little> LE(Out);
d635 2
a636 2
    LE.write<uint16_t>(KeyLen);
    LE.write<uint16_t>(DataLen);
a645 1
    using namespace llvm::support;
d647 1
a647 1
      endian::Writer<little>(Out).write<uint32_t>(Data[I]);
d695 1
a695 1
    llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
a708 1
      using namespace llvm::support;
d711 1
a711 1
      endian::Writer<little>(Out).write<uint32_t>(0);
d837 1
a837 1
    StringRef Next() override {
@


1.1.1.4
log
@Import clang 3.6svn r215315.
@
text
@d242 3
a244 3
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
      llvm::MemoryBuffer::getFile(IndexPath.c_str());
  if (!BufferOrErr)
a245 1
  std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
d791 1
a791 1
  std::error_code EC;
@


1.1.1.4.2.1
log
@Update LLVM to 3.6.1, requested by joerg in ticket 824.
@
text
@d125 1
a125 1
GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
d127 3
a129 2
    : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
      NumIdentifierLookupHits() {
d263 1
a263 1
  return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
d497 3
a499 2

  auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
d506 4
a509 3
  InStreamFile.init((const unsigned char *)(*Buffer)->getBufferStart(),
                    (const unsigned char *)(*Buffer)->getBufferEnd());
  llvm::BitstreamCursor InStream(InStreamFile);
a593 4
        // Skip the stored signature.
        // FIXME: we could read the signature out of the import and validate it.
        Idx++;

@


1.1.1.5
log
@Import Clang 3.6RC1 r227398.
@
text
@d125 1
a125 1
GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
d127 3
a129 2
    : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
      NumIdentifierLookupHits() {
d263 1
a263 1
  return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
d497 3
a499 2

  auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
d506 4
a509 3
  InStreamFile.init((const unsigned char *)(*Buffer)->getBufferStart(),
                    (const unsigned char *)(*Buffer)->getBufferEnd());
  llvm::BitstreamCursor InStream(InStreamFile);
a593 4
        // Skip the stored signature.
        // FIXME: we could read the signature out of the import and validate it.
        Idx++;

@


1.1.1.6
log
@Import Clang 3.8.0rc3 r261930.
@
text
@a14 1
#include "clang/Frontend/PCHContainerOperations.h"
a386 1
    const PCHContainerReader &PCHContainerRdr;
d419 1
a419 3
    explicit GlobalModuleIndexBuilder(
        FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
        : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
d504 2
a505 1
  PCHContainerRdr.ExtractPCH((*Buffer)->getMemBufferRef(), InStreamFile);
d757 4
a760 2
    uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
    Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
d767 1
a767 3
GlobalModuleIndex::writeIndex(FileManager &FileMgr,
                              const PCHContainerReader &PCHContainerRdr,
                              StringRef Path) {
d790 2
a791 2
  GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);

d844 1
a844 1
  llvm::sys::fs::remove(IndexPath);
d847 1
a847 1
  if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
d849 1
a849 1
    llvm::sys::fs::remove(IndexTmpPath);
@


1.1.1.6.2.1
log
@Sync with HEAD
@
text
@d248 4
d253 1
a253 1
  llvm::BitstreamCursor Cursor(*Buffer);
d357 1
a357 1
LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
d463 1
a463 1
  Stream.EnterBlockInfoBlock();
d507 3
a509 1
  llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
d753 1
a753 1
    auto Abbrev = std::make_shared<BitCodeAbbrev>();
d757 1
a757 1
    unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
@


1.1.1.7
log
@Import Clang pre-4.0.0 r291444.
@
text
@d248 4
d253 1
a253 1
  llvm::BitstreamCursor Cursor(*Buffer);
d357 1
a357 1
LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
d463 1
a463 1
  Stream.EnterBlockInfoBlock();
d507 3
a509 1
  llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
d753 1
a753 1
    auto Abbrev = std::make_shared<BitCodeAbbrev>();
d757 1
a757 1
    unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
@


1.1.1.8
log
@Import clang r309604 from branches/release_50
@
text
@a378 9
    ASTFileSignature Signature;
  };

  struct ImportedModuleFileInfo {
    off_t StoredSize;
    time_t StoredModTime;
    ASTFileSignature StoredSignature;
    ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
        : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
d386 1
a386 1
    /// Mapping from files to module file information.
d389 1
a389 1
    /// Information about each of the known module files.
a391 8
    /// \brief Mapping from the imported module file to the imported
    /// information.
    typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
        ImportedModuleFilesMap;

    /// \brief Information about each importing of a module file.
    ImportedModuleFilesMap ImportedModuleFiles;

d427 1
a427 2
    /// \returns true if an error occurred, false otherwise.
    bool writeIndex(llvm::BitstreamWriter &Stream);
d518 1
a518 1
  enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
a555 9
      if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
        if (InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID))
          return true;

        // Found the Diagnostic Options block.
        State = DiagnosticOptionsBlock;
        continue;
      }

d590 1
a590 4
        ASTFileSignature StoredSignature = {
            {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
              (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
              (uint32_t)Record[Idx++]}}};
d602 3
a604 2

        if (!DependsOnFile)
a606 6
        // Save the information in ImportedModuleFileInfo so we can verify after
        // loading all pcms.
        ImportedModuleFiles.insert(std::make_pair(
            DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
                                                  StoredSignature)));

a634 6
    // Get Signature.
    if (State == DiagnosticOptionsBlock && Code == SIGNATURE)
      getModuleFileInfo(File).Signature = {
          {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
            (uint32_t)Record[3], (uint32_t)Record[4]}}};

d683 1
a683 14
bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
  for (auto MapEntry : ImportedModuleFiles) {
    auto *File = MapEntry.first;
    ImportedModuleFileInfo &Info = MapEntry.second;
    if (getModuleFileInfo(File).Signature) {
      if (getModuleFileInfo(File).Signature != Info.StoredSignature)
        // Verify Signature.
        return true;
    } else if (Info.StoredSize != File->getSize() ||
               Info.StoredModTime != File->getModificationTime())
      // Verify Size and ModTime.
      return true;
  }

a758 1
  return false;
d819 1
a819 2
    if (Builder.writeIndex(OutputStream))
      return EC_IOError;
@


1.1.1.8.4.1
log
@Sync with HEAD
@
text
@d24 1
a26 1
#include "llvm/Support/DJB.h"
d41 1
a41 1
    /// The block containing the index.
d45 1
a45 1
  /// Describes the record types in the index.
d47 1
a47 1
    /// Contains version information and potentially other metadata,
d50 1
a50 1
    /// Describes a module, including its file name and dependencies.
d52 1
a52 1
    /// The index for identifiers.
d57 1
a57 1
/// The name of the global index file.
d60 1
a60 1
/// The global index file version.
d69 1
a69 1
/// Trait used to read the identifier index from the on-disk hash
d84 1
a84 1
    return llvm::djbHash(a);
d248 1
a248 1
  /// The main bitstream cursor for the main block.
d292 1
a292 1

d371 1
a371 1
  /// Provides information about a specific module file.
d373 1
a373 1
    /// The numberic ID for this module file.
d376 1
a376 1
    /// The set of modules on which this module depends. Each entry is
d390 1
a390 1
  /// Builder that generates the global module index file.
d401 1
a401 1
    /// Mapping from the imported module file to the imported
d406 1
a406 1
    /// Information about each importing of a module file.
d409 1
a409 1
    /// Mapping from identifiers to the list of module file IDs that
d413 1
a413 1
    /// A mapping from all interesting identifiers to the set of module
d416 2
a417 2

    /// Write the block-info block for the global module index file.
d420 1
a420 1
    /// Retrieve the module file information for the given file.
d438 1
a438 1
    /// Load the contents of the given module file into the builder.
d443 1
a443 1
    /// Write the index to the given bitstream.
d496 1
a496 1
    /// The identifier and whether it is "interesting".
d611 1
a611 1
        // Load stored size/modification time.
a621 4
        // Skip the module name (currently this is only used for prebuilt
        // modules while here we are only dealing with cached).
        Idx += Record[Idx] + 1;

d684 1
a684 1
/// Trait used to generate the identifier index as an on-disk hash
d696 1
a696 1
    return llvm::djbHash(Key);
d702 1
a702 1
    endian::Writer LE(Out, little);
d709 1
a709 1

d718 1
a718 1
      endian::write<uint32_t>(Out, Data[I], little);
d739 1
a739 1

d788 1
a788 1

d796 1
a796 1
      endian::write<uint32_t>(Out, 0, little);
d901 1
a901 1
    // Rename failed; just remove the
d912 1
a912 1
    /// The current position within the identifier lookup table.
d915 1
a915 1
    /// The end position within the identifier lookup table.
@


1.1.1.8.4.2
log
@Mostly merge changes from HEAD upto 20200411
@
text
@@


1.1.1.8.2.1
log
@Sync with HEAD
@
text
@d24 1
a26 1
#include "llvm/Support/DJB.h"
d41 1
a41 1
    /// The block containing the index.
d45 1
a45 1
  /// Describes the record types in the index.
d47 1
a47 1
    /// Contains version information and potentially other metadata,
d50 1
a50 1
    /// Describes a module, including its file name and dependencies.
d52 1
a52 1
    /// The index for identifiers.
d57 1
a57 1
/// The name of the global index file.
d60 1
a60 1
/// The global index file version.
d69 1
a69 1
/// Trait used to read the identifier index from the on-disk hash
d84 1
a84 1
    return llvm::djbHash(a);
d248 1
a248 1
  /// The main bitstream cursor for the main block.
d292 1
a292 1

d371 1
a371 1
  /// Provides information about a specific module file.
d373 1
a373 1
    /// The numberic ID for this module file.
d376 1
a376 1
    /// The set of modules on which this module depends. Each entry is
d390 1
a390 1
  /// Builder that generates the global module index file.
d401 1
a401 1
    /// Mapping from the imported module file to the imported
d406 1
a406 1
    /// Information about each importing of a module file.
d409 1
a409 1
    /// Mapping from identifiers to the list of module file IDs that
d413 1
a413 1
    /// A mapping from all interesting identifiers to the set of module
d416 2
a417 2

    /// Write the block-info block for the global module index file.
d420 1
a420 1
    /// Retrieve the module file information for the given file.
d438 1
a438 1
    /// Load the contents of the given module file into the builder.
d443 1
a443 1
    /// Write the index to the given bitstream.
d496 1
a496 1
    /// The identifier and whether it is "interesting".
d611 1
a611 1
        // Load stored size/modification time.
a621 4
        // Skip the module name (currently this is only used for prebuilt
        // modules while here we are only dealing with cached).
        Idx += Record[Idx] + 1;

d684 1
a684 1
/// Trait used to generate the identifier index as an on-disk hash
d696 1
a696 1
    return llvm::djbHash(Key);
d702 1
a702 1
    endian::Writer LE(Out, little);
d709 1
a709 1

d718 1
a718 1
      endian::write<uint32_t>(Out, Data[I], little);
d739 1
a739 1

d788 1
a788 1

d796 1
a796 1
      endian::write<uint32_t>(Out, 0, little);
d901 1
a901 1
    // Rename failed; just remove the
d912 1
a912 1
    /// The current position within the identifier lookup table.
d915 1
a915 1
    /// The end position within the identifier lookup table.
@


1.1.1.9
log
@Import clang r337282 from trunk
@
text
@d24 1
a26 1
#include "llvm/Support/DJB.h"
d41 1
a41 1
    /// The block containing the index.
d45 1
a45 1
  /// Describes the record types in the index.
d47 1
a47 1
    /// Contains version information and potentially other metadata,
d50 1
a50 1
    /// Describes a module, including its file name and dependencies.
d52 1
a52 1
    /// The index for identifiers.
d57 1
a57 1
/// The name of the global index file.
d60 1
a60 1
/// The global index file version.
d69 1
a69 1
/// Trait used to read the identifier index from the on-disk hash
d84 1
a84 1
    return llvm::djbHash(a);
d248 1
a248 1
  /// The main bitstream cursor for the main block.
d292 1
a292 1

d371 1
a371 1
  /// Provides information about a specific module file.
d373 1
a373 1
    /// The numberic ID for this module file.
d376 1
a376 1
    /// The set of modules on which this module depends. Each entry is
d390 1
a390 1
  /// Builder that generates the global module index file.
d401 1
a401 1
    /// Mapping from the imported module file to the imported
d406 1
a406 1
    /// Information about each importing of a module file.
d409 1
a409 1
    /// Mapping from identifiers to the list of module file IDs that
d413 1
a413 1
    /// A mapping from all interesting identifiers to the set of module
d416 2
a417 2

    /// Write the block-info block for the global module index file.
d420 1
a420 1
    /// Retrieve the module file information for the given file.
d438 1
a438 1
    /// Load the contents of the given module file into the builder.
d443 1
a443 1
    /// Write the index to the given bitstream.
d496 1
a496 1
    /// The identifier and whether it is "interesting".
d611 1
a611 1
        // Load stored size/modification time.
a621 4
        // Skip the module name (currently this is only used for prebuilt
        // modules while here we are only dealing with cached).
        Idx += Record[Idx] + 1;

d684 1
a684 1
/// Trait used to generate the identifier index as an on-disk hash
d696 1
a696 1
    return llvm::djbHash(Key);
d702 1
a702 1
    endian::Writer LE(Out, little);
d709 1
a709 1

d718 1
a718 1
      endian::write<uint32_t>(Out, Data[I], little);
d739 1
a739 1

d788 1
a788 1

d796 1
a796 1
      endian::write<uint32_t>(Out, 0, little);
d901 1
a901 1
    // Rename failed; just remove the
d912 1
a912 1
    /// The current position within the identifier lookup table.
d915 1
a915 1
    /// The end position within the identifier lookup table.
@


1.1.1.10
log
@Mark old LLVM instance as dead.
@
text
@@


1.1.1.4.4.1
log
@file GlobalModuleIndex.cpp was added on branch tls-maxphys on 2014-08-19 23:47:31 +0000
@
text
@d1 885
@


1.1.1.4.4.2
log
@Rebase to HEAD as of a few days ago.
@
text
@a0 885
//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements the GlobalModuleIndex class.
//
//===----------------------------------------------------------------------===//

#include "ASTReaderInternals.h"
#include "clang/Basic/FileManager.h"
#include "clang/Lex/HeaderSearch.h"
#include "clang/Serialization/ASTBitCodes.h"
#include "clang/Serialization/GlobalModuleIndex.h"
#include "clang/Serialization/Module.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Bitcode/BitstreamReader.h"
#include "llvm/Bitcode/BitstreamWriter.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/LockFileManager.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/OnDiskHashTable.h"
#include "llvm/Support/Path.h"
#include <cstdio>
using namespace clang;
using namespace serialization;

//----------------------------------------------------------------------------//
// Shared constants
//----------------------------------------------------------------------------//
namespace {
  enum {
    /// \brief The block containing the index.
    GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
  };

  /// \brief Describes the record types in the index.
  enum IndexRecordTypes {
    /// \brief Contains version information and potentially other metadata,
    /// used to determine if we can read this global index file.
    INDEX_METADATA,
    /// \brief Describes a module, including its file name and dependencies.
    MODULE,
    /// \brief The index for identifiers.
    IDENTIFIER_INDEX
  };
}

/// \brief The name of the global index file.
static const char * const IndexFileName = "modules.idx";

/// \brief The global index file version.
static const unsigned CurrentVersion = 1;

//----------------------------------------------------------------------------//
// Global module index reader.
//----------------------------------------------------------------------------//

namespace {

/// \brief Trait used to read the identifier index from the on-disk hash
/// table.
class IdentifierIndexReaderTrait {
public:
  typedef StringRef external_key_type;
  typedef StringRef internal_key_type;
  typedef SmallVector<unsigned, 2> data_type;
  typedef unsigned hash_value_type;
  typedef unsigned offset_type;

  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
    return a == b;
  }

  static hash_value_type ComputeHash(const internal_key_type& a) {
    return llvm::HashString(a);
  }

  static std::pair<unsigned, unsigned>
  ReadKeyDataLength(const unsigned char*& d) {
    using namespace llvm::support;
    unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
    unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
    return std::make_pair(KeyLen, DataLen);
  }

  static const internal_key_type&
  GetInternalKey(const external_key_type& x) { return x; }

  static const external_key_type&
  GetExternalKey(const internal_key_type& x) { return x; }

  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
    return StringRef((const char *)d, n);
  }

  static data_type ReadData(const internal_key_type& k,
                            const unsigned char* d,
                            unsigned DataLen) {
    using namespace llvm::support;

    data_type Result;
    while (DataLen > 0) {
      unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
      Result.push_back(ID);
      DataLen -= 4;
    }

    return Result;
  }
};

typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
    IdentifierIndexTable;

}

GlobalModuleIndex::GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
                                     llvm::BitstreamCursor Cursor)
  : Buffer(Buffer), IdentifierIndex(),
    NumIdentifierLookups(), NumIdentifierLookupHits()
{
  // Read the global index.
  bool InGlobalIndexBlock = false;
  bool Done = false;
  while (!Done) {
    llvm::BitstreamEntry Entry = Cursor.advance();

    switch (Entry.Kind) {
    case llvm::BitstreamEntry::Error:
      return;

    case llvm::BitstreamEntry::EndBlock:
      if (InGlobalIndexBlock) {
        InGlobalIndexBlock = false;
        Done = true;
        continue;
      }
      return;


    case llvm::BitstreamEntry::Record:
      // Entries in the global index block are handled below.
      if (InGlobalIndexBlock)
        break;

      return;

    case llvm::BitstreamEntry::SubBlock:
      if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
        if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
          return;

        InGlobalIndexBlock = true;
      } else if (Cursor.SkipBlock()) {
        return;
      }
      continue;
    }

    SmallVector<uint64_t, 64> Record;
    StringRef Blob;
    switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
    case INDEX_METADATA:
      // Make sure that the version matches.
      if (Record.size() < 1 || Record[0] != CurrentVersion)
        return;
      break;

    case MODULE: {
      unsigned Idx = 0;
      unsigned ID = Record[Idx++];

      // Make room for this module's information.
      if (ID == Modules.size())
        Modules.push_back(ModuleInfo());
      else
        Modules.resize(ID + 1);

      // Size/modification time for this module file at the time the
      // global index was built.
      Modules[ID].Size = Record[Idx++];
      Modules[ID].ModTime = Record[Idx++];

      // File name.
      unsigned NameLen = Record[Idx++];
      Modules[ID].FileName.assign(Record.begin() + Idx,
                                  Record.begin() + Idx + NameLen);
      Idx += NameLen;

      // Dependencies
      unsigned NumDeps = Record[Idx++];
      Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
                                      Record.begin() + Idx,
                                      Record.begin() + Idx + NumDeps);
      Idx += NumDeps;

      // Make sure we're at the end of the record.
      assert(Idx == Record.size() && "More module info?");

      // Record this module as an unresolved module.
      // FIXME: this doesn't work correctly for module names containing path
      // separators.
      StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
      // Remove the -<hash of ModuleMapPath>
      ModuleName = ModuleName.rsplit('-').first;
      UnresolvedModules[ModuleName] = ID;
      break;
    }

    case IDENTIFIER_INDEX:
      // Wire up the identifier index.
      if (Record[0]) {
        IdentifierIndex = IdentifierIndexTable::Create(
            (const unsigned char *)Blob.data() + Record[0],
            (const unsigned char *)Blob.data() + sizeof(uint32_t),
            (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
      }
      break;
    }
  }
}

GlobalModuleIndex::~GlobalModuleIndex() {
  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
}

std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
GlobalModuleIndex::readIndex(StringRef Path) {
  // Load the index file, if it's there.
  llvm::SmallString<128> IndexPath;
  IndexPath += Path;
  llvm::sys::path::append(IndexPath, IndexFileName);

  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
      llvm::MemoryBuffer::getFile(IndexPath.c_str());
  if (!BufferOrErr)
    return std::make_pair(nullptr, EC_NotFound);
  std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());

  /// \brief The bitstream reader from which we'll read the AST file.
  llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(),
                               (const unsigned char *)Buffer->getBufferEnd());

  /// \brief The main bitstream cursor for the main block.
  llvm::BitstreamCursor Cursor(Reader);

  // Sniff for the signature.
  if (Cursor.Read(8) != 'B' ||
      Cursor.Read(8) != 'C' ||
      Cursor.Read(8) != 'G' ||
      Cursor.Read(8) != 'I') {
    return std::make_pair(nullptr, EC_IOError);
  }

  return std::make_pair(new GlobalModuleIndex(Buffer.release(), Cursor),
                        EC_None);
}

void
GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
  ModuleFiles.clear();
  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
    if (ModuleFile *MF = Modules[I].File)
      ModuleFiles.push_back(MF);
  }
}

void GlobalModuleIndex::getModuleDependencies(
       ModuleFile *File,
       SmallVectorImpl<ModuleFile *> &Dependencies) {
  // Look for information about this module file.
  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
    = ModulesByFile.find(File);
  if (Known == ModulesByFile.end())
    return;

  // Record dependencies.
  Dependencies.clear();
  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
    if (ModuleFile *MF = Modules[I].File)
      Dependencies.push_back(MF);
  }
}

bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
  Hits.clear();
  
  // If there's no identifier index, there is nothing we can do.
  if (!IdentifierIndex)
    return false;

  // Look into the identifier index.
  ++NumIdentifierLookups;
  IdentifierIndexTable &Table
    = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
  IdentifierIndexTable::iterator Known = Table.find(Name);
  if (Known == Table.end()) {
    return true;
  }

  SmallVector<unsigned, 2> ModuleIDs = *Known;
  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
    if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
      Hits.insert(MF);
  }

  ++NumIdentifierLookupHits;
  return true;
}

bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
  // Look for the module in the global module index based on the module name.
  StringRef Name = File->ModuleName;
  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
  if (Known == UnresolvedModules.end()) {
    return true;
  }

  // Rectify this module with the global module index.
  ModuleInfo &Info = Modules[Known->second];

  //  If the size and modification time match what we expected, record this
  // module file.
  bool Failed = true;
  if (File->File->getSize() == Info.Size &&
      File->File->getModificationTime() == Info.ModTime) {
    Info.File = File;
    ModulesByFile[File] = Known->second;

    Failed = false;
  }

  // One way or another, we have resolved this module file.
  UnresolvedModules.erase(Known);
  return Failed;
}

void GlobalModuleIndex::printStats() {
  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
  if (NumIdentifierLookups) {
    fprintf(stderr, "  %u / %u identifier lookups succeeded (%f%%)\n",
            NumIdentifierLookupHits, NumIdentifierLookups,
            (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
  }
  std::fprintf(stderr, "\n");
}

void GlobalModuleIndex::dump() {
  llvm::errs() << "*** Global Module Index Dump:\n";
  llvm::errs() << "Module files:\n";
  for (auto &MI : Modules) {
    llvm::errs() << "** " << MI.FileName << "\n";
    if (MI.File)
      MI.File->dump();
    else
      llvm::errs() << "\n";
  }
  llvm::errs() << "\n";
}

//----------------------------------------------------------------------------//
// Global module index writer.
//----------------------------------------------------------------------------//

namespace {
  /// \brief Provides information about a specific module file.
  struct ModuleFileInfo {
    /// \brief The numberic ID for this module file.
    unsigned ID;

    /// \brief The set of modules on which this module depends. Each entry is
    /// a module ID.
    SmallVector<unsigned, 4> Dependencies;
  };

  /// \brief Builder that generates the global module index file.
  class GlobalModuleIndexBuilder {
    FileManager &FileMgr;

    /// \brief Mapping from files to module file information.
    typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;

    /// \brief Information about each of the known module files.
    ModuleFilesMap ModuleFiles;

    /// \brief Mapping from identifiers to the list of module file IDs that
    /// consider this identifier to be interesting.
    typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;

    /// \brief A mapping from all interesting identifiers to the set of module
    /// files in which those identifiers are considered interesting.
    InterestingIdentifierMap InterestingIdentifiers;
    
    /// \brief Write the block-info block for the global module index file.
    void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);

    /// \brief Retrieve the module file information for the given file.
    ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
      llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
        = ModuleFiles.find(File);
      if (Known != ModuleFiles.end())
        return Known->second;

      unsigned NewID = ModuleFiles.size();
      ModuleFileInfo &Info = ModuleFiles[File];
      Info.ID = NewID;
      return Info;
    }

  public:
    explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){}

    /// \brief Load the contents of the given module file into the builder.
    ///
    /// \returns true if an error occurred, false otherwise.
    bool loadModuleFile(const FileEntry *File);

    /// \brief Write the index to the given bitstream.
    void writeIndex(llvm::BitstreamWriter &Stream);
  };
}

static void emitBlockID(unsigned ID, const char *Name,
                        llvm::BitstreamWriter &Stream,
                        SmallVectorImpl<uint64_t> &Record) {
  Record.clear();
  Record.push_back(ID);
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);

  // Emit the block name if present.
  if (!Name || Name[0] == 0) return;
  Record.clear();
  while (*Name)
    Record.push_back(*Name++);
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
}

static void emitRecordID(unsigned ID, const char *Name,
                         llvm::BitstreamWriter &Stream,
                         SmallVectorImpl<uint64_t> &Record) {
  Record.clear();
  Record.push_back(ID);
  while (*Name)
    Record.push_back(*Name++);
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
}

void
GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
  SmallVector<uint64_t, 64> Record;
  Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);

#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
#define RECORD(X) emitRecordID(X, #X, Stream, Record)
  BLOCK(GLOBAL_INDEX_BLOCK);
  RECORD(INDEX_METADATA);
  RECORD(MODULE);
  RECORD(IDENTIFIER_INDEX);
#undef RECORD
#undef BLOCK

  Stream.ExitBlock();
}

namespace {
  class InterestingASTIdentifierLookupTrait
    : public serialization::reader::ASTIdentifierLookupTraitBase {

  public:
    /// \brief The identifier and whether it is "interesting".
    typedef std::pair<StringRef, bool> data_type;

    data_type ReadData(const internal_key_type& k,
                       const unsigned char* d,
                       unsigned DataLen) {
      // The first bit indicates whether this identifier is interesting.
      // That's all we care about.
      using namespace llvm::support;
      unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
      bool IsInteresting = RawID & 0x01;
      return std::make_pair(k, IsInteresting);
    }
  };
}

bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
  // Open the module file.
  std::unique_ptr<llvm::MemoryBuffer> Buffer;
  std::string ErrorStr;
  Buffer.reset(FileMgr.getBufferForFile(File, &ErrorStr, /*isVolatile=*/true));
  if (!Buffer) {
    return true;
  }

  // Initialize the input stream
  llvm::BitstreamReader InStreamFile;
  llvm::BitstreamCursor InStream;
  InStreamFile.init((const unsigned char *)Buffer->getBufferStart(),
                  (const unsigned char *)Buffer->getBufferEnd());
  InStream.init(InStreamFile);

  // Sniff for the signature.
  if (InStream.Read(8) != 'C' ||
      InStream.Read(8) != 'P' ||
      InStream.Read(8) != 'C' ||
      InStream.Read(8) != 'H') {
    return true;
  }

  // Record this module file and assign it a unique ID (if it doesn't have
  // one already).
  unsigned ID = getModuleFileInfo(File).ID;

  // Search for the blocks and records we care about.
  enum { Other, ControlBlock, ASTBlock } State = Other;
  bool Done = false;
  while (!Done) {
    llvm::BitstreamEntry Entry = InStream.advance();
    switch (Entry.Kind) {
    case llvm::BitstreamEntry::Error:
      Done = true;
      continue;

    case llvm::BitstreamEntry::Record:
      // In the 'other' state, just skip the record. We don't care.
      if (State == Other) {
        InStream.skipRecord(Entry.ID);
        continue;
      }

      // Handle potentially-interesting records below.
      break;

    case llvm::BitstreamEntry::SubBlock:
      if (Entry.ID == CONTROL_BLOCK_ID) {
        if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
          return true;

        // Found the control block.
        State = ControlBlock;
        continue;
      }

      if (Entry.ID == AST_BLOCK_ID) {
        if (InStream.EnterSubBlock(AST_BLOCK_ID))
          return true;

        // Found the AST block.
        State = ASTBlock;
        continue;
      }

      if (InStream.SkipBlock())
        return true;

      continue;

    case llvm::BitstreamEntry::EndBlock:
      State = Other;
      continue;
    }

    // Read the given record.
    SmallVector<uint64_t, 64> Record;
    StringRef Blob;
    unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);

    // Handle module dependencies.
    if (State == ControlBlock && Code == IMPORTS) {
      // Load each of the imported PCH files.
      unsigned Idx = 0, N = Record.size();
      while (Idx < N) {
        // Read information about the AST file.

        // Skip the imported kind
        ++Idx;

        // Skip the import location
        ++Idx;

        // Load stored size/modification time. 
        off_t StoredSize = (off_t)Record[Idx++];
        time_t StoredModTime = (time_t)Record[Idx++];

        // Retrieve the imported file name.
        unsigned Length = Record[Idx++];
        SmallString<128> ImportedFile(Record.begin() + Idx,
                                      Record.begin() + Idx + Length);
        Idx += Length;

        // Find the imported module file.
        const FileEntry *DependsOnFile
          = FileMgr.getFile(ImportedFile, /*openFile=*/false,
                            /*cacheFailure=*/false);
        if (!DependsOnFile ||
            (StoredSize != DependsOnFile->getSize()) ||
            (StoredModTime != DependsOnFile->getModificationTime()))
          return true;

        // Record the dependency.
        unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
        getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
      }

      continue;
    }

    // Handle the identifier table
    if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
      typedef llvm::OnDiskIterableChainedHashTable<
          InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
      std::unique_ptr<InterestingIdentifierTable> Table(
          InterestingIdentifierTable::Create(
              (const unsigned char *)Blob.data() + Record[0],
              (const unsigned char *)Blob.data() + sizeof(uint32_t),
              (const unsigned char *)Blob.data()));
      for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
                                                     DEnd = Table->data_end();
           D != DEnd; ++D) {
        std::pair<StringRef, bool> Ident = *D;
        if (Ident.second)
          InterestingIdentifiers[Ident.first].push_back(ID);
        else
          (void)InterestingIdentifiers[Ident.first];
      }
    }

    // We don't care about this record.
  }

  return false;
}

namespace {

/// \brief Trait used to generate the identifier index as an on-disk hash
/// table.
class IdentifierIndexWriterTrait {
public:
  typedef StringRef key_type;
  typedef StringRef key_type_ref;
  typedef SmallVector<unsigned, 2> data_type;
  typedef const SmallVector<unsigned, 2> &data_type_ref;
  typedef unsigned hash_value_type;
  typedef unsigned offset_type;

  static hash_value_type ComputeHash(key_type_ref Key) {
    return llvm::HashString(Key);
  }

  std::pair<unsigned,unsigned>
  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
    using namespace llvm::support;
    endian::Writer<little> LE(Out);
    unsigned KeyLen = Key.size();
    unsigned DataLen = Data.size() * 4;
    LE.write<uint16_t>(KeyLen);
    LE.write<uint16_t>(DataLen);
    return std::make_pair(KeyLen, DataLen);
  }
  
  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
    Out.write(Key.data(), KeyLen);
  }

  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
                unsigned DataLen) {
    using namespace llvm::support;
    for (unsigned I = 0, N = Data.size(); I != N; ++I)
      endian::Writer<little>(Out).write<uint32_t>(Data[I]);
  }
};

}

void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
  using namespace llvm;
  
  // Emit the file header.
  Stream.Emit((unsigned)'B', 8);
  Stream.Emit((unsigned)'C', 8);
  Stream.Emit((unsigned)'G', 8);
  Stream.Emit((unsigned)'I', 8);

  // Write the block-info block, which describes the records in this bitcode
  // file.
  emitBlockInfoBlock(Stream);

  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);

  // Write the metadata.
  SmallVector<uint64_t, 2> Record;
  Record.push_back(CurrentVersion);
  Stream.EmitRecord(INDEX_METADATA, Record);

  // Write the set of known module files.
  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
                                MEnd = ModuleFiles.end();
       M != MEnd; ++M) {
    Record.clear();
    Record.push_back(M->second.ID);
    Record.push_back(M->first->getSize());
    Record.push_back(M->first->getModificationTime());

    // File name
    StringRef Name(M->first->getName());
    Record.push_back(Name.size());
    Record.append(Name.begin(), Name.end());

    // Dependencies
    Record.push_back(M->second.Dependencies.size());
    Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
    Stream.EmitRecord(MODULE, Record);
  }

  // Write the identifier -> module file mapping.
  {
    llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
    IdentifierIndexWriterTrait Trait;

    // Populate the hash table.
    for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
                                            IEnd = InterestingIdentifiers.end();
         I != IEnd; ++I) {
      Generator.insert(I->first(), I->second, Trait);
    }
    
    // Create the on-disk hash table in a buffer.
    SmallString<4096> IdentifierTable;
    uint32_t BucketOffset;
    {
      using namespace llvm::support;
      llvm::raw_svector_ostream Out(IdentifierTable);
      // Make sure that no bucket is at offset 0
      endian::Writer<little>(Out).write<uint32_t>(0);
      BucketOffset = Generator.Emit(Out, Trait);
    }

    // Create a blob abbreviation
    BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
    Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
    unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);

    // Write the identifier table
    Record.clear();
    Record.push_back(IDENTIFIER_INDEX);
    Record.push_back(BucketOffset);
    Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str());
  }

  Stream.ExitBlock();
}

GlobalModuleIndex::ErrorCode
GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) {
  llvm::SmallString<128> IndexPath;
  IndexPath += Path;
  llvm::sys::path::append(IndexPath, IndexFileName);

  // Coordinate building the global index file with other processes that might
  // try to do the same.
  llvm::LockFileManager Locked(IndexPath);
  switch (Locked) {
  case llvm::LockFileManager::LFS_Error:
    return EC_IOError;

  case llvm::LockFileManager::LFS_Owned:
    // We're responsible for building the index ourselves. Do so below.
    break;

  case llvm::LockFileManager::LFS_Shared:
    // Someone else is responsible for building the index. We don't care
    // when they finish, so we're done.
    return EC_Building;
  }

  // The module index builder.
  GlobalModuleIndexBuilder Builder(FileMgr);
  
  // Load each of the module files.
  std::error_code EC;
  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
       D != DEnd && !EC;
       D.increment(EC)) {
    // If this isn't a module file, we don't care.
    if (llvm::sys::path::extension(D->path()) != ".pcm") {
      // ... unless it's a .pcm.lock file, which indicates that someone is
      // in the process of rebuilding a module. They'll rebuild the index
      // at the end of that translation unit, so we don't have to.
      if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
        return EC_Building;

      continue;
    }

    // If we can't find the module file, skip it.
    const FileEntry *ModuleFile = FileMgr.getFile(D->path());
    if (!ModuleFile)
      continue;

    // Load this module file.
    if (Builder.loadModuleFile(ModuleFile))
      return EC_IOError;
  }

  // The output buffer, into which the global index will be written.
  SmallVector<char, 16> OutputBuffer;
  {
    llvm::BitstreamWriter OutputStream(OutputBuffer);
    Builder.writeIndex(OutputStream);
  }

  // Write the global index file to a temporary file.
  llvm::SmallString<128> IndexTmpPath;
  int TmpFD;
  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
                                      IndexTmpPath))
    return EC_IOError;

  // Open the temporary global index file for output.
  llvm::raw_fd_ostream Out(TmpFD, true);
  if (Out.has_error())
    return EC_IOError;

  // Write the index.
  Out.write(OutputBuffer.data(), OutputBuffer.size());
  Out.close();
  if (Out.has_error())
    return EC_IOError;

  // Remove the old index file. It isn't relevant any more.
  llvm::sys::fs::remove(IndexPath.str());

  // Rename the newly-written index file to the proper name.
  if (llvm::sys::fs::rename(IndexTmpPath.str(), IndexPath.str())) {
    // Rename failed; just remove the 
    llvm::sys::fs::remove(IndexTmpPath.str());
    return EC_IOError;
  }

  // We're done.
  return EC_None;
}

namespace {
  class GlobalIndexIdentifierIterator : public IdentifierIterator {
    /// \brief The current position within the identifier lookup table.
    IdentifierIndexTable::key_iterator Current;

    /// \brief The end position within the identifier lookup table.
    IdentifierIndexTable::key_iterator End;

  public:
    explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
      Current = Idx.key_begin();
      End = Idx.key_end();
    }

    StringRef Next() override {
      if (Current == End)
        return StringRef();

      StringRef Result = *Current;
      ++Current;
      return Result;
    }
  };
}

IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
  IdentifierIndexTable &Table =
    *static_cast<IdentifierIndexTable *>(IdentifierIndex);
  return new GlobalIndexIdentifierIterator(Table);
}
@


1.1.1.2.4.1
log
@file GlobalModuleIndex.cpp was added on branch yamt-pagecache on 2014-05-22 16:18:30 +0000
@
text
@d1 852
@


1.1.1.2.4.2
log
@sync with head.

for a reference, the tree before this commit was tagged
as yamt-pagecache-tag8.

this commit was splitted into small chunks to avoid
a limitation of cvs.  ("Protocol error: too many arguments")
@
text
@a0 852
//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements the GlobalModuleIndex class.
//
//===----------------------------------------------------------------------===//

#include "ASTReaderInternals.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/OnDiskHashTable.h"
#include "clang/Serialization/ASTBitCodes.h"
#include "clang/Serialization/GlobalModuleIndex.h"
#include "clang/Serialization/Module.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Bitcode/BitstreamReader.h"
#include "llvm/Bitcode/BitstreamWriter.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/LockFileManager.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include <cstdio>
using namespace clang;
using namespace serialization;

//----------------------------------------------------------------------------//
// Shared constants
//----------------------------------------------------------------------------//
namespace {
  enum {
    /// \brief The block containing the index.
    GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
  };

  /// \brief Describes the record types in the index.
  enum IndexRecordTypes {
    /// \brief Contains version information and potentially other metadata,
    /// used to determine if we can read this global index file.
    INDEX_METADATA,
    /// \brief Describes a module, including its file name and dependencies.
    MODULE,
    /// \brief The index for identifiers.
    IDENTIFIER_INDEX
  };
}

/// \brief The name of the global index file.
static const char * const IndexFileName = "modules.idx";

/// \brief The global index file version.
static const unsigned CurrentVersion = 1;

//----------------------------------------------------------------------------//
// Global module index reader.
//----------------------------------------------------------------------------//

namespace {

/// \brief Trait used to read the identifier index from the on-disk hash
/// table.
class IdentifierIndexReaderTrait {
public:
  typedef StringRef external_key_type;
  typedef StringRef internal_key_type;
  typedef SmallVector<unsigned, 2> data_type;

  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
    return a == b;
  }

  static unsigned ComputeHash(const internal_key_type& a) {
    return llvm::HashString(a);
  }

  static std::pair<unsigned, unsigned>
  ReadKeyDataLength(const unsigned char*& d) {
    using namespace clang::io;
    unsigned KeyLen = ReadUnalignedLE16(d);
    unsigned DataLen = ReadUnalignedLE16(d);
    return std::make_pair(KeyLen, DataLen);
  }

  static const internal_key_type&
  GetInternalKey(const external_key_type& x) { return x; }

  static const external_key_type&
  GetExternalKey(const internal_key_type& x) { return x; }

  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
    return StringRef((const char *)d, n);
  }

  static data_type ReadData(const internal_key_type& k,
                            const unsigned char* d,
                            unsigned DataLen) {
    using namespace clang::io;

    data_type Result;
    while (DataLen > 0) {
      unsigned ID = ReadUnalignedLE32(d);
      Result.push_back(ID);
      DataLen -= 4;
    }

    return Result;
  }
};

typedef OnDiskChainedHashTable<IdentifierIndexReaderTrait> IdentifierIndexTable;

}

GlobalModuleIndex::GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
                                     llvm::BitstreamCursor Cursor)
  : Buffer(Buffer), IdentifierIndex(),
    NumIdentifierLookups(), NumIdentifierLookupHits()
{
  // Read the global index.
  bool InGlobalIndexBlock = false;
  bool Done = false;
  while (!Done) {
    llvm::BitstreamEntry Entry = Cursor.advance();

    switch (Entry.Kind) {
    case llvm::BitstreamEntry::Error:
      return;

    case llvm::BitstreamEntry::EndBlock:
      if (InGlobalIndexBlock) {
        InGlobalIndexBlock = false;
        Done = true;
        continue;
      }
      return;


    case llvm::BitstreamEntry::Record:
      // Entries in the global index block are handled below.
      if (InGlobalIndexBlock)
        break;

      return;

    case llvm::BitstreamEntry::SubBlock:
      if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
        if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
          return;

        InGlobalIndexBlock = true;
      } else if (Cursor.SkipBlock()) {
        return;
      }
      continue;
    }

    SmallVector<uint64_t, 64> Record;
    StringRef Blob;
    switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
    case INDEX_METADATA:
      // Make sure that the version matches.
      if (Record.size() < 1 || Record[0] != CurrentVersion)
        return;
      break;

    case MODULE: {
      unsigned Idx = 0;
      unsigned ID = Record[Idx++];

      // Make room for this module's information.
      if (ID == Modules.size())
        Modules.push_back(ModuleInfo());
      else
        Modules.resize(ID + 1);

      // Size/modification time for this module file at the time the
      // global index was built.
      Modules[ID].Size = Record[Idx++];
      Modules[ID].ModTime = Record[Idx++];

      // File name.
      unsigned NameLen = Record[Idx++];
      Modules[ID].FileName.assign(Record.begin() + Idx,
                                  Record.begin() + Idx + NameLen);
      Idx += NameLen;

      // Dependencies
      unsigned NumDeps = Record[Idx++];
      Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
                                      Record.begin() + Idx,
                                      Record.begin() + Idx + NumDeps);
      Idx += NumDeps;

      // Make sure we're at the end of the record.
      assert(Idx == Record.size() && "More module info?");

      // Record this module as an unresolved module.
      UnresolvedModules[llvm::sys::path::stem(Modules[ID].FileName)] = ID;
      break;
    }

    case IDENTIFIER_INDEX:
      // Wire up the identifier index.
      if (Record[0]) {
        IdentifierIndex = IdentifierIndexTable::Create(
                            (const unsigned char *)Blob.data() + Record[0],
                            (const unsigned char *)Blob.data(),
                            IdentifierIndexReaderTrait());
      }
      break;
    }
  }
}

GlobalModuleIndex::~GlobalModuleIndex() { }

std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
GlobalModuleIndex::readIndex(StringRef Path) {
  // Load the index file, if it's there.
  llvm::SmallString<128> IndexPath;
  IndexPath += Path;
  llvm::sys::path::append(IndexPath, IndexFileName);

  llvm::OwningPtr<llvm::MemoryBuffer> Buffer;
  if (llvm::MemoryBuffer::getFile(IndexPath.c_str(), Buffer) !=
      llvm::errc::success)
    return std::make_pair((GlobalModuleIndex *)0, EC_NotFound);

  /// \brief The bitstream reader from which we'll read the AST file.
  llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(),
                               (const unsigned char *)Buffer->getBufferEnd());

  /// \brief The main bitstream cursor for the main block.
  llvm::BitstreamCursor Cursor(Reader);

  // Sniff for the signature.
  if (Cursor.Read(8) != 'B' ||
      Cursor.Read(8) != 'C' ||
      Cursor.Read(8) != 'G' ||
      Cursor.Read(8) != 'I') {
    return std::make_pair((GlobalModuleIndex *)0, EC_IOError);
  }
  
  return std::make_pair(new GlobalModuleIndex(Buffer.take(), Cursor), EC_None);
}

void
GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
  ModuleFiles.clear();
  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
    if (ModuleFile *MF = Modules[I].File)
      ModuleFiles.push_back(MF);
  }
}

void GlobalModuleIndex::getModuleDependencies(
       ModuleFile *File,
       SmallVectorImpl<ModuleFile *> &Dependencies) {
  // Look for information about this module file.
  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
    = ModulesByFile.find(File);
  if (Known == ModulesByFile.end())
    return;

  // Record dependencies.
  Dependencies.clear();
  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
    if (ModuleFile *MF = Modules[I].File)
      Dependencies.push_back(MF);
  }
}

bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
  Hits.clear();
  
  // If there's no identifier index, there is nothing we can do.
  if (!IdentifierIndex)
    return false;

  // Look into the identifier index.
  ++NumIdentifierLookups;
  IdentifierIndexTable &Table
    = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
  IdentifierIndexTable::iterator Known = Table.find(Name);
  if (Known == Table.end()) {
    return true;
  }

  SmallVector<unsigned, 2> ModuleIDs = *Known;
  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
    if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
      Hits.insert(MF);
  }

  ++NumIdentifierLookupHits;
  return true;
}

bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
  // Look for the module in the global module index based on the module name.
  StringRef Name = llvm::sys::path::stem(File->FileName);
  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
  if (Known == UnresolvedModules.end()) {
    return true;
  }

  // Rectify this module with the global module index.
  ModuleInfo &Info = Modules[Known->second];

  //  If the size and modification time match what we expected, record this
  // module file.
  bool Failed = true;
  if (File->File->getSize() == Info.Size &&
      File->File->getModificationTime() == Info.ModTime) {
    Info.File = File;
    ModulesByFile[File] = Known->second;

    Failed = false;
  }

  // One way or another, we have resolved this module file.
  UnresolvedModules.erase(Known);
  return Failed;
}

void GlobalModuleIndex::printStats() {
  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
  if (NumIdentifierLookups) {
    fprintf(stderr, "  %u / %u identifier lookups succeeded (%f%%)\n",
            NumIdentifierLookupHits, NumIdentifierLookups,
            (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
  }
  std::fprintf(stderr, "\n");
}

//----------------------------------------------------------------------------//
// Global module index writer.
//----------------------------------------------------------------------------//

namespace {
  /// \brief Provides information about a specific module file.
  struct ModuleFileInfo {
    /// \brief The numberic ID for this module file.
    unsigned ID;

    /// \brief The set of modules on which this module depends. Each entry is
    /// a module ID.
    SmallVector<unsigned, 4> Dependencies;
  };

  /// \brief Builder that generates the global module index file.
  class GlobalModuleIndexBuilder {
    FileManager &FileMgr;

    /// \brief Mapping from files to module file information.
    typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;

    /// \brief Information about each of the known module files.
    ModuleFilesMap ModuleFiles;

    /// \brief Mapping from identifiers to the list of module file IDs that
    /// consider this identifier to be interesting.
    typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;

    /// \brief A mapping from all interesting identifiers to the set of module
    /// files in which those identifiers are considered interesting.
    InterestingIdentifierMap InterestingIdentifiers;
    
    /// \brief Write the block-info block for the global module index file.
    void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);

    /// \brief Retrieve the module file information for the given file.
    ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
      llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
        = ModuleFiles.find(File);
      if (Known != ModuleFiles.end())
        return Known->second;

      unsigned NewID = ModuleFiles.size();
      ModuleFileInfo &Info = ModuleFiles[File];
      Info.ID = NewID;
      return Info;
    }

  public:
    explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){}

    /// \brief Load the contents of the given module file into the builder.
    ///
    /// \returns true if an error occurred, false otherwise.
    bool loadModuleFile(const FileEntry *File);

    /// \brief Write the index to the given bitstream.
    void writeIndex(llvm::BitstreamWriter &Stream);
  };
}

static void emitBlockID(unsigned ID, const char *Name,
                        llvm::BitstreamWriter &Stream,
                        SmallVectorImpl<uint64_t> &Record) {
  Record.clear();
  Record.push_back(ID);
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);

  // Emit the block name if present.
  if (Name == 0 || Name[0] == 0) return;
  Record.clear();
  while (*Name)
    Record.push_back(*Name++);
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
}

static void emitRecordID(unsigned ID, const char *Name,
                         llvm::BitstreamWriter &Stream,
                         SmallVectorImpl<uint64_t> &Record) {
  Record.clear();
  Record.push_back(ID);
  while (*Name)
    Record.push_back(*Name++);
  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
}

void
GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
  SmallVector<uint64_t, 64> Record;
  Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);

#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
#define RECORD(X) emitRecordID(X, #X, Stream, Record)
  BLOCK(GLOBAL_INDEX_BLOCK);
  RECORD(INDEX_METADATA);
  RECORD(MODULE);
  RECORD(IDENTIFIER_INDEX);
#undef RECORD
#undef BLOCK

  Stream.ExitBlock();
}

namespace {
  class InterestingASTIdentifierLookupTrait
    : public serialization::reader::ASTIdentifierLookupTraitBase {

  public:
    /// \brief The identifier and whether it is "interesting".
    typedef std::pair<StringRef, bool> data_type;

    data_type ReadData(const internal_key_type& k,
                       const unsigned char* d,
                       unsigned DataLen) {
      // The first bit indicates whether this identifier is interesting.
      // That's all we care about.
      using namespace clang::io;
      unsigned RawID = ReadUnalignedLE32(d);
      bool IsInteresting = RawID & 0x01;
      return std::make_pair(k, IsInteresting);
    }
  };
}

bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
  // Open the module file.
  OwningPtr<llvm::MemoryBuffer> Buffer;
  std::string ErrorStr;
  Buffer.reset(FileMgr.getBufferForFile(File, &ErrorStr, /*isVolatile=*/true));
  if (!Buffer) {
    return true;
  }

  // Initialize the input stream
  llvm::BitstreamReader InStreamFile;
  llvm::BitstreamCursor InStream;
  InStreamFile.init((const unsigned char *)Buffer->getBufferStart(),
                  (const unsigned char *)Buffer->getBufferEnd());
  InStream.init(InStreamFile);

  // Sniff for the signature.
  if (InStream.Read(8) != 'C' ||
      InStream.Read(8) != 'P' ||
      InStream.Read(8) != 'C' ||
      InStream.Read(8) != 'H') {
    return true;
  }

  // Record this module file and assign it a unique ID (if it doesn't have
  // one already).
  unsigned ID = getModuleFileInfo(File).ID;

  // Search for the blocks and records we care about.
  enum { Other, ControlBlock, ASTBlock } State = Other;
  bool Done = false;
  while (!Done) {
    llvm::BitstreamEntry Entry = InStream.advance();
    switch (Entry.Kind) {
    case llvm::BitstreamEntry::Error:
      Done = true;
      continue;

    case llvm::BitstreamEntry::Record:
      // In the 'other' state, just skip the record. We don't care.
      if (State == Other) {
        InStream.skipRecord(Entry.ID);
        continue;
      }

      // Handle potentially-interesting records below.
      break;

    case llvm::BitstreamEntry::SubBlock:
      if (Entry.ID == CONTROL_BLOCK_ID) {
        if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
          return true;

        // Found the control block.
        State = ControlBlock;
        continue;
      }

      if (Entry.ID == AST_BLOCK_ID) {
        if (InStream.EnterSubBlock(AST_BLOCK_ID))
          return true;

        // Found the AST block.
        State = ASTBlock;
        continue;
      }

      if (InStream.SkipBlock())
        return true;

      continue;

    case llvm::BitstreamEntry::EndBlock:
      State = Other;
      continue;
    }

    // Read the given record.
    SmallVector<uint64_t, 64> Record;
    StringRef Blob;
    unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);

    // Handle module dependencies.
    if (State == ControlBlock && Code == IMPORTS) {
      // Load each of the imported PCH files.
      unsigned Idx = 0, N = Record.size();
      while (Idx < N) {
        // Read information about the AST file.

        // Skip the imported kind
        ++Idx;

        // Skip the import location
        ++Idx;

        // Load stored size/modification time. 
        off_t StoredSize = (off_t)Record[Idx++];
        time_t StoredModTime = (time_t)Record[Idx++];

        // Retrieve the imported file name.
        unsigned Length = Record[Idx++];
        SmallString<128> ImportedFile(Record.begin() + Idx,
                                      Record.begin() + Idx + Length);
        Idx += Length;

        // Find the imported module file.
        const FileEntry *DependsOnFile
          = FileMgr.getFile(ImportedFile, /*openFile=*/false,
                            /*cacheFailure=*/false);
        if (!DependsOnFile ||
            (StoredSize != DependsOnFile->getSize()) ||
            (StoredModTime != DependsOnFile->getModificationTime()))
          return true;

        // Record the dependency.
        unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
        getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
      }

      continue;
    }

    // Handle the identifier table
    if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
      typedef OnDiskChainedHashTable<InterestingASTIdentifierLookupTrait>
        InterestingIdentifierTable;
      llvm::OwningPtr<InterestingIdentifierTable>
        Table(InterestingIdentifierTable::Create(
                (const unsigned char *)Blob.data() + Record[0],
                (const unsigned char *)Blob.data()));
      for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
                                                     DEnd = Table->data_end();
           D != DEnd; ++D) {
        std::pair<StringRef, bool> Ident = *D;
        if (Ident.second)
          InterestingIdentifiers[Ident.first].push_back(ID);
        else
          (void)InterestingIdentifiers[Ident.first];
      }
    }

    // We don't care about this record.
  }

  return false;
}

namespace {

/// \brief Trait used to generate the identifier index as an on-disk hash
/// table.
class IdentifierIndexWriterTrait {
public:
  typedef StringRef key_type;
  typedef StringRef key_type_ref;
  typedef SmallVector<unsigned, 2> data_type;
  typedef const SmallVector<unsigned, 2> &data_type_ref;

  static unsigned ComputeHash(key_type_ref Key) {
    return llvm::HashString(Key);
  }

  std::pair<unsigned,unsigned>
  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
    unsigned KeyLen = Key.size();
    unsigned DataLen = Data.size() * 4;
    clang::io::Emit16(Out, KeyLen);
    clang::io::Emit16(Out, DataLen);
    return std::make_pair(KeyLen, DataLen);
  }
  
  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
    Out.write(Key.data(), KeyLen);
  }

  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
                unsigned DataLen) {
    for (unsigned I = 0, N = Data.size(); I != N; ++I)
      clang::io::Emit32(Out, Data[I]);
  }
};

}

void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
  using namespace llvm;
  
  // Emit the file header.
  Stream.Emit((unsigned)'B', 8);
  Stream.Emit((unsigned)'C', 8);
  Stream.Emit((unsigned)'G', 8);
  Stream.Emit((unsigned)'I', 8);

  // Write the block-info block, which describes the records in this bitcode
  // file.
  emitBlockInfoBlock(Stream);

  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);

  // Write the metadata.
  SmallVector<uint64_t, 2> Record;
  Record.push_back(CurrentVersion);
  Stream.EmitRecord(INDEX_METADATA, Record);

  // Write the set of known module files.
  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
                                MEnd = ModuleFiles.end();
       M != MEnd; ++M) {
    Record.clear();
    Record.push_back(M->second.ID);
    Record.push_back(M->first->getSize());
    Record.push_back(M->first->getModificationTime());

    // File name
    StringRef Name(M->first->getName());
    Record.push_back(Name.size());
    Record.append(Name.begin(), Name.end());

    // Dependencies
    Record.push_back(M->second.Dependencies.size());
    Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
    Stream.EmitRecord(MODULE, Record);
  }

  // Write the identifier -> module file mapping.
  {
    OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
    IdentifierIndexWriterTrait Trait;

    // Populate the hash table.
    for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
                                            IEnd = InterestingIdentifiers.end();
         I != IEnd; ++I) {
      Generator.insert(I->first(), I->second, Trait);
    }
    
    // Create the on-disk hash table in a buffer.
    SmallString<4096> IdentifierTable;
    uint32_t BucketOffset;
    {
      llvm::raw_svector_ostream Out(IdentifierTable);
      // Make sure that no bucket is at offset 0
      clang::io::Emit32(Out, 0);
      BucketOffset = Generator.Emit(Out, Trait);
    }

    // Create a blob abbreviation
    BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
    Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
    unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);

    // Write the identifier table
    Record.clear();
    Record.push_back(IDENTIFIER_INDEX);
    Record.push_back(BucketOffset);
    Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str());
  }

  Stream.ExitBlock();
}

GlobalModuleIndex::ErrorCode
GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) {
  llvm::SmallString<128> IndexPath;
  IndexPath += Path;
  llvm::sys::path::append(IndexPath, IndexFileName);

  // Coordinate building the global index file with other processes that might
  // try to do the same.
  llvm::LockFileManager Locked(IndexPath);
  switch (Locked) {
  case llvm::LockFileManager::LFS_Error:
    return EC_IOError;

  case llvm::LockFileManager::LFS_Owned:
    // We're responsible for building the index ourselves. Do so below.
    break;

  case llvm::LockFileManager::LFS_Shared:
    // Someone else is responsible for building the index. We don't care
    // when they finish, so we're done.
    return EC_Building;
  }

  // The module index builder.
  GlobalModuleIndexBuilder Builder(FileMgr);
  
  // Load each of the module files.
  llvm::error_code EC;
  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
       D != DEnd && !EC;
       D.increment(EC)) {
    // If this isn't a module file, we don't care.
    if (llvm::sys::path::extension(D->path()) != ".pcm") {
      // ... unless it's a .pcm.lock file, which indicates that someone is
      // in the process of rebuilding a module. They'll rebuild the index
      // at the end of that translation unit, so we don't have to.
      if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
        return EC_Building;

      continue;
    }

    // If we can't find the module file, skip it.
    const FileEntry *ModuleFile = FileMgr.getFile(D->path());
    if (!ModuleFile)
      continue;

    // Load this module file.
    if (Builder.loadModuleFile(ModuleFile))
      return EC_IOError;
  }

  // The output buffer, into which the global index will be written.
  SmallVector<char, 16> OutputBuffer;
  {
    llvm::BitstreamWriter OutputStream(OutputBuffer);
    Builder.writeIndex(OutputStream);
  }

  // Write the global index file to a temporary file.
  llvm::SmallString<128> IndexTmpPath;
  int TmpFD;
  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
                                      IndexTmpPath))
    return EC_IOError;

  // Open the temporary global index file for output.
  llvm::raw_fd_ostream Out(TmpFD, true);
  if (Out.has_error())
    return EC_IOError;

  // Write the index.
  Out.write(OutputBuffer.data(), OutputBuffer.size());
  Out.close();
  if (Out.has_error())
    return EC_IOError;

  // Remove the old index file. It isn't relevant any more.
  llvm::sys::fs::remove(IndexPath.str());

  // Rename the newly-written index file to the proper name.
  if (llvm::sys::fs::rename(IndexTmpPath.str(), IndexPath.str())) {
    // Rename failed; just remove the 
    llvm::sys::fs::remove(IndexTmpPath.str());
    return EC_IOError;
  }

  // We're done.
  return EC_None;
}

namespace {
  class GlobalIndexIdentifierIterator : public IdentifierIterator {
    /// \brief The current position within the identifier lookup table.
    IdentifierIndexTable::key_iterator Current;

    /// \brief The end position within the identifier lookup table.
    IdentifierIndexTable::key_iterator End;

  public:
    explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
      Current = Idx.key_begin();
      End = Idx.key_end();
    }

    virtual StringRef Next() {
      if (Current == End)
        return StringRef();

      StringRef Result = *Current;
      ++Current;
      return Result;
    }
  };
}

IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
  IdentifierIndexTable &Table =
    *static_cast<IdentifierIndexTable *>(IdentifierIndex);
  return new GlobalIndexIdentifierIterator(Table);
}
@