[lld-link] Add /reproduce: support for several flags

/reproduce: now works correctly with: - /call-graph-ordering-file: - /def: - /natvis: - /order: - /pdbstream: I went through all instances of MemoryBuffer::getFile() and made sure everything that didn't already do so called takeBuffer(). For natvis, that wasn't possible since DebugInfo/PDB wants to take owernship of the natvis buffer. For that case, I'm manually adding the tar file entry. /natvis: and /pdbstream: is slightly awkward, since createResponseFile() always adds these flags to the response file but createPDB() (which ultimately adds the files referenced by the flags) is only called if /debug is also passed. So when using /natvis: without /debug with /reproduce:, lld won't warn, but when linking using the response file from the archive, it won't find the natvis file since it's not in the tar. This isn't a new issue though, and after this patch things at least work with using /natvis: _with_ debug with /reproduce:. (Same for /pdbstream:) Differential Revison: https://reviews.llvm.org/D97212
2021-02-22 14:29:55 -05:00 · 2021-02-22 14:29:55 -05:00 · e6d1f261a5
parent f47a654a39
commit e6d1f261a5
6 changed files with 127 additions and 20 deletions
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@ -54,7 +54,7 @@
 using namespace llvm;
 using namespace llvm::object;
 using namespace llvm::COFF;
-using llvm::sys::Process;
+using namespace llvm::sys;

 namespace lld {
 namespace coff {
@ -622,6 +622,14 @@ static uint64_t getDefaultImageBase() {
  return config->dll ? 0x10000000 : 0x400000;
 }

+static std::string rewritePath(StringRef s) {
+  if (fs::exists(s))
+    return relativeToRoot(s);
+  return std::string(s);
+}
+
+// Reconstructs command line arguments so that so that you can re-run
+// the same command with the same inputs. This is for --reproduce.
 static std::string createResponseFile(const opt::InputArgList &args,
                                      ArrayRef<StringRef> filePaths,
                                      ArrayRef<StringRef> searchPaths) {
@ -642,6 +650,24 @@ static std::string createResponseFile(const opt::InputArgList &args,
    case OPT_manifestinput:
    case OPT_manifestuac:
      break;
+    case OPT_call_graph_ordering_file:
+    case OPT_deffile:
+    case OPT_natvis:
+      os << arg->getSpelling() << quote(rewritePath(arg->getValue())) << '\n';
+      break;
+    case OPT_order: {
+      StringRef orderFile = arg->getValue();
+      orderFile.consume_front("@");
+      os << arg->getSpelling() << '@' << quote(rewritePath(orderFile)) << '\n';
+      break;
+    }
+    case OPT_pdbstream: {
+      const std::pair<StringRef, StringRef> nameFile =
+          StringRef(arg->getValue()).split("=");
+      os << arg->getSpelling() << nameFile.first << '='
+         << quote(rewritePath(nameFile.second)) << '\n';
+      break;
+    }
    case OPT_implib:
    case OPT_pdb:
    case OPT_pdbstripped:
@ -838,6 +864,9 @@ static void parseModuleDefs(StringRef path) {
  COFFModuleDefinition m = check(parseCOFFModuleDefinition(
      mb->getMemBufferRef(), config->machine, config->mingw));

+  // Include in /reproduce: output if applicable.
+  driver->takeBuffer(std::move(mb));
+
  if (config->outputFile.empty())
    config->outputFile = std::string(saver.save(m.OutputFile));
  config->importName = std::string(saver.save(m.ImportName));
@ -938,6 +967,9 @@ static void parseOrderFile(StringRef arg) {
    else
      config->order[s] = INT_MIN + config->order.size();
  }
+
+  // Include in /reproduce: output if applicable.
+  driver->takeBuffer(std::move(mb));
 }

 static void parseCallGraphFile(StringRef path) {
@ -978,6 +1010,9 @@ static void parseCallGraphFile(StringRef path) {
      if (SectionChunk *to = findSection(fields[1]))
        config->callGraphProfile[{from, to}] += count;
  }
+
+  // Include in /reproduce: output if applicable.
+  driver->takeBuffer(std::move(mb));
 }

 static void readCallGraphsFromObjectFiles() {
--- a/lld/COFF/Driver.h
+++ b/lld/COFF/Driver.h
@ -93,9 +93,9 @@ public:

  void enqueuePath(StringRef path, bool wholeArchive, bool lazy);

-private:
  std::unique_ptr<llvm::TarWriter> tar; // for /linkrepro

+private:
  // Searches a file from search paths.
  Optional<StringRef> findFile(StringRef filename);
  Optional<StringRef> findLib(StringRef filename);
--- a/lld/COFF/PDB.cpp
+++ b/lld/COFF/PDB.cpp
@ -1093,7 +1093,14 @@ void PDBLinker::addNatvisFiles() {
      warn("Cannot open input file: " + file);
      continue;
    }
-    builder.addInjectedSource(file, std::move(*dataOrErr));
+    std::unique_ptr<MemoryBuffer> data = std::move(*dataOrErr);
+
+    // Can't use takeBuffer() here since addInjectedSource() takes ownership.
+    if (driver->tar)
+      driver->tar->append(relativeToRoot(data->getBufferIdentifier()),
+                          data->getBuffer());
+
+    builder.addInjectedSource(file, std::move(data));
  }
 }

@ -1106,7 +1113,9 @@ void PDBLinker::addNamedStreams() {
      warn("Cannot open input file: " + file);
      continue;
    }
-    exitOnErr(builder.addNamedStream(stream, (*dataOrErr)->getBuffer()));
+    std::unique_ptr<MemoryBuffer> data = std::move(*dataOrErr);
+    exitOnErr(builder.addNamedStream(stream, data->getBuffer()));
+    driver->takeBuffer(std::move(data));
  }
 }

--- a/lld/test/COFF/linkrepro-manifest.test
+++ b/lld/test/COFF/linkrepro-manifest.test
@ -1,7 +1,13 @@
 REQUIRES: x86, gnutar, manifest_tool

+manifest-related files are compiled to a .res file and the .res file is
+added to the repro archive, instead of adding the inputs.
+
 RUN: rm -rf %t && mkdir %t && cd %t
-RUN: lld-link -entry:__ImageBase -nodefaultlib -linkrepro:%t -manifest:embed %p/Inputs/std32.lib -subsystem:console
+RUN: lld-link -entry:__ImageBase -nodefaultlib -linkrepro:%t \
+RUN:     -manifest:embed %p/Inputs/std32.lib -subsystem:console \
+RUN:     -manifestinput:%p/Inputs/manifestinput.test
+
 RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 RUN: tar xOf repro.tar repro/response.txt | FileCheck %s

--- a/lld/test/COFF/linkrepro-pdb.test
+++ b/lld/test/COFF/linkrepro-pdb.test
@ -4,19 +4,31 @@ RUN: rm -rf %t && mkdir -p %t && cd %t
 RUN: yaml2obj %S/Inputs/pdb-type-server-simple-a.yaml -o a.obj
 RUN: yaml2obj %S/Inputs/pdb-type-server-simple-b.yaml -o b.obj
 RUN: llvm-pdbutil yaml2pdb %S/Inputs/pdb-type-server-simple-ts.yaml -pdb ts.pdb
-RUN: lld-link a.obj b.obj -entry:main -debug -out:%t.exe -pdb:%t.pdb -nodefaultlib -linkrepro:.
-RUN: tar xOf repro.tar repro/%:t/ts.pdb > repro-ts.pdb
-RUN: diff ts.pdb repro-ts.pdb
-
+RUN: cp %p/Inputs/natvis-1.natvis %t.natvis
+RUN: cp %p/Inputs/stream.txt %t.txt
+RUN: lld-link a.obj b.obj -entry:main -debug -out:%t.exe -pdb:%t.pdb \
+RUN:     -nodefaultlib -linkrepro:. -natvis:%t.natvis \
+RUN:     -pdbstream:srcsrv=%t.txt
+RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 RUN: tar xf repro.tar
-RUN: cat repro/response.txt | FileCheck -check-prefix=PDB %s
+RUN: diff ts.pdb repro/%:t/ts.pdb
+RUN: diff %t.natvis repro/%:t.natvis
+RUN: diff %t.txt repro/%:t.txt
+RUN: cat repro/response.txt | FileCheck -check-prefix=RSP %s

-PDB: -out:linkrepro-pdb.test.tmp.exe
-PDB-NEXT: -pdb:linkrepro-pdb.test.tmp.pdb
+LIST: .obj
+LIST: response.txt
+LIST: .natvis
+
+RSP: -out:linkrepro-pdb.test.tmp.exe
+RSP-NEXT: -pdb:linkrepro-pdb.test.tmp.pdb
+RSP-NEXT: -nodefaultlib
+RSP-NOT: -natvis:/
+RSP-NOT: -pdbstream:srcsrv=/

 RUN: yaml2obj %p/Inputs/export.yaml -o %t1.obj
-RUN: lld-link /out:%t1.dll /dll %t1.obj /implib:%t1.lib /export:exportfn1 /export:exportfn2 /linkrepro:.
-RUN: tar xf repro.tar
-RUN: cat repro/response.txt | FileCheck -check-prefix=IMP %s
+RUN: lld-link /out:%t1.dll /dll %t1.obj /implib:%t1.lib /export:exportfn1 /export:exportfn2 /reproduce:repro2.tar
+RUN: tar xf repro2.tar
+RUN: cat repro2/response.txt | FileCheck -check-prefix=IMP %s

 IMP: /implib:linkrepro-pdb.test.tmp1.lib
--- a/lld/test/COFF/linkrepro.test
+++ b/lld/test/COFF/linkrepro.test
@ -1,9 +1,13 @@
 # REQUIRES: x86, shell

 # RUN: rm -rf %t.dir
-# RUN: mkdir -p %t.dir/build1 %t.dir/build2 %t.dir/build3 %t.dir/build4
 # RUN: yaml2obj %p/Inputs/hello32.yaml -o %t.obj
+# RUN: echo '_main@0' > %t.order
+# RUN: touch %t.def
+# RUN: touch %t.cg

+Test link.exe-style /linkrepro: flag.
+# RUN: mkdir -p %t.dir/build1
 # RUN: cd %t.dir/build1
 # RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \
 # RUN:   /entry:main@0 /linkrepro:. /out:%t.exe
@ -12,7 +16,7 @@
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt

-# RUN: cd %t.dir/build1
+Test lld-style /reproduce: flag.
 # RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \
 # RUN:   /entry:main@0 /reproduce:repro2.tar /out:%t.exe
 # RUN: tar xf repro2.tar
@ -20,6 +24,8 @@
 # RUN: diff %p/Inputs/std32.lib repro2/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro2/response.txt

+Test LLD_REPRODUCE env var.
+# RUN: mkdir -p %t.dir/build2
 # RUN: cd %t.dir/build2
 # RUN: env LLD_REPRODUCE=repro.tar lld-link %t.obj %p/Inputs/std32.lib \
 # RUN:    /subsystem:console /entry:main@0 /out:%t.exe
@ -28,26 +34,65 @@
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt

+Test adding .lib files with /libpath: to repro archive,
+and various other flags.
+# RUN: mkdir -p %t.dir/build3
 # RUN: cd %t.dir/build3
 # RUN: lld-link %t.obj /libpath:%p/Inputs /defaultlib:std32 /subsystem:console \
-# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe
+# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe /order:@%t.order /def:%t.def
+# RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 # RUN: tar xf repro.tar
 # RUN: diff %t.obj repro/%:t.obj
+# RUN: diff %t.order repro/%:t.order
+# RUN: diff %t.def repro/%:t.def
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
+# RUN: cd repro; lld-link @response.txt

+Test adding .lib files with LIB env var to repro archive,
+and various other flags.
+# RUN: mkdir -p %t.dir/build4
 # RUN: cd %t.dir/build4
 # RUN: env LIB=%p/Inputs lld-link %t.obj /defaultlib:std32 /subsystem:console \
-# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe
+# RUN:   /entry:main@0 /linkrepro:. /out:%t.exe /order:@%t.order /def:%t.def
+# RUN: tar tf repro.tar | FileCheck --check-prefix=LIST %s
 # RUN: tar xf repro.tar
 # RUN: diff %t.obj repro/%:t.obj
+# RUN: diff %t.order repro/%:t.order
+# RUN: diff %t.def repro/%:t.def
 # RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib
 # RUN: FileCheck %s --check-prefix=RSP < repro/response.txt
+# RUN: cd repro; lld-link @response.txt
+
+# LIST: .obj
+# LIST: std32.lib
+# LIST: response.txt
+# LIST: .def
+# LIST: .order

 # RSP: /subsystem:console
 # RSP: /entry:main@0
-# RSP-NOT: /linkrepro:
 # RSP: /out:
+# RSP-NOT: /order:@/
+# RSP-NOT: /def:/
 # RSP: linkrepro.test.tmp.obj
 # RSP-NOT: defaultlib
 # RSP: std32.lib
+
+Test /call-graph-ordering-file (can't be used with /order:, needs separate test)
+# RUN: mkdir -p %t.dir/build5
+# RUN: cd %t.dir/build5
+# RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \
+# RUN:     /entry:main@0 /linkrepro:. /out:%t.exe /call-graph-ordering-file:%t.cg
+# RUN: tar tf repro.tar | FileCheck --check-prefix=LISTCG %s
+# RUN: tar xf repro.tar
+# RUN: diff %t.obj repro/%:t.obj
+# RUN: diff %t.cg repro/%:t.cg
+# RUN: FileCheck %s --check-prefix=RSPCG < repro/response.txt
+# RUN: cd repro; lld-link @response.txt
+
+# LISTCG: .obj
+# LISTCG: response.txt
+# LISTCG: .cg
+
+# RSPCG-NOT: /call-graph-ordering-file:/