squiid/llvm
1
0
Fork 0
Code Issues Pull requests Activity

[flang] Module file improvements

Browse source 
Verify that checksums are correct when reading a module file.

Don't write a module file if its current contents are correct.

Don't make .mod files read-only. It doesn't do much to prevent
users from editing them, checksum verification will detect when
it happens, and having them read-only causes problems if you then
compile with another compiler (e.g. PGI).

Original-commit: flang-compiler/f18@21d82aef6e
Reviewed-on: https://github.com/flang-compiler/f18/pull/164
Tree-same-pre-rewrite: false
This commit is contained in:
Tim Keith 2018-08-08 11:36:24 -07:00
parent 70dfdf979d
commit 3d43ea663b
2 changed files with 88 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

125
flang/lib/semantics/mod-file.cc
View file

@ -36,7 +36,7 @@ using namespace parser::literals;
// The extension used for module files.
static constexpr auto extension{".mod"};
// The initial characters of a file that identify it as a .mod file.
static constexpr auto magic{"!mod$"};
static const auto magic{"!mod$ v1 sum:"s};
// Helpers for creating error messages.
static parser::Message Error(
@ -56,8 +56,11 @@ static std::ostream &PutAttrs(
static std::ostream &PutLower(std::ostream &, const Symbol &);
static std::ostream &PutLower(std::ostream &, const DeclTypeSpec &);
static std::ostream &PutLower(std::ostream &, const std::string &);
static std::string CheckSum(const std::string &);
static bool MakeReadonly(const std::string &);
static bool WriteFile(const std::string &, std::string &&);
static bool FileContentsMatch(
std::fstream &, const std::string &, const std::string &);
static std::string GetHeader(const std::string &);
static std::size_t GetFileSize(const std::string &);
bool ModFileWriter::WriteAll() {
WriteChildren(Scope::globalScope);
@ -85,21 +88,10 @@ void ModFileWriter::Write(const Symbol &symbol) {
auto *ancestor{symbol.get<ModuleDetails>().ancestor()};
auto ancestorName{ancestor ? ancestor->name().ToString() : ""s};
auto path{ModFilePath(dir_, symbol.name(), ancestorName)};
unlink(path.data());
std::ofstream os{path};
PutSymbols(*symbol.scope());
std::string all{GetAsString(symbol)};
auto header{GetHeader(all)};
os << header << all;
os.close();
if (!os) {
if (!WriteFile(path, GetAsString(symbol))) {
errors_.emplace_back(
"Error writing %s: %s"_err_en_US, path.c_str(), std::strerror(errno));
return;
}
if (!MakeReadonly(path)) {
errors_.emplace_back("Error changing permissions on %s: %s"_en_US,
path.c_str(), std::strerror(errno));
}
}
@ -134,13 +126,6 @@ std::string ModFileWriter::GetAsString(const Symbol &symbol) {
return all.str();
}
// Return the header for this mod file.
std::string ModFileWriter::GetHeader(const std::string &all) {
std::stringstream ss;
ss << magic << " v" << version_ << " sum:" << CheckSum(all) << '\n';
return ss.str();
}
// Put out the visible symbols from scope.
void ModFileWriter::PutSymbols(const Scope &scope) {
for (const auto *symbol : SortSymbols(CollectSymbols(scope))) {
@ -362,12 +347,53 @@ std::ostream &PutLower(std::ostream &os, const std::string &str) {
return os;
}
// Write the module file at path, prepending header. Return false on error.
static bool WriteFile(const std::string &path, std::string &&contents) {
std::fstream stream;
auto header{GetHeader(contents)};
auto size{GetFileSize(path)};
if (size == header.size() + 1 + contents.size()) {
// file exists and has the right size, check the contents
stream.open(path, std::ios::in | std::ios::out);
if (FileContentsMatch(stream, header, contents)) {
return true;
}
stream.seekp(0);
} else {
stream.open(path, std::ios::out);
}
stream << header << '\n' << contents;
stream.close();
return !stream.fail();
}
// Return true if the stream matches what we would write for the mod file.
static bool FileContentsMatch(std::fstream &stream, const std::string &header,
const std::string &contents) {
char c;
for (std::size_t i{0}; i < header.size(); ++i) {
if (!stream.get(c) || c != header[i]) {
return false;
}
}
if (!stream.get(c) || c != '\n') {
return false;
}
for (std::size_t i{0}; i < contents.size(); ++i) {
if (!stream.get(c) || c != contents[i]) {
return false;
}
}
return true;
}
// Compute a simple hash of the contents of a module file and
// return it as a string of hex digits.
// This uses the Fowler-Noll-Vo hash function.
std::string CheckSum(const std::string &str) {
template<typename Iter> static std::string CheckSum(Iter begin, Iter end) {
std::uint64_t hash{0xcbf29ce484222325ull};
for (char c : str) {
for (auto it{begin}; it != end; ++it) {
char c{*it};
hash ^= c & 0xff;
hash *= 0x100000001b3;
}
@ -379,6 +405,34 @@ std::string CheckSum(const std::string &str) {
return result;
}
static bool VerifyHeader(const std::string &path) {
std::fstream stream{path};
std::string header;
std::getline(stream, header);
if (header.compare(0, magic.size(), magic) != 0) {
return false;
}
std::string expectSum{header.substr(magic.size(), 16)};
std::string actualSum{CheckSum(std::istreambuf_iterator<char>(stream),
std::istreambuf_iterator<char>())};
return expectSum == actualSum;
}
static std::string GetHeader(const std::string &all) {
std::stringstream ss;
ss << magic << CheckSum(all.begin(), all.end());
return ss.str();
}
static std::size_t GetFileSize(const std::string &path) {
struct stat statbuf;
if (stat(path.c_str(), &statbuf) == 0) {
return static_cast<std::size_t>(statbuf.st_size);
} else {
return 0;
}
}
Scope *ModFileReader::Read(const SourceName &name, Scope *ancestor) {
std::string ancestorName; // empty for module
if (ancestor) {
@ -396,6 +450,14 @@ Scope *ModFileReader::Read(const SourceName &name, Scope *ancestor) {
if (!path.has_value()) {
return nullptr;
}
// TODO: We are reading the file once to verify the checksum and then again
// to parse. Do it only reading the file once.
if (!VerifyHeader(*path)) {
errors_.push_back(
Error(name, "Module file for '%s' has invalid checksum: %s"_err_en_US,
name.ToString(), *path));
return nullptr;
}
// TODO: Construct parsing with an AllSources reference to share provenance
parser::Parsing parsing;
parser::Options options;
@ -441,9 +503,8 @@ std::optional<std::string> ModFileReader::FindModFile(
Error(name, "%s: %s"_en_US, path, std::string{std::strerror(errno)}));
} else {
std::string line;
ifstream >> line;
if (std::equal(line.begin(), line.end(), std::string{magic}.begin())) {
// TODO: verify rest of header line: version, checksum, etc.
std::getline(ifstream, line);
if (line.compare(0, magic.size(), magic) == 0) {
return path;
}
errors.push_back(Error(name, "%s: Not a valid module file"_en_US, path));
@ -490,16 +551,6 @@ static std::string ModFilePath(const std::string &dir, const SourceName &name,
return path.str();
}
static bool MakeReadonly(const std::string &path) {
struct stat statbuf;
if (stat(path.c_str(), &statbuf) != 0) {
return false;
}
auto mode{statbuf.st_mode};
mode &= S_IRUSR | S_IRGRP | S_IROTH;
return chmod(path.data(), mode) == 0;
}
static parser::Message Error(const SourceName &location,
parser::MessageFixedText fixedText, const std::string &arg) {
return parser::Message{

4
flang/lib/semantics/mod-file.h
View file

@ -37,8 +37,6 @@ class Scope;
class ModFileWriter {
public:
// The .mod file format version number.
void set_version(int version) { version_ = version; }
// The directory to write .mod files in.
void set_directory(const std::string &dir) { dir_ = dir; }
@ -54,7 +52,6 @@ private:
using symbolSet = std::set<const Symbol *>;
using symbolVector = std::vector<const Symbol *>;
int version_{1};
std::string dir_{"."};
// The mod file consists of uses, declarations, and contained subprograms:
std::stringstream uses_;
@ -68,7 +65,6 @@ private:
void WriteOne(const Scope &);
void Write(const Symbol &);
std::string GetAsString(const Symbol &);
std::string GetHeader(const std::string &);
void PutSymbols(const Scope &);
symbolVector SortSymbols(const symbolSet);
symbolSet CollectSymbols(const Scope &);