2018-05-01 21:50:34 +02:00
|
|
|
// Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2018-02-16 20:42:17 +01:00
|
|
|
#ifndef FORTRAN_PARSER_PRESCAN_H_
|
|
|
|
#define FORTRAN_PARSER_PRESCAN_H_
|
2018-01-30 20:54:47 +01:00
|
|
|
|
|
|
|
// Defines a fast Fortran source prescanning phase that implements some
|
|
|
|
// character-level features of the language that can be inefficient to
|
|
|
|
// support directly in a backtracking parser. This phase handles Fortran
|
|
|
|
// line continuation, comment removal, card image margins, padding out
|
2018-02-13 21:24:54 +01:00
|
|
|
// fixed form character literals on truncated card images, file
|
|
|
|
// inclusion, and driving the Fortran source preprocessor.
|
2018-01-30 20:54:47 +01:00
|
|
|
|
2018-03-01 01:56:10 +01:00
|
|
|
#include "characters.h"
|
2018-07-18 20:19:21 +02:00
|
|
|
#include "features.h"
|
2018-02-17 01:57:40 +01:00
|
|
|
#include "message.h"
|
2018-02-09 23:04:11 +01:00
|
|
|
#include "provenance.h"
|
2018-02-13 21:50:47 +01:00
|
|
|
#include "token-sequence.h"
|
2018-03-20 18:59:07 +01:00
|
|
|
#include <bitset>
|
2018-01-30 20:54:47 +01:00
|
|
|
#include <optional>
|
2018-02-09 23:04:11 +01:00
|
|
|
#include <string>
|
2018-03-20 18:59:07 +01:00
|
|
|
#include <unordered_set>
|
2018-01-30 20:54:47 +01:00
|
|
|
|
2018-05-02 22:48:12 +02:00
|
|
|
namespace Fortran::parser {
|
2018-01-30 20:54:47 +01:00
|
|
|
|
2018-02-13 21:50:47 +01:00
|
|
|
class Messages;
|
|
|
|
class Preprocessor;
|
|
|
|
|
2018-01-30 20:54:47 +01:00
|
|
|
class Prescanner {
|
2018-02-05 21:54:36 +01:00
|
|
|
public:
|
2018-07-18 20:19:21 +02:00
|
|
|
Prescanner(
|
|
|
|
Messages &, CookedSource &, Preprocessor &, LanguageFeatureControl);
|
2018-02-13 23:22:08 +01:00
|
|
|
Prescanner(const Prescanner &);
|
2018-02-02 00:01:23 +01:00
|
|
|
|
2018-03-23 23:14:52 +01:00
|
|
|
Messages &messages() const { return messages_; }
|
2018-01-30 20:54:47 +01:00
|
|
|
|
|
|
|
Prescanner &set_fixedForm(bool yes) {
|
|
|
|
inFixedForm_ = yes;
|
|
|
|
return *this;
|
|
|
|
}
|
2018-03-01 01:56:10 +01:00
|
|
|
Prescanner &set_encoding(Encoding code) {
|
|
|
|
encoding_ = code;
|
|
|
|
return *this;
|
|
|
|
}
|
2018-01-30 20:54:47 +01:00
|
|
|
Prescanner &set_fixedFormColumnLimit(int limit) {
|
|
|
|
fixedFormColumnLimit_ = limit;
|
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
|
2018-03-20 18:59:07 +01:00
|
|
|
Prescanner &AddCompilerDirectiveSentinel(const std::string &);
|
|
|
|
|
2018-04-03 00:51:04 +02:00
|
|
|
void Prescan(ProvenanceRange);
|
2018-03-23 23:14:52 +01:00
|
|
|
void Statement();
|
2018-03-17 00:58:35 +01:00
|
|
|
void NextLine();
|
2018-02-13 23:22:08 +01:00
|
|
|
|
|
|
|
// Callbacks for use by Preprocessor.
|
2018-03-17 00:58:35 +01:00
|
|
|
bool IsAtEnd() const { return lineStart_ >= limit_; }
|
|
|
|
bool IsNextLinePreprocessorDirective() const;
|
|
|
|
TokenSequence TokenizePreprocessorDirective();
|
2018-02-09 23:04:11 +01:00
|
|
|
Provenance GetCurrentProvenance() const { return GetProvenance(at_); }
|
2018-03-19 19:48:49 +01:00
|
|
|
|
2018-04-03 00:51:04 +02:00
|
|
|
void Say(Message &&);
|
2018-04-23 21:17:11 +02:00
|
|
|
void Say(MessageFixedText, ProvenanceRange);
|
|
|
|
void Say(MessageFormattedText &&, ProvenanceRange);
|
2018-01-30 20:54:47 +01:00
|
|
|
|
2018-02-05 21:54:36 +01:00
|
|
|
private:
|
2018-03-23 23:14:52 +01:00
|
|
|
struct LineClassification {
|
|
|
|
enum class Kind {
|
|
|
|
Comment,
|
2018-05-03 01:09:12 +02:00
|
|
|
ConditionalCompilationDirective,
|
2018-03-23 23:14:52 +01:00
|
|
|
PreprocessorDirective,
|
|
|
|
Include,
|
|
|
|
CompilerDirective,
|
|
|
|
Source
|
|
|
|
};
|
|
|
|
LineClassification(Kind k, std::size_t po = 0, const char *s = nullptr)
|
|
|
|
: kind{k}, payloadOffset{po}, sentinel{s} {}
|
|
|
|
LineClassification(LineClassification &&) = default;
|
|
|
|
Kind kind;
|
|
|
|
std::size_t payloadOffset; // byte offset of content
|
|
|
|
const char *sentinel; // if it's a compiler directive
|
|
|
|
};
|
|
|
|
|
2018-02-01 21:08:02 +01:00
|
|
|
void BeginSourceLine(const char *at) {
|
2018-01-30 20:54:47 +01:00
|
|
|
at_ = at;
|
2018-02-12 20:56:42 +01:00
|
|
|
column_ = 1;
|
2018-01-30 20:54:47 +01:00
|
|
|
tabInCurrentLine_ = false;
|
2018-04-09 20:47:00 +02:00
|
|
|
slashInCurrentLine_ = false;
|
2018-01-30 20:54:47 +01:00
|
|
|
preventHollerith_ = false;
|
2018-01-31 00:22:26 +01:00
|
|
|
delimiterNesting_ = 0;
|
2018-01-30 20:54:47 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void BeginSourceLineAndAdvance() {
|
|
|
|
BeginSourceLine(lineStart_);
|
|
|
|
NextLine();
|
|
|
|
}
|
|
|
|
|
2018-02-09 23:04:11 +01:00
|
|
|
Provenance GetProvenance(const char *sourceChar) const {
|
2018-02-15 19:42:36 +01:00
|
|
|
return startProvenance_ + (sourceChar - start_);
|
2018-02-09 23:04:11 +01:00
|
|
|
}
|
|
|
|
|
2018-04-23 21:17:11 +02:00
|
|
|
ProvenanceRange GetProvenanceRange(
|
|
|
|
const char *first, const char *afterLast) const {
|
|
|
|
std::size_t bytes = afterLast - first;
|
|
|
|
return {startProvenance_ + (first - start_), bytes};
|
|
|
|
}
|
|
|
|
|
2018-05-10 22:48:55 +02:00
|
|
|
void EmitChar(TokenSequence &tokens, char ch) {
|
|
|
|
tokens.PutNextTokenChar(ch, GetCurrentProvenance());
|
2018-02-09 23:04:11 +01:00
|
|
|
}
|
|
|
|
|
2018-05-10 22:48:55 +02:00
|
|
|
void EmitInsertedChar(TokenSequence &tokens, char ch) {
|
2018-03-23 23:14:52 +01:00
|
|
|
Provenance provenance{cooked_.allSources().CompilerInsertionProvenance(ch)};
|
2018-05-10 22:48:55 +02:00
|
|
|
tokens.PutNextTokenChar(ch, provenance);
|
2018-02-16 20:14:11 +01:00
|
|
|
}
|
|
|
|
|
2018-05-10 22:48:55 +02:00
|
|
|
char EmitCharAndAdvance(TokenSequence &tokens, char ch) {
|
2018-02-09 23:04:11 +01:00
|
|
|
EmitChar(tokens, ch);
|
2018-01-30 20:54:47 +01:00
|
|
|
NextChar();
|
|
|
|
return *at_;
|
|
|
|
}
|
|
|
|
|
2018-05-10 22:48:55 +02:00
|
|
|
bool InCompilerDirective() const { return directiveSentinel_ != nullptr; }
|
|
|
|
bool InFixedFormSource() const {
|
|
|
|
return inFixedForm_ && !inPreprocessorDirective_ && !InCompilerDirective();
|
|
|
|
}
|
|
|
|
|
|
|
|
void LabelField(TokenSequence &);
|
2018-05-03 20:09:04 +02:00
|
|
|
void SkipToEndOfLine();
|
2018-01-30 20:54:47 +01:00
|
|
|
void NextChar();
|
|
|
|
void SkipSpaces();
|
2018-05-10 22:48:55 +02:00
|
|
|
bool NextToken(TokenSequence &);
|
|
|
|
bool ExponentAndKind(TokenSequence &);
|
|
|
|
void QuotedCharacterLiteral(TokenSequence &);
|
|
|
|
void Hollerith(TokenSequence &, int);
|
|
|
|
bool PadOutCharacterLiteral(TokenSequence &);
|
2018-05-10 00:32:22 +02:00
|
|
|
bool SkipCommentLine();
|
2018-03-20 18:59:07 +01:00
|
|
|
bool IsFixedFormCommentLine(const char *) const;
|
|
|
|
bool IsFreeFormComment(const char *) const;
|
2018-03-23 23:14:52 +01:00
|
|
|
std::optional<std::size_t> IsIncludeLine(const char *) const;
|
2018-04-04 17:06:15 +02:00
|
|
|
void FortranInclude(const char *quote);
|
2018-05-03 01:09:12 +02:00
|
|
|
const char *IsPreprocessorDirectiveLine(const char *) const;
|
2018-05-10 22:48:55 +02:00
|
|
|
const char *FixedFormContinuationLine(bool mightNeedSpace);
|
2018-05-10 00:32:22 +02:00
|
|
|
const char *FreeFormContinuationLine(bool ampersand);
|
2018-05-10 22:48:55 +02:00
|
|
|
bool FixedFormContinuation(bool mightNeedSpace);
|
2018-01-30 20:54:47 +01:00
|
|
|
bool FreeFormContinuation();
|
2018-03-23 23:14:52 +01:00
|
|
|
std::optional<LineClassification> IsFixedFormCompilerDirectiveLine(
|
|
|
|
const char *) const;
|
|
|
|
std::optional<LineClassification> IsFreeFormCompilerDirectiveLine(
|
|
|
|
const char *) const;
|
|
|
|
const char *IsCompilerDirectiveSentinel(const char *) const;
|
|
|
|
LineClassification ClassifyLine(const char *) const;
|
|
|
|
void SourceFormChange(std::string &&);
|
|
|
|
|
|
|
|
Messages &messages_;
|
|
|
|
CookedSource &cooked_;
|
|
|
|
Preprocessor &preprocessor_;
|
2018-07-18 20:19:21 +02:00
|
|
|
LanguageFeatureControl features_;
|
2018-03-23 23:14:52 +01:00
|
|
|
bool inFixedForm_{false};
|
|
|
|
int fixedFormColumnLimit_{72};
|
|
|
|
Encoding encoding_{Encoding::UTF8};
|
|
|
|
int delimiterNesting_{0};
|
2018-04-06 23:20:29 +02:00
|
|
|
int prescannerNesting_{0};
|
2018-02-09 23:04:11 +01:00
|
|
|
|
2018-02-15 19:42:36 +01:00
|
|
|
Provenance startProvenance_;
|
2018-02-13 21:24:54 +01:00
|
|
|
const char *start_{nullptr}; // beginning of current source file content
|
|
|
|
const char *limit_{nullptr}; // first address after end of current source
|
2018-03-23 23:14:52 +01:00
|
|
|
const char *lineStart_{nullptr}; // next line to process; <= limit_
|
|
|
|
const char *directiveSentinel_{nullptr}; // current compiler directive
|
|
|
|
|
|
|
|
// This data members are state for processing the source line containing
|
|
|
|
// "at_", which goes to up to the newline character before "lineStart_".
|
2018-01-30 20:54:47 +01:00
|
|
|
const char *at_{nullptr}; // next character to process; < lineStart_
|
|
|
|
int column_{1}; // card image column position of next character
|
2018-02-09 23:04:11 +01:00
|
|
|
bool tabInCurrentLine_{false};
|
2018-04-09 20:47:00 +02:00
|
|
|
bool slashInCurrentLine_{false};
|
2018-02-09 23:04:11 +01:00
|
|
|
bool preventHollerith_{false};
|
2018-01-30 20:54:47 +01:00
|
|
|
bool inCharLiteral_{false};
|
|
|
|
bool inPreprocessorDirective_{false};
|
2018-03-23 23:14:52 +01:00
|
|
|
|
2018-05-10 22:48:55 +02:00
|
|
|
// In some edge cases of compiler directive continuation lines, it
|
|
|
|
// is necessary to treat the line break as a space character by
|
|
|
|
// setting this flag, which is cleared by EmitChar().
|
|
|
|
bool insertASpace_{false};
|
|
|
|
|
2018-03-23 23:14:52 +01:00
|
|
|
const Provenance spaceProvenance_{
|
|
|
|
cooked_.allSources().CompilerInsertionProvenance(' ')};
|
|
|
|
const Provenance backslashProvenance_{
|
|
|
|
cooked_.allSources().CompilerInsertionProvenance('\\')};
|
|
|
|
const ProvenanceRange sixSpaceProvenance_{
|
|
|
|
cooked_.allSources().AddCompilerInsertion(" "s)};
|
2018-03-20 18:59:07 +01:00
|
|
|
|
|
|
|
// To avoid probing the set of active compiler directive sentinel strings
|
|
|
|
// on every comment line, they're checked first with a cheap Bloom filter.
|
|
|
|
static const int prime1{1019}, prime2{1021};
|
|
|
|
std::bitset<prime2> compilerDirectiveBloomFilter_; // 128 bytes
|
|
|
|
std::unordered_set<std::string> compilerDirectiveSentinels_;
|
2018-01-30 20:54:47 +01:00
|
|
|
};
|
2018-05-02 22:48:12 +02:00
|
|
|
|
|
|
|
} // namespace Fortran::parser
|
2018-02-16 20:42:17 +01:00
|
|
|
#endif // FORTRAN_PARSER_PRESCAN_H_
|