[flang] Review comments and last (?) bugs

Original-commit: flang-compiler/f18@db8302e3ab
Reviewed-on: https://github.com/flang-compiler/f18/pull/496

flang/lib/evaluate/formatting.cc

@@ -56,7 +56,7 @@ std::ostream &ConstantBase<RESULT, VALUE>::AsFortran(std::ostream &o) const {
         Result::category == TypeCategory::Complex) {
       value.AsFortran(o, Result::kind);
     } else if constexpr (Result::category == TypeCategory::Character) {
-      o << Result::kind << '_' << parser::QuoteCharacterLiteral(value, false);
+      o << Result::kind << '_' << parser::QuoteCharacterLiteral(value, true);
     } else if constexpr (Result::category == TypeCategory::Logical) {
       if (value.IsTrue()) {
         o << ".true.";

flang/lib/parser/characters.cc

@@ -131,12 +131,12 @@ template<> EncodedCharacter EncodeCharacter<Encoding::EUC_JP>(char32_t ucs) {
     result.buffer[0] = ucs;
     result.bytes = 1;
   } else if (ucs <= 0xff) {
-    result.buffer[0] = 0x8e;  // JIS X 0201
+    result.buffer[0] = '\x8e';  // JIS X 0201
     result.buffer[1] = ucs;
     result.bytes = 2;
   } else if (IsUCSJIS_0212(ucs)) {  // JIS X 0212
     char32_t jis{UCSToJIS(ucs)};
-    result.buffer[0] = 0x8f;
+    result.buffer[0] = '\x8f';
     result.buffer[1] = 0x80 ^ (jis >> 8);
     result.buffer[2] = 0x80 ^ jis;
     result.bytes = 3;

flang/lib/parser/prescan.cc

@@ -106,6 +106,7 @@ void Prescanner::Statement() {
     return;
   case LineClassification::Kind::ConditionalCompilationDirective:
   case LineClassification::Kind::IncludeDirective:
+  case LineClassification::Kind::DefinitionDirective:
   case LineClassification::Kind::PreprocessorDirective:
     preprocessor_.Directive(TokenizePreprocessorDirective(), this);
     return;
@@ -181,6 +182,7 @@ void Prescanner::Statement() {
       break;
     case LineClassification::Kind::ConditionalCompilationDirective:
     case LineClassification::Kind::IncludeDirective:
+    case LineClassification::Kind::DefinitionDirective:
     case LineClassification::Kind::PreprocessorDirective:
       Say(preprocessed->GetProvenanceRange(),
           "Preprocessed line resembles a preprocessor directive"_en_US);
@@ -487,11 +489,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
     do {
     } while (IsLegalInIdentifier(EmitCharAndAdvance(tokens, *at_)));
     if (*at_ == '\'' || *at_ == '"') {
-      // Look for prefix of NC'...' legacy PGI "Kanji" NCHARACTER literal
-      CharBlock prefix{tokens.CurrentOpenToken()};
-      bool isKanji{prefix.size() == 2 && ToLowerCaseLetter(prefix[0]) == 'n' &&
-          ToLowerCaseLetter(prefix[1]) == 'c'};
-      QuotedCharacterLiteral(tokens, start, isKanji);
+      QuotedCharacterLiteral(tokens, start);
       preventHollerith_ = false;
     } else {
       // Subtle: Don't misrecognize labeled DO statement label as Hollerith
@@ -552,7 +550,7 @@ bool Prescanner::ExponentAndKind(TokenSequence &tokens) {
 }
 
 void Prescanner::QuotedCharacterLiteral(
-    TokenSequence &tokens, const char *start, bool isKanji) {
+    TokenSequence &tokens, const char *start) {
   char quote{*at_};
   const char *end{at_ + 1};
   inCharLiteral_ = true;
@@ -808,7 +806,8 @@ bool Prescanner::SkipCommentLine(bool afterAmpersand) {
       lineClass.kind == LineClassification::Kind::PreprocessorDirective) {
     // Allow conditional compilation directives (e.g., #ifdef) to affect
     // continuation lines.
-    // Allow other preprocessor directives, too, except #include.
+    // Allow other preprocessor directives, too, except #include,
+    // #define, & #undef.
     preprocessor_.Directive(TokenizePreprocessorDirective(), this);
     return true;
   } else if (afterAmpersand &&
@@ -1114,6 +1113,9 @@ Prescanner::LineClassification Prescanner::ClassifyLine(
       return {LineClassification::Kind::ConditionalCompilationDirective};
     } else if (std::memcmp(dir, "include", 7) == 0) {
       return {LineClassification::Kind::IncludeDirective};
+    } else if (std::memcmp(dir, "define", 6) == 0 ||
+        std::memcmp(dir, "undef", 5) == 0) {
+      return {LineClassification::Kind::DefinitionDirective};
     } else {
       return {LineClassification::Kind::PreprocessorDirective};
     }

flang/lib/parser/prescan.h

@@ -83,6 +83,7 @@ private:
       Comment,
       ConditionalCompilationDirective,
       IncludeDirective,  // #include
+      DefinitionDirective,  // #define & #undef
       PreprocessorDirective,
       IncludeLine,  // Fortran INCLUDE
       CompilerDirective,
@@ -158,8 +159,7 @@ private:
   const char *SkipCComment(const char *) const;
   bool NextToken(TokenSequence &);
   bool ExponentAndKind(TokenSequence &);
-  void QuotedCharacterLiteral(
-      TokenSequence &, const char *start, bool isKanji = false);
+  void QuotedCharacterLiteral(TokenSequence &, const char *start);
   void Hollerith(TokenSequence &, int count, const char *start);
   bool PadOutCharacterLiteral(TokenSequence &);
   bool SkipCommentLine(bool afterAmpersand);

flang/lib/parser/unparse.cc

@@ -192,15 +192,14 @@ public:
         Word("NC");
         std::u16string decoded{DecodeString<Encoding::EUC_JP>(str, true)};
         std::string encoded{EncodeString<Encoding::EUC_JP>(decoded)};
-        Put(QuoteCharacterLiteral(encoded, backslashEscapes_, Encoding::LATIN_1));
-        return;
+        Put(QuoteCharacterLiteral(encoded, backslashEscapes_));
       } else {
         Walk(*k), Put('_');
+        PutNormalized(str);
       }
+    } else {
+      PutNormalized(str);
     }
-    std::string decoded{DecodeString<Encoding::LATIN_1>(str, true)};
-    std::string encoded{EncodeString<Encoding::LATIN_1>(decoded)};
-    Put(QuoteCharacterLiteral(encoded, backslashEscapes_, Encoding::LATIN_1));
   }
   void Unparse(const HollerithLiteralConstant &x) {
     std::u32string ucs{DecodeString<Encoding::UTF_8>(x.v, false)};
@@ -1435,9 +1434,7 @@ public:
     }
     std::visit(
         common::visitors{
-            [&](const std::string &y) {
-              Put(QuoteCharacterLiteral(y, backslashEscapes_));
-            },
+            [&](const std::string &y) { PutNormalized(y); },
             [&](const std::list<format::FormatItem> &y) {
               Walk("(", y, ",", ")");
             },
@@ -2504,6 +2501,7 @@ private:
   void Put(char);
   void Put(const char *);
   void Put(const std::string &);
+  void PutNormalized(const std::string &);
   void PutKeywordLetter(char);
   void Word(const char *);
   void Word(const std::string &);
@@ -2635,6 +2633,12 @@ void UnparseVisitor::Put(const std::string &str) {
   }
 }
 
+void UnparseVisitor::PutNormalized(const std::string &str) {
+  std::string decoded{DecodeString<Encoding::LATIN_1>(str, true)};
+  std::string encoded{EncodeString<Encoding::LATIN_1>(decoded)};
+  Put(QuoteCharacterLiteral(encoded, backslashEscapes_));
+}
+
 void UnparseVisitor::PutKeywordLetter(char ch) {
   if (capitalizeKeywords_) {
     Put(ToUpperCaseLetter(ch));