Replace token count,len with string_view, closes #517

Author: hsutter

regression-tests/test-results/version

@@ -1,5 +1,5 @@
 
-cppfront compiler v0.2.1   Build 8618:1005
+cppfront compiler v0.2.1   Build 8618:1139
 Copyright(c) Herb Sutter   All rights reserved
 
 SPDX-License-Identifier: CC-BY-NC-ND-4.0

source/build.info

@@ -1 +1 @@
-"8618:1005"
+"8618:1139"

source/cppfront.cpp

@@ -6028,8 +6028,7 @@ auto main(
             exit_status = EXIT_FAILURE;
         }
 
-        //  In any case, emit the debug information (during early development this is
-        //  on by default, later we'll flip the switch to turn it on instead of off)
+        //  And, if requested, the debug information
         if (enable_debug_output_files) {
             c.debug_print();
         }

source/lex.h

@@ -226,10 +226,9 @@ class token
         source_position pos,
         lexeme          type
     )
-      : start   {start}
-      , count   {int16_t(count)}
-      , pos     {pos  }
-      , lex_type{type }
+      : sv      {start, unsafe_narrow<ulong>(count)}
+      , pos     {pos}
+      , lex_type{type}
     {
     }
 
@@ -238,18 +237,17 @@ class token
         source_position pos,
         lexeme          type
     )
-      : start   {sz}
-      , count   {__as<int16_t>(std::strlen(sz))}
-      , pos     {pos  }
-      , lex_type{type }
+      : sv      {sz}
+      , pos     {pos}
+      , lex_type{type}
     {
     }
 
     auto as_string_view() const
         -> std::string_view
     {
-        assert (start);
-        return {start, static_cast<unsigned>(count)};
+        assert (sv.data());
+        return sv;
     }
 
     operator std::string_view() const
@@ -272,7 +270,7 @@ class token
     auto to_string( bool text_only = false ) const
         -> std::string
     {
-        auto text = std::string{start, static_cast<unsigned>(count)};
+        auto text = std::string{sv};
         if (text_only) {
             return text;
         }
@@ -294,13 +292,13 @@ class token
         pos.colno += offset;
     }
 
-    auto position() const -> source_position { return pos;      }
+    auto position() const -> source_position { return pos;       }
 
-    auto length  () const -> int             { return count;    }
+    auto length  () const -> int             { return sv.size(); }
 
-    auto type    () const -> lexeme          { return lex_type; }
+    auto type    () const -> lexeme          { return lex_type;  }
 
-    auto set_type(lexeme l) -> void          { lex_type = l;    }
+    auto set_type(lexeme l) -> void          { lex_type = l;     }
 
     auto visit(auto& v, int depth) const
         -> void
@@ -309,12 +307,9 @@ class token
     }
 
 private:
-    //  Store (char*,count) because it's smaller than a string_view
-    //
-    char const*     start;
-    int16_t         count;
-    source_position pos;
-    lexeme          lex_type;
+    std::string_view sv;
+    source_position  pos;
+    lexeme           lex_type;
 };
 
 static_assert (CHAR_BIT == 8);