Merge pull request python#55 from lysnikolaou/fstring-cleanup-grammar-actions

Cleanup grammar and actions // Remove unnecessary fstring code

Author: pablogsal

Grammar/python.gram

@@ -877,30 +877,29 @@ lambda_param[arg_ty]: a=NAME { _PyAST_arg(a->v.Name.id, NULL, NULL, EXTRA) }
 # LITERALS
 # ========
 
-gstring_middle[expr_ty]:
-    | gstring_replacement_field
-    | t=FSTRING_MIDDLE { _PyPegen_constant_from_token2(p, t) }
-# There are some shenanigans with the gstring_format_spec: Don't try to put it in its own rule
+fstring_middle[expr_ty]:
+    | fstring_replacement_field
+    | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
+# There are some shenanigans with the fstring_format_spec: Don't try to put it in its own rule
 # or otherwise it will try to parse the first token with the regular tokenizer mode (due to the EXTRA).
 # TODO: (Ideally we need a way similar to 'memo' so the parser can set the tokenize mode on fstring/normal)
-gstring_replacement_field[expr_ty]:
+fstring_replacement_field[expr_ty]:
     | expr_start='{' a=(yield_expr | star_expressions) debug_expr="="? conversion=[
         conv_token="!" conv=NAME { _PyPegen_check_fstring_conversion(p, conv_token, conv) ? NULL : conv }
     ] format=[
-        ':' spec=gstring_format_spec* { spec ? _PyAST_JoinedStr((asdl_expr_seq*)spec, EXTRA) : NULL }
+        ':' spec=fstring_format_spec* { spec ? _PyAST_JoinedStr((asdl_expr_seq*)spec, EXTRA) : NULL }
     ] &&'}' {
         _PyPegen_formatted_value(p, a, debug_expr, conversion, format, EXTRA)
     }
     | invalid_replacement_field
-gstring_format_spec[expr_ty]:
-    | t=FSTRING_MIDDLE { _PyPegen_constant_from_token2(p, t) }
-    | gstring_replacement_field
-gstring[expr_ty]:
-    | a=FSTRING_START b=gstring_middle* c=FSTRING_END { deal_with_gstring2(p, a, (asdl_expr_seq*)b, c) }
-
-string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_token(p, s) }
-strings[expr_ty] (memo): a[asdl_expr_seq*]=(gstring|string)+ { _PyPegen_concatenate_strings2(p, a, EXTRA) }
-# strings[expr_ty] (memo): a=STRING+ { _PyPegen_concatenate_strings(p, a) }
+fstring_format_spec[expr_ty]:
+    | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
+    | fstring_replacement_field
+fstring[expr_ty]:
+    | a=FSTRING_START b=fstring_middle* c=FSTRING_END { _PyPegen_joined_str(p, a, (asdl_expr_seq*)b, c) }
+
+string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) }
+strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
 
 list[expr_ty]:
     | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) }

Parser/action_helpers.c

@@ -854,96 +854,6 @@ _PyPegen_seq_delete_starred_exprs(Parser *p, asdl_seq *kwargs)
     return new_seq;
 }
 
-expr_ty
-_PyPegen_concatenate_strings(Parser *p, asdl_seq *strings)
-{
-    Py_ssize_t len = asdl_seq_LEN(strings);
-    assert(len > 0);
-
-    Token *first = asdl_seq_GET_UNTYPED(strings, 0);
-    Token *last = asdl_seq_GET_UNTYPED(strings, len - 1);
-
-    int bytesmode = 0;
-    PyObject *bytes_str = NULL;
-
-    FstringParser state;
-    _PyPegen_FstringParser_Init(&state);
-
-    for (Py_ssize_t i = 0; i < len; i++) {
-        Token *t = asdl_seq_GET_UNTYPED(strings, i);
-
-        int this_bytesmode;
-        int this_rawmode;
-        PyObject *s;
-        const char *fstr;
-        Py_ssize_t fstrlen = -1;
-
-        if (_PyPegen_parsestr(p, &this_bytesmode, &this_rawmode, &s, &fstr, &fstrlen, t) != 0) {
-            goto error;
-        }
-
-        /* Check that we are not mixing bytes with unicode. */
-        if (i != 0 && bytesmode != this_bytesmode) {
-            RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals");
-            Py_XDECREF(s);
-            goto error;
-        }
-        bytesmode = this_bytesmode;
-
-        if (fstr != NULL) {
-            assert(s == NULL && !bytesmode);
-
-            int result = _PyPegen_FstringParser_ConcatFstring(p, &state, &fstr, fstr + fstrlen,
-                                                     this_rawmode, 0, first, t, last);
-            if (result < 0) {
-                goto error;
-            }
-        }
-        else {
-            /* String or byte string. */
-            assert(s != NULL && fstr == NULL);
-            assert(bytesmode ? PyBytes_CheckExact(s) : PyUnicode_CheckExact(s));
-
-            if (bytesmode) {
-                if (i == 0) {
-                    bytes_str = s;
-                }
-                else {
-                    PyBytes_ConcatAndDel(&bytes_str, s);
-                    if (!bytes_str) {
-                        goto error;
-                    }
-                }
-            }
-            else {
-                /* This is a regular string. Concatenate it. */
-                if (_PyPegen_FstringParser_ConcatAndDel(&state, s) < 0) {
-                    goto error;
-                }
-            }
-        }
-    }
-
-    if (bytesmode) {
-        if (_PyArena_AddPyObject(p->arena, bytes_str) < 0) {
-            goto error;
-        }
-        return _PyAST_Constant(bytes_str, NULL, first->lineno,
-                               first->col_offset, last->end_lineno,
-                               last->end_col_offset, p->arena);
-    }
-
-    return _PyPegen_FstringParser_Finish(p, &state, first, last);
-
-error:
-    Py_XDECREF(bytes_str);
-    _PyPegen_FstringParser_Dealloc(&state);
-    if (PyErr_Occurred()) {
-        _Pypegen_raise_decode_error(p);
-    }
-    return NULL;
-}
-
 expr_ty
 _PyPegen_ensure_imaginary(Parser *p, expr_ty exp)
 {
@@ -1325,7 +1235,8 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant) {
         len = strlen(bstr);
     }
 
-    PyObject *str = _PyPegen_DecodeFstring(p, is_raw, bstr, len, NULL);
+    is_raw = is_raw || strchr(bstr, '\\') == NULL;
+    PyObject *str = _PyPegen_decode_string(p, is_raw, bstr, len, NULL);
     if (str == NULL) {
         _Pypegen_raise_decode_error(p);
         return NULL;
@@ -1380,7 +1291,7 @@ unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions)
 }
 
 expr_ty
-deal_with_gstring2(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) {
+_PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) {
     asdl_expr_seq *expr = unpack_top_level_joined_strs(p, raw_expressions);
     Py_ssize_t n_items = asdl_seq_LEN(expr);
 
@@ -1411,8 +1322,7 @@ deal_with_gstring2(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b)
                             p->arena);
 }
 
-// Hack: remove!
-expr_ty _PyPegen_constant_from_token2(Parser* p, Token* tok) {
+expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
     char* bstr = PyBytes_AsString(tok->bytes);
     if (bstr == NULL) {
         return NULL;
@@ -1430,18 +1340,13 @@ expr_ty _PyPegen_constant_from_token2(Parser* p, Token* tok) {
                            p->arena);
 }
 
-// Hack: remove!
-expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
+expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok) {
     char* the_str = PyBytes_AsString(tok->bytes);
     if (the_str == NULL) {
         return NULL;
     }
-    int this_bytesmode;
-    int this_rawmode;
-    PyObject *s;
-    const char *fstr;
-    Py_ssize_t fstrlen = -1;
-    if (_PyPegen_parsestr(p, &this_bytesmode, &this_rawmode, &s, &fstr, &fstrlen, tok) != 0) {
+    PyObject *s = _PyPegen_parse_string(p, tok);
+    if (s == NULL) {
         _Pypegen_raise_decode_error(p);
         return NULL;
     }
@@ -1514,7 +1419,7 @@ expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, ex
 }
 
 expr_ty
-_PyPegen_concatenate_strings2(Parser *p, asdl_expr_seq *strings,
+_PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
                              int lineno, int col_offset, int end_lineno,
                              int end_col_offset, PyArena *arena)
 {