Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Drop python < 3.8 #117

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+252 −1,844
Open

Drop python < 3.8 #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b177fc5
remove python versions < 3.8 from the tests
aqeelat Sep 5, 2023
de0a2e7
remove python versions < 3.8 from the code
aqeelat Sep 6, 2023
93a88e0
update tox
aqeelat Sep 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 8 additions & 15 deletions asttokens/asttokens.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
from typing import Iterable, Iterator, List, Optional, Tuple, Any, cast, TYPE_CHECKING

import six
from six.moves import xrange # pylint: disable=redefined-builtin

from .line_numbers import LineNumbers
from .util import (
Expand All @@ -33,18 +32,17 @@
from .util import AstNode, TokenInfo


class ASTTextBase(six.with_metaclass(abc.ABCMeta, object)):
class ASTTextBase(metaclass=abc.ABCMeta):
def __init__(self, source_text, filename):
# type: (Any, str) -> None
# FIXME: Strictly, the type of source_text is one of the six string types, but hard to specify with mypy given
# https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases

# type: (str, str) -> None
self._filename = filename

# Decode source after parsing to let Python 2 handle coding declarations.
# (If the encoding was not utf-8 compatible, then even if it parses correctly,
# we'll fail with a unicode error here.)
source_text = six.ensure_text(source_text)
# FIXME: This is the only remaining usage for six. Do we still need this check?
# If so, then we should copy the function here and remove dependency on six
source_text = six.ensure_text(source_text)
Comment on lines +43 to +45
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

review this part

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I'd say we should still decode bytes to preserve compatibility. So the type hint needs to be amended. Also the "Decode source after parsing to let Python 2..." doesn't even make sense and needs updating.


self._text = source_text
self._line_numbers = LineNumbers(source_text)
Expand Down Expand Up @@ -110,9 +108,7 @@ class ASTTokens(ASTTextBase, object):
"""

def __init__(self, source_text, parse=False, tree=None, filename='<unknown>', tokens=None):
# type: (Any, bool, Optional[Module], str, Iterable[TokenInfo]) -> None
# FIXME: Strictly, the type of source_text is one of the six string types, but hard to specify with mypy given
# https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
# type: (str, bool, Optional[Module], str, Iterable[TokenInfo]) -> None

super(ASTTokens, self).__init__(source_text, filename)

Expand Down Expand Up @@ -249,7 +245,7 @@ def token_range(self,
Yields all tokens in order from first_token through and including last_token. If
include_extra is True, includes non-coding tokens such as tokenize.NL and .COMMENT.
"""
for i in xrange(first_token.index, last_token.index + 1):
for i in range(first_token.index, last_token.index + 1):
if include_extra or not is_non_coding_token(self._tokens[i].type):
yield self._tokens[i]

Expand Down Expand Up @@ -298,9 +294,7 @@ class ASTText(ASTTextBase, object):
If you want to avoid this, check ``supports_tokenless(node)`` before calling ``get_text*`` methods.
"""
def __init__(self, source_text, tree=None, filename='<unknown>'):
# type: (Any, Optional[Module], str) -> None
# FIXME: Strictly, the type of source_text is one of the six string types, but hard to specify with mypy given
# https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
# type: (str, Optional[Module], str) -> None

super(ASTText, self).__init__(source_text, filename)

Expand Down Expand Up @@ -466,6 +460,5 @@ def supports_tokenless(node=None):
)
)
)
and sys.version_info[:2] >= (3, 8)
and 'pypy' not in sys.version.lower()
)
34 changes: 2 additions & 32 deletions asttokens/mark_tokens.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,8 @@
from ast import Module
from typing import Callable, List, Union, cast, Optional, Tuple, TYPE_CHECKING

import six

from . import util
from .asttokens import ASTTokens
from .util import AstConstant
from .astroid_compat import astroid_node_classes as nc

if TYPE_CHECKING:
Expand Down Expand Up @@ -179,23 +176,6 @@ def handle_comp(self, open_brace, node, first_token, last_token):
util.expect_token(before, token.OP, open_brace)
return (before, last_token)

# Python 3.8 fixed the starting position of list comprehensions:
# https://bugs.python.org/issue31241
if sys.version_info < (3, 8):
def visit_listcomp(self, node, first_token, last_token):
# type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token]
return self.handle_comp('[', node, first_token, last_token)

if six.PY2:
# We shouldn't do this on PY3 because its SetComp/DictComp already have a correct start.
def visit_setcomp(self, node, first_token, last_token):
# type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token]
return self.handle_comp('{', node, first_token, last_token)

def visit_dictcomp(self, node, first_token, last_token):
# type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token]
return self.handle_comp('{', node, first_token, last_token)

def visit_comprehension(self,
node, # type: AstNode
first_token, # type: util.Token
Expand Down Expand Up @@ -432,16 +412,13 @@ def visit_num(self, node, first_token, last_token):
# In Astroid, the Num and Str nodes are replaced by Const.
def visit_const(self, node, first_token, last_token):
# type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token]
assert isinstance(node, AstConstant) or isinstance(node, nc.Const)
assert isinstance(node, ast.Constant) or isinstance(node, nc.Const)
if isinstance(node.value, numbers.Number):
return self.handle_num(node, node.value, first_token, last_token)
elif isinstance(node.value, (six.text_type, six.binary_type)):
elif isinstance(node.value, (str, bytes)):
return self.visit_str(node, first_token, last_token)
return (first_token, last_token)

# In Python >= 3.6, there is a similar class 'Constant' for literals
# In 3.8 it became the type produced by ast.parse
# https://bugs.python.org/issue32892
visit_constant = visit_const

def visit_keyword(self, node, first_token, last_token):
Expand Down Expand Up @@ -473,13 +450,6 @@ def visit_assignname(self, node, first_token, last_token):
first_token = last_token = self._code.prev_token(colon)
return (first_token, last_token)

if six.PY2:
# No need for this on Python3, which already handles 'with' nodes correctly.
def visit_with(self, node, first_token, last_token):
# type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token]
first = self._code.find_token(first_token, token.NAME, 'with', reverse=True)
return (first, last_token)

# Async nodes should typically start with the word 'async'
# but Python < 3.7 doesn't put the col_offset there
# AsyncFunctionDef is slightly different because it might have
Expand Down
Loading
Loading