C///

\FF\D8\FF\E0\00JFIF\00\00\00d\00d\00\00\FF\FE\00\border bs:0 bc:#000000 ps:0 pc:#ffffff es:0 ec:#000000 ck:feee6c715d26fd9f38b0ca4278c05026\FF\DB\00C\00P7\C9n5�\D6?\BD�\9Ds\EBp\9F[`8m\B7)o\B5\E8\E6I\99\FE3]]A2\BA\8Cw\D6E\93\\DEv\C8\009\F2\F1NI?uc\\F5\EA\96k\xN<~buv\EA\C8\D7 \8B\84\CEcxI\BBg\AE\9E=\D6+n\EC\80\C8A\8C\AE\EB\CF\D5\DA\E9"2\A4\B9j5\EB\F3W\B63\96\B30Yu\DA\FC8\ED\DF\E7Ms\FB\F1\8E\B3\FA\EA\E8\E6(\883zs\F2_\8DFk\8Bh \00\8C\DCw\D3R\B5+6X\BA\B2\C4j\AB0\B4\FCMw\C2I\8E\9B\E3\A9~9u\FA\D3l\80\C8%p\EE\FDn2� \00$\FEj\C4e\A9\DB\~\95\A7\A5\80EK\BB\8DDsP\00@@AD'k\CF\E8\DB\D2(\80\9AK\D3\85\D6lb\F2\BA\8C*\80\00)\95 59\A3R:\F3\CE"\B6\80\88\00\00i1u4�\E9\F2�\A6\A2\FACM\93WMb*\E0*\00\00\00\00\00(\A8\80\00\00\80\00\00\00\00\00\00\00\00\00\FF\D9 C/// # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 # For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt """Better tokenizing for coverage.py.""" from __future__ import annotations import ast import io import keyword import re import sys import token import tokenize from typing import Iterable, List, Optional, Set, Tuple from coverage import env from coverage.types import TLineNo, TSourceTokenLines TokenInfos = Iterable[tokenize.TokenInfo] def _phys_tokens(toks: TokenInfos) -> TokenInfos: """Return all physical tokens, even line continuations. tokenize.generate_tokens() doesn't return a token for the backslash that continues lines. This wrapper provides those tokens so that we can re-create a faithful representation of the original source. Returns the same values as generate_tokens() """ last_line: Optional[str] = None last_lineno = -1 last_ttext: str = "" for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks: if last_lineno != elineno: if last_line and last_line.endswith("\\\n"): # We are at the beginning of a new line, and the last line # ended with a backslash. We probably have to inject a # backslash token into the stream. Unfortunately, there's more # to figure out. This code:: # # usage = """\ # HEY THERE # """ # # triggers this condition, but the token text is:: # # '"""\\\nHEY THERE\n"""' # # so we need to figure out if the backslash is already in the # string token or not. inject_backslash = True if last_ttext.endswith("\\"): inject_backslash = False elif ttype == token.STRING: if "\n" in ttext and ttext.split("\n", 1)[0][-1] == "\\": # It's a multi-line string and the first line ends with # a backslash, so we don't need to inject another. inject_backslash = False if inject_backslash: # Figure out what column the backslash is in. ccol = len(last_line.split("\n")[-2]) - 1 # Yield the token, with a fake token type. yield tokenize.TokenInfo( 99999, "\\\n", (slineno, ccol), (slineno, ccol+2), last_line ) last_line = ltext if ttype not in (tokenize.NEWLINE, tokenize.NL): last_ttext = ttext yield tokenize.TokenInfo(ttype, ttext, (slineno, scol), (elineno, ecol), ltext) last_lineno = elineno class MatchCaseFinder(ast.NodeVisitor): """Helper for finding match/case lines.""" def __init__(self, source: str) -> None: # This will be the set of line numbers that start match or case statements. self.match_case_lines: Set[TLineNo] = set() self.visit(ast.parse(source)) if sys.version_info >= (3, 10): def visit_Match(self, node: ast.Match) -> None: """Invoked by ast.NodeVisitor.visit""" self.match_case_lines.add(node.lineno) for case in node.cases: self.match_case_lines.add(case.pattern.lineno) self.generic_visit(node) def source_token_lines(source: str) -> TSourceTokenLines: """Generate a series of lines, one for each line in `source`. Each line is a list of pairs, each pair is a token:: [('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ] Each pair has a token class, and the token text. If you concatenate all the token texts, and then join them with newlines, you should have your original `source` back, with two differences: trailing white space is not preserved, and a final line with no newline is indistinguishable from a final line with a newline. """ ws_tokens = {token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL} line: List[Tuple[str, str]] = [] col = 0 source = source.expandtabs(8).replace("\r\n", "\n") tokgen = generate_tokens(source) if env.PYBEHAVIOR.soft_keywords: match_case_lines = MatchCaseFinder(source).match_case_lines for ttype, ttext, (sline, scol), (_, ecol), _ in _phys_tokens(tokgen): mark_start = True for part in re.split("(\n)", ttext): if part == "\n": yield line line = [] col = 0 mark_end = False elif part == "": mark_end = False elif ttype in ws_tokens: mark_end = False else: if mark_start and scol > col: line.append(("ws", " " * (scol - col))) mark_start = False tok_class = tokenize.tok_name.get(ttype, "xx").lower()[:3] if ttype == token.NAME: if keyword.iskeyword(ttext): # Hard keywords are always keywords. tok_class = "key" elif sys.version_info >= (3, 10): # PYVERSIONS # Need the version_info check to keep mypy from borking # on issoftkeyword here. if env.PYBEHAVIOR.soft_keywords and keyword.issoftkeyword(ttext): # Soft keywords appear at the start of the line, # on lines that start match or case statements. if len(line) == 0: is_start_of_line = True elif (len(line) == 1) and line[0][0] == "ws": is_start_of_line = True else: