{-# LANGUAGE CPP, DeriveDataTypeable #-}
module Language.Python.Common.Token (
Token (..),
debugTokenString,
tokenString,
hasLiteral,
TokenClass (..),
classifyToken
) where
import Language.Python.Common.Pretty
import Language.Python.Common.SrcLocation (SrcSpan (..), Span(getSpan))
import Data.Data
data Token
= IndentToken { Token -> SrcSpan
token_span :: !SrcSpan }
| DedentToken { token_span :: !SrcSpan }
| NewlineToken { token_span :: !SrcSpan }
| LineJoinToken { token_span :: !SrcSpan }
| { token_span :: !SrcSpan, Token -> String
token_literal :: !String }
| IdentifierToken { token_span :: !SrcSpan, token_literal :: !String }
| StringToken { token_span :: !SrcSpan, token_literal :: !String }
| ByteStringToken { token_span :: !SrcSpan, token_literal :: !String }
| UnicodeStringToken { token_span :: !SrcSpan, token_literal :: !String }
| IntegerToken { token_span :: !SrcSpan, token_literal :: !String, Token -> Integer
token_integer :: !Integer }
| LongIntegerToken { token_span :: !SrcSpan, token_literal :: !String, token_integer :: !Integer }
| FloatToken { token_span :: !SrcSpan, token_literal :: !String, Token -> Double
token_double :: !Double }
| ImaginaryToken { token_span :: !SrcSpan, token_literal :: !String, token_double :: !Double }
| DefToken { token_span :: !SrcSpan }
| WhileToken { token_span :: !SrcSpan }
| IfToken { token_span :: !SrcSpan }
| TrueToken { token_span :: !SrcSpan }
| FalseToken { token_span :: !SrcSpan }
| ReturnToken { token_span :: !SrcSpan }
| TryToken { token_span :: !SrcSpan }
| ExceptToken { token_span :: !SrcSpan }
| RaiseToken { token_span :: !SrcSpan }
| InToken { token_span :: !SrcSpan }
| IsToken { token_span :: !SrcSpan }
| LambdaToken { token_span :: !SrcSpan }
| ClassToken { token_span :: !SrcSpan }
| FinallyToken { token_span :: !SrcSpan }
| NoneToken { token_span :: !SrcSpan }
| ForToken { token_span :: !SrcSpan }
| FromToken { token_span :: !SrcSpan }
| GlobalToken { token_span :: !SrcSpan }
| WithToken { token_span :: !SrcSpan }
| AsToken { token_span :: !SrcSpan }
| ElifToken { token_span :: !SrcSpan }
| YieldToken { token_span :: !SrcSpan }
| AssertToken { token_span :: !SrcSpan }
| ImportToken { token_span :: !SrcSpan }
| PassToken { token_span :: !SrcSpan }
| BreakToken { token_span :: !SrcSpan }
| ContinueToken { token_span :: !SrcSpan }
| DeleteToken { token_span :: !SrcSpan }
| ElseToken { token_span :: !SrcSpan }
| NotToken { token_span :: !SrcSpan }
| AndToken { token_span :: !SrcSpan }
| OrToken { token_span :: !SrcSpan }
| NonLocalToken { token_span :: !SrcSpan }
| AsyncToken { token_span :: !SrcSpan }
| AwaitToken { token_span :: !SrcSpan }
| PrintToken { token_span :: !SrcSpan }
| ExecToken { token_span :: !SrcSpan }
| AtToken { token_span :: !SrcSpan }
| LeftRoundBracketToken { token_span :: !SrcSpan }
| RightRoundBracketToken { token_span :: !SrcSpan }
| LeftSquareBracketToken { token_span :: !SrcSpan }
| RightSquareBracketToken { token_span :: !SrcSpan }
| LeftBraceToken { token_span :: !SrcSpan }
| RightBraceToken { token_span :: !SrcSpan }
| DotToken { token_span :: !SrcSpan }
| CommaToken { token_span :: !SrcSpan }
| SemiColonToken { token_span :: !SrcSpan }
| ColonToken { token_span :: !SrcSpan }
| EllipsisToken { token_span :: !SrcSpan }
| RightArrowToken { token_span :: !SrcSpan }
| AssignToken { token_span :: !SrcSpan }
| PlusAssignToken { token_span :: !SrcSpan }
| MinusAssignToken { token_span :: !SrcSpan }
| MultAssignToken { token_span :: !SrcSpan }
| DivAssignToken { token_span :: !SrcSpan }
| ModAssignToken { token_span :: !SrcSpan }
| PowAssignToken { token_span :: !SrcSpan }
| BinAndAssignToken { token_span :: !SrcSpan }
| BinOrAssignToken { token_span :: !SrcSpan }
| BinXorAssignToken { token_span :: !SrcSpan }
| LeftShiftAssignToken { token_span :: !SrcSpan }
| RightShiftAssignToken { token_span :: !SrcSpan }
| FloorDivAssignToken { token_span :: !SrcSpan }
| MatrixMultAssignToken { token_span :: !SrcSpan }
| BackQuoteToken { token_span :: !SrcSpan }
| PlusToken { token_span :: !SrcSpan }
| MinusToken { token_span :: !SrcSpan }
| MultToken { token_span :: !SrcSpan }
| DivToken { token_span :: !SrcSpan }
| GreaterThanToken { token_span :: !SrcSpan }
| LessThanToken { token_span :: !SrcSpan }
| EqualityToken { token_span :: !SrcSpan }
| GreaterThanEqualsToken { token_span :: !SrcSpan }
| LessThanEqualsToken { token_span :: !SrcSpan }
| ExponentToken { token_span :: !SrcSpan }
| BinaryOrToken { token_span :: !SrcSpan }
| XorToken { token_span :: !SrcSpan }
| BinaryAndToken { token_span :: !SrcSpan }
| ShiftLeftToken { token_span :: !SrcSpan }
| ShiftRightToken { token_span :: !SrcSpan }
| ModuloToken { token_span :: !SrcSpan }
| FloorDivToken { token_span :: !SrcSpan }
| TildeToken { token_span :: !SrcSpan }
| NotEqualsToken { token_span :: !SrcSpan }
| NotEqualsV2Token { token_span :: !SrcSpan }
| EOFToken { token_span :: !SrcSpan }
deriving (Token -> Token -> Bool
(Token -> Token -> Bool) -> (Token -> Token -> Bool) -> Eq Token
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
/= :: Token -> Token -> Bool
$c/= :: Token -> Token -> Bool
== :: Token -> Token -> Bool
$c== :: Token -> Token -> Bool
Eq,Eq Token
Eq Token =>
(Token -> Token -> Ordering)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Token)
-> (Token -> Token -> Token)
-> Ord Token
Token -> Token -> Bool
Token -> Token -> Ordering
Token -> Token -> Token
forall a.
Eq a =>
(a -> a -> Ordering)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> a)
-> (a -> a -> a)
-> Ord a
min :: Token -> Token -> Token
$cmin :: Token -> Token -> Token
max :: Token -> Token -> Token
$cmax :: Token -> Token -> Token
>= :: Token -> Token -> Bool
$c>= :: Token -> Token -> Bool
> :: Token -> Token -> Bool
$c> :: Token -> Token -> Bool
<= :: Token -> Token -> Bool
$c<= :: Token -> Token -> Bool
< :: Token -> Token -> Bool
$c< :: Token -> Token -> Bool
compare :: Token -> Token -> Ordering
$ccompare :: Token -> Token -> Ordering
$cp1Ord :: Eq Token
Ord,Int -> Token -> ShowS
[Token] -> ShowS
Token -> String
(Int -> Token -> ShowS)
-> (Token -> String) -> ([Token] -> ShowS) -> Show Token
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
showList :: [Token] -> ShowS
$cshowList :: [Token] -> ShowS
show :: Token -> String
$cshow :: Token -> String
showsPrec :: Int -> Token -> ShowS
$cshowsPrec :: Int -> Token -> ShowS
Show,Typeable,)
instance Span Token where
getSpan :: Token -> SrcSpan
getSpan = Token -> SrcSpan
token_span
debugTokenString :: Token -> String
debugTokenString :: Token -> String
debugTokenString token :: Token
token =
Doc -> String
render (String -> Doc
text (Constr -> String
forall a. Show a => a -> String
show (Constr -> String) -> Constr -> String
forall a b. (a -> b) -> a -> b
$ Token -> Constr
forall a. Data a => a -> Constr
toConstr Token
token) Doc -> Doc -> Doc
<+> SrcSpan -> Doc
forall a. Pretty a => a -> Doc
pretty (Token -> SrcSpan
token_span Token
token) Doc -> Doc -> Doc
<+>
if Token -> Bool
hasLiteral Token
token then String -> Doc
text (Token -> String
token_literal Token
token) else Doc
empty)
hasLiteral :: Token -> Bool
hasLiteral :: Token -> Bool
hasLiteral token :: Token
token =
case Token
token of
CommentToken {} -> Bool
True
IdentifierToken {} -> Bool
True
StringToken {} -> Bool
True
ByteStringToken {} -> Bool
True
UnicodeStringToken {} -> Bool
True
IntegerToken {} -> Bool
True
LongIntegerToken {} -> Bool
True
FloatToken {} -> Bool
True
ImaginaryToken {} -> Bool
True
other :: Token
other -> Bool
False
data TokenClass
=
| Number
| Identifier
| Punctuation
| Bracket
| Layout
| Keyword
| String
| Operator
| Assignment
deriving (Int -> TokenClass -> ShowS
[TokenClass] -> ShowS
TokenClass -> String
(Int -> TokenClass -> ShowS)
-> (TokenClass -> String)
-> ([TokenClass] -> ShowS)
-> Show TokenClass
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
showList :: [TokenClass] -> ShowS
$cshowList :: [TokenClass] -> ShowS
show :: TokenClass -> String
$cshow :: TokenClass -> String
showsPrec :: Int -> TokenClass -> ShowS
$cshowsPrec :: Int -> TokenClass -> ShowS
Show, TokenClass -> TokenClass -> Bool
(TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool) -> Eq TokenClass
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
/= :: TokenClass -> TokenClass -> Bool
$c/= :: TokenClass -> TokenClass -> Bool
== :: TokenClass -> TokenClass -> Bool
$c== :: TokenClass -> TokenClass -> Bool
Eq, Eq TokenClass
Eq TokenClass =>
(TokenClass -> TokenClass -> Ordering)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> TokenClass)
-> (TokenClass -> TokenClass -> TokenClass)
-> Ord TokenClass
TokenClass -> TokenClass -> Bool
TokenClass -> TokenClass -> Ordering
TokenClass -> TokenClass -> TokenClass
forall a.
Eq a =>
(a -> a -> Ordering)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> a)
-> (a -> a -> a)
-> Ord a
min :: TokenClass -> TokenClass -> TokenClass
$cmin :: TokenClass -> TokenClass -> TokenClass
max :: TokenClass -> TokenClass -> TokenClass
$cmax :: TokenClass -> TokenClass -> TokenClass
>= :: TokenClass -> TokenClass -> Bool
$c>= :: TokenClass -> TokenClass -> Bool
> :: TokenClass -> TokenClass -> Bool
$c> :: TokenClass -> TokenClass -> Bool
<= :: TokenClass -> TokenClass -> Bool
$c<= :: TokenClass -> TokenClass -> Bool
< :: TokenClass -> TokenClass -> Bool
$c< :: TokenClass -> TokenClass -> Bool
compare :: TokenClass -> TokenClass -> Ordering
$ccompare :: TokenClass -> TokenClass -> Ordering
$cp1Ord :: Eq TokenClass
Ord)
classifyToken :: Token -> TokenClass
classifyToken :: Token -> TokenClass
classifyToken token :: Token
token =
case Token
token of
IndentToken {} -> TokenClass
Layout
DedentToken {} -> TokenClass
Layout
NewlineToken {} -> TokenClass
Layout
CommentToken {} -> TokenClass
Comment
IdentifierToken {} -> TokenClass
Identifier
StringToken {} -> TokenClass
String
ByteStringToken {} -> TokenClass
String
UnicodeStringToken {} -> TokenClass
String
IntegerToken {} -> TokenClass
Number
LongIntegerToken {} -> TokenClass
Number
FloatToken {} -> TokenClass
Number
ImaginaryToken {} -> TokenClass
Number
DefToken {} -> TokenClass
Keyword
WhileToken {} -> TokenClass
Keyword
IfToken {} -> TokenClass
Keyword
TrueToken {} -> TokenClass
Keyword
FalseToken {} -> TokenClass
Keyword
ReturnToken {} -> TokenClass
Keyword
TryToken {} -> TokenClass
Keyword
ExceptToken {} -> TokenClass
Keyword
RaiseToken {} -> TokenClass
Keyword
InToken {} -> TokenClass
Keyword
IsToken {} -> TokenClass
Keyword
LambdaToken {} -> TokenClass
Keyword
ClassToken {} -> TokenClass
Keyword
FinallyToken {} -> TokenClass
Keyword
NoneToken {} -> TokenClass
Keyword
ForToken {} -> TokenClass
Keyword
FromToken {} -> TokenClass
Keyword
GlobalToken {} -> TokenClass
Keyword
WithToken {} -> TokenClass
Keyword
AsToken {} -> TokenClass
Keyword
ElifToken {} -> TokenClass
Keyword
YieldToken {} -> TokenClass
Keyword
AsyncToken {} -> TokenClass
Keyword
AwaitToken {} -> TokenClass
Keyword
AssertToken {} -> TokenClass
Keyword
ImportToken {} -> TokenClass
Keyword
PassToken {} -> TokenClass
Keyword
BreakToken {} -> TokenClass
Keyword
ContinueToken {} -> TokenClass
Keyword
DeleteToken {} -> TokenClass
Keyword
ElseToken {} -> TokenClass
Keyword
NotToken {} -> TokenClass
Keyword
AndToken {} -> TokenClass
Keyword
OrToken {} -> TokenClass
Keyword
NonLocalToken {} -> TokenClass
Keyword
PrintToken {} -> TokenClass
Keyword
ExecToken {} -> TokenClass
Keyword
AtToken {} -> TokenClass
Keyword
LeftRoundBracketToken {} -> TokenClass
Bracket
RightRoundBracketToken {} -> TokenClass
Bracket
LeftSquareBracketToken {} -> TokenClass
Bracket
RightSquareBracketToken {} -> TokenClass
Bracket
LeftBraceToken {} -> TokenClass
Bracket
RightBraceToken {} -> TokenClass
Bracket
DotToken {} -> TokenClass
Operator
CommaToken {} -> TokenClass
Punctuation
SemiColonToken {} -> TokenClass
Punctuation
ColonToken {} -> TokenClass
Punctuation
EllipsisToken {} -> TokenClass
Keyword
RightArrowToken {} -> TokenClass
Punctuation
AssignToken {} -> TokenClass
Assignment
PlusAssignToken {} -> TokenClass
Assignment
MinusAssignToken {} -> TokenClass
Assignment
MultAssignToken {} -> TokenClass
Assignment
DivAssignToken {} -> TokenClass
Assignment
ModAssignToken {} -> TokenClass
Assignment
PowAssignToken {} -> TokenClass
Assignment
BinAndAssignToken {} -> TokenClass
Assignment
BinOrAssignToken {} -> TokenClass
Assignment
BinXorAssignToken {} -> TokenClass
Assignment
LeftShiftAssignToken {} -> TokenClass
Assignment
RightShiftAssignToken {} -> TokenClass
Assignment
FloorDivAssignToken {} -> TokenClass
Assignment
MatrixMultAssignToken {} -> TokenClass
Assignment
BackQuoteToken {} -> TokenClass
Punctuation
PlusToken {} -> TokenClass
Operator
MinusToken {} -> TokenClass
Operator
MultToken {} -> TokenClass
Operator
DivToken {} -> TokenClass
Operator
GreaterThanToken {} -> TokenClass
Operator
LessThanToken {} -> TokenClass
Operator
EqualityToken {} -> TokenClass
Operator
GreaterThanEqualsToken {} -> TokenClass
Operator
LessThanEqualsToken {} -> TokenClass
Operator
ExponentToken {} -> TokenClass
Operator
BinaryOrToken {} -> TokenClass
Operator
XorToken {} -> TokenClass
Operator
BinaryAndToken {} -> TokenClass
Operator
ShiftLeftToken {} -> TokenClass
Operator
ShiftRightToken {} -> TokenClass
Operator
ModuloToken {} -> TokenClass
Operator
FloorDivToken {} -> TokenClass
Operator
TildeToken {} -> TokenClass
Operator
NotEqualsToken {} -> TokenClass
Operator
NotEqualsV2Token {} -> TokenClass
Operator
LineJoinToken {} -> TokenClass
Layout
EOFToken {} -> TokenClass
Layout
tokenString :: Token -> String
tokenString :: Token -> String
tokenString token :: Token
token =
case Token
token of
IndentToken {} -> ""
DedentToken {} -> ""
NewlineToken {} -> ""
CommentToken {} -> Token -> String
token_literal Token
token
IdentifierToken {} -> Token -> String
token_literal Token
token
StringToken {} -> Token -> String
token_literal Token
token
ByteStringToken {} -> Token -> String
token_literal Token
token
UnicodeStringToken {} -> Token -> String
token_literal Token
token
IntegerToken {} -> Token -> String
token_literal Token
token
LongIntegerToken {} -> Token -> String
token_literal Token
token
FloatToken {} -> Token -> String
token_literal Token
token
ImaginaryToken {} -> Token -> String
token_literal Token
token
DefToken {} -> "def"
WhileToken {} -> "while"
IfToken {} -> "if"
TrueToken {} -> "True"
FalseToken {} -> "False"
ReturnToken {} -> "return"
TryToken {} -> "try"
ExceptToken {} -> "except"
RaiseToken {} -> "raise"
InToken {} -> "in"
IsToken {} -> "is"
LambdaToken {} -> "lambda"
ClassToken {} -> "class"
FinallyToken {} -> "finally"
NoneToken {} -> "None"
ForToken {} -> "for"
FromToken {} -> "from"
GlobalToken {} -> "global"
WithToken {} -> "with"
AsToken {} -> "as"
ElifToken {} -> "elif"
YieldToken {} -> "yield"
AsyncToken {} -> "async"
AwaitToken {} -> "await"
AssertToken {} -> "assert"
ImportToken {} -> "import"
PassToken {} -> "pass"
BreakToken {} -> "break"
ContinueToken {} -> "continue"
DeleteToken {} -> "delete"
ElseToken {} -> "else"
NotToken {} -> "not"
AndToken {} -> "and"
OrToken {} -> "or"
NonLocalToken {} -> "nonlocal"
PrintToken {} -> "print"
ExecToken {} -> "exec"
AtToken {} -> "at"
LeftRoundBracketToken {} -> "("
RightRoundBracketToken {} -> ")"
LeftSquareBracketToken {} -> "["
RightSquareBracketToken {} -> "]"
LeftBraceToken {} -> "{"
RightBraceToken {} -> "}"
DotToken {} -> "."
CommaToken {} -> ","
SemiColonToken {} -> ";"
ColonToken {} -> ":"
EllipsisToken {} -> "..."
RightArrowToken {} -> "->"
AssignToken {} -> "="
PlusAssignToken {} -> "+="
MinusAssignToken {} -> "-="
MultAssignToken {} -> "*="
DivAssignToken {} -> "/="
ModAssignToken {} -> "%="
PowAssignToken {} -> "**="
BinAndAssignToken {} -> "&="
BinOrAssignToken {} -> "|="
BinXorAssignToken {} -> "^="
LeftShiftAssignToken {} -> "<<="
RightShiftAssignToken {} -> ">>="
FloorDivAssignToken {} -> "//="
MatrixMultAssignToken {} -> "@="
BackQuoteToken {} -> "`"
PlusToken {} -> "+"
MinusToken {} -> "-"
MultToken {} -> "*"
DivToken {} -> "/"
GreaterThanToken {} -> ">"
LessThanToken {} -> "<"
EqualityToken {} -> "=="
GreaterThanEqualsToken {} -> ">="
LessThanEqualsToken {} -> "<="
ExponentToken {} -> "**"
BinaryOrToken {} -> "|"
XorToken {} -> "^"
BinaryAndToken {} -> "&"
ShiftLeftToken {} -> "<<"
ShiftRightToken {} -> ">>"
ModuloToken {} -> "%"
FloorDivToken {} -> "//"
TildeToken {} -> "~"
NotEqualsToken {} -> "!="
NotEqualsV2Token {} -> "<>"
LineJoinToken {} -> "\\"
EOFToken {} -> ""