{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE LambdaCase #-}
{-# LANGUAGE MultiWayIf #-}
{-# LANGUAGE RankNTypes #-}
{-# LANGUAGE RecordWildCards #-}
{-# LANGUAGE ScopedTypeVariables #-}
{-# LANGUAGE TypeFamilies #-}
module Text.Megaparsec.Stream
( Stream (..) )
where
import Data.Char (chr)
import Data.Foldable (foldl')
import Data.Kind (Type)
import Data.List.NonEmpty (NonEmpty (..))
import Data.Maybe (fromMaybe)
import Data.Proxy
import Data.Word (Word8)
import Text.Megaparsec.Pos
import Text.Megaparsec.State
import qualified Data.ByteString as B
import qualified Data.ByteString.Char8 as B8
import qualified Data.ByteString.Lazy as BL
import qualified Data.ByteString.Lazy.Char8 as BL8
import qualified Data.List.NonEmpty as NE
import qualified Data.Text as T
import qualified Data.Text.Lazy as TL
class (Ord (Token s), Ord (Tokens s)) => Stream s where
type Token s :: Type
type Tokens s :: Type
tokenToChunk :: Proxy s -> Token s -> Tokens s
tokenToChunk pxy :: Proxy s
pxy = Proxy s -> [Token s] -> Tokens s
forall s. Stream s => Proxy s -> [Token s] -> Tokens s
tokensToChunk Proxy s
pxy ([Token s] -> Tokens s)
-> (Token s -> [Token s]) -> Token s -> Tokens s
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Token s -> [Token s]
forall (f :: * -> *) a. Applicative f => a -> f a
pure
tokensToChunk :: Proxy s -> [Token s] -> Tokens s
chunkToTokens :: Proxy s -> Tokens s -> [Token s]
chunkLength :: Proxy s -> Tokens s -> Int
chunkEmpty :: Proxy s -> Tokens s -> Bool
chunkEmpty pxy :: Proxy s
pxy ts :: Tokens s
ts = Proxy s -> Tokens s -> Int
forall s. Stream s => Proxy s -> Tokens s -> Int
chunkLength Proxy s
pxy Tokens s
ts Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= 0
take1_ :: s -> Maybe (Token s, s)
takeN_ :: Int -> s -> Maybe (Tokens s, s)
takeWhile_ :: (Token s -> Bool) -> s -> (Tokens s, s)
showTokens :: Proxy s -> NonEmpty (Token s) -> String
tokensLength :: Proxy s -> NonEmpty (Token s) -> Int
tokensLength Proxy = NonEmpty (Token s) -> Int
forall a. NonEmpty a -> Int
NE.length
reachOffset
:: Int
-> PosState s
-> (String, PosState s)
reachOffsetNoLine
:: Int
-> PosState s
-> PosState s
reachOffsetNoLine o :: Int
o pst :: PosState s
pst =
(String, PosState s) -> PosState s
forall a b. (a, b) -> b
snd (Int -> PosState s -> (String, PosState s)
forall s. Stream s => Int -> PosState s -> (String, PosState s)
reachOffset Int
o PosState s
pst)
instance Stream String where
type Token String = Char
type Tokens String = String
tokenToChunk :: Proxy String -> Token String -> Tokens String
tokenToChunk Proxy = Token String -> Tokens String
forall (f :: * -> *) a. Applicative f => a -> f a
pure
tokensToChunk :: Proxy String -> [Token String] -> Tokens String
tokensToChunk Proxy = [Token String] -> Tokens String
forall a. a -> a
id
chunkToTokens :: Proxy String -> Tokens String -> [Token String]
chunkToTokens Proxy = Tokens String -> [Token String]
forall a. a -> a
id
chunkLength :: Proxy String -> Tokens String -> Int
chunkLength Proxy = Tokens String -> Int
forall (t :: * -> *) a. Foldable t => t a -> Int
length
chunkEmpty :: Proxy String -> Tokens String -> Bool
chunkEmpty Proxy = Tokens String -> Bool
forall (t :: * -> *) a. Foldable t => t a -> Bool
null
take1_ :: String -> Maybe (Token String, String)
take1_ [] = Maybe (Token String, String)
forall a. Maybe a
Nothing
take1_ (t :: Char
t:ts :: String
ts) = (Char, String) -> Maybe (Char, String)
forall a. a -> Maybe a
Just (Char
t, String
ts)
takeN_ :: Int -> String -> Maybe (Tokens String, String)
takeN_ n :: Int
n s :: String
s
| Int
n Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= 0 = (String, String) -> Maybe (String, String)
forall a. a -> Maybe a
Just ("", String
s)
| String -> Bool
forall (t :: * -> *) a. Foldable t => t a -> Bool
null String
s = Maybe (Tokens String, String)
forall a. Maybe a
Nothing
| Bool
otherwise = (String, String) -> Maybe (String, String)
forall a. a -> Maybe a
Just (Int -> String -> (String, String)
forall a. Int -> [a] -> ([a], [a])
splitAt Int
n String
s)
takeWhile_ :: (Token String -> Bool) -> String -> (Tokens String, String)
takeWhile_ = (Token String -> Bool) -> String -> (Tokens String, String)
forall a. (a -> Bool) -> [a] -> ([a], [a])
span
showTokens :: Proxy String -> NonEmpty (Token String) -> String
showTokens Proxy = NonEmpty Char -> String
NonEmpty (Token String) -> String
stringPretty
reachOffset :: Int -> PosState String -> (String, PosState String)
reachOffset o :: Int
o pst :: PosState String
pst =
(Int -> String -> (Tokens String, String))
-> (forall b. (b -> Token String -> b) -> b -> Tokens String -> b)
-> (Tokens String -> String)
-> (Token String -> Char)
-> (Token String, Token String)
-> Int
-> PosState String
-> (String, PosState String)
forall s.
Stream s =>
(Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Tokens s -> String)
-> (Token s -> Char)
-> (Token s, Token s)
-> Int
-> PosState s
-> (String, PosState s)
reachOffset' Int -> String -> (Tokens String, String)
forall a. Int -> [a] -> ([a], [a])
splitAt forall b. (b -> Token String -> b) -> b -> Tokens String -> b
forall (t :: * -> *) b a.
Foldable t =>
(b -> a -> b) -> b -> t a -> b
foldl' Tokens String -> String
forall a. a -> a
id Token String -> Char
forall a. a -> a
id (Token String
'\n',Token String
'\t') Int
o PosState String
pst
reachOffsetNoLine :: Int -> PosState String -> PosState String
reachOffsetNoLine o :: Int
o pst :: PosState String
pst =
(Int -> String -> (Tokens String, String))
-> (forall b. (b -> Token String -> b) -> b -> Tokens String -> b)
-> (Token String, Token String)
-> Int
-> PosState String
-> PosState String
forall s.
Stream s =>
(Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Token s, Token s)
-> Int
-> PosState s
-> PosState s
reachOffsetNoLine' Int -> String -> (Tokens String, String)
forall a. Int -> [a] -> ([a], [a])
splitAt forall b. (b -> Token String -> b) -> b -> Tokens String -> b
forall (t :: * -> *) b a.
Foldable t =>
(b -> a -> b) -> b -> t a -> b
foldl' (Token String
'\n', Token String
'\t') Int
o PosState String
pst
instance Stream B.ByteString where
type Token B.ByteString = Word8
type Tokens B.ByteString = B.ByteString
tokenToChunk :: Proxy ByteString -> Token ByteString -> Tokens ByteString
tokenToChunk Proxy = Word8 -> ByteString
Token ByteString -> Tokens ByteString
B.singleton
tokensToChunk :: Proxy ByteString -> [Token ByteString] -> Tokens ByteString
tokensToChunk Proxy = [Word8] -> ByteString
[Token ByteString] -> Tokens ByteString
B.pack
chunkToTokens :: Proxy ByteString -> Tokens ByteString -> [Token ByteString]
chunkToTokens Proxy = ByteString -> [Word8]
Tokens ByteString -> [Token ByteString]
B.unpack
chunkLength :: Proxy ByteString -> Tokens ByteString -> Int
chunkLength Proxy = ByteString -> Int
Tokens ByteString -> Int
B.length
chunkEmpty :: Proxy ByteString -> Tokens ByteString -> Bool
chunkEmpty Proxy = ByteString -> Bool
Tokens ByteString -> Bool
B.null
take1_ :: ByteString -> Maybe (Token ByteString, ByteString)
take1_ = ByteString -> Maybe (Word8, ByteString)
ByteString -> Maybe (Token ByteString, ByteString)
B.uncons
takeN_ :: Int -> ByteString -> Maybe (Tokens ByteString, ByteString)
takeN_ n :: Int
n s :: ByteString
s
| Int
n Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= 0 = (ByteString, ByteString) -> Maybe (ByteString, ByteString)
forall a. a -> Maybe a
Just (ByteString
B.empty, ByteString
s)
| ByteString -> Bool
B.null ByteString
s = Maybe (Tokens ByteString, ByteString)
forall a. Maybe a
Nothing
| Bool
otherwise = (ByteString, ByteString) -> Maybe (ByteString, ByteString)
forall a. a -> Maybe a
Just (Int -> ByteString -> (ByteString, ByteString)
B.splitAt Int
n ByteString
s)
takeWhile_ :: (Token ByteString -> Bool)
-> ByteString -> (Tokens ByteString, ByteString)
takeWhile_ = (Word8 -> Bool) -> ByteString -> (ByteString, ByteString)
(Token ByteString -> Bool)
-> ByteString -> (Tokens ByteString, ByteString)
B.span
showTokens :: Proxy ByteString -> NonEmpty (Token ByteString) -> String
showTokens Proxy = NonEmpty Char -> String
stringPretty (NonEmpty Char -> String)
-> (NonEmpty Word8 -> NonEmpty Char) -> NonEmpty Word8 -> String
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Word8 -> Char) -> NonEmpty Word8 -> NonEmpty Char
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap (Int -> Char
chr (Int -> Char) -> (Word8 -> Int) -> Word8 -> Char
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Word8 -> Int
forall a b. (Integral a, Num b) => a -> b
fromIntegral)
reachOffset :: Int -> PosState ByteString -> (String, PosState ByteString)
reachOffset o :: Int
o pst :: PosState ByteString
pst =
(Int -> ByteString -> (Tokens ByteString, ByteString))
-> (forall b.
(b -> Token ByteString -> b) -> b -> Tokens ByteString -> b)
-> (Tokens ByteString -> String)
-> (Token ByteString -> Char)
-> (Token ByteString, Token ByteString)
-> Int
-> PosState ByteString
-> (String, PosState ByteString)
forall s.
Stream s =>
(Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Tokens s -> String)
-> (Token s -> Char)
-> (Token s, Token s)
-> Int
-> PosState s
-> (String, PosState s)
reachOffset' Int -> ByteString -> (ByteString, ByteString)
Int -> ByteString -> (Tokens ByteString, ByteString)
B.splitAt forall a. (a -> Word8 -> a) -> a -> ByteString -> a
forall b.
(b -> Token ByteString -> b) -> b -> Tokens ByteString -> b
B.foldl' ByteString -> String
Tokens ByteString -> String
B8.unpack (Int -> Char
chr (Int -> Char) -> (Word8 -> Int) -> Word8 -> Char
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Word8 -> Int
forall a b. (Integral a, Num b) => a -> b
fromIntegral) (10, 9) Int
o PosState ByteString
pst
reachOffsetNoLine :: Int -> PosState ByteString -> PosState ByteString
reachOffsetNoLine o :: Int
o pst :: PosState ByteString
pst =
(Int -> ByteString -> (Tokens ByteString, ByteString))
-> (forall b.
(b -> Token ByteString -> b) -> b -> Tokens ByteString -> b)
-> (Token ByteString, Token ByteString)
-> Int
-> PosState ByteString
-> PosState ByteString
forall s.
Stream s =>
(Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Token s, Token s)
-> Int
-> PosState s
-> PosState s
reachOffsetNoLine' Int -> ByteString -> (ByteString, ByteString)
Int -> ByteString -> (Tokens ByteString, ByteString)
B.splitAt forall a. (a -> Word8 -> a) -> a -> ByteString -> a
forall b.
(b -> Token ByteString -> b) -> b -> Tokens ByteString -> b
B.foldl' (10, 9) Int
o PosState ByteString
pst
instance Stream BL.ByteString where
type Token BL.ByteString = Word8
type Tokens BL.ByteString = BL.ByteString
tokenToChunk :: Proxy ByteString -> Token ByteString -> Tokens ByteString
tokenToChunk Proxy = Word8 -> ByteString
Token ByteString -> Tokens ByteString
BL.singleton
tokensToChunk :: Proxy ByteString -> [Token ByteString] -> Tokens ByteString
tokensToChunk Proxy = [Word8] -> ByteString
[Token ByteString] -> Tokens ByteString
BL.pack
chunkToTokens :: Proxy ByteString -> Tokens ByteString -> [Token ByteString]
chunkToTokens Proxy = ByteString -> [Word8]
Tokens ByteString -> [Token ByteString]
BL.unpack
chunkLength :: Proxy ByteString -> Tokens ByteString -> Int
chunkLength Proxy = Int64 -> Int
forall a b. (Integral a, Num b) => a -> b
fromIntegral (Int64 -> Int) -> (ByteString -> Int64) -> ByteString -> Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. ByteString -> Int64
BL.length
chunkEmpty :: Proxy ByteString -> Tokens ByteString -> Bool
chunkEmpty Proxy = ByteString -> Bool
Tokens ByteString -> Bool
BL.null
take1_ :: ByteString -> Maybe (Token ByteString, ByteString)
take1_ = ByteString -> Maybe (Word8, ByteString)
ByteString -> Maybe (Token ByteString, ByteString)
BL.uncons
takeN_ :: Int -> ByteString -> Maybe (Tokens ByteString, ByteString)
takeN_ n :: Int
n s :: ByteString
s
| Int
n Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= 0 = (ByteString, ByteString) -> Maybe (ByteString, ByteString)
forall a. a -> Maybe a
Just (ByteString
BL.empty, ByteString
s)
| ByteString -> Bool
BL.null ByteString
s = Maybe (Tokens ByteString, ByteString)
forall a. Maybe a
Nothing
| Bool
otherwise = (ByteString, ByteString) -> Maybe (ByteString, ByteString)
forall a. a -> Maybe a
Just (Int64 -> ByteString -> (ByteString, ByteString)
BL.splitAt (Int -> Int64
forall a b. (Integral a, Num b) => a -> b
fromIntegral Int
n) ByteString
s)
takeWhile_ :: (Token ByteString -> Bool)
-> ByteString -> (Tokens ByteString, ByteString)
takeWhile_ = (Word8 -> Bool) -> ByteString -> (ByteString, ByteString)
(Token ByteString -> Bool)
-> ByteString -> (Tokens ByteString, ByteString)
BL.span
showTokens :: Proxy ByteString -> NonEmpty (Token ByteString) -> String
showTokens Proxy = NonEmpty Char -> String
stringPretty (NonEmpty Char -> String)
-> (NonEmpty Word8 -> NonEmpty Char) -> NonEmpty Word8 -> String
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Word8 -> Char) -> NonEmpty Word8 -> NonEmpty Char
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap (Int -> Char
chr (Int -> Char) -> (Word8 -> Int) -> Word8 -> Char
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Word8 -> Int
forall a b. (Integral a, Num b) => a -> b
fromIntegral)
reachOffset :: Int -> PosState ByteString -> (String, PosState ByteString)
reachOffset o :: Int
o pst :: PosState ByteString
pst =
(Int -> ByteString -> (Tokens ByteString, ByteString))
-> (forall b.
(b -> Token ByteString -> b) -> b -> Tokens ByteString -> b)
-> (Tokens ByteString -> String)
-> (Token ByteString -> Char)
-> (Token ByteString, Token ByteString)
-> Int
-> PosState ByteString
-> (String, PosState ByteString)
forall s.
Stream s =>
(Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Tokens s -> String)
-> (Token s -> Char)
-> (Token s, Token s)
-> Int
-> PosState s
-> (String, PosState s)
reachOffset' Int -> ByteString -> (ByteString, ByteString)
Int -> ByteString -> (Tokens ByteString, ByteString)
splitAtBL forall a. (a -> Word8 -> a) -> a -> ByteString -> a
forall b.
(b -> Token ByteString -> b) -> b -> Tokens ByteString -> b
BL.foldl' ByteString -> String
Tokens ByteString -> String
BL8.unpack (Int -> Char
chr (Int -> Char) -> (Word8 -> Int) -> Word8 -> Char
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Word8 -> Int
forall a b. (Integral a, Num b) => a -> b
fromIntegral) (10, 9) Int
o PosState ByteString
pst
reachOffsetNoLine :: Int -> PosState ByteString -> PosState ByteString
reachOffsetNoLine o :: Int
o pst :: PosState ByteString
pst =
(Int -> ByteString -> (Tokens ByteString, ByteString))
-> (forall b.
(b -> Token ByteString -> b) -> b -> Tokens ByteString -> b)
-> (Token ByteString, Token ByteString)
-> Int
-> PosState ByteString
-> PosState ByteString
forall s.
Stream s =>
(Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Token s, Token s)
-> Int
-> PosState s
-> PosState s
reachOffsetNoLine' Int -> ByteString -> (ByteString, ByteString)
Int -> ByteString -> (Tokens ByteString, ByteString)
splitAtBL forall a. (a -> Word8 -> a) -> a -> ByteString -> a
forall b.
(b -> Token ByteString -> b) -> b -> Tokens ByteString -> b
BL.foldl' (10, 9) Int
o PosState ByteString
pst
instance Stream T.Text where
type Token T.Text = Char
type Tokens T.Text = T.Text
tokenToChunk :: Proxy Text -> Token Text -> Tokens Text
tokenToChunk Proxy = Char -> Text
Token Text -> Tokens Text
T.singleton
tokensToChunk :: Proxy Text -> [Token Text] -> Tokens Text
tokensToChunk Proxy = String -> Text
[Token Text] -> Tokens Text
T.pack
chunkToTokens :: Proxy Text -> Tokens Text -> [Token Text]
chunkToTokens Proxy = Text -> String
Tokens Text -> [Token Text]
T.unpack
chunkLength :: Proxy Text -> Tokens Text -> Int
chunkLength Proxy = Text -> Int
Tokens Text -> Int
T.length
chunkEmpty :: Proxy Text -> Tokens Text -> Bool
chunkEmpty Proxy = Text -> Bool
Tokens Text -> Bool
T.null
take1_ :: Text -> Maybe (Token Text, Text)
take1_ = Text -> Maybe (Char, Text)
Text -> Maybe (Token Text, Text)
T.uncons
takeN_ :: Int -> Text -> Maybe (Tokens Text, Text)
takeN_ n :: Int
n s :: Text
s
| Int
n Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= 0 = (Text, Text) -> Maybe (Text, Text)
forall a. a -> Maybe a
Just (Text
T.empty, Text
s)
| Text -> Bool
T.null Text
s = Maybe (Tokens Text, Text)
forall a. Maybe a
Nothing
| Bool
otherwise = (Text, Text) -> Maybe (Text, Text)
forall a. a -> Maybe a
Just (Int -> Text -> (Text, Text)
T.splitAt Int
n Text
s)
takeWhile_ :: (Token Text -> Bool) -> Text -> (Tokens Text, Text)
takeWhile_ = (Char -> Bool) -> Text -> (Text, Text)
(Token Text -> Bool) -> Text -> (Tokens Text, Text)
T.span
showTokens :: Proxy Text -> NonEmpty (Token Text) -> String
showTokens Proxy = NonEmpty Char -> String
NonEmpty (Token Text) -> String
stringPretty
reachOffset :: Int -> PosState Text -> (String, PosState Text)
reachOffset o :: Int
o pst :: PosState Text
pst =
(Int -> Text -> (Tokens Text, Text))
-> (forall b. (b -> Token Text -> b) -> b -> Tokens Text -> b)
-> (Tokens Text -> String)
-> (Token Text -> Char)
-> (Token Text, Token Text)
-> Int
-> PosState Text
-> (String, PosState Text)
forall s.
Stream s =>
(Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Tokens s -> String)
-> (Token s -> Char)
-> (Token s, Token s)
-> Int
-> PosState s
-> (String, PosState s)
reachOffset' Int -> Text -> (Text, Text)
Int -> Text -> (Tokens Text, Text)
T.splitAt forall a. (a -> Char -> a) -> a -> Text -> a
forall b. (b -> Token Text -> b) -> b -> Tokens Text -> b
T.foldl' Text -> String
Tokens Text -> String
T.unpack Token Text -> Char
forall a. a -> a
id (Token Text
'\n', Token Text
'\t') Int
o PosState Text
pst
reachOffsetNoLine :: Int -> PosState Text -> PosState Text
reachOffsetNoLine o :: Int
o pst :: PosState Text
pst =
(Int -> Text -> (Tokens Text, Text))
-> (forall b. (b -> Token Text -> b) -> b -> Tokens Text -> b)
-> (Token Text, Token Text)
-> Int
-> PosState Text
-> PosState Text
forall s.
Stream s =>
(Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Token s, Token s)
-> Int
-> PosState s
-> PosState s
reachOffsetNoLine' Int -> Text -> (Text, Text)
Int -> Text -> (Tokens Text, Text)
T.splitAt forall a. (a -> Char -> a) -> a -> Text -> a
forall b. (b -> Token Text -> b) -> b -> Tokens Text -> b
T.foldl' (Token Text
'\n', Token Text
'\t') Int
o PosState Text
pst
instance Stream TL.Text where
type Token TL.Text = Char
type Tokens TL.Text = TL.Text
tokenToChunk :: Proxy Text -> Token Text -> Tokens Text
tokenToChunk Proxy = Char -> Text
Token Text -> Tokens Text
TL.singleton
tokensToChunk :: Proxy Text -> [Token Text] -> Tokens Text
tokensToChunk Proxy = String -> Text
[Token Text] -> Tokens Text
TL.pack
chunkToTokens :: Proxy Text -> Tokens Text -> [Token Text]
chunkToTokens Proxy = Text -> String
Tokens Text -> [Token Text]
TL.unpack
chunkLength :: Proxy Text -> Tokens Text -> Int
chunkLength Proxy = Int64 -> Int
forall a b. (Integral a, Num b) => a -> b
fromIntegral (Int64 -> Int) -> (Text -> Int64) -> Text -> Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Text -> Int64
TL.length
chunkEmpty :: Proxy Text -> Tokens Text -> Bool
chunkEmpty Proxy = Text -> Bool
Tokens Text -> Bool
TL.null
take1_ :: Text -> Maybe (Token Text, Text)
take1_ = Text -> Maybe (Char, Text)
Text -> Maybe (Token Text, Text)
TL.uncons
takeN_ :: Int -> Text -> Maybe (Tokens Text, Text)
takeN_ n :: Int
n s :: Text
s
| Int
n Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
<= 0 = (Text, Text) -> Maybe (Text, Text)
forall a. a -> Maybe a
Just (Text
TL.empty, Text
s)
| Text -> Bool
TL.null Text
s = Maybe (Tokens Text, Text)
forall a. Maybe a
Nothing
| Bool
otherwise = (Text, Text) -> Maybe (Text, Text)
forall a. a -> Maybe a
Just (Int64 -> Text -> (Text, Text)
TL.splitAt (Int -> Int64
forall a b. (Integral a, Num b) => a -> b
fromIntegral Int
n) Text
s)
takeWhile_ :: (Token Text -> Bool) -> Text -> (Tokens Text, Text)
takeWhile_ = (Char -> Bool) -> Text -> (Text, Text)
(Token Text -> Bool) -> Text -> (Tokens Text, Text)
TL.span
showTokens :: Proxy Text -> NonEmpty (Token Text) -> String
showTokens Proxy = NonEmpty Char -> String
NonEmpty (Token Text) -> String
stringPretty
reachOffset :: Int -> PosState Text -> (String, PosState Text)
reachOffset o :: Int
o pst :: PosState Text
pst =
(Int -> Text -> (Tokens Text, Text))
-> (forall b. (b -> Token Text -> b) -> b -> Tokens Text -> b)
-> (Tokens Text -> String)
-> (Token Text -> Char)
-> (Token Text, Token Text)
-> Int
-> PosState Text
-> (String, PosState Text)
forall s.
Stream s =>
(Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Tokens s -> String)
-> (Token s -> Char)
-> (Token s, Token s)
-> Int
-> PosState s
-> (String, PosState s)
reachOffset' Int -> Text -> (Text, Text)
Int -> Text -> (Tokens Text, Text)
splitAtTL forall a. (a -> Char -> a) -> a -> Text -> a
forall b. (b -> Token Text -> b) -> b -> Tokens Text -> b
TL.foldl' Text -> String
Tokens Text -> String
TL.unpack Token Text -> Char
forall a. a -> a
id (Token Text
'\n', Token Text
'\t') Int
o PosState Text
pst
reachOffsetNoLine :: Int -> PosState Text -> PosState Text
reachOffsetNoLine o :: Int
o pst :: PosState Text
pst =
(Int -> Text -> (Tokens Text, Text))
-> (forall b. (b -> Token Text -> b) -> b -> Tokens Text -> b)
-> (Token Text, Token Text)
-> Int
-> PosState Text
-> PosState Text
forall s.
Stream s =>
(Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Token s, Token s)
-> Int
-> PosState s
-> PosState s
reachOffsetNoLine' Int -> Text -> (Text, Text)
Int -> Text -> (Tokens Text, Text)
splitAtTL forall a. (a -> Char -> a) -> a -> Text -> a
forall b. (b -> Token Text -> b) -> b -> Tokens Text -> b
TL.foldl' (Token Text
'\n', Token Text
'\t') Int
o PosState Text
pst
data St = St SourcePos ShowS
reachOffset'
:: forall s. Stream s
=> (Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Tokens s -> String)
-> (Token s -> Char)
-> (Token s, Token s)
-> Int
-> PosState s
-> (String, PosState s)
reachOffset' :: (Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Tokens s -> String)
-> (Token s -> Char)
-> (Token s, Token s)
-> Int
-> PosState s
-> (String, PosState s)
reachOffset' splitAt' :: Int -> s -> (Tokens s, s)
splitAt'
foldl'' :: forall b. (b -> Token s -> b) -> b -> Tokens s -> b
foldl''
fromToks :: Tokens s -> String
fromToks
fromTok :: Token s -> Char
fromTok
(newlineTok :: Token s
newlineTok, tabTok :: Token s
tabTok)
o :: Int
o
PosState {..} =
( case Pos -> String -> String
expandTab Pos
pstateTabWidth
(String -> String)
-> ((Tokens s, s) -> String) -> (Tokens s, s) -> String
forall b c a. (b -> c) -> (a -> b) -> a -> c
. String -> String
addPrefix
(String -> String)
-> ((Tokens s, s) -> String) -> (Tokens s, s) -> String
forall b c a. (b -> c) -> (a -> b) -> a -> c
. String -> String
f
(String -> String)
-> ((Tokens s, s) -> String) -> (Tokens s, s) -> String
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Tokens s -> String
fromToks
(Tokens s -> String)
-> ((Tokens s, s) -> Tokens s) -> (Tokens s, s) -> String
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Tokens s, s) -> Tokens s
forall a b. (a, b) -> a
fst
((Tokens s, s) -> String) -> (Tokens s, s) -> String
forall a b. (a -> b) -> a -> b
$ (Token s -> Bool) -> s -> (Tokens s, s)
forall s. Stream s => (Token s -> Bool) -> s -> (Tokens s, s)
takeWhile_ (Token s -> Token s -> Bool
forall a. Eq a => a -> a -> Bool
/= Token s
newlineTok) s
post of
"" -> "<empty line>"
xs :: String
xs -> String
xs
, $WPosState :: forall s. s -> Int -> SourcePos -> Pos -> String -> PosState s
PosState
{ pstateInput :: s
pstateInput = s
post
, pstateOffset :: Int
pstateOffset = Int -> Int -> Int
forall a. Ord a => a -> a -> a
max Int
pstateOffset Int
o
, pstateSourcePos :: SourcePos
pstateSourcePos = SourcePos
spos
, pstateTabWidth :: Pos
pstateTabWidth = Pos
pstateTabWidth
, pstateLinePrefix :: String
pstateLinePrefix =
if Bool
sameLine
then String
pstateLinePrefix String -> String -> String
forall a. [a] -> [a] -> [a]
++ String -> String
f ""
else String -> String
f ""
}
)
where
addPrefix :: String -> String
addPrefix xs :: String
xs =
if Bool
sameLine
then String
pstateLinePrefix String -> String -> String
forall a. [a] -> [a] -> [a]
++ String
xs
else String
xs
sameLine :: Bool
sameLine = SourcePos -> Pos
sourceLine SourcePos
spos Pos -> Pos -> Bool
forall a. Eq a => a -> a -> Bool
== SourcePos -> Pos
sourceLine SourcePos
pstateSourcePos
(pre :: Tokens s
pre, post :: s
post) = Int -> s -> (Tokens s, s)
splitAt' (Int
o Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
pstateOffset) s
pstateInput
St spos :: SourcePos
spos f :: String -> String
f = (St -> Token s -> St) -> St -> Tokens s -> St
forall b. (b -> Token s -> b) -> b -> Tokens s -> b
foldl'' St -> Token s -> St
go (SourcePos -> (String -> String) -> St
St SourcePos
pstateSourcePos String -> String
forall a. a -> a
id) Tokens s
pre
go :: St -> Token s -> St
go (St apos :: SourcePos
apos g :: String -> String
g) ch :: Token s
ch =
let SourcePos n :: String
n l :: Pos
l c :: Pos
c = SourcePos
apos
c' :: Int
c' = Pos -> Int
unPos Pos
c
w :: Int
w = Pos -> Int
unPos Pos
pstateTabWidth
in if | Token s
ch Token s -> Token s -> Bool
forall a. Eq a => a -> a -> Bool
== Token s
newlineTok ->
SourcePos -> (String -> String) -> St
St (String -> Pos -> Pos -> SourcePos
SourcePos String
n (Pos
l Pos -> Pos -> Pos
forall a. Semigroup a => a -> a -> a
<> Pos
pos1) Pos
pos1)
String -> String
forall a. a -> a
id
| Token s
ch Token s -> Token s -> Bool
forall a. Eq a => a -> a -> Bool
== Token s
tabTok ->
SourcePos -> (String -> String) -> St
St (String -> Pos -> Pos -> SourcePos
SourcePos String
n Pos
l (Int -> Pos
mkPos (Int -> Pos) -> Int -> Pos
forall a b. (a -> b) -> a -> b
$ Int
c' Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
w Int -> Int -> Int
forall a. Num a => a -> a -> a
- ((Int
c' Int -> Int -> Int
forall a. Num a => a -> a -> a
- 1) Int -> Int -> Int
forall a. Integral a => a -> a -> a
`rem` Int
w)))
(String -> String
g (String -> String) -> (String -> String) -> String -> String
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Token s -> Char
fromTok Token s
ch Char -> String -> String
forall a. a -> [a] -> [a]
:))
| Bool
otherwise ->
SourcePos -> (String -> String) -> St
St (String -> Pos -> Pos -> SourcePos
SourcePos String
n Pos
l (Pos
c Pos -> Pos -> Pos
forall a. Semigroup a => a -> a -> a
<> Pos
pos1))
(String -> String
g (String -> String) -> (String -> String) -> String -> String
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Token s -> Char
fromTok Token s
ch Char -> String -> String
forall a. a -> [a] -> [a]
:))
{-# INLINE reachOffset' #-}
reachOffsetNoLine'
:: forall s. Stream s
=> (Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Token s, Token s)
-> Int
-> PosState s
-> PosState s
reachOffsetNoLine' :: (Int -> s -> (Tokens s, s))
-> (forall b. (b -> Token s -> b) -> b -> Tokens s -> b)
-> (Token s, Token s)
-> Int
-> PosState s
-> PosState s
reachOffsetNoLine' splitAt' :: Int -> s -> (Tokens s, s)
splitAt'
foldl'' :: forall b. (b -> Token s -> b) -> b -> Tokens s -> b
foldl''
(newlineTok :: Token s
newlineTok, tabTok :: Token s
tabTok)
o :: Int
o
PosState {..} =
( $WPosState :: forall s. s -> Int -> SourcePos -> Pos -> String -> PosState s
PosState
{ pstateInput :: s
pstateInput = s
post
, pstateOffset :: Int
pstateOffset = Int -> Int -> Int
forall a. Ord a => a -> a -> a
max Int
pstateOffset Int
o
, pstateSourcePos :: SourcePos
pstateSourcePos = SourcePos
spos
, pstateTabWidth :: Pos
pstateTabWidth = Pos
pstateTabWidth
, pstateLinePrefix :: String
pstateLinePrefix = String
pstateLinePrefix
}
)
where
spos :: SourcePos
spos = (SourcePos -> Token s -> SourcePos)
-> SourcePos -> Tokens s -> SourcePos
forall b. (b -> Token s -> b) -> b -> Tokens s -> b
foldl'' SourcePos -> Token s -> SourcePos
go SourcePos
pstateSourcePos Tokens s
pre
(pre :: Tokens s
pre, post :: s
post) = Int -> s -> (Tokens s, s)
splitAt' (Int
o Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
pstateOffset) s
pstateInput
go :: SourcePos -> Token s -> SourcePos
go (SourcePos n :: String
n l :: Pos
l c :: Pos
c) ch :: Token s
ch =
let c' :: Int
c' = Pos -> Int
unPos Pos
c
w :: Int
w = Pos -> Int
unPos Pos
pstateTabWidth
in if | Token s
ch Token s -> Token s -> Bool
forall a. Eq a => a -> a -> Bool
== Token s
newlineTok ->
String -> Pos -> Pos -> SourcePos
SourcePos String
n (Pos
l Pos -> Pos -> Pos
forall a. Semigroup a => a -> a -> a
<> Pos
pos1) Pos
pos1
| Token s
ch Token s -> Token s -> Bool
forall a. Eq a => a -> a -> Bool
== Token s
tabTok ->
String -> Pos -> Pos -> SourcePos
SourcePos String
n Pos
l (Int -> Pos
mkPos (Int -> Pos) -> Int -> Pos
forall a b. (a -> b) -> a -> b
$ Int
c' Int -> Int -> Int
forall a. Num a => a -> a -> a
+ Int
w Int -> Int -> Int
forall a. Num a => a -> a -> a
- ((Int
c' Int -> Int -> Int
forall a. Num a => a -> a -> a
- 1) Int -> Int -> Int
forall a. Integral a => a -> a -> a
`rem` Int
w))
| Bool
otherwise ->
String -> Pos -> Pos -> SourcePos
SourcePos String
n Pos
l (Pos
c Pos -> Pos -> Pos
forall a. Semigroup a => a -> a -> a
<> Pos
pos1)
{-# INLINE reachOffsetNoLine' #-}
splitAtBL :: Int -> BL.ByteString -> (BL.ByteString, BL.ByteString)
splitAtBL :: Int -> ByteString -> (ByteString, ByteString)
splitAtBL n :: Int
n = Int64 -> ByteString -> (ByteString, ByteString)
BL.splitAt (Int -> Int64
forall a b. (Integral a, Num b) => a -> b
fromIntegral Int
n)
{-# INLINE splitAtBL #-}
splitAtTL :: Int -> TL.Text -> (TL.Text, TL.Text)
splitAtTL :: Int -> Text -> (Text, Text)
splitAtTL n :: Int
n = Int64 -> Text -> (Text, Text)
TL.splitAt (Int -> Int64
forall a b. (Integral a, Num b) => a -> b
fromIntegral Int
n)
{-# INLINE splitAtTL #-}
stringPretty :: NonEmpty Char -> String
stringPretty :: NonEmpty Char -> String
stringPretty (x :: Char
x:|[]) = Char -> String
charPretty Char
x
stringPretty ('\r':|"\n") = "crlf newline"
stringPretty xs :: NonEmpty Char
xs = "\"" String -> String -> String
forall a. Semigroup a => a -> a -> a
<> (Char -> String) -> String -> String
forall (t :: * -> *) a b. Foldable t => (a -> [b]) -> t a -> [b]
concatMap Char -> String
f (NonEmpty Char -> String
forall a. NonEmpty a -> [a]
NE.toList NonEmpty Char
xs) String -> String -> String
forall a. Semigroup a => a -> a -> a
<> "\""
where
f :: Char -> String
f ch :: Char
ch =
case Char -> Maybe String
charPretty' Char
ch of
Nothing -> [Char
ch]
Just pretty :: String
pretty -> "<" String -> String -> String
forall a. Semigroup a => a -> a -> a
<> String
pretty String -> String -> String
forall a. Semigroup a => a -> a -> a
<> ">"
charPretty :: Char -> String
charPretty :: Char -> String
charPretty ' ' = "space"
charPretty ch :: Char
ch = String -> Maybe String -> String
forall a. a -> Maybe a -> a
fromMaybe ("'" String -> String -> String
forall a. Semigroup a => a -> a -> a
<> [Char
ch] String -> String -> String
forall a. Semigroup a => a -> a -> a
<> "'") (Char -> Maybe String
charPretty' Char
ch)
charPretty' :: Char -> Maybe String
charPretty' :: Char -> Maybe String
charPretty' = \case
'\NUL' -> String -> Maybe String
forall a. a -> Maybe a
Just "null"
'\SOH' -> String -> Maybe String
forall a. a -> Maybe a
Just "start of heading"
'\STX' -> String -> Maybe String
forall a. a -> Maybe a
Just "start of text"
'\ETX' -> String -> Maybe String
forall a. a -> Maybe a
Just "end of text"
'\EOT' -> String -> Maybe String
forall a. a -> Maybe a
Just "end of transmission"
'\ENQ' -> String -> Maybe String
forall a. a -> Maybe a
Just "enquiry"
'\ACK' -> String -> Maybe String
forall a. a -> Maybe a
Just "acknowledge"
'\BEL' -> String -> Maybe String
forall a. a -> Maybe a
Just "bell"
'\BS' -> String -> Maybe String
forall a. a -> Maybe a
Just "backspace"
'\t' -> String -> Maybe String
forall a. a -> Maybe a
Just "tab"
'\n' -> String -> Maybe String
forall a. a -> Maybe a
Just "newline"
'\v' -> String -> Maybe String
forall a. a -> Maybe a
Just "vertical tab"
'\f' -> String -> Maybe String
forall a. a -> Maybe a
Just "form feed"
'\r' -> String -> Maybe String
forall a. a -> Maybe a
Just "carriage return"
'\SO' -> String -> Maybe String
forall a. a -> Maybe a
Just "shift out"
'\SI' -> String -> Maybe String
forall a. a -> Maybe a
Just "shift in"
'\DLE' -> String -> Maybe String
forall a. a -> Maybe a
Just "data link escape"
'\DC1' -> String -> Maybe String
forall a. a -> Maybe a
Just "device control one"
'\DC2' -> String -> Maybe String
forall a. a -> Maybe a
Just "device control two"
'\DC3' -> String -> Maybe String
forall a. a -> Maybe a
Just "device control three"
'\DC4' -> String -> Maybe String
forall a. a -> Maybe a
Just "device control four"
'\NAK' -> String -> Maybe String
forall a. a -> Maybe a
Just "negative acknowledge"
'\SYN' -> String -> Maybe String
forall a. a -> Maybe a
Just "synchronous idle"
'\ETB' -> String -> Maybe String
forall a. a -> Maybe a
Just "end of transmission block"
'\CAN' -> String -> Maybe String
forall a. a -> Maybe a
Just "cancel"
'\EM' -> String -> Maybe String
forall a. a -> Maybe a
Just "end of medium"
'\SUB' -> String -> Maybe String
forall a. a -> Maybe a
Just "substitute"
'\ESC' -> String -> Maybe String
forall a. a -> Maybe a
Just "escape"
'\FS' -> String -> Maybe String
forall a. a -> Maybe a
Just "file separator"
'\GS' -> String -> Maybe String
forall a. a -> Maybe a
Just "group separator"
'\RS' -> String -> Maybe String
forall a. a -> Maybe a
Just "record separator"
'\US' -> String -> Maybe String
forall a. a -> Maybe a
Just "unit separator"
'\DEL' -> String -> Maybe String
forall a. a -> Maybe a
Just "delete"
'\160' -> String -> Maybe String
forall a. a -> Maybe a
Just "non-breaking space"
_ -> Maybe String
forall a. Maybe a
Nothing
expandTab
:: Pos
-> String
-> String
expandTab :: Pos -> String -> String
expandTab w' :: Pos
w' = Int -> String -> String
go 0
where
go :: Int -> String -> String
go 0 [] = []
go 0 ('\t':xs :: String
xs) = Int -> String -> String
go Int
w String
xs
go 0 (x :: Char
x:xs :: String
xs) = Char
x Char -> String -> String
forall a. a -> [a] -> [a]
: Int -> String -> String
go 0 String
xs
go n :: Int
n xs :: String
xs = ' ' Char -> String -> String
forall a. a -> [a] -> [a]
: Int -> String -> String
go (Int
n Int -> Int -> Int
forall a. Num a => a -> a -> a
- 1) String
xs
w :: Int
w = Pos -> Int
unPos Pos
w'