Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | 51x 586x 51x 63x 251x 49x 202x 8x 194x 23x 171x 22x 149x 5x 144x 70x 74x 58x 16x 16x 251x 12x 1x 11x 11x 4x 246x 51x | import {
isFunctionSymbol,
isNumeric,
toNumber,
isOperatorSymbol,
isVariable,
isConstant,
} from "./helpers";
import { Token } from "./types";
const identity = <T>(i: T): T => i;
const tokenizer = (input: string): Token[] => {
return (
input
.replace(/\s+/g, "")
.split(
/(@[a-f0-9-]{36}@|@engine::[^@]+@|<<|>>|==|!=|>=|>|<=|<|&&|\|\||[+\-*/^%&|~!@(),])/,
)
.filter(identity)
.map((token): Token => {
if (isNumeric(token)) {
return {
type: "VAL",
value: toNumber(token),
};
}
if (isFunctionSymbol(token)) {
return {
type: "FUN",
function: token,
};
}
if (token === "(") {
return {
type: "LBRACE",
};
}
if (token === ")") {
return {
type: "RBRACE",
};
}
if (token === ",") {
return {
type: "SEPERATOR",
};
}
if (isOperatorSymbol(token)) {
return {
type: "OP",
operator: token,
};
}
if (isVariable(token)) {
return {
type: "VAR",
symbol: token,
};
}
if (isConstant(token)) {
return {
type: "CONST",
symbol: token.replaceAll(/@/g, ""),
};
}
throw new Error(`Unexpected token ${token}`);
})
.filter(identity) as Token[]
)
.map((token, i, tokens): Token[] => {
// Handle unary negation
if (token.type === "OP" && token.operator === "-") {
if (i === 0) {
return [
{
type: "OP",
operator: "neg",
},
];
}
const previous = tokens[i - 1];
if (
previous.type === "LBRACE" ||
previous.type === "SEPERATOR" ||
(previous.type === "OP" && isOperatorSymbol(previous.operator))
) {
return [
{
type: "OP",
operator: "neg",
},
];
}
}
return [token];
})
.flat();
};
export default tokenizer;
|