index.d.ts 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593
  1. /**
  2. * Tokenize CSS following the {@link https://drafts.csswg.org/css-syntax/#tokenization | CSS Syntax Level 3 specification}.
  3. *
  4. * @remarks
  5. * The tokenizing and parsing tools provided by CSS Tools are designed to be low level and generic with strong ties to their respective specifications.
  6. *
  7. * Any analysis or mutation of CSS source code should be done with the least powerful tool that can accomplish the task.
  8. * For many applications it is sufficient to work with tokens.
  9. * For others you might need to use {@link https://github.com/csstools/postcss-plugins/tree/main/packages/css-parser-algorithms | @csstools/css-parser-algorithms} or a more specific parser.
  10. *
  11. * @example
  12. * Tokenize a string of CSS into an array of tokens:
  13. * ```js
  14. * import { tokenize } from '@csstools/css-tokenizer';
  15. *
  16. * const myCSS = `@media only screen and (min-width: 768rem) {
  17. * .foo {
  18. * content: 'Some content!' !important;
  19. * }
  20. * }
  21. * `;
  22. *
  23. * const tokens = tokenize({
  24. * css: myCSS,
  25. * });
  26. *
  27. * console.log(tokens);
  28. * ```
  29. *
  30. * @packageDocumentation
  31. */
  32. /**
  33. * Deep clone a list of tokens.
  34. * Useful for mutations without altering the original list.
  35. */
  36. export declare function cloneTokens(tokens: Array<CSSToken>): Array<CSSToken>;
  37. /**
  38. * The union of all possible CSS tokens
  39. */
  40. export declare type CSSToken = TokenAtKeyword | TokenBadString | TokenBadURL | TokenCDC | TokenCDO | TokenColon | TokenComma | TokenComment | TokenDelim | TokenDimension | TokenEOF | TokenFunction | TokenHash | TokenIdent | TokenNumber | TokenPercentage | TokenSemicolon | TokenString | TokenURL | TokenWhitespace | TokenOpenParen | TokenCloseParen | TokenOpenSquare | TokenCloseSquare | TokenOpenCurly | TokenCloseCurly | TokenUnicodeRange;
  41. /**
  42. * The type of hash token
  43. */
  44. export declare enum HashType {
  45. /**
  46. * The hash token did not start with an ident sequence (e.g. `#-2`)
  47. */
  48. Unrestricted = "unrestricted",
  49. /**
  50. * The hash token started with an ident sequence (e.g. `#foo`)
  51. * Only hash tokens with the "id" type are valid ID selectors.
  52. */
  53. ID = "id"
  54. }
  55. /**
  56. * Assert that a given value has the general structure of a CSS token:
  57. * 1. is an array.
  58. * 2. has at least four items.
  59. * 3. has a known token type.
  60. * 4. has a string representation.
  61. * 5. has a start position.
  62. * 6. has an end position.
  63. */
  64. export declare function isToken(x: any): x is CSSToken;
  65. export declare function isTokenAtKeyword(x?: CSSToken | null): x is TokenAtKeyword;
  66. export declare function isTokenBadString(x?: CSSToken | null): x is TokenBadString;
  67. export declare function isTokenBadURL(x?: CSSToken | null): x is TokenBadURL;
  68. export declare function isTokenCDC(x?: CSSToken | null): x is TokenCDC;
  69. export declare function isTokenCDO(x?: CSSToken | null): x is TokenCDO;
  70. export declare function isTokenCloseCurly(x?: CSSToken | null): x is TokenCloseCurly;
  71. export declare function isTokenCloseParen(x?: CSSToken | null): x is TokenCloseParen;
  72. export declare function isTokenCloseSquare(x?: CSSToken | null): x is TokenCloseSquare;
  73. export declare function isTokenColon(x?: CSSToken | null): x is TokenColon;
  74. export declare function isTokenComma(x?: CSSToken | null): x is TokenComma;
  75. export declare function isTokenComment(x?: CSSToken | null): x is TokenComment;
  76. export declare function isTokenDelim(x?: CSSToken | null): x is TokenDelim;
  77. export declare function isTokenDimension(x?: CSSToken | null): x is TokenDimension;
  78. export declare function isTokenEOF(x?: CSSToken | null): x is TokenEOF;
  79. export declare function isTokenFunction(x?: CSSToken | null): x is TokenFunction;
  80. export declare function isTokenHash(x?: CSSToken | null): x is TokenHash;
  81. export declare function isTokenIdent(x?: CSSToken | null): x is TokenIdent;
  82. export declare function isTokenNumber(x?: CSSToken | null): x is TokenNumber;
  83. /**
  84. * Assert that a token is a numeric token
  85. */
  86. export declare function isTokenNumeric(x?: CSSToken | null): x is NumericToken;
  87. export declare function isTokenOpenCurly(x?: CSSToken | null): x is TokenOpenCurly;
  88. export declare function isTokenOpenParen(x?: CSSToken | null): x is TokenOpenParen;
  89. export declare function isTokenOpenSquare(x?: CSSToken | null): x is TokenOpenSquare;
  90. export declare function isTokenPercentage(x?: CSSToken | null): x is TokenPercentage;
  91. export declare function isTokenSemicolon(x?: CSSToken | null): x is TokenSemicolon;
  92. export declare function isTokenString(x?: CSSToken | null): x is TokenString;
  93. export declare function isTokenUnicodeRange(x?: CSSToken | null): x is TokenUnicodeRange;
  94. export declare function isTokenURL(x?: CSSToken | null): x is TokenURL;
  95. export declare function isTokenWhitespace(x?: CSSToken | null): x is TokenWhitespace;
  96. /**
  97. * Assert that a token is a whitespace or comment token
  98. */
  99. export declare function isTokenWhiteSpaceOrComment(x?: CSSToken | null): x is TokenWhitespace | TokenComment;
  100. /**
  101. * Get the mirror variant of a given token
  102. *
  103. * @example
  104. *
  105. * ```js
  106. * const input = [TokenType.OpenParen, '(', 0, 1, undefined];
  107. * const output = mirrorVariant(input);
  108. *
  109. * console.log(output); // [TokenType.CloseParen, ')', -1, -1, undefined]
  110. * ```
  111. */
  112. export declare function mirrorVariant(token: CSSToken): CSSToken | null;
  113. /**
  114. * Get the mirror variant type of a given token type
  115. *
  116. * @example
  117. *
  118. * ```js
  119. * const input = TokenType.OpenParen;
  120. * const output = mirrorVariantType(input);
  121. *
  122. * console.log(output); // TokenType.CloseParen
  123. * ```
  124. */
  125. export declare function mirrorVariantType(type: TokenType): TokenType | null;
  126. /**
  127. * Set the ident value and update the string representation.
  128. * This handles escaping.
  129. */
  130. export declare function mutateIdent(ident: TokenIdent, newValue: string): void;
  131. /**
  132. * Set the unit and update the string representation.
  133. * This handles escaping.
  134. */
  135. export declare function mutateUnit(ident: TokenDimension, newUnit: string): void;
  136. /**
  137. * The type of number token
  138. * Either `integer` or `number`
  139. */
  140. export declare enum NumberType {
  141. Integer = "integer",
  142. Number = "number"
  143. }
  144. /**
  145. * The union of all possible CSS tokens that represent a numeric value
  146. */
  147. export declare type NumericToken = TokenDimension | TokenNumber | TokenPercentage;
  148. /**
  149. * The CSS Tokenizer is forgiving and will never throw on invalid input.
  150. * Any errors are reported through the `onParseError` callback.
  151. */
  152. export declare class ParseError extends Error {
  153. /** The index of the start character of the current token. */
  154. sourceStart: number;
  155. /** The index of the end character of the current token. */
  156. sourceEnd: number;
  157. /** The parser steps that preceded the error. */
  158. parserState: Array<string>;
  159. constructor(message: string, sourceStart: number, sourceEnd: number, parserState: Array<string>);
  160. }
  161. export declare const ParseErrorMessage: {
  162. UnexpectedNewLineInString: string;
  163. UnexpectedEOFInString: string;
  164. UnexpectedEOFInComment: string;
  165. UnexpectedEOFInURL: string;
  166. UnexpectedEOFInEscapedCodePoint: string;
  167. UnexpectedCharacterInURL: string;
  168. InvalidEscapeSequenceInURL: string;
  169. InvalidEscapeSequenceAfterBackslash: string;
  170. };
  171. export declare class ParseErrorWithToken extends ParseError {
  172. /** The associated token. */
  173. token: CSSToken;
  174. constructor(message: string, sourceStart: number, sourceEnd: number, parserState: Array<string>, token: CSSToken);
  175. }
  176. /**
  177. * Concatenate the string representation of a list of tokens.
  178. * This is not a proper serializer that will handle escaping and whitespace.
  179. * It only produces valid CSS for a token list that is also valid.
  180. */
  181. export declare function stringify(...tokens: Array<CSSToken>): string;
  182. /**
  183. * The CSS Token interface
  184. *
  185. * @remarks
  186. * CSS Tokens are fully typed and have a strict structure.
  187. * This makes it easier to iterate and analyze a token stream.
  188. *
  189. * The string representation and the parsed value are stored separately for many token types.
  190. * It is always assumed that the string representation will be used when stringifying, while the parsed value should be used when analyzing tokens.
  191. */
  192. export declare interface Token<T extends TokenType, U> extends Array<T | string | number | U> {
  193. /**
  194. * The type of token
  195. */
  196. 0: T;
  197. /**
  198. * The token representation
  199. *
  200. * @remarks
  201. * This field will be used when stringifying the token.
  202. * Any stored value is assumed to be valid CSS.
  203. *
  204. * You should never use this field when analyzing the token when there is a parsed value available.
  205. * But you must store mutated values here.
  206. */
  207. 1: string;
  208. /**
  209. * Start position of representation
  210. */
  211. 2: number;
  212. /**
  213. * End position of representation
  214. */
  215. 3: number;
  216. /**
  217. * Extra data
  218. *
  219. * @remarks
  220. * This holds the parsed value of each token.
  221. * These values are unescaped, unquoted, converted to numbers, etc.
  222. *
  223. * You should always use this field when analyzing the token.
  224. * But you must not assume that mutating only this field will have any effect.
  225. */
  226. 4: U;
  227. }
  228. export declare interface TokenAtKeyword extends Token<TokenType.AtKeyword, {
  229. /**
  230. * The unescaped at-keyword name without the leading `@`.
  231. */
  232. value: string;
  233. }> {
  234. }
  235. export declare interface TokenBadString extends Token<TokenType.BadString, undefined> {
  236. }
  237. export declare interface TokenBadURL extends Token<TokenType.BadURL, undefined> {
  238. }
  239. export declare interface TokenCDC extends Token<TokenType.CDC, undefined> {
  240. }
  241. export declare interface TokenCDO extends Token<TokenType.CDO, undefined> {
  242. }
  243. export declare interface TokenCloseCurly extends Token<TokenType.CloseCurly, undefined> {
  244. }
  245. export declare interface TokenCloseParen extends Token<TokenType.CloseParen, undefined> {
  246. }
  247. export declare interface TokenCloseSquare extends Token<TokenType.CloseSquare, undefined> {
  248. }
  249. export declare interface TokenColon extends Token<TokenType.Colon, undefined> {
  250. }
  251. export declare interface TokenComma extends Token<TokenType.Comma, undefined> {
  252. }
  253. export declare interface TokenComment extends Token<TokenType.Comment, undefined> {
  254. }
  255. export declare interface TokenDelim extends Token<TokenType.Delim, {
  256. /**
  257. * The delim character.
  258. */
  259. value: string;
  260. }> {
  261. }
  262. export declare interface TokenDimension extends Token<TokenType.Dimension, {
  263. /**
  264. * The numeric value.
  265. */
  266. value: number;
  267. /**
  268. * The unescaped unit name.
  269. */
  270. unit: string;
  271. /**
  272. * `integer` or `number`
  273. */
  274. type: NumberType;
  275. /**
  276. * The sign character as it appeared in the source.
  277. * This is only useful if you need to determine if a value was written as "2px" or "+2px".
  278. */
  279. signCharacter?: '+' | '-';
  280. }> {
  281. }
  282. export declare interface TokenEOF extends Token<TokenType.EOF, undefined> {
  283. }
  284. export declare interface TokenFunction extends Token<TokenType.Function, {
  285. /**
  286. * The unescaped function name without the trailing `(`.
  287. */
  288. value: string;
  289. }> {
  290. }
  291. export declare interface TokenHash extends Token<TokenType.Hash, {
  292. /**
  293. * The unescaped hash value without the leading `#`.
  294. */
  295. value: string;
  296. /**
  297. * The hash type.
  298. */
  299. type: HashType;
  300. }> {
  301. }
  302. export declare interface TokenIdent extends Token<TokenType.Ident, {
  303. /**
  304. * The unescaped ident value.
  305. */
  306. value: string;
  307. }> {
  308. }
  309. /**
  310. * Tokenize a CSS string into a list of tokens.
  311. */
  312. export declare function tokenize(input: {
  313. css: {
  314. valueOf(): string;
  315. };
  316. unicodeRangesAllowed?: boolean;
  317. }, options?: {
  318. onParseError?: (error: ParseError) => void;
  319. }): Array<CSSToken>;
  320. /**
  321. * Create a tokenizer for a CSS string.
  322. */
  323. export declare function tokenizer(input: {
  324. css: {
  325. valueOf(): string;
  326. };
  327. unicodeRangesAllowed?: boolean;
  328. }, options?: {
  329. onParseError?: (error: ParseError) => void;
  330. }): {
  331. nextToken: () => CSSToken;
  332. endOfFile: () => boolean;
  333. };
  334. export declare interface TokenNumber extends Token<TokenType.Number, {
  335. /**
  336. * The numeric value.
  337. */
  338. value: number;
  339. /**
  340. * `integer` or `number`
  341. */
  342. type: NumberType;
  343. /**
  344. * The sign character as it appeared in the source.
  345. * This is only useful if you need to determine if a value was written as "2" or "+2".
  346. */
  347. signCharacter?: '+' | '-';
  348. }> {
  349. }
  350. export declare interface TokenOpenCurly extends Token<TokenType.OpenCurly, undefined> {
  351. }
  352. export declare interface TokenOpenParen extends Token<TokenType.OpenParen, undefined> {
  353. }
  354. export declare interface TokenOpenSquare extends Token<TokenType.OpenSquare, undefined> {
  355. }
  356. export declare interface TokenPercentage extends Token<TokenType.Percentage, {
  357. /**
  358. * The numeric value.
  359. */
  360. value: number;
  361. /**
  362. * The sign character as it appeared in the source.
  363. * This is only useful if you need to determine if a value was written as "2%" or "+2%".
  364. */
  365. signCharacter?: '+' | '-';
  366. }> {
  367. }
  368. export declare interface TokenSemicolon extends Token<TokenType.Semicolon, undefined> {
  369. }
  370. export declare interface TokenString extends Token<TokenType.String, {
  371. /**
  372. * The unescaped string value without the leading and trailing quotes.
  373. */
  374. value: string;
  375. }> {
  376. }
  377. /**
  378. * All possible CSS token types
  379. */
  380. export declare enum TokenType {
  381. /**
  382. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#comment-diagram}
  383. */
  384. Comment = "comment",
  385. /**
  386. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-at-keyword-token}
  387. */
  388. AtKeyword = "at-keyword-token",
  389. /**
  390. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-bad-string-token}
  391. */
  392. BadString = "bad-string-token",
  393. /**
  394. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-bad-url-token}
  395. */
  396. BadURL = "bad-url-token",
  397. /**
  398. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-cdc-token}
  399. */
  400. CDC = "CDC-token",
  401. /**
  402. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-cdo-token}
  403. */
  404. CDO = "CDO-token",
  405. /**
  406. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-colon-token}
  407. */
  408. Colon = "colon-token",
  409. /**
  410. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-comma-token}
  411. */
  412. Comma = "comma-token",
  413. /**
  414. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-delim-token}
  415. */
  416. Delim = "delim-token",
  417. /**
  418. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-dimension-token}
  419. */
  420. Dimension = "dimension-token",
  421. /**
  422. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-eof-token}
  423. */
  424. EOF = "EOF-token",
  425. /**
  426. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-function-token}
  427. */
  428. Function = "function-token",
  429. /**
  430. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-hash-token}
  431. */
  432. Hash = "hash-token",
  433. /**
  434. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-ident-token}
  435. */
  436. Ident = "ident-token",
  437. /**
  438. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-percentage-token}
  439. */
  440. Number = "number-token",
  441. /**
  442. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-percentage-token}
  443. */
  444. Percentage = "percentage-token",
  445. /**
  446. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-semicolon-token}
  447. */
  448. Semicolon = "semicolon-token",
  449. /**
  450. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-string-token}
  451. */
  452. String = "string-token",
  453. /**
  454. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-url-token}
  455. */
  456. URL = "url-token",
  457. /**
  458. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-whitespace-token}
  459. */
  460. Whitespace = "whitespace-token",
  461. /**
  462. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-open-paren}
  463. */
  464. OpenParen = "(-token",
  465. /**
  466. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-close-paren}
  467. */
  468. CloseParen = ")-token",
  469. /**
  470. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-open-square}
  471. */
  472. OpenSquare = "[-token",
  473. /**
  474. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-close-square}
  475. */
  476. CloseSquare = "]-token",
  477. /**
  478. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-open-curly}
  479. */
  480. OpenCurly = "{-token",
  481. /**
  482. * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-close-curly}
  483. */
  484. CloseCurly = "}-token",
  485. /**
  486. * Only appears in the token stream when the `unicodeRangesAllowed` option is set to true.
  487. *
  488. * @example
  489. * ```js
  490. * import { tokenize } from '@csstools/css-tokenizer';
  491. *
  492. * const tokens = tokenize({
  493. * css: `U+0025-00FF, U+4??`,
  494. * unicodeRangesAllowed: true,
  495. * });
  496. *
  497. * console.log(tokens);
  498. * ```
  499. *
  500. * @see {@link https://drafts.csswg.org/css-syntax/#typedef-unicode-range-token}
  501. */
  502. UnicodeRange = "unicode-range-token"
  503. }
  504. export declare interface TokenUnicodeRange extends Token<TokenType.UnicodeRange, {
  505. startOfRange: number;
  506. endOfRange: number;
  507. }> {
  508. }
  509. export declare interface TokenURL extends Token<TokenType.URL, {
  510. /**
  511. * The unescaped URL value without the leading `url(` and trailing `)`.
  512. */
  513. value: string;
  514. }> {
  515. }
  516. export declare interface TokenWhitespace extends Token<TokenType.Whitespace, undefined> {
  517. }
  518. export { }