From 0c20178c83a9082f75fc7d80907ad3b093b8b774 Mon Sep 17 00:00:00 2001
From: Carlos Kelly <me@carlos.dev>
Date: Mon, 18 Dec 2023 07:19:26 -0600
Subject: [PATCH] Export `normalizeTokens` and `useTokenize` utility functions.

---
 .changeset/tasty-cats-check.md             |  5 +++
 README.md                                  | 40 ++++++++++++++++++++++
 packages/prism-react-renderer/src/index.ts |  5 ++-
 3 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 .changeset/tasty-cats-check.md

diff --git a/.changeset/tasty-cats-check.md b/.changeset/tasty-cats-check.md
new file mode 100644
index 0000000..b801e6a
--- /dev/null
+++ b/.changeset/tasty-cats-check.md
@@ -0,0 +1,5 @@
+---
+"prism-react-renderer": patch
+---
+
+Export `normalizeTokens` and `useTokenize` utility functions.
diff --git a/README.md b/README.md
index 6a38fc8..6b82f49 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,9 @@ _(If you just want to use your Prism CSS-file themes, that's also no problem)_
   - [prop getters](#prop-getters)
     - [`getLineProps`](#getlineprops)
     - [`getTokenProps`](#gettokenprops)
+- [Utility Functions](#utility-functions)
+  - [`normalizeTokens`](#normalizetokens)
+  - [`useTokenize`](#usetokenize)
 - [Theming](#theming)
 - [LICENSE](#license)
 - [Maintenance Status](#maintenance-status)
@@ -277,6 +280,43 @@ to the input.
 The `className` will always contain `.token`. This also provides full compatibility with
 your old Prism CSS-file themes.
 
+## Utility Functions
+
+### `useTokenize`
+
+> `(options: TokenizeOptions) => Token[][]`
+
+```ts
+type TokenizeOptions = {
+  prism: PrismLib
+  code: string
+  grammar?: PrismGrammar
+  language: Language
+}
+
+```
+
+This is a React hook that tokenizes code using Prism. It returns an array of tokens that can be rendered using the built-in `<Highlight />` component or your own custom component. It uses `normalizeTokens` internally to convert the tokens into a shape that can be rendered.
+
+- `prism: PrismLib`: the Prism library to use for tokenization. This can be the vendored version of Prism that is included with `prism-react-renderer` or a custom version of Prism that you have configured.
+
+- `code: string`: a string containing the code to tokenize.
+- `grammar?: PrismGrammar`: a Prism grammar object to use for tokenization. If this is omitted, the tokens will just be normalized. A grammar can be obtained from `Prism.languages` or by importing a language from `prismjs/components/`.
+- `language: Language`: the language to use for tokenization. This should be a language that Prism supports.
+
+### `normalizeTokens`
+
+> `(tokens: (PrismToken | string)[]) => Token[][]`
+
+Takes an array of Prism’s tokens and groups them by line, converting strings into tokens. Tokens can become recursive in some cases which means that their types are concatenated. Plain-string tokens however are always of type `plain`.
+
+- `PrismToken` is an internal alias for `Token` exported by `prismjs` and is defined [here](https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/prismjs/index.d.ts#L347).
+
+- `Token` is an internal object that represents a slice of tokenized content for Prism with three properties:
+  - `types: string[]`: an array of types that indicate the purpose and styling of a piece of text
+  - `content: string`: the content of the token
+  - `empty: boolean`: a flag indicating whether the token is empty or not.
+
 ## Theming
 
 The `defaultProps` you'd typically apply in a basic use-case, contain a default theme.
diff --git a/packages/prism-react-renderer/src/index.ts b/packages/prism-react-renderer/src/index.ts
index 6def32a..60b9ba6 100644
--- a/packages/prism-react-renderer/src/index.ts
+++ b/packages/prism-react-renderer/src/index.ts
@@ -3,6 +3,8 @@ import * as themes from "./themes"
 import { createElement } from "react"
 import { Highlight as InternalHighlight } from "./components/highlight"
 import { HighlightProps, PrismLib } from "./types"
+import normalizeTokens from "./utils/normalizeTokens"
+import { useTokenize } from "./components/useTokenize"
 export * from "./types"
 
 /**
@@ -17,4 +19,5 @@ const Highlight = (props: HighlightProps) =>
     code: props.code,
     language: props.language,
   })
-export { Highlight, Prism, themes }
+
+export { Highlight, Prism, themes, normalizeTokens, useTokenize }