mdream Copy package name

Copy code

# npm
npm install mdream

# pnpm
pnpm add mdream

# yarn
yarn add mdream

Copy code

pnpm add @mdream/js

Copy code

// next.config.js
const nextConfig = {
  serverExternalPackages: ['mdream'],
}

Copy code

externals: ['mdream']

Copy code

- import { htmlToMarkdown } from 'mdream'
- const markdown = (await htmlToMarkdown(html)).markdown
+ import { htmlToMarkdown } from 'mdream/browser'
+ const markdown = await htmlToMarkdown(html)

Copy code

- const markdown = window.mdream.htmlToMarkdown(html).markdown
+ const markdown = await window.mdream.htmlToMarkdown(html)

Copy code

// md.worker.ts
import { htmlToMarkdown } from 'mdream/browser'

addEventListener('message', async (event: MessageEvent<string>) => {
  postMessage(await htmlToMarkdown(event.data, { minimal: true }))
})

Copy code

// main.ts
const worker = new Worker(new URL('./md.worker.ts', import.meta.url), { type: 'module' })
worker.onmessage = (event: MessageEvent<string>) => console.log(event.data)
worker.postMessage('<h1>Hello</h1>')

Copy code

import { htmlToMarkdown } from 'mdream'

htmlToMarkdown(html, { minimal: true, filter: false })

htmlToMarkdown(html, {
  frontmatter: true,
  isolateMain: true,
  tailwind: true,
  clean: true,
  filter: { exclude: ['footer'] },
})

Copy code

import type { MdreamOptions } from 'mdream'

declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string

// `mdream/browser` and the CDN script return Promise<string> instead.

Copy code

import { htmlToMarkdown } from '@mdream/js'
import { htmlToSafeHtml } from '@mdream/js/html'
import { htmlToText } from '@mdream/js/text'

Copy code

import type { MdreamOptions } from '@mdream/js'

declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string
declare function htmlToText(html: string, options?: Partial<MdreamOptions>): string
declare function htmlToSafeHtml(html: string, options?: Partial<MdreamOptions>): string

Copy code

import { htmlToMarkdown } from 'mdream'

const markdown = htmlToMarkdown('<h1>Hello World</h1><p>Some content.</p>')
// # Hello World
//
// Some content.

const text = htmlToMarkdown('<h1>Hello <strong>World</strong></h1>', {
  format: 'text',
})
// Hello World

const html = htmlToMarkdown('<h1>Hello <strong>World</strong></h1>', {
  format: 'html',
})
// <h1 id="hello-world">Hello <strong>World</strong></h1>

Copy code

import type { MdreamOptions } from 'mdream'

declare function streamHtmlToMarkdown(
  htmlStream: ReadableStream<Uint8Array | string> | null,
  options?: Partial<MdreamOptions>,
): AsyncIterable<string>

Copy code

import { streamHtmlToMarkdown } from 'mdream'

const response = await fetch('https://example.com')
const stream = response.body

for await (const chunk of streamHtmlToMarkdown(stream, {
  origin: 'https://example.com',
})) {
  process.stdout.write(chunk)
}

Copy code

// JavaScript engine for explicit plugins and smaller bundles
import { htmlToMarkdown } from '@mdream/js'

// Rust NAPI engine (auto-selected in Node.js)
import { htmlToMarkdown as htmlToMarkdownRust } from 'mdream'

Copy code

interface MdreamOptions {
  /** Base URL for resolving relative links and images. */
  origin?: string

  /**
   * Enable minimal preset (frontmatter, isolateMain, tailwind, filter).
   * Default: false
   */
  minimal?: boolean

  /**
   * Post-processing cleanup. Pass `true` for all cleanup, or an object for specific features.
   * Enabled by default when `minimal` is true.
   */
  clean?: boolean | CleanOptions

  /**
   * Extract frontmatter from HTML <head>.
   * - `true`: enable with defaults
   * - `(fm) => void`: enable and receive structured data via callback
   * - `FrontmatterConfig`: enable with config and optional callback
   */
  frontmatter?: boolean | ((frontmatter: Record<string, string>) => void) | FrontmatterConfig

  /** Isolate main content area. Default when minimal: true */
  isolateMain?: boolean

  /** Convert Tailwind utility classes to Markdown. Default when minimal: true */
  tailwind?: boolean

  /** Filter elements by CSS selectors. Default when minimal: excludes form, nav, footer, etc. */
  filter?: false | { include?: string[], exclude?: string[], processChildren?: boolean }

  /** Extract elements matching CSS selectors during conversion. */
  extraction?: Record<string, (element: ExtractedElement) => void>

  /** Override tag rendering behavior. String values act as aliases. */
  tagOverrides?: Record<string, TagOverride | string>

  /**
   * Hard-wrap prose at this many characters, breaking on word boundaries.
   * Applied inline during conversion (zero-cost when unset). Code blocks,
   * tables, and headings are never wrapped. `0` (or unset) disables wrapping.
   */
  wrapWidth?: number

  /** Output Markdown, plain text, or HTML. Default: 'markdown' */
  format?: 'markdown' | 'text' | 'html'
}

Copy code

interface MdreamOptions extends EngineOptions {
  /** Explicit plugins, applied in array order. */
  plugins?: Plugin[]
}

interface EngineOptions {
  origin?: string
  /** Cleanup rules from clean() in @mdream/js/clean */
  clean?: Cleaner
  tagOverrides?: Record<string, TagOverride | string>

  /**
   * Hard-wrap prose at this many characters, breaking on word boundaries.
   * Code blocks, tables, and headings are never wrapped. `0` (or unset)
   * disables wrapping.
   */
  wrapWidth?: number

}

Copy code

interface CleanOptions {
  /** Strip tracking query parameters (utm_*, fbclid, gclid, etc.) from URLs */
  urls?: boolean
  /** Strip fragment-only links that don't match any heading in the output */
  fragments?: boolean
  /** Strip links with meaningless hrefs (#, javascript:void(0)) to plain text */
  emptyLinks?: boolean
  /** Strip links where text equals URL: [https://x.com](https://x.com) becomes https://x.com */
  redundantLinks?: boolean
  /** Strip self-referencing heading anchors: ## [Title](#title) becomes ## Title */
  selfLinkHeadings?: boolean
  /** Strip images with no alt text (decorative/tracking pixels) */
  emptyImages?: boolean
  /** Drop links that produce no visible text: [](url) is removed entirely */
  emptyLinkText?: boolean
}

Copy code

import { htmlToMarkdown } from 'mdream'

const markdown = htmlToMarkdown(html, {
  clean: {
    urls: true,
    emptyLinks: true,
    emptyImages: true,
  },
})

Copy code

interface FrontmatterConfig {
  /** Additional static fields to include in frontmatter */
  additionalFields?: Record<string, string>
  /**
   * Meta tag names to extract beyond the defaults.
   * Defaults: description, keywords, author, date,
   * og:title, og:description, twitter:title, twitter:description
   */
  metaFields?: string[]
  /** Callback to receive structured frontmatter data after conversion */
  onExtract?: (frontmatter: Record<string, string>) => void
}

Copy code

interface TagOverride {
  /** Markdown string to insert when entering this tag */
  enter?: string
  /** Markdown string to insert when exiting this tag */
  exit?: string
  /** Spacing: [newlines before, newlines after] */
  spacing?: number[]
  /** Whether this tag should be treated as inline */
  isInline?: boolean
  /** Whether this tag is self-closing */
  isSelfClosing?: boolean
  /** Whether whitespace inside this tag should be collapsed */
  collapsesInnerWhiteSpace?: boolean
  /** Alias this tag to another tag's handler */
  alias?: string
}

Copy code

import { htmlToMarkdown } from 'mdream'

const markdown = htmlToMarkdown(html, {
  tagOverrides: {
    // Treat <x-heading> like <h2>
    'x-heading': 'h2',
    // Custom rendering for <callout>
    'callout': {
      enter: '> **Note:** ',
      exit: '',
      spacing: [2, 2],
    },
  },
})

Copy code

import { htmlToMarkdown } from 'mdream'

htmlToMarkdown(html, {
  tagOverrides: {
    em: { enter: '_', exit: '_', isInline: true }, // *x*  →  _x_
    strong: { enter: '__', exit: '__', isInline: true }, // **x** →  __x__
    hr: { enter: '* * *', exit: '' }, // ---  →  * * *
  },
})

Copy code

interface FilterOptions {
  /** CSS selectors or tag names for elements to include (all others excluded) */
  include?: string[]
  /** CSS selectors or tag names for elements to exclude */
  exclude?: string[]
  /** Whether to also process children of matched elements. Default: true */
  processChildren?: boolean
}

Copy code

import { htmlToMarkdown } from 'mdream'

// Excludes h1 as well as form, nav, footer, and the rest
htmlToMarkdown(html, { minimal: true, filter: { exclude: ['h1'] } })

// Keeps forms and nav
htmlToMarkdown(html, { minimal: true, filter: false })

Copy code

import { htmlToMarkdown } from 'mdream'

const markdown = htmlToMarkdown(html, {
  origin: 'https://example.com',
  minimal: true,
})

Copy code

import { htmlToMarkdown } from '@mdream/js'
import { withMinimalPreset } from '@mdream/js/preset/minimal'

const markdown = htmlToMarkdown(html, withMinimalPreset({
  origin: 'https://example.com',
}))

Copy code

import { htmlToMarkdown } from '@mdream/js'
import { filterPlugin } from '@mdream/js/plugins'
import { withMinimalPreset } from '@mdream/js/preset/minimal'

const markdown = htmlToMarkdown(html, withMinimalPreset({
  plugins: [filterPlugin({ exclude: ['.cookie-banner'] }), myPlugin],
}))

Copy code

import { htmlToMarkdown } from 'mdream'

// Enable with defaults
htmlToMarkdown(html, { frontmatter: true })

// With callback to receive structured data
htmlToMarkdown(html, {
  frontmatter: (fm) => {
    console.log(fm.title)
    console.log(fm.description)
  },
})

// With full config
htmlToMarkdown(html, {
  frontmatter: {
    additionalFields: { source: 'https://example.com' },
    metaFields: ['robots', 'viewport'],
    onExtract: fm => console.log(fm),
  },
})

Copy code

---
title: My Page Title
meta:
  description: A page description
  'og:title': My Page Title
---

Copy code

import { htmlToMarkdown } from 'mdream'

htmlToMarkdown(html, { isolateMain: true })

Copy code

import { htmlToMarkdown } from 'mdream'

htmlToMarkdown(html, { tailwind: true })

Copy code

import { htmlToMarkdown } from 'mdream'

// Exclude navigation, sidebar, footer
htmlToMarkdown(html, {
  filter: {
    exclude: ['nav', '#sidebar', '.footer', 'aside'],
  },
})

// Include only specific elements
htmlToMarkdown(html, {
  filter: {
    include: ['article', 'main'],
  },
})

Copy code

import { htmlToMarkdown } from '@mdream/js'
import { filterPlugin } from '@mdream/js/plugins'

htmlToMarkdown(html, {
  plugins: [filterPlugin({ exclude: ['nav', 'footer'] })],
})

Copy code

import { htmlToMarkdown } from 'mdream'

htmlToMarkdown(html, {
  extraction: {
    'h2': (el) => {
      console.log('Heading:', el.textContent)
    },
    'img[alt]': (el) => {
      console.log('Image:', el.attributes.src, el.attributes.alt)
    },
    'a[href]': (el) => {
      console.log('Link:', el.textContent, el.attributes.href)
    },
  },
})

Copy code

interface ExtractedElement {
  selector: string
  tagName: string
  textContent: string
  attributes: Record<string, string>
}

Copy code

import { htmlToMarkdown } from '@mdream/js'
import { createPlugin } from '@mdream/js/plugins'

const myPlugin = createPlugin({
  onNodeEnter(node) {
    if (node.name === 'h1')
      return '** '
  },
  processTextNode(textNode) {
    if (textNode.parent?.attributes?.id === 'highlight') {
      return { content: `**${textNode.value}**`, skip: false }
    }
  },
})

const markdown = htmlToMarkdown(html, { plugins: [myPlugin] })

Copy code

interface PluginState {
  /** Options passed to the converter. */
  readonly options: Readonly<EngineOptions>
  /** Output that the running converter writes. */
  readonly outputFormat: 'markdown' | 'text' | 'html'
  /** Nesting depth of the current node. It is 0 before the first node. */
  readonly depth: number
}

interface TransformPlugin {
  /**
   * Called before any node processing. Return { skip: true } to skip the node.
   */
  beforeNodeProcess?: (
    event: NodeEvent,
    state: PluginState,
  ) => undefined | void | { skip: boolean }

  /**
   * Called when entering an element node.
   * Return a string to prepend to the output.
   */
  onNodeEnter?: (
    node: ElementNode,
    state: PluginState,
  ) => string | undefined | void

  /**
   * Called when exiting an element node.
   * Return a string to append to the output.
   */
  onNodeExit?: (
    node: ElementNode,
    state: PluginState,
  ) => string | undefined | void

  /**
   * Called to process element attributes (e.g., extracting Tailwind classes).
   */
  processAttributes?: (
    node: ElementNode,
    state: PluginState,
  ) => void

  /**
   * Called for each text node. Return { content, skip } to transform text.
   * Return undefined for no transformation.
   */
  processTextNode?: (
    node: TextNode,
    state: PluginState,
  ) => { content: string, skip: boolean } | undefined

  /**
   * Called once after the whole document is converted, including for streams.
   */
  onDocumentEnd?: (state: PluginState) => void
}

Copy code

import type { ElementNode } from '@mdream/js'
import { ELEMENT_NODE } from '@mdream/js'
import { createPlugin } from '@mdream/js/plugins'

const plugin = createPlugin({
  beforeNodeProcess({ node }) {
    // Skip all div elements with class "ad"
    if (node.type === ELEMENT_NODE) {
      const element = node as ElementNode
      if (element.name === 'div' && element.attributes.class?.split(/\s+/).includes('ad')) {
        return { skip: true }
      }
    }
  },
})

Copy code

import {
  createPlugin,
  extractionPlugin,
  filterPlugin,
  frontmatterPlugin,
  isolateMainPlugin,
  tailwindPlugin,
} from '@mdream/js/plugins'

Copy code

import { htmlToMarkdownSplitChunks } from '@mdream/js/splitter'

const html = `
  <h1>Documentation</h1>
  <h2>Installation</h2>
  <p>Install via npm...</p>
  <h2>Usage</h2>
  <p>Use it like this...</p>
`

const chunks = htmlToMarkdownSplitChunks(html, {
  headersToSplitOn: [2],
  chunkSize: 1000,
  chunkOverlap: 200,
  stripHeaders: true,
})

chunks.forEach((chunk) => {
  console.log(chunk.content)
  console.log(chunk.metadata.headers) // { h1: "Documentation", h2: "Installation" }
  console.log(chunk.metadata.code) // Language if chunk contains code
  console.log(chunk.metadata.loc) // { lines: { from: 1, to: 5 } }
})

Copy code

import { htmlToMarkdownSplitChunksStream } from '@mdream/js/splitter'

for (const chunk of htmlToMarkdownSplitChunksStream(html, options)) {
  await processChunk(chunk)

  // Early termination supported
  if (foundTarget)
    break
}

Copy code

interface SplitterOptions {
  // --- Structural splitting ---

  /**
   * Heading levels that start a new chunk, from 1 (<h1>) to 6 (<h6>).
   * Another value throws a TypeError. Default: [2, 3, 4, 5, 6]
   */
  headersToSplitOn?: (1 | 2 | 3 | 4 | 5 | 6)[]

  // --- Size-based splitting ---

  /** Target chunk size in characters. Complete code fences can exceed it. Default: 1000 */
  chunkSize?: number

  /** Overlap between chunks for context preservation. Default: 200 */
  chunkOverlap?: number

  /**
   * Custom length function (e.g., a token counter for LLM applications).
   * Default: (text) => text.length
   */
  lengthFunction?: (text: string) => number

  // --- Output formatting ---

  /** Remove headers from chunk content. Default: true */
  stripHeaders?: boolean

  /** Split into individual lines. Default: false */
  returnEachLine?: boolean

  /** The splitter drops whitespace between chunks for either value. */
  keepSeparator?: boolean

  // --- Standard options ---

  /** Base URL for resolving relative links/images */
  origin?: string

  /** Explicit plugins, applied in array order */
  plugins?: Plugin[]

  /** Custom tag output or aliases */
  tagOverrides?: Record<string, TagOverride | string>

  /** Cleanup rules from clean() in @mdream/js/clean */
  clean?: Cleaner
}

Copy code

interface MarkdownChunk {
  content: string
  metadata: {
    /** Header hierarchy at this chunk position */
    headers?: Record<string, string> // { h1: "Title", h2: "Section" }
    /** Code block language if chunk contains code */
    code?: string
    /** Inclusive, one-based line range in converted Markdown, excluding stripped headings */
    loc?: {
      lines: { from: number, to: number }
    }
  }
}

Copy code

import { withMinimalPreset } from '@mdream/js/preset/minimal'
import { htmlToMarkdownSplitChunks } from '@mdream/js/splitter'

const chunks = htmlToMarkdownSplitChunks(html, withMinimalPreset({
  headersToSplitOn: [2],
  chunkSize: 500,
  origin: 'https://example.com',
}))

Copy code

import { shouldServeMarkdown } from '@mdream/js/negotiate'

export function markdownResponse(request: Request, markdown: string): Response | undefined {
  if (shouldServeMarkdown(
    request.headers.get('accept') ?? undefined,
    request.headers.get('sec-fetch-dest') ?? undefined,
  )) {
    return new Response(markdown, {
      headers: { 'Content-Type': 'text/markdown' },
    })
  }
}

Copy code

import type { ElementNode } from '@mdream/js'
import { ELEMENT_NODE, NodeEventEnter } from '@mdream/js'
import { parseHtml } from '@mdream/js/parse'

const html = '<div><h1>Title</h1><p>Content</p></div>'
const { events, remainingHtml } = parseHtml(html)

events.forEach((event) => {
  if (event.type === NodeEventEnter && event.node.type === ELEMENT_NODE) {
    console.log('Entering element:', (event.node as ElementNode).name)
  }
})

Copy code

curl -s https://en.wikipedia.org/wiki/Markdown \
  | npx mdream --origin https://en.wikipedia.org --preset minimal \
  | tee output.md

Copy code

cat index.html \
  | npx mdream --preset minimal \
  | tee output.md

Copy code

cat index.html \
  | npx mdream --format text \
  | tee output.txt

Copy code

cat index.html \
  | npx mdream --format html \
  | tee output.html

Copy code

import { htmlToMarkdown } from 'mdream/browser'

const markdown = await htmlToMarkdown('<h1>Hello</h1>', { minimal: true })

Copy code

import { htmlToMarkdown, streamHtmlToMarkdown } from 'mdream'

// WASM engine auto-selected via export conditions
const markdown = htmlToMarkdown('<h1>Hello World</h1>')

// Streaming works the same as Node.js
const response = await fetch('https://example.com')
for await (const chunk of streamHtmlToMarkdown(response.body)) {
  // process chunk
}

Copy code

import { htmlToMarkdown } from 'mdream'

export function convertResponse(html: string): Response {
  try {
    return new Response(htmlToMarkdown(html))
  }
  catch (error) {
    console.error(error)
    return new Response('Conversion failed', { status: 500 })
  }
}

Copy code

import init, { htmlToMarkdown } from 'mdream/wasm'
import wasmModule from 'mdream/wasm/mdream_edge_bg.wasm'

await init({ module_or_path: wasmModule })
const markdown = htmlToMarkdown('<h1>Hello</h1>', { minimal: true })

Copy code

<script src="https://unpkg.com/mdream/dist/iife.js"></script>
<script>
  window.mdream.htmlToMarkdown('<h1>Hello</h1><p>World</p>')
    .then(markdown => console.log(markdown)) // # Hello\n\nWorld
</script>

Copy code

import { Readability } from '@mozilla/readability'
import { JSDOM } from 'jsdom'
import { htmlToMarkdown } from 'mdream'

const dom = new JSDOM(html, { url: 'https://example.com' })
const article = new Readability(dom.window.document).parse()

if (article?.content) {
  const markdown = htmlToMarkdown(article.content)
  // article.title, article.excerpt, article.byline also available
}

Copy code

import { generateLlmsTxtArtifacts } from '@mdream/js/llms-txt'
import { htmlToMarkdown } from 'mdream'

const result = await generateLlmsTxtArtifacts({
  files: [
    { title: 'Home', url: '/', content: htmlToMarkdown(homeHtml) },
    { title: 'About', url: '/about', content: htmlToMarkdown(aboutHtml) },
  ],
  siteName: 'My Site',
  origin: 'https://example.com',
  generateFull: true,
})

console.log(result.llmsTxt) // llms.txt content
console.log(result.llmsFullTxt) // llms-full.txt content