{"componentChunkName":"component---src-templates-tag-page-js","path":"/tags/nlp/","result":{"data":{"site":{"siteMetadata":{"title":"M.Hassan Ahmed","author":"Hassan11196"}},"allMarkdownRemark":{"totalCount":1,"edges":[{"node":{"excerpt":"How Byte-Pair Encoding Tokenizes Text for LLMs You count characters. The model counts tokens. That gap is behind a surprising number of the…","fields":{"slug":"/2026-08-09-byte-pair-encoding-llm-tokenization/"},"frontmatter":{"date":"2026-08-09T00:00:00.000Z","title":"How Byte-Pair Encoding Tokenizes Text for LLMs","description":"Byte-pair encoding turns text into the tokens an LLM bills and reasons over. How BPE merges are learned, why token counts drive cost, and where it breaks.","tags":["AI","LLM","Tokenization","NLP","Python"],"thumbnail":null}}}]}},"pageContext":{"tag":"NLP"}},"staticQueryHashes":["32046230"]}