Repository navigation
Expand file tree
/
Copy pathslugify.py
More file actions
229 lines (205 loc) · 9.44 KB
/
Copy pathslugify.py
File metadata and controls
229 lines (205 loc) · 9.44 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
from __future__ import annotations
import re
import unicodedata
from collections.abc import Iterable
from html.entities import name2codepoint
from importlib import import_module
from typing import Literal
from ._legacy import slugify as _legacy_slugify, smart_truncate
__all__ = ['slugify', 'smart_truncate', 'Backend', 'ReplacementStage', 'Algorithm']
CHAR_ENTITY_PATTERN = re.compile(r'&(%s);' % '|'.join(name2codepoint))
DECIMAL_PATTERN = re.compile(r'&#(\d+);')
MODERN_HEX_PATTERN = re.compile(r'&#[xX]([\da-fA-F]+);')
QUOTE_PATTERN = re.compile(r"[']+")
DISALLOWED_CHARS_PATTERN = re.compile(r'[^-a-zA-Z0-9]+')
DISALLOWED_UNICODE_CHARS_PATTERN = re.compile(r'[\W_]+')
DUPLICATE_DASH_PATTERN = re.compile(r'-{2,}')
NUMBERS_PATTERN = re.compile(r'(?<=\d),(?=\d)')
DEFAULT_SEPARATOR = '-'
Backend = Literal['auto', 'text-unidecode', 'unidecode', 'anyascii']
ReplacementStage = Literal['both', 'pre', 'post']
Algorithm = Literal['legacy', 'modern']
def _numeric_reference(match: re.Match[str], base: int) -> str:
"""Leave invalid references for ordinary filtering, independently of neighbors."""
try:
value = int(match.group(1), base)
if 0xD800 <= value <= 0xDFFF:
return match.group(0)
return chr(value)
except (ValueError, OverflowError):
return match.group(0)
def _decode_entities(text: str, entities: bool, decimal: bool, hexadecimal: bool) -> str:
if entities:
text = CHAR_ENTITY_PATTERN.sub(lambda m: chr(name2codepoint[m.group(1)]), text)
if decimal:
text = DECIMAL_PATTERN.sub(lambda m: _numeric_reference(m, 10), text)
if hexadecimal:
text = MODERN_HEX_PATTERN.sub(lambda m: _numeric_reference(m, 16), text)
return text
def _transliterate(text: str, backend: Backend) -> str:
# Every backend maps 7-bit ASCII to itself; skip importing one (text-unidecode's table is ~3 MB).
if text.isascii():
return text
if backend == 'auto':
try:
module = import_module('unidecode')
except ModuleNotFoundError as error:
if error.name != 'unidecode':
raise
module = import_module('text_unidecode')
else:
module = import_module(backend.replace('-', '_'))
# These third-party modules share a string-to-string API; some are untyped.
result: str = getattr(module, 'anyascii' if backend == 'anyascii' else 'unidecode')(text)
return result
def _modern_truncate(text: str, max_length: int, word_boundary: bool, separator: str, save_order: bool) -> str:
"""Budget internal dash-separated tokens before mapping output delimiters.
Output delimiters may also occur in words; never strip or split emitted text.
Post replacements retain the historical global dash-to-separator mapping.
"""
if max_length <= 0:
return text.replace(DEFAULT_SEPARATOR, separator)
output_length = len(text) + text.count(DEFAULT_SEPARATOR) * (len(separator) - 1)
if output_length <= max_length:
return text.replace(DEFAULT_SEPARATOR, separator)
tokens = text.split(DEFAULT_SEPARATOR)
if word_boundary:
words: list[str] = []
length = 0
previous_index = 0
first_word = True
for index, word in enumerate(tokens):
if not word:
continue
# Keep the full delimiter run attached to each retained word.
# Skipping an oversized first word must not add a leading delimiter.
delimiter = separator * (index - previous_index) if words or first_word else ''
previous_index = index
first_word = False
next_length = length + len(delimiter) + len(word)
if next_length <= max_length:
words.append(delimiter + word)
length = next_length
elif save_order:
break
if words:
return ''.join(words)
# A hard cut may shorten a word but must not emit a partial/trailing delimiter.
parts: list[str] = []
length = 0
previous_index = 0
for index, word in enumerate(tokens):
if not word:
continue
# Keep repeated delimiters with the next nonempty word until it fits.
delimiter = separator * (index - previous_index)
remaining = max_length - length - len(delimiter)
if remaining <= 0:
break
parts.append(delimiter + word[:remaining])
length += len(delimiter) + min(len(word), remaining)
previous_index = index
if len(word) > remaining:
break
return ''.join(parts)
def _modern_slugify(
text: str | bytes | bytearray,
entities: bool,
decimal: bool,
hexadecimal: bool,
max_length: int,
word_boundary: bool,
separator: str,
save_order: bool,
stopwords: Iterable[str],
regex_pattern: re.Pattern[str] | str | None,
lowercase: bool,
replacements: Iterable[Iterable[str]],
allow_unicode: bool,
replacement_stage: ReplacementStage,
backend: Backend,
) -> str:
# bool is an int subclass: legacy silently treats max_length=True as 1.
# Modern rejects it, and a non-str separator, up front.
if isinstance(max_length, bool) or not isinstance(max_length, int):
raise TypeError(f"max_length must be an int, not {type(max_length).__name__}")
if not isinstance(separator, str):
raise TypeError(f"separator must be str, not {type(separator).__name__}")
if not isinstance(text, str):
if not isinstance(text, (bytes, bytearray)):
raise TypeError(f'text must be str, bytes or bytearray, not {type(text).__name__}')
text = text.decode('utf-8', 'ignore')
if replacement_stage not in ('both', 'pre', 'post'):
raise ValueError("replacement_stage must be 'both', 'pre' or 'post'")
if backend not in ('auto', 'text-unidecode', 'unidecode', 'anyascii'):
raise ValueError("backend must be 'auto', 'text-unidecode', 'unidecode' or 'anyascii'")
# Materialize rules once: consumption of an iterator would affect output.
rules = tuple((old, new) for old, new in replacements) if replacements else ()
if rules and replacement_stage in ('both', 'pre'):
for old, new in rules:
text = text.replace(old, new)
text = _decode_entities(text, entities, decimal, hexadecimal)
text = QUOTE_PATTERN.sub(DEFAULT_SEPARATOR, text)
if allow_unicode:
text = unicodedata.normalize('NFKC', text)
else:
text = _transliterate(unicodedata.normalize('NFKD', text), backend)
text = unicodedata.normalize('NFKC' if allow_unicode else 'NFKD', text)
if lowercase:
text = text.lower()
text = QUOTE_PATTERN.sub('', text)
text = NUMBERS_PATTERN.sub('', text)
pattern = regex_pattern or (DISALLOWED_UNICODE_CHARS_PATTERN if allow_unicode else DISALLOWED_CHARS_PATTERN)
text = re.sub(pattern, DEFAULT_SEPARATOR, text)
text = DUPLICATE_DASH_PATTERN.sub(DEFAULT_SEPARATOR, text).strip(DEFAULT_SEPARATOR)
if stopwords:
excluded = {word.lower() if lowercase else word for word in stopwords}
text = DEFAULT_SEPARATOR.join(word for word in text.split(DEFAULT_SEPARATOR) if word not in excluded)
if rules and replacement_stage in ('both', 'post'):
for old, new in rules:
text = text.replace(old, new)
return _modern_truncate(text, max_length, word_boundary, separator, save_order)
def slugify(
text: str | bytes | bytearray,
entities: bool = True,
decimal: bool = True,
hexadecimal: bool = True,
max_length: int = 0,
word_boundary: bool = False,
separator: str = DEFAULT_SEPARATOR,
save_order: bool = False,
stopwords: Iterable[str] = (),
regex_pattern: re.Pattern[str] | str | None = None,
lowercase: bool = True,
replacements: Iterable[Iterable[str]] = (),
allow_unicode: bool = False,
*,
replacement_stage: ReplacementStage = 'both',
backend: Backend = 'auto',
algorithm: Algorithm = 'legacy',
) -> str:
"""Make a slug with the legacy output pipeline by default, permanently.
algorithm='legacy' (the default) dispatches to the frozen legacy pipeline in
slugify._legacy, which must never change. algorithm='modern' opts into early
entity decoding, reusable iterator rules, stable stopword membership, a final
emitted-character length budget, and up-front argument type validation.
Bytes and bytearray are decoded as UTF-8, ignoring invalid bytes.
replacements are ordered literal rules, before and after cleanup by default;
replacement_stage selects 'pre', 'post' or 'both'. Post rules are unfiltered.
regex_pattern matches disallowed characters; stopwords match internal words.
allow_unicode uses NFKC without transliteration. backend='auto' prefers
installed Unidecode, falling back to text-unidecode; explicit choices do not
fall back. All new controls are keyword-only.
"""
if algorithm not in ('legacy', 'modern'):
raise ValueError("algorithm must be 'legacy' or 'modern'")
if algorithm == 'legacy':
return _legacy_slugify(
text, entities, decimal, hexadecimal, max_length, word_boundary,
separator, save_order, stopwords, regex_pattern, lowercase,
replacements, allow_unicode,
replacement_stage=replacement_stage, backend=backend)
return _modern_slugify(
text, entities, decimal, hexadecimal, max_length, word_boundary,
separator, save_order, stopwords, regex_pattern, lowercase,
replacements, allow_unicode, replacement_stage, backend)