1"""Multi-line needs in source comments: comment runs, prefix stripping, the block grammar.
2
3A multi-line need is one need written across the lines of a comment. This module turns
4the comment nodes of one file into comment RUNS, strips each line's comment prefix by the
5comment's kind, and parses a run's lines with the block grammar. It knows nothing of
6Sphinx or of a need's fields: the record is built by ``analyse.py``, and the need is
7validated and its content parsed by whoever consumes the record.
8
9**Runs.** One block comment (``/* … */``, ``/** … */``, ``/*! … */``) or one Python string
10statement (a docstring or any bare string statement) is a run on its own. Consecutive line comments form one run when they sit on consecutive rows,
11use the same delimiter, and have only whitespace before them on their rows; a line
12comment after code on its row neither starts nor continues a run.
13
14**Prefix stripping** follows the comment's kind and never searches the text, so each
15logical line maps to exactly one source row:
16
17======================================== ================================================== ===========================================
18comment kind stripped from each line languages
19======================================== ================================================== ===========================================
20line comment ``//``, ``///``, ``//!`` the delimiter, then ONE space if present C/C++, C#, Rust, Go, TS/JS, JSONC
21line comment ``#`` the delimiter, then ONE space if present Python, YAML, Bash
22block comment ``/* … */``, ``/** … */``, a first row holding only the opener (``/*`` with C/C++, C#, Rust, Go, TS/JS, JSONC
23``/*! … */`` any stars, or ``/*!``) and a last row holding only
24 the closer (stars then ``*/``) are dropped; the
25 delimiters are removed; then a ``*`` leader (``*``
26 and one space, or a lone ``*``) only if EVERY
27 remaining non-blank line after the opener's row
28 carries one (the doxygen rule)
29Python docstring the quotes and any string prefix; the interior Python
30 dedented as ``inspect.cleandoc`` does (no line is
31 removed)
32line comment ``--`` one row to add when the language lands VHDL, SQL (sphinx-needs #2077)
33block comment ``<!-- … -->`` one row to add when the language lands Markdown (sphinx-needs #1895)
34======================================== ================================================== ===========================================
35
36A comment whose text does not start with a delimiter its kind allows -- impossible for a
37tree-sitter node, possible for a libclang one -- keeps its lines unstripped, and a debug
38message says so.
39
40**The grammar** (after stripping) is the one in ``docs/components/analyse.rst``: the open
41line ``<open>[<markup>] <type>: <title>``, option lines ``:key: value`` (a line indented
42deeper than its option line continues the value), the body in the declared markup, and the
43close word alone on its line. :func:`parse_run` returns the blocks, the refusals and the
44rows a block claims; the claimed rows are hidden from the one-line and ``@need-ids:``
45extractors.
46"""
47
48from __future__ import annotations
49
50import re
51from collections.abc import Iterable, Sequence
52from dataclasses import dataclass, field
53from typing import Literal
54
55from sphinx_codelinks.analyse.models import SourceComment, WarningSubTypeEnum
56from sphinx_codelinks.config import CommentCategory, MultilineNeedsConfig
57from sphinx_codelinks.logger import get_logger
58
59logger = get_logger(__name__)
60
61CommentKind = Literal["line", "block", "docstring", "unknown"]
62
63#: line-comment delimiters, longest first so that ``///`` is not read as ``//``
64LINE_DELIMITERS: tuple[str, ...] = ("///", "//!", "//", "#")
65
66_LEADER = re.compile(r"^[ \t]*\*(?: |$)")
67"""A doxygen leader: optional whitespace, ``*``, then one space or the line end."""
68
69_OPENER_ONLY = re.compile(r"^/\*+!?\s*$")
70"""A block comment's first row holding nothing but its opener (``/*``, ``/*****``, ``/*!``)."""
71
72_CLOSER_ONLY = re.compile(r"^\s*\*+/\s*$")
73"""A block comment's last row holding nothing but its closer (``*/``, `` ***/``)."""
74
75_STARS = re.compile(r"^\s*\*+\s*")
76"""Stars at the start of a line, with the whitespace around them."""
77
78_STARS_ONLY = re.compile(r"^\s*\*+\s*$")
79"""A line of stars only: a separator, counted as a leader line and read as blank."""
80
81_DOCSTRING_OPEN = re.compile(r"""^([rRuUbBfF]{0,2})(\"\"\"|'''|"|')""")
82
83_HEADER = re.compile(r"^(?:\[([^\]]*)\])?[ \t]+([A-Za-z0-9_-]+):(?:[ \t]+(.*))?$")
84"""What follows the open word: ``[<markup>]``, whitespace, ``<type>:``, the title."""
85
86_OPTION = re.compile(r"^:([A-Za-z0-9_-]+):(?:[ \t]+(.*))?$")
87"""An option line, leading whitespace removed: ``:<key>:`` then the value."""
88
89RESERVED_KEYS: frozenset[str] = frozenset({"type", "title", "content", "doctype"})
90"""Keys the record sets itself: an option naming one duplicates it."""
91
92
93@dataclass(frozen=True)
94class LogicalLine:
95 """One line of a comment run with its comment prefix stripped."""
96
97 row: int
98 """The 0-based source row."""
99 col: int
100 """The 0-based source column (in characters) of ``text[0]``."""
101 text: str
102 """The line after prefix stripping."""
103
104
105@dataclass
106class CommentRun:
107 """The comments one block grammar pass reads, and their logical lines."""
108
109 comments: list[SourceComment]
110 lines: list[LogicalLine]
111 leaderless_block: bool = False
112 """A block comment whose lines did not all carry a ``*`` leader, so none was
113 stripped: an open word behind a star is reported, not silently lost."""
114 mixed_leaders: bool = False
115 """A leaderless block in which some lines DO carry the leader: the missing leader is
116 then the likely cause of an unterminated block (a plain block has none to miss)."""
117
118
119@dataclass
120class ParsedBlock:
121 """One multi-line need as the grammar found it, in source coordinates (0-based)."""
122
123 open_row: int
124 open_col: int
125 close_row: int
126 end_col: int
127 """One past the close word's last character."""
128 markup: str
129 doctype: str
130 need: dict[str, str]
131 option_rows: dict[str, int]
132 content_start: tuple[int, int] | None
133
134
135@dataclass(frozen=True)
136class ParseIssue:
137 """A refusal or a note of the grammar, at a 0-based source row."""
138
139 kind: WarningSubTypeEnum
140 row: int
141 msg: str
142
143
144@dataclass
145class ParseResult:
146 blocks: list[ParsedBlock] = field(default_factory=list)
147 issues: list[ParseIssue] = field(default_factory=list)
148 claimed_rows: set[int] = field(default_factory=set)
149
150
151@dataclass
152class _Comment:
153 comment: SourceComment
154 row: int
155 col: int
156 text: str
157 kind: CommentKind
158 delimiter: str
159 alone: bool
160 """Only whitespace before the comment on its row (or not knowable: libclang
161 without the row's text)."""
162
163
164def _text(comment: SourceComment) -> str:
165 return comment.text.decode("utf-8", errors="replace")
166
167
168def _column(comment: SourceComment, text: str, row_text: str | None) -> int | None:
169 """The comment's 0-based column in characters, or ``None`` when not knowable.
170
171 A tree-sitter node gives a byte column; a libclang comment gives none, so its
172 column is read from the row: its first line is the row's suffix, or else its first
173 occurrence in the row (a one-row block comment followed by code).
174 """
175 node = comment.node
176 if getattr(node, "is_libclang", False):
177 if row_text is None:
178 return None
179 first = text.split("\n", 1)[0]
180 if row_text.endswith(first):
181 return len(row_text) - len(first)
182 index = row_text.find(first)
183 return index if index != -1 else None
184 byte_col = node.start_point.column
185 if row_text is None:
186 return byte_col
187 return len(row_text.encode("utf-8")[:byte_col].decode("utf-8", errors="replace"))
188
189
190def _classify(comment: SourceComment, text: str) -> tuple[CommentKind, str]:
191 # a libclang comment has no ``type``; only tree-sitter's Python grammar has docstrings
192 if getattr(comment.node, "type", None) == CommentCategory.docstring:
193 return "docstring", ""
194 stripped = text.lstrip()
195 if stripped.startswith("/*"):
196 return "block", "/*"
197 for delimiter in LINE_DELIMITERS:
198 if stripped.startswith(delimiter):
199 return "line", delimiter
200 return "unknown", ""
201
202
203def _view(comment: SourceComment, source_lines: Sequence[str] | None) -> _Comment:
204 text = _text(comment)
205 row = comment.node.start_point.row
206 row_text = (
207 source_lines[row]
208 if source_lines is not None and row < len(source_lines)
209 else None
210 )
211 col = _column(comment, text, row_text)
212 kind, delimiter = _classify(comment, text)
213 alone = True
214 if row_text is not None and col is not None:
215 alone = not row_text[:col].strip()
216 return _Comment(comment, row, col or 0, text, kind, delimiter, alone)
217
218
219def form_runs(
220 comments: Iterable[SourceComment], source_lines: Sequence[str] | None = None
221) -> list[CommentRun]:
222 """Group one file's comments into runs, in source order.
223
224 :param comments: The file's comment nodes, in any order (tree-sitter's captures are
225 not in source order).
226 :param source_lines: The file's rows, for what precedes a comment on its row; without
227 them every line comment counts as alone on its row.
228 """
229 views = sorted(
230 (_view(comment, source_lines) for comment in comments),
231 key=lambda view: (view.row, view.col),
232 )
233 runs: list[CommentRun] = []
234 current: list[_Comment] = []
235 for view in views:
236 if current and _continues(current[-1], view):
237 current.append(view)
238 continue
239 if current:
240 runs.append(_make_run(current))
241 current = [view]
242 if current:
243 runs.append(_make_run(current))
244 return runs
245
246
247def _continues(previous: _Comment, view: _Comment) -> bool:
248 return (
249 previous.kind == "line"
250 and view.kind == "line"
251 and previous.alone
252 and view.alone
253 and previous.delimiter == view.delimiter
254 and view.row == previous.row + 1
255 )
256
257
258def _make_run(views: list[_Comment]) -> CommentRun:
259 if len(views) == 1 and views[0].kind == "block":
260 lines, leader_stripped, mixed = _block_comment_lines(views[0])
261 return CommentRun([views[0].comment], lines, not leader_stripped, mixed)
262 run_lines: list[LogicalLine] = []
263 for view in views:
264 run_lines.extend(_logical_lines(view))
265 return CommentRun([view.comment for view in views], run_lines)
266
267
268def _logical_lines(view: _Comment) -> list[LogicalLine]:
269 if view.kind == "line":
270 return [_line_comment_line(view)]
271 if view.kind == "block":
272 return _block_comment_lines(view)[0]
273 if view.kind == "docstring":
274 return _docstring_lines(view)
275 logger.debug(
276 f"codelinks: a comment at row {view.row + 1} starts with no comment delimiter; "
277 "its lines are read unstripped"
278 )
279 return [
280 LogicalLine(view.row + offset, view.col if offset == 0 else 0, line)
281 for offset, line in enumerate(view.text.split("\n"))
282 ]
283
284
285def _line_comment_line(view: _Comment) -> LogicalLine:
286 text = view.text[:-1] if view.text.endswith("\n") else view.text
287 stripped = text.lstrip()
288 col = view.col + len(text) - len(stripped) + len(view.delimiter)
289 rest = stripped[len(view.delimiter) :]
290 if rest.startswith(" "):
291 rest = rest[1:]
292 col += 1
293 return LogicalLine(view.row, col, rest)
294
295
296def _block_comment_lines(view: _Comment) -> tuple[list[LogicalLine], bool, bool]:
297 """The block comment's logical lines, whether a ``*`` leader was stripped, and
298 whether, unstripped, some of its lines carry one (a mixed block)."""
299 texts = view.text.split("\n")
300 count = len(texts)
301 cols = [view.col] + [0] * (count - 1)
302 keep = [True] * count
303 if count > 1 and _OPENER_ONLY.match(texts[0].strip()):
304 keep[0] = False
305 else:
306 lead = len(texts[0]) - len(texts[0].lstrip())
307 body = texts[0].lstrip()
308 if body.startswith("/*!"):
309 opener = "/*!"
310 elif body.startswith("/**") and len(view.text.strip()) > 4:
311 opener = "/**"
312 else:
313 opener = "/*"
314 cols[0] += lead + len(opener)
315 texts[0] = body[len(opener) :]
316 if count > 1 and _CLOSER_ONLY.match(texts[-1]):
317 keep[-1] = False
318 else:
319 last = texts[-1].rstrip()
320 if last.endswith("*/"):
321 texts[-1] = last[: -len("*/")]
322
323 # the leader test reads the rows after the opener's, delimiter-only rows dropped
324 later = [i for i in range(1, count) if keep[i]]
325 non_blank = [i for i in later if texts[i].strip()]
326 leader_lines = [
327 i for i in non_blank if _LEADER.match(texts[i]) or _STARS_ONLY.match(texts[i])
328 ]
329 leader_stripped = bool(non_blank) and len(leader_lines) == len(non_blank)
330 mixed = not leader_stripped and bool(leader_lines)
331 if leader_stripped:
332 for i in later:
333 match = _LEADER.match(texts[i])
334 if match:
335 cols[i] += match.end()
336 texts[i] = texts[i][match.end() :]
337 elif _STARS_ONLY.match(texts[i]):
338 texts[i] = ""
339
340 if not texts[0].strip():
341 keep[0] = False
342 if count > 1 and not texts[-1].strip():
343 keep[-1] = False
344 lines = [
345 LogicalLine(view.row + i, cols[i], texts[i]) for i in range(count) if keep[i]
346 ]
347 return lines, leader_stripped, mixed
348
349
350def _docstring_lines(view: _Comment) -> list[LogicalLine]:
351 match = _DOCSTRING_OPEN.match(view.text)
352 if match is None:
353 return [LogicalLine(view.row, view.col, view.text)]
354 quote = match.group(2)
355 inner = view.text[match.end() :]
356 if inner.endswith(quote):
357 inner = inner[: -len(quote)]
358 raw = inner.split("\n")
359 first = raw[0]
360 lines = [
361 LogicalLine(
362 view.row,
363 view.col + match.end() + len(first) - len(first.lstrip()),
364 first.lstrip(),
365 )
366 ]
367 indents = [len(line) - len(line.lstrip()) for line in raw[1:] if line.strip()]
368 margin = min(indents, default=0)
369 for offset, line in enumerate(raw[1:], start=1):
370 text = line[margin:] if line.strip() else ""
371 lines.append(LogicalLine(view.row + offset, margin, text))
372 return lines
373
374
375def _indent(text: str) -> int:
376 return len(text) - len(text.lstrip())
377
378
379def open_rest(text: str, start_sequence: str) -> str | None:
380 """What follows the open word, if ``text`` starts with it as a whole word.
381
382 The word must be followed by ``[``, whitespace or the line end. So the default
383 reference marker ``@need-ids:``, which starts with the default open word, does not
384 open a block, and neither does ``@needle``.
385 """
386 stripped = text.lstrip()
387 if not stripped.startswith(start_sequence):
388 return None
389 rest = stripped[len(start_sequence) :]
390 if rest and rest[0] != "[" and not rest[0].isspace():
391 return None
392 return rest
393
394
[docs]395# @Parse multi-line need blocks in a comment run, IMPL_MLN_2, impl, [FE_MULTILINE_NEEDS]
396def parse_run(
397 lines: Sequence[LogicalLine],
398 config: MultilineNeedsConfig,
399 *,
400 leaderless_block: bool = False,
401 mixed_leaders: bool = False,
402) -> ParseResult:
403 """Find the multi-line needs in one comment run.
404
405 At each line opening a block: the close word ending the same line is the one-line
406 form, refused, and its line is claimed; no close line further down the run is an
407 unterminated block, refused, and the rest of the run is not scanned; an open line
408 that does not match the grammar is refused, and the lines up to its close are still
409 consumed. A block, a refused header and a refused one-line form claim their lines.
410
411 :param leaderless_block: The run is one block comment whose ``*`` leader was not
412 stripped: an open word behind a star is refused as a header, with the cause, and
413 consumed up to the first close line, which may then carry the stars too. The
414 refusal order is the same as for any open line: the one-line form first.
415 :param mixed_leaders: Some lines of that block do carry the leader: an unterminated
416 open then names the missing leader as the likely cause.
417 """
418 result = ParseResult()
419 start, end = config.start_sequence, config.end_sequence
420 unterminated = f"no '{end}' line before the comment ends; the block is skipped"
421 if leaderless_block and mixed_leaders:
422 unterminated += (
423 "; the likely cause: the block comment's lines do not all carry the '*' "
424 "leader"
425 )
426 index = 0
427 while index < len(lines):
428 line = lines[index]
429 rest = open_rest(line.text, start)
430 if rest is None:
431 stars = _STARS.match(line.text) if leaderless_block else None
432 starred_rest = open_rest(line.text[stars.end() :], start) if stars else None
433 if starred_rest is not None and _ends_with_word(starred_rest, end):
434 result.issues.append(_oneline_form(line.row, start, end))
435 result.claimed_rows.add(line.row)
436 index += 1
437 continue
438 if starred_rest is not None:
439 close = next(
440 (
441 k
442 for k in range(index + 1, len(lines))
443 if _is_close(lines[k].text, end, starred=True)
444 ),
445 None,
446 )
447 if close is None:
448 result.issues.append(
449 ParseIssue(
450 WarningSubTypeEnum.multiline_need_unterminated,
451 line.row,
452 unterminated,
453 )
454 )
455 break
456 result.issues.append(
457 ParseIssue(
458 WarningSubTypeEnum.multiline_need_header,
459 line.row,
460 f"'{start}' sits behind a '*' leader in a block comment "
461 "whose other lines carry none; give every line the leader "
462 "or none",
463 )
464 )
465 result.claimed_rows.update(
466 lines[k].row for k in range(index, close + 1)
467 )
468 index = close + 1
469 continue
470 index += 1
471 continue
472 if _ends_with_word(rest, end):
473 result.issues.append(_oneline_form(line.row, start, end))
474 result.claimed_rows.add(line.row)
475 index += 1
476 continue
477 close = next(
478 (
479 k
480 for k in range(index + 1, len(lines))
481 if _is_close(lines[k].text, end, starred=False)
482 ),
483 None,
484 )
485 if close is None:
486 result.issues.append(
487 ParseIssue(
488 WarningSubTypeEnum.multiline_need_unterminated,
489 line.row,
490 unterminated,
491 )
492 )
493 break
494 result.claimed_rows.update(lines[k].row for k in range(index, close + 1))
495 header = _HEADER.match(rest.rstrip())
496 if header is None:
497 result.issues.append(
498 ParseIssue(
499 WarningSubTypeEnum.multiline_need_header,
500 line.row,
501 f"the open line does not match '{start}[<markup>] <type>: <title>'; "
502 f"the block up to '{end}' is skipped",
503 )
504 )
505 else:
506 result.blocks.append(
507 _block(lines, index, close, header, config, result.issues)
508 )
509 index = close + 1
510 return result
511
512
513def _oneline_form(row: int, start: str, end: str) -> ParseIssue:
514 return ParseIssue(
515 WarningSubTypeEnum.multiline_need_oneline_form,
516 row,
517 f"'{start}' and '{end}' on one line: a one-line need is written with the "
518 "one-line marker; the block is skipped",
519 )
520
521
522def _is_close(text: str, end: str, *, starred: bool) -> bool:
523 """Whether ``text`` is a close line; ``starred`` also accepts stars before the word."""
524 stripped = text.strip()
525 if stripped == end:
526 return True
527 if starred:
528 stars = _STARS.match(stripped)
529 return stars is not None and stripped[stars.end() :] == end
530 return False
531
532
533def _ends_with_word(text: str, word: str) -> bool:
534 """Whether ``text`` ends with ``word`` preceded by whitespace or nothing."""
535 stripped = text.rstrip()
536 if not stripped.endswith(word):
537 return False
538 before = stripped[: -len(word)]
539 return not before or before[-1].isspace()
540
541
542def _block(
543 lines: Sequence[LogicalLine],
544 open_index: int,
545 close_index: int,
546 header: re.Match[str],
547 config: MultilineNeedsConfig,
548 issues: list[ParseIssue],
549) -> ParsedBlock:
550 open_line, close_line = lines[open_index], lines[close_index]
551 tag, need_type, title = header.group(1), header.group(2), header.group(3) or ""
552 markup = config.default_markup
553 if tag is not None:
554 if tag in config.markups:
555 markup = tag
556 else:
557 issues.append(
558 ParseIssue(
559 WarningSubTypeEnum.multiline_need_markup,
560 open_line.row,
561 f"unknown markup '{tag}' (known: "
562 f"{', '.join(sorted(config.markups))}); the project default "
563 f"'{config.default_markup}' is used",
564 )
565 )
566
567 options: dict[str, str] = {}
568 option_rows: dict[str, int] = {}
569 index = open_index + 1
570 # the option a continuation line extends: its key (None when the option was
571 # refused as a duplicate) and its indentation
572 current: tuple[str | None, int] | None = None
573 while index < close_index:
574 text = lines[index].text
575 if current is not None and text.strip() and _indent(text) > current[1]:
576 key = current[0]
577 if key is not None:
578 options[key] = " ".join(filter(None, (options[key], text.strip())))
579 index += 1
580 continue
581 option = _OPTION.match(text.strip())
582 if option is None:
583 break
584 key, value = option.group(1), (option.group(2) or "").strip()
585 if key in options or key in RESERVED_KEYS:
586 what = "the block's own" if key in RESERVED_KEYS else "an earlier"
587 issues.append(
588 ParseIssue(
589 WarningSubTypeEnum.multiline_need_duplicate_option,
590 lines[index].row,
591 f"option '{key}' repeats {what} '{key}'; the first value is kept",
592 )
593 )
594 current = (None, _indent(text))
595 else:
596 options[key] = value
597 option_rows[key] = lines[index].row + 1
598 current = (key, _indent(text))
599 index += 1
600
601 body = list(lines[index:close_index])
602 while body and not body[0].text.strip():
603 body.pop(0)
604 while body and not body[-1].text.strip():
605 body.pop()
606 content_lines: list[str] = []
607 content_start: tuple[int, int] | None = None
608 if body:
609 content_start = (body[0].row, body[0].col + _indent(body[0].text))
610 margin = min(_indent(line.text) for line in body if line.text.strip())
611 for line in body:
612 text = line.text[margin:] if line.text.strip() else ""
613 stripped = text.lstrip()
614 if stripped.startswith("\\" + config.end_sequence):
615 # tested past the indentation, as the close is; the indentation stays
616 text = text[: len(text) - len(stripped)] + stripped[1:]
617 elif open_rest(text, config.start_sequence) is not None:
618 issues.append(
619 ParseIssue(
620 WarningSubTypeEnum.multiline_need_nested_open,
621 line.row,
622 f"'{config.start_sequence}' inside a multi-line need's body is "
623 "body text: one need per block",
624 )
625 )
626 content_lines.append(text)
627
628 need: dict[str, str] = {"type": need_type, "title": title.strip()}
629 need.update(options)
630 need["content"] = "\n".join(content_lines)
631 need["doctype"] = config.markups[markup]
632 return ParsedBlock(
633 open_row=open_line.row,
634 open_col=open_line.col + _indent(open_line.text),
635 close_row=close_line.row,
636 end_col=close_line.col
637 + close_line.text.index(config.end_sequence)
638 + len(config.end_sequence),
639 markup=markup,
640 doctype=need["doctype"],
641 need=need,
642 option_rows=option_rows,
643 content_start=content_start,
644 )
645
646
647def blank_rows(text: str, first_row: int, rows: set[int]) -> str:
648 """``text`` with each of ``rows`` (0-based source rows) replaced by an empty line.
649
650 The text keeps its line count, so the row arithmetic of the other extractors holds.
651 """
652 lines = text.split("\n")
653 return "\n".join(
654 "" if first_row + offset in rows else line for offset, line in enumerate(lines)
655 )
Back to top