diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0f4b9ee..741f7502 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,6 +76,9 @@ jobs: - name: Build & test Rust tokenizer (rust_tokenizer) run: devenv shell -- bash -c 'cd tokenize/rustTokenizer && make && make test' + - name: Test shell tokenizer (sh_tokenizer.py) + run: devenv shell -- bash -c 'cd tokenize/shTokenizer && python3 -m unittest discover -s tests' + - name: "Tokenizer worker: same output as tokenBySha.pl" run: devenv shell -- bash tests/test_token_worker.sh diff --git a/README.md b/README.md index dfae65fc..d5bce41d 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,9 @@ the checkout — the repo ships an [`.envrc`](./.envrc), so the devenv shell the activates automatically whenever you `cd` in, and typing `devenv shell` is no longer needed. -By default every file the tokenizer can parse is tokenized — C, C++, Java and -Rust, case-insensitively. The default mask is derived from the tokenizer's own -extension table, so it cannot drift from it; print it with `perl +By default every file the tokenizer can parse is tokenized — C, C++, Java, +Rust and sh, case-insensitively. The default mask is derived from the +tokenizer's own extension table, so it cannot drift from it; print it with `perl tokenize/fileMask.pl`. Pass `--mask` to narrow it to one language. `.am`/`.ac` are routed to the m4 parser but are not in the default mask, because that parser mis-lexes real autotools quoting (see `%MASKED_LANGUAGES` in @@ -90,6 +90,7 @@ This builds, in dependency order: | ------------------------------------------------------------- | ------------------- | ------------------------- | | `tokenize/srcMLtoken/srcml2token` | srcML tokenizer | gcc + xerces-c + libsrcml | | `tokenize/rustTokenizer` binary | Rust tokenizer | cargo | +| `tokenize/shTokenizer/sh_tokenizer.py` (no build step) | sh tokenizer | python3 + tree-sitter-bash | | `blobExec/target/scala-2.13/blobExec-0.1.0-assembly.jar` | tokenization driver | sbt, JDK 21 | | `{slickGitLog,persons,remapCommits}/target/scala-2.10/*-one-jar.jar` | history / persons / remap tools | sbt 0.13, JDK 8 | @@ -161,7 +162,7 @@ Flags (see `./run_pipeline_process.sh --help` for the full list): | `--repo-url` | git URL or local path of the repository to process | **required** | | `--repo-name` | short name prefixed to the output files | derived from `--repo-url` | | `--commit-url` | base URL for commit links in the generated HTML | `/commit/` | -| `--mask` | regex of files to tokenize (C, C++, Java, Rust); quote it | `perl tokenize/fileMask.pl` | +| `--mask` | regex of files to tokenize (C, C++, Java, Rust, sh); quote it | `perl tokenize/fileMask.pl` | | `--work` | working/output directory | `../cregit-files` | | `--memo-dir` | where to memoize tokenized blobs; outside `--work` it survives the full-run wipe | `/memo` | | `--mode` / `--shards` | tokenizer walk mode / shard count for `sharded` | `pipeline` / `4` | diff --git a/devenv.nix b/devenv.nix index f9608171..ad1a2f57 100644 --- a/devenv.nix +++ b/devenv.nix @@ -56,7 +56,8 @@ in perlEnv # pytest must come from this python: the system one's duckdb will not load. - (pkgs.python3.withPackages (ps: [ ps.duckdb ps.pytest ])) + # tree-sitter + tree-sitter-bash: the shell tokenizer. + (pkgs.python3.withPackages (ps: [ ps.duckdb ps.pytest ps.tree-sitter ps.tree-sitter-bash ])) ]; # Scala 2.13 + sbt 1.x needs a modern JDK; the legacy sbt 0.13 hack is gone. @@ -73,7 +74,8 @@ in # in `packages`: that would change the shell env in srcml2token's RUNPATH, and # so the tokenizer identity of every C, C++, Java and Rust blob. enterTest = '' - git ls-files -z '*.sh' | xargs -0 --no-run-if-empty ${pkgs.shellcheck}/bin/shellcheck -S error + # The sh tokenizer's fixtures are its input, and some are invalid shell on purpose. + git ls-files -z -- '*.sh' ':!:tokenize/shTokenizer/tests/fixtures/' | xargs -0 --no-run-if-empty ${pkgs.shellcheck}/bin/shellcheck -S error echo "shellcheck: no errors" pytest -q prove -r $(git ls-files '*/t/*.t' | xargs -n1 dirname | sort -u) diff --git a/tests/fixtures/tiny.sh b/tests/fixtures/tiny.sh new file mode 100644 index 00000000..3a14998d --- /dev/null +++ b/tests/fixtures/tiny.sh @@ -0,0 +1,4 @@ +#!/bin/sh +add () { + echo "$(( $1 + $2 ))" # "x|y" +} diff --git a/tests/t/tokenBySha.t b/tests/t/tokenBySha.t index 214744c7..1902debd 100644 --- a/tests/t/tokenBySha.t +++ b/tests/t/tokenBySha.t @@ -2,7 +2,7 @@ use strict; use warnings; -use Test::More tests => 24; +use Test::More tests => 26; use FindBin; use File::Temp qw(tempdir); use Digest::SHA qw(sha1_hex); @@ -89,6 +89,17 @@ my $memoFile = "$memoDir/" . substr($sha1, 0, 2) . "/" . substr($sha1, 2, 2) . " like($out, qr/^STUB-TOKENIZER\n--language=C\+\+\n/, ".cpp maps to --language=C++"); } +{ + my ($status, $out, $err) = run_tokenbysha("echo hi\n", + BFG_MEMO_DIR => $memoDir, + BFG_TOKENIZE_CMD => $stub, + BFG_BLOB => "6" x 40, + BFG_FILENAME => "t0000-basic.sh", + ); + is($status, 0, ".sh run succeeds"); + like($out, qr/^STUB-TOKENIZER\n--language=Sh\n/, ".sh maps to --language=Sh"); +} + { my $failedContent = "int failure;\n"; my $failedSha1 = sha1_hex($failedContent); diff --git a/tests/t/tokenizeSh.t b/tests/t/tokenizeSh.t new file mode 100644 index 00000000..b398e1e2 --- /dev/null +++ b/tests/t/tokenizeSh.t @@ -0,0 +1,56 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use Test::More; +use FindBin; +use File::Temp qw(tempdir); + +my $root = "$FindBin::Bin/../.."; # tests/t -> repo root + +my $dispatcher = "$root/tokenize/tokenize.pl"; +my $shDir = "$root/tokenize/shTokenizer"; +my $fixture = "$shDir/tests/fixtures/01_test_body.sh"; +my $expected = "$shDir/tests/expected/01_test_body.sh.nopos.token"; + +plan skip_all => "python3 with tree-sitter-bash not available (run inside devenv shell)" + unless system("python3 -c 'import tree_sitter, tree_sitter_bash' >/dev/null 2>&1") == 0; + +plan tests => 7; + +my $workdir = tempdir(CLEANUP => 1); + +sub slurp { + my ($file) = @_; + open(my $fh, '<', $file) or die "unable to read [$file]: $!"; + local $/; + my $content = <$fh>; + return defined $content ? $content : ''; +} + +sub run_dispatcher { + my (@args) = @_; + my $out = "$workdir/stdout"; + my $err = "$workdir/stderr"; + my $status = system("perl '$dispatcher' " . join(' ', @args) . " > '$out' 2> '$err'"); + return ($status, slurp($out), slurp($err)); +} + +{ + my ($status, $out) = run_dispatcher("--language=Sh", "'$fixture'"); + is($status, 0, "dispatching a .sh file as Sh succeeds"); + is($out, slurp($expected), "dispatcher output matches the shTokenizer expected output"); +} + +{ + my ($status, $out) = run_dispatcher("'$fixture'"); + is($status, 0, "autodetecting .sh succeeds"); + is($out, slurp($expected), "autodetected output matches the explicit --language=Sh run"); +} + +{ + my ($status, $out) = run_dispatcher("--language=Sh", "--position", "'$fixture'"); + is($status, 0, "--position is forwarded to the shell tokenizer"); + like($out, qr/\A-:-\|begin_unit\|/, "markers are prefixed with -:-|"); + like($out, qr/^2:1\|command\|test_expect_success$/m, "tokens are prefixed with LINE:COL|"); +} diff --git a/tests/t/tokenizerIdentity.t b/tests/t/tokenizerIdentity.t index 49e4d145..5fa8fd5e 100644 --- a/tests/t/tokenizerIdentity.t +++ b/tests/t/tokenizerIdentity.t @@ -30,6 +30,10 @@ sub fixture { write_file("$tok/rustTokenizer/target/release/rust_tokenizer", "RUST TOKENIZER v1\n"); make_path("$tok/m4Tokenizer"); write_file("$tok/m4Tokenizer/m4.py", "M4 v1\n"); + make_path("$tok/shTokenizer"); + write_sh_tokenizer($dir, "v1"); + write_file("$dir/tree-sitter", "TREE-SITTER v1\n"); + write_file("$dir/tree-sitter-bash", "TREE-SITTER-BASH v1\n"); write_srcml2token($dir, "v1"); write_file("$dir/libsrcml", "LIBSRCML v1\n"); write_file("$dir/ctags", "CTAGS v1\n"); @@ -49,6 +53,12 @@ sub write_srcml2token { write_file("$dir/srcml2token", "#!/bin/sh\n# SRCML2TOKEN $version\necho '$dir/libsrcml'\n"); } +sub write_sh_tokenizer { + my ($dir, $version) = @_; + write_file("$dir/tokenize/shTokenizer/sh_tokenizer.py", + "#!/bin/sh\n# SH TOKENIZER $version\necho '$dir/tree-sitter'\necho '$dir/tree-sitter-bash'\n"); +} + sub identity { my ($dir, @extra) = @_; my @cmd = ("perl", "$dir/tokenize/tokenizerIdentity.pl", @@ -113,6 +123,18 @@ for my $component (qw(srcml2token libsrcml ctags)) { is($a{rs}, $b{rs}, "changing $component leaves .rs alone"); } +for my $component (qw(sh_tokenizer.py tree-sitter tree-sitter-bash)) { + my $d = fixture(); + my ($before) = identity($d); + if ($component eq "sh_tokenizer.py") { write_sh_tokenizer($d, "v2") } + else { write_file("$d/$component", "$component v2\n") } + my ($after) = identity($d); + my %b = as_map($before); + my %a = as_map($after); + isnt($a{sh}, $b{sh}, "changing $component moves the .sh identity"); + is($a{c}, $b{c}, "changing $component leaves .c alone"); +} + # CregitSrcMl.pm is tokenizeSrcMl.pl's copy in the tokenizer worker. for my $script (qw(tokenizeSrcMl.pl CregitSrcMl.pm)) { my $d = fixture(); @@ -169,6 +191,14 @@ for my $shared (qw(tokenize.pl CregitLanguages.pm)) { is($out, "", "and prints no identity"); } +{ + my $d = fixture(); + write_file("$d/tokenize/shTokenizer/sh_tokenizer.py", "#!/bin/sh\nexit 0\n"); + my ($out, $status) = identity($d); + isnt($status, 0, "an sh tokenizer that does not name its tree-sitter libraries is fatal"); + is($out, "", "and prints no identity"); +} + # --------------------------------------------------------------------------- { my ($all) = identity($dir, "--all-extensions"); diff --git a/tokenize/CregitLanguages.pm b/tokenize/CregitLanguages.pm index 6d85fd04..076ce137 100644 --- a/tokenize/CregitLanguages.pm +++ b/tokenize/CregitLanguages.pm @@ -70,6 +70,8 @@ our %EXT_LANG = ( 'am' => 'M4', # Rust 'rs' => 'Rust', + # Shell + 'sh' => 'Sh', ); # Languages the pipeline's file mask is allowed to select, i.e. the ones whose @@ -102,6 +104,7 @@ our %MASKED_LANGUAGES = ( 'C++' => 1, 'Java' => 1, 'Rust' => 1, + 'Sh' => 1, ); # Language -> parser, relative to the tokenize/ directory. Every value must @@ -112,6 +115,7 @@ our %LANG_PARSER_REL = ( 'Java' => 'tokenizeSrcMl.pl', 'M4' => 'm4Tokenizer/m4.py', 'Rust' => 'rustTokenizer/target/release/rust_tokenizer', + 'Sh' => 'shTokenizer/sh_tokenizer.py', ); # The parser every srcML-routed language shares. Named so callers can ask "is diff --git a/tokenize/shTokenizer/Makefile b/tokenize/shTokenizer/Makefile new file mode 100644 index 00000000..6d204c9e --- /dev/null +++ b/tokenize/shTokenizer/Makefile @@ -0,0 +1,14 @@ +.PHONY: test expected + +PYTHON ?= python3 + +test: + $(PYTHON) -m unittest discover -s tests + +expected: + mkdir -p tests/expected + for fixture in tests/fixtures/*.sh; do \ + name=$$(basename "$$fixture"); \ + $(PYTHON) sh_tokenizer.py --position "$$fixture" > "tests/expected/$$name.token"; \ + $(PYTHON) sh_tokenizer.py "$$fixture" > "tests/expected/$$name.nopos.token"; \ + done diff --git a/tokenize/shTokenizer/README.md b/tokenize/shTokenizer/README.md new file mode 100644 index 00000000..8743c594 --- /dev/null +++ b/tokenize/shTokenizer/README.md @@ -0,0 +1,147 @@ +# shTokenizer + +`sh_tokenizer.py` tokenizes POSIX sh / bash scripts into the line-oriented view used +by cregit, the format consumed by `blameRepo` and `generate_dataset`. + +It uses [tree-sitter-bash](https://github.com/tree-sitter/tree-sitter-bash) for the +syntax, plus its own heredoc locator: tree-sitter-bash misreads heredoc bodies in common +forms, such as a `&&` after the delimiter (`cat >expect <<-\EOF &&`), and reads their +text as code. + +## Requirements + +Python 3 with the `tree-sitter` and `tree-sitter-bash` packages (provided by +`devenv shell`). + +## How to use + +```sh +sh_tokenizer.py [--language=Sh] [--position] [--verbose] [] +sh_tokenizer.py --library-paths +``` + +The token stream goes to ``, or to _stdout_ without it. `--language` and +`--verbose` are accepted so the `tokenize.pl` dispatcher can call it like the other +tokenizers. `--position` prefixes every line with `LINE:COL|` (tokens) or `-:-|` +(markers). + +`--library-paths` prints the files of the compiled tree-sitter runtime and bash +grammar. They decide the tokens as much as `sh_tokenizer.py` does, so +`tokenize/tokenizerIdentity.pl` digests them in the tokenizer identity of `.sh`. + +Exit status: 0 whenever the file can be read, including files with syntax errors; +1 if the file cannot be read; 2 on a usage error. + +## Output format + +One line per token or marker: + +``` +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +kind|value +... +end_unit + +``` + +Every `end_*` marker, `end_unit` included, is followed by an empty line (the line +`-:-|` with `--position`), exactly as `tokenizeSrcMl.pl` does for C, C++ and Java. +`generate_dataset` classifies these lines as structural blanks. + +If a file cannot be tokenized (an internal error, or an output that fails the checks of +this format), a warning goes to _stderr_ and the output is the plain split of the file +into words, all of kind `unknown`, so the pipeline never stops on a file. + +### Token kinds + +| kind | what | +|---|---| +| `keyword` | alphabetic reserved words: `if then else elif fi case esac for select while until do done in function` | +| `command` | the command name of a simple command (`git`, `:`, `.`, `return`, `time`), and the builtins `export`, `local`, `readonly`, `declare`, `typeset`, `unset` and `unsetenv` | +| `word` | any other unquoted word: arguments, options, paths, patterns, test operators and the numbers they compare (`-z`, `-eq 1`), the name in a function definition | +| `variable` | variable names in expansions, assignments and declaration commands, special parameters (`1`, `@`, `#`, `?`) | +| `number` | numbers in arithmetic (`$(( ))`, `$[ ]`, `(( ))`, the `(( ))` of `for`), file descriptors in redirections (`2>`, `6<<`, `>&2`) | +| `op` | operators and punctuation: `&& \|\| \| ; ;; & ( ) { } ! = < > >> <& >& << <<- $ ${ $( ) $(( ))`, backquotes, the brackets `[ ] [[ ]]` of a test command, arithmetic operators, the `\` of a line continuation | +| `literal` | a whole string: `'...'`, `"..."` or `$'...'` | +| `heredoc` | a whole heredoc body | +| `heredoc_delim` | the delimiter word after `<<` / `<<-` and the terminator line | +| `comment` | a whole comment, `#` to end of line | + +### Values + +- The value is the exact source text of the token, never the value the shell would + compute: quotes, backslashes and escapes are kept. +- CR and LF inside a value are replaced by a space, so every token fits on one line. +- Values are never empty. +- Values are raw bytes: files that are not valid UTF-8 are copied through unchanged. + Tokens are only split at ASCII characters, never inside a multi-byte sequence. +- Concatenating all values after removing whitespace (space, tab, CR, LF) gives the + source file after removing whitespace. No non-whitespace byte is dropped or added. + +### Strings + +A string is one `literal` token holding its whole source text, quotes and expansions +included, as the other tokenizers emit a string literal. In a concatenation such as +`'a'"b"c`, each quoted piece is a literal of its own and `c` is a `word`. A heredoc +inside a string (in `"$(cat <` / `end_function` | a function definition | + +### Positions + +With `--position`, `LINE:COL` is the 1-based position of the first byte of the token; +`COL` counts the characters of the line decoded as UTF-8 with invalid bytes replaced, as +`generate_dataset` reads the source: a truncated multi-byte sequence counts as one +character, and so does each other invalid byte. + +## Limitations + +- A string or heredoc body that spans several lines is one line of the tokenized file, + so its blame is coarser than the line blame of git. Code passed as a string is a + string too: in the git test suite, the whole body of `test_expect_success 'title' '...'` + is one `literal`. +- The heredoc locator reads `((` at the start of a word as arithmetic; bash reads it as + two nested subshells when it is not valid arithmetic (`((cd x) )`). +- Inside backquotes, a backquote within quotes opens a nested command substitution; + bash ends the backquotes there. +- tree-sitter-bash cuts an unquoted word at a form feed or a vertical tab, which the shell + reads as part of the word; that byte is then a `word` token of its own. +- When tree-sitter-bash pairs the quotes of a line wrongly (as in the `case` pattern + `?*' '?*)`) and reaches a quote it cannot close, everything from that quote to the end + of the file is one error token, which comes out as a single `word`. + +## How to test + +```sh +cd tokenize/shTokenizer +make test +``` + +`make expected` regenerates the expected outputs in `tests/expected/` from +`tests/fixtures/`: `NAME.sh.token` with `--position` and `NAME.sh.nopos.token` without. +Check the diff of every regenerated file. + +## License + +GPL-3.0+ diff --git a/tokenize/shTokenizer/sh_tokenizer.py b/tokenize/shTokenizer/sh_tokenizer.py new file mode 100755 index 00000000..2a9d0aba --- /dev/null +++ b/tokenize/shTokenizer/sh_tokenizer.py @@ -0,0 +1,595 @@ +#!/usr/bin/env python3 +"""POSIX sh / bash tokenizer for cregit. See README.md for the output format.""" +import argparse +import bisect +import re +import sys +from dataclasses import dataclass + +import tree_sitter._binding +import tree_sitter_bash +import tree_sitter_bash._binding +from tree_sitter import Language, Parser + +REVISION = "0.1.0" +CREGIT_VERSION = "0.0.1" +UNIT_HEADER = f"begin_unit|revision:{REVISION};language:Sh;cregit-version:{CREGIT_VERSION}".encode() + +# tree-sitter-bash parses these as keywords; for the shell they are commands. +KEYWORD_COMMANDS = {b"export", b"local", b"readonly", b"declare", b"typeset", b"unset", b"unsetenv"} + +WHITESPACE = b" \t\r\n" +WORD = re.compile(rb"[^ \t\r\n]+") +WORD_END = b" \t\n;&|()<>" # as for the shell, CR does not end a word + +PARSER = Parser(Language(tree_sitter_bash.language())) + + +@dataclass(slots=True) +class Token: + offset: int + kind: bytes + value: bytes + + +@dataclass(slots=True) +class Marker: + text: bytes + + +BLANK = Marker(b"") + +END_RANK, BEGIN_RANK, TOKEN_RANK = 0, 1, 2 + + +class Output: + """Tokens and markers, sorted by offset at the end, because the heredocs are emitted + after the tree walk. At one offset: end markers (innermost first), begin markers + (outermost first), then the token.""" + + def __init__(self): + self.items = [] + + def add(self, offset: int, rank: int, tie: int, records: list) -> None: + self.items.append(((offset, rank, tie, len(self.items)), records)) + + def token(self, offset: int, kind: bytes, value: bytes) -> None: + self.add(offset, TOKEN_RANK, 0, [Token(offset, kind, value)]) + + def begin(self, offset: int, depth: int, name: bytes, decl: bytes) -> None: + self.add(offset, BEGIN_RANK, depth, [Marker(b"begin_" + name), Marker(decl)]) + + def end(self, offset: int, depth: int, name: bytes) -> None: + self.add(offset, END_RANK, -depth, [Marker(b"end_" + name), BLANK]) + + def records(self): + for _key, records in sorted(self.items, key=lambda item: item[0]): + yield from records + + +BACKSLASH, SINGLE_QUOTE, DOUBLE_QUOTE, BACKQUOTE = b"\\'\"`" +DOLLAR, HASH, LESS, NEWLINE, SPACE = b"$#<\n " +OPEN_PAREN, CLOSE_PAREN, OPEN_BRACE, CLOSE_BRACE = b"(){}" +DIGITS = b"0123456789" +CLOSER = {OPEN_PAREN: CLOSE_PAREN, OPEN_BRACE: CLOSE_BRACE} +QUOTE_CONTEXTS = {SINGLE_QUOTE: "single", DOUBLE_QUOTE: "double", BACKQUOTE: "backquote"} + + +@dataclass +class Heredoc: + io_start: int # start of the fd digits, or of the operator + op_start: int + op_end: int + word_start: int + word_end: int + strip_tabs: bool + delimiter: bytes # unquoted + body_start: int = 0 + body_end: int = 0 + term_start: int | None = None # None when the terminator is missing + term_end: int | None = None + line_end: int = 0 # end of the terminator line, or of the body + + def spans(self) -> tuple[tuple[int, int], tuple[int, int]]: + return (self.io_start, self.word_end), (self.body_start, self.line_end) + + +@dataclass +class Context: + kind: str + depth: int = 0 # brackets still open + opener: int | None = None # ( or { of $( ) or ${ } + expansion: bool = False # $( ), ${ } or $(( )) + + +def scan_heredocs(text: bytes) -> list[Heredoc]: + """Locate heredocs by the rules of the shell: tree-sitter-bash reads many of them as + code (README.md, Heredocs).""" + return HeredocScanner(text).scan() + + +class HeredocScanner: + def __init__(self, text: bytes): + self.text = text + self.stack = [Context("code")] + self.heredocs: list[Heredoc] = [] + self.pending: list[Heredoc] = [] # operators of the current line + self.expansion_end = -1 + + def scan(self) -> list[Heredoc]: + i = 0 + while i < len(self.text): + context = self.stack[-1] + if context.kind == "single": + i = self.single_quoted(i) + elif context.kind == "double": + i = self.double_quoted(i) + elif context.kind == "arith": + i = self.arithmetic(i, context) + else: + i = self.code(i, context) + for heredoc in self.pending: # operators on the last line, without a newline: empty bodies + heredoc.body_start = heredoc.body_end = heredoc.line_end = len(self.text) + return self.heredocs + self.pending + + def single_quoted(self, i: int) -> int: + close = self.text.find(b"'", i) + if close < 0: + return len(self.text) + self.stack.pop() + return close + 1 + + def double_quoted(self, i: int) -> int: + byte = self.text[i] + if byte == BACKSLASH: + return i + 2 + if byte == DOUBLE_QUOTE: + self.stack.pop() + return i + 1 + if byte == DOLLAR: + after = self.open_expansion(i) + if after is not None: + return after + if byte == BACKQUOTE: + self.stack.append(Context("backquote")) + return i + 1 + + def arithmetic(self, i: int, context: Context) -> int: + byte = self.text[i] + if byte == OPEN_PAREN: + context.depth += 1 + elif byte == CLOSE_PAREN: + context.depth -= 1 + if context.depth == 0: + self.close(context, i + 1) + return i + 1 + + def close(self, context: Context, after: int) -> None: + self.stack.pop() + if context.expansion: + self.expansion_end = after + + def at_word_start(self, i: int) -> bool: + """Right after an expansion the word goes on: bash reads $(echo a)#b as a#b.""" + return i == 0 or (self.text[i - 1] in WORD_END and i != self.expansion_end) + + def code(self, i: int, context: Context) -> int: + text = self.text + byte = text[i] + if byte == BACKSLASH: + return i + 2 + if byte == BACKQUOTE and context.kind == "backquote": + self.stack.pop() + return i + 1 + if byte in QUOTE_CONTEXTS: + self.stack.append(Context(QUOTE_CONTEXTS[byte])) + return i + 1 + if byte == DOLLAR: + after = self.open_expansion(i) + if after is not None: + return after + if byte == OPEN_PAREN and text[i + 1:i + 2] == b"(" and self.at_word_start(i): + self.stack.append(Context("arith", depth=2)) # (( )) or for (( )) + return i + 2 + if context.opener is not None: + if byte == context.opener: + context.depth += 1 + return i + 1 + if byte == CLOSER[context.opener]: + context.depth -= 1 + if context.depth == 0: + self.close(context, i + 1) + return i + 1 + if byte == HASH and context.opener != OPEN_BRACE and self.at_word_start(i): + ends = [text.find(b"\n", i)] + if context.kind == "backquote": # the closing backquote also ends the comment + ends.append(text.find(b"`", i)) + ends = [end for end in ends if end >= 0] + return min(ends) if ends else len(text) + if byte == LESS and text[i + 1:i + 2] == b"<": + return self.operator(i) + if byte == NEWLINE: + i += 1 + if self.pending: + i = read_bodies(text, i, self.pending) + self.heredocs.extend(self.pending) + self.pending = [] + return i + return i + 1 + + def open_expansion(self, i: int) -> int | None: + text = self.text + if text[i + 1:i + 3] == b"((": + self.stack.append(Context("arith", depth=2, expansion=True)) + return i + 3 + if text[i + 1:i + 2] in (b"(", b"{"): + self.stack.append(Context("code", depth=1, opener=text[i + 1], expansion=True)) + return i + 2 + return None + + def operator(self, i: int) -> int: + text = self.text + if text[i + 2:i + 3] == b"<": # here-string + return i + 3 + heredoc = heredoc_at(text, i) + if heredoc is None: # no delimiter word + return i + 2 + (text[i + 2:i + 3] == b"-") + self.pending.append(heredoc) + return heredoc.word_end + + +def heredoc_at(text: bytes, i: int) -> Heredoc | None: + strip_tabs = text[i + 2:i + 3] == b"-" + op_end = i + 2 + strip_tabs + word_start = op_end + while word_start < len(text) and text[word_start] in b" \t": + word_start += 1 + word_end = skip_quoted_word(text, word_start) + if word_end == word_start: + return None + io_start = i + while io_start > 0 and text[io_start - 1] in DIGITS: + io_start -= 1 + if io_start < i and io_start > 0 and text[io_start - 1] not in WORD_END: + io_start = i # digits belong to a longer word, not an fd + word = text[word_start:word_end].replace(b"\\\n", b"") # line continuations + return Heredoc(io_start=io_start, op_start=i, op_end=op_end, word_start=word_start, + word_end=word_end, strip_tabs=strip_tabs, + delimiter=bytes(b for b in word if b not in b"\\'\"")) + + +def skip_quoted_word(text: bytes, i: int) -> int: + n = len(text) + while i < n and text[i] not in WORD_END: + byte = text[i] + if byte == BACKSLASH: + i += 2 + elif byte in (SINGLE_QUOTE, DOUBLE_QUOTE): + close = text.find(bytes([byte]), i + 1) + i = n if close < 0 else close + 1 + else: + i += 1 + return min(i, n) + + +def read_bodies(text: bytes, i: int, pending: list[Heredoc]) -> int: + n = len(text) + for heredoc in pending: + heredoc.body_start = i + pos = i + while pos < n: + newline = text.find(b"\n", pos) + line_end = n if newline < 0 else newline + line = text[pos:line_end] + stripped = line.lstrip(b"\t") if heredoc.strip_tabs else line + if stripped == heredoc.delimiter: + heredoc.body_end = pos + heredoc.term_start = pos + len(line) - len(stripped) + heredoc.term_end = heredoc.line_end = line_end + i = min(line_end + 1, n) + break + pos = line_end + 1 + else: + heredoc.body_end = heredoc.line_end = i = n + return i + + +def blank_heredocs(text: bytes, heredocs: list[Heredoc]) -> bytes: + out = bytearray(text) + for heredoc in heredocs: + for start, end in heredoc.spans(): + for k in range(start, end): + if out[k] != NEWLINE: + out[k] = SPACE + return bytes(out) + + +def split_words(src: bytes, start: int, end: int): + for m in WORD.finditer(src, start, end): + yield m.start(), m.end() + + +def flatten(value: bytes) -> bytes: + return value.replace(b"\r", b" ").replace(b"\n", b" ").rstrip(b" \t") + + +class Tokenizer: + def __init__(self, src: bytes): + self.src = src + self.out = Output() + self.covered = bytearray(len(src)) + self.heredocs = scan_heredocs(src) + # Heredoc bytes, for the heredoc rules only: a tree-sitter leaf can still span them. + self.reserved = bytearray(len(src)) + for heredoc in self.heredocs: + for start, end in heredoc.spans(): + self.reserved[start:end] = b"\x01" * (end - start) + + def run(self) -> Output: + tree = PARSER.parse(blank_heredocs(self.src, self.heredocs)) + self.walk(tree.root_node, 0, False) + self.emit_heredocs() + self.fill_gaps() + return self.out + + def emit(self, start: int, end: int, kind: bytes, reserved_ok: bool = False) -> None: + k = start + while k < end: + if self.covered[k] or (self.reserved[k] and not reserved_ok): + k += 1 + continue + j = k + while j < end and not self.covered[j] and (reserved_ok or not self.reserved[j]): + j += 1 + self.emit_span(k, j, kind) + k = j + + def emit_span(self, start: int, end: int, kind: bytes) -> None: + while start < end and self.src[start] in WHITESPACE: + start += 1 + value = flatten(self.src[start:end]) + if not value: + return + self.out.token(start, kind, value) + self.covered[start:end] = b"\x01" * (end - start) + + def emit_heredocs(self) -> None: + for heredoc in self.heredocs: + if heredoc.io_start < heredoc.op_start: + self.emit(heredoc.io_start, heredoc.op_start, b"number", reserved_ok=True) + self.emit(heredoc.op_start, heredoc.op_end, b"op", reserved_ok=True) + self.emit(heredoc.word_start, heredoc.word_end, b"heredoc_delim", reserved_ok=True) + self.emit(heredoc.body_start, heredoc.body_end, b"heredoc", reserved_ok=True) + if heredoc.term_start is not None: + self.emit(heredoc.term_start, heredoc.term_end, b"heredoc_delim", reserved_ok=True) + + def fill_gaps(self) -> None: + for word_start, word_end in split_words(self.src, 0, len(self.src)): + k = word_start + while k < word_end: + if self.covered[k]: + k += 1 + continue + run_end = k + while run_end < word_end and not self.covered[run_end]: + run_end += 1 + continuation = self.src[k:run_end] == b"\\" and self.src[run_end:run_end + 1] in (b"\n", b"\r") + self.emit(k, run_end, b"op" if continuation else b"word") + k = run_end + + def walk(self, node, depth: int, in_arith: bool) -> None: + stack = [(node, depth, in_arith)] + while stack: + current, current_depth, current_arith = stack.pop() + if self.visit(current, current_depth, current_arith): + continue + for child in reversed(current.children): + stack.append((child, current_depth + 1, current_arith or opens_arithmetic(current, child))) + + def visit(self, node, depth: int, in_arith: bool) -> bool: + """Handle node; True when its children must not be walked.""" + if node.is_missing or node.start_byte == node.end_byte: + return True + node_type = node.type + start, end = node.start_byte, node.end_byte + + if node_type == "function_definition": + name = node.child_by_field_name("name") + label = self.src[name.start_byte:name.end_byte] if name else b"" + self.out.begin(start, depth, b"function", b"DECL|function|" + flatten(label)) + self.out.end(end, depth, b"function") + return False + + if node_type in ("string", "raw_string", "ansi_c_string"): + # A heredoc inside the string ("$(cat < 0: + return False + + self.emit(start, end, leaf_kind(node, self.src[start:end], in_arith)) + return True + + def visit_concatenation(self, node, depth: int, in_arith: bool) -> None: + """tree-sitter-bash cuts a word at every byte that is not valid UTF-8; the pieces + are joined back, so no token ends inside a multi-byte sequence.""" + children = node.children + i = 0 + while i < len(children): + j = i + while (j + 1 < len(children) and children[j].type == children[j + 1].type == "word" + and children[j].end_byte == children[j + 1].start_byte + and self.non_ascii_cut(children[j].end_byte)): + j += 1 + if j > i: + self.emit(children[i].start_byte, children[j].end_byte, b"word") + else: + self.walk(children[i], depth + 1, in_arith) + i = j + 1 + + def non_ascii_cut(self, offset: int) -> bool: + return self.src[offset - 1] >= 0x80 or self.src[offset] >= 0x80 + + def visit_error(self, node, depth: int) -> None: + """Inside a syntax error the kinds of loose leaves are unreliable (`-u` comes out as + `-` and the keyword `u`), so leaves that touch are merged into one `word`.""" + run_start = run_end = None + for child in node.children: + if child.child_count == 0 and child.type not in ("comment", "raw_string", "ansi_c_string"): + if run_end is not None and child.start_byte == run_end: + run_end = child.end_byte + continue + if run_start is not None: + self.emit(run_start, run_end, b"word") + run_start, run_end = child.start_byte, child.end_byte + continue + if run_start is not None: + self.emit(run_start, run_end, b"word") + run_start = run_end = None + self.walk(child, depth + 1, False) + if run_start is not None: + self.emit(run_start, run_end, b"word") + + +def opens_arithmetic(parent, child) -> bool: + """True when child is inside $(( )), $[ ], (( )) or the header of for (( )).""" + if parent.type == "arithmetic_expansion": + return True + if parent.type == "compound_statement": + return parent.children[0].type == "((" + if parent.type == "c_style_for_statement": + return child.type not in ("do_group", "compound_statement") + return False + + +def leaf_kind(node, text: bytes, in_arith: bool) -> bytes: + node_type = node.type + if node_type == "comment": + return b"comment" + if not node.is_named: + if node_type.isalpha(): + return b"command" if text in KEYWORD_COMMANDS else b"keyword" + return b"op" + if node_type in ("variable_name", "special_variable_name"): + return b"variable" + if node_type == "file_descriptor": + return b"number" + if node_type == "number": + parent, previous = node.parent, node.prev_sibling + fd_target = (parent is not None and parent.type == "file_redirect" and previous is not None + and previous.type in (">&", "<&")) + return b"number" if in_arith or fd_target else b"word" + if node_type == "word" and node.parent is not None and node.parent.type == "command_name": + return b"command" + return b"word" + + +def position_of(src: bytes, line_starts: list[int], offset: int) -> bytes: + """LINE:COL of offset; COL counts the characters of the line decoded as UTF-8.""" + line = bisect.bisect_right(line_starts, offset) - 1 + prefix = src[line_starts[line]:offset].decode("utf-8", errors="replace") + return f"{line + 1}:{len(prefix) + 1}".encode() + + +def render(src: bytes, records, with_position: bool) -> bytes: + marker_prefix = b"-:-|" if with_position else b"" + line_starts = [0] + [m.end() for m in re.finditer(rb"\n", src)] + lines = [marker_prefix + UNIT_HEADER] + for record in records: + if isinstance(record, Token): + prefix = position_of(src, line_starts, record.offset) + b"|" if with_position else b"" + lines.append(prefix + record.kind + b"|" + record.value) + else: + lines.append(marker_prefix + record.text) + lines += [marker_prefix + b"end_unit", marker_prefix] + return b"\n".join(lines) + b"\n" + + +def check(src: bytes, records: list) -> str | None: + values = [record.value for record in records if isinstance(record, Token)] + if b"".join(WORD.findall(b"".join(values))) != b"".join(WORD.findall(src)): + return "token values do not rebuild the source" + open_markers = [] + for record in records: + if not isinstance(record, Marker): + continue + m = re.fullmatch(rb"(begin|end)_([a-z_]+)", record.text) + if m is None: + continue + if m.group(1) == b"begin": + open_markers.append(m.group(2)) + elif not open_markers or open_markers.pop() != m.group(2): + return "unbalanced markers" + if open_markers: + return "unbalanced markers" + return None + + +def fallback_records(src: bytes) -> list: + return [Token(start, b"unknown", src[start:end]) for start, end in split_words(src, 0, len(src))] + + +def tokenize(src: bytes, name: str = "") -> list: + try: + records = list(Tokenizer(src).run().records()) + reason = check(src, records) + except Exception as exc: # noqa: BLE001 - the tokenizer must never fail + reason = f"internal error: {type(exc).__name__}: {exc}" + if reason is None: + return records + print(f"sh_tokenizer: falling back to word split for [{name}]: {reason}", file=sys.stderr) + return fallback_records(src) + + +def library_paths() -> list[str]: + """The compiled tree-sitter libraries, which tokenize/tokenizerIdentity.pl digests.""" + return [tree_sitter._binding.__file__, tree_sitter_bash._binding.__file__] + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description="POSIX sh / bash tokenizer for cregit") + parser.add_argument("--language", default="Sh", help="Language of the source; only Sh is accepted") + parser.add_argument("--position", action="store_true", + help="Prefix every line with LINE:COL| (tokens) or -:-| (markers)") + parser.add_argument("--verbose", action="store_true", help="Report the file being tokenized on stderr") + parser.add_argument("--library-paths", action="store_true", + help="Print the files of the tree-sitter runtime and of the bash grammar, and exit") + parser.add_argument("source", nargs="?", help="File to tokenize") + parser.add_argument("output", nargs="?", help="Output file (default: stdout)") + args = parser.parse_args(argv) + if args.library_paths: + print("\n".join(library_paths())) + return 0 + if args.source is None: + parser.error("the following arguments are required: source") + if args.language.lower() != "sh": + print(f"sh_tokenizer: can only tokenize Sh, not [{args.language}]", file=sys.stderr) + return 2 + try: + with open(args.source, "rb") as f: + src = f.read() + except OSError as exc: + print(f"sh_tokenizer: cannot read {args.source}: {exc}", file=sys.stderr) + return 1 + if args.verbose: + print(f"sh_tokenizer: tokenizing {args.source}", file=sys.stderr) + data = render(src, tokenize(src, args.source), args.position) + if args.output: + with open(args.output, "wb") as f: + f.write(data) + else: + sys.stdout.buffer.write(data) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tokenize/shTokenizer/tests/expected/01_test_body.sh.nopos.token b/tokenize/shTokenizer/tests/expected/01_test_body.sh.nopos.token new file mode 100644 index 00000000..1ea6be48 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/01_test_body.sh.nopos.token @@ -0,0 +1,7 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: t/t0000-basic.sh:66-68 @ 18e66859 +command|test_expect_success +literal|'success is reported like this' +literal|' : ' +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/01_test_body.sh.token b/tokenize/shTokenizer/tests/expected/01_test_body.sh.token new file mode 100644 index 00000000..5b0696bb --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/01_test_body.sh.token @@ -0,0 +1,7 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: t/t0000-basic.sh:66-68 @ 18e66859 +2:1|command|test_expect_success +2:21|literal|'success is reported like this' +2:53|literal|' : ' +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/02_functions.sh.nopos.token b/tokenize/shTokenizer/tests/expected/02_functions.sh.nopos.token new file mode 100644 index 00000000..00704f06 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/02_functions.sh.nopos.token @@ -0,0 +1,37 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: git-sh-setup.sh:49-58 @ 18e66859 +begin_function +DECL|function|die +word|die +op|( +op|) +op|{ +command|die_with_status +word|1 +literal|"$@" +op|} +end_function + +begin_function +DECL|function|die_with_status +word|die_with_status +op|( +op|) +op|{ +variable|status +op|= +op|$ +variable|1 +command|shift +command|printf +op|>& +number|2 +literal|'%s\n' +literal|"$*" +command|exit +literal|"$status" +op|} +end_function + +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/02_functions.sh.token b/tokenize/shTokenizer/tests/expected/02_functions.sh.token new file mode 100644 index 00000000..14436a53 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/02_functions.sh.token @@ -0,0 +1,37 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: git-sh-setup.sh:49-58 @ 18e66859 +-:-|begin_function +-:-|DECL|function|die +2:1|word|die +2:5|op|( +2:6|op|) +2:8|op|{ +3:2|command|die_with_status +3:18|word|1 +3:20|literal|"$@" +4:1|op|} +-:-|end_function +-:-| +-:-|begin_function +-:-|DECL|function|die_with_status +6:1|word|die_with_status +6:17|op|( +6:18|op|) +6:20|op|{ +7:2|variable|status +7:8|op|= +7:9|op|$ +7:10|variable|1 +8:2|command|shift +9:2|command|printf +9:9|op|>& +9:11|number|2 +9:13|literal|'%s\n' +9:20|literal|"$*" +10:2|command|exit +10:7|literal|"$status" +11:1|op|} +-:-|end_function +-:-| +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/03_if_case_multiline_string.sh.nopos.token b/tokenize/shTokenizer/tests/expected/03_if_case_multiline_string.sh.nopos.token new file mode 100644 index 00000000..7f6962dc --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/03_if_case_multiline_string.sh.nopos.token @@ -0,0 +1,27 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: git-sh-setup.sh:82-96 @ 18e66859 +keyword|if +op|[ +word|-z +literal|"$LONG_USAGE" +op|] +keyword|then +variable|LONG_USAGE +op|= +literal|"$(eval_gettext "usage: \$dashless \$USAGE")" +keyword|else +variable|LONG_USAGE +op|= +literal|"$(eval_gettext "usage: \$dashless \$USAGE $LONG_USAGE")" +keyword|fi +keyword|case +literal|"$1" +keyword|in +word|-h +op|) +command|echo +literal|"$LONG_USAGE" +command|exit +keyword|esac +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/03_if_case_multiline_string.sh.token b/tokenize/shTokenizer/tests/expected/03_if_case_multiline_string.sh.token new file mode 100644 index 00000000..ad01a201 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/03_if_case_multiline_string.sh.token @@ -0,0 +1,27 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: git-sh-setup.sh:82-96 @ 18e66859 +3:2|keyword|if +3:5|op|[ +3:7|word|-z +3:10|literal|"$LONG_USAGE" +3:24|op|] +4:2|keyword|then +5:3|variable|LONG_USAGE +5:13|op|= +5:14|literal|"$(eval_gettext "usage: \$dashless \$USAGE")" +6:2|keyword|else +7:3|variable|LONG_USAGE +7:13|op|= +7:14|literal|"$(eval_gettext "usage: \$dashless \$USAGE $LONG_USAGE")" +10:2|keyword|fi +12:2|keyword|case +12:7|literal|"$1" +12:12|keyword|in +13:3|word|-h +13:5|op|) +14:3|command|echo +14:8|literal|"$LONG_USAGE" +15:3|command|exit +16:2|keyword|esac +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/04_heredoc_operator_in_string.sh.nopos.token b/tokenize/shTokenizer/tests/expected/04_heredoc_operator_in_string.sh.nopos.token new file mode 100644 index 00000000..bda7e582 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/04_heredoc_operator_in_string.sh.nopos.token @@ -0,0 +1,44 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: t/lib-t6000.sh:125-135 @ 18e66859 +begin_function +DECL|function|test_output_expect_success +word|test_output_expect_success +op|( +op|) +op|{ +variable|_description +op|= +op|$ +variable|1 +variable|_test +op|= +op|$ +variable|2 +command|test +op|$ +variable|# +word|-eq +word|2 +op||| +command|error +literal|"usage: test_output_expect_success description test < +literal|"$_name.expected" +command|test_expect_success +literal|"$_description" +literal|"check_output $_name \"$_test\"" +op|} +end_function + +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/04_heredoc_operator_in_string.sh.token b/tokenize/shTokenizer/tests/expected/04_heredoc_operator_in_string.sh.token new file mode 100644 index 00000000..ab1b28c8 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/04_heredoc_operator_in_string.sh.token @@ -0,0 +1,44 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: t/lib-t6000.sh:125-135 @ 18e66859 +-:-|begin_function +-:-|DECL|function|test_output_expect_success +2:1|word|test_output_expect_success +2:27|op|( +2:28|op|) +3:1|op|{ +4:2|variable|_description +4:14|op|= +4:15|op|$ +4:16|variable|1 +5:2|variable|_test +5:7|op|= +5:8|op|$ +5:9|variable|2 +6:2|command|test +6:7|op|$ +6:8|variable|# +6:10|word|-eq +6:14|word|2 +6:16|op||| +7:2|command|error +7:8|literal|"usage: test_output_expect_success description test < +10:7|literal|"$_name.expected" +11:2|command|test_expect_success +11:22|literal|"$_description" +11:38|literal|"check_output $_name \"$_test\"" +12:1|op|} +-:-|end_function +-:-| +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/05_heredoc_with_heredoc_text.sh.nopos.token b/tokenize/shTokenizer/tests/expected/05_heredoc_with_heredoc_text.sh.nopos.token new file mode 100644 index 00000000..6828aa62 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/05_heredoc_with_heredoc_text.sh.nopos.token @@ -0,0 +1,11 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: t/t0600-reffiles-backend.sh:93-110 @ 18e66859 +command|test_expect_success +literal|'non-empty directory blocks create' +word|- +op|<< +heredoc_delim|\EOT +heredoc|prefix=refs/ne-create && mkdir -p .git/$prefix/foo/bar && : >.git/$prefix/foo/bar/baz.lock && test_when_finished "rm -f .git/$prefix/foo/bar/baz.lock" && cat >expected <<-EOF && fatal: cannot lock ref '$prefix/foo': there is a non-empty directory '.git/$prefix/foo' blocking reference '$prefix/foo' EOF printf "%s\n" "update $prefix/foo $C" | test_must_fail git update-ref --stdin 2>output.err && test_cmp expected output.err && cat >expected <<-EOF && fatal: cannot lock ref '$prefix/foo': unable to resolve reference '$prefix/foo' EOF printf "%s\n" "update $prefix/foo $D $C" | test_must_fail git update-ref --stdin 2>output.err && test_cmp expected output.err +heredoc_delim|EOT +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/05_heredoc_with_heredoc_text.sh.token b/tokenize/shTokenizer/tests/expected/05_heredoc_with_heredoc_text.sh.token new file mode 100644 index 00000000..ec2bb454 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/05_heredoc_with_heredoc_text.sh.token @@ -0,0 +1,11 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: t/t0600-reffiles-backend.sh:93-110 @ 18e66859 +2:1|command|test_expect_success +2:21|literal|'non-empty directory blocks create' +2:57|word|- +2:59|op|<< +2:61|heredoc_delim|\EOT +3:2|heredoc|prefix=refs/ne-create && mkdir -p .git/$prefix/foo/bar && : >.git/$prefix/foo/bar/baz.lock && test_when_finished "rm -f .git/$prefix/foo/bar/baz.lock" && cat >expected <<-EOF && fatal: cannot lock ref '$prefix/foo': there is a non-empty directory '.git/$prefix/foo' blocking reference '$prefix/foo' EOF printf "%s\n" "update $prefix/foo $C" | test_must_fail git update-ref --stdin 2>output.err && test_cmp expected output.err && cat >expected <<-EOF && fatal: cannot lock ref '$prefix/foo': unable to resolve reference '$prefix/foo' EOF printf "%s\n" "update $prefix/foo $D $C" | test_must_fail git update-ref --stdin 2>output.err && test_cmp expected output.err +19:1|heredoc_delim|EOT +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/06_heredoc_in_substitution.sh.nopos.token b/tokenize/shTokenizer/tests/expected/06_heredoc_in_substitution.sh.nopos.token new file mode 100644 index 00000000..7347ac64 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/06_heredoc_in_substitution.sh.nopos.token @@ -0,0 +1,68 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: contrib/rerere-train.sh:1-39 @ 18e66859 +comment|#!/bin/sh +comment|# Copyright (c) 2008, Nanako Shiraishi +comment|# Prime rerere database from existing merge commits +variable|me +op|= +word|rerere-train +variable|USAGE +op|= +op|$( +command|cat +op|<<- +heredoc_delim|EOF +heredoc|usage: $me [--overwrite] -h, --help show the help -o, --overwrite overwrite any existing rerere cache +heredoc_delim|EOF +op|) +variable|SUBDIRECTORY_OK +op|= +word|Yes +variable|overwrite +op|= +word|0 +keyword|while +command|test +op|$ +variable|# +word|-gt +word|0 +keyword|do +variable|opt +op|= +literal|"$1" +keyword|case +literal|"$opt" +keyword|in +word|-h +op|| +word|--help +op|) +command|echo +literal|"$USAGE" +command|exit +word|0 +op|;; +word|-o +op|| +word|--overwrite +op|) +variable|overwrite +op|= +word|1 +command|shift +command|break +op|;; +word|-- +op|) +command|shift +command|break +op|;; +word|* +op|) +command|break +op|;; +keyword|esac +keyword|done +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/06_heredoc_in_substitution.sh.token b/tokenize/shTokenizer/tests/expected/06_heredoc_in_substitution.sh.token new file mode 100644 index 00000000..17ef34c3 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/06_heredoc_in_substitution.sh.token @@ -0,0 +1,68 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: contrib/rerere-train.sh:1-39 @ 18e66859 +2:1|comment|#!/bin/sh +3:1|comment|# Copyright (c) 2008, Nanako Shiraishi +4:1|comment|# Prime rerere database from existing merge commits +6:1|variable|me +6:3|op|= +6:4|word|rerere-train +7:1|variable|USAGE +7:6|op|= +7:7|op|$( +7:9|command|cat +7:13|op|<<- +7:16|heredoc_delim|EOF +8:1|heredoc|usage: $me [--overwrite] -h, --help show the help -o, --overwrite overwrite any existing rerere cache +12:1|heredoc_delim|EOF +13:1|op|) +15:1|variable|SUBDIRECTORY_OK +15:16|op|= +15:17|word|Yes +17:1|variable|overwrite +17:10|op|= +17:11|word|0 +19:1|keyword|while +19:7|command|test +19:12|op|$ +19:13|variable|# +19:15|word|-gt +19:19|word|0 +20:1|keyword|do +21:2|variable|opt +21:5|op|= +21:6|literal|"$1" +22:2|keyword|case +22:7|literal|"$opt" +22:14|keyword|in +23:2|word|-h +23:4|op|| +23:5|word|--help +23:11|op|) +24:3|command|echo +24:8|literal|"$USAGE" +25:3|command|exit +25:8|word|0 +26:3|op|;; +27:2|word|-o +27:4|op|| +27:5|word|--overwrite +27:16|op|) +28:3|variable|overwrite +28:12|op|= +28:13|word|1 +29:3|command|shift +30:3|command|break +31:3|op|;; +32:2|word|-- +32:4|op|) +33:3|command|shift +34:3|command|break +35:3|op|;; +36:2|word|* +36:3|op|) +37:3|command|break +38:3|op|;; +39:2|keyword|esac +40:1|keyword|done +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/07_toplevel_heredoc_perl.sh.nopos.token b/tokenize/shTokenizer/tests/expected/07_toplevel_heredoc_perl.sh.nopos.token new file mode 100644 index 00000000..00b4d985 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/07_toplevel_heredoc_perl.sh.nopos.token @@ -0,0 +1,12 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: t/t9500-gitweb-standalone-no-errors.sh:739-741 @ 18e66859 +command|cat +op|>> +word|gitweb_config.perl +op|<< +heredoc_delim|\EOF +op|&& +heredoc|$feature{'forks'}{'default'} = [1]; +heredoc_delim|EOF +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/07_toplevel_heredoc_perl.sh.token b/tokenize/shTokenizer/tests/expected/07_toplevel_heredoc_perl.sh.token new file mode 100644 index 00000000..f63f5eeb --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/07_toplevel_heredoc_perl.sh.token @@ -0,0 +1,12 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: t/t9500-gitweb-standalone-no-errors.sh:739-741 @ 18e66859 +2:1|command|cat +2:5|op|>> +2:7|word|gitweb_config.perl +2:26|op|<< +2:28|heredoc_delim|\EOF +2:33|op|&& +3:1|heredoc|$feature{'forks'}{'default'} = [1]; +4:1|heredoc_delim|EOF +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/08_arithmetic.sh.nopos.token b/tokenize/shTokenizer/tests/expected/08_arithmetic.sh.nopos.token new file mode 100644 index 00000000..517a892d --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/08_arithmetic.sh.nopos.token @@ -0,0 +1,18 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: t/test-lib-functions.sh:543-544 @ 18e66859 +variable|n +op|= +op|$(( +variable|n +op|+ +number|1 +op|)) +variable|total +op|= +op|$(( +variable|total +op|- +number|1 +op|)) +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/08_arithmetic.sh.token b/tokenize/shTokenizer/tests/expected/08_arithmetic.sh.token new file mode 100644 index 00000000..7314d567 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/08_arithmetic.sh.token @@ -0,0 +1,18 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: t/test-lib-functions.sh:543-544 @ 18e66859 +2:3|variable|n +2:4|op|= +2:5|op|$(( +2:8|variable|n +2:10|op|+ +2:12|number|1 +2:13|op|)) +3:3|variable|total +3:8|op|= +3:9|op|$(( +3:12|variable|total +3:18|op|- +3:20|number|1 +3:21|op|)) +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/09_line_continuation.sh.nopos.token b/tokenize/shTokenizer/tests/expected/09_line_continuation.sh.nopos.token new file mode 100644 index 00000000..fdcfee78 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/09_line_continuation.sh.nopos.token @@ -0,0 +1,17 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: t/test-lib-functions.sh:547-549 @ 18e66859 +command|git +word|-C +literal|"$indir" +op|\ +word|-c +word|fastimport.unpacklimit=0 +op|\ +word|fast-import +op|< +literal|"$tmpfile" +op||| +command|return +word|1 +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/09_line_continuation.sh.token b/tokenize/shTokenizer/tests/expected/09_line_continuation.sh.token new file mode 100644 index 00000000..7970af6b --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/09_line_continuation.sh.token @@ -0,0 +1,17 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: t/test-lib-functions.sh:547-549 @ 18e66859 +2:2|command|git +2:6|word|-C +2:9|literal|"$indir" +2:18|op|\ +3:6|word|-c +3:9|word|fastimport.unpacklimit=0 +3:34|op|\ +4:6|word|fast-import +4:18|op|< +4:19|literal|"$tmpfile" +4:30|op||| +4:33|command|return +4:40|word|1 +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/10_file_header.sh.nopos.token b/tokenize/shTokenizer/tests/expected/10_file_header.sh.nopos.token new file mode 100644 index 00000000..637cf132 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/10_file_header.sh.nopos.token @@ -0,0 +1,13 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: t/t1400-update-ref.sh:1-8 @ 18e66859 +comment|#!/bin/sh +comment|# +comment|# Copyright (c) 2006 Shawn Pearce +comment|# +variable|test_description +op|= +literal|'Test git update-ref and basic ref logging' +command|. +word|./test-lib.sh +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/10_file_header.sh.token b/tokenize/shTokenizer/tests/expected/10_file_header.sh.token new file mode 100644 index 00000000..71fbb4fb --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/10_file_header.sh.token @@ -0,0 +1,13 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: t/t1400-update-ref.sh:1-8 @ 18e66859 +2:1|comment|#!/bin/sh +3:1|comment|# +4:1|comment|# Copyright (c) 2006 Shawn Pearce +5:1|comment|# +7:1|variable|test_description +7:17|op|= +7:18|literal|'Test git update-ref and basic ref logging' +9:1|command|. +9:3|word|./test-lib.sh +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/11_template_placeholder.sh.nopos.token b/tokenize/shTokenizer/tests/expected/11_template_placeholder.sh.nopos.token new file mode 100644 index 00000000..fcb40525 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/11_template_placeholder.sh.nopos.token @@ -0,0 +1,60 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: git-sh-setup.sh:279-289 @ 18e66859 +comment|# Generate a virtual base file for a two-file merge. Uses git apply to +comment|# remove lines from $1 that are not in $2, leaving only common lines. +begin_function +DECL|function|create_virtual_base +word|create_virtual_base +op|( +op|) +op|{ +variable|sz0 +op|= +op|$( +command|wc +word|-c +op|< +literal|"$1" +op|) +word|@DIFF@ +word|-u +word|- +word|La/ +literal|"$1" +word|-Lb/ +literal|"$1" +literal|"$1" +literal|"$2" +op|| +command|git +word|apply +word|--no-add +variable|sz1 +op|= +op|$( +command|wc +word|-c +op|< +literal|"$1" +op|) +comment|# If we do not have enough common material, it is not +comment|# worth trying two-file merge using common subsections. +command|expr +op|$ +variable|sz0 +word|\< +op|$ +variable|sz1 +word|\* +word|2 +op|> +word|/dev/null +op||| +command|: +op|> +literal|"$1" +op|} +end_function + +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/11_template_placeholder.sh.token b/tokenize/shTokenizer/tests/expected/11_template_placeholder.sh.token new file mode 100644 index 00000000..93dbd391 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/11_template_placeholder.sh.token @@ -0,0 +1,60 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: git-sh-setup.sh:279-289 @ 18e66859 +2:1|comment|# Generate a virtual base file for a two-file merge. Uses git apply to +3:1|comment|# remove lines from $1 that are not in $2, leaving only common lines. +-:-|begin_function +-:-|DECL|function|create_virtual_base +4:1|word|create_virtual_base +4:20|op|( +4:21|op|) +4:23|op|{ +5:2|variable|sz0 +5:5|op|= +5:6|op|$( +5:8|command|wc +5:11|word|-c +5:14|op|< +5:15|literal|"$1" +5:19|op|) +6:2|word|@DIFF@ +6:9|word|-u +6:12|word|- +6:13|word|La/ +6:16|literal|"$1" +6:21|word|-Lb/ +6:25|literal|"$1" +6:30|literal|"$1" +6:35|literal|"$2" +6:40|op|| +6:42|command|git +6:46|word|apply +6:52|word|--no-add +7:2|variable|sz1 +7:5|op|= +7:6|op|$( +7:8|command|wc +7:11|word|-c +7:14|op|< +7:15|literal|"$1" +7:19|op|) +9:2|comment|# If we do not have enough common material, it is not +10:2|comment|# worth trying two-file merge using common subsections. +11:2|command|expr +11:7|op|$ +11:8|variable|sz0 +11:12|word|\< +11:15|op|$ +11:16|variable|sz1 +11:20|word|\* +11:23|word|2 +11:25|op|> +11:26|word|/dev/null +11:36|op||| +11:39|command|: +11:41|op|> +11:42|literal|"$1" +12:1|op|} +-:-|end_function +-:-| +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/expected/12_non_utf8.sh.nopos.token b/tokenize/shTokenizer/tests/expected/12_non_utf8.sh.nopos.token new file mode 100644 index 00000000..789bd877 --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/12_non_utf8.sh.nopos.token @@ -0,0 +1,12 @@ +begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +comment|# fixture: t/t7813-grep-icase-iso.sh:7-17 @ 18e66859 +command|test_expect_success +word|GETTEXT_ISO_LOCALE +literal|'setup' +literal|' printf "TILRAUN: Halló Heimur!" >file && git add file && LC_ALL="$is_IS_iso_locale" && export LC_ALL ' +command|test_expect_success +word|GETTEXT_ISO_LOCALE,PCRE +literal|'grep pcre string' +literal|' git grep --perl-regexp -i "TILRAUN: H.lló Heimur!" && git grep --perl-regexp -i "TILRAUN: H.LLÓ HEIMUR!" ' +end_unit + diff --git a/tokenize/shTokenizer/tests/expected/12_non_utf8.sh.token b/tokenize/shTokenizer/tests/expected/12_non_utf8.sh.token new file mode 100644 index 00000000..88052e7e --- /dev/null +++ b/tokenize/shTokenizer/tests/expected/12_non_utf8.sh.token @@ -0,0 +1,12 @@ +-:-|begin_unit|revision:0.1.0;language:Sh;cregit-version:0.0.1 +1:1|comment|# fixture: t/t7813-grep-icase-iso.sh:7-17 @ 18e66859 +2:1|command|test_expect_success +2:21|word|GETTEXT_ISO_LOCALE +2:40|literal|'setup' +2:48|literal|' printf "TILRAUN: Halló Heimur!" >file && git add file && LC_ALL="$is_IS_iso_locale" && export LC_ALL ' +9:1|command|test_expect_success +9:21|word|GETTEXT_ISO_LOCALE,PCRE +9:45|literal|'grep pcre string' +9:64|literal|' git grep --perl-regexp -i "TILRAUN: H.lló Heimur!" && git grep --perl-regexp -i "TILRAUN: H.LLÓ HEIMUR!" ' +-:-|end_unit +-:-| diff --git a/tokenize/shTokenizer/tests/fixtures/01_test_body.sh b/tokenize/shTokenizer/tests/fixtures/01_test_body.sh new file mode 100644 index 00000000..042da8a6 --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/01_test_body.sh @@ -0,0 +1,4 @@ +# fixture: t/t0000-basic.sh:66-68 @ 18e66859 +test_expect_success 'success is reported like this' ' + : +' diff --git a/tokenize/shTokenizer/tests/fixtures/02_functions.sh b/tokenize/shTokenizer/tests/fixtures/02_functions.sh new file mode 100644 index 00000000..69e87925 --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/02_functions.sh @@ -0,0 +1,11 @@ +# fixture: git-sh-setup.sh:49-58 @ 18e66859 +die () { + die_with_status 1 "$@" +} + +die_with_status () { + status=$1 + shift + printf >&2 '%s\n' "$*" + exit "$status" +} diff --git a/tokenize/shTokenizer/tests/fixtures/03_if_case_multiline_string.sh b/tokenize/shTokenizer/tests/fixtures/03_if_case_multiline_string.sh new file mode 100644 index 00000000..32cd7e73 --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/03_if_case_multiline_string.sh @@ -0,0 +1,16 @@ +# fixture: git-sh-setup.sh:82-96 @ 18e66859 + + if [ -z "$LONG_USAGE" ] + then + LONG_USAGE="$(eval_gettext "usage: \$dashless \$USAGE")" + else + LONG_USAGE="$(eval_gettext "usage: \$dashless \$USAGE + +$LONG_USAGE")" + fi + + case "$1" in + -h) + echo "$LONG_USAGE" + exit + esac diff --git a/tokenize/shTokenizer/tests/fixtures/04_heredoc_operator_in_string.sh b/tokenize/shTokenizer/tests/fixtures/04_heredoc_operator_in_string.sh new file mode 100644 index 00000000..f780bda6 --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/04_heredoc_operator_in_string.sh @@ -0,0 +1,12 @@ +# fixture: t/lib-t6000.sh:125-135 @ 18e66859 +test_output_expect_success() +{ + _description=$1 + _test=$2 + test $# -eq 2 || + error "usage: test_output_expect_success description test <"$_name.expected" + test_expect_success "$_description" "check_output $_name \"$_test\"" +} diff --git a/tokenize/shTokenizer/tests/fixtures/05_heredoc_with_heredoc_text.sh b/tokenize/shTokenizer/tests/fixtures/05_heredoc_with_heredoc_text.sh new file mode 100644 index 00000000..1aef4815 --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/05_heredoc_with_heredoc_text.sh @@ -0,0 +1,19 @@ +# fixture: t/t0600-reffiles-backend.sh:93-110 @ 18e66859 +test_expect_success 'non-empty directory blocks create' - <<\EOT + prefix=refs/ne-create && + mkdir -p .git/$prefix/foo/bar && + : >.git/$prefix/foo/bar/baz.lock && + test_when_finished "rm -f .git/$prefix/foo/bar/baz.lock" && + cat >expected <<-EOF && + fatal: cannot lock ref '$prefix/foo': there is a non-empty directory '.git/$prefix/foo' blocking reference '$prefix/foo' + EOF + printf "%s\n" "update $prefix/foo $C" | + test_must_fail git update-ref --stdin 2>output.err && + test_cmp expected output.err && + cat >expected <<-EOF && + fatal: cannot lock ref '$prefix/foo': unable to resolve reference '$prefix/foo' + EOF + printf "%s\n" "update $prefix/foo $D $C" | + test_must_fail git update-ref --stdin 2>output.err && + test_cmp expected output.err +EOT diff --git a/tokenize/shTokenizer/tests/fixtures/06_heredoc_in_substitution.sh b/tokenize/shTokenizer/tests/fixtures/06_heredoc_in_substitution.sh new file mode 100644 index 00000000..111b3346 --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/06_heredoc_in_substitution.sh @@ -0,0 +1,40 @@ +# fixture: contrib/rerere-train.sh:1-39 @ 18e66859 +#!/bin/sh +# Copyright (c) 2008, Nanako Shiraishi +# Prime rerere database from existing merge commits + +me=rerere-train +USAGE=$(cat <<-EOF +usage: $me [--overwrite] + + -h, --help show the help + -o, --overwrite overwrite any existing rerere cache +EOF +) + +SUBDIRECTORY_OK=Yes + +overwrite=0 + +while test $# -gt 0 +do + opt="$1" + case "$opt" in + -h|--help) + echo "$USAGE" + exit 0 + ;; + -o|--overwrite) + overwrite=1 + shift + break + ;; + --) + shift + break + ;; + *) + break + ;; + esac +done diff --git a/tokenize/shTokenizer/tests/fixtures/07_toplevel_heredoc_perl.sh b/tokenize/shTokenizer/tests/fixtures/07_toplevel_heredoc_perl.sh new file mode 100644 index 00000000..e1dba997 --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/07_toplevel_heredoc_perl.sh @@ -0,0 +1,4 @@ +# fixture: t/t9500-gitweb-standalone-no-errors.sh:739-741 @ 18e66859 +cat >>gitweb_config.perl <<\EOF && +$feature{'forks'}{'default'} = [1]; +EOF diff --git a/tokenize/shTokenizer/tests/fixtures/08_arithmetic.sh b/tokenize/shTokenizer/tests/fixtures/08_arithmetic.sh new file mode 100644 index 00000000..30a5fd8f --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/08_arithmetic.sh @@ -0,0 +1,3 @@ +# fixture: t/test-lib-functions.sh:543-544 @ 18e66859 + n=$((n + 1)) + total=$((total - 1)) diff --git a/tokenize/shTokenizer/tests/fixtures/09_line_continuation.sh b/tokenize/shTokenizer/tests/fixtures/09_line_continuation.sh new file mode 100644 index 00000000..fe006013 --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/09_line_continuation.sh @@ -0,0 +1,4 @@ +# fixture: t/test-lib-functions.sh:547-549 @ 18e66859 + git -C "$indir" \ + -c fastimport.unpacklimit=0 \ + fast-import <"$tmpfile" || return 1 diff --git a/tokenize/shTokenizer/tests/fixtures/10_file_header.sh b/tokenize/shTokenizer/tests/fixtures/10_file_header.sh new file mode 100644 index 00000000..b17cfe6e --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/10_file_header.sh @@ -0,0 +1,9 @@ +# fixture: t/t1400-update-ref.sh:1-8 @ 18e66859 +#!/bin/sh +# +# Copyright (c) 2006 Shawn Pearce +# + +test_description='Test git update-ref and basic ref logging' + +. ./test-lib.sh diff --git a/tokenize/shTokenizer/tests/fixtures/11_template_placeholder.sh b/tokenize/shTokenizer/tests/fixtures/11_template_placeholder.sh new file mode 100644 index 00000000..7cd5e1c1 --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/11_template_placeholder.sh @@ -0,0 +1,12 @@ +# fixture: git-sh-setup.sh:279-289 @ 18e66859 +# Generate a virtual base file for a two-file merge. Uses git apply to +# remove lines from $1 that are not in $2, leaving only common lines. +create_virtual_base() { + sz0=$(wc -c <"$1") + @DIFF@ -u -La/"$1" -Lb/"$1" "$1" "$2" | git apply --no-add + sz1=$(wc -c <"$1") + + # If we do not have enough common material, it is not + # worth trying two-file merge using common subsections. + expr $sz0 \< $sz1 \* 2 >/dev/null || : >"$1" +} diff --git a/tokenize/shTokenizer/tests/fixtures/12_non_utf8.sh b/tokenize/shTokenizer/tests/fixtures/12_non_utf8.sh new file mode 100644 index 00000000..74833733 --- /dev/null +++ b/tokenize/shTokenizer/tests/fixtures/12_non_utf8.sh @@ -0,0 +1,12 @@ +# fixture: t/t7813-grep-icase-iso.sh:7-17 @ 18e66859 +test_expect_success GETTEXT_ISO_LOCALE 'setup' ' + printf "TILRAUN: Halló Heimur!" >file && + git add file && + LC_ALL="$is_IS_iso_locale" && + export LC_ALL +' + +test_expect_success GETTEXT_ISO_LOCALE,PCRE 'grep pcre string' ' + git grep --perl-regexp -i "TILRAUN: H.lló Heimur!" && + git grep --perl-regexp -i "TILRAUN: H.LLÓ HEIMUR!" +' diff --git a/tokenize/shTokenizer/tests/test_sh_tokenizer.py b/tokenize/shTokenizer/tests/test_sh_tokenizer.py new file mode 100644 index 00000000..b374550c --- /dev/null +++ b/tokenize/shTokenizer/tests/test_sh_tokenizer.py @@ -0,0 +1,272 @@ +import contextlib +import io +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +sys.path.insert(0, str(ROOT)) + +import sh_tokenizer # noqa: E402 + +FIXTURES = HERE / "fixtures" +EXPECTED = HERE / "expected" +SCRIPT = ROOT / "sh_tokenizer.py" + + +def tokenize(src: bytes) -> bytes: + return sh_tokenizer.render(src, sh_tokenizer.tokenize(src), False) + + +def token_lines(src: bytes) -> list[bytes]: + out = tokenize(src).split(b"\n") + return [line for line in out[1:] if b"|" in line and not line.startswith(b"DECL|")] + + +def is_valid(src: bytes) -> bool: + return sh_tokenizer.check(src, list(sh_tokenizer.Tokenizer(src).run().records())) is None + + +class GoldenFixturesTest(unittest.TestCase): + + def test_fixtures_match_expected_output(self): + for fixture in sorted(FIXTURES.glob("*.sh")): + src = fixture.read_bytes() + records = sh_tokenizer.tokenize(src) + for suffix, with_position in ((".token", True), (".nopos.token", False)): + expected = EXPECTED / (fixture.name + suffix) + with self.subTest(expected.name): + self.assertEqual(sh_tokenizer.render(src, records, with_position), expected.read_bytes()) + + +class HeredocScannerTest(unittest.TestCase): + + def bodies(self, src: bytes) -> list[bytes]: + return [src[h.body_start:h.body_end] for h in sh_tokenizer.scan_heredocs(src)] + + def test_heredoc_followed_by_and_and(self): + self.assertEqual(self.bodies(b"cat >x <<-\\EOF &&\n\ta\n\tEOF\necho\n"), [b"\ta\n"]) + + def test_several_heredocs_on_one_line(self): + src = b"cmd <x <<-\\EOF &&\n\tif true; then echo 'a b'; fi\n\tEOF\necho\n" + self.assertEqual(token_lines(src), [ + b"command|cat", b"op|>", b"word|x", b"op|<<-", b"heredoc_delim|\\EOF", b"op|&&", + b"heredoc|if true; then echo 'a b'; fi", b"heredoc_delim|EOF", b"command|echo", + ]) + + def test_heredoc_inside_a_string_cuts_it(self): + src = b'x="$(cat <<-EOF\n\tbody\n\tEOF\n)" &&\necho\n' + self.assertEqual(token_lines(src), [ + b"variable|x", b"op|=", b'literal|"$(cat', b"op|<<-", b"heredoc_delim|EOF", + b"heredoc|body", b"heredoc_delim|EOF", b'literal|)"', b"op|&&", b"command|echo", + ]) + + def test_empty_heredoc_body_has_no_token(self): + self.assertEqual(token_lines(b"cat <&2 2>e\ny=$((1 + 2))\n"), [ + b"command|exit", b"word|1", + b"command|echo", b"word|x", b"op|>&", b"number|2", b"number|2", b"op|>", b"word|e", + b"variable|y", b"op|=", b"op|$((", b"number|1", b"op|+", b"number|2", b"op|))", + ]) + self.assertEqual(token_lines(b"[ $x -eq 1 ]\n[[ $x -eq 1 ]]\ntest $x -eq 1\n"), [ + b"op|[", b"op|$", b"variable|x", b"word|-eq", b"word|1", b"op|]", + b"op|[[", b"op|$", b"variable|x", b"word|-eq", b"word|1", b"op|]]", + b"command|test", b"op|$", b"variable|x", b"word|-eq", b"word|1", + ]) + self.assertEqual(token_lines(b"(( 2 ))\nfor ((i = 0; i < 3; i++)); do echo 4; done\n"), [ + b"op|((", b"number|2", b"op|))", + b"keyword|for", b"op|((", b"variable|i", b"op|=", b"number|0", b"op|;", b"word|i", b"op|<", + b"number|3", b"op|;", b"word|i", b"op|++", b"op|))", b"op|;", + b"keyword|do", b"command|echo", b"word|4", b"op|;", b"keyword|done", + ]) + + def test_line_continuation(self): + self.assertEqual(token_lines(b"a \\\n b\n"), [b"command|a", b"op|\\", b"word|b"]) + + def test_positions(self): + out = sh_tokenizer.render(b"a\n b\n", sh_tokenizer.tokenize(b"a\n b\n"), True).split(b"\n") + self.assertIn(b"1:1|command|a", out) + self.assertIn(b"2:3|command|b", out) + self.assertEqual(out[0][:4], b"-:-|") + + def test_positions_count_characters(self): + # An invalid byte counts as one character. + for src in ("echo é x\n".encode(), b"echo \xe9 x\n"): + with self.subTest(src): + out = sh_tokenizer.render(src, sh_tokenizer.tokenize(src), True).split(b"\n") + self.assertIn(b"1:8|word|x", out) + + def test_word_with_invalid_utf8_is_one_token(self): + self.assertEqual(token_lines(b"echo \xe2\x82 b\n"), [b"command|echo", b"word|\xe2\x82", b"word|b"]) + self.assertEqual(token_lines(b"echo a\xe2\x82b c\n"), [b"command|echo", b"word|a\xe2\x82b", b"word|c"]) + self.assertEqual(token_lines(b"echo a\xe9$y\xe9b\n"), + [b"command|echo", b"word|a\xe9", b"op|$", b"variable|y", b"word|\xe9b"]) + + def test_form_feed_and_vertical_tab_are_kept(self): + # tree-sitter-bash still cuts a word at them. + self.assertEqual(token_lines(b"echo \x0c x\x0by\n"), + [b"command|echo", b"word|\x0c", b"word|x", b"word|\x0b", b"word|y"]) + + def test_garbage_never_fails(self): + for src in (b"", b"\n\n", b"if then fi fi ((( ''' \"", b"cat <<\n", b"\xff\xfe\x00 \x80", + b"echo 'x", b"f() {"): + with self.subTest(src): + out = tokenize(src) + self.assertTrue(out.startswith(sh_tokenizer.UNIT_HEADER)) + self.assertTrue(out.endswith(b"end_unit\n\n")) + self.assertNotIn(b"\nunknown|", out) + self.assertTrue(is_valid(src)) + + def test_loose_words_inside_a_syntax_error(self): + self.assertEqual(token_lines(b"case 'a b' in\n"), [b"word|case", b"literal|'a b'", b"word|in"]) + self.assertIn(b"word|-u", token_lines(b"if then -u fi\n")) + + def test_syntax_errors_around_heredocs_keep_the_output_valid(self): + # Fuzzing inputs: error recovery ends a node inside a body, or adds a MISSING node. + for src in (b'e(){case"1" in)ro<&2&&"): + with self.subTest(src): + self.assertTrue(is_valid(src)) + + def test_fallback_is_a_word_split(self): + src = b"echo 'a b'\n" + failures = {"check fails": mock.patch.object(sh_tokenizer, "check", return_value="forced"), + "tokenizer raises": mock.patch.object(sh_tokenizer.Tokenizer, "run", side_effect=RuntimeError)} + for label, failure in failures.items(): + with self.subTest(label): + with failure, contextlib.redirect_stderr(io.StringIO()) as err: + lines = token_lines(src) + self.assertEqual(lines, [b"unknown|echo", b"unknown|'a", b"unknown|b'"]) + self.assertIn("falling back to word split", err.getvalue()) + + +class CommandLineTest(unittest.TestCase): + + def run_cli(self, *args): + return subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True) + + def test_dispatcher_style_call(self): + fixture = FIXTURES / "02_functions.sh" + result = self.run_cli("--language=Sh", str(fixture)) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, (EXPECTED / "02_functions.sh.nopos.token").read_bytes()) + + def test_output_argument(self): + fixture = FIXTURES / "02_functions.sh" + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / "out.token" + result = self.run_cli(str(fixture), str(output)) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, b"") + self.assertEqual(output.read_bytes(), (EXPECTED / "02_functions.sh.nopos.token").read_bytes()) + + def test_position_option(self): + result = self.run_cli("--position", str(FIXTURES / "12_non_utf8.sh")) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, (EXPECTED / "12_non_utf8.sh.token").read_bytes()) + + def test_library_paths(self): + result = self.run_cli("--library-paths") + self.assertEqual(result.returncode, 0) + paths = result.stdout.decode().split() + self.assertEqual(len(paths), 2) + for path in paths: + self.assertTrue(Path(path).is_file(), path) + + def test_unreadable_file(self): + result = self.run_cli(str(HERE / "does-not-exist.sh")) + self.assertEqual(result.returncode, 1) + self.assertIn(b"cannot read", result.stderr) + + def test_wrong_language(self): + self.assertEqual(self.run_cli("--language=C", str(FIXTURES / "02_functions.sh")).returncode, 2) + + def test_usage_errors(self): + fixture = str(FIXTURES / "02_functions.sh") + for args in (("--bogus", fixture), (fixture, os.devnull, "extra"), ()): + with self.subTest(args): + result = self.run_cli(*args) + self.assertEqual(result.returncode, 2) + self.assertIn(b"usage:", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tokenize/tokenize.pl b/tokenize/tokenize.pl index ea4737b4..5c8a8633 100755 --- a/tokenize/tokenize.pl +++ b/tokenize/tokenize.pl @@ -42,7 +42,7 @@ Usage $0 [options] * Options: - --language= + --language= --position --srcml= (forwarded to srcML-based parsers only) --srcml2token= (forwarded to srcML-based parsers only) @@ -55,7 +55,7 @@ my $position = 0; # srcML-specific options. Accepted here so a single caller (e.g. BFG_TOKENIZE_CMD) # can pin the binary paths once and have them reach tokenizeSrcMl.pl without knowing -# which language each blob is. For non-srcML parsers (Rust, M4) these are ignored. +# which language each blob is. For non-srcML parsers (Rust, M4, Sh) these are ignored. my $srcmlPath = ""; my $srcml2tokenPath = ""; my $ctagsPath = ""; diff --git a/tokenize/tokenizerIdentity.pl b/tokenize/tokenizerIdentity.pl index af630f37..400c1c16 100755 --- a/tokenize/tokenizerIdentity.pl +++ b/tokenize/tokenizerIdentity.pl @@ -15,7 +15,8 @@ # Prints blobExec's --tokenizer-identity, ext=,...: a digest of each # extension's parser, tokenize.pl, CregitLanguages.pm and, for srcML languages, -# srcml2token, libsrcml, ctags and CregitSrcMl.pm. A change refuses old caches. +# srcml2token, libsrcml, ctags and CregitSrcMl.pm; for sh, the tree-sitter runtime and +# bash grammar that sh_tokenizer.py names. A change refuses old caches. use strict; use warnings; @@ -89,6 +90,21 @@ sub libsrcml_of { return $path; } +# The tree-sitter runtime and the bash grammar decide sh tokens as libsrcml decides +# srcML's, and like srcml2token the tokenizer names the files it loads. +sub sh_libraries_of { + my ($shTokenizer) = @_; + open(my $ph, '-|', $shTokenizer, "--library-paths") + or die "tokenizerIdentity: cannot run [$shTokenizer]: $!\n"; + my @paths = <$ph>; + close $ph; + chomp @paths; + die "tokenizerIdentity: [$shTokenizer] does not answer --library-paths, so the tree-sitter\n" + . " libraries its tokens depend on cannot be digested. Run it inside devenv shell.\n" + if $? != 0 or not @paths; + return @paths; +} + my %digestCache; sub cached_digest { my ($path) = @_; @@ -98,6 +114,7 @@ sub cached_digest { my @pairs; my $libsrcml; +my $shLibraries; for my $ext (@extensions) { my $language = $CregitLanguages::EXT_LANG{$ext}; die "tokenizerIdentity: no language for extension [$ext]\n" unless defined $language; @@ -118,6 +135,10 @@ sub cached_digest { $libsrcml //= libsrcml_of($srcml2tokenPath); push @components, $libsrcml, "$basedir/CregitSrcMl.pm"; } + if ($language eq "Sh") { + $shLibraries //= [sh_libraries_of($parser)]; + push @components, @$shLibraries; + } my $combined = sha256_hex(join("\n", map { cached_digest($_) } @components)); push @pairs, "$ext=$combined";