diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efb85c2..b54f0b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,8 +18,8 @@ jobs: run: cargo fmt --check - name: Lint run: cargo clippy --all-targets -- -D warnings - - name: Lint with self-update - run: cargo clippy --all-targets --features self-update -- -D warnings + - name: Lint with self-update and fetch + run: cargo clippy --all-targets --features self-update,fetch -- -D warnings - name: Build run: cargo build --verbose # the golden programs are also built for aarch64 and run under qemu-user, the built-in @@ -31,6 +31,8 @@ jobs: run: cargo test --verbose - name: Run self-update tests run: cargo test --verbose --features self-update --bin stone + - name: Run package download tests + run: cargo test --verbose --features fetch --lib --test cli - name: Lint the scripts run: shellcheck install.sh scripts/*.sh - name: Test the install script @@ -86,14 +88,16 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Lint run: cargo clippy --all-targets -- -D warnings - - name: Lint with self-update - run: cargo clippy --all-targets --features self-update -- -D warnings + - name: Lint with self-update and fetch + run: cargo clippy --all-targets --features self-update,fetch -- -D warnings - name: Build run: cargo build --verbose - name: Run tests run: cargo test --verbose --no-fail-fast - name: Run self-update tests run: cargo test --verbose --features self-update --bin stone + - name: Run package download tests + run: cargo test --verbose --features fetch --lib --test cli - name: Test the install script shell: pwsh run: ./scripts/test-install.ps1 diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml deleted file mode 100644 index 24d0823..0000000 --- a/.github/workflows/codacy.yml +++ /dev/null @@ -1,62 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -# This workflow checks out code, performs a Codacy security scan -# and integrates the results with the -# GitHub Advanced Security code scanning feature. For more information on -# the Codacy security scan action usage and parameters, see -# https://github.com/codacy/codacy-analysis-cli-action. -# For more information on Codacy Analysis CLI in general, see -# https://github.com/codacy/codacy-analysis-cli. - -name: Codacy Security Scan - -on: - push: - branches: [ "main" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "main" ] - schedule: - - cron: '18 5 * * 0' - -permissions: - contents: read - -jobs: - codacy-security-scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - name: Codacy Security Scan - # codacy-analysis-cli 4.0.0 uses Docker API 1.39, which ubuntu-26.04's Docker rejects - runs-on: ubuntu-24.04 - steps: - # Checkout the repository to the GitHub Actions runner - - name: Checkout code - uses: actions/checkout@v4 - - # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis - - name: Run Codacy Analysis CLI - uses: codacy/codacy-analysis-cli-action@d840f886c4bd4edc059706d09c6a1586111c540b - with: - # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository - # You can also omit the token and run the tools that support default configurations - project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} - verbose: true - output: results.sarif - format: sarif - # Adjust severity of non-security issues - gh-code-scanning-compat: true - # Force 0 exit code to allow SARIF file generation - # This will handover control about PR rejection to the GitHub side - max-allowed-issues: 2147483647 - - # Upload the SARIF file generated in the previous step - - name: Upload SARIF results file - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: results.sarif diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 972d51d..3991db3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -78,7 +78,7 @@ jobs: - name: Build run: | export PATH="$HOME/.cargo/bin:$PATH" - cargo build --release --locked -p stone --features self-update --target ${{ matrix.target }} + cargo build --release --locked -p stone --features self-update,fetch --target ${{ matrix.target }} - name: Smoke test if: matrix.smoke run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ab88a4..4dc6f17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,28 @@ changelog in `editors/vscode/CHANGELOG.md`. The format follows ### Added +- Packages, first step: a program can depend on other packages through a `stone.toml` next to + its entry file. Each `[dependencies]` entry names a directory for now + (`geo = { path = "../geo" }`), and its files use the package by that name: `use geo.shapes` + loads the package's `shapes.stn`, and `use geo` loads its `lib.stn`. A package's files see + only their own modules and their own dependencies, and two versions of one package with + different major versions can both be used. +- Dependencies can name packages from GitHub and GitLab releases + (`json = { github = "alice/stone-json", version = "1.2.0" }`, or `gitlab` with an optional + `host`), read from the package store in `$STONE_HOME/packages` (`~/.stone/packages`, or + `%LOCALAPPDATA%\stone\packages` on Windows). Each dependency gives the oldest version it works + with, and the newest version any package in the program asks for is used, as Go selects + them, so the same manifests always give the same versions. `stone.lock`, written next to + `stone.toml`, records a hash of each package's files, and a package whose files no longer + match fails to build. +- `stone add github:owner/repo` (or `gitlab:group/project`, a link to either, `--host` for + another GitLab, and `@1.2` for the newest 1.2.x or `@latest`, the default) adds a package's + newest release to `stone.toml` and downloads it with everything it uses; `stone remove`, + `stone fetch [--locked]`, and `stone tree` manage the rest. `run`, `build`, and `check` + download what a program uses but the store lacks, unless given `--offline`. A version is a + release whose tag is the version (`v1.2.0` or `1.2.0`), and only the `stone.toml` and `.stn` + files of its source archive are kept. `GITHUB_TOKEN` and `GITLAB_TOKEN` reach private + repositories. Downloading needs a build with the `fetch` feature, as release builds are. - The `gui` module works on Windows, with Win32 windows, under both `stone run` and in built programs: the same `gui.window` API, events, and `gui.run` as on Linux and macOS. `next_frame` waits for the desktop's next frame, and the window repaints itself from the last diff --git a/CLAUDE.md b/CLAUDE.md index d8f6dea..0145efc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ stone is a small Python-like language implemented in Rust (edition 2024, only de ## Commands - Build: `cargo build` -- Test: `cargo test`. CI also runs `cargo fmt --check` and `cargo clippy --all-targets -- -D warnings` +- Test: `cargo test`. CI also runs `cargo fmt --check` and `cargo clippy --all-targets -- -D warnings`, plus clippy with `--features self-update,fetch` and the package download tests with `cargo test --features fetch --lib --test cli` - Single test: `cargo test ` (e.g. `cargo test simple_functions`). Unit tests live in `#[cfg(test)]` modules in `src/lexer.rs`, `src/ast.rs`, `src/parser/tests.rs`, `src/checker/tests.rs`, `src/project/tests.rs`, `src/lift.rs`, `src/last_use.rs`, `src/unused.rs`, `src/interpreter.rs`, `src/format.rs`, `src/codegen/ir/lower/tests.rs`, `src/codegen/ir/opt.rs`, `src/codegen/ir/inline.rs`, `src/codegen/ir/liveness.rs`, `src/codegen/regalloc.rs`, `src/codegen.rs`, `src/codegen/device.rs`, `src/codegen/asm.rs`, `src/codegen/asm/x64.rs`, `src/codegen/asm/arm64.rs`, `src/codegen/elf.rs`, `src/codegen/macho/tests.rs`, `src/codegen/pe/tests.rs`, `src/codegen/sha256.rs`, `src/codegen/x64.rs`, `src/codegen/arm64.rs`, `src/stdlib.rs`, `src/stdlib/os.rs`, `src/stdlib/math.rs`, `src/stdlib/random.rs`, `src/stdlib/time.rs`, `src/stdlib/fs.rs`, `src/stdlib/net.rs` (over loopback, timeouts included), `src/driver.rs`, `src/repl.rs`, `src/uninstall.rs`, and `src/update.rs` (which needs `--features self-update`). `tests/cli.rs` runs the binary to check error output and exit codes, that built programs need only the system (static on Linux; on macOS, `codesign --verify` passes and `otool -L` lists only libSystem), that `os.hostname()` and `os.cpu_count()` match between backends, that both backends resolve names alike through a fake DNS server on a free UDP port (`dns_reply`, through `STONE_RESOLV_CONF`) and give up connecting to an unanswered address after a timeout (skipping if the network refuses at once), and that they print and parse thousands of random floats exactly as the interpreter does. `tests/assembler.rs` assembles every golden program for both targets with the built-in assembler and with GNU as plus GNU ld at the same section addresses, and requires identical `.text`, `.rodata`, and `.data` bytes and `.bss` size (needs `as`/`ld` for the host and `aarch64-linux-gnu-as`/`-ld` or `x86_64-linux-gnu-as`/`-ld` for the other target, skipping a target whose tools are missing) - Golden-output tests: `tests/programs.rs` runs every `.stn` in `examples/`, `tests/programs/`, and `docs/examples/` through both `stone run` and `stone build`, comparing stdout with the sibling `.out` file. A program that should fail at runtime also has a `.err` file holding text its stderr must contain, the position line without the path (`:3:7: error: division by zero`), so both backends must point at the same place. A sibling `.stderr` file holds the exact stderr a run must write under every backend (for `eprint`; without one, a successful run's stderr is not checked). Each backend runs a program in a new empty working directory of its own (`//.cwd`), first filled with a copy of a sibling `.files` directory if there is one (`fs_fixture.files/` beside `fs_fixture.stn`, `main.files/` beside a `main.stn`), which discovery skips, so programs that write files never meet. A sibling `.in` file becomes the program's stdin (otherwise it is empty), a `.args` file holds its command-line arguments, one per line, and a `.env` file holds `NAME=value` lines that every backend adds to its environment (for `os.env`). A subdirectory of one of those directories is a program of several files, run from its `main.stn`, with `main.out` (and any `.err`, `.in`, `.args`, `.env`) next to it, as in `tests/programs/modules/`. Both backends must agree on everything the checker accepts, so `SKIPS` should stay empty. `stone build` targets the host, and a third pass (`build-aarch64` on x86-64, `build-x86_64` on arm64) cross-builds every program with `--target` and runs it under qemu-user when `qemu-aarch64`/`qemu-x86_64` is on PATH, skipping with a note otherwise; compiled programs are static, so qemu needs no target libc. Off macOS, `every_program_links_for_macos` builds every program with `--target aarch64-macos` without running it. Off Windows, a `build-x86_64-windows` pass (`Backend::Windows`, `WindowsRunner`) builds every program as an `.exe` and runs it through WSL interop (found from `/proc/sys/fs/binfmt_misc/WSLInterop*`, passing `STONE_LEAK_CHECK` and `.env` names through `WSLENV`, which cannot pass an empty value, so `os_env` is skipped there, and with its working directory under the Windows `%TEMP%`, since the WSL file share gives spurious sharing violations) or else `wine`, and otherwise only links it (`windows_compiler_matches_expected`); a program whose check reports `driver::unsupported`'s message (`net`, `os.run`) is skipped for Windows, as is any program in `NOT_ON_WINDOWS` (`gui_no_display`, which is about Linux display variables). A `windows-latest` CI job runs `cargo test` natively, so there the build pass runs every program as a PE, plus clippy and `scripts/test-install.ps1` under both PowerShells. CI's x86-64 job installs `qemu-user binutils-aarch64-linux-gnu`, a `ubuntu-24.04-arm` job runs everything natively and installs `qemu-user binutils-x86-64-linux-gnu`, and a `macos-15` job runs `cargo test` (so every golden program runs as a Mach-O) plus the fuzz crate's tests natively. Built binaries run with `STONE_LEAK_CHECK=1`, so a program that ends with any string or list still allocated fails (see "Memory" below). Add a program plus its `.out` to cover new behavior - Fuzzing: the `fuzz/` crate (cargo-fuzz, nightly) has crash targets `lex`, `parse`, `format` (formatting its own output changes nothing), `interpret`, `codegen`, and `structured`, plus `differential`, which compares `stone run` with `stone build` on generated programs. Run one with `cargo +nightly fuzz run parse -- -dict=fuzz/stone.dict`. `cargo test --manifest-path fuzz/Cargo.toml` checks the program generator on stable, including that generated programs pass the checker and format stably, that 2,000 of them assemble and link for every target, and that 150 of them print the same under both backends and free everything, plus the same checks on longer inputs (`compound_sources`), which reach compound values more often. The generator writes strings too, keeping them from failing: only ASCII "words" of at least two bytes are indexed and sliced. It also declares structs (with `impl` blocks) and enums and works with them, options, maps, and lists of them through a type-directed `Generator::value` in `fuzz/src/compound.rs`, whose module doc lists the rules that keep those programs valid and from failing (no list shrinks, map variables only gain entries, bare `none` only where the type is fixed, no unreachable `case`). `STONE_FUZZ_TARGET=aarch64` makes that check (and `differential`) build for aarch64 and run under qemu, like the cross golden pass. Turn each crash into a regression test, as a unit test or a `tests/programs/` program @@ -53,6 +53,7 @@ Key pieces: - `src/span.rs`: `Pos` (1-based line and col, counted in chars), `FileId` (the entry file is `FileId(0)`, and the prelude `FileId::PRELUDE`, which `SourceMap::file` answers but `files` leaves out), and half-open `Span`, which holds a `FileId`. Every token and AST node carries one. `Span::new` puts a span in the entry file, and `Lexer::with_file` lexes another file. - `src/prelude.stn`: the prelude (`stdlib::PRELUDE`), which `project::link` parses in `FileId::PRELUDE` and puts first in every linked module, keeping its names (`option`); the parser lets only it name a variant `none`. `Rewriter::prelude_variant` rewrites a bare `some`/`none` (not a local) to `option.some`/`option.none` with a zero-width span on the unwritten `option`, `pattern` does the same to `["some"]`, and a written `option.x` is an error. Code that parses without linking (unit tests through `driver::parse_program`, the `codegen` fuzz target) must link, or options do not exist. `none` parses to `ExprKind::Name`, since `Constant` has no `None`. It also defines `enum result[T, E]` (tags 0 `ok`, 1 `err`, which `stone.parse_int_result` and `stone.parse_float_result` make by tag), `enum ordering` (tags 0 `less`, 1 `equal`, 2 `greater`, which builtin `cmp` makes by tag), the traits `eq`, `ord`, `display`, `from[T]`, and `try_from[T]`, and `fn sort[T: ord]`, a stable merge sort in stone that `xs.sort()` uses for anything but int/float/str (`stdlib::PRELUDE_SORT`). Since a prelude enum's name never comes before a written variant, a value may take it (`result = []`): the linker writes the enum's name before a bare variant with an empty span, which `Expr::is_unwritten` recognizes and `Expr::children`, `last_use::expr_names`, `AssignmentCheck`, and `ParameterUse` skip, `checker::may_name_enum` tells the backends whether a name before a dot may be an enum (never a written prelude enum name), the checker's `value_enum` keeps only other enums' names from variables, and `Rewriter::globals` (the entry file's top-level names) and `locals` exempt a variable from the `result.ok` error. `stdlib::PRELUDE_VARIANTS` pairs each bare variant with its enum (`stdlib::prelude_variant`), which `prelude_variant`, `pattern` (a bare `PatternKind::Name` such as `case less:` too), and `prelude_qualified` use, and `prints_bare` covers `ordering`. `project::prelude` renames each prelude `fn` to `prelude.` (`PRELUDE_MODULE`, so a root `prelude.stn` is an error); a prelude function calls no other, so nothing else is rewritten. `Context::types` skips prelude spans, so the unused `sort` body pulls in no runtime. - `src/project.rs`: multi-file programs. The entry file's directory is the root, and `a/b.stn` is the module `a.b`; directories are only parts of names, so there are no barrel files. `use a.b` binds the module (called as `b.f()`), `use a.b.f` binds a `pub fn` of `a.b` (a module file wins over a function of the same path), and `as` renames. Library modules hold only `use`, `fn`, and `enum`, so importing runs nothing and cycles are fine, and their functions cannot see the entry file's names, types and traits included: the entry file's types and traits keep their plain names, so `checker::visible_type` (a qualified name, a prelude type or trait, or code in `FileId::ENTRY`) hides them from other files in the checker's `resolve_name`, `value_enum`, `value_struct`, `written_type`, `declare_type_params`, `bounds`, and `impl_trait`, and in both backends through `may_name_enum` and `may_name_struct`, so the prelude's local `keys` stays a variable beside a `struct keys`. `pub enum` is imported like a function (`use zoo.Animal`, `Target::Enum`, `public_enums`), and `Rewriter` renames enums to `.` in signatures and variants (`type_expr`), in front of a variant (`name`, `qualified_enum` for `zoo.Animal.Fish`), and in patterns (`pattern`). `link` loads files through `Sources` (`FsSources`, or `MapSources` in tests), reports `use`/`pub` errors with the checker's wording, and merges every file into one `Mod` in which each library function is renamed to `.` and each use of it rewritten to that name, so the checker and both backends run unchanged (`fn.geometry.shapes.area` is a valid label). `link_in` picks another root, for editors. `pub struct` works as `pub enum` does: `public_types` lists a file's enums and structs together, `is_struct` tells `use geo.Point` to give `Target::Struct` rather than `Target::Enum` (both bind `Binding::Type`), and `module_member` rewrites `geo.Point(x=1)` to a call of the linked name `geo.Point`. `os`, `math`, `random`, `time`, and `fs` are builtin modules with no file (`stdlib::MODULES`, `Loader::resolve_builtin_module`, `Target::BuiltinModule`): `use math`, `use math.sqrt`, and `use math as m` work anywhere, calls are rewritten to linked names such as `math.sqrt` like library functions, and a `math.stn` (or `os.stn`, ...) in the root is an error. The checker reports any `use` left in the module, which can only be a nested one. Tests in `src/project/tests.rs`. An `impl` block must name a struct of its own file (`own_structs`; an enum or an imported struct is an error), is renamed like the struct in a library, and its functions are rewritten as `fn`s are (`Rewriter::function`) but keep their names, since their linked name is the struct's plus theirs; `report_method_collisions` reports a module function with the same linked name as a method. `Rewriter::set_type_params` keeps a generic definition's type parameters (and, for the functions of an `impl` block, its struct's, from `own_structs`) in `type_params`, which `type_expr` never renames, and reports one named like a type the file imports or, in a library, defines (the checker reports the rest). Traits: `public_types` lists `trait`s too, `use geo.Shape` gives `Target::Trait` (bound as `Binding::Type`), a library's traits are renamed `.`, and `Rewriter::trait_name` rewrites bounds (`Rewriter::bounds`, from `function`) and an `impl` block's trait (`geo.Shape` through `module_type`). `impl Tr for Ty` must be in the file of `Ty` (a struct) or of `Tr` (`Rewriter::impl_type`, with `struct_params`/`trait_names` of every file, for an imported struct's type parameters and a builtin type), except a prelude trait on a builtin type, which the checker reports; `type_expr` never renames `Self`. `report_method_collisions` covers trait default methods (`Trait.m`). A module of `stdlib::SOURCE_MODULES` (only `net`, whose source is `stdlib::net::SOURCE`) is part of stone but written in stone: `Loader::module` adds its embedded source as the library module of that name, at the path `` (`SourceFile::is_builtin`), so it links, checks, and runs like a user's library (functions `net.connect`, ...), `check_builtin_name` rejects a root `net.stn`, and `is_entry` never matches it. Such files, and only they (`Loader::own`), may `use` an internal builtin module of `stdlib::INTERNAL_MODULES` (`net_sys`, whose functions `stdlib::module` and `module_function` find but `MODULES`, the docs test, and editors do not list); anywhere else it is `no module named 'net_sys'`. +- `src/package.rs`: packages. `Manifest::parse` reads a `stone.toml` (`[package]` `name`, `version`, `stone` minimum; `[dependencies]` entries `{ github = "o/r", version }`, `{ gitlab = "g/p", version, host }`, or `{ path }`) through `package::toml`, a hand-written TOML subset reader whose keys and values keep spans, and `package::version::Version` is semver with `class` (the major, or `0.minor` before 1.0) and `from_tag` (`v1.2.0` or `1.2.0`). `dependency_name_error` rejects builtin module, prelude, and builtin names. `package::resolve::select` is Go's minimal version selection over `Target`s (`Path(dir)` or `Remote(source, version)`, whose `identity` is the directory or the source plus class), asking a closure for each target's `Requirement`s, breadth first, the newest version named winning (`Selection::resolve`, `used`); it never lists releases. `package::store` is the store (`default_dir`: `$STONE_HOME/packages`, `~/.stone/packages`, `%LOCALAPPDATA%\stone\packages`; `package_dir`: `///`; `content_hash`: SHA-256 of ` ` lines of the kept files, `stone.toml` and `.stn`), and `package::lock::Lock` reads and writes `stone.lock` (`[[package]]` name, github or gitlab, host, version, `hash = "sha256:..."`, sorted). In `project.rs`, `Loader::load_packages` reads the root's `stone.toml` through `Sources` (so `MapSources` tests and the LSP see packages too; `Sources::store` gives the store, `MapSources::set_store` in tests, and `Sources::write` writes, which only `FsSources` and `MapSources`, recording `written`, do), runs `select` with `read_target` (a directory's or the store's manifest; a missing download is an error at the dependency's key and goes in `Linked::missing`; a release's manifest must give the version it was released as, and cannot use a directory), makes a `Package` (root, prefix, name, deps by key, `unavailable` names whose `use` is silent) per used target, and `check_lock` compares each used remote package's hash with `stone.lock`, an error at the locked hash if they differ, else noting `Linked::lock`, which `driver::load` and `check_program` write (`write_lock`) when linking found no errors. Each file records its package (`file_package`, public as `Linked::file_packages`, indexes into `Linked::packages`, whose `PackageInfo` has `root` and `prefix` too), and manifests and the lock are files of the `SourceMap` with no statements (`add_manifest`, `SourceFile::is_package_file`), so their errors render like any other. A dependency's modules link as `..` (`linked_module`; `geo.1.shapes`, `geo.1` for its `lib.stn`, a digit segment so no module of the program can collide, `.2` and on for two directories of one name and class), `resolve` sends a path whose first name is a dependency of the file's package to `resolve_in` that package (`use geo` is its `lib.stn`), and otherwise resolves in the file's own package root, so a package sees only its own modules and dependencies. `SourceFile::package` names a dependency's files (`is_dependency`), which `unused.rs` never warns about. Downloading is `package::fetch`, behind the `fetch` cargo feature (optional `ureq` with rustls, `flate2`, `tar`; release.yml and CI's feature runs turn it on): `Client` lists a source's releases (GitHub `{api}/repos/{repo}/releases`, GitLab `{api}/projects/{encoded}/releases`, read with `package::json`, a small JSON reader, since serde_json's `PartialEq` impls break `assert_eq!(x, [])` inference in tests), finds a version's tag (`release`, `newest` for a `package::edit::Wanted`), and `download`s `{api}/repos/{repo}/tarball/{tag}` or `.../repository/archive.tar.gz?sha={tag}` into the store through `unpack` (only kept files, nothing outside the archive's top directory, a `.partial` sibling renamed into place); `STONE_GITHUB_API`/`STONE_GITLAB_API` replace the APIs and `GITHUB_TOKEN`/`GITLAB_TOKEN` are sent. `fetch_missing(entry)` links the manifests (an empty entry source) again and again, downloading `Linked::missing`, until nothing is. `package::edit` parses `stone add` specs (`parse_spec`, `Wanted`) and edits `[dependencies]` in place (`set_dependency`, `remove_dependency`, `dependency_value`). The CLI is `src/deps.rs` (`add`, `remove`, `fetch`, `tree`, which prints `Linked::packages`, and `prepare`, which `run`, `build`, and `check` call unless `--offline`); without the feature, `add` and `fetch` explain how to get a build that downloads. `tests/cli/packages.rs` (feature only) runs the commands against a fake GitHub on loopback (`FakeGitHub`, tarballs made by hand in stored gzip). Golden program `tests/programs/packages/` uses packages in `tests/packages/`, and `tests/cli.rs`'s `a_package_from_the_store_runs_and_is_locked` uses a store through `STONE_HOME`. - `src/unused.rs`: `find(linked, analysis)` warns about what a program defines and never uses, for `stone check` and the LSP only (nothing if linking failed). A symbol is read when it has a `Reference` that is neither its definition nor in `Facts::writes` (bare `Name` targets of `Assign` and `For`, and pattern bindings; `x += 1` and a change's root count as reads), and a function, method, or type is used when a reference lies outside its own definition (and, for a struct, its `impl` blocks), so recursion alone is no use. Never reported: names starting with `_`, `self`, lambdas, the prelude and builtin source files, `pub` definitions and a `pub struct`'s methods (`Facts::exempt`, in every file, since the LSP checks an unused library as an entry file), and the functions and parameters of a trait or an `impl Trait for T` block. Imports come from `project::Import::used`, which the linker sets from `Rewriter::binding`, the one lookup of `use` bindings that records each name it finds. The LSP keeps the warnings in `Document::unused` and tags them `DiagnosticTag::UNNECESSARY`. - `src/diagnostic.rs`: `Diagnostic` (severity, span, message) with `render` for the CLI, and `Diagnostics` for returning several as one error. Every stage's errors convert to it. - `src/token.rs`: `Token`/`TokenType` and `RESERVED_KEYWORDS`. `Keyword` and `Operator` hold `&'static str`s (the lexer hands out the entry of `RESERVED_KEYWORDS` or its operator table), so `expect(TokenType::Keyword("fn"))` allocates nothing. The lexer (`src/lexer.rs`) emits Python-style `Indent`/`Dedent`/`Newline` tokens, and `->` as `Arrow`, lexes a string with values as `StringStart(text)`, each value's own tokens, `StringMiddle(text)`, ..., `StringEnd(text)` (`lex_text`, with a `holes` stack of bracket depths so `}` ends a value only at depth 0, and errors for `{}`, a lone `}`, a depth-0 `:`, and a newline inside a value), a string without as `String`, and reads string escapes (`\"`, `\'`, `\\`, `\n`, `\t`, `\r`, `\xhh` up to `\x7f`, never `\x00`, since compiled strings are null-terminated). `Lexer::lex` stops at the first error; `Lexer::lex_recovering`, which `project::Loader::add` uses, records each error, skips the rest of its line (closing any open values), and goes on, and `Loader::add` drops parser errors on a line the lexer cut short, so an unfinished value such as `print("{xs.` keeps the rest of the file analyzed. `context::escape` writes other control characters into `.string` directives as three octal digits, which `asm::parse_string` reads. @@ -101,8 +102,9 @@ Key pieces: Language server (`lsp/`, crate `stone-lsp`, depends on `lsp-server`, `lsp-types`, `serde_json`): -- `src/lib.rs`: `run(connection)` does the initialize handshake (preferring UTF-8 positions, falling back to UTF-16), keeps open documents with full-text sync, and dispatches requests through `Server::answer`. On every open/change/close it reanalyzes every open document, reading files through `Workspace` (open documents' text over the disk), and publishes each one's diagnostics, the changed document first. -- `src/features.rs`: one pure function per request (`diagnostics`, `hover`, `definition`, `references`, `prepare_rename`, `rename`, `document_symbols`, `completion`) over a `Document`, which holds the text, the `stone::driver::analyze_linked` result of the whole program it belongs to (`Document::in_program`: the nearest `main.stn` at or above it, or the file alone if that program does not use it), its `FileId`, the `SourceMap`, the program's imports, and a `LineIndex`. Spans in other files become locations through `Document::location`, and a library function's symbol is named `.`, which `Document::local_name` shortens. An enum hovers with its variants and outlines with them as children, a variant hovers as `Shape.Circle(float)`, and completion after an enum's name and a `.` offers its variants (`enum_variants`). A struct hovers with its fields and outlines with them as children, a field hovers as `Point.x: float` and renames everywhere, including `x=` in a construction, and completion after a variable holding a struct and a `.` offers its fields (`struct_fields`). Builtins have no stone source, so `run` writes `stdlib::builtins_reference()` (comment-only, documenting each builtin type from `stdlib::TYPE_DOCS`, then each builtin from `stdlib::BUILTIN_DOCS`) to `/stone-lsp-/builtins.stn`, and `definition` of a builtin jumps to its line there (`Builtins`). A builtin type's name not followed by `(` (`builtin_type_at`) hovers and leads to its type's line, so `int` in `x: int` is the type and in `int(x)` the conversion, except `fn`, which is the type `fn(T1, ...) -> R` (in `TYPE_DOCS`) only when followed by `(`. `Document::tokens` (from `lex_recovering`) feed `Document::in_code`: in a string's text (strictly inside a string token's span, so a value's contents and the spot before its `}` are code) or after a `//` outside one, `hover` shows only the expression's type, `definition` gives nothing, and `completion` offers nothing. `completion` asks `slot` whether a type (`Slot::Type`: after `->`, a definition's or lambda's `name: `, a struct field's line, a variant's parentheses, or the brackets of a type, found from the line's tokens before the word, so a string's text counts for nothing, and `headers`, the lines opening its blocks) or a bound (`Slot::Bound`, after `[T: ` in a definition) comes next, and `type_completion` offers `TYPE_DOCS`, type parameters of the enclosing function and its struct or trait, `Self` in an `impl`/`trait` block, and the file's, imports', and prelude's structs and enums (or traits). After a `.`, a variable's type comes from its symbol and any other receiver's (`xs[0].`, `f(x).`, `p.items.`) from `receiver_type`, which checks the program again through an `Overlay` of `Sources` with the line minus the `.` and the member (then with a `}` there, for an unfinished string value) and takes the type of the expression ending at the dot that starts last, skipping a version with an error on that line; `member_completion` then offers the type's members, always with `display`. `use_completion` offers `SOURCE_MODULES` too, analyzing `use net` on its own for `use net.` before the program uses it. It writes `stdlib::PRELUDE` beside it as `prelude.stn`, where `definition` of `some`, `none`, or `option` goes (`Builtins::prelude_location`); `Document::location` gives nothing for a prelude span, and its symbols cannot be renamed. It writes each of `stdlib::SOURCE_MODULES` there too (`net.stn`), where `definition` of a name in such a file, or of `use net`, goes (`Builtins::builtin_location`), and those names cannot be renamed either. Add a feature here with unit tests in `src/features/tests.rs`, then wire it into `Server::handle_request` and `capabilities`. A method or associated function hovers as `fn Point.shift(self, dx: float)` (`function_signature`), outlines under its struct after the fields (`struct_methods`, `function_outline`), and completion offers a struct value's methods with `self` (`has_self`) after its fields, and a struct's associated functions after its name and a `.`. A trait hovers with its methods' signatures and outlines with them (`trait_methods`, `SymbolKind::INTERFACE`), a bounded type parameter hovers as `(type parameter) T: Shape` and signatures show bounds (`bounds_of`, from `Analysis::bounds`), completion after a value of a bounded `T` (or `Self` in a default) offers its traits' methods (`bound_methods`) and after a struct value the defaults of the traits it implements (`Analysis::implemented`), and `method_applies` offers `eq`/`cmp` on builtin types. +- `src/lib.rs`: `run(connection)` does the initialize handshake (preferring UTF-8 positions, falling back to UTF-16), keeps open documents with full-text sync, and dispatches requests through `Server::answer`. On every open/change/close it reanalyzes every open document (`Server::reanalyze`), reading files through `Workspace` (open documents' text over the disk), and publishes each one's diagnostics, the changed document first, then each `stone.toml`/`stone.lock` of their programs (`features::package_diagnostics`, merged across documents, with `Server::published` clearing a file no program has anymore). If the client offers dynamic registration, it registers (`watch_packages`) a `workspace/didChangeWatchedFiles` watcher of `**/stone.toml` and `**/stone.lock`, whose events reanalyze everything, so a manifest edit or `stone fetch` (which writes the lock) takes effect. +- Packages in the LSP: linking puts a dependency that cannot be read in `diagnostics` once, at the manifest's key, and an error at each `use` of it in `Linked::unavailable` (from `Loader::failed`, the reason `read_target` gave, plus the hint to run `stone fetch` for a missing download), which only `features::diagnostics` shows. `program_of` takes the nearest directory with a `main.stn` or a `stone.toml` as the root, its `main.stn` or else `lib.stn` as the entry, so a file of a library (or of a downloaded package opened from the store) links with its own dependencies. `hover` and `completion` (wrapping `hover_at`/`completion_at`) and diagnostic messages go through `project::shown_names`, which writes a dependency's prefix (`geo.1`) as its name wherever it starts a name, unless two packages share the name. `Document::package_owning` (a dependency's file, or any file under `Sources::store`) refuses rename. `use_completion` offers the file's package's dependency keys (`package geo 1.2.0`) and, after `geo.`, that package's directory and its `lib.stn`'s functions, linking a one-line `use` program if the program does not load it yet. +- `src/features.rs`: one pure function per request (`diagnostics`, `hover`, `definition`, `references`, `prepare_rename`, `rename`, `document_symbols`, `completion`) over a `Document`, which holds the text, the `stone::driver::analyze_linked` result of the whole program it belongs to (`Document::in_program`: the nearest `main.stn`, or `stone.toml` with its `lib.stn`, at or above it, or the file alone if that program does not use it), its `FileId`, the `SourceMap`, the program's imports, and a `LineIndex`. Spans in other files become locations through `Document::location`, and a library function's symbol is named `.`, which `Document::local_name` shortens. An enum hovers with its variants and outlines with them as children, a variant hovers as `Shape.Circle(float)`, and completion after an enum's name and a `.` offers its variants (`enum_variants`). A struct hovers with its fields and outlines with them as children, a field hovers as `Point.x: float` and renames everywhere, including `x=` in a construction, and completion after a variable holding a struct and a `.` offers its fields (`struct_fields`). Builtins have no stone source, so `run` writes `stdlib::builtins_reference()` (comment-only, documenting each builtin type from `stdlib::TYPE_DOCS`, then each builtin from `stdlib::BUILTIN_DOCS`) to `/stone-lsp-/builtins.stn`, and `definition` of a builtin jumps to its line there (`Builtins`). A builtin type's name not followed by `(` (`builtin_type_at`) hovers and leads to its type's line, so `int` in `x: int` is the type and in `int(x)` the conversion, except `fn`, which is the type `fn(T1, ...) -> R` (in `TYPE_DOCS`) only when followed by `(`. `Document::tokens` (from `lex_recovering`) feed `Document::in_code`: in a string's text (strictly inside a string token's span, so a value's contents and the spot before its `}` are code) or after a `//` outside one, `hover` shows only the expression's type, `definition` gives nothing, and `completion` offers nothing. `completion` asks `slot` whether a type (`Slot::Type`: after `->`, a definition's or lambda's `name: `, a struct field's line, a variant's parentheses, or the brackets of a type, found from the line's tokens before the word, so a string's text counts for nothing, and `headers`, the lines opening its blocks) or a bound (`Slot::Bound`, after `[T: ` in a definition) comes next, and `type_completion` offers `TYPE_DOCS`, type parameters of the enclosing function and its struct or trait, `Self` in an `impl`/`trait` block, and the file's, imports', and prelude's structs and enums (or traits). After a `.`, a variable's type comes from its symbol and any other receiver's (`xs[0].`, `f(x).`, `p.items.`) from `receiver_type`, which checks the program again through an `Overlay` of `Sources` with the line minus the `.` and the member (then with a `}` there, for an unfinished string value) and takes the type of the expression ending at the dot that starts last, skipping a version with an error on that line; `member_completion` then offers the type's members, always with `display`. `use_completion` offers `SOURCE_MODULES` too, analyzing `use net` on its own for `use net.` before the program uses it. It writes `stdlib::PRELUDE` beside it as `prelude.stn`, where `definition` of `some`, `none`, or `option` goes (`Builtins::prelude_location`); `Document::location` gives nothing for a prelude span, and its symbols cannot be renamed. It writes each of `stdlib::SOURCE_MODULES` there too (`net.stn`), where `definition` of a name in such a file, or of `use net`, goes (`Builtins::builtin_location`), and those names cannot be renamed either. Add a feature here with unit tests in `src/features/tests.rs`, then wire it into `Server::handle_request` and `capabilities`. A method or associated function hovers as `fn Point.shift(self, dx: float)` (`function_signature`), outlines under its struct after the fields (`struct_methods`, `function_outline`), and completion offers a struct value's methods with `self` (`has_self`) after its fields, and a struct's associated functions after its name and a `.`. A trait hovers with its methods' signatures and outlines with them (`trait_methods`, `SymbolKind::INTERFACE`), a bounded type parameter hovers as `(type parameter) T: Shape` and signatures show bounds (`bounds_of`, from `Analysis::bounds`), completion after a value of a bounded `T` (or `Self` in a default) offers its traits' methods (`bound_methods`) and after a struct value the defaults of the traits it implements (`Analysis::implemented`), and `method_applies` offers `eq`/`cmp` on builtin types. - `src/line_index.rs`: converts stone's 1-based char positions to LSP's 0-based UTF-16 or UTF-8 positions. - `src/uri.rs`: converts between `file://` URIs and paths. - Tests: unit tests for features and positions, `tests/server.rs` over an in-memory connection, and `tests/stdio.rs` against the real binary. diff --git a/Cargo.lock b/Cargo.lock index 09c6e46..724bb6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1793,7 +1793,10 @@ name = "stone" version = "0.4.1" dependencies = [ "clap", + "flate2", "self_update", + "tar", + "ureq", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c58d4d0..4629e76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,8 +8,8 @@ license = "Apache-2.0" repository = "https://github.com/tarolling/stone" readme = "README.md" -# the language server is a separate crate, and self-update is opt-in, so a default build of -# stone depends only on clap +# the language server is a separate crate, and self-update and fetch are opt-in, so a default +# build of stone depends only on clap [workspace] members = ["lsp"] default-members = [".", "lsp"] @@ -28,6 +28,11 @@ self_update = { version = "1.3", default-features = false, features = [ "progress-bar", ], optional = true } +# `stone add` and `stone fetch`, which download packages over HTTPS. release.yml turns it on +ureq = { version = "3.4", default-features = false, features = ["rustls"], optional = true } +flate2 = { version = "1.1", optional = true } +tar = { version = "0.4", default-features = false, optional = true } + [lints.rust] # cargo fuzz sets --cfg fuzzing, which `debug!` checks unexpected_cfgs = { level = "warn", check-cfg = ["cfg(fuzzing)"] } @@ -35,6 +40,8 @@ unexpected_cfgs = { level = "warn", check-cfg = ["cfg(fuzzing)"] } [features] # `stone self-update`, which downloads releases over HTTPS. release.yml turns it on self-update = ["dep:self_update"] +# downloading packages from GitHub and GitLab releases. release.yml turns it on +fetch = ["dep:ureq", "dep:flate2", "dep:tar"] # `cargo run -- build` is about 6x slower unoptimized; level 1 keeps debug info and assertions [profile.dev] diff --git a/docs/agents/stone.md b/docs/agents/stone.md index 02a4734..01cee62 100644 --- a/docs/agents/stone.md +++ b/docs/agents/stone.md @@ -611,6 +611,21 @@ directory: `geometry/shapes.stn` is `use geometry.shapes`, called as `shapes.are `fn`, `struct`, `enum`, `trait`, and `impl`, never top-level statements. A program of several files conventionally starts at `main.stn`. +Packages come from GitHub or GitLab releases, named in a `stone.toml` beside `main.stn`, which +`stone add github:owner/repo` writes. Never write a version range: `version` is the oldest +version the program works with, and stone picks the newest that any package asks for. A path +starting with the dependency's name is in the package: `use json` loads its `lib.stn`, and +`use json.parse` its `parse.stn` (or `lib.stn`'s `pub fn parse`). + +```toml +[dependencies] +json = { github = "alice/stone-json", version = "1.2.0" } +geo = { path = "../geo" } +``` + +`stone run` downloads what is missing, `stone tree` shows what is used, and `stone.lock`, which +stone writes, holds each package's hash and belongs in version control. + ## Style Indent with four spaces. Write comment lines directly above a function to document it (editors diff --git a/docs/development/language-server.md b/docs/development/language-server.md index e8fbfa8..05bdd80 100644 --- a/docs/development/language-server.md +++ b/docs/development/language-server.md @@ -20,6 +20,19 @@ symbol with all its references in every file. The analysis recovers from syntax by statement, so features keep working around a typo, and it hides type errors until the syntax and `use` errors are fixed. +A program's root is the nearest directory holding a `main.stn` or a package's `stone.toml`, and +its entry file that `main.stn` or, for a library, its `lib.stn`. Packages need nothing of their +own: `project::link` reads manifests through the same `Sources`, so the editor sees the program's +dependencies, from directories and from the package store, without downloading. Linking reports a +package that cannot be read once, in the manifest, and notes an error at each `use` of it in +`Linked::unavailable`, which only the editor shows. `features::package_diagnostics` gives the +errors of every `stone.toml` and `stone.lock`, which the server publishes to those files' URIs, +and when the client allows it the server registers a watcher for both, so editing a manifest or +running `stone fetch` reanalyzes every open document. Hover, completion, and diagnostics show a +dependency's linked names as the program writes them (`project::shown_names`, `geo.shapes.area` +for `geo.1.shapes.area`), and a symbol in a dependency, or in a downloaded package opened on its +own, cannot be renamed. + | file | role | | --- | --- | | `lsp/src/lib.rs` | the initialize handshake, open documents, diagnostics, and request dispatch | diff --git a/docs/guides/packages.md b/docs/guides/packages.md new file mode 100644 index 0000000..4fc086e --- /dev/null +++ b/docs/guides/packages.md @@ -0,0 +1,156 @@ +# Using packages + +A package is a directory of stone modules with a `stone.toml` that names it. A program uses other +people's packages by naming them in its own `stone.toml`, next to its entry file. stone downloads +them from GitHub or GitLab releases, so there is no registry to sign up for: any repository whose +releases hold a stone package is one. + +## Adding a package + +In the directory of your program's `main.stn`, run + +```sh +stone add github:alice/stone-json +``` + +This finds the newest release of `alice/stone-json`, downloads it, and writes the dependency to +`stone.toml`, making the file if there is none: + +```toml +[dependencies] +json = { github = "alice/stone-json", version = "1.4.0" } +``` + +The key, `json`, is the name the package gives itself, and the name your files use it under. +`--name` picks another (`stone add github:alice/stone-json --name js`). The package can be named +in several ways: + +| you write | you get | +| --- | --- | +| `github:alice/stone-json` | the newest release | +| `github:alice/stone-json@1.2` | the newest 1.2.x release | +| `github:alice/stone-json@1.2.3` | exactly 1.2.3 | +| `https://github.com/alice/stone-json` | the same as `github:alice/stone-json` | +| `gitlab:group/project`, `gitlab:group/subgroup/project` | a project on gitlab.com | +| `gitlab:group/project --host gitlab.example.com` | a project on your own GitLab | +| `github:github.example.com/alice/stone-json` | a repository on GitHub Enterprise | + +`@latest` is the default, and it skips pre-releases such as `2.0.0-beta.1`. To use one, give it +exactly. + +## Using it in code + +A package's modules are named by the dependency's name, then their path in the package: + +```stone +use json // the package's lib.stn +use json.encode // encode.stn, or the pub fn encode of lib.stn +use json.parse.value // parse/value.stn, or the pub fn value of parse.stn + +print(json.decode("[1, 2]")) +``` + +`use json` loads the package's `lib.stn`, the module a package keeps its main functions in. A +package's files see only their own modules and their own dependencies, never your program's, so +two packages can each have a `util.stn` without meaning the same module. + +Everything else works as with your own modules: only `pub` functions and types can be used, and +unused functions are never compiled in. See {doc}`../reference/modules`. + +## Packages in a directory + +A dependency can also be a directory, for a package you are writing yourself or one that lives +next to your program in the same repository: + +```toml +[dependencies] +geo = { path = "../geo" } +``` + +The path is relative to the `stone.toml`, and the directory must have a `stone.toml` of its own +that gives the package a name. stone uses the directory as it is, so a change there shows up the +next time you run. Directories have no versions and are never downloaded or locked. + +## How versions are chosen + +Each dependency gives the oldest version it works with, and that is all a manifest ever says +about versions. stone uses the newest of the versions that your program and the packages it uses +ask for, which is how Go chooses versions: + +- Your program asks for `json` 1.2.0, and `geo` asks for `json` 1.3.0, so both get `json` 1.3.0. +- A newer 1.9.0 being released changes nothing, until something asks for it. + +So the same manifests always give the same versions, on every machine and on every day, and +nothing is ever upgraded behind your back. To upgrade, ask for the newer version: run +`stone add github:alice/stone-json` again, or edit the version in `stone.toml`. + +Versions are [semantic versions](https://semver.org). A new major version may break the code that +uses it, so 1.x and 2.x are treated as different packages: if one part of your program uses +`json` 1.4.0 and another `json` 2.1.0, both are linked in, each under its own name, and neither +can see the other. Before 1.0, each minor version is its own: 0.3.x and 0.4.x are different +packages too. + +## stone.lock + +The first time a package is downloaded, stone writes a hash of its files to `stone.lock`, next to +`stone.toml`: + +```toml +# written by stone: the hash of what each package downloaded held +[[package]] +name = "json" +github = "alice/stone-json" +version = "1.4.0" +hash = "sha256:4f1c..." +``` + +The lockfile does not choose versions, since the manifests already do. It makes sure that a +package is the same code every time: if a release is later moved to other code, or a file in the +download is changed, the program fails to build. If you trust the change (say, the author fixed a +release on purpose), delete the package's entry from `stone.lock`, and the next build records the +new hash. Commit `stone.lock`, so everyone who builds your program checks against the same +hashes. + +## Downloading + +`stone run`, `stone build`, and `stone check` download whatever the program uses that is not +downloaded yet, so after cloning a program, running it is enough. The commands for managing +packages are: + +- `stone fetch` downloads everything the program uses and brings `stone.lock` up to date. With + `--locked` it fails instead if `stone.lock` would change, which is what CI should run. +- `stone tree` shows each package the program uses and the packages they use: + + ```text + app + └── json 1.4.0 (github:alice/stone-json) + └── num 0.3.0 (github:bob/num) + ``` + +- `stone remove json` takes a dependency out of `stone.toml`, and out of `stone.lock` with any + package only it used. +- `--offline`, given to `run`, `build`, or `check`, uses only what is already downloaded and + never connects to anything. + +Downloaded packages are kept in one place for every program, the package store: +`~/.stone/packages`, or `%LOCALAPPDATA%\stone\packages` on Windows. Set `STONE_HOME` to keep them +in `$STONE_HOME/packages` instead. Each version of each package is downloaded once, and only its +`stone.toml` and `.stn` files are kept. Don't change files in the store, since they no longer +match `stone.lock` once changed. + +For a private repository, or to avoid GitHub's limit on requests without a token, set +`GITHUB_TOKEN` (or `GITLAB_TOKEN`) to a token that can read it. + +Release builds of stone can download. A build from source needs the `fetch` feature for that +(`cargo install --features fetch`), and without it reads only packages already downloaded. + +## In the editor + +The language server reads packages as the compiler does, from the store and from directories, but +never downloads. If a package is not downloaded yet, each `use` of it shows the error with the +hint to run `stone fetch`, and so does the dependency in `stone.toml`. Once `stone fetch` is done, +the errors clear on their own. Go to definition opens a package's files, and a package's names +cannot be renamed from your program, since other programs use them too. + +See {doc}`publishing` to make a package of your own, and {doc}`../reference/manifest` for every +key `stone.toml` can hold. diff --git a/docs/guides/publishing.md b/docs/guides/publishing.md new file mode 100644 index 0000000..7ab1ca4 --- /dev/null +++ b/docs/guides/publishing.md @@ -0,0 +1,115 @@ +# Publishing a package + +A package is published by making a release of a GitHub or GitLab repository. There is no registry +and nothing to upload: a program that names the repository in its `stone.toml` downloads the +release straight from the host. + +## Laying out the repository + +The repository's root is the package. It holds a `stone.toml` that names the package and gives +its version, and the package's modules, as any stone program's root does: + +```text +stone-json/ + stone.toml + lib.stn + encode.stn + parse/ + value.stn +``` + +```toml +[package] +name = "json" +version = "1.0.0" +description = "Reading and writing JSON" +license = "MIT" +``` + +`name` is how programs name the package by default. It is made of ASCII letters, digits, and +`_`, and does not start with a digit. Avoid the name of a builtin module or builtin, such as +`math` or `print`, which programs cannot use, so each would have to pick another name for it. +`version` is the version you are about to release. See {doc}`../reference/manifest` for every +key. + +`lib.stn` is the module a program gets from `use json`, so it is where the package's main +functions go. The other modules are named by their paths, as in any program: a program writes +`use json.encode` for `encode.stn` and `use json.parse.value` for `parse/value.stn`. A package +needs no `lib.stn` if it has none to offer, and then programs use its modules by name. + +Only `pub` functions, structs, enums, and traits can be used from outside the package, and every +module of a package holds only `use`, functions, and types, as library modules do, so using the +package never runs any code. A package's own files use each other by their paths within the +package (`use parse.value`), and its dependencies by their names. + +A downloaded package keeps only its `stone.toml` and its `.stn` files. Tests, examples, and +anything else in the repository are left out, so they cost the programs that use it nothing. + +## Depending on other packages + +A package names its own dependencies in its `stone.toml`, as a program does: + +```toml +[dependencies] +num = { github = "bob/num", version = "0.3.0" } +``` + +Give the oldest version your package works with, not the newest there is: a program that needs a +newer `num` asks for it, and gets that version for every part of the program (see +{doc}`packages`). A package from a host cannot depend on a directory, since the directory is not +part of its release. + +## Trying it out + +Before releasing, use the package from a program through a path dependency, which reads the +directory as it is: + +```toml +[dependencies] +json = { path = "../stone-json" } +``` + +`stone check` and `stone run` in that program then report errors in the package's files too, and +the editor checks the package's files as you change them. Opening a file of the package itself in +the editor checks it as part of the package, with its own dependencies. + +## Releasing a version + +1. Set `version` in `stone.toml` to the new version, and commit it. +2. Tag the commit with the version, as `v1.0.0` or `1.0.0`, and push the tag. +3. Make a release of the tag: + + ```sh + gh release create v1.0.0 --notes "First release" # GitHub + glab release create v1.0.0 --notes "First release" # GitLab + ``` + + or with the "Draft a new release" button on GitHub, or Deploy, Releases on GitLab. + +That's all: `stone add github:you/stone-json` now finds it. A release is found by its tag, which +must be a version, and drafts are left out. stone downloads the source archive the host makes of +the tag, so no files need to be attached to the release. + +The `version` in the released `stone.toml` must be the version of the tag, or programs that use +the release fail to build. A tag such as `v1.0.0-beta.1` is a pre-release, which +`stone add` only picks when asked for exactly. + +Never move a tag once it is released. Programs that used the release recorded a hash of its files +in their `stone.lock`, and stop building if the release changes, so fix a mistake with a new +version instead. + +## Choosing the next version + +Versions are [semantic versions](https://semver.org), `major.minor.patch`: + +- **patch** (1.0.0 to 1.0.1): fixes that change nothing a program relies on. +- **minor** (1.0.1 to 1.1.0): new functions, types, or modules, with everything that worked + before still working. +- **major** (1.1.0 to 2.0.0): anything that could break a program that uses the package, such as + removing or renaming a `pub` function, changing a signature, or adding a field to a `pub` + struct, which every construction of it must then name. + +A new major version is a separate package to stone: a program may use 1.x and 2.x side by side, +and upgrading to 2.x is always something its author asks for. Before 1.0, a new minor version +(0.3 to 0.4) is treated as a new major one, so a package can still change freely while it is +young. diff --git a/docs/index.md b/docs/index.md index bdc3d6b..66f1c12 100644 --- a/docs/index.md +++ b/docs/index.md @@ -48,12 +48,21 @@ tutorial/index philosophy ``` +```{toctree} +:maxdepth: 2 +:caption: Guides + +guides/packages +guides/publishing +``` + ```{toctree} :maxdepth: 2 :caption: Reference reference/syntax reference/modules +reference/manifest reference/types reference/builtins reference/runtime-errors diff --git a/docs/philosophy.md b/docs/philosophy.md index f3143d2..7ee5023 100644 --- a/docs/philosophy.md +++ b/docs/philosophy.md @@ -75,6 +75,17 @@ What it should infer next: - **Dead code.** Functions that are never called should be left out, not only unused runtime routines. +Status: packages apply the same idea to versions. A `stone.toml` gives each dependency only the +oldest version it works with, and stone uses the newest version anything in the program asks +for, as Go does, so there are no version ranges to write, no solver to fail, and the same +manifests always give the same versions without a lockfile choosing them. `stone.lock` only +records a hash of what each package held when it was first downloaded, so a release that changes +fails to build rather than changing a program. Two major versions of one package can be used at +once, linked under different names, so there is no conflict to resolve by hand. Packages are +downloaded from GitHub and GitLab releases, so there is no registry to run or to trust beyond the +host and the hashes. Not done yet: packages in a subdirectory of a repository, sources other than +GitHub and GitLab, and a shared database of checksums beyond trust on first use. + ## Simple like Python The code a programmer reads and writes should look like Python's and need as few concepts. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index c8ee225..0d2be12 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -10,7 +10,12 @@ finds errors, the command prints all of them and exits with status 1 without run Errors print as `file:line:col: error: message`, followed by the source line and a caret under the problem. -## `stone run FILE [ARGS...]` +A program with a `stone.toml` next to its entry file uses packages (see +{doc}`../guides/packages`). `run`, `build`, and `check` first download any package the program +uses that is not downloaded yet, unless given `--offline`, which uses only what is already in +the package store and never connects to anything. + +## `stone run [--offline] FILE [ARGS...]` Interprets the program. `stone FILE` is shorthand for `stone run FILE`. Everything after the file is passed to the program, which reads it with `args()`, even arguments that start with `-`. @@ -21,7 +26,7 @@ stone run examples/basics.stn stone run sum.stn 1 2 3 < numbers.txt ``` -## `stone build FILE... [-o OUTPUT] [--target TARGET]... [--windows-gui]` +## `stone build FILE... [-o OUTPUT] [--target TARGET]... [--windows-gui] [--offline]` Compiles each program to a native executable in its build directory (see [Where executables go](#where-executables-go)), writing the assembly next to it with a `.s` @@ -122,7 +127,7 @@ stone clean examples/basics.stn # removes examples/build stone clean # removes ./build ``` -## `stone check FILE` +## `stone check [--offline] FILE` Reports every syntax and type error without running the program, and exits with status 1 if there are any. Warnings alone print but leave the status at 0. @@ -147,6 +152,45 @@ main.stn:3:17: warning: parameter 'h' is never used; remove it or name it '_h' stone check examples/basics.stn ``` +## `stone add PACKAGE [--name NAME] [--host HOST]` + +Adds a package from a GitHub or GitLab release to the program in the current directory: it finds +the release, downloads it with every package it uses, names it under `[dependencies]` in +`stone.toml` (making the file if there is none), and updates `stone.lock`. `PACKAGE` is +`github:owner/repo`, `gitlab:group/project`, or a link to either, optionally followed by +`@latest` (the default, the newest release that is not a pre-release), `@1.2` (the newest 1.2.x), +or an exact version such as `@1.2.3`. The dependency is named by the package's own name unless +`--name` gives another, and `--host` names a GitLab of your own, such as `gitlab.example.com`. +Adding a package that is already there changes its version to the one found. + +```sh +stone add github:alice/stone-json +stone add gitlab:bob/num@0.3 --name numbers +``` + +## `stone remove NAME` + +Removes the dependency `NAME` from the current directory's `stone.toml`, and from `stone.lock` +along with any package only it used. The downloaded files stay in the package store. + +## `stone fetch [PATH] [--locked]` + +Downloads every package the program in `PATH` (an entry file or a directory, by default the +current directory) uses, and brings its `stone.lock` up to date. With `--locked`, it fails if +`stone.lock` would change instead of changing it, so CI builds only what was locked. + +## `stone tree [PATH]` + +Prints the packages the program in `PATH` uses, each with its version and source, and below each +the packages it uses in turn. A package used under a name other than its own shows as +`name = package`, and one that appears again is marked `(*)` and not expanded again. + +```text +app +└── json 1.2.0 (github:alice/stone-json) + └── num 0.3.0 (github:bob/num) +``` + ## `stone fmt [PATH...] [--check]` Rewrites files in stone's one standard layout. A `PATH` is a file, or a directory, in which diff --git a/docs/reference/editor.md b/docs/reference/editor.md index 2359a8f..3c2958d 100644 --- a/docs/reference/editor.md +++ b/docs/reference/editor.md @@ -19,6 +19,12 @@ arguments, speaking LSP over stdin and stdout, for files ending in `.stn`. - **Outline** of functions and variables, with each function's parameters and locals. - **Completion** of names visible at the cursor, builtins, and keywords. - **Formatting**: Format Document (and format on save) rewrites the file as `stone fmt` does. +- **Packages** (see {doc}`../guides/packages`) are read from the package store and from + directories, as the compiler reads them, but never downloaded: a package that is not + downloaded yet is an error at each `use` of it and in `stone.toml`, with the hint to run + `stone fetch`, and the errors clear once it has run. Go to definition opens a package's files, + and a package's names cannot be renamed. A file inside a package is checked as part of it, + with the package's own dependencies. The VS Code extension also provides syntax highlighting, comment toggling, bracket matching, and indentation after a line ending in `:`. diff --git a/docs/reference/manifest.md b/docs/reference/manifest.md new file mode 100644 index 0000000..30ec7b6 --- /dev/null +++ b/docs/reference/manifest.md @@ -0,0 +1,109 @@ +# stone.toml + +A `stone.toml` makes a directory a package. A program's sits next to its entry file, and a +package's at the root of its repository. The guides {doc}`../guides/packages` and +{doc}`../guides/publishing` show how they are used. + +```toml +# a program that uses three packages +[package] +name = "app" +version = "0.1.0" +stone = "0.4" + +[dependencies] +json = { github = "alice/stone-json", version = "1.2.0" } +num = { gitlab = "bob/num", version = "0.3" } +geo = { path = "../geo" } +``` + +A manifest holds two tables, `[package]` and `[dependencies]`, both optional. Any other key is an +error, as is a key one of them does not hold, so a misspelling is caught. + +## [package] + +| key | value | meaning | +| --- | --- | --- | +| `name` | a stone name | The package's name, which programs use it under unless they pick another. A package that others use must have one; a program needs none. | +| `version` | a version | The package's version. A released package's must be the version of its release's tag. | +| `stone` | a version | The oldest stone the package works with. An older stone reports an error at this key. | +| `description` | a string | What the package is for. | +| `license` | a string | Its license, as an SPDX expression such as `MIT` or `Apache-2.0`. | + +A **stone name** is made of ASCII letters, digits, and `_`, does not start with a digit, and is +not a keyword. A `-` is not allowed, so the package in a repository `stone-json` is named +something like `json` or `stone_json`. + +## [dependencies] + +Each key is a dependency's name, which the package's files write in `use` (`use json`, +`use json.encode`). Besides being a stone name, it cannot be a builtin module (`os`, `math`, +`net`, ...), `prelude`, or a builtin (`print`, `some`, ...), and it cannot also be a module or +directory of the package. Its value is an inline table that says where it comes from: + +| form | where it comes from | +| --- | --- | +| `{ github = "owner/repo", version = "1.2.0" }` | releases of a repository on github.com | +| `{ github = "owner/repo", version = "1.2.0", host = "github.example.com" }` | releases on GitHub Enterprise | +| `{ gitlab = "group/project", version = "1.2.0" }` | releases of a project on gitlab.com, whose path may have subgroups, as `group/subgroup/project` | +| `{ gitlab = "group/project", version = "1.2.0", host = "gitlab.example.com" }` | releases on another GitLab | +| `{ path = "../geo" }` | the directory `../geo`, relative to this `stone.toml` | + +A dependency from a host must give a `version`: the oldest version it works with. stone uses the +newest version that any package of the program asks for, so this is a minimum, never a range or +an exact pin. A directory has no version and no host, and a package from a host cannot depend on +one. `host` is a host name, such as `gitlab.example.com`, without `https://`. + +## Versions + +A version is `major.minor.patch`, as `1.2.0`. A missing minor or patch is 0, so `1.2` is `1.2.0` +and `1` is `1.0.0`. A pre-release follows a `-`, as `2.0.0-beta.1`, and build metadata after a `+` +is ignored. Versions in a manifest have no `v`, but release tags may: `v1.2.0` and `1.2.0` are +both version 1.2.0. + +Versions with the same major number, or before 1.0 the same minor number, are compatible, and a +program uses one version of each such class: 1.2.0 and 1.5.3 are one class, and 2.0.0, 0.3.1, and +0.4.0 each another. A program can use several classes of one package at once, which are linked +apart. + +## The TOML it reads + +stone reads `stone.toml` and `stone.lock` itself, and reads the part of TOML they use: comments, +`[table]` and `[[array of tables]]` headers with a single name, `key = value` lines with bare or +quoted keys, and values that are strings (basic `"..."` or literal `'...'`, on one line), +integers, booleans, arrays, and inline tables (on one line). Dotted keys, multi-line strings, +floats, and dates are errors. + +## stone.lock + +stone writes `stone.lock` next to `stone.toml` the first time it downloads a package, and keeps +it current, so it is never edited by hand except to forget a package's hash. It has one entry per +package from a host that the program uses, sorted: + +```toml +# written by stone: the hash of what each package downloaded held +[[package]] +name = "json" +github = "alice/stone-json" +version = "1.2.0" +hash = "sha256:4f1c..." +``` + +`gitlab` replaces `github` for a project on GitLab, and `host` follows for another host. `hash` is +the SHA-256 of a listing of the package's kept files, one line per file of its own SHA-256, two +spaces, and its path. A package whose files no longer match is an error. Only a program's own +`stone.lock` is read; a package's is left out of its release. + +## The package store + +Downloaded packages are kept in `$STONE_HOME/packages` if `STONE_HOME` is set, and otherwise in +`~/.stone/packages` (`%LOCALAPPDATA%\stone\packages` on Windows), one directory per version: + +```text +~/.stone/packages/ + github.com/alice/stone-json/1.2.0/ + gitlab.com/bob/num/0.3.0/ +``` + +Each holds the release's `stone.toml` and `.stn` files only. Every program shares the store, and +`stone run --offline` and the editor read only from it. diff --git a/docs/reference/modules.md b/docs/reference/modules.md index 30c7ccc..68ed450 100644 --- a/docs/reference/modules.md +++ b/docs/reference/modules.md @@ -112,6 +112,16 @@ A module's functions see their parameters and locals, their own file's functions the names their file binds with `use`. They never see the entry file's variables or functions: reading one is `undefined name` or `undefined function`, as for any unknown name. +## Packages + +A program whose entry file has a `stone.toml` beside it can use packages, other directories of +modules that it names under `[dependencies]` (see {doc}`../guides/packages` and +{doc}`manifest`). A path that starts with a dependency's name is in that package: `use geo` loads +the package's `lib.stn`, `use geo.shapes` its `shapes.stn`, and `use geo.area` the `pub fn area` +of its `lib.stn`. A dependency cannot share its name with a module or directory of the program. +A package's files follow the same rules as any module, see only the package's own modules and +dependencies, and name their own modules by their paths within the package. + ## Types The checker checks the whole program at once, the same way it does within one file. A module's diff --git a/editors/vscode/CHANGELOG.md b/editors/vscode/CHANGELOG.md index f204451..1dfa247 100644 --- a/editors/vscode/CHANGELOG.md +++ b/editors/vscode/CHANGELOG.md @@ -10,6 +10,23 @@ follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and versions f ## [Unreleased] +### Added + +- Packages work in the editor. A program's dependencies, from directories and from the package + store, are checked with it, go to definition opens their files, and `use` completes their + names and modules. A package that is not downloaded yet is an error at each `use` of it, with + the hint to run `stone fetch`. +- Errors in `stone.toml` and `stone.lock` are shown in those files, and editing either, or + running `stone fetch`, checks every open file again. +- A file inside a package (a directory with a `stone.toml`) is checked as part of the package, + with its own dependencies, even when the package has no `main.stn`. + +### Changed + +- A package's functions and types show with the names the program writes, as + `geo.shapes.area` rather than `geo.1.shapes.area`. +- A package's names cannot be renamed, since other programs use them too. + ## [0.4.0] - 2026-10-30 ### Added diff --git a/lsp/src/features.rs b/lsp/src/features.rs index 264e1b6..2702102 100644 --- a/lsp/src/features.rs +++ b/lsp/src/features.rs @@ -18,9 +18,10 @@ use stone::checker::{ }; use stone::diagnostic::Severity; use stone::lexer::Lexer; +use stone::package::{LIB, MANIFEST}; use stone::project::{ - DEFAULT_ENTRY, Import, Linked, MapSources, SourceFile, SourceMap, Sources, Target, link, - link_in, + DEFAULT_ENTRY, Import, Linked, MapSources, PackageInfo, SourceFile, SourceMap, Sources, Target, + link, link_in, shown_names, }; use stone::span::{FileId, Pos, Span}; use stone::stdlib::{ @@ -54,6 +55,14 @@ pub struct Document { pub public: HashSet, /// The directory module paths start from. pub root: PathBuf, + /// An error at each `use` of a package that could not be read, saying why. + unavailable: Vec, + /// The program, then every package it uses. + packages: Vec, + /// The package each file belongs to, by index into `packages`. + file_packages: Vec, + /// Where downloaded packages are kept, whose files are never changed. + store: Option, /// The document's tokens, which tell the text of its strings from code. tokens: Vec, encoding: Encoding, @@ -64,23 +73,29 @@ impl Document { /// requests until the text changes. pub fn new(text: String, encoding: Encoding) -> Self { let linked = link(Path::new(DEFAULT_ENTRY), &text, &MapSources::default()); - Self::from_linked(linked, FileId(0), encoding) + Self::from_linked(linked, FileId(0), None, encoding) } /// Analyzes the file at `path` as part of its program, reading every file from `sources`, or /// returns `None` if there is no file at `path`. /// - /// The program is the one whose entry file is the nearest `main.stn` in the file's directory or - /// one above it. If that program does not use the file yet, the file is checked as an entry - /// file of its own, from the program's root if it is in a directory under it. + /// The program's root is the nearest directory, the file's own or one above it, that holds a + /// `main.stn` or a package's `stone.toml`, and its entry file is that `main.stn` or, for a + /// library, its `lib.stn`. If that program does not use the file yet, the file is checked as + /// an entry file of its own, from the program's root if it is in a directory under it. pub fn in_program(path: &Path, sources: &dyn Sources, encoding: Encoding) -> Option { let text = sources.read(path)?; let linked = program_of(path, &text, sources); let file = linked.sources.find(path).unwrap_or(FileId(0)); - Some(Self::from_linked(linked, file, encoding)) + Some(Self::from_linked(linked, file, sources.store(), encoding)) } - fn from_linked(linked: Linked, file: FileId, encoding: Encoding) -> Self { + fn from_linked( + linked: Linked, + file: FileId, + store: Option, + encoding: Encoding, + ) -> Self { let analysis = stone::driver::analyze_linked(&linked); let unused = stone::unused::find(&linked, &analysis); let text = linked.sources.file(file).source.clone(); @@ -97,10 +112,43 @@ impl Document { imports: linked.imports, public: linked.public, root: linked.root, + unavailable: linked.unavailable, + packages: linked.packages, + file_packages: linked.file_packages, + store, encoding, } } + /// Returns `text` with the linked names of packages' modules written as the program's files + /// write them, as `geo.shapes.area` for `geo.1.shapes.area`. + fn shown(&self, text: &str) -> String { + shown_names(text, &self.packages) + } + + /// Returns the package `file` belongs to: its index in `packages`. + fn package_of(&self, file: FileId) -> usize { + self.file_packages + .get(file.0 as usize) + .copied() + .unwrap_or(0) + } + + /// Returns the name of the package whose code `file` is, if it is not the program's own to + /// change: a package the program depends on, or a program in the store, which is a + /// downloaded package opened on its own. + fn package_owning(&self, file: FileId) -> Option { + let source = self.sources.file(file); + if let Some(package) = &source.package { + return Some(package.clone()); + } + let stored = self + .store + .as_ref() + .is_some_and(|store| source.path.starts_with(store)); + stored.then(|| self.packages[self.package_of(file)].name.clone()) + } + /// Returns the symbol whose name is at `position`, with the span of that appearance. fn symbol_at(&self, position: Position) -> Option<(usize, Span)> { let reference = self @@ -219,19 +267,26 @@ impl Document { /// Links the program the file at `path` belongs to. See [`Document::in_program`]. fn program_of(path: &Path, text: &str, sources: &dyn Sources) -> Linked { let dir = path.parent().unwrap_or(Path::new("")); + let holds = |d: &Path, name: &str| sources.read(&d.join(name)).is_some(); let root = dir .ancestors() - .find(|d| sources.read(&d.join(DEFAULT_ENTRY)).is_some()); + .find(|d| holds(d, DEFAULT_ENTRY) || holds(d, MANIFEST)); let Some(root) = root else { return link(path, text, sources); }; - let main = root.join(DEFAULT_ENTRY); - if main == path { - return link(path, text, sources); - } - let linked = link(&main, &sources.read(&main).unwrap_or_default(), sources); - if linked.sources.find(path).is_some() { - return linked; + let lib = format!("{LIB}.stn"); + let entry = [DEFAULT_ENTRY, lib.as_str()] + .into_iter() + .map(|name| root.join(name)) + .find(|entry| sources.read(entry).is_some()); + if let Some(entry) = entry { + if entry == path { + return link(path, text, sources); + } + let linked = link(&entry, &sources.read(&entry).unwrap_or_default(), sources); + if linked.sources.find(path).is_some() { + return linked; + } } // not used by the program yet if dir == root { @@ -244,29 +299,65 @@ fn program_of(path: &Path, text: &str, sources: &dyn Sources) -> Linked { /// Returns the document's problems as LSP diagnostics. pub fn diagnostics(doc: &Document) -> Vec { let found = doc.analysis.diagnostics.iter().map(|d| (d, false)); + // the program's errors say once, in the manifest, why a package cannot be used, and an + // editor shows it at each `use` of it too + let unavailable = doc.unavailable.iter().map(|d| (d, false)); // an editor fades what is unused as well as underlining it let unused = doc.unused.iter().map(|d| (d, true)); let mut diagnostics: Vec<_> = found + .chain(unavailable) .chain(unused) .filter(|(d, _)| d.span.file == doc.file) .collect(); diagnostics.sort_by_key(|(d, _)| d.span.start); diagnostics .into_iter() - .map(|(d, unused)| Diagnostic { - range: doc.index.range(d.span), - severity: Some(match d.severity { - Severity::Error => DiagnosticSeverity::ERROR, - Severity::Warning => DiagnosticSeverity::WARNING, - }), - source: Some("stone".to_string()), - message: d.message.clone(), - tags: unused.then(|| vec![DiagnosticTag::UNNECESSARY]), - ..Diagnostic::default() + .map(|(d, unused)| lsp_diagnostic(doc, &doc.index, d, unused)) + .collect() +} + +/// Returns the diagnostics of each `stone.toml` and `stone.lock` of the program `doc` belongs +/// to, by the file's URI. A file with none is included too, so an editor clears what it showed +/// before. +pub fn package_diagnostics(doc: &Document) -> Vec<(Uri, Vec)> { + doc.sources + .files() + .filter(|(_, file)| file.is_package_file()) + .filter_map(|(id, file)| { + let index = LineIndex::new(&file.source, doc.encoding); + let found = doc + .analysis + .diagnostics + .iter() + .filter(|d| d.span.file == id) + .map(|d| lsp_diagnostic(doc, &index, d, false)) + .collect(); + Some((file_uri(&file.path)?, found)) }) .collect() } +/// Converts a diagnostic of a file whose lines `index` knows to the protocol's, tagged as +/// `unused` code if it is one. +fn lsp_diagnostic( + doc: &Document, + index: &LineIndex, + d: &stone::diagnostic::Diagnostic, + unused: bool, +) -> Diagnostic { + Diagnostic { + range: index.range(d.span), + severity: Some(match d.severity { + Severity::Error => DiagnosticSeverity::ERROR, + Severity::Warning => DiagnosticSeverity::WARNING, + }), + source: Some("stone".to_string()), + message: doc.shown(&d.message), + tags: unused.then(|| vec![DiagnosticTag::UNNECESSARY]), + ..Diagnostic::default() + } +} + /// Describes a symbol the way stone would declare it, such as `fn add(a: int) -> int`, /// `fn greet(name: str)`, or `(local) sum: int`. fn signature(analysis: &Analysis, symbol: &Symbol) -> String { @@ -710,6 +801,15 @@ fn markdown(code: &str, text: Option<&str>) -> HoverContents { /// Describes what is at `position`: a symbol's declaration with the comment above its definition /// (see [`doc_comment`]), a builtin's documentation, or the type of the innermost expression there. pub fn hover(doc: &Document, position: Position) -> Option { + let mut hover = hover_at(doc, position)?; + if let HoverContents::Markup(markup) = &mut hover.contents { + markup.value = doc.shown(&markup.value); + } + Some(hover) +} + +/// Does the work of [`hover`], with names as they are linked. +fn hover_at(doc: &Document, position: Position) -> Option { let pos = doc.index.pos(position); if !doc.in_code(pos) { return expression_hover(doc, pos); @@ -978,9 +1078,12 @@ pub fn references( pub fn prepare_rename(doc: &Document, position: Position) -> Option { let (symbol, span) = doc.symbol_at(position)?; // the prelude's names, such as `some`, and those of modules like `net` belong to the - // language + // language, and a package's to the package let file = doc.analysis.symbols[symbol].span.file; - if file == FileId::PRELUDE || doc.sources.file(file).is_builtin() { + if file == FileId::PRELUDE + || doc.sources.file(file).is_builtin() + || doc.package_owning(file).is_some() + { return None; } Some(PrepareRenameResponse::Range(doc.index.range(span))) @@ -1012,6 +1115,13 @@ pub fn rename( if symbol.span.file == FileId::PRELUDE || doc.sources.file(symbol.span.file).is_builtin() { return Err(format!("'{}' is a builtin", doc.local_name(symbol))); } + // renaming it would change the package's files, which other programs use too + if let Some(package) = doc.package_owning(symbol.span.file) { + return Err(format!( + "'{}' belongs to the package '{package}'", + doc.local_name(symbol) + )); + } if is_reserved(new_name) { return Err(format!("'{new_name}' is a builtin")); } @@ -1218,6 +1328,17 @@ pub fn completion( position: Position, sources: &dyn Sources, ) -> Vec { + let mut items = completion_at(doc, position, sources); + for item in &mut items { + if let Some(detail) = &mut item.detail { + *detail = doc.shown(detail); + } + } + items +} + +/// Does the work of [`completion`], with names as they are linked. +fn completion_at(doc: &Document, position: Position, sources: &dyn Sources) -> Vec { let pos = doc.index.pos(position); if !doc.in_code(pos) { return vec![]; @@ -1917,15 +2038,27 @@ fn builtin_module_functions(module: &str) -> Vec { fn use_completion(doc: &Document, path: &str, sources: &dyn Sources) -> Vec { let parent = path.rsplit_once('.').map_or("", |(parent, _)| parent); let names: Vec<&str> = parent.split('.').filter(|n| !n.is_empty()).collect(); - let dir = names.iter().fold(doc.root.clone(), |dir, n| dir.join(n)); + // a path that starts with a dependency's name is in that package's directory + let package = &doc.packages[doc.package_of(doc.file)]; + let dependency = names.first().and_then(|first| { + let (_, i) = package.deps.iter().find(|(name, _)| name == first)?; + Some(&doc.packages[*i]) + }); + let (root, within) = match dependency { + Some(dependency) => (dependency.root.clone(), &names[1..]), + None => (doc.root.clone(), &names[..]), + }; + let dir = within.iter().fold(root, |dir, n| dir.join(n)); let mut items: Vec = vec![]; for entry in sources.entries(&dir) { let Some(name) = entry.file_stem().and_then(|s| s.to_str()) else { continue; }; let kind = if entry.extension().is_some_and(|ext| ext == "stn") { - // the entry file cannot be imported - if doc.sources.find(&entry) == Some(FileId(0)) { + // the entry file cannot be imported, and a package's lib.stn is the package itself + if doc.sources.find(&entry) == Some(FileId(0)) + || (dependency.is_some() && within.is_empty() && name == LIB) + { continue; } CompletionItemKind::MODULE @@ -1944,6 +2077,22 @@ fn use_completion(doc: &Document, path: &str, sources: &dyn Sources) -> Vec Vec dependency.prefix.clone(), + within => format!("{}.{}", dependency.prefix, within.join(".")), + }; + if let Some((file, _)) = doc.sources.files().find(|(_, f)| f.module == module) { + items.extend(module_functions(doc, file)); + } else if doc.package_of(doc.file) == 0 { + // a module the program does not use yet, loaded as the program would load it + let entry = doc.root.join(DEFAULT_ENTRY); + let used = link(&entry, &format!("use {}\n", names.join(".")), sources); + let used = Document::from_linked(used, FileId(0), None, doc.encoding); + if let Some((file, _)) = used.sources.files().find(|(_, f)| f.module == module) { + items.extend(module_functions(&used, file)); + } + } + } else if let [name] = names[..] && stone::stdlib::module(name).is_some() { items.extend(builtin_module_functions(name)); @@ -1972,7 +2138,7 @@ fn use_completion(doc: &Document, path: &str, sources: &dyn Sources) -> Vec MapSources { + let mut sources = MapSources::default(); + for (path, text) in files { + sources.insert(*path, *text); + } + sources.set_store("/store"); + sources +} + +/// Opens the file at `path` of `sources` as part of the program it belongs to. +fn packaged_doc(path: &str, sources: &MapSources) -> Document { + Document::in_program(Path::new(path), sources, Encoding::Utf16).expect("the file should exist") +} + +const APP_MANIFEST: (&str, &str) = ( + "/p/stone.toml", + "[dependencies]\ngeo = { path = \"../geo\" }\n\ + json = { github = \"alice/json\", version = \"1.2\" }\n", +); +const APP_MAIN: (&str, &str) = ( + "/p/main.stn", + "use geo.shapes\nuse json\nprint(shapes.area(2), json.v())\n", +); +const GEO_FILES: [(&str, &str); 3] = [ + ( + "/geo/stone.toml", + "[package]\nname = \"geo\"\nversion = \"1.2.0\"\n", + ), + ( + "/geo/shapes.stn", + "pub fn area(x: int) -> int:\n ret x * x\n", + ), + ("/geo/lib.stn", "pub fn unit() -> int:\n ret 1\n"), +]; +const JSON_FILES: [(&str, &str); 2] = [ + ( + "/store/github.com/alice/json/1.2.0/stone.toml", + "[package]\nname = \"json\"\nversion = \"1.2.0\"\n", + ), + ( + "/store/github.com/alice/json/1.2.0/lib.stn", + "pub fn v() -> str:\n ret \"1.2.0\"\n", + ), +]; + +/// The files of a program that uses a package in a directory and one downloaded into the store. +fn app_files() -> Vec<(&'static str, &'static str)> { + let mut files = vec![APP_MAIN, APP_MANIFEST]; + files.extend(GEO_FILES); + files.extend(JSON_FILES); + files +} + +#[test] +fn a_program_with_packages_has_no_errors() { + let doc = packaged_doc("/p/main.stn", &packaged(&app_files())); + assert_eq!(diagnostics(&doc), []); +} + +#[test] +fn a_use_of_a_package_that_is_not_downloaded_says_to_fetch_it() { + let mut files = vec![APP_MAIN, APP_MANIFEST]; + files.extend(GEO_FILES); + let doc = packaged_doc("/p/main.stn", &packaged(&files)); + let found = diagnostics(&doc); + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].range, Range::new(at(2, 5), at(2, 9))); + assert_eq!( + found[0].message, + "'json' cannot be used: json 1.2.0 (github:alice/json) is not downloaded; run `stone fetch`" + ); +} + +#[test] +fn errors_in_a_manifest_are_published_for_its_file() { + let mut files = vec![APP_MAIN, APP_MANIFEST]; + files.extend(GEO_FILES); + let doc = packaged_doc("/p/main.stn", &packaged(&files)); + let published = package_diagnostics(&doc); + let manifest: Uri = "file:///p/stone.toml".parse().unwrap(); + let (_, found) = published + .iter() + .find(|(uri, _)| *uri == manifest) + .expect("the program's manifest"); + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].range.start, at(3, 1)); + assert_eq!( + found[0].message, + "json 1.2.0 (github:alice/json) is not downloaded" + ); + // every manifest is published, so one that is fixed has its errors cleared + let geo: Uri = "file:///geo/stone.toml".parse().unwrap(); + assert!( + published + .iter() + .any(|(uri, found)| *uri == geo && found.is_empty()) + ); +} + +#[test] +fn definition_opens_a_packages_file() { + let doc = packaged_doc("/p/main.stn", &packaged(&app_files())); + let location = |pos| definition(&doc, &uri_of("main.stn"), pos, None).unwrap(); + // `area` in `shapes.area(2)`, from a directory + let area = location(at(3, 15)); + assert_eq!(area.uri.as_str(), "file:///geo/shapes.stn"); + assert_eq!(area.range.start, at(1, 8)); + // `v` in `json.v()`, from the store + let v = location(at(3, 28)); + assert_eq!( + v.uri.as_str(), + "file:///store/github.com/alice/json/1.2.0/lib.stn" + ); + // the `use` of a package's module opens it + assert_eq!(location(at(1, 10)).uri.as_str(), "file:///geo/shapes.stn"); +} + +#[test] +fn a_packages_names_cannot_be_renamed() { + let doc = packaged_doc("/p/main.stn", &packaged(&app_files())); + assert_eq!(prepare_rename(&doc, at(3, 15)), None); + assert_eq!( + rename(&doc, &uri_of("main.stn"), at(3, 15), "size"), + Err("'area' belongs to the package 'geo'".to_string()) + ); +} + +#[test] +fn a_downloaded_packages_own_files_cannot_be_renamed() { + let sources = packaged(&app_files()); + let doc = packaged_doc("/store/github.com/alice/json/1.2.0/lib.stn", &sources); + assert_eq!(prepare_rename(&doc, at(1, 8)), None); + assert_eq!( + rename(&doc, &uri_of("lib.stn"), at(1, 8), "w"), + Err("'v' belongs to the package 'json'".to_string()) + ); +} + +#[test] +fn hover_names_a_packages_function_as_the_program_does() { + let doc = packaged_doc("/p/main.stn", &packaged(&app_files())); + assert_eq!( + hover_text(&doc, at(3, 15)).unwrap(), + "```stone\nfn geo.shapes.area(x: int) -> int\n```" + ); + assert_eq!( + hover_text(&doc, at(1, 10)).unwrap(), + "```stone\nmodule geo.shapes\n```" + ); +} + +#[test] +fn use_completion_offers_dependencies_and_their_modules() { + let files = [ + ("/p/main.stn", "use \nuse geo.\nuse json.\n"), + APP_MANIFEST, + GEO_FILES[0], + GEO_FILES[1], + GEO_FILES[2], + JSON_FILES[0], + JSON_FILES[1], + ]; + let sources = packaged(&files); + let doc = packaged_doc("/p/main.stn", &sources); + let items = |pos: Position| -> Vec<(String, Option)> { + let mut items: Vec<(String, Option)> = completion(&doc, pos, &sources) + .into_iter() + .map(|i| (i.label, i.detail)) + .collect(); + items.sort(); + items + }; + let first = items(at(1, 5)); + assert!( + first.contains(&("geo".to_string(), Some("package geo 1.2.0".to_string()))), + "{first:?}" + ); + assert!(first.contains(&("json".to_string(), Some("package json 1.2.0".to_string())))); + // a package's modules, and the functions of its lib.stn, which is not a module of its own + assert_eq!( + items(at(2, 9)), + [ + ("shapes".to_string(), None), + ("unit".to_string(), Some("fn geo.unit() -> int".to_string())), + ] + ); + assert_eq!( + items(at(3, 10)), + [("v".to_string(), Some("fn json.v() -> str".to_string()))] + ); +} + +#[test] +fn a_file_in_a_librarys_directory_is_checked_with_its_dependencies() { + let files = [ + ( + "/lib/stone.toml", + "[package]\nname = \"lib\"\n\n[dependencies]\ngeo = { path = \"../geo\" }\n", + ), + ( + "/lib/lib.stn", + "use parts.sum\npub fn total() -> int:\n ret sum.two()\n", + ), + ( + "/lib/parts/sum.stn", + "use geo\npub fn two() -> int:\n ret geo.unit() + geo.unit()\n", + ), + GEO_FILES[0], + GEO_FILES[2], + ]; + let doc = packaged_doc("/lib/parts/sum.stn", &packaged(&files)); + assert_eq!(diagnostics(&doc), []); + assert_eq!( + doc.file, + doc.sources.find(Path::new("/lib/parts/sum.stn")).unwrap() + ); +} diff --git a/lsp/src/lib.rs b/lsp/src/lib.rs index 2011f8c..741d697 100644 --- a/lsp/src/lib.rs +++ b/lsp/src/lib.rs @@ -5,25 +5,26 @@ //! document changes, which publishes its diagnostics, and requests like hover and rename are //! answered from that analysis by [`features`]. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::error::Error; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use lsp_server::{Connection, ErrorCode, ExtractError, Message, Notification, Request, Response}; use lsp_types::notification::{ - DidChangeTextDocument, DidCloseTextDocument, DidOpenTextDocument, + DidChangeTextDocument, DidChangeWatchedFiles, DidCloseTextDocument, DidOpenTextDocument, Notification as LspNotification, PublishDiagnostics, }; use lsp_types::request::{ Completion, DocumentSymbolRequest, Formatting, GotoDefinition, HoverRequest, - PrepareRenameRequest, References, Rename, Request as LspRequest, + PrepareRenameRequest, References, RegisterCapability, Rename, Request as LspRequest, }; use lsp_types::{ - CompletionOptions, CompletionResponse, DocumentSymbolResponse, GotoDefinitionResponse, + CompletionOptions, CompletionResponse, Diagnostic, DidChangeWatchedFilesRegistrationOptions, + DocumentSymbolResponse, FileSystemWatcher, GlobPattern, GotoDefinitionResponse, HoverProviderCapability, InitializeParams, InitializeResult, OneOf, PositionEncodingKind, - PublishDiagnosticsParams, RenameOptions, ServerCapabilities, ServerInfo, - TextDocumentSyncCapability, TextDocumentSyncKind, Uri, WorkDoneProgressOptions, + PublishDiagnosticsParams, Registration, RegistrationParams, RenameOptions, ServerCapabilities, + ServerInfo, TextDocumentSyncCapability, TextDocumentSyncKind, Uri, WorkDoneProgressOptions, }; use stone::project::{FsSources, Sources}; @@ -52,11 +53,15 @@ pub fn run(connection: Connection) -> Result<(), Box> { }), }; connection.initialize_finish(id, serde_json::to_value(result)?)?; + if watches_files(¶ms) { + connection.sender.send(watch_packages()?.into())?; + } let mut server = Server { encoding, texts: HashMap::new(), documents: HashMap::new(), + published: HashSet::new(), builtins: write_builtins_reference().map(Builtins::new), }; for message in &connection.receiver { @@ -110,6 +115,41 @@ fn replace_file(path: &Path, text: &str) -> Option<()> { std::fs::rename(&temporary, path).ok() } +/// Returns whether the client can be asked to say when files change on disk. +fn watches_files(params: &InitializeParams) -> bool { + let watched = params + .capabilities + .workspace + .as_ref() + .and_then(|workspace| workspace.did_change_watched_files.as_ref()); + watched.and_then(|w| w.dynamic_registration) == Some(true) +} + +/// Returns the request that asks the client to say when a `stone.toml` or `stone.lock` changes, +/// as an edit to a manifest or `stone fetch` does, since either can change what is wrong in any +/// program and neither is a document the server is sent. +fn watch_packages() -> Result { + let watcher = |pattern: &str| FileSystemWatcher { + glob_pattern: GlobPattern::String(pattern.to_string()), + kind: None, + }; + let options = DidChangeWatchedFilesRegistrationOptions { + watchers: vec![watcher("**/stone.toml"), watcher("**/stone.lock")], + }; + let params = RegistrationParams { + registrations: vec![Registration { + id: "stone-packages".to_string(), + method: DidChangeWatchedFiles::METHOD.to_string(), + register_options: Some(serde_json::to_value(options)?), + }], + }; + Ok(Request::new( + "stone-packages".to_string().into(), + RegisterCapability::METHOD.to_string(), + params, + )) +} + /// Picks UTF-8 positions if the client supports them, and UTF-16 otherwise. fn negotiate_encoding(params: &InitializeParams) -> Encoding { let offered = params @@ -156,6 +196,9 @@ struct Server { texts: HashMap, /// The analysis of each open document. documents: HashMap, + /// The `stone.toml` and `stone.lock` files whose diagnostics were published last, which are + /// cleared once no open document's program has them. + published: HashSet, /// Where definitions of builtins go, or `None` if the reference file could not be written. builtins: Option, } @@ -291,6 +334,8 @@ impl Server { }; (params.text_document.uri, None) } + // a manifest or lockfile changed on disk + DidChangeWatchedFiles::METHOD => return self.reanalyze(None), _ => return vec![], }; @@ -303,7 +348,13 @@ impl Server { self.documents.remove(&uri); } } + self.reanalyze(Some(uri)) + } + /// Analyzes every open document again, returning the diagnostics to publish: those of + /// `changed` first, which may have been closed, then the other documents', then those of + /// the packages' files. + fn reanalyze(&mut self, changed: Option) -> Vec { // a change to one file can change what is wrong in any other file of its program let workspace = self.workspace(); self.documents = self @@ -318,10 +369,53 @@ impl Server { .collect(); // the changed document first, and a closed one's diagnostics are cleared - let mut uris: Vec<&Uri> = self.documents.keys().filter(|u| **u != uri).collect(); + let mut uris: Vec<&Uri> = self + .documents + .keys() + .filter(|u| Some(*u) != changed.as_ref()) + .collect(); uris.sort_by_key(|u| u.as_str()); - let mut replies = vec![publish(&uri, self.documents.get(&uri))]; + let mut replies: Vec = changed + .iter() + .map(|uri| publish(uri, self.documents.get(uri))) + .collect(); replies.extend(uris.into_iter().map(|u| publish(u, self.documents.get(u)))); + + // a package file may belong to several programs, which find the same errors in it + let mut packages: Vec<(Uri, Vec)> = vec![]; + for doc in self.documents.values() { + for (uri, found) in features::package_diagnostics(doc) { + let at = match packages.iter().position(|(u, _)| *u == uri) { + Some(at) => at, + None => { + packages.push((uri, vec![])); + packages.len() - 1 + } + }; + for diagnostic in found { + if !packages[at].1.contains(&diagnostic) { + packages[at].1.push(diagnostic); + } + } + } + } + packages.sort_by_key(|(u, _)| u.as_str().to_string()); + let mut cleared: Vec = self + .published + .iter() + .filter(|uri| !packages.iter().any(|(u, _)| u == *uri)) + .cloned() + .collect(); + cleared.sort_by_key(|u| u.as_str().to_string()); + self.published = packages.iter().map(|(uri, _)| uri.clone()).collect(); + for (uri, found) in packages { + let params = PublishDiagnosticsParams::new(uri, found, None); + replies.push(Notification::new( + PublishDiagnostics::METHOD.to_string(), + params, + )); + } + replies.extend(cleared.iter().map(|uri| publish(uri, None))); replies } diff --git a/lsp/tests/server.rs b/lsp/tests/server.rs index dbf84c2..eafc8d9 100644 --- a/lsp/tests/server.rs +++ b/lsp/tests/server.rs @@ -1,5 +1,6 @@ //! Drives the language server over an in-memory connection, the way an editor would. +use std::path::{Path, PathBuf}; use std::thread::JoinHandle; use std::time::Duration; @@ -19,6 +20,15 @@ impl Session { /// Starts a server and completes the initialize handshake, offering `encodings` as the /// client's position encodings. Returns the session and the server's capabilities. fn start(encodings: Option<&[&str]>) -> (Session, Value) { + let capabilities = match encodings { + Some(encodings) => json!({ "general": { "positionEncodings": encodings } }), + None => json!({}), + }; + Session::start_with(capabilities) + } + + /// Starts a server like [`Session::start`], for a client with these capabilities. + fn start_with(capabilities: Value) -> (Session, Value) { let (server, client) = Connection::memory(); let handle = std::thread::spawn(move || stone_lsp::run(server).unwrap()); let mut session = Session { @@ -26,10 +36,6 @@ impl Session { server: Some(handle), next_id: 0, }; - let capabilities = match encodings { - Some(encodings) => json!({ "general": { "positionEncodings": encodings } }), - None => json!({}), - }; let result = session.request("initialize", json!({ "capabilities": capabilities })); session.notify("initialized", json!({})); (session, result["capabilities"].clone()) @@ -76,11 +82,18 @@ impl Session { /// Returns the next diagnostics the server publishes. fn diagnostics(&self) -> Value { + let (uri, diagnostics) = self.published(); + assert_eq!(uri, URI); + diagnostics + } + + /// Returns the next diagnostics the server publishes, and the URI of the file they are for. + fn published(&self) -> (String, Value) { match self.receive() { - Message::Notification(n) if n.method == "textDocument/publishDiagnostics" => { - assert_eq!(n.params["uri"], URI); - n.params["diagnostics"].clone() - } + Message::Notification(n) if n.method == "textDocument/publishDiagnostics" => ( + n.params["uri"].as_str().unwrap().to_string(), + n.params["diagnostics"].clone(), + ), other => panic!("expected diagnostics, got {other:?}"), } } @@ -330,3 +343,84 @@ fn a_change_to_one_file_republishes_the_others_in_its_program() { assert_eq!(published(2), [(util.to_string(), 0), (main.to_string(), 1)]); session.shut_down(); } + +/// Returns a new empty directory for a test named `name`. +fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("stone-lsp-test-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn uri_of(path: &Path) -> String { + stone_lsp::uri::file_uri(path).unwrap().as_str().to_string() +} + +#[test] +fn watches_manifests_and_publishes_their_errors() { + let dir = scratch("manifests"); + let (app, geo) = (dir.join("app"), dir.join("geo")); + std::fs::create_dir_all(&app).unwrap(); + std::fs::create_dir_all(&geo).unwrap(); + let main = "use geo\nprint(geo.one())\n"; + std::fs::write(app.join("main.stn"), main).unwrap(); + let manifest = "[dependencies]\ngeo = { path = \"../geo\" }\n"; + std::fs::write(app.join("stone.toml"), manifest).unwrap(); + std::fs::write(geo.join("lib.stn"), "pub fn one() -> int:\n ret 1\n").unwrap(); + + let (session, _) = Session::start_with(json!({ + "workspace": { "didChangeWatchedFiles": { "dynamicRegistration": true } } + })); + // the server asks to hear of changes to manifests and lockfiles + let request = match session.receive() { + Message::Request(request) => request, + other => panic!("expected a request, got {other:?}"), + }; + assert_eq!(request.method, "client/registerCapability"); + let registration = &request.params["registrations"][0]; + assert_eq!(registration["method"], "workspace/didChangeWatchedFiles"); + assert_eq!( + registration["registerOptions"]["watchers"], + json!([{ "globPattern": "**/stone.toml" }, { "globPattern": "**/stone.lock" }]) + ); + let reply = Response::new_ok(request.id, Value::Null); + session.client.sender.send(reply.into()).unwrap(); + + let main_uri = uri_of(&app.join("main.stn")); + session.notify( + "textDocument/didOpen", + json!({ + "textDocument": { "uri": main_uri, "languageId": "stone", "version": 1, "text": main } + }), + ); + // `geo` has no stone.toml yet, which the `use` and the program's manifest both say + let (uri, found) = session.published(); + assert_eq!(uri, main_uri); + assert_eq!( + found[0]["message"], + "'geo' cannot be used: '../geo' has no stone.toml, so it is not a package" + ); + let manifest_uri = uri_of(&app.join("stone.toml")); + let (uri, found) = session.published(); + assert_eq!(uri, manifest_uri); + assert_eq!( + found[0]["range"]["start"], + json!({ "line": 1, "character": 0 }) + ); + + std::fs::write(geo.join("stone.toml"), "[package]\nname = \"geo\"\n").unwrap(); + let geo_manifest = uri_of(&geo.join("stone.toml")); + session.notify( + "workspace/didChangeWatchedFiles", + json!({ "changes": [{ "uri": geo_manifest, "type": 1 }] }), + ); + assert_eq!(session.published(), (main_uri, json!([]))); + // both manifests, the fixed one cleared + let mut rest = vec![session.published(), session.published()]; + rest.sort_by(|a, b| a.0.cmp(&b.0)); + let mut expected = vec![(manifest_uri, json!([])), (geo_manifest, json!([]))]; + expected.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!(rest, expected); + session.shut_down(); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/src/deps.rs b/src/deps.rs new file mode 100644 index 0000000..6784173 --- /dev/null +++ b/src/deps.rs @@ -0,0 +1,261 @@ +//! `stone add`, `stone remove`, `stone fetch`, and `stone tree`, which manage the packages a +//! program uses (see `stone::package`), and the downloading `run`, `build`, and `check` do first. +//! +//! Downloading needs the `fetch` feature, which release builds have. Without it, `remove` and +//! `tree` still work, and so does building a program whose packages are already in the store. + +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use stone::diagnostic::Severity; +use stone::package::MANIFEST; +use stone::package::edit; +use stone::project::{self, DEFAULT_ENTRY, FsSources, Linked, PackageInfo, Sources}; + +/// What a build without the `fetch` feature says when asked to download. +#[cfg(not(feature = "fetch"))] +const NO_FETCH: &str = "this build of stone cannot download packages; install a release of stone \ + from https://github.com/tarolling/stone/releases, or build it with \ + `cargo install --features fetch`"; + +/// Returns the entry file of the program `path` names: the file itself, or the `main.stn` of a +/// directory, or of the current directory if there is no path. Only its manifest matters, so it +/// need not exist. +fn entry_of(path: Option<&Path>) -> PathBuf { + match path { + None => PathBuf::from(DEFAULT_ENTRY), + Some(path) if path.is_dir() => path.join(DEFAULT_ENTRY), + Some(path) => path.to_path_buf(), + } +} + +/// Downloads every package the program whose entry file is `entry` uses that is not in the +/// store yet, before `run`, `build`, or `check` links it, unless `offline`. Any error is printed, +/// and linking then reports which packages are missing. +pub fn prepare(entry: &Path, offline: bool) { + let has_manifest = entry + .parent() + .map_or(Path::new(MANIFEST).to_path_buf(), |dir| dir.join(MANIFEST)) + .is_file(); + #[cfg(feature = "fetch")] + if !offline && has_manifest { + let mut client = stone::package::fetch::Client::new(); + if let Err(e) = stone::package::fetch::fetch_missing(entry, &mut client, &mut log) { + eprintln!("error: {e}"); + } + } + #[cfg(not(feature = "fetch"))] + let _ = (offline, has_manifest); +} + +/// Prints a line about downloading. +#[cfg(feature = "fetch")] +fn log(line: String) { + eprintln!("{line}"); +} + +/// Prints the errors of a link of manifests and writes its lockfile if it should change, +/// returning whether there were no errors. +fn finish(linked: &Linked) -> bool { + let errors: Vec<_> = linked + .diagnostics + .iter() + .filter(|d| d.severity == Severity::Error) + .collect(); + let rendered: String = errors.iter().map(|d| linked.sources.render(d)).collect(); + eprint!("{rendered}"); + if !errors.is_empty() { + return false; + } + if let Some((path, text)) = &linked.lock + && let Err(e) = FsSources.write(path, text) + { + eprintln!("error: {e}"); + return false; + } + true +} + +/// Turns the outcome of a command into its exit code, printing its error. +fn exit(result: Result) -> ExitCode { + match result { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::FAILURE, + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } +} + +/// Runs `stone add`: finds the release `spec` asks for, downloads it, names it in the current +/// directory's `stone.toml` (made if there is none) under `name` or the package's own name, and +/// downloads whatever else the program now uses. +pub fn add(spec: &str, name: Option<&str>, host: Option<&str>) -> ExitCode { + exit(add_package(spec, name, host)) +} + +#[cfg(not(feature = "fetch"))] +fn add_package(_: &str, _: Option<&str>, _: Option<&str>) -> Result { + Err(NO_FETCH.to_string()) +} + +#[cfg(feature = "fetch")] +fn add_package(spec: &str, name: Option<&str>, host: Option<&str>) -> Result { + use stone::package::{Manifest, dependency_name_error}; + use stone::span::FileId; + + let (source, wanted) = edit::parse_spec(spec, host)?; + let mut client = stone::package::fetch::Client::new(); + let release = client.newest(&source, &wanted)?; + let version = release.version; + let store = FsSources + .store() + .ok_or("there is no directory to keep packages in; set STONE_HOME")?; + let dir = stone::package::store::package_dir(&store, &source, &version); + if !dir.join(MANIFEST).is_file() { + log(format!("downloading {source} {version}")); + } + let dir = client + .download(&store, &source, &version) + .map_err(|e| format!("cannot download {source} {version}: {e}"))?; + let text = std::fs::read_to_string(dir.join(MANIFEST)).map_err(|e| e.to_string())?; + let (manifest, _) = Manifest::parse(&text, FileId(0)); + let name = match (name, manifest.name) { + (Some(name), _) => name.to_string(), + (None, Some((name, _))) => name, + (None, None) => { + return Err(format!( + "{source} {version} does not name itself in its {MANIFEST}, so give it a name \ + with --name" + )); + } + }; + if let Some(message) = dependency_name_error(&name) { + return Err(format!("{message}, with --name")); + } + let old = std::fs::read_to_string(MANIFEST).unwrap_or_default(); + let new = edit::set_dependency(&old, &name, &edit::dependency_value(&source, &version))?; + std::fs::write(MANIFEST, new).map_err(|e| format!("cannot write {MANIFEST}: {e}"))?; + let linked = + stone::package::fetch::fetch_missing(Path::new(DEFAULT_ENTRY), &mut client, &mut log)?; + let ok = finish(&linked); + if ok { + println!("added {name} {version} ({source})"); + } + Ok(ok) +} + +/// Runs `stone remove`: takes the dependency `name` out of the current directory's +/// `stone.toml`, and out of `stone.lock` with any package only it used. +pub fn remove(name: &str) -> ExitCode { + exit((|| { + let old = std::fs::read_to_string(MANIFEST) + .map_err(|_| format!("there is no {MANIFEST} in this directory"))?; + let new = edit::remove_dependency(&old, name)?; + std::fs::write(MANIFEST, new).map_err(|e| format!("cannot write {MANIFEST}: {e}"))?; + let ok = finish(&project::link(Path::new(DEFAULT_ENTRY), "", &FsSources)); + if ok { + println!("removed {name}"); + } + Ok(ok) + })()) +} + +/// Runs `stone fetch`: downloads every package the program in `path` uses and brings its +/// `stone.lock` up to date, which with `locked` must already be. +pub fn fetch(path: Option<&Path>, locked: bool) -> ExitCode { + let entry = entry_of(path); + exit(fetch_all(&entry, locked)) +} + +#[cfg(not(feature = "fetch"))] +fn fetch_all(_: &Path, _: bool) -> Result { + Err(NO_FETCH.to_string()) +} + +#[cfg(feature = "fetch")] +fn fetch_all(entry: &Path, locked: bool) -> Result { + let mut client = stone::package::fetch::Client::new(); + let linked = stone::package::fetch::fetch_missing(entry, &mut client, &mut log)?; + if locked && let Some((path, _)) = &linked.lock { + let errors = linked + .diagnostics + .iter() + .any(|d| d.severity == Severity::Error); + if !errors { + return Err(format!( + "{} is not up to date, and --locked keeps it as it is; run `stone fetch` to \ + update it", + path.display() + )); + } + } + Ok(finish(&linked)) +} + +/// Runs `stone tree`: prints the packages the program in `path` uses, and the ones they use, as +/// a tree. A package seen before is marked `(*)` and not shown again. +pub fn tree(path: Option<&Path>) -> ExitCode { + let entry = entry_of(path); + let linked = project::link(&entry, "", &FsSources); + if linked.packages.is_empty() { + return ExitCode::SUCCESS; + } + let root = &linked.packages[0]; + let mut title = match root.name.as_str() { + "" => std::path::absolute(entry) + .ok() + .and_then(|entry| Some(entry.parent()?.file_name()?.to_string_lossy().into_owned())) + .unwrap_or("program".to_string()), + name => name.to_string(), + }; + if let Some(version) = &root.version { + title.push_str(&format!(" {version}")); + } + let mut lines = vec![title]; + let mut seen = vec![false; linked.packages.len()]; + seen[0] = true; + branches(&linked.packages, 0, "", &mut seen, &mut lines); + for line in lines { + println!("{line}"); + } + match finish(&linked) { + true => ExitCode::SUCCESS, + false => ExitCode::FAILURE, + } +} + +/// Adds the lines of the packages `packages[at]` uses, each after `indent`. +fn branches( + packages: &[PackageInfo], + at: usize, + indent: &str, + seen: &mut [bool], + lines: &mut Vec, +) { + let deps = &packages[at].deps; + for (n, (key, i)) in deps.iter().enumerate() { + let last = n + 1 == deps.len(); + let package = &packages[*i]; + let mut line = format!("{indent}{}", if last { "└── " } else { "├── " }); + if *key != package.name { + line.push_str(&format!("{key} = ")); + } + line.push_str(&package.name); + if let Some(version) = &package.version { + line.push_str(&format!(" {version}")); + } + if let Some(source) = &package.source { + line.push_str(&format!(" ({source})")); + } + if seen[*i] { + line.push_str(" (*)"); + lines.push(line); + continue; + } + seen[*i] = true; + lines.push(line); + let deeper = format!("{indent}{}", if last { " " } else { "│ " }); + branches(packages, *i, &deeper, seen, lines); + } +} diff --git a/src/driver.rs b/src/driver.rs index 2cea219..7c41835 100644 --- a/src/driver.rs +++ b/src/driver.rs @@ -78,12 +78,15 @@ pub fn load( source: &str, sources: &dyn Sources, ) -> (SourceMap, Result) { + let linked = project::link(entry, source, sources); + let lock_error = write_lock(&linked, sources); let Linked { module, sources, - diagnostics, + mut diagnostics, .. - } = project::link(entry, source, sources); + } = linked; + diagnostics.extend(lock_error); if has_errors(&diagnostics) { return (sources, Err(Diagnostics(diagnostics))); } @@ -100,6 +103,20 @@ pub fn load( (sources, Ok(checked)) } +/// Writes the `stone.lock` that linking found should change, if the program linked without +/// errors, returning the error if it cannot be written. +fn write_lock(linked: &Linked, sources: &dyn Sources) -> Option { + let (path, text) = linked.lock.as_ref()?; + if has_errors(&linked.diagnostics) { + return None; + } + let error = sources.write(path, text).err()?; + Some(Diagnostic::error( + crate::span::Span::empty(crate::span::Pos::new(1, 1)), + error, + )) +} + fn has_errors(diagnostics: &[Diagnostic]) -> bool { diagnostics.iter().any(|d| d.severity == Severity::Error) } @@ -154,6 +171,7 @@ pub fn check_program( let linked = project::link(entry, source, sources); let analysis = analyze_linked(&linked); let mut diagnostics = crate::unused::find(&linked, &analysis); + diagnostics.extend(write_lock(&linked, sources)); diagnostics.extend(analysis.diagnostics); // a program the host cannot run is an error where it runs if let Ok(host) = Target::host() { diff --git a/src/lib.rs b/src/lib.rs index bbf5fd6..ce270be 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ pub mod interpreter; pub mod last_use; pub mod lexer; pub mod lift; +pub mod package; pub mod parser; pub mod project; pub mod repl; diff --git a/src/main.rs b/src/main.rs index c5cfd25..1cb06b9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ use stone::format; use stone::interpreter::{Exit, Limits}; use stone::project::{FsSources, SourceMap}; +mod deps; mod uninstall; #[cfg(feature = "self-update")] mod update; @@ -34,6 +35,9 @@ struct Args { enum Command { /// Runs a file with the interpreter. Run { + /// Use only packages already downloaded. + #[arg(long)] + offline: bool, /// The file to run. file: String, /// Arguments for the program, which it reads with `args()`. @@ -61,6 +65,9 @@ enum Command { /// its output is redirected. Needs a Windows target. #[arg(long)] windows_gui: bool, + /// Use only packages already downloaded. + #[arg(long)] + offline: bool, }, /// Removes the build directory that `stone build` made. Clean { @@ -72,6 +79,41 @@ enum Command { Check { /// The file to check. file: String, + /// Use only packages already downloaded. + #[arg(long)] + offline: bool, + }, + /// Adds a package from a GitHub or GitLab release to the program in the current directory, + /// naming it in `stone.toml` and downloading it. + Add { + /// The package, as `github:owner/repo` or `gitlab:group/project`, or a link to one, + /// optionally followed by `@latest` (the default) or a version such as `@1.2`, which is + /// the newest 1.2.x. + package: String, + /// The name the program uses it under, instead of the package's own. + #[arg(long)] + name: Option, + /// The host of a GitLab of its own, such as `gitlab.example.com`. + #[arg(long)] + host: Option, + }, + /// Removes a package from the program in the current directory. + Remove { + /// The name the program uses it under. + name: String, + }, + /// Downloads every package a program uses and brings its `stone.lock` up to date. + Fetch { + /// A program's entry file or directory. Defaults to the current directory. + path: Option, + /// Fail if `stone.lock` would change, as in CI. + #[arg(long)] + locked: bool, + }, + /// Shows the packages a program uses, and the packages they use. + Tree { + /// A program's entry file or directory. Defaults to the current directory. + path: Option, }, /// Rewrites files in the standard layout. Fmt { @@ -108,8 +150,23 @@ enum ExecutionMode { targets: Vec, /// Whether Windows executables start with no console. windows_gui: bool, + offline: bool, + }, + Check { + file: String, + offline: bool, }, - Check(String), + Add { + package: String, + name: Option, + host: Option, + }, + Remove(String), + Fetch { + path: Option, + locked: bool, + }, + Tree(Option), Clean(Option), Fmt { paths: Vec, @@ -119,6 +176,7 @@ enum ExecutionMode { Run { file: String, args: Vec, + offline: bool, }, Uninstall { yes: bool, @@ -133,19 +191,41 @@ fn main() -> ExitCode { let args = Args::parse(); let mode = match args.command { - Some(Command::Run { file, args }) => ExecutionMode::Run { file, args }, + Some(Command::Run { + file, + args, + offline, + }) => ExecutionMode::Run { + file, + args, + offline, + }, Some(Command::Build { files, output, targets, windows_gui, + offline, }) => ExecutionMode::Build { files, output, targets, windows_gui, + offline, + }, + Some(Command::Check { file, offline }) => ExecutionMode::Check { file, offline }, + Some(Command::Add { + package, + name, + host, + }) => ExecutionMode::Add { + package, + name, + host, }, - Some(Command::Check { file }) => ExecutionMode::Check(file), + Some(Command::Remove { name }) => ExecutionMode::Remove(name), + Some(Command::Fetch { path, locked }) => ExecutionMode::Fetch { path, locked }, + Some(Command::Tree { path }) => ExecutionMode::Tree(path), Some(Command::Clean { path }) => ExecutionMode::Clean(path), Some(Command::Fmt { paths, check }) => ExecutionMode::Fmt { paths, check }, Some(Command::Update { check, version }) => ExecutionMode::Update { @@ -158,6 +238,7 @@ fn main() -> ExitCode { ExecutionMode::Run { file, args: args.args, + offline: false, } } else { ExecutionMode::Repl @@ -171,17 +252,27 @@ fn main() -> ExitCode { output, targets, windows_gui, + offline, } => { let subsystem = match windows_gui { true => Subsystem::Gui, false => Subsystem::Console, }; - build(&files, output.as_deref(), &targets, subsystem) + build(&files, output.as_deref(), &targets, subsystem, offline) } - ExecutionMode::Check(file) => { + ExecutionMode::Add { + package, + name, + host, + } => deps::add(&package, name.as_deref(), host.as_deref()), + ExecutionMode::Remove(name) => deps::remove(&name), + ExecutionMode::Fetch { path, locked } => deps::fetch(path.as_deref(), locked), + ExecutionMode::Tree(path) => deps::tree(path.as_deref()), + ExecutionMode::Check { file, offline } => { let Some(source) = read(&file) else { return ExitCode::FAILURE; }; + deps::prepare(Path::new(&file), offline); let (sources, diagnostics) = driver::check_program(Path::new(&file), &source, &FsSources); // warnings alone still pass @@ -222,7 +313,11 @@ fn main() -> ExitCode { }, } } - ExecutionMode::Run { file, args } => with_program(&file, |checked, sources| { + ExecutionMode::Run { + file, + args, + offline, + } => with_program(&file, offline, |checked, sources| { // the interpreter runs only what the machine it runs on can if let Ok(host) = Target::host() { let unsupported = driver::unsupported(&checked.module, sources, host.os); @@ -262,6 +357,7 @@ fn build( output: Option<&Path>, targets: &[Target], subsystem: Subsystem, + offline: bool, ) -> ExitCode { let mut unique: Vec = Vec::new(); for &target in targets { @@ -283,7 +379,7 @@ fn build( } let mut status = ExitCode::SUCCESS; for file in files { - let code = with_program(file, |checked, sources| { + let code = with_program(file, offline, |checked, sources| { let targets = if unique.is_empty() { vec![Target::host()?] } else { @@ -467,14 +563,17 @@ fn read(file: &str) -> Option { /// /// Diagnostics are rendered with the path of the file they are in, the line, and a caret under /// the problem, while other errors, such as runtime errors, are printed as `file: error: message`. -/// A program that ended with `os.exit` prints nothing more and exits with its status. +/// A program that ended with `os.exit` prints nothing more and exits with its status. The +/// packages it uses are downloaded first, unless `offline`. fn with_program( file: &str, + offline: bool, action: impl FnOnce(&Checked, &SourceMap) -> Result<(), Box>, ) -> ExitCode { let Some(source) = read(file) else { return ExitCode::FAILURE; }; + deps::prepare(Path::new(file), offline); let (sources, result) = driver::load(Path::new(file), &source, &FsSources); let err = match result { Ok(checked) => { diff --git a/src/package.rs b/src/package.rs new file mode 100644 index 0000000..b2df8a5 --- /dev/null +++ b/src/package.rs @@ -0,0 +1,601 @@ +//! Packages: programs and libraries that depend on others, named in a `stone.toml` manifest. +//! +//! A package is a directory with a `stone.toml`. A program's manifest sits next to its entry +//! file, and each of its `[dependencies]` names a package by the name its files use it under: +//! +//! ```toml +//! [package] +//! name = "app" +//! version = "0.1.0" +//! +//! [dependencies] +//! geo = { path = "../geo" } +//! json = { github = "alice/stone-json", version = "1.2.0" } +//! ``` +//! +//! With that, `use geo.shapes` loads `shapes.stn` from the `geo` package's directory, and +//! `use geo` loads its `lib.stn`. [`crate::project`] links a package's files like any other +//! modules, under names that start with the package's own name and its versions' compatibility +//! class (see [`Version::class`]), so two packages never collide. + +pub mod edit; +#[cfg(feature = "fetch")] +pub mod fetch; +pub mod json; +pub mod lock; +pub mod resolve; +pub mod store; +pub mod toml; +pub mod version; + +use std::fmt; +use std::path::{Path, PathBuf}; + +use crate::diagnostic::Diagnostic; +use crate::span::{FileId, Span}; +use crate::token::RESERVED_KEYWORDS; +use toml::{Entry, Table, Value}; +pub use version::Version; + +/// The manifest's file name, in a package's directory. +pub const MANIFEST: &str = "stone.toml"; + +/// The lockfile's file name, next to the manifest. +pub const LOCK: &str = "stone.lock"; + +/// The file a package's `use` of the package's own name loads, as `use geo` loads `geo`'s. +pub const LIB: &str = "lib"; + +/// What a `stone.toml` says. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Manifest { + /// The package's name, which a library must have. + pub name: Option<(String, Span)>, + pub version: Option<(Version, Span)>, + /// The oldest stone the package works with. + pub stone: Option<(Version, Span)>, + pub dependencies: Vec, +} + +/// One entry of `[dependencies]`. +#[derive(Debug, Clone, PartialEq)] +pub struct Dependency { + /// The name the package's files use it under, which is the entry's key. + pub name: String, + /// The span of the key. + pub span: Span, + pub source: Source, + /// The oldest version to use, for a package from a host. + pub version: Option<(Version, Span)>, +} + +/// Where a dependency comes from. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Source { + /// A repository on GitHub, as `owner/repo`, whose releases are its versions. + GitHub { host: Option, repo: String }, + /// A project on GitLab, as `group/project` or `group/subgroup/project`. + GitLab { host: Option, path: String }, + /// A directory, relative to the manifest's. + Path(PathBuf), +} + +impl fmt::Display for Source { + /// Writes the source as `stone add` reads it, such as `github:alice/json`. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Source::GitHub { host: None, repo } => write!(f, "github:{repo}"), + Source::GitHub { + host: Some(host), + repo, + } => write!(f, "github:{host}/{repo}"), + Source::GitLab { host: None, path } => write!(f, "gitlab:{path}"), + Source::GitLab { + host: Some(host), + path, + } => write!(f, "gitlab:{host}/{path}"), + Source::Path(path) => write!(f, "path:{}", path.display()), + } + } +} + +impl Manifest { + /// Reads the manifest `source` in `file`, returning what it could read and every error. + /// + /// For example, a manifest holding `[dependencies]\ngeo = { path = "../geo" }\n` has one + /// dependency, `geo`, from the directory `../geo`. + pub fn parse(source: &str, file: FileId) -> (Manifest, Vec) { + let mut manifest = Manifest::default(); + let mut errors = vec![]; + let document = match toml::parse(source, file) { + Ok(document) => document, + Err(error) => return (manifest, vec![error]), + }; + for entry in &document.entries { + match (entry.key.as_str(), &entry.item.value) { + ("package", Value::Table(table)) => manifest.package(table, &mut errors), + ("dependencies", Value::Table(table)) => { + for entry in &table.entries { + if let Some(dependency) = dependency(entry, &mut errors) { + manifest.dependencies.push(dependency); + } + } + } + ("package" | "dependencies", _) => errors.push(Diagnostic::error( + entry.key_span, + format!("'{}' is a table, written as '[{}]'", entry.key, entry.key), + )), + (key, _) => errors.push(Diagnostic::error( + entry.key_span, + format!("unknown key '{key}'; a manifest holds [package] and [dependencies]"), + )), + } + } + (manifest, errors) + } + + fn package(&mut self, table: &Table, errors: &mut Vec) { + for entry in &table.entries { + let span = entry.item.span; + match entry.key.as_str() { + "name" => { + if let Some(name) = string(entry, errors) { + match identifier_error(&name, "a package's name") { + Some(message) => errors.push(Diagnostic::error(span, message)), + None => self.name = Some((name, span)), + } + } + } + "version" => self.version = version(entry, errors), + "stone" => { + self.stone = version(entry, errors); + if let Some((needed, span)) = &self.stone + && let Ok(running) = Version::parse(env!("CARGO_PKG_VERSION")) + && *needed > running + { + errors.push(Diagnostic::error( + *span, + format!("this package needs stone {needed} or newer, and this is stone {running}"), + )); + } + } + "description" | "license" => { + string(entry, errors); + } + key => errors.push(Diagnostic::error( + entry.key_span, + format!( + "unknown key '{key}' in [package], which holds name, version, stone, \ + description, and license" + ), + )), + } + } + } +} + +/// Reads one entry of `[dependencies]`. +fn dependency(entry: &Entry, errors: &mut Vec) -> Option { + let Value::Table(table) = &entry.item.value else { + errors.push(Diagnostic::error( + entry.item.span, + format!( + "write the dependency as {{ github = \"owner/repo\", version = \"1.0.0\" }}, \ + {{ gitlab = \"group/project\", version = \"1.0.0\" }}, or {{ path = \"../{}\" }}", + entry.key + ), + )); + return None; + }; + if let Some(message) = dependency_name_error(&entry.key) { + errors.push(Diagnostic::error(entry.key_span, message)); + return None; + } + let mut sources = vec![]; + let mut host = None; + let mut version_found = None; + let before = errors.len(); + for field in &table.entries { + match field.key.as_str() { + "github" | "gitlab" | "path" => { + if let Some(text) = string(field, errors) { + sources.push((field.key.clone(), text, field.item.span)); + } + } + "host" => { + if let Some(text) = string(field, errors) { + let valid = !text.is_empty() + && text + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-.:".contains(&b)); + match valid { + true => host = Some((text, field.key_span)), + false => errors.push(Diagnostic::error( + field.item.span, + "a host is a name such as 'gitlab.example.com', without 'https://'", + )), + } + } + } + "version" => version_found = Some((version(field, errors), field.key_span)), + key => errors.push(Diagnostic::error( + field.key_span, + format!( + "unknown key '{key}' in a dependency, which holds github, gitlab, or path, \ + and version and host" + ), + )), + } + } + if errors.len() > before { + return None; + } + let (kind, text, span) = match sources.as_slice() { + [one] => one.clone(), + [] => { + errors.push(Diagnostic::error( + entry.item.span, + "a dependency names where it comes from with github, gitlab, or path", + )); + return None; + } + [_, (_, _, span), ..] => { + errors.push(Diagnostic::error( + *span, + "a dependency comes from one place, so give one of github, gitlab, and path", + )); + return None; + } + }; + let source = match kind.as_str() { + "github" => { + let segments: Vec<&str> = text.split('/').collect(); + if segments.len() != 2 || !segments.iter().all(|s| is_repository_name(s)) { + errors.push(Diagnostic::error( + span, + "a GitHub repository is written as 'owner/repo'", + )); + return None; + } + Source::GitHub { + host: host.as_ref().map(|(h, _)| h.clone()), + repo: text, + } + } + "gitlab" => { + let segments: Vec<&str> = text.split('/').collect(); + if segments.len() < 2 || !segments.iter().all(|s| is_repository_name(s)) { + errors.push(Diagnostic::error( + span, + "a GitLab project is written as 'group/project' or 'group/subgroup/project'", + )); + return None; + } + Source::GitLab { + host: host.as_ref().map(|(h, _)| h.clone()), + path: text, + } + } + _ => { + if let Some((_, span)) = host { + errors.push(Diagnostic::error(span, "a path dependency has no host")); + return None; + } + if let Some((_, span)) = version_found { + errors.push(Diagnostic::error( + span, + "a path dependency uses the directory as it is, so it has no version", + )); + return None; + } + if text.is_empty() { + errors.push(Diagnostic::error(span, "a path names a directory")); + return None; + } + Source::Path(PathBuf::from(text)) + } + }; + let version = match version_found { + Some((version, _)) => version, + None if !matches!(source, Source::Path(_)) => { + errors.push(Diagnostic::error( + entry.item.span, + "a dependency from a host gives the oldest version it works with, as \ + version = \"1.0.0\"", + )); + return None; + } + None => None, + }; + Some(Dependency { + name: entry.key.clone(), + span: entry.key_span, + source, + version, + }) +} + +/// Reads an entry's string, reporting any other value. +fn string(entry: &Entry, errors: &mut Vec) -> Option { + match &entry.item.value { + Value::Str(text) => Some(text.clone()), + other => { + errors.push(Diagnostic::error( + entry.item.span, + format!("'{}' is a string, not {}", entry.key, other.kind()), + )); + None + } + } +} + +/// Reads an entry's version, reporting anything else. +fn version(entry: &Entry, errors: &mut Vec) -> Option<(Version, Span)> { + let text = string(entry, errors)?; + match Version::parse(&text) { + Ok(version) => Some((version, entry.item.span)), + Err(message) => { + errors.push(Diagnostic::error(entry.item.span, message)); + None + } + } +} + +/// Returns why `name` cannot be a package's name, called `what` in the message, if it cannot: +/// it must be a stone name, since modules are named after it. +/// +/// For example, `identifier_error("my-geo", "a package's name")` says to use `_`. +pub fn identifier_error(name: &str, what: &str) -> Option { + let mut chars = name.chars(); + let valid = chars + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_'); + if !valid { + let hint = match name.contains('-') { + true => format!(", so write '{}'", name.replace('-', "_")), + false => String::new(), + }; + return Some(format!( + "'{name}' cannot be {what}, which is a name of letters, digits, and '_'{hint}" + )); + } + if RESERVED_KEYWORDS.contains(&name) { + return Some(format!("'{name}' is a keyword, so it cannot be {what}")); + } + None +} + +/// Returns why a dependency cannot be named `name`, if it cannot: it must be a stone name that no +/// builtin module, the prelude, or a builtin function takes. +/// +/// For example, `dependency_name_error("os")` says `os` is a builtin module. +pub fn dependency_name_error(name: &str) -> Option { + if let Some(message) = identifier_error(name, "a dependency's name") { + return Some(message); + } + let builtin_module = crate::stdlib::module(name).is_some() + || crate::stdlib::source_module(name).is_some() + || crate::stdlib::is_internal_module(name); + if builtin_module { + return Some(format!( + "'{name}' is a builtin module, so give the dependency another name" + )); + } + if name == crate::project::PRELUDE_MODULE { + return Some(format!( + "'{name}' is the name of stone's prelude, so give the dependency another name" + )); + } + if crate::stdlib::is_reserved(name) { + return Some(format!( + "'{name}' is a builtin, so give the dependency another name" + )); + } + None +} + +/// Returns whether `name` may be a segment of a repository's path on a host. +fn is_repository_name(name: &str) -> bool { + !name.is_empty() + && name != "." + && name != ".." + && name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b)) +} + +/// Joins `path` onto `dir` and removes `.` and `..` components without reading the filesystem, +/// so the same directory reached two ways is one package, in memory as on disk. +/// +/// For example, `join_normal("app", "../geo")` is `geo`, and `join_normal("", "../geo")` is +/// `../geo`. +pub fn join_normal(dir: &Path, path: &Path) -> PathBuf { + use std::path::Component; + let joined = dir.join(path); + let mut parts: Vec = vec![]; + for component in joined.components() { + match component { + Component::CurDir => {} + Component::ParentDir => match parts.last() { + Some(Component::Normal(_)) => { + parts.pop(); + } + Some(Component::RootDir | Component::Prefix(_)) => {} + _ => parts.push(component), + }, + other => parts.push(other), + } + } + parts.iter().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(source: &str) -> Manifest { + let (manifest, errors) = Manifest::parse(source, FileId(1)); + assert_eq!(errors, [], "should have no errors"); + manifest + } + + fn errors(source: &str) -> Vec<(String, usize, usize)> { + Manifest::parse(source, FileId(1)) + .1 + .into_iter() + .map(|e| (e.message, e.span.start.line, e.span.start.col)) + .collect() + } + + fn message(source: &str) -> String { + let errors = errors(source); + assert_eq!(errors.len(), 1, "one error: {errors:?}"); + errors[0].0.clone() + } + + #[test] + fn a_manifest_names_its_package_and_dependencies() { + let manifest = read( + "[package]\nname = \"app\"\nversion = \"0.1\"\n\n[dependencies]\n\ + geo = { path = \"../geo\" }\n\ + json = { github = \"alice/stone-json\", version = \"1.2.0\" }\n\ + http = { gitlab = \"group/sub/http\", version = \"0.4\", host = \"git.corp.com\" }\n", + ); + assert_eq!(manifest.name.unwrap().0, "app"); + assert_eq!(manifest.version.unwrap().0, Version::new(0, 1, 0)); + let deps: Vec<(String, String, Option)> = manifest + .dependencies + .iter() + .map(|d| { + ( + d.name.clone(), + d.source.to_string(), + d.version.as_ref().map(|(v, _)| v.to_string()), + ) + }) + .collect(); + assert_eq!( + deps, + [ + ("geo".into(), "path:../geo".into(), None), + ( + "json".into(), + "github:alice/stone-json".into(), + Some("1.2.0".into()) + ), + ( + "http".into(), + "gitlab:git.corp.com/group/sub/http".into(), + Some("0.4.0".into()) + ), + ] + ); + } + + #[test] + fn an_empty_manifest_is_valid() { + assert_eq!(read(""), Manifest::default()); + } + + #[test] + fn manifest_errors_point_at_their_keys_and_values() { + assert_eq!( + errors("[package]\nname = \"my-geo\"\n"), + [( + "'my-geo' cannot be a package's name, which is a name of letters, digits, and \ + '_', so write 'my_geo'" + .to_string(), + 2, + 8 + )] + ); + assert_eq!( + errors("[dependecies]\n"), + [( + "unknown key 'dependecies'; a manifest holds [package] and [dependencies]" + .to_string(), + 1, + 2 + )] + ); + assert_eq!( + message("[package]\nversion = \"v1.0\"\n"), + "'v1.0' is not a version; write it without the 'v', as '1.0'" + ); + assert_eq!( + message("[package]\nname = \"fn\"\n"), + "'fn' is a keyword, so it cannot be a package's name" + ); + assert_eq!( + message("[package]\nauthor = \"me\"\n"), + "unknown key 'author' in [package], which holds name, version, stone, description, \ + and license" + ); + } + + #[test] + fn dependency_errors_explain_the_forms() { + assert_eq!( + message("[dependencies]\njson = \"1.0\"\n"), + "write the dependency as { github = \"owner/repo\", version = \"1.0.0\" }, \ + { gitlab = \"group/project\", version = \"1.0.0\" }, or { path = \"../json\" }" + ); + assert_eq!( + message("[dependencies]\njson = { github = \"alice/json\" }\n"), + "a dependency from a host gives the oldest version it works with, as \ + version = \"1.0.0\"" + ); + assert_eq!( + message("[dependencies]\njson = { github = \"json\", version = \"1\" }\n"), + "a GitHub repository is written as 'owner/repo'" + ); + assert_eq!( + message("[dependencies]\njson = { path = \"../json\", version = \"1\" }\n"), + "a path dependency uses the directory as it is, so it has no version" + ); + assert_eq!( + message( + "[dependencies]\njson = { path = \"../json\", github = \"a/b\", version = \"1\" }\n" + ), + "a dependency comes from one place, so give one of github, gitlab, and path" + ); + assert_eq!( + message("[dependencies]\njson = { version = \"1\" }\n"), + "a dependency names where it comes from with github, gitlab, or path" + ); + assert_eq!( + message("[dependencies]\nos = { path = \"../os\" }\n"), + "'os' is a builtin module, so give the dependency another name" + ); + assert_eq!( + message("[dependencies]\nprint = { path = \"../p\" }\n"), + "'print' is a builtin, so give the dependency another name" + ); + assert_eq!( + message("[dependencies]\nprelude = { path = \"../p\" }\n"), + "'prelude' is the name of stone's prelude, so give the dependency another name" + ); + } + + #[test] + fn a_package_may_need_a_newer_stone() { + assert_eq!( + message("[package]\nstone = \"999.0\"\n"), + format!( + "this package needs stone 999.0.0 or newer, and this is stone {}", + env!("CARGO_PKG_VERSION") + ) + ); + read("[package]\nstone = \"0.1\"\n"); + } + + #[test] + fn paths_join_without_dot_components() { + let join = |dir: &str, path: &str| join_normal(Path::new(dir), Path::new(path)); + assert_eq!(join("app", "../geo"), PathBuf::from("geo")); + assert_eq!(join("", "../geo"), PathBuf::from("../geo")); + assert_eq!(join("a/b", "./../../c/./d"), PathBuf::from("c/d")); + assert_eq!(join("/x", "../../y"), PathBuf::from("/y")); + } +} diff --git a/src/package/edit.rs b/src/package/edit.rs new file mode 100644 index 0000000..7227d9f --- /dev/null +++ b/src/package/edit.rs @@ -0,0 +1,345 @@ +//! What `stone add` and `stone remove` do to a manifest: reading the package a command names, and +//! changing `[dependencies]` in place, so the rest of the file, comments included, stays as +//! written. +//! +//! For example, `stone add github:alice/json@1.2` names `alice/json` on GitHub and the newest +//! release 1.2.x, and adds `json = { github = "alice/json", version = "1.2.3" }` to the manifest. + +use super::toml::{self, Value, quote}; +use super::{Source, Version}; +use crate::span::{FileId, Pos}; + +/// Which release a command asks for. +#[derive(Debug, Clone, PartialEq)] +pub enum Wanted { + /// The newest release that is not a pre-release. + Latest, + /// The newest release whose version starts with these numbers, as `1.2` for the newest + /// 1.2.x, or exactly this version, when it has three numbers. + Matching { numbers: Vec, exact: Version }, +} + +impl Wanted { + /// Reads what follows `@`: `latest` or a version, possibly short. + /// + /// For example, `Wanted::parse("1.2")` matches 1.2.0 and 1.2.7 but not 1.3.0. + pub fn parse(text: &str) -> Result { + if text == "latest" { + return Ok(Wanted::Latest); + } + let exact = Version::parse(text)?; + let written = text.split(['-', '+']).next().unwrap_or(text); + let numbers = written.split('.').filter_map(|n| n.parse().ok()).collect(); + Ok(Wanted::Matching { numbers, exact }) + } + + /// Returns whether a release of `version` is what was asked for. + pub fn matches(&self, version: &Version) -> bool { + match self { + Wanted::Latest => !version.is_pre(), + Wanted::Matching { numbers, exact } if numbers.len() == 3 => version == exact, + Wanted::Matching { numbers, .. } => { + let parts = [version.major, version.minor, version.patch]; + !version.is_pre() && numbers.iter().zip(parts).all(|(n, part)| *n == part) + } + } + } +} + +/// Reads the package a command names: `github:owner/repo`, `github:host/owner/repo` for +/// GitHub Enterprise, `gitlab:group/project` (with `host` for another GitLab), or a link to one on +/// `github.com` or `gitlab.com`, each optionally followed by `@` and what [`Wanted::parse`] reads. +/// +/// For example, `parse_spec("https://github.com/alice/json.git@latest", None)` is +/// `github:alice/json` at its latest release. +pub fn parse_spec(spec: &str, host: Option<&str>) -> Result<(Source, Wanted), String> { + let (place, wanted) = match spec.rsplit_once('@') { + Some((place, wanted)) => (place, Wanted::parse(wanted)?), + None => (spec, Wanted::Latest), + }; + let link = place + .strip_prefix("https://") + .or_else(|| place.strip_prefix("http://")); + let (kind, path) = match link { + Some(link) => { + let link = link.trim_end_matches('/'); + let link = link.strip_suffix(".git").unwrap_or(link); + match link.split_once('/') { + Some(("github.com", path)) => ("github", path.to_string()), + Some(("gitlab.com", path)) => ("gitlab", path.to_string()), + Some((other, path)) => { + return Err(format!( + "say which host {other} is: github:{other}/{path} for GitHub Enterprise, \ + or gitlab:{path} --host {other} for GitLab" + )); + } + None => return Err(format!("'{place}' does not name a repository")), + } + } + None => match place.split_once(':') { + Some((kind @ ("github" | "gitlab"), path)) => (kind, path.to_string()), + _ => { + return Err(format!( + "'{place}' does not name a package; write github:owner/repo or \ + gitlab:group/project" + )); + } + }, + }; + let segments: Vec<&str> = path.split('/').collect(); + let valid = segments.iter().all(|s| { + !s.is_empty() + && *s != "." + && *s != ".." + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b)) + }); + let source = match (kind, segments.as_slice()) { + _ if !valid => return Err(format!("'{path}' is not a repository's path")), + ("github", [_, _]) => Source::GitHub { + host: host.map(str::to_string), + repo: path, + }, + ("github", [enterprise, owner, repo]) if host.is_none() => Source::GitHub { + host: Some(enterprise.to_string()), + repo: format!("{owner}/{repo}"), + }, + ("github", _) => return Err("a GitHub repository is written as owner/repo".to_string()), + (_, [_, _, ..]) => Source::GitLab { + host: host.map(str::to_string), + path, + }, + _ => { + return Err( + "a GitLab project is written as group/project or group/subgroup/project" + .to_string(), + ); + } + }; + Ok((source, wanted)) +} + +/// Writes the inline table a manifest names a dependency with. +/// +/// For example, `json` 1.2.0 from `github:alice/json` is +/// `{ github = "alice/json", version = "1.2.0" }`. +pub fn dependency_value(source: &Source, version: &Version) -> String { + let (kind, host, path) = match source { + Source::GitHub { host, repo } => ("github", host, repo.clone()), + Source::GitLab { host, path } => ("gitlab", host, path.clone()), + Source::Path(dir) => return format!("{{ path = {} }}", quote(&dir.display().to_string())), + }; + let host = match host { + Some(host) => format!(", host = {}", quote(host)), + None => String::new(), + }; + format!( + "{{ {kind} = {}, version = {}{host} }}", + quote(&path), + quote(&version.to_string()) + ) +} + +/// Returns the manifest `text` with the dependency `name` set to `value`, replacing an entry of +/// that name where it is, or else adding one after the last of `[dependencies]`, which is added if +/// the manifest has none. +pub fn set_dependency(text: &str, name: &str, value: &str) -> Result { + let document = toml::parse(text, FileId(0)).map_err(|e| format!("stone.toml:{e}"))?; + let line = format!("{} = {value}", bare_or_quoted(name)); + let Some(entry) = document.get("dependencies") else { + let mut text = text.to_string(); + if !text.is_empty() && !text.ends_with('\n') { + text.push('\n'); + } + if !text.is_empty() { + text.push('\n'); + } + text.push_str(&format!("[dependencies]\n{line}\n")); + return Ok(text); + }; + let Value::Table(table) = &entry.item.value else { + return Err("'dependencies' is not a table".to_string()); + }; + if let Some(existing) = table.get(name) { + let start = offset(text, existing.key_span.start); + let end = offset(text, existing.item.span.end); + return Ok(format!("{}{line}{}", &text[..start], &text[end..])); + } + // after the last entry, or else the header + let after = table + .entries + .last() + .map_or(entry.item.span.end, |last| last.item.span.end); + let at = match text[offset(text, after)..].find('\n') { + Some(newline) => offset(text, after) + newline + 1, + None => text.len(), + }; + let mut changed = text[..at].to_string(); + if !changed.ends_with('\n') { + changed.push('\n'); + } + changed.push_str(&line); + changed.push('\n'); + changed.push_str(&text[at..]); + Ok(changed) +} + +/// Returns the manifest `text` without the dependency `name`, whose lines are removed. +pub fn remove_dependency(text: &str, name: &str) -> Result { + let document = toml::parse(text, FileId(0)).map_err(|e| format!("stone.toml:{e}"))?; + let entry = document + .get("dependencies") + .and_then(|deps| match &deps.item.value { + Value::Table(table) => table.get(name), + _ => None, + }) + .ok_or_else(|| format!("there is no dependency named '{name}'"))?; + let start = offset(text, Pos::new(entry.key_span.start.line, 1)); + let end = offset(text, entry.item.span.end); + let end = match text[end..].find('\n') { + Some(newline) => end + newline + 1, + None => text.len(), + }; + Ok(format!("{}{}", &text[..start], &text[end..])) +} + +/// Writes a key bare if it can be, and quoted otherwise. +fn bare_or_quoted(key: &str) -> String { + match key + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') + && !key.is_empty() + { + true => key.to_string(), + false => quote(key), + } +} + +/// Returns the byte offset of `pos`, a line and a column of characters, in `text`. +fn offset(text: &str, pos: Pos) -> usize { + let mut line = 1; + let mut col = 1; + for (i, c) in text.char_indices() { + if line == pos.line && col == pos.col { + return i; + } + if c == '\n' { + if line == pos.line { + return i; + } + line += 1; + col = 1; + } else { + col += 1; + } + } + text.len() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn version(text: &str) -> Version { + Version::parse(text).unwrap() + } + + #[test] + fn specs_name_a_host_a_repository_and_a_release() { + let github = |repo: &str| Source::GitHub { + host: None, + repo: repo.to_string(), + }; + assert_eq!( + parse_spec("github:alice/json", None), + Ok((github("alice/json"), Wanted::Latest)) + ); + assert_eq!( + parse_spec("https://github.com/alice/json.git", None), + Ok((github("alice/json"), Wanted::Latest)) + ); + let (source, wanted) = + parse_spec("gitlab:group/sub/http@1.2", Some("git.corp.com")).unwrap(); + assert_eq!(source.to_string(), "gitlab:git.corp.com/group/sub/http"); + assert!(wanted.matches(&version("1.2.7"))); + assert!(!wanted.matches(&version("1.3.0"))); + assert!(!wanted.matches(&version("1.2.8-rc.1"))); + let (source, _) = parse_spec("github:ghe.corp.com/team/lib", None).unwrap(); + assert_eq!(source.to_string(), "github:ghe.corp.com/team/lib"); + let (_, wanted) = parse_spec("github:a/b@1.0.0-rc.1", None).unwrap(); + assert!(wanted.matches(&version("1.0.0-rc.1"))); + assert!(!Wanted::Latest.matches(&version("1.0.0-rc.1"))); + } + + #[test] + fn bad_specs_say_how_to_write_one() { + assert_eq!( + parse_spec("alice/json", None).unwrap_err(), + "'alice/json' does not name a package; write github:owner/repo or gitlab:group/project" + ); + assert_eq!( + parse_spec("https://git.corp.com/g/p", None).unwrap_err(), + "say which host git.corp.com is: github:git.corp.com/g/p for GitHub Enterprise, or \ + gitlab:g/p --host git.corp.com for GitLab" + ); + assert!(parse_spec("github:alice", None).is_err()); + assert!(parse_spec("github:a/../b", None).is_err()); + assert!(parse_spec("github:a/b@v1", None).is_err()); + } + + #[test] + fn dependencies_are_written_as_inline_tables() { + let source = Source::GitLab { + host: Some("git.corp.com".into()), + path: "g/p".into(), + }; + assert_eq!( + dependency_value(&source, &version("1.2")), + "{ gitlab = \"g/p\", version = \"1.2.0\", host = \"git.corp.com\" }" + ); + } + + #[test] + fn a_dependency_is_added_after_the_others() { + let text = "[package]\nname = \"app\"\n\n[dependencies]\n# geometry\ngeo = { path = \"../geo\" }\n\n[other]\n"; + assert_eq!( + set_dependency(text, "json", "{ x = 1 }").unwrap(), + "[package]\nname = \"app\"\n\n[dependencies]\n# geometry\ngeo = { path = \"../geo\" }\n\ + json = { x = 1 }\n\n[other]\n" + ); + assert_eq!( + set_dependency("[dependencies]\n", "json", "{ x = 1 }").unwrap(), + "[dependencies]\njson = { x = 1 }\n" + ); + assert_eq!( + set_dependency("[package]\nname = \"app\"", "json", "{ x = 1 }").unwrap(), + "[package]\nname = \"app\"\n\n[dependencies]\njson = { x = 1 }\n" + ); + assert_eq!( + set_dependency("", "json", "{ x = 1 }").unwrap(), + "[dependencies]\njson = { x = 1 }\n" + ); + } + + #[test] + fn a_dependency_is_replaced_where_it_is() { + let text = "[dependencies]\njson = { x = 1 } # keep me\ngeo = { y = 2 }\n"; + assert_eq!( + set_dependency(text, "json", "{ x = 2 }").unwrap(), + "[dependencies]\njson = { x = 2 } # keep me\ngeo = { y = 2 }\n" + ); + } + + #[test] + fn a_dependency_is_removed_with_its_line() { + let text = "[dependencies]\njson = { x = 1 }\ngeo = { y = 2 }\n"; + assert_eq!( + remove_dependency(text, "json").unwrap(), + "[dependencies]\ngeo = { y = 2 }\n" + ); + assert_eq!( + remove_dependency(text, "http").unwrap_err(), + "there is no dependency named 'http'" + ); + } +} diff --git a/src/package/fetch.rs b/src/package/fetch.rs new file mode 100644 index 0000000..816a4d9 --- /dev/null +++ b/src/package/fetch.rs @@ -0,0 +1,521 @@ +//! Downloading packages from GitHub and GitLab releases into the store (see [`super::store`]), +//! behind the `fetch` cargo feature, which brings in an HTTPS client. +//! +//! A version of a package is a release, found by its tag (`v1.2.0` or `1.2.0`), and its files +//! come from the source archive the host makes of the tag. Only the archive's `stone.toml` and +//! `.stn` files are kept. Selecting versions never lists releases (see [`super::resolve`]), so +//! only `stone add` asks for the newest, and downloading asks once per package for the tag of a +//! version. +//! +//! `GITHUB_TOKEN` and `GITLAB_TOKEN` are sent if set, for private repositories and higher rate +//! limits, and `STONE_GITHUB_API` and `STONE_GITLAB_API` replace `https://api.github.com` and +//! `https://gitlab.com/api/v4`, for tests. + +use std::collections::{HashMap, HashSet}; +use std::io::Read; +use std::path::{Component, Path, PathBuf}; +use std::time::Duration; + +use super::store::{is_kept, package_dir}; +use super::{MANIFEST, Source, Version}; +use crate::project::{self, FsSources, Linked, Sources}; + +/// The most an archive may hold, unpacked or not. +const MAX_ARCHIVE: u64 = 64 << 20; + +/// The most pages of releases read, 100 to a page. +const MAX_PAGES: usize = 10; + +/// A release of a package. +#[derive(Debug, Clone, PartialEq)] +pub struct Release { + pub tag: String, + pub version: Version, +} + +/// Talks to the hosts packages come from. +pub struct Client { + agent: ureq::Agent, + /// Each package's releases, once read. + releases: HashMap>, +} + +impl Default for Client { + fn default() -> Self { + Client::new() + } +} + +impl Client { + pub fn new() -> Client { + let config = ureq::Agent::config_builder() + .timeout_global(Some(Duration::from_secs(120))) + .user_agent(format!("stone/{}", env!("CARGO_PKG_VERSION"))) + .build(); + Client { + agent: config.into(), + releases: HashMap::new(), + } + } + + /// Returns every release of the package from `source` whose tag is a version, drafts and + /// GitLab's upcoming releases left out. + pub fn releases(&mut self, source: &Source) -> Result, String> { + if let Some(releases) = self.releases.get(source) { + return Ok(releases.clone()); + } + let mut releases = vec![]; + for page in 1..=MAX_PAGES { + let url = match source { + Source::GitHub { repo, .. } => { + format!( + "{}/repos/{repo}/releases?per_page=100&page={page}", + api(source) + ) + } + Source::GitLab { path, .. } => format!( + "{}/projects/{}/releases?per_page=100&page={page}", + api(source), + encode(path) + ), + Source::Path(_) => return Ok(vec![]), + }; + let body = self.get(source, &url)?; + let text = String::from_utf8(body).map_err(|_| "the host's answer is not text")?; + let (found, count) = parse_releases(&text)?; + releases.extend(found); + if count < 100 { + break; + } + } + self.releases.insert(source.clone(), releases.clone()); + Ok(releases) + } + + /// Returns the release of `version`, or an error listing the versions there are. + pub fn release(&mut self, source: &Source, version: &Version) -> Result { + let releases = self.releases(source)?; + if let Some(release) = releases.iter().find(|r| r.version == *version) { + return Ok(release.clone()); + } + Err(format!( + "{source} has no release {version}{}", + listing(&releases) + )) + } + + /// Returns the newest release that `wanted` matches. + pub fn newest( + &mut self, + source: &Source, + wanted: &super::edit::Wanted, + ) -> Result { + let releases = self.releases(source)?; + releases + .iter() + .filter(|r| wanted.matches(&r.version)) + .max_by(|a, b| a.version.cmp(&b.version)) + .cloned() + .ok_or_else(|| match wanted { + super::edit::Wanted::Latest => { + format!("{source} has no release yet{}", listing(&releases)) + } + super::edit::Wanted::Matching { exact, .. } => { + format!( + "{source} has no release matching {exact}{}", + listing(&releases) + ) + } + }) + } + + /// Downloads the release `version` of the package from `source` into the store `store`, + /// unless it is there already. + pub fn download( + &mut self, + store: &Path, + source: &Source, + version: &Version, + ) -> Result { + let dir = package_dir(store, source, version); + if dir.join(MANIFEST).is_file() { + return Ok(dir); + } + let release = self.release(source, version)?; + let tag = encode(&release.tag); + let url = match source { + Source::GitHub { repo, .. } => format!("{}/repos/{repo}/tarball/{tag}", api(source)), + Source::GitLab { path, .. } => format!( + "{}/projects/{}/repository/archive.tar.gz?sha={tag}", + api(source), + encode(path) + ), + Source::Path(dir) => return Ok(dir.clone()), + }; + let archive = self.get(source, &url)?; + unpack(&archive, &dir)?; + if !dir.join(MANIFEST).is_file() { + let _ = std::fs::remove_dir_all(&dir); + return Err(format!( + "its release {} has no {MANIFEST} at the top, so it is not a stone package", + release.tag + )); + } + Ok(dir) + } + + /// Reads `url` from the host of `source`, sending its token if one is set. + fn get(&self, source: &Source, url: &str) -> Result, String> { + let mut request = self.agent.get(url); + match source { + Source::GitHub { .. } => { + request = request.header("Accept", "application/vnd.github+json"); + if let Some(token) = token("GITHUB_TOKEN") { + request = request.header("Authorization", format!("Bearer {token}")); + } + } + Source::GitLab { .. } => { + if let Some(token) = token("GITLAB_TOKEN") { + request = request.header("PRIVATE-TOKEN", token); + } + } + Source::Path(_) => {} + } + let mut response = request.call().map_err(|e| match e { + ureq::Error::StatusCode(404) => { + let token = match source { + Source::GitLab { .. } => "GITLAB_TOKEN", + _ => "GITHUB_TOKEN", + }; + format!( + "{source} was not found (if it is private, set {token} to a token that can \ + read it)" + ) + } + ureq::Error::StatusCode(code @ (403 | 429)) => format!( + "the host refused with status {code}, which may be its rate limit; setting a \ + token raises it" + ), + ureq::Error::StatusCode(code) => format!("the host answered with status {code}"), + other => other.to_string(), + })?; + response + .body_mut() + .with_config() + .limit(MAX_ARCHIVE) + .read_to_vec() + .map_err(|e| e.to_string()) + } +} + +/// Links the program whose entry file is `entry` again and again, downloading each package it +/// uses that is not in the store, until every one is there, and returns the last link, whose +/// diagnostics say what else is wrong. `log` is told of each download. +/// +/// Only the manifests count, so the entry file is linked as if it were empty. +pub fn fetch_missing( + entry: &Path, + client: &mut Client, + log: &mut dyn FnMut(String), +) -> Result { + let store = FsSources + .store() + .ok_or("there is no directory to keep packages in; set STONE_HOME")?; + let mut tried = HashSet::new(); + loop { + let linked = project::link(entry, "", &FsSources); + let missing: Vec<(Source, Version)> = linked + .missing + .iter() + .filter(|m| !tried.contains(*m)) + .cloned() + .collect(); + if missing.is_empty() { + return Ok(linked); + } + for (source, version) in missing { + log(format!("downloading {source} {version}")); + client + .download(&store, &source, &version) + .map_err(|e| format!("cannot download {source} {version}: {e}"))?; + tried.insert((source, version)); + } + } +} + +/// Returns a source's API, as `https://api.github.com`. +fn api(source: &Source) -> String { + let from_env = |name: &str| std::env::var(name).ok().filter(|v| !v.is_empty()); + let url = match source { + Source::GitHub { host: None, .. } => { + from_env("STONE_GITHUB_API").unwrap_or("https://api.github.com".to_string()) + } + Source::GitHub { + host: Some(host), .. + } => format!("https://{host}/api/v3"), + Source::GitLab { host: None, .. } => { + from_env("STONE_GITLAB_API").unwrap_or("https://gitlab.com/api/v4".to_string()) + } + Source::GitLab { + host: Some(host), .. + } => format!("https://{host}/api/v4"), + Source::Path(_) => String::new(), + }; + url.trim_end_matches('/').to_string() +} + +/// Returns the environment variable `name` if it is set and not empty. +fn token(name: &str) -> Option { + std::env::var(name).ok().filter(|t| !t.is_empty()) +} + +/// Percent-encodes everything but letters, digits, and `-._~`, as a path segment or query value +/// needs, so `group/project` is `group%2Fproject`. +fn encode(text: &str) -> String { + text.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { + (b as char).to_string() + } + b => format!("%{b:02X}"), + }) + .collect() +} + +/// Writes the versions of `releases`, newest first, to end an error, or nothing if there are none. +fn listing(releases: &[Release]) -> String { + let mut versions: Vec<&Version> = releases.iter().map(|r| &r.version).collect(); + versions.sort_by(|a, b| b.cmp(a)); + match versions.as_slice() { + [] => " (it has no releases whose tags are versions, as v1.0.0)".to_string(), + versions => { + let shown: Vec = versions.iter().take(8).map(|v| v.to_string()).collect(); + let more = if versions.len() > 8 { ", ..." } else { "" }; + format!(" (its releases are {}{more})", shown.join(", ")) + } + } +} + +/// Reads a page of GitHub's or GitLab's releases, returning those whose tags are versions and +/// how many the page held. +fn parse_releases(text: &str) -> Result<(Vec, usize), String> { + let value = + super::json::parse(text).map_err(|e| format!("the host's answer is not JSON: {e}"))?; + let items = value + .as_array() + .ok_or("the host's answer is not a list of releases")?; + let releases = items + .iter() + .filter(|item| { + let flag = |key: &str| item.get(key).and_then(|v| v.as_bool()).unwrap_or(false); + !flag("draft") && !flag("upcoming_release") + }) + .filter_map(|item| { + let tag = item.get("tag_name")?.as_str()?; + Some(Release { + tag: tag.to_string(), + version: Version::from_tag(tag)?, + }) + }) + .collect(); + Ok((releases, items.len())) +} + +/// Unpacks the files a package keeps from the `.tar.gz` `archive`, whose names start with one +/// directory, as hosts make them, into `dir`, which must not exist yet. The files go into a +/// directory beside it first, which is renamed, so `dir` is never half written. +pub fn unpack(archive: &[u8], dir: &Path) -> Result<(), String> { + let parent = dir.parent().ok_or("the store has no directory")?; + std::fs::create_dir_all(parent) + .map_err(|e| format!("cannot make {}: {e}", parent.display()))?; + let name = dir.file_name().unwrap_or_default().to_string_lossy(); + let temporary = parent.join(format!(".{name}.{}.partial", std::process::id())); + let _ = std::fs::remove_dir_all(&temporary); + let result = unpack_into(archive, &temporary).and_then(|()| { + match std::fs::rename(&temporary, dir) { + Ok(()) => Ok(()), + // another stone got there first + Err(_) if dir.join(MANIFEST).is_file() => Ok(()), + Err(e) => Err(format!("cannot move it to {}: {e}", dir.display())), + } + }); + let _ = std::fs::remove_dir_all(&temporary); + result +} + +fn unpack_into(archive: &[u8], dir: &Path) -> Result<(), String> { + std::fs::create_dir_all(dir).map_err(|e| format!("cannot make {}: {e}", dir.display()))?; + let decoder = flate2::read::GzDecoder::new(archive).take(MAX_ARCHIVE); + let mut archive = tar::Archive::new(decoder); + let entries = archive.entries().map_err(|e| format!("bad archive: {e}"))?; + for entry in entries { + let mut entry = entry.map_err(|e| format!("bad archive: {e}"))?; + if !entry.header().entry_type().is_file() { + continue; + } + let path = entry + .path() + .map_err(|e| format!("bad archive: {e}"))? + .into_owned(); + let Some(relative) = inside_top(&path) else { + return Err(format!( + "the archive holds '{}', which is outside its directory", + path.display() + )); + }; + if relative.as_os_str().is_empty() || !is_kept(&relative) { + continue; + } + let mut text = String::new(); + entry + .read_to_string(&mut text) + .map_err(|_| format!("'{}' is not text", relative.display()))?; + let target = dir.join(&relative); + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("cannot make {}: {e}", parent.display()))?; + } + std::fs::write(&target, text) + .map_err(|e| format!("cannot write {}: {e}", target.display()))?; + } + Ok(()) +} + +/// Returns `path` without its first directory, or `None` if it leaves it, as `..` or an absolute +/// path would. +fn inside_top(path: &Path) -> Option { + let mut components = path.components(); + match components.next()? { + Component::Normal(_) => {} + _ => return None, + } + let mut relative = PathBuf::new(); + for component in components { + match component { + Component::Normal(name) => relative.push(name), + Component::CurDir => {} + _ => return None, + } + } + Some(relative) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn archive(files: &[(&str, &str)]) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + for (path, text) in files { + let mut header = tar::Header::new_gnu(); + header.set_size(text.len() as u64); + header.set_mode(0o644); + header.set_entry_type(tar::EntryType::Regular); + // set_path refuses '..', which an archive may still hold + header.as_gnu_mut().unwrap().name[..path.len()].copy_from_slice(path.as_bytes()); + header.set_cksum(); + builder.append(&header, text.as_bytes()).unwrap(); + } + let tar = builder.into_inner().unwrap(); + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + std::io::Write::write_all(&mut encoder, &tar).unwrap(); + encoder.finish().unwrap() + } + + fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("stone-fetch-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + dir + } + + #[test] + fn an_archive_unpacks_only_the_files_a_package_keeps() { + let dir = scratch("keeps"); + let data = archive(&[ + ( + "alice-json-abc123/stone.toml", + "[package]\nname = \"json\"\n", + ), + ( + "alice-json-abc123/lib.stn", + "pub fn one() -> int:\n ret 1\n", + ), + ( + "alice-json-abc123/sub/two.stn", + "pub fn two() -> int:\n ret 2\n", + ), + ("alice-json-abc123/README.md", "# json\n"), + ]); + let target = dir.join("1.0.0"); + unpack(&data, &target).unwrap(); + let mut found: Vec = vec![]; + for path in ["stone.toml", "lib.stn", "sub/two.stn", "README.md"] { + if target.join(path).is_file() { + found.push(path.to_string()); + } + } + assert_eq!(found, ["stone.toml", "lib.stn", "sub/two.stn"]); + // nothing is left beside it + let left: Vec<_> = std::fs::read_dir(&dir).unwrap().collect(); + assert_eq!(left.len(), 1); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn an_archive_cannot_write_outside_its_directory() { + let dir = scratch("outside"); + let data = archive(&[("top/../../evil.stn", "x = 1\n")]); + let error = unpack(&data, &dir.join("1.0.0")).unwrap_err(); + assert!(error.contains("outside its directory"), "{error}"); + assert!(!dir.join("1.0.0").exists()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn releases_are_read_from_either_hosts_answer() { + let (releases, count) = parse_releases( + r#"[{"tag_name": "v1.2.0", "draft": false, "prerelease": false}, + {"tag_name": "v1.3.0", "draft": true}, + {"tag_name": "nightly"}, + {"tag_name": "2.0.0-rc.1", "prerelease": true}, + {"tag_name": "v0.9.0", "upcoming_release": true}]"#, + ) + .unwrap(); + assert_eq!(count, 5); + let found: Vec<(String, String)> = releases + .iter() + .map(|r| (r.tag.clone(), r.version.to_string())) + .collect(); + assert_eq!( + found, + [ + ("v1.2.0".to_string(), "1.2.0".to_string()), + ("2.0.0-rc.1".to_string(), "2.0.0-rc.1".to_string()) + ] + ); + assert!(parse_releases("{\"message\": \"Not Found\"}").is_err()); + } + + #[test] + fn paths_and_tags_are_encoded() { + assert_eq!(encode("group/sub/proj"), "group%2Fsub%2Fproj"); + assert_eq!(encode("v1.2.0-rc.1"), "v1.2.0-rc.1"); + assert_eq!(encode("a b"), "a%20b"); + } + + #[test] + fn errors_list_the_releases_there_are() { + let release = |v: &str| Release { + tag: format!("v{v}"), + version: Version::parse(v).unwrap(), + }; + assert_eq!( + listing(&[release("1.0.0"), release("1.3.0")]), + " (its releases are 1.3.0, 1.0.0)" + ); + assert_eq!( + listing(&[]), + " (it has no releases whose tags are versions, as v1.0.0)" + ); + } +} diff --git a/src/package/json.rs b/src/package/json.rs new file mode 100644 index 0000000..ef112e0 --- /dev/null +++ b/src/package/json.rs @@ -0,0 +1,260 @@ +//! A small JSON reader, for the lists of releases GitHub and GitLab answer with. +//! +//! For example, `parse(r#"[{"tag_name": "v1.0.0", "draft": false}]"#)` gives an array of one +//! object, whose `tag_name` is the string `v1.0.0`. + +/// A JSON value. Numbers keep their text, since releases need none of them. +#[derive(Debug, Clone, PartialEq)] +pub enum Json { + Null, + Bool(bool), + Number(String), + Str(String), + Array(Vec), + Object(Vec<(String, Json)>), +} + +impl Json { + /// Returns the value of `key`, if this is an object that has it. + pub fn get(&self, key: &str) -> Option<&Json> { + match self { + Json::Object(entries) => entries.iter().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + + pub fn as_str(&self) -> Option<&str> { + match self { + Json::Str(text) => Some(text), + _ => None, + } + } + + pub fn as_bool(&self) -> Option { + match self { + Json::Bool(b) => Some(*b), + _ => None, + } + } + + pub fn as_array(&self) -> Option<&[Json]> { + match self { + Json::Array(items) => Some(items), + _ => None, + } + } +} + +/// The deepest arrays and objects may nest, so a hostile answer cannot overflow the stack. +const MAX_DEPTH: usize = 128; + +/// Reads a JSON document, or says where it is not JSON. +pub fn parse(text: &str) -> Result { + let mut reader = Reader { + bytes: text.as_bytes(), + at: 0, + }; + let value = reader.value(0)?; + reader.blanks(); + if reader.at != reader.bytes.len() { + return Err(reader.error("expected the end")); + } + Ok(value) +} + +struct Reader<'a> { + bytes: &'a [u8], + at: usize, +} + +impl Reader<'_> { + fn error(&self, message: &str) -> String { + format!("{message} at byte {}", self.at) + } + + fn blanks(&mut self) { + while matches!(self.bytes.get(self.at), Some(b' ' | b'\t' | b'\n' | b'\r')) { + self.at += 1; + } + } + + fn eat(&mut self, byte: u8) -> bool { + self.blanks(); + if self.bytes.get(self.at) == Some(&byte) { + self.at += 1; + true + } else { + false + } + } + + fn word(&mut self, word: &str, value: Json) -> Result { + if self.bytes[self.at..].starts_with(word.as_bytes()) { + self.at += word.len(); + Ok(value) + } else { + Err(self.error("expected a value")) + } + } + + fn value(&mut self, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(self.error("nested too deeply")); + } + self.blanks(); + match self.bytes.get(self.at) { + Some(b'{') => { + self.at += 1; + let mut entries = vec![]; + if self.eat(b'}') { + return Ok(Json::Object(entries)); + } + loop { + self.blanks(); + let key = self.string()?; + if !self.eat(b':') { + return Err(self.error("expected ':'")); + } + entries.push((key, self.value(depth + 1)?)); + if self.eat(b'}') { + return Ok(Json::Object(entries)); + } + if !self.eat(b',') { + return Err(self.error("expected ',' or '}'")); + } + } + } + Some(b'[') => { + self.at += 1; + let mut items = vec![]; + if self.eat(b']') { + return Ok(Json::Array(items)); + } + loop { + items.push(self.value(depth + 1)?); + if self.eat(b']') { + return Ok(Json::Array(items)); + } + if !self.eat(b',') { + return Err(self.error("expected ',' or ']'")); + } + } + } + Some(b'"') => Ok(Json::Str(self.string()?)), + Some(b't') => self.word("true", Json::Bool(true)), + Some(b'f') => self.word("false", Json::Bool(false)), + Some(b'n') => self.word("null", Json::Null), + Some(b'-' | b'0'..=b'9') => { + let start = self.at; + while matches!( + self.bytes.get(self.at), + Some(b'-' | b'+' | b'.' | b'e' | b'E' | b'0'..=b'9') + ) { + self.at += 1; + } + let text = std::str::from_utf8(&self.bytes[start..self.at]).unwrap_or_default(); + Ok(Json::Number(text.to_string())) + } + _ => Err(self.error("expected a value")), + } + } + + fn string(&mut self) -> Result { + if self.bytes.get(self.at) != Some(&b'"') { + return Err(self.error("expected a string")); + } + self.at += 1; + let mut text = Vec::new(); + loop { + let Some(&byte) = self.bytes.get(self.at) else { + return Err(self.error("this string is not closed")); + }; + self.at += 1; + match byte { + b'"' => { + return String::from_utf8(text).map_err(|_| self.error("not UTF-8")); + } + b'\\' => { + let Some(&escape) = self.bytes.get(self.at) else { + return Err(self.error("this string is not closed")); + }; + self.at += 1; + let c = match escape { + b'"' => '"', + b'\\' => '\\', + b'/' => '/', + b'b' => '\u{8}', + b'f' => '\u{c}', + b'n' => '\n', + b'r' => '\r', + b't' => '\t', + b'u' => { + let first = self.hex4()?; + let code = if (0xd800..0xdc00).contains(&first) { + if !self.bytes[self.at..].starts_with(b"\\u") { + return Err(self.error("a surrogate is not paired")); + } + self.at += 2; + let second = self.hex4()?; + 0x10000 + ((first - 0xd800) << 10) + (second.wrapping_sub(0xdc00)) + } else { + first + }; + char::from_u32(code).ok_or_else(|| self.error("not a character"))? + } + _ => return Err(self.error("unknown escape")), + }; + let mut buffer = [0; 4]; + text.extend_from_slice(c.encode_utf8(&mut buffer).as_bytes()); + } + byte => text.push(byte), + } + } + } + + fn hex4(&mut self) -> Result { + let digits = self + .bytes + .get(self.at..self.at + 4) + .and_then(|d| std::str::from_utf8(d).ok()) + .and_then(|d| u32::from_str_radix(d, 16).ok()) + .ok_or_else(|| self.error("expected four hexadecimal digits"))?; + self.at += 4; + Ok(digits) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn values_of_every_kind_are_read() { + let value = + parse(r#" { "a": [1, -2.5e3, true, false, null], "b": "x\"é😀", "c": {} } "#).unwrap(); + assert_eq!( + value.get("a").unwrap().as_array().unwrap(), + [ + Json::Number("1".into()), + Json::Number("-2.5e3".into()), + Json::Bool(true), + Json::Bool(false), + Json::Null + ] + ); + assert_eq!(value.get("b").unwrap().as_str(), Some("x\"\u{e9}\u{1f600}")); + assert_eq!(value.get("c"), Some(&Json::Object(vec![]))); + } + + #[test] + fn bad_documents_are_errors() { + for bad in ["", "[1,]", "{\"a\" 1}", "\"open", "[1] 2", "tru"] { + assert!(parse(bad).is_err(), "{bad}"); + } + assert!( + parse(&"[".repeat(1000)) + .unwrap_err() + .contains("nested too deeply") + ); + } +} diff --git a/src/package/lock.rs b/src/package/lock.rs new file mode 100644 index 0000000..0bc5639 --- /dev/null +++ b/src/package/lock.rs @@ -0,0 +1,233 @@ +//! `stone.lock`: what each package from a host held when it was first downloaded. +//! +//! Minimal version selection always picks the same versions from the same manifests (see +//! [`super::resolve`]), so the lockfile does not choose versions. It records a hash of each used +//! package's files, so a release whose tag later moves to other code fails to build rather than +//! changing the program. +//! +//! ```toml +//! [[package]] +//! name = "json" +//! github = "alice/stone-json" +//! version = "1.4.0" +//! hash = "sha256:4f1c..." +//! ``` + +use super::toml::{self, Table, Value, quote}; +use super::{Source, Version}; +use crate::diagnostic::Diagnostic; +use crate::span::{FileId, Span}; + +/// The comment that starts every lockfile stone writes. +const HEADER: &str = "# written by stone: the hash of what each package downloaded held\n"; + +/// What a `stone.lock` says. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Lock { + pub entries: Vec, +} + +/// One package of a lockfile. +#[derive(Debug, Clone, PartialEq)] +pub struct Locked { + pub name: String, + pub source: Source, + pub version: Version, + /// The hash of its files, as `sha256:` and hexadecimal digits (see + /// [`super::store::content_hash`]). + pub hash: String, + /// The span of the hash, for errors, or empty for an entry not read from a file. + pub span: Span, +} + +impl Lock { + /// Reads the lockfile `source` in `file`, returning what it could read and every error. + pub fn parse(source: &str, file: FileId) -> (Lock, Vec) { + let mut lock = Lock::default(); + let document = match toml::parse(source, file) { + Ok(document) => document, + Err(error) => return (lock, vec![error]), + }; + let mut errors = vec![]; + for entry in &document.entries { + match (&*entry.key, &entry.item.value) { + ("package", Value::Array(items)) => { + for item in items { + match &item.value { + Value::Table(table) => match locked(table) { + Ok(locked) => lock.entries.push(locked), + Err(message) => { + errors.push(Diagnostic::error(item.span, message)); + } + }, + _ => errors.push(Diagnostic::error(item.span, "expected a table")), + } + } + } + _ => errors.push(Diagnostic::error( + entry.key_span, + "a lockfile holds only [[package]] tables, which stone writes", + )), + } + } + (lock, errors) + } + + /// Returns the entry for `version` of the package from `source`, if there is one. + pub fn find(&self, source: &Source, version: &Version) -> Option<&Locked> { + self.entries + .iter() + .find(|e| e.source == *source && e.version == *version) + } + + /// Writes the lockfile, its entries sorted by name, source, and version, so the same + /// packages always give the same text. + pub fn write(&self) -> String { + let mut entries: Vec<&Locked> = self.entries.iter().collect(); + entries.sort_by(|a, b| { + (&a.name, a.source.to_string(), &a.version).cmp(&( + &b.name, + b.source.to_string(), + &b.version, + )) + }); + let mut text = HEADER.to_string(); + for entry in entries { + text.push_str("\n[[package]]\n"); + text.push_str(&format!("name = {}\n", quote(&entry.name))); + let (kind, host, path) = match &entry.source { + Source::GitHub { host, repo } => ("github", host, repo), + Source::GitLab { host, path } => ("gitlab", host, path), + Source::Path(_) => unreachable!("a directory is never locked"), + }; + text.push_str(&format!("{kind} = {}\n", quote(path))); + if let Some(host) = host { + text.push_str(&format!("host = {}\n", quote(host))); + } + text.push_str(&format!( + "version = {}\n", + quote(&entry.version.to_string()) + )); + text.push_str(&format!("hash = {}\n", quote(&entry.hash))); + } + text + } +} + +/// Reads one `[[package]]` table. +fn locked(table: &Table) -> Result { + let text = |key: &str| match table.get(key).map(|e| &e.item.value) { + Some(Value::Str(text)) => Ok(Some(text.clone())), + Some(_) => Err(format!("'{key}' is a string")), + None => Ok(None), + }; + let required = |key: &str| text(key)?.ok_or_else(|| format!("a package names its {key}")); + let host = text("host")?; + let source = match (text("github")?, text("gitlab")?) { + (Some(repo), None) => Source::GitHub { host, repo }, + (None, Some(path)) => Source::GitLab { host, path }, + _ => return Err("a package comes from one of github and gitlab".to_string()), + }; + let hash = required("hash")?; + if !is_hash(&hash) { + return Err(format!( + "'{hash}' is not a hash, written as 'sha256:' and 64 digits" + )); + } + Ok(Locked { + name: required("name")?, + source, + version: Version::parse(&required("version")?)?, + hash, + span: table.get("hash").map_or(Span::default(), |e| e.item.span), + }) +} + +/// Returns whether `text` is a hash as lockfiles write them. +fn is_hash(text: &str) -> bool { + text.strip_prefix("sha256:") + .is_some_and(|hex| hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(name: &str, source: Source, version: &str) -> Locked { + Locked { + name: name.to_string(), + source, + version: Version::parse(version).unwrap(), + hash: format!("sha256:{}", "ab".repeat(32)), + span: Span::default(), + } + } + + #[test] + fn a_lock_reads_back_what_it_writes() { + let lock = Lock { + entries: vec![ + entry( + "json", + Source::GitHub { + host: None, + repo: "alice/json".into(), + }, + "1.4.0", + ), + entry( + "http", + Source::GitLab { + host: Some("git.corp.com".into()), + path: "group/http".into(), + }, + "0.4.1", + ), + ], + }; + let text = lock.write(); + assert!(text.starts_with(HEADER)); + // sorted by name + assert!(text.find("http").unwrap() < text.find("json").unwrap()); + let (read, errors) = Lock::parse(&text, FileId(1)); + assert_eq!(errors, []); + let without_spans = |lock: &Lock| { + let mut entries: Vec<(String, String)> = lock + .entries + .iter() + .map(|e| { + ( + e.name.clone(), + format!("{} {} {}", e.source, e.version, e.hash), + ) + }) + .collect(); + entries.sort(); + entries + }; + assert_eq!(without_spans(&read), without_spans(&lock)); + assert_eq!(read.write(), text); + } + + #[test] + fn bad_entries_are_errors() { + let message = |text: &str| { + let (_, errors) = Lock::parse(text, FileId(1)); + errors[0].message.clone() + }; + assert_eq!( + message( + "[[package]]\nname = \"a\"\ngithub = \"o/a\"\nversion = \"1.0\"\nhash = \"x\"\n" + ), + "'x' is not a hash, written as 'sha256:' and 64 digits" + ); + assert_eq!( + message("[package]\n"), + "a lockfile holds only [[package]] tables, which stone writes" + ); + assert_eq!( + message("[[package]]\nname = \"a\"\nversion = \"1.0\"\n"), + "a package comes from one of github and gitlab" + ); + } +} diff --git a/src/package/resolve.rs b/src/package/resolve.rs new file mode 100644 index 0000000..70acb65 --- /dev/null +++ b/src/package/resolve.rs @@ -0,0 +1,262 @@ +//! Minimal version selection, which picks the version of every package a program uses. +//! +//! Each dependency gives the oldest version of a package it works with. The version used is the +//! newest of those any package in the graph asks for, among the versions of the same +//! compatibility class (see [`Version::class`]), which are different packages. Selection never +//! asks a host which versions exist, so it never picks one that no manifest names, and the same +//! manifests always give the same versions. This is Go's algorithm: every (package, version) +//! reachable from the program is read, the versions of a package that are not chosen included, +//! since they may still raise what another package needs. +//! +//! For example, if the program needs `a` 1.0 and `b` 1.0, and `b` 1.0 needs `a` 1.2, then `a` +//! 1.2 is used. A release `a` 1.5 that nothing names is not. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::path::PathBuf; + +use super::{Source, Version}; +use crate::span::Span; + +/// What a dependency names: a directory, or the oldest version of a package from a host. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Target { + Path(PathBuf), + Remote(Source, Version), +} + +impl Target { + /// Returns the package the target is a version of: a directory, or a source with a + /// compatibility class, so `json` 1.x and 2.x are two packages. + pub fn identity(&self) -> Identity { + match self { + Target::Path(dir) => Identity::Path(dir.clone()), + Target::Remote(source, version) => Identity::Remote(source.clone(), version.class()), + } + } +} + +/// A package, whatever its version. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Identity { + Path(PathBuf), + Remote(Source, String), +} + +/// One dependency of a package, by the name its files use it under. +#[derive(Debug, Clone, PartialEq)] +pub struct Requirement { + pub name: String, + /// The span of the dependency's key in its manifest. + pub span: Span, + pub target: Target, +} + +/// The versions selection chose, and what it read on the way. +#[derive(Debug, Default)] +pub struct Selection { + /// Every target read, in the order first reached, breadth first from the program. + pub order: Vec, + /// Each target's requirements, for the targets whose manifests could be read. + pub requirements: HashMap>, + /// The version chosen for each package from a host. + chosen: HashMap, +} + +impl Selection { + /// Returns the version of `target`'s package that is used: `target` itself for a directory, + /// and the chosen version for a package from a host. + pub fn resolve(&self, target: &Target) -> Target { + match target { + Target::Path(_) => target.clone(), + Target::Remote(source, version) => { + let chosen = self.chosen.get(&target.identity()).unwrap_or(version); + Target::Remote(source.clone(), chosen.clone()) + } + } + } + + /// Returns every target that is used, in the order first reached. + pub fn used(&self) -> impl Iterator { + self.order.iter().filter(|t| self.resolve(t) == **t) + } +} + +/// Selects the versions of every package reachable from `root`, the program's requirements, +/// asking `requirements` for each target's own, which it gives as `None` if it cannot read them, +/// as for a version that is not downloaded. +/// +/// For example, with `root` needing `a` 1.0 and `b` 1.0, and `b` 1.0 needing `a` 1.2, +/// `resolve` of `a` 1.0 is `a` 1.2. +pub fn select( + root: &[Requirement], + mut requirements: impl FnMut(&Target) -> Option>, +) -> Selection { + let mut selection = Selection::default(); + let mut seen = HashSet::new(); + let mut queue: VecDeque = root.iter().map(|r| r.target.clone()).collect(); + while let Some(target) = queue.pop_front() { + if !seen.insert(target.clone()) { + continue; + } + if let Target::Remote(_, version) = &target { + let chosen = selection + .chosen + .entry(target.identity()) + .or_insert_with(|| version.clone()); + if version > chosen { + *chosen = version.clone(); + } + } + selection.order.push(target.clone()); + if let Some(found) = requirements(&target) { + queue.extend(found.iter().map(|r| r.target.clone())); + selection.requirements.insert(target, found); + } + } + selection +} + +#[cfg(test)] +mod tests { + use super::*; + + fn source(name: &str) -> Source { + Source::GitHub { + host: None, + repo: format!("o/{name}"), + } + } + + fn remote(name: &str, version: &str) -> Target { + Target::Remote(source(name), Version::parse(version).unwrap()) + } + + fn requirement(name: &str, version: &str) -> Requirement { + Requirement { + name: name.to_string(), + span: Span::default(), + target: remote(name, version), + } + } + + /// A manifest of a test registry: a package's name, its version, and its dependencies with + /// their versions. + type Manifest<'a> = (&'a str, &'a str, &'a [(&'a str, &'a str)]); + + /// Selects from a registry of manifests, returning each used package as `name version`, in + /// order. + fn used(root: &[(&str, &str)], registry: &[Manifest]) -> Vec { + let root: Vec = root.iter().map(|(n, v)| requirement(n, v)).collect(); + let selection = select(&root, |target| { + registry.iter().find_map(|(name, version, deps)| { + (remote(name, version) == *target) + .then(|| deps.iter().map(|(n, v)| requirement(n, v)).collect()) + }) + }); + selection + .used() + .map(|target| match target { + Target::Remote(Source::GitHub { repo, .. }, version) => { + format!("{} {version}", &repo[2..]) + } + other => panic!("{other:?}"), + }) + .collect() + } + + #[test] + fn the_newest_version_asked_for_is_used() { + assert_eq!( + used( + &[("a", "1.0"), ("b", "1.0")], + &[ + ("a", "1.0.0", &[]), + ("a", "1.2.0", &[]), + ("a", "1.5.0", &[]), + ("b", "1.0.0", &[("a", "1.2")]), + ], + ), + ["b 1.0.0", "a 1.2.0"] + ); + } + + #[test] + fn a_diamond_meets_at_the_newer_version() { + assert_eq!( + used( + &[("b", "1.0"), ("c", "1.0")], + &[ + ("b", "1.0.0", &[("d", "1.1")]), + ("c", "1.0.0", &[("d", "1.3")]), + ("d", "1.1.0", &[]), + ("d", "1.3.0", &[]), + ], + ), + ["b 1.0.0", "c 1.0.0", "d 1.3.0"] + ); + } + + #[test] + fn a_version_that_is_not_used_still_raises_others() { + // c 1.0 is replaced by c 1.1, but what c 1.0 needs still counts, as in Go + assert_eq!( + used( + &[("b", "1.0"), ("c", "1.0")], + &[ + ("b", "1.0.0", &[("c", "1.1")]), + ("c", "1.0.0", &[("d", "1.2")]), + ("c", "1.1.0", &[("d", "1.0")]), + ("d", "1.0.0", &[]), + ("d", "1.2.0", &[]), + ], + ), + ["b 1.0.0", "c 1.1.0", "d 1.2.0"] + ); + } + + #[test] + fn major_versions_are_different_packages() { + assert_eq!( + used( + &[("a", "1.4"), ("b", "1.0")], + &[ + ("a", "1.4.0", &[]), + ("a", "2.1.0", &[]), + ("b", "1.0.0", &[("a", "2.1")]), + ], + ), + ["a 1.4.0", "b 1.0.0", "a 2.1.0"] + ); + // before 1.0, each minor version is its own class + assert_eq!( + used( + &[("a", "0.3"), ("a", "0.4")], + &[("a", "0.3.0", &[]), ("a", "0.4.0", &[])] + ), + ["a 0.3.0", "a 0.4.0"] + ); + } + + #[test] + fn cycles_end() { + assert_eq!( + used( + &[("a", "1.0")], + &[ + ("a", "1.0.0", &[("b", "1.0")]), + ("b", "1.0.0", &[("a", "1.1")]), + ("a", "1.1.0", &[("b", "1.0")]), + ], + ), + ["b 1.0.0", "a 1.1.0"] + ); + } + + #[test] + fn a_package_that_cannot_be_read_is_still_selected() { + let root = [requirement("a", "1.0")]; + let selection = select(&root, |_| None); + assert_eq!(selection.used().count(), 1); + assert!(selection.requirements.is_empty()); + } +} diff --git a/src/package/store.rs b/src/package/store.rs new file mode 100644 index 0000000..fded91d --- /dev/null +++ b/src/package/store.rs @@ -0,0 +1,141 @@ +//! The package store: where packages from hosts are downloaded, one directory per version. +//! +//! The store is `$STONE_HOME/packages`, which defaults to `~/.stone/packages`, or +//! `%LOCALAPPDATA%\stone\packages` on Windows. `json` 1.4.0 from `github:alice/stone-json` is in +//! `/github.com/alice/stone-json/1.4.0/`, holding only the package's `stone.toml` and +//! `.stn` files. + +use std::path::{Path, PathBuf}; + +use super::{MANIFEST, Source, Version}; +use crate::codegen::sha256::sha256; +use crate::project::Sources; + +/// Returns the store's directory, from `STONE_HOME` or else the home directory, or `None` if +/// neither is known. +pub fn default_dir() -> Option { + let home = match std::env::var_os("STONE_HOME") { + Some(home) if !home.is_empty() => PathBuf::from(home), + _ if cfg!(windows) => PathBuf::from(std::env::var_os("LOCALAPPDATA")?).join("stone"), + _ => PathBuf::from(std::env::var_os("HOME")?).join(".stone"), + }; + Some(home.join("packages")) +} + +/// Returns the host a source's packages come from, as `github.com`. +pub fn host(source: &Source) -> &str { + match source { + Source::GitHub { host, .. } => host.as_deref().unwrap_or("github.com"), + Source::GitLab { host, .. } => host.as_deref().unwrap_or("gitlab.com"), + Source::Path(_) => "", + } +} + +/// Returns the directory of `version` of the package from `source` in the store `store`. +/// +/// For example, `json` 1.4.0 from `github:alice/json` is `/github.com/alice/json/1.4.0`. +pub fn package_dir(store: &Path, source: &Source, version: &Version) -> PathBuf { + let path = match source { + Source::GitHub { repo, .. } => repo, + Source::GitLab { path, .. } => path, + Source::Path(dir) => return dir.clone(), + }; + let mut dir = store.join(host(source).replace(':', "_")); + dir.extend(path.split('/')); + dir.push(version.to_string()); + dir +} + +/// Returns whether a package keeps the file at `path`: its manifest and its stone files. +pub fn is_kept(path: &Path) -> bool { + path.file_name() == Some(std::ffi::OsStr::new(MANIFEST)) + || path.extension().is_some_and(|e| e == "stn") +} + +/// Returns the hash of the files a package keeps under `dir`, as `sha256:` and 64 hexadecimal +/// digits: the SHA-256 of a line ` ` per file, sorted by its path +/// from `dir` with `/` between names, so the hash depends on nothing but those files. +pub fn content_hash(sources: &dyn Sources, dir: &Path) -> String { + let mut files = vec![]; + let mut dirs = vec![(dir.to_path_buf(), String::new())]; + while let Some((at, relative)) = dirs.pop() { + for entry in sources.entries(&at) { + let Some(name) = entry.file_name().map(|n| n.to_string_lossy().into_owned()) else { + continue; + }; + let path = match relative.is_empty() { + true => name, + false => format!("{relative}/{name}"), + }; + if sources.is_dir(&entry) { + dirs.push((entry, path)); + } else if is_kept(&entry) + && let Some(text) = sources.read(&entry) + { + files.push((path, hex(&sha256(text.as_bytes())))); + } + } + } + files.sort(); + let listing: String = files + .iter() + .map(|(path, hash)| format!("{hash} {path}\n")) + .collect(); + format!("sha256:{}", hex(&sha256(listing.as_bytes()))) +} + +/// Writes bytes as lowercase hexadecimal digits. +pub fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::project::MapSources; + + #[test] + fn packages_are_stored_by_host_path_and_version() { + let source = Source::GitLab { + host: Some("git.corp.com:8443".into()), + path: "group/sub/http".into(), + }; + let dir = package_dir(Path::new("store"), &source, &Version::new(0, 4, 1)); + assert_eq!( + dir, + [ + "store", + "git.corp.com_8443", + "group", + "sub", + "http", + "0.4.1" + ] + .iter() + .collect::() + ); + } + + #[test] + fn the_hash_depends_only_on_the_kept_files() { + let mut sources = MapSources::default(); + sources.insert("p/stone.toml", "[package]\nname = \"p\"\n"); + sources.insert("p/lib.stn", "pub fn one() -> int:\n ret 1\n"); + sources.insert("p/sub/two.stn", "pub fn two() -> int:\n ret 2\n"); + let hash = content_hash(&sources, Path::new("p")); + assert!(hash.starts_with("sha256:") && hash.len() == 71); + + let mut moved = sources.clone(); + moved.insert("q/stone.toml", "[package]\nname = \"p\"\n"); + moved.insert("q/lib.stn", "pub fn one() -> int:\n ret 1\n"); + moved.insert("q/sub/two.stn", "pub fn two() -> int:\n ret 2\n"); + moved.insert("q/README.md", "not kept\n"); + assert_eq!(content_hash(&moved, Path::new("q")), hash); + + let mut changed = MapSources::default(); + changed.insert("p/stone.toml", "[package]\nname = \"p\"\n"); + changed.insert("p/lib.stn", "pub fn one() -> int:\n ret 2\n"); + changed.insert("p/sub/two.stn", "pub fn two() -> int:\n ret 2\n"); + assert_ne!(content_hash(&changed, Path::new("p")), hash); + } +} diff --git a/src/package/toml.rs b/src/package/toml.rs new file mode 100644 index 0000000..5f2eaa5 --- /dev/null +++ b/src/package/toml.rs @@ -0,0 +1,683 @@ +//! A reader for the part of TOML that `stone.toml` and `stone.lock` use, since stone depends on no +//! TOML library. +//! +//! It reads comments, `[table]` and `[[array of tables]]` headers with one name, `key = value` +//! lines with bare or quoted keys, and values that are strings (basic or literal, on one line), +//! integers, booleans, arrays (which may span lines), and inline tables. Every key and value keeps +//! its span, so errors point at the text, and `stone add` can edit a manifest in place. +//! +//! For example, reading `[dependencies]\njson = { github = "alice/json", version = "1.2.0" }\n` +//! gives a table with one entry, `dependencies`, whose table has one entry, `json`. + +use crate::diagnostic::Diagnostic; +use crate::span::{FileId, Pos, Span}; + +/// A value with the span of its text. +#[derive(Debug, Clone, PartialEq)] +pub struct Item { + pub value: Value, + pub span: Span, +} + +/// A TOML value. +#[derive(Debug, Clone, PartialEq)] +pub enum Value { + Str(String), + Int(i64), + Bool(bool), + Array(Vec), + Table(Table), +} + +impl Value { + /// Returns what the value is called in an error, such as `a string`. + pub fn kind(&self) -> &'static str { + match self { + Value::Str(_) => "a string", + Value::Int(_) => "an integer", + Value::Bool(_) => "a boolean", + Value::Array(_) => "an array", + Value::Table(_) => "a table", + } + } +} + +/// A table: its entries in the order they were written. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Table { + pub entries: Vec, +} + +impl Table { + /// Returns the entry named `key`, if there is one. + pub fn get(&self, key: &str) -> Option<&Entry> { + self.entries.iter().find(|entry| entry.key == key) + } +} + +/// One `key = value` of a table, or a `[key]` table or `[[key]]` array of tables of the document. +#[derive(Debug, Clone, PartialEq)] +pub struct Entry { + pub key: String, + pub key_span: Span, + pub item: Item, +} + +/// Reads a TOML document in `file`, returning its top-level table or the first error. +/// +/// For example, `parse("name = \"geo\"\n", file)` has one entry, `name`, holding the string +/// `geo`, and `parse("name = geo\n", file)` fails at `geo`. +pub fn parse(source: &str, file: FileId) -> Result { + let mut reader = Reader { + chars: source.chars().collect(), + at: 0, + pos: Pos::new(1, 1), + file, + }; + reader.document() +} + +/// Where a `[header]` puts the lines after it. +enum Current { + Root, + /// The top-level entry the header made, by index. + Table(usize), + /// The last table of the top-level array of tables, by index. + ArrayTable(usize), +} + +struct Reader { + chars: Vec, + at: usize, + pos: Pos, + file: FileId, +} + +impl Reader { + fn peek(&self) -> Option { + self.chars.get(self.at).copied() + } + + fn bump(&mut self) -> Option { + let c = self.peek()?; + self.at += 1; + if c == '\n' { + self.pos = Pos::new(self.pos.line + 1, 1); + } else { + self.pos.col += 1; + } + Some(c) + } + + fn span_from(&self, start: Pos) -> Span { + Span::new(start, self.pos).in_file(self.file) + } + + fn error(&self, message: impl Into) -> Diagnostic { + Diagnostic::error( + Span::new(self.pos, Pos::new(self.pos.line, self.pos.col + 1)).in_file(self.file), + message, + ) + } + + /// Returns what is next, for errors: the character in quotes, or `end of line`. + fn found(&self) -> String { + match self.peek() { + None => "end of file".to_string(), + Some('\n' | '\r') => "end of line".to_string(), + Some(c) => format!("'{c}'"), + } + } + + /// Skips spaces and tabs. + fn blanks(&mut self) { + while matches!(self.peek(), Some(' ' | '\t')) { + self.bump(); + } + } + + /// Skips blanks and a comment, then requires the end of the line, which it skips too. + fn end_of_line(&mut self) -> Result<(), Diagnostic> { + self.blanks(); + if self.peek() == Some('#') { + while !matches!(self.peek(), None | Some('\n')) { + self.bump(); + } + } + if self.peek() == Some('\r') { + self.bump(); + } + match self.peek() { + None => Ok(()), + Some('\n') => { + self.bump(); + Ok(()) + } + _ => Err(self.error(format!( + "expected the end of the line, found {}", + self.found() + ))), + } + } + + /// Skips blank lines, comments, and newlines, as between the values of an array. + fn whitespace(&mut self) { + loop { + match self.peek() { + Some(' ' | '\t' | '\r' | '\n') => { + self.bump(); + } + Some('#') => { + while !matches!(self.peek(), None | Some('\n')) { + self.bump(); + } + } + _ => return, + } + } + } + + fn document(&mut self) -> Result { + let mut root = Table::default(); + let mut current = Current::Root; + loop { + self.blanks(); + match self.peek() { + None => return Ok(root), + Some('\n' | '\r' | '#') => self.end_of_line()?, + Some('[') => { + current = self.header(&mut root)?; + self.end_of_line()?; + } + Some(_) => { + let entry = self.key_value()?; + let table = match current { + Current::Root => &mut root, + Current::Table(i) => match &mut root.entries[i].item.value { + Value::Table(table) => table, + _ => unreachable!("a header makes a table"), + }, + Current::ArrayTable(i) => match &mut root.entries[i].item.value { + Value::Array(items) => match &mut items.last_mut().unwrap().value { + Value::Table(table) => table, + _ => unreachable!("an array header makes tables"), + }, + _ => unreachable!("an array header makes an array"), + }, + }; + insert(table, entry)?; + self.end_of_line()?; + } + } + } + } + + /// Reads a `[name]` or `[[name]]` header, adding its table to `root`. + fn header(&mut self, root: &mut Table) -> Result { + let start = self.pos; + self.bump(); + let array = self.peek() == Some('['); + if array { + self.bump(); + } + self.blanks(); + let (key, key_span) = self.key()?; + self.blanks(); + if self.peek() == Some('.') { + return Err(self.error("dotted table names are not supported, so write one name")); + } + for _ in 0..if array { 2 } else { 1 } { + if self.peek() != Some(']') { + return Err(self.error(format!("expected ']', found {}", self.found()))); + } + self.bump(); + } + let span = self.span_from(start); + let table = Item { + value: Value::Table(Table::default()), + span, + }; + let existing = root.entries.iter().position(|entry| entry.key == key); + match (existing, array) { + (None, false) => { + root.entries.push(Entry { + key, + key_span, + item: table, + }); + Ok(Current::Table(root.entries.len() - 1)) + } + (None, true) => { + root.entries.push(Entry { + key, + key_span, + item: Item { + value: Value::Array(vec![table]), + span, + }, + }); + Ok(Current::ArrayTable(root.entries.len() - 1)) + } + (Some(i), true) + if matches!(&root.entries[i].item.value, Value::Array(items) + if items.iter().all(|item| matches!(item.value, Value::Table(_)))) => + { + if let Value::Array(items) = &mut root.entries[i].item.value { + items.push(table); + } + Ok(Current::ArrayTable(i)) + } + (Some(_), _) => Err(Diagnostic::error( + key_span, + format!("'{key}' is already defined"), + )), + } + } + + /// Reads a bare key (letters, digits, `_`, and `-`) or a quoted one. + fn key(&mut self) -> Result<(String, Span), Diagnostic> { + let start = self.pos; + let key = match self.peek() { + Some('"') => self.basic_string()?, + Some('\'') => self.literal_string()?, + Some(c) if is_bare(c) => { + let mut key = String::new(); + while let Some(c) = self.peek().filter(|c| is_bare(*c)) { + key.push(c); + self.bump(); + } + key + } + _ => return Err(self.error(format!("expected a key, found {}", self.found()))), + }; + Ok((key, self.span_from(start))) + } + + /// Reads `key = value`. + fn key_value(&mut self) -> Result { + let (key, key_span) = self.key()?; + self.blanks(); + if self.peek() == Some('.') { + return Err(self.error("dotted keys are not supported, so write one name")); + } + if self.peek() != Some('=') { + return Err(self.error(format!( + "expected '=' after the key, found {}", + self.found() + ))); + } + self.bump(); + self.blanks(); + let item = self.value()?; + Ok(Entry { + key, + key_span, + item, + }) + } + + fn value(&mut self) -> Result { + let start = self.pos; + let value = match self.peek() { + Some('"') => { + if self.chars[self.at..].starts_with(&['"', '"', '"']) { + return Err(self.error("multi-line strings are not supported")); + } + Value::Str(self.basic_string()?) + } + Some('\'') => { + if self.chars[self.at..].starts_with(&['\'', '\'', '\'']) { + return Err(self.error("multi-line strings are not supported")); + } + Value::Str(self.literal_string()?) + } + Some('[') => self.array()?, + Some('{') => self.inline_table()?, + Some(c) if c.is_ascii_digit() || c == '-' || c == '+' => self.integer()?, + Some(c) if c.is_ascii_alphabetic() => { + let mut word = String::new(); + while let Some(c) = self.peek().filter(|c| c.is_ascii_alphanumeric()) { + word.push(c); + self.bump(); + } + match word.as_str() { + "true" => Value::Bool(true), + "false" => Value::Bool(false), + _ => { + return Err(Diagnostic::error( + self.span_from(start), + format!("expected a value, found '{word}' (write a string in quotes)"), + )); + } + } + } + _ => return Err(self.error(format!("expected a value, found {}", self.found()))), + }; + Ok(Item { + value, + span: self.span_from(start), + }) + } + + fn integer(&mut self) -> Result { + let start = self.pos; + let mut text = String::new(); + if let Some(sign @ ('-' | '+')) = self.peek() { + text.push(sign); + self.bump(); + } + while let Some(c) = self.peek().filter(|c| c.is_ascii_digit() || *c == '_') { + if c != '_' { + text.push(c); + } + self.bump(); + } + text.parse().map(Value::Int).map_err(|_| { + Diagnostic::error( + self.span_from(start), + "expected an integer (floats and dates are not supported)", + ) + }) + } + + fn array(&mut self) -> Result { + self.bump(); + let mut items = vec![]; + loop { + self.whitespace(); + if self.peek() == Some(']') { + self.bump(); + return Ok(Value::Array(items)); + } + items.push(self.value()?); + self.whitespace(); + match self.peek() { + Some(',') => { + self.bump(); + } + Some(']') => {} + _ => return Err(self.error(format!("expected ',' or ']', found {}", self.found()))), + } + } + } + + fn inline_table(&mut self) -> Result { + self.bump(); + let mut table = Table::default(); + self.blanks(); + if self.peek() == Some('}') { + self.bump(); + return Ok(Value::Table(table)); + } + loop { + self.blanks(); + let entry = self.key_value()?; + insert(&mut table, entry)?; + self.blanks(); + match self.peek() { + Some(',') => { + self.bump(); + } + Some('}') => { + self.bump(); + return Ok(Value::Table(table)); + } + _ => { + return Err(self.error(format!( + "expected ',' or '}}' in an inline table, which stays on one line, \ + found {}", + self.found() + ))); + } + } + } + } + + fn basic_string(&mut self) -> Result { + self.bump(); + let mut text = String::new(); + loop { + match self.peek() { + None | Some('\n' | '\r') => return Err(self.error("this string is not closed")), + Some('"') => { + self.bump(); + return Ok(text); + } + Some('\\') => { + self.bump(); + let escaped = match self.peek() { + Some('"') => '"', + Some('\\') => '\\', + Some('n') => '\n', + Some('t') => '\t', + Some('r') => '\r', + Some('b') => '\u{8}', + Some('f') => '\u{c}', + Some(c @ ('u' | 'U')) => { + self.bump(); + let digits = if c == 'u' { 4 } else { 8 }; + let mut code = 0; + for _ in 0..digits { + let Some(digit) = self.peek().and_then(|c| c.to_digit(16)) else { + return Err(self.error("expected a hexadecimal digit")); + }; + code = code * 16 + digit; + self.bump(); + } + let Some(c) = char::from_u32(code) else { + return Err(self.error("this escape is not a character")); + }; + text.push(c); + continue; + } + _ => return Err(self.error(format!("unknown escape {}", self.found()))), + }; + self.bump(); + text.push(escaped); + } + Some(c) => { + self.bump(); + text.push(c); + } + } + } + } + + fn literal_string(&mut self) -> Result { + self.bump(); + let mut text = String::new(); + loop { + match self.peek() { + None | Some('\n' | '\r') => return Err(self.error("this string is not closed")), + Some('\'') => { + self.bump(); + return Ok(text); + } + Some(c) => { + self.bump(); + text.push(c); + } + } + } + } +} + +/// Adds `entry` to `table`, unless it already has the key. +fn insert(table: &mut Table, entry: Entry) -> Result<(), Diagnostic> { + if table.get(&entry.key).is_some() { + return Err(Diagnostic::error( + entry.key_span, + format!("'{}' is already defined", entry.key), + )); + } + table.entries.push(entry); + Ok(()) +} + +/// Returns whether `c` may be in a bare key. +fn is_bare(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '_' || c == '-' +} + +/// Writes `text` as a TOML basic string, quoted and escaped. +/// +/// For example, `quote("a\"b")` returns `"a\"b"` with the quote escaped. +pub fn quote(text: &str) -> String { + let mut quoted = String::from("\""); + for c in text.chars() { + match c { + '"' => quoted.push_str("\\\""), + '\\' => quoted.push_str("\\\\"), + '\n' => quoted.push_str("\\n"), + '\t' => quoted.push_str("\\t"), + '\r' => quoted.push_str("\\r"), + c if c.is_control() => quoted.push_str(&format!("\\u{:04X}", c as u32)), + c => quoted.push(c), + } + } + quoted.push('"'); + quoted +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(source: &str) -> Table { + parse(source, FileId(1)).unwrap_or_else(|e| panic!("should parse: {e}")) + } + + fn error(source: &str) -> (String, usize, usize) { + let e = parse(source, FileId(1)).expect_err("should fail"); + (e.message, e.span.start.line, e.span.start.col) + } + + fn string(table: &Table, key: &str) -> String { + match &table.get(key).expect("key").item.value { + Value::Str(s) => s.clone(), + other => panic!("not a string: {other:?}"), + } + } + + fn table<'a>(table: &'a Table, key: &str) -> &'a Table { + match &table.get(key).expect("key").item.value { + Value::Table(t) => t, + other => panic!("not a table: {other:?}"), + } + } + + #[test] + fn keys_hold_strings_integers_and_booleans() { + let doc = + read("name = \"geo\" # a comment\ncount = 3\nyes = true\n'quoted key' = 'raw\\'\n"); + assert_eq!(string(&doc, "name"), "geo"); + assert_eq!(doc.get("count").unwrap().item.value, Value::Int(3)); + assert_eq!(doc.get("yes").unwrap().item.value, Value::Bool(true)); + assert_eq!(string(&doc, "quoted key"), "raw\\"); + } + + #[test] + fn headers_hold_the_lines_after_them() { + let doc = read( + "top = \"t\"\n\n[package]\nname = \"geo\"\n\n[dependencies]\n\ + json = { github = \"alice/json\", version = \"1.2.0\" }\n", + ); + assert_eq!(string(&doc, "top"), "t"); + assert_eq!(string(table(&doc, "package"), "name"), "geo"); + let json = table(table(&doc, "dependencies"), "json"); + assert_eq!(string(json, "github"), "alice/json"); + assert_eq!(string(json, "version"), "1.2.0"); + } + + #[test] + fn array_headers_make_a_table_each() { + let doc = read("[[package]]\nname = \"a\"\n[[package]]\nname = \"b\"\n"); + let Value::Array(items) = &doc.get("package").unwrap().item.value else { + panic!("an array"); + }; + let names: Vec = items + .iter() + .map(|item| match &item.value { + Value::Table(t) => string(t, "name"), + _ => panic!("tables"), + }) + .collect(); + assert_eq!(names, ["a", "b"]); + } + + #[test] + fn arrays_may_span_lines() { + let doc = read("xs = [\n \"a\", # first\n \"b\",\n]\n"); + let Value::Array(items) = &doc.get("xs").unwrap().item.value else { + panic!("an array"); + }; + assert_eq!(items.len(), 2); + } + + #[test] + fn strings_read_escapes() { + let doc = read("s = \"a\\\"b\\u00e9\\n\"\n"); + assert_eq!(string(&doc, "s"), "a\"b\u{e9}\n"); + assert_eq!(quote("a\"b\n"), "\"a\\\"b\\n\""); + } + + #[test] + fn keys_and_values_keep_their_spans() { + let doc = read("[dependencies]\njson = { version = \"1.0\" }\n"); + let deps = table(&doc, "dependencies"); + let json = deps.get("json").unwrap(); + assert_eq!( + (json.key_span.start, json.key_span.end), + (Pos::new(2, 1), Pos::new(2, 5)) + ); + assert_eq!(json.key_span.file, FileId(1)); + let version = table(deps, "json").get("version").unwrap(); + assert_eq!(version.item.span.start, Pos::new(2, 20)); + } + + #[test] + fn errors_point_at_the_mistake() { + assert_eq!( + error("name = geo\n"), + ( + "expected a value, found 'geo' (write a string in quotes)".to_string(), + 1, + 8 + ) + ); + assert_eq!( + error("name \"geo\"\n"), + ("expected '=' after the key, found '\"'".to_string(), 1, 6) + ); + assert_eq!( + error("a = 1\na = 2\n"), + ("'a' is already defined".to_string(), 2, 1) + ); + assert_eq!( + error("[package]\n[package]\n"), + ("'package' is already defined".to_string(), 2, 2) + ); + assert_eq!( + error("s = \"open\n"), + ("this string is not closed".to_string(), 1, 10) + ); + assert_eq!( + error("a = 1 2\n"), + ("expected the end of the line, found '2'".to_string(), 1, 7) + ); + assert_eq!( + error("t = { a = 1,\n b = 2 }\n").0, + "expected a key, found end of line" + ); + assert_eq!( + error("a.b = 1\n").0, + "dotted keys are not supported, so write one name" + ); + } + + #[test] + fn windows_line_endings_are_read() { + let doc = read("a = 1\r\nb = 2\r\n"); + assert_eq!(doc.entries.len(), 2); + } +} diff --git a/src/package/version.rs b/src/package/version.rs new file mode 100644 index 0000000..05c8736 --- /dev/null +++ b/src/package/version.rs @@ -0,0 +1,238 @@ +//! Semantic versions, as packages and their release tags are numbered. +//! +//! For example, `1.2.0` is newer than `1.1.9` and than `1.2.0-beta.1`, and is compatible with +//! `1.0.0`, while `0.4.2` is compatible with `0.4.0` but not with `0.5.0`. + +use std::cmp::Ordering; +use std::fmt; + +/// A version `major.minor.patch`, possibly with a pre-release such as `-beta.1`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Version { + pub major: u64, + pub minor: u64, + pub patch: u64, + /// The pre-release identifiers after `-`, such as `["beta", "1"]`, or empty for a release. + pub pre: Vec, +} + +impl Version { + pub fn new(major: u64, minor: u64, patch: u64) -> Version { + Version { + major, + minor, + patch, + pre: vec![], + } + } + + /// Reads a version written as `1`, `1.2`, or `1.2.3`, optionally followed by a pre-release + /// such as `-beta.1` and build metadata such as `+linux`, which is ignored. A missing minor or + /// patch number is 0. + /// + /// For example, `Version::parse("1.2")` is `1.2.0`, and `Version::parse("v1.2")` fails, since + /// the `v` belongs to tags, not versions. + pub fn parse(text: &str) -> Result { + let text = text.split_once('+').map_or(text, |(version, _)| version); + let (numbers, pre) = match text.split_once('-') { + Some((numbers, pre)) => (numbers, Some(pre)), + None => (text, None), + }; + if text.starts_with(['v', 'V']) { + return Err(format!( + "'{text}' is not a version; write it without the 'v', as '{}'", + &text[1..] + )); + } + let parts: Vec<&str> = numbers.split('.').collect(); + if parts.len() > 3 { + return Err(format!( + "'{text}' is not a version, which has at most three numbers" + )); + } + let mut values = [0u64; 3]; + for (value, part) in values.iter_mut().zip(&parts) { + let valid = !part.is_empty() + && part.bytes().all(|b| b.is_ascii_digit()) + && (part.len() == 1 || !part.starts_with('0')); + *value = match part.parse() { + Ok(n) if valid => n, + _ => { + return Err(format!( + "'{text}' is not a version, which is numbers separated by '.', as '1.2.0'" + )); + } + }; + } + let pre = match pre { + None => vec![], + Some(pre) => { + let identifiers: Vec = pre.split('.').map(str::to_string).collect(); + let valid = identifiers.iter().all(|id| { + !id.is_empty() && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') + }); + if !valid { + return Err(format!("'{text}' has a pre-release that is not valid")); + } + identifiers + } + }; + Ok(Version { + major: values[0], + minor: values[1], + patch: values[2], + pre, + }) + } + + /// Reads a release tag, which is a version with or without a leading `v`. + /// + /// For example, `v1.2.0` and `1.2.0` are both `1.2.0`, and `release-1` is `None`. + pub fn from_tag(tag: &str) -> Option { + let text = tag.strip_prefix('v').unwrap_or(tag); + // a tag names a whole version + if text.split(['-', '+']).next()?.split('.').count() != 3 { + return None; + } + Version::parse(text).ok() + } + + /// Returns the versions this one is compatible with as a name: the major version from 1.0 on, + /// and the major and minor before that, as Cargo treats them. + /// + /// For example, `1.4.2` gives `1`, and `0.4.2` gives `0.4`. + pub fn class(&self) -> String { + if self.major == 0 { + format!("0.{}", self.minor) + } else { + self.major.to_string() + } + } + + /// Returns whether this is a pre-release, such as `1.0.0-rc.1`. + pub fn is_pre(&self) -> bool { + !self.pre.is_empty() + } +} + +impl fmt::Display for Version { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}.{}.{}", self.major, self.minor, self.patch)?; + if !self.pre.is_empty() { + write!(f, "-{}", self.pre.join("."))?; + } + Ok(()) + } +} + +impl Ord for Version { + fn cmp(&self, other: &Self) -> Ordering { + (self.major, self.minor, self.patch) + .cmp(&(other.major, other.minor, other.patch)) + .then_with(|| compare_pre(&self.pre, &other.pre)) + } +} + +impl PartialOrd for Version { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// Orders pre-releases as semver does: a release after every pre-release of it, numeric +/// identifiers by value and before others, and a longer list after its prefix. +fn compare_pre(a: &[String], b: &[String]) -> Ordering { + match (a.is_empty(), b.is_empty()) { + (true, true) => return Ordering::Equal, + (true, false) => return Ordering::Greater, + (false, true) => return Ordering::Less, + (false, false) => {} + } + for (x, y) in a.iter().zip(b) { + let order = match (x.parse::(), y.parse::()) { + (Ok(x), Ok(y)) => x.cmp(&y), + (Ok(_), Err(_)) => Ordering::Less, + (Err(_), Ok(_)) => Ordering::Greater, + (Err(_), Err(_)) => x.cmp(y), + }; + if order != Ordering::Equal { + return order; + } + } + a.len().cmp(&b.len()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v(text: &str) -> Version { + Version::parse(text).unwrap() + } + + #[test] + fn short_versions_fill_in_zeros() { + assert_eq!(v("1"), Version::new(1, 0, 0)); + assert_eq!(v("1.2"), Version::new(1, 2, 0)); + assert_eq!(v("1.2.3+build").to_string(), "1.2.3"); + assert_eq!(v("1.2.3-beta.1").to_string(), "1.2.3-beta.1"); + } + + #[test] + fn bad_versions_are_errors() { + assert!( + Version::parse("v1.2") + .unwrap_err() + .contains("without the 'v', as '1.2'") + ); + for bad in ["", "1.", "1.2.3.4", "01.2", "a.b", "1.2-", "1.2-be ta"] { + assert!(Version::parse(bad).is_err(), "{bad}"); + } + } + + #[test] + fn versions_order_as_semver_does() { + let mut versions = [ + v("1.0.0"), + v("1.0.0-rc.1"), + v("1.0.0-beta.11"), + v("1.0.0-beta.2"), + v("1.0.0-beta"), + v("1.0.0-alpha.1"), + v("0.9.10"), + v("0.9.9"), + ]; + versions.sort(); + let sorted: Vec = versions.iter().map(Version::to_string).collect(); + assert_eq!( + sorted, + [ + "0.9.9", + "0.9.10", + "1.0.0-alpha.1", + "1.0.0-beta", + "1.0.0-beta.2", + "1.0.0-beta.11", + "1.0.0-rc.1", + "1.0.0" + ] + ); + } + + #[test] + fn classes_split_majors_and_zero_minors() { + assert_eq!(v("1.4.2").class(), "1"); + assert_eq!(v("2.0.0").class(), "2"); + assert_eq!(v("0.4.2").class(), "0.4"); + assert_eq!(v("0.0.3").class(), "0.0"); + } + + #[test] + fn tags_are_whole_versions_with_or_without_v() { + assert_eq!(Version::from_tag("v1.2.0"), Some(v("1.2.0"))); + assert_eq!(Version::from_tag("1.2.0"), Some(v("1.2.0"))); + assert_eq!(Version::from_tag("v1.2.0-rc.1"), Some(v("1.2.0-rc.1"))); + assert_eq!(Version::from_tag("v1.2"), None); + assert_eq!(Version::from_tag("release-1"), None); + } +} diff --git a/src/project.rs b/src/project.rs index 0be81d5..996773d 100644 --- a/src/project.rs +++ b/src/project.rs @@ -30,6 +30,10 @@ use crate::ast::{ use crate::checker::{DISPLAY, EQ, FROM, OPTION, ORD, RESULT, TRY_FROM, collect_assigned}; use crate::diagnostic::{Diagnostic, excerpt, location}; use crate::lexer::Lexer; +use crate::package::lock::{Lock, Locked}; +use crate::package::resolve::{self, Requirement}; +use crate::package::store::{content_hash, package_dir}; +use crate::package::{LIB, LOCK, MANIFEST, Manifest, Source, Version, join_normal}; use crate::parser::Parser; use crate::span::{FileId, Pos, Span}; use crate::stdlib::{ @@ -49,6 +53,17 @@ pub trait Sources { /// Returns the files and directories directly in the directory `dir`, for editors to suggest /// modules. fn entries(&self, dir: &Path) -> Vec; + /// Returns the directory packages from hosts are downloaded into (see + /// [`crate::package::store`]), if there is one. + fn store(&self) -> Option { + crate::package::store::default_dir() + } + /// Writes `text` to the file at `path`, as linking does to keep `stone.lock` current. Sources + /// that only read, as an editor's, write nothing. + fn write(&self, path: &Path, text: &str) -> Result<(), String> { + let _ = (path, text); + Ok(()) + } } /// Reads modules from the filesystem. @@ -72,6 +87,10 @@ impl Sources for FsSources { }; entries.filter_map(|e| e.ok().map(|e| e.path())).collect() } + + fn write(&self, path: &Path, text: &str) -> Result<(), String> { + std::fs::write(path, text).map_err(|e| format!("cannot write {}: {e}", path.display())) + } } /// Reads modules from a map of paths to sources, where a directory is any path that one of the @@ -81,12 +100,26 @@ impl Sources for FsSources { #[derive(Debug, Default, Clone)] pub struct MapSources { files: HashMap, + /// The package store's directory, which is none unless set. + store: Option, + /// Every file written, in order, shared with clones. + written: std::sync::Arc>>, } impl MapSources { pub fn insert(&mut self, path: impl Into, source: impl Into) { self.files.insert(path.into(), source.into()); } + + /// Makes `dir` the package store, whose packages are files under it like any other. + pub fn set_store(&mut self, dir: impl Into) { + self.store = Some(dir.into()); + } + + /// Returns every file written through [`Sources::write`], in order. + pub fn written(&self) -> Vec<(PathBuf, String)> { + self.written.lock().expect("not poisoned").clone() + } } impl Sources for MapSources { @@ -98,6 +131,18 @@ impl Sources for MapSources { self.files.keys().any(|p| p != path && p.starts_with(path)) } + fn store(&self) -> Option { + self.store.clone() + } + + fn write(&self, path: &Path, text: &str) -> Result<(), String> { + self.written + .lock() + .expect("not poisoned") + .push((path.to_path_buf(), text.to_string())); + Ok(()) + } + fn entries(&self, dir: &Path) -> Vec { let mut entries: Vec = self .files @@ -117,9 +162,14 @@ pub struct SourceFile { /// The path diagnostics show, which is the root joined with the module's path, such as /// `app/geometry/shapes.stn`. pub path: PathBuf, - /// The module's dotted name, such as `geometry.shapes`, or empty for the entry file. + /// The module's dotted name, such as `geometry.shapes`, or empty for the entry file and a + /// manifest. A module of a dependency is named by its package's linked prefix, as + /// `geo.1.shapes` (see [`crate::package`]). pub module: String, pub source: String, + /// The name of the package the file belongs to, if it is a dependency's rather than the + /// program's own, such as `geo`. + pub package: Option, } impl SourceFile { @@ -130,6 +180,19 @@ impl SourceFile { crate::stdlib::source_module(&self.module).is_some() && self.path == Path::new(&format!("<{}>", self.module)) } + + /// Returns whether the file belongs to a package the program depends on, whose code the + /// program's author did not write. + pub fn is_dependency(&self) -> bool { + self.package.is_some() + } + + /// Returns whether the file is a package's `stone.toml` or `stone.lock` rather than stone + /// source. + pub fn is_package_file(&self) -> bool { + let name = self.path.file_name(); + name == Some(std::ffi::OsStr::new(MANIFEST)) || name == Some(std::ffi::OsStr::new(LOCK)) + } } /// Every file of a program, indexed by [`FileId`], with the entry file first. @@ -143,6 +206,7 @@ static PRELUDE_FILE: LazyLock = LazyLock::new(|| SourceFile { path: PathBuf::from(""), module: String::new(), source: PRELUDE.to_string(), + package: None, }); impl SourceMap { @@ -242,6 +306,78 @@ pub struct Linked { pub imports: Vec, /// The linked name of every `pub` function of a library module, such as `geometry.vec.dot`. pub public: HashSet, + /// The packages from hosts the program uses that are not downloaded, which are errors too. + pub missing: Vec<(Source, Version)>, + /// An error at each `use` of a dependency that could not be read, saying why, for editors. + /// These are not in `diagnostics`, which report each such dependency once, where its + /// manifest names it. + pub unavailable: Vec, + /// The package each file belongs to, by index into `packages`. + pub file_packages: Vec, + /// The program's `stone.lock` and the text to write to it, if it should change. + pub lock: Option<(PathBuf, String)>, + /// The program, then every package it uses, for `stone tree`. + pub packages: Vec, +} + +/// A package a program is linked with, as `stone tree` shows it. +#[derive(Debug, Clone, PartialEq)] +pub struct PackageInfo { + /// Its name, which is empty for a program whose manifest names none. + pub name: String, + /// The version used, if it has one. + pub version: Option, + /// Where it comes from, which is `None` for the program. + pub source: Option, + /// Each package it uses, by the name it uses it under and its index in + /// [`Linked::packages`], sorted by name. + pub deps: Vec<(String, usize)>, + /// The directory its module paths start from. + pub root: PathBuf, + /// What its modules' linked names start with, as `geo.1`, or empty for the program. + pub prefix: String, +} + +/// Returns `text` with each dependency's linked prefix written as the package's name, the way +/// the program's files name it, wherever the prefix starts a name. +/// +/// For example, with `geo` 1.2.0 a dependency, `fn geo.1.shapes.area(p: geo.1.Point)` becomes +/// `fn geo.shapes.area(p: geo.Point)`. A prefix has a segment of digits, which no name of the +/// program's own has, so nothing else changes. Two packages of one name keep their prefixes, +/// which are what tell them apart. +pub fn shown_names(text: &str, packages: &[PackageInfo]) -> String { + let dependencies = || packages.iter().filter(|p| !p.prefix.is_empty()); + let prefixes: Vec<(&str, &str)> = dependencies() + .filter(|p| dependencies().filter(|q| q.name == p.name).count() == 1) + .map(|p| (p.prefix.as_str(), p.name.as_str())) + .collect(); + if prefixes.is_empty() { + return text.to_string(); + } + let is_word = |c: char| c.is_alphanumeric() || c == '_'; + let mut shown = String::with_capacity(text.len()); + let mut rest = text; + let mut previous: Option = None; + while let Some(c) = rest.chars().next() { + let starts_name = previous.is_none_or(|p| !is_word(p) && p != '.'); + let found = prefixes.iter().find(|(prefix, _)| { + rest.starts_with(prefix) + && rest[prefix.len()..] + .chars() + .next() + .is_none_or(|n| !is_word(n)) + }); + if let Some((prefix, name)) = found.filter(|_| starts_name) { + shown.push_str(name); + rest = &rest[prefix.len()..]; + previous = name.chars().last(); + continue; + } + shown.push(c); + rest = &rest[c.len_utf8()..]; + previous = Some(c); + } + shown } /// The entry file's name for programs given as a single source string, such as by @@ -278,9 +414,26 @@ pub fn link_in(root: &Path, entry: &Path, source: &str, sources: &dyn Sources) - by_name: HashMap::new(), uses: vec![], own: HashSet::new(), + packages: vec![Package { + root: root.to_path_buf(), + prefix: String::new(), + name: String::new(), + deps: HashMap::new(), + unavailable: HashMap::new(), + source: None, + version: None, + }], + file_package: vec![], + nodes: HashMap::new(), + named: HashMap::new(), + missing: vec![], + failed: HashMap::new(), + unavailable: vec![], + lock: None, diagnostics: vec![], }; - loader.add(entry.to_path_buf(), String::new(), source.to_string()); + loader.add(entry.to_path_buf(), String::new(), source.to_string(), 0); + loader.load_packages(); // each file's uses can add files, which are resolved in turn let mut next = 0; @@ -330,6 +483,38 @@ struct Use { target: Option, } +/// A package whose files are part of the program: the program itself, or one it depends on, +/// directly or not. +struct Package { + /// The directory its module paths start from. + root: PathBuf, + /// What its modules' linked names start with, as `geo.1` (see [`Loader::load_packages`]), + /// or empty for the program, whose modules are named by their paths alone. + prefix: String, + /// Its name, from its manifest, as `geo`. + name: String, + /// Each package its files use, by the name they use it under. + deps: HashMap, + /// The names of its dependencies that could not be read, whose errors are reported where + /// the manifest names them, so a `use` of one says nothing more, with the reason, if one + /// was given, for editors (see [`Linked::unavailable`]). + unavailable: HashMap>, + /// Where it comes from and its version, for `stone tree`. + source: Option, + version: Option, +} + +/// A package whose manifest was read while selecting versions. +#[derive(Clone)] +struct Node { + dir: PathBuf, + name: String, + /// The version it was released as, or its manifest's for a directory. + version: Version, + /// The version its manifest gives, if any. + written: Option, +} + /// State for loading every file a program uses. struct Loader<'a> { sources: &'a dyn Sources, @@ -345,12 +530,31 @@ struct Loader<'a> { /// The files that are part of stone, written in stone, such as `net`'s, which alone may use /// the internal builtin modules. own: HashSet, + /// The program first, then every package it depends on, directly or not. + packages: Vec, + /// Each file's package, by index into `packages`. + file_package: Vec, + /// Every package read while selecting versions. + nodes: HashMap, + /// Where each target was first named, by which dependency, and as what: its directory as + /// written, or the dependency's name for a package from a host. + named: HashMap, + /// The packages from hosts that are not in the store. + missing: Vec<(Source, Version)>, + /// Why each package that could not be read could not be, when that is said where it is + /// named. + failed: HashMap, + /// The errors at each `use` of a dependency that could not be read. + unavailable: Vec, + /// The lockfile to write, if it should change. + lock: Option<(PathBuf, String)>, diagnostics: Vec, } impl Loader<'_> { - /// Parses a file and adds it to the program, recording its syntax errors. - fn add(&mut self, path: PathBuf, module: String, source: String) -> FileId { + /// Parses a file of the package `package` and adds it to the program, recording its syntax + /// errors. + fn add(&mut self, path: PathBuf, module: String, source: String, package: usize) -> FileId { let id = FileId(self.files.len() as u32); let (tokens, lex_errors) = Lexer::with_file(&source, id).lex_recovering(); let (mut parsed, parse_errors) = Parser::new(&tokens).parse_recovering(); @@ -370,40 +574,368 @@ impl Loader<'_> { path, module, source, + package: (package != 0).then(|| self.packages[package].name.clone()), }); self.modules.push(parsed); self.uses.push(vec![]); + self.file_package.push(package); + id + } + + /// Adds a package's `stone.toml` or `stone.lock` to the program's files, so its errors + /// render like any other, with no statements. `package` names the dependency it belongs to, + /// if it is not the program's. + fn add_manifest(&mut self, path: PathBuf, source: String, package: Option) -> FileId { + let id = FileId(self.files.len() as u32); + self.files.push(SourceFile { + path, + module: String::new(), + source, + package, + }); + self.modules.push(Mod::Module { body: vec![] }); + self.uses.push(vec![]); + self.file_package.push(0); id } - /// Returns the file of the module with these dotted names, loading it if it exists. + /// Reads the program's `stone.toml`, if it has one, selects the version of every package it + /// depends on, directly or not (see [`crate::package::resolve`]), and adds each package used. + /// A package from a host is read from the store, and checked against `stone.lock`. /// - /// For example, `["geometry", "shapes"]` loads `/geometry/shapes.stn`. - fn module(&mut self, names: &[String]) -> Option { - if self.is_entry(names) { + /// A dependency's modules are linked under a prefix of its name and the compatibility class + /// of its version, as `geo.1` for `geo` 1.2.0, so `use geo.shapes` in the program and + /// `use shapes` in `geo` both load the module `geo.1.shapes`. A module's name never starts + /// with a digit, so no prefix is ever a module of the program. Two packages with the same + /// name and class, which can only be different directories, are told apart by a number, as + /// `geo.1.2`. + fn load_packages(&mut self) { + let root = self.packages[0].root.clone(); + let path = root.join(MANIFEST); + let Some(source) = self.sources.read(&path) else { + return; + }; + let file = self.add_manifest(path, source.clone(), None); + let (manifest, errors) = Manifest::parse(&source, file); + self.diagnostics.extend(errors); + if let Some((name, _)) = &manifest.name { + self.packages[0].name = name.clone(); + } + self.packages[0].version = manifest.version.as_ref().map(|(v, _)| v.clone()); + let root_requirements = self.requirements(&root, &manifest, false); + let selection = resolve::select(&root_requirements, |target| self.read_target(target)); + + // every package used, in the order first reached + let mut index: HashMap = HashMap::new(); + for target in selection.used() { + let Some(node) = self.nodes.get(target) else { + continue; + }; + let class = format!("{}.{}", node.name, node.version.class()); + let mut prefix = class.clone(); + let mut n = 1; + while self.packages.iter().any(|p| p.prefix == prefix) { + n += 1; + prefix = format!("{class}.{n}"); + } + let (source, version) = match target { + resolve::Target::Path(dir) => (Source::Path(dir.clone()), node.written.clone()), + resolve::Target::Remote(source, version) => (source.clone(), Some(version.clone())), + }; + self.packages.push(Package { + root: node.dir.clone(), + prefix, + name: node.name.clone(), + deps: HashMap::new(), + unavailable: HashMap::new(), + source: Some(source), + version, + }); + index.insert(target.clone(), self.packages.len() - 1); + } + let failed = std::mem::take(&mut self.failed); + let bind = |package: &mut Package, requirements: &[Requirement]| { + for r in requirements { + let target = selection.resolve(&r.target); + match index.get(&target) { + Some(&i) => { + package.deps.insert(r.name.clone(), i); + } + None => { + let reason = failed.get(&target).or_else(|| failed.get(&r.target)); + package.unavailable.insert(r.name.clone(), reason.cloned()); + } + } + } + }; + bind(&mut self.packages[0], &root_requirements); + for (target, &i) in &index { + if let Some(requirements) = selection.requirements.get(target) { + bind(&mut self.packages[i], requirements); + } + } + self.check_lock(&root, &selection); + } + + /// Returns the dependencies of the package in `dir` with `manifest` as requirements, which + /// for a package from a host (`remote`) cannot be directories. + fn requirements(&mut self, dir: &Path, manifest: &Manifest, remote: bool) -> Vec { + let mut requirements = vec![]; + for dependency in &manifest.dependencies { + let names = [dependency.name.clone()]; + let local = self.sources.read(&dir.join(module_path(&names))).is_some() + || self.sources.is_dir(&dir.join(&dependency.name)); + if local { + let whose = match dir == self.packages[0].root { + true => "program", + false => "package", + }; + self.diagnostics.push(Diagnostic::error( + dependency.span, + format!( + "'{}' is both a dependency and a module of this {whose}, so rename one", + dependency.name + ), + )); + continue; + } + let (target, written) = match (&dependency.source, &dependency.version) { + (Source::Path(path), _) if remote => { + self.diagnostics.push(Diagnostic::error( + dependency.span, + "a package from a host cannot use a directory, since the directory is \ + not part of its release", + )); + continue; + } + (Source::Path(path), _) => ( + resolve::Target::Path(join_normal(dir, path)), + path.display().to_string(), + ), + (source, Some((version, _))) => ( + resolve::Target::Remote(source.clone(), version.clone()), + dependency.name.clone(), + ), + // already reported + (_, None) => continue, + }; + self.named + .entry(target.clone()) + .or_insert_with(|| (dependency.span, dependency.name.clone(), written)); + requirements.push(Requirement { + name: dependency.name.clone(), + span: dependency.span, + target, + }); + } + requirements + } + + /// Reads the manifest of the package `target` names, from its directory or the store, + /// returning its requirements, or `None` with the error if it cannot. + fn read_target(&mut self, target: &resolve::Target) -> Option> { + let (span, key, written) = self.named.get(target)?.clone(); + let dir = match target { + resolve::Target::Path(dir) => dir.clone(), + resolve::Target::Remote(source, version) => match self.sources.store() { + Some(store) => package_dir(&store, source, version), + None => { + let message = format!( + "{key} {version} ({source}) cannot be downloaded, since there is no \ + directory to keep packages in; set STONE_HOME" + ); + self.failed.insert(target.clone(), message.clone()); + self.diagnostics.push(Diagnostic::error(span, message)); + return None; + } + }, + }; + let path = dir.join(MANIFEST); + let Some(source) = self.sources.read(&path) else { + let (message, hint) = match target { + resolve::Target::Path(_) => ( + format!("'{written}' has no {MANIFEST}, so it is not a package"), + "", + ), + resolve::Target::Remote(source, version) => { + self.missing.push((source.clone(), version.clone())); + ( + format!("{key} {version} ({source}) is not downloaded"), + "; run `stone fetch`", + ) + } + }; + self.failed + .insert(target.clone(), format!("{message}{hint}")); + self.diagnostics.push(Diagnostic::error(span, message)); return None; + }; + let file = self.add_manifest(path, source.clone(), None); + let (manifest, errors) = Manifest::parse(&source, file); + let clean = errors.is_empty(); + self.diagnostics.extend(errors); + let Some((name, _)) = manifest.name.clone() else { + // a mistake in the manifest may be why it has no name + if clean { + self.diagnostics.push(Diagnostic::error( + Span::empty(Pos::new(1, 1)).in_file(file), + format!( + "a package that others use has a name, as name = \"{key}\" under \ + [package]" + ), + )); + } + return None; + }; + self.files[file.0 as usize].package = Some(name.clone()); + let written_version = manifest.version.as_ref().map(|(v, _)| v.clone()); + let written = written_version.clone(); + let version = match target { + resolve::Target::Remote(_, released) => { + if let Some((said, span)) = &manifest.version + && said != released + { + self.diagnostics.push(Diagnostic::error( + *span, + format!( + "this package says it is version {said}, but it was released as \ + {released}" + ), + )); + } + released.clone() + } + resolve::Target::Path(_) => written_version.unwrap_or(Version::new(0, 0, 0)), + }; + let remote = matches!(target, resolve::Target::Remote(..)); + let requirements = self.requirements(&dir, &manifest, remote); + self.nodes.insert( + target.clone(), + Node { + dir, + name, + version, + written, + }, + ); + Some(requirements) + } + + /// Checks every package from a host that `selection` uses against the program's + /// `stone.lock`, noting the lockfile to write if it lacks one or holds one no longer used. + fn check_lock(&mut self, root: &Path, selection: &resolve::Selection) { + let path = root.join(LOCK); + let existing = self.sources.read(&path); + let used: Vec<(Source, Version, Node)> = selection + .used() + .filter_map(|target| match target { + resolve::Target::Remote(source, version) => self + .nodes + .get(target) + .map(|node| (source.clone(), version.clone(), node.clone())), + resolve::Target::Path(_) => None, + }) + .collect(); + if used.is_empty() && existing.is_none() { + return; } - let name = names.join("."); - if let Some(&id) = self.by_name.get(&name) { - return Some(id); + let lock = match &existing { + Some(text) => { + let file = self.add_manifest(path.clone(), text.clone(), None); + let (lock, errors) = Lock::parse(text, file); + if !errors.is_empty() { + self.diagnostics.extend(errors); + return; + } + lock + } + None => Lock::default(), + }; + let mut entries = vec![]; + let mut changed = false; + for (source, version, node) in used { + let hash = content_hash(self.sources, &node.dir); + if let Some(locked) = lock.find(&source, &version) + && locked.hash != hash + { + self.diagnostics.push(Diagnostic::error( + locked.span, + format!( + "{} {version} ({source}) does not hold what it held when it was locked: \ + its hash was {}, and is now {hash}; if its release changed on \ + purpose, remove its entry from {LOCK}", + node.name, locked.hash + ), + )); + changed = true; + continue; + } + entries.push(Locked { + name: node.name, + source, + version, + hash, + span: Span::default(), + }); } + if changed { + return; + } + let text = Lock { entries }.write(); + if existing.as_deref() != Some(text.as_str()) { + self.lock = Some((path, text)); + } + } + + /// Returns the linked name of the module of `package` at the path `names` from its root. + /// + /// For example, in a package with prefix `geo.1`, `["shapes"]` is `geo.1.shapes` and + /// `["lib"]` is `geo.1`, while in the program `["shapes"]` is `shapes`. + fn linked_module(&self, package: usize, names: &[String]) -> String { + let prefix = &self.packages[package].prefix; + if package == 0 { + names.join(".") + } else if names == [LIB] { + prefix.clone() + } else { + format!("{prefix}.{}", names.join(".")) + } + } + + /// Returns the file of the module of `package` with these dotted names, loading it if it + /// exists. With `builtin`, a module written in stone, such as `net`, takes its name. + /// + /// For example, `["geometry", "shapes"]` loads `/geometry/shapes.stn`. + fn module(&mut self, package: usize, names: &[String], builtin: bool) -> Option { // a module written in stone has no file of the program, and no modules inside it - if let Some(source) = crate::stdlib::source_module(&names[0]) { + if builtin && let Some(source) = crate::stdlib::source_module(&names[0]) { if names.len() != 1 { return None; } + let name = names[0].clone(); + if let Some(&id) = self.by_name.get(&name) { + return Some(id); + } let id = self.add( PathBuf::from(format!("<{name}>")), name.clone(), source.to_string(), + 0, ); self.own.insert(id); self.by_name.insert(name, id); return Some(id); } - let path = self.root.join(module_path(names)); + if package == 0 && self.is_entry(names) { + return None; + } + let name = self.linked_module(package, names); + if let Some(&id) = self.by_name.get(&name) { + return Some(id); + } + let path = self.packages[package].root.join(module_path(names)); let source = self.sources.read(&path)?; - let id = self.add(path, name.clone(), source); + let id = self.add(path, name.clone(), source, package); self.by_name.insert(name, id); Some(id) } @@ -431,9 +963,26 @@ impl Loader<'_> { let Some(last) = path.last().cloned() else { continue; }; - let target = match self.resolve(file, &path) { - Ok(target) => Some(target), - Err(e) => { + let unavailable = self.packages[self.file_package[file.0 as usize]] + .unavailable + .get(&path[0].0) + .cloned(); + let target = match (self.resolve(file, &path), unavailable) { + (Ok(target), _) => Some(target), + // the manifest's error says why, and editors show it at the `use` too + (Err(_), Some(reason)) => { + let name = &path[0].0; + let message = match reason { + Some(reason) => format!("'{name}' cannot be used: {reason}"), + None => format!( + "'{name}' cannot be used, because of an error in its {MANIFEST}" + ), + }; + let span = path[0].1.to(path[path.len() - 1].1); + self.unavailable.push(Diagnostic::error(span, message)); + None + } + (Err(e), None) => { self.diagnostics.push(e); None } @@ -449,13 +998,18 @@ impl Loader<'_> { } /// Finds what a `use` path in `file` names: a module file if there is one, otherwise a public - /// function of the module the path ends in. + /// function of the module the path ends in. A path that starts with the name of a dependency + /// of the file's package names a module of that package. /// /// For example, `use util.pad` is the module `util/pad.stn` if it exists, and otherwise the /// function `pad` of `util.stn`. fn resolve(&mut self, file: FileId, path: &[(String, Span)]) -> Result { let names: Vec = path.iter().map(|(name, _)| name.clone()).collect(); let span = path[0].1.to(path[path.len() - 1].1); + let package = self.file_package[file.0 as usize]; + if let Some(&dependency) = self.packages[package].deps.get(&names[0]) { + return self.resolve_in(file, dependency, path, 1); + } // the prelude's functions are linked as if in this module if names == [PRELUDE_MODULE] { return Err(Diagnostic::error( @@ -473,11 +1027,31 @@ impl Loader<'_> { )); } if let Some(module) = crate::stdlib::module(&names[0]) { - return self.resolve_builtin_module(module, path, span); + return self.resolve_builtin_module(package, module, path, span); } if crate::stdlib::source_module(&names[0]).is_some() { - self.check_builtin_name(&names[0], span)?; + self.check_builtin_name(package, &names[0], span)?; } + self.resolve_in(file, package, path, 0) + } + + /// Finds what a `use` path in `file` names in `package`, whose module paths are the path's + /// names after the first `skip`: the first name is a dependency's when `skip` is 1, and then + /// no more names is the package's `lib.stn`. + fn resolve_in( + &mut self, + file: FileId, + package: usize, + path: &[(String, Span)], + skip: usize, + ) -> Result { + let names: Vec = path.iter().map(|(name, _)| name.clone()).collect(); + let span = path[0].1.to(path[path.len() - 1].1); + let builtin = skip == 0; + let lib_or = |names: &[String]| match names { + [] => vec![LIB.to_string()], + names => names.to_vec(), + }; let entry = self.entry_names.join("."); let entry_error = || { Diagnostic::error( @@ -486,24 +1060,36 @@ impl Loader<'_> { ) }; - if self.is_entry(&names) { + let within = &names[skip..]; + if package == 0 && self.is_entry(within) { return Err(entry_error()); } - if let Some(id) = self.module(&names) { + if let Some(id) = self.module(package, &lib_or(within), builtin) { return Ok(Target::Module(id)); } + if within.is_empty() { + return Err(Diagnostic::error( + span, + format!( + "package '{}' has no {LIB}.stn, so use one of its modules", + names[0] + ), + )); + } if let [parent @ .., (function, function_span)] = path && !parent.is_empty() { - let parent_names = &names[..parent.len()]; - if self.is_entry(parent_names) { + let parent_within = &names[skip..parent.len()]; + if package == 0 && self.is_entry(parent_within) { return Err(entry_error()); } - if let Some(id) = self.module(parent_names) { - let module = parent_names.join("."); + if let Some(id) = self.module(package, &lib_or(parent_within), builtin) { + let linked_module = self.files[id.0 as usize].module.clone(); + // errors name the module as the file wrote it + let module = names[..parent.len()].join("."); let parsed = &self.modules[id.0 as usize]; if let Some(&public) = public_types(parsed).get(function) { - let linked = format!("{module}.{function}"); + let linked = format!("{linked_module}.{function}"); return if !(public || id == file) { Err(Diagnostic::error( *function_span, @@ -519,7 +1105,7 @@ impl Loader<'_> { } return match public_functions(parsed).get(function) { Some(&public) if public || id == file => { - Ok(Target::Function(format!("{module}.{function}"))) + Ok(Target::Function(format!("{linked_module}.{function}"))) } Some(_) => Err(Diagnostic::error( *function_span, @@ -534,7 +1120,9 @@ impl Loader<'_> { } let dotted = names.join("."); - let dir = self.root.join(names.iter().collect::()); + let dir = self.packages[package] + .root + .join(within.iter().collect::()); if self.sources.is_dir(&dir) { return Err(Diagnostic::error( span, @@ -554,12 +1142,13 @@ impl Loader<'_> { /// `os.stn` next to the entry file would be ambiguous, so it is an error to import it. fn resolve_builtin_module( &self, + package: usize, module: &'static BuiltinModule, path: &[(String, Span)], span: Span, ) -> Result { let name = module.name; - self.check_builtin_name(name, span)?; + self.check_builtin_name(package, name, span)?; match path { [_] => Ok(Target::BuiltinModule(name.to_string())), [_, (function, function_span)] => { @@ -583,14 +1172,14 @@ impl Loader<'_> { } } - /// Fails if a file next to the entry file takes the name of the builtin module `name`, which - /// would make `use` ambiguous. - fn check_builtin_name(&self, name: &str, span: Span) -> Result<(), Diagnostic> { + /// Fails if a file at the root of `package` takes the name of the builtin module `name`, + /// which would make `use` ambiguous. + fn check_builtin_name(&self, package: usize, name: &str, span: Span) -> Result<(), Diagnostic> { let names = [name.to_string()]; - let taken = names != *self.entry_names + let taken = (package != 0 || names != *self.entry_names) && self .sources - .read(&self.root.join(module_path(&names))) + .read(&self.packages[package].root.join(module_path(&names))) .is_some(); match taken { true => Err(Diagnostic::error( @@ -887,6 +1476,26 @@ impl Loader<'_> { } report_method_collisions(&library_body, &entry_body, &mut self.diagnostics); + let packages = self + .packages + .iter() + .map(|package| { + let mut deps: Vec<(String, usize)> = package + .deps + .iter() + .map(|(name, &i)| (name.clone(), i)) + .collect(); + deps.sort(); + PackageInfo { + name: package.name.clone(), + version: package.version.clone(), + source: package.source.clone(), + deps, + root: package.root.clone(), + prefix: package.prefix.clone(), + } + }) + .collect(); let mut body = prelude(&mut self.diagnostics); let public = functions .iter() @@ -910,6 +1519,11 @@ impl Loader<'_> { diagnostics: self.diagnostics, imports, public, + missing: self.missing, + unavailable: self.unavailable, + file_packages: self.file_package, + lock: self.lock, + packages, } } } diff --git a/src/project/tests.rs b/src/project/tests.rs index e0582ac..5963b38 100644 --- a/src/project/tests.rs +++ b/src/project/tests.rs @@ -1821,3 +1821,618 @@ fn a_library_cannot_name_a_trait_of_the_entry_file_as_a_type() { 13, ); } + +const GEO_MANIFEST: (&str, &str) = ( + "../geo/stone.toml", + "[package]\nname = \"geo\"\nversion = \"1.2.0\"\n", +); + +const GEO_SHAPES: (&str, &str) = ( + "../geo/shapes.stn", + "pub fn area(x: int) -> int:\n ret x * x\n\nfn hidden() -> int:\n ret 0\n", +); + +const USES_GEO: (&str, &str) = ( + "stone.toml", + "[dependencies]\ngeo = { path = \"../geo\" }\n", +); + +#[test] +fn a_path_dependency_is_used_by_its_name() { + assert_eq!( + output(&[ + ( + "main.stn", + "use geo.shapes\nuse geo.shapes.area\nprint(shapes.area(2) + area(3))\n" + ), + USES_GEO, + GEO_MANIFEST, + GEO_SHAPES, + ]), + "13\n" + ); +} + +#[test] +fn a_packages_own_name_loads_its_lib() { + let lib = ("../geo/lib.stn", "pub fn one() -> int:\n ret 1\n"); + assert_eq!( + output(&[ + ( + "main.stn", + "use geo\nuse geo.one\nprint(geo.one() + one())\n" + ), + USES_GEO, + GEO_MANIFEST, + lib, + ]), + "2\n" + ); + assert_error( + &[ + ("main.stn", "use geo\n"), + USES_GEO, + GEO_MANIFEST, + GEO_SHAPES, + ], + "main.stn", + "package 'geo' has no lib.stn, so use one of its modules", + 1, + 5, + ); +} + +#[test] +fn a_packages_files_use_its_own_modules_and_dependencies() { + // the program and the package each have a module `util`, and they do not meet + assert_eq!( + output(&[ + ( + "main.stn", + "use util\nuse geo.shapes\nprint(util.n() + shapes.n())\n" + ), + ("util.stn", "pub fn n() -> int:\n ret 100\n"), + USES_GEO, + ( + "../geo/stone.toml", + "[package]\nname = \"geo\"\n\n[dependencies]\nnum = { path = \"../num\" }\n", + ), + ( + "../geo/shapes.stn", + "use util\nuse num\npub fn n() -> int:\n ret util.n() + num.n()\n", + ), + ("../geo/util.stn", "pub fn n() -> int:\n ret 20\n"), + ("../num/stone.toml", "[package]\nname = \"num\"\n"), + ("../num/lib.stn", "pub fn n() -> int:\n ret 3\n"), + ]), + "123\n" + ); +} + +#[test] +fn a_package_cannot_see_the_programs_modules_or_its_dependencies() { + assert_error( + &[ + ("main.stn", "use geo.shapes\nprint(shapes.area(1))\n"), + ("helper.stn", "pub fn n() -> int:\n ret 1\n"), + USES_GEO, + GEO_MANIFEST, + ( + "../geo/shapes.stn", + "use helper\npub fn area(x: int) -> int:\n ret x\n", + ), + ], + "../geo/shapes.stn", + "no module named 'helper'", + 1, + 5, + ); + // a dependency's own dependencies are its alone + assert_error( + &[ + ("main.stn", "use num\n"), + USES_GEO, + ( + "../geo/stone.toml", + "[package]\nname = \"geo\"\n\n[dependencies]\nnum = { path = \"../num\" }\n", + ), + ("../num/stone.toml", "[package]\nname = \"num\"\n"), + ("../num/lib.stn", "pub fn n() -> int:\n ret 3\n"), + ], + "main.stn", + "no module named 'num'", + 1, + 5, + ); +} + +#[test] +fn a_packages_private_functions_stay_private() { + assert_error( + &[ + ("main.stn", "use geo.shapes.hidden\n"), + USES_GEO, + GEO_MANIFEST, + GEO_SHAPES, + ], + "main.stn", + "'hidden' is private to module 'geo.shapes'", + 1, + 16, + ); +} + +#[test] +fn a_missing_package_module_is_named_as_written() { + assert_error( + &[ + ("main.stn", "use geo.lines\n"), + USES_GEO, + GEO_MANIFEST, + GEO_SHAPES, + ], + "main.stn", + "no module named 'geo.lines'", + 1, + 5, + ); +} + +#[test] +fn a_dependency_cannot_share_a_name_with_a_module_of_the_program() { + for local in ["geo.stn", "geo/lines.stn"] { + assert_error( + &[ + ("main.stn", "print(1)\n"), + (local, "pub fn n() -> int:\n ret 1\n"), + USES_GEO, + GEO_MANIFEST, + ], + "stone.toml", + "'geo' is both a dependency and a module of this program, so rename one", + 2, + 1, + ); + } +} + +#[test] +fn a_path_dependency_needs_a_manifest() { + assert_error( + &[("main.stn", "print(1)\n"), USES_GEO, GEO_SHAPES], + "stone.toml", + "'../geo' has no stone.toml, so it is not a package", + 2, + 1, + ); +} + +#[test] +fn manifest_errors_point_into_the_manifest() { + assert_error( + &[ + ("main.stn", "print(1)\n"), + ("stone.toml", "[dependencies]\ngeo = { path = 1 }\n"), + ], + "stone.toml", + "'path' is a string, not an integer", + 2, + 16, + ); + assert_error( + &[ + ("main.stn", "print(1)\n"), + USES_GEO, + ("../geo/stone.toml", "[package]\nnam = \"geo\"\n"), + ], + "../geo/stone.toml", + "unknown key 'nam' in [package], which holds name, version, stone, description, and \ + license", + 2, + 1, + ); +} + +#[test] +fn a_dependency_needs_a_name() { + assert_error( + &[ + ("main.stn", "print(1)\n"), + USES_GEO, + ("../geo/stone.toml", "[package]\nversion = \"1.0\"\n"), + ], + "../geo/stone.toml", + "a package that others use has a name, as name = \"geo\" under [package]", + 1, + 1, + ); +} + +#[test] +fn packages_are_linked_under_their_names_and_compatibility_classes() { + let a = |version: &str, n: &str| { + [ + ( + format!("../a{n}/stone.toml"), + format!("[package]\nname = \"a\"\nversion = \"{version}\"\n"), + ), + ( + format!("../a{n}/lib.stn"), + format!("pub fn make() -> int:\n ret {n}\n"), + ), + ] + }; + let mut files: Vec<(String, String)> = vec![ + ( + "main.stn".into(), + "use one\nuse two\nuse three\nprint(one.make() + two.make() + three.make())\n".into(), + ), + ( + "stone.toml".into(), + "[dependencies]\none = { path = \"../a1\" }\ntwo = { path = \"../a2\" }\n\ + three = { path = \"../a3\" }\n" + .into(), + ), + ]; + files.extend(a("1.4.0", "1")); + files.extend(a("2.0.1", "2")); + files.extend(a("1.0.0", "3")); + let files: Vec<(&str, &str)> = files + .iter() + .map(|(path, source)| (path.as_str(), source.as_str())) + .collect(); + assert_eq!(output(&files), "6\n"); + assert_eq!( + defined_functions(&files), + ["a.1.make", "a.2.make", "a.1.2.make"] + ); +} + +#[test] +fn one_directory_is_one_package_however_it_is_reached() { + let files = [ + ( + "main.stn", + "use geo\nuse other\nprint(geo.one() + other.one())\n", + ), + ( + "stone.toml", + "[dependencies]\ngeo = { path = \"../geo\" }\nother = { path = \"../x/../geo\" }\n", + ), + GEO_MANIFEST, + ( + "../geo/lib.stn", + "use lib\npub fn one() -> int:\n ret 1\n", + ), + ]; + assert_eq!(output(&files), "2\n"); + assert_eq!(defined_functions(&files), ["geo.1.one"]); +} + +#[test] +fn a_dependencys_files_are_never_reported_unused() { + let (map, diagnostics) = driver::check_program( + Path::new("main.stn"), + "use geo.shapes\nprint(shapes.area(2))\n", + &sources_of(&[USES_GEO, GEO_MANIFEST, GEO_SHAPES]), + ); + let rendered: Vec = diagnostics.iter().map(|d| map.render(d)).collect(); + assert_eq!(rendered, Vec::::new()); +} + +/// Builds in-memory sources from `(path, source)` pairs, with `store` as the package store. +fn stored_sources(files: &[(&str, &str)]) -> MapSources { + let mut sources = sources_of(files); + sources.set_store("store"); + sources +} + +/// The files of `version` of the package `json` from `github:alice/json` in the store, whose +/// `lib.stn` has `pub fn v() -> str` giving the version, plus `extra` dependencies in its +/// manifest. +fn stored_json(version: &str, extra: &str) -> Vec<(String, String)> { + let dir = format!("store/github.com/alice/json/{version}"); + vec![ + ( + format!("{dir}/stone.toml"), + format!("[package]\nname = \"json\"\nversion = \"{version}\"\n{extra}"), + ), + ( + format!("{dir}/lib.stn"), + format!("pub fn v() -> str:\n ret \"{version}\"\n"), + ), + ] +} + +/// Loads a program whose entry file is the first of `files` from sources with a store, returning +/// what it prints (or its rendered errors) and every file linking wrote. +fn stored_run(files: &[(String, String)]) -> (String, Vec<(PathBuf, String)>) { + let pairs: Vec<(&str, &str)> = files + .iter() + .map(|(p, s)| (p.as_str(), s.as_str())) + .collect(); + let sources = stored_sources(&pairs[1..]); + let (map, result) = driver::load(Path::new(pairs[0].0), pairs[0].1, &sources); + let printed = match result { + Ok(checked) => { + let mut out = vec![]; + driver::run_module( + &checked.module, + &checked.calls, + &mut std::io::empty(), + &mut out, + &[], + Limits::DEFAULT, + ) + .expect("program should run"); + String::from_utf8(out).unwrap() + } + // the expected paths are written with windows' separator as `/` + Err(errors) => errors + .0 + .iter() + .map(|d| map.render(d).replace('\\', "/")) + .collect(), + }; + (printed, sources.written()) +} + +fn owned(files: &[(&str, &str)]) -> Vec<(String, String)> { + files + .iter() + .map(|(p, s)| (p.to_string(), s.to_string())) + .collect() +} + +const USES_JSON: (&str, &str) = ( + "stone.toml", + "[dependencies]\njson = { github = \"alice/json\", version = \"1.2\" }\n", +); + +#[test] +fn a_package_from_a_host_is_read_from_the_store_and_locked() { + let mut files = owned(&[("main.stn", "use json\nprint(json.v())\n"), USES_JSON]); + files.extend(stored_json("1.2.0", "")); + let (printed, written) = stored_run(&files); + assert_eq!(printed, "1.2.0\n"); + let [(path, lock)] = written.as_slice() else { + panic!("one file written: {written:?}"); + }; + assert_eq!(path, Path::new("stone.lock")); + let (lock, errors) = crate::package::lock::Lock::parse(lock, FileId(0)); + assert_eq!(errors, []); + let entry = &lock.entries[0]; + assert_eq!( + ( + entry.name.as_str(), + entry.source.to_string(), + entry.version.to_string() + ), + ("json", "github:alice/json".to_string(), "1.2.0".to_string()) + ); + + // with the lock written, linking again writes nothing + let text = written[0].1.clone(); + files.push(("stone.lock".into(), text)); + assert_eq!(stored_run(&files), ("1.2.0\n".to_string(), vec![])); +} + +#[test] +fn the_newest_version_any_package_needs_is_used() { + let mut files = owned(&[ + ( + "main.stn", + "use json\nuse geo\nprint(json.v())\nprint(geo.v())\n", + ), + ( + "stone.toml", + "[dependencies]\njson = { github = \"alice/json\", version = \"1.2\" }\n\ + geo = { path = \"../geo\" }\n", + ), + ( + "../geo/stone.toml", + "[package]\nname = \"geo\"\n\n[dependencies]\n\ + j = { github = \"alice/json\", version = \"1.3.0\" }\n", + ), + ( + "../geo/lib.stn", + "use j\npub fn v() -> str:\n ret j.v()\n", + ), + ]); + files.extend(stored_json("1.2.0", "")); + files.extend(stored_json("1.3.0", "")); + files.extend(stored_json("1.9.0", "")); + assert_eq!(stored_run(&files).0, "1.3.0\n1.3.0\n"); +} + +#[test] +fn a_package_that_is_not_downloaded_is_an_error_where_it_is_named() { + let files = owned(&[("main.stn", "use json\nprint(json.v())\n"), USES_JSON]); + let (printed, written) = stored_run(&files); + assert!( + printed + .starts_with("stone.toml:2:1: error: json 1.2.0 (github:alice/json) is not downloaded"), + "{printed}" + ); + assert_eq!(written, []); +} + +/// Returns the first line of each of `diagnostics` rendered from `linked`'s files. +fn first_lines(linked: &Linked, diagnostics: &[Diagnostic]) -> Vec { + diagnostics + .iter() + .map(|d| linked.sources.render(d).lines().next().unwrap().to_string()) + .collect() +} + +#[test] +fn a_use_of_a_package_that_is_not_downloaded_is_noted_for_editors() { + let sources = stored_sources(&[USES_JSON]); + let linked = link( + Path::new("main.stn"), + "use json\nprint(json.v())\n", + &sources, + ); + // the program's errors say it once, where the manifest names the package + assert_eq!( + first_lines(&linked, &linked.diagnostics), + ["stone.toml:2:1: error: json 1.2.0 (github:alice/json) is not downloaded"] + ); + assert_eq!( + first_lines(&linked, &linked.unavailable), + [ + "main.stn:1:5: error: 'json' cannot be used: json 1.2.0 (github:alice/json) is not \ + downloaded; run `stone fetch`" + ] + ); +} + +#[test] +fn a_use_of_a_directory_that_is_not_a_package_is_noted_for_editors() { + let sources = sources_of(&[USES_GEO]); + let linked = link(Path::new("main.stn"), "use geo.shapes\n", &sources); + assert_eq!( + first_lines(&linked, &linked.unavailable), + [ + "main.stn:1:5: error: 'geo' cannot be used: '../geo' has no stone.toml, so it is not \ + a package" + ] + ); +} + +#[test] +fn a_use_of_a_package_whose_manifest_has_errors_points_at_it() { + let sources = sources_of(&[USES_GEO, ("../geo/stone.toml", "[package]\nname = 1\n")]); + let linked = link(Path::new("main.stn"), "use geo\n", &sources); + assert_eq!( + first_lines(&linked, &linked.unavailable), + ["main.stn:1:5: error: 'geo' cannot be used, because of an error in its stone.toml"] + ); +} + +#[test] +fn linking_tells_which_package_each_file_belongs_to() { + let sources = sources_of(&[USES_GEO, GEO_MANIFEST, GEO_SHAPES]); + let linked = link( + Path::new("main.stn"), + "use geo.shapes\nprint(shapes.area(2))\n", + &sources, + ); + assert_eq!(linked.diagnostics, []); + let geo = &linked.packages[1]; + assert_eq!( + (geo.name.as_str(), geo.prefix.as_str(), geo.root.as_path()), + ("geo", "geo.1", Path::new("../geo")) + ); + assert_eq!(linked.packages[0].prefix, ""); + let shapes = linked.sources.find(Path::new("../geo/shapes.stn")).unwrap(); + assert_eq!(linked.file_packages[shapes.0 as usize], 1); + assert_eq!(linked.file_packages[0], 0); + assert_eq!(linked.file_packages.len(), linked.sources.files().count()); +} + +#[test] +fn shown_names_leave_out_compatibility_classes() { + let package = |name: &str, prefix: &str| PackageInfo { + name: name.to_string(), + version: None, + source: None, + deps: vec![], + root: PathBuf::new(), + prefix: prefix.to_string(), + }; + let one = [package("app", ""), package("geo", "geo.1")]; + assert_eq!( + shown_names("fn geo.1.shapes.area(x: geo.1.Point) -> geo.1", &one), + "fn geo.shapes.area(x: geo.Point) -> geo" + ); + // only where a name starts, so other names are left alone + assert_eq!( + shown_names("ageo.1.x xgeo.1 geo.12", &one), + "ageo.1.x xgeo.1 geo.12" + ); + // two packages of one name keep what tells them apart + let two = [ + package("app", ""), + package("geo", "geo.1"), + package("geo", "geo.2"), + package("geo", "geo.1.2"), + ]; + assert_eq!( + shown_names("geo.1.a geo.2.b geo.1.2.c", &two), + "geo.1.a geo.2.b geo.1.2.c" + ); +} + +#[test] +fn a_package_whose_files_changed_does_not_match_its_lock() { + let mut files = owned(&[("main.stn", "use json\nprint(json.v())\n"), USES_JSON]); + files.extend(stored_json("1.2.0", "")); + let wrong = format!("sha256:{}", "0".repeat(64)); + files.push(( + "stone.lock".into(), + format!( + "[[package]]\nname = \"json\"\ngithub = \"alice/json\"\nversion = \"1.2.0\"\n\ + hash = \"{wrong}\"\n" + ), + )); + let (printed, written) = stored_run(&files); + assert!( + printed.starts_with(&format!( + "stone.lock:5:8: error: json 1.2.0 (github:alice/json) does not hold what it held \ + when it was locked: its hash was {wrong}, and is now sha256:" + )), + "{printed}" + ); + assert_eq!(written, []); +} + +#[test] +fn a_package_from_a_host_must_be_the_version_it_was_released_as() { + let mut files = owned(&[("main.stn", "use json\nprint(json.v())\n"), USES_JSON]); + files.push(( + "store/github.com/alice/json/1.2.0/stone.toml".into(), + "[package]\nname = \"json\"\nversion = \"1.1.0\"\n".into(), + )); + files.push(( + "store/github.com/alice/json/1.2.0/lib.stn".into(), + "pub fn v() -> str:\n ret \"1.1.0\"\n".into(), + )); + let (printed, _) = stored_run(&files); + assert!( + printed.starts_with( + "store/github.com/alice/json/1.2.0/stone.toml:3:11: error: this package says it is \ + version 1.1.0, but it was released as 1.2.0" + ), + "{printed}" + ); +} + +#[test] +fn a_package_from_a_host_cannot_use_a_directory() { + let mut files = owned(&[("main.stn", "use json\nprint(json.v())\n"), USES_JSON]); + files.extend(stored_json( + "1.2.0", + "\n[dependencies]\nnum = { path = \"../num\" }\n", + )); + let (printed, _) = stored_run(&files); + assert!( + printed.starts_with( + "store/github.com/alice/json/1.2.0/stone.toml:6:1: error: a package from a host \ + cannot use a directory, since the directory is not part of its release" + ), + "{printed}" + ); +} + +#[test] +fn packages_from_hosts_need_a_store() { + assert_error( + &[("main.stn", "print(1)\n"), USES_JSON], + "stone.toml", + "json 1.2.0 (github:alice/json) cannot be downloaded, since there is no directory to \ + keep packages in; set STONE_HOME", + 2, + 1, + ); +} diff --git a/src/stdlib.rs b/src/stdlib.rs index e0e214e..d63ef6a 100644 --- a/src/stdlib.rs +++ b/src/stdlib.rs @@ -1535,8 +1535,9 @@ mod tests { "{start:?}:{stop:?}:{step:?}" ); } - assert_eq!(sliced(0, None, None, Some(-1)), []); - assert_eq!(sliced(0, Some(3), Some(-3), None), []); + // typed, since a dependency's impls can make `[]` ambiguous + assert_eq!(sliced(0, None, None, Some(-1)), [0i64; 0]); + assert_eq!(sliced(0, Some(3), Some(-3), None), [0i64; 0]); } /// Slices `text` as `text[start:stop:step]` does. diff --git a/src/unused.rs b/src/unused.rs index 2ee38a0..44b5df8 100644 --- a/src/unused.rs +++ b/src/unused.rs @@ -59,8 +59,11 @@ pub fn find(linked: &Linked, analysis: &Analysis) -> Vec { } } - let skipped = - |span: Span| span.file == FileId::PRELUDE || linked.sources.file(span.file).is_builtin(); + // stone's own files, and the files of the packages a program depends on, are not its author's + let skipped = |span: Span| { + let file = linked.sources.file(span.file); + span.file == FileId::PRELUDE || file.is_builtin() || file.is_dependency() + }; let mut warnings = vec![]; for (symbol, reads) in analysis.symbols.iter().zip(&reads) { let short = symbol.name.rsplit('.').next().unwrap_or(&symbol.name); diff --git a/tests/cli.rs b/tests/cli.rs index f54db74..f2d6496 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1411,7 +1411,119 @@ fn file_errors_are_worded_the_same_in_both_backends() { #[path = "cli/unix.rs"] mod unix; +/// Tests of downloading packages from a fake GitHub, which need the `fetch` feature. +#[cfg(feature = "fetch")] +#[path = "cli/packages.rs"] +mod packages; + +#[cfg(not(feature = "fetch"))] +#[test] +fn add_without_feature_explains_how_to_get_a_build_that_downloads() { + let dir = project("package_no_fetch", &[("main.stn", "print(1)\n")]); + let output = Command::new(env!("CARGO_BIN_EXE_stone")) + .args(["add", "github:alice/json"]) + .current_dir(&dir) + .output() + .expect("stone should run"); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("cannot download packages"), "{stderr}"); + assert!(!dir.join("stone.toml").exists()); +} + +#[test] +fn tree_shows_the_packages_a_program_uses() { + let dir = project( + "package_tree", + &[ + ("app/main.stn", "print(1)\n"), + ( + "app/stone.toml", + "[package]\nname = \"app\"\nversion = \"0.2.0\"\n\n[dependencies]\n\ + geo = { path = \"../geo\" }\nshapes = { path = \"../shapes\" }\n", + ), + ( + "geo/stone.toml", + "[package]\nname = \"geo\"\nversion = \"1.0.0\"\n\n[dependencies]\n\ + nums = { path = \"../num\" }\n", + ), + ( + "shapes/stone.toml", + "[package]\nname = \"shapes\"\n\n[dependencies]\nnum = { path = \"../num\" }\n", + ), + ("num/stone.toml", "[package]\nname = \"num\"\n"), + ], + ); + let output = Command::new(env!("CARGO_BIN_EXE_stone")) + .args(["tree", "app"]) + .current_dir(&dir) + .output() + .expect("stone should run"); + assert_eq!(String::from_utf8_lossy(&output.stderr), ""); + assert_eq!( + String::from_utf8_lossy(&output.stdout).replace('\\', "/"), + "app 0.2.0\n\ + ├── geo 1.0.0 (path:geo)\n\ + │ └── nums = num (path:num)\n\ + └── shapes (path:shapes)\n \ + └── num (path:num) (*)\n" + ); +} + /// Tests of the `gui` module's Win32 windows, which run where Windows programs and PowerShell /// do: on Windows, and under WSL. #[path = "cli/windows_gui.rs"] mod windows_gui; + +#[test] +fn a_package_from_the_store_runs_and_is_locked() { + let dir = project( + "package_store", + &[ + ("app/main.stn", "use json\nprint(json.v())\n"), + ( + "app/stone.toml", + "[dependencies]\njson = { github = \"alice/json\", version = \"1.2\" }\n", + ), + ( + "home/packages/github.com/alice/json/1.2.0/stone.toml", + "[package]\nname = \"json\"\nversion = \"1.2.0\"\n", + ), + ( + "home/packages/github.com/alice/json/1.2.0/lib.stn", + "pub fn v() -> str:\n ret \"1.2.0\"\n", + ), + ], + ); + let run = |command: &str| { + Command::new(env!("CARGO_BIN_EXE_stone")) + .args([command, "app/main.stn"]) + .current_dir(&dir) + .env("STONE_HOME", dir.join("home")) + .output() + .expect("stone should run") + }; + let output = run("run"); + assert_eq!(String::from_utf8_lossy(&output.stderr), ""); + assert_eq!(String::from_utf8_lossy(&output.stdout), "1.2.0\n"); + let lock = std::fs::read_to_string(dir.join("app/stone.lock")).expect("lock written"); + assert!(lock.contains("name = \"json\"\ngithub = \"alice/json\"\nversion = \"1.2.0\"\n")); + + // a changed release no longer matches the lock + std::fs::write( + dir.join("home/packages/github.com/alice/json/1.2.0/lib.stn"), + "pub fn v() -> str:\n ret \"changed\"\n", + ) + .unwrap(); + let output = run("check"); + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("error: json 1.2.0 (github:alice/json) does not hold what it held"), + "{stderr}" + ); + assert_eq!( + std::fs::read_to_string(dir.join("app/stone.lock")).unwrap(), + lock + ); +} diff --git a/tests/cli/packages.rs b/tests/cli/packages.rs new file mode 100644 index 0000000..acc3a01 --- /dev/null +++ b/tests/cli/packages.rs @@ -0,0 +1,345 @@ +//! The tests of `cli.rs` for downloading packages, which need the `fetch` feature: `stone add`, +//! `remove`, `fetch`, and `tree`, and the downloads `run` makes, against a fake GitHub on +//! loopback that `STONE_GITHUB_API` points stone at. + +use super::*; +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; + +/// Returns the CRC-32 of `data`, as gzip needs it. +fn crc32(data: &[u8]) -> u32 { + let mut crc = !0u32; + for &byte in data { + crc ^= byte as u32; + for _ in 0..8 { + crc = if crc & 1 == 1 { + (crc >> 1) ^ 0xedb8_8320 + } else { + crc >> 1 + }; + } + } + !crc +} + +/// Wraps `data` in gzip without compressing it, in stored deflate blocks. +fn gzip(data: &[u8]) -> Vec { + let mut out = vec![0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 0xff]; + let chunks: Vec<&[u8]> = match data.is_empty() { + true => vec![&[]], + false => data.chunks(0xffff).collect(), + }; + for (i, chunk) in chunks.iter().enumerate() { + out.push((i + 1 == chunks.len()) as u8); + let len = chunk.len() as u16; + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&(!len).to_le_bytes()); + out.extend_from_slice(chunk); + } + out.extend_from_slice(&crc32(data).to_le_bytes()); + out.extend_from_slice(&(data.len() as u32).to_le_bytes()); + out +} + +/// Makes a `.tar.gz` of `files`, each under the directory `top`, as GitHub's archives are. +fn tarball(top: &str, files: &[(&str, &str)]) -> Vec { + let mut tar = vec![]; + for (path, text) in files { + let mut header = [0u8; 512]; + let name = format!("{top}/{path}"); + header[..name.len()].copy_from_slice(name.as_bytes()); + header[100..108].copy_from_slice(b"0000644\0"); + header[108..116].copy_from_slice(b"0000000\0"); + header[116..124].copy_from_slice(b"0000000\0"); + header[124..136].copy_from_slice(format!("{:011o}\0", text.len()).as_bytes()); + header[136..148].copy_from_slice(b"00000000000\0"); + header[148..156].copy_from_slice(b" "); + header[156] = b'0'; + header[257..263].copy_from_slice(b"ustar\0"); + header[263..265].copy_from_slice(b"00"); + let sum: u32 = header.iter().map(|&b| b as u32).sum(); + header[148..156].copy_from_slice(format!("{sum:06o}\0 ").as_bytes()); + tar.extend_from_slice(&header); + tar.extend_from_slice(text.as_bytes()); + tar.resize(tar.len().div_ceil(512) * 512, 0); + } + tar.resize(tar.len() + 1024, 0); + gzip(&tar) +} + +/// A fake GitHub API: each path (with its query) and the body it answers with, which tests may +/// change, and every path asked for. +#[derive(Clone, Default)] +struct FakeGitHub { + routes: Arc>>>, + asked: Arc>>, +} + +impl FakeGitHub { + /// Starts serving on a free loopback port, returning the API's URL. + fn serve(&self) -> String { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let fake = self.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + let mut request = vec![]; + let mut byte = [0]; + while !request.ends_with(b"\r\n\r\n") && stream.read(&mut byte).unwrap_or(0) == 1 { + request.push(byte[0]); + } + let request = String::from_utf8_lossy(&request); + let path = request.split(' ').nth(1).unwrap_or("").to_string(); + fake.asked.lock().unwrap().push(path.clone()); + let body = fake.routes.lock().unwrap().get(&path).cloned(); + let (status, body) = match body { + Some(body) => ("200 OK", body), + None => ("404 Not Found", b"{\"message\": \"Not Found\"}".to_vec()), + }; + let head = format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(head.as_bytes()); + let _ = stream.write_all(&body); + } + }); + url + } + + /// Publishes releases of `repo` with these tags, each holding `files`. + fn release(&self, repo: &str, releases: &[(&str, Vec<(&str, String)>)]) { + let mut routes = self.routes.lock().unwrap(); + let listing: Vec = releases + .iter() + .map(|(tag, _)| format!("{{\"tag_name\": \"{tag}\", \"draft\": false}}")) + .collect(); + routes.insert( + format!("/repos/{repo}/releases?per_page=100&page=1"), + format!("[{}]", listing.join(", ")).into_bytes(), + ); + for (tag, files) in releases { + let files: Vec<(&str, &str)> = files.iter().map(|(p, t)| (*p, t.as_str())).collect(); + let top = format!("{}-{tag}", repo.replace('/', "-")); + routes.insert( + format!("/repos/{repo}/tarball/{tag}"), + tarball(&top, &files), + ); + } + } + + /// Returns how many requests asked for `path`. + fn count(&self, path: &str) -> usize { + self.asked + .lock() + .unwrap() + .iter() + .filter(|p| *p == path) + .count() + } +} + +/// The files of a release of `json`, whose `v()` gives `version` and the `num` it uses. +fn json_release(version: &str) -> Vec<(&'static str, String)> { + vec![ + ( + "stone.toml", + format!( + "[package]\nname = \"json\"\nversion = \"{version}\"\n\n[dependencies]\n\ + num = {{ github = \"bob/num\", version = \"0.3\" }}\n" + ), + ), + ( + "lib.stn", + format!("use num\n\npub fn v() -> str:\n ret \"json {version}, num \" + num.v()\n"), + ), + ("README.md", "# json\n".to_string()), + ] +} + +fn num_release(version: &str) -> Vec<(&'static str, String)> { + vec![ + ( + "stone.toml", + format!("[package]\nname = \"num\"\nversion = \"{version}\"\n"), + ), + ( + "lib.stn", + format!("pub fn v() -> str:\n ret \"{version}\"\n"), + ), + ] +} + +/// Runs stone with `args` in `dir/app`, with the store in `dir/home` and the fake API at `api`. +fn stone_in(dir: &Path, api: &str, args: &[&str]) -> (bool, String, String) { + let output = Command::new(env!("CARGO_BIN_EXE_stone")) + .args(args) + .current_dir(dir.join("app")) + .env("STONE_HOME", dir.join("home")) + .env("STONE_GITHUB_API", api) + .env_remove("GITHUB_TOKEN") + .output() + .expect("stone should run"); + ( + output.status.success(), + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) +} + +#[test] +fn packages_are_added_downloaded_locked_and_removed() { + let github = FakeGitHub::default(); + github.release( + "alice/json", + &[ + ("v1.0.0", json_release("1.0.0")), + ("v1.2.0", json_release("1.2.0")), + ("v2.0.0-rc.1", json_release("2.0.0-rc.1")), + ], + ); + github.release( + "bob/num", + &[ + ("v0.3.0", num_release("0.3.0")), + ("v0.3.5", num_release("0.3.5")), + ], + ); + let api = github.serve(); + let dir = project( + "package_fetching", + &[("app/main.stn", "use json\nprint(json.v())\n")], + ); + + // the newest release that is not a pre-release, and the oldest num it names + let (ok, out, err) = stone_in(&dir, &api, &["add", "github:alice/json"]); + assert!(ok, "{err}"); + assert_eq!(out, "added json 1.2.0 (github:alice/json)\n"); + assert_eq!( + err, + "downloading github:alice/json 1.2.0\ndownloading github:bob/num 0.3.0\n" + ); + let manifest = std::fs::read_to_string(dir.join("app/stone.toml")).unwrap(); + assert_eq!( + manifest, + "[dependencies]\njson = { github = \"alice/json\", version = \"1.2.0\" }\n" + ); + let lock = std::fs::read_to_string(dir.join("app/stone.lock")).unwrap(); + assert!( + lock.contains("name = \"json\"") && lock.contains("name = \"num\""), + "{lock}" + ); + let stored = dir.join("home/packages/github.com/alice/json/1.2.0"); + assert!(stored.join("lib.stn").is_file()); + assert!(!stored.join("README.md").exists()); + + // nothing more is downloaded or asked for + let asked = github.asked.lock().unwrap().len(); + let (ok, out, err) = stone_in(&dir, &api, &["run", "main.stn"]); + assert!(ok, "{err}"); + assert_eq!( + (out.as_str(), err.as_str()), + ("json 1.2.0, num 0.3.0\n", "") + ); + assert_eq!(github.asked.lock().unwrap().len(), asked); + + let (ok, out, _) = stone_in(&dir, &api, &["tree"]); + assert!(ok); + assert_eq!( + out, + "app\n└── json 1.2.0 (github:alice/json)\n └── num 0.3.0 (github:bob/num)\n" + ); + let (ok, _, err) = stone_in(&dir, &api, &["fetch", "--locked"]); + assert!(ok, "{err}"); + + // without the store, --offline says what is missing, and a run downloads it again + std::fs::remove_dir_all(dir.join("home")).unwrap(); + let (ok, _, err) = stone_in(&dir, &api, &["run", "--offline", "main.stn"]); + assert!(!ok); + assert!( + err.contains("error: json 1.2.0 (github:alice/json) is not downloaded"), + "{err}" + ); + let (ok, out, err) = stone_in(&dir, &api, &["run", "main.stn"]); + assert!(ok, "{err}"); + assert_eq!(out, "json 1.2.0, num 0.3.0\n"); + // each package's releases were read once per run that downloaded + assert_eq!( + github.count("/repos/alice/json/releases?per_page=100&page=1"), + 2 + ); + + // a release whose tag moved no longer matches the lock + github.release("bob/num", &[("v0.3.0", num_release("0.3.9"))]); + std::fs::remove_dir_all(dir.join("home")).unwrap(); + let (ok, _, err) = stone_in(&dir, &api, &["run", "main.stn"]); + assert!(!ok); + assert!( + err.contains("stone.toml:3:11: error: this package says it is version 0.3.9, but it was released as 0.3.0"), + "{err}" + ); + github.release( + "bob/num", + &[("v0.3.0", { + let mut files = num_release("0.3.0"); + files[1].1.push_str("\npub fn extra() -> int:\n ret 1\n"); + files + })], + ); + std::fs::remove_dir_all(dir.join("home")).unwrap(); + let (ok, _, err) = stone_in(&dir, &api, &["run", "main.stn"]); + assert!(!ok); + assert!( + err.contains( + "error: num 0.3.0 (github:bob/num) does not hold what it held when it was locked" + ), + "{err}" + ); + github.release( + "bob/num", + &[ + ("v0.3.0", num_release("0.3.0")), + ("v0.3.5", num_release("0.3.5")), + ], + ); + std::fs::remove_dir_all(dir.join("home")).unwrap(); + + // an older release by its version, then a missing one + let (ok, out, err) = stone_in(&dir, &api, &["add", "github:alice/json@1.0"]); + assert!(ok, "{err}"); + assert_eq!(out, "added json 1.0.0 (github:alice/json)\n"); + let (ok, _, err) = stone_in(&dir, &api, &["add", "github:alice/json@3"]); + assert!(!ok); + assert_eq!( + err, + "error: github:alice/json has no release matching 3.0.0 (its releases are 2.0.0-rc.1, \ + 1.2.0, 1.0.0)\n" + ); + + let (ok, out, err) = stone_in(&dir, &api, &["remove", "json"]); + assert!(ok, "{err}"); + assert_eq!(out, "removed json\n"); + assert_eq!( + std::fs::read_to_string(dir.join("app/stone.toml")).unwrap(), + "[dependencies]\n" + ); + let lock = std::fs::read_to_string(dir.join("app/stone.lock")).unwrap(); + assert!(!lock.contains("[[package]]"), "{lock}"); +} + +#[test] +fn a_missing_repository_says_how_to_reach_a_private_one() { + let api = FakeGitHub::default().serve(); + let dir = project("package_missing_repo", &[("app/main.stn", "print(1)\n")]); + let (ok, _, err) = stone_in(&dir, &api, &["add", "github:alice/secret"]); + assert!(!ok); + assert_eq!( + err, + "error: github:alice/secret was not found (if it is private, set GITHUB_TOKEN to a token \ + that can read it)\n" + ); + assert!(!dir.join("app/stone.toml").exists()); +} diff --git a/tests/cli/windows_gui.ps1 b/tests/cli/windows_gui.ps1 index 9ca554e..7964cd4 100644 --- a/tests/cli/windows_gui.ps1 +++ b/tests/cli/windows_gui.ps1 @@ -10,6 +10,8 @@ # up the window's thread until it ends, checks that the program goes on drawing meanwhile, ends # the loop, and asks the window to close. # +# It parks the real pointer in the screen's corner first, so a resize puts no window under it. +# # It prints the window's scale, then `ok`, or what failed. param([string]$Title, [int]$Color, [double]$Scale = 0, [int]$Width = 0, [int]$Height = 0, [string]$Mode = "events") @@ -30,6 +32,10 @@ public static class Win { public static extern bool SetWindowPos(IntPtr window, IntPtr after, int x, int y, int width, int height, uint flags); [DllImport("user32.dll")] + public static extern bool SetCursorPos(int x, int y); + [DllImport("user32.dll")] + public static extern int GetSystemMetrics(int index); + [DllImport("user32.dll")] public static extern bool SetProcessDpiAwarenessContext(IntPtr context); [DllImport("user32.dll")] public static extern uint GetDpiForWindow(IntPtr window); @@ -67,6 +73,9 @@ public static class Win { "@ [Win]::SetProcessDpiAwarenessContext([IntPtr](-4)) | Out-Null +# park the real pointer in the screen's corner, since a window that grows under it would make +# Windows send a mouse move that the posted ones do not account for +[Win]::SetCursorPos([Win]::GetSystemMetrics(0) - 1, [Win]::GetSystemMetrics(1) - 1) | Out-Null $window = [IntPtr]::Zero for ($i = 0; $i -lt 300 -and $window -eq [IntPtr]::Zero; $i++) { Start-Sleep -Milliseconds 100 diff --git a/tests/packages/geo/lib.stn b/tests/packages/geo/lib.stn new file mode 100644 index 0000000..b0bd2f3 --- /dev/null +++ b/tests/packages/geo/lib.stn @@ -0,0 +1,15 @@ +use num + +pub struct Point: + x: int + y: int + +impl Point: + fn shifted(self, dx: int) -> Point: + ret Point(x=self.x + dx, y=self.y) + +pub fn origin() -> Point: + ret Point(x=0, y=0) + +pub fn dist2(a: Point, b: Point) -> int: + ret num.square(a.x - b.x) + num.square(a.y - b.y) diff --git a/tests/packages/geo/shapes.stn b/tests/packages/geo/shapes.stn new file mode 100644 index 0000000..0018028 --- /dev/null +++ b/tests/packages/geo/shapes.stn @@ -0,0 +1,5 @@ +// a module of the package, which uses another of its modules by its path +use sum + +pub fn perimeter(sides: list[int]) -> int: + ret sum.total(sides) diff --git a/tests/packages/geo/stone.toml b/tests/packages/geo/stone.toml new file mode 100644 index 0000000..ea55f73 --- /dev/null +++ b/tests/packages/geo/stone.toml @@ -0,0 +1,6 @@ +[package] +name = "geo" +version = "1.2.0" + +[dependencies] +num = { path = "../num" } diff --git a/tests/packages/geo/sum.stn b/tests/packages/geo/sum.stn new file mode 100644 index 0000000..e0d3d6c --- /dev/null +++ b/tests/packages/geo/sum.stn @@ -0,0 +1,5 @@ +pub fn total(xs: list[int]) -> int: + t = 0 + for x in xs: + t += x + ret t diff --git a/tests/packages/num/lib.stn b/tests/packages/num/lib.stn new file mode 100644 index 0000000..7aa795f --- /dev/null +++ b/tests/packages/num/lib.stn @@ -0,0 +1,2 @@ +pub fn square(x: int) -> int: + ret x * x diff --git a/tests/packages/num/stone.toml b/tests/packages/num/stone.toml new file mode 100644 index 0000000..a36860e --- /dev/null +++ b/tests/packages/num/stone.toml @@ -0,0 +1,3 @@ +[package] +name = "num" +version = "0.3.0" diff --git a/tests/packages/num_one/lib.stn b/tests/packages/num_one/lib.stn new file mode 100644 index 0000000..c4addce --- /dev/null +++ b/tests/packages/num_one/lib.stn @@ -0,0 +1,5 @@ +pub fn square(x: int) -> int: + ret x * x + +pub fn cube(x: int) -> int: + ret square(x) * x diff --git a/tests/packages/num_one/stone.toml b/tests/packages/num_one/stone.toml new file mode 100644 index 0000000..60f5e31 --- /dev/null +++ b/tests/packages/num_one/stone.toml @@ -0,0 +1,3 @@ +[package] +name = "num" +version = "1.0.0" diff --git a/tests/programs/packages/main.out b/tests/programs/packages/main.out new file mode 100644 index 0000000..79a7daf --- /dev/null +++ b/tests/programs/packages/main.out @@ -0,0 +1,4 @@ +25 +Point(x=5, y=4) +12 +8 diff --git a/tests/programs/packages/main.stn b/tests/programs/packages/main.stn new file mode 100644 index 0000000..1b5ef89 --- /dev/null +++ b/tests/programs/packages/main.stn @@ -0,0 +1,10 @@ +use geo +use geo.Point +use geo.shapes +use num + +p = Point(x=3, y=4) +print(geo.dist2(geo.origin(), p)) +print(p.shifted(2)) +print(shapes.perimeter([3, 4, 5])) +print(num.cube(2)) diff --git a/tests/programs/packages/stone.toml b/tests/programs/packages/stone.toml new file mode 100644 index 0000000..8ad2f00 --- /dev/null +++ b/tests/programs/packages/stone.toml @@ -0,0 +1,9 @@ +# a program whose dependencies are directories beside it, in tests/packages +[package] +name = "packages" +version = "0.1.0" + +[dependencies] +geo = { path = "../../packages/geo" } +# the same package as geo's num, but a newer major version, so both are linked +num = { path = "../../packages/num_one" }