diff --git a/pkgs/edit/.cargo/release-windows-ms.toml b/pkgs/edit/.cargo/release-windows-ms.toml new file mode 100644 index 0000000..0e5f50a --- /dev/null +++ b/pkgs/edit/.cargo/release-windows-ms.toml @@ -0,0 +1,32 @@ +# vvv The following parts are identical to release.toml vvv + +# Avoid linking with vcruntime140.dll by statically linking everything, +# and then explicitly linking with ucrtbase.dll dynamically. +# We do this, because vcruntime140.dll is an optional Windows component. +[target.'cfg(target_os = "windows")'] +rustflags = [ + "-Ctarget-feature=+crt-static", + "-Clink-args=/DEFAULTLIB:ucrt.lib", + "-Clink-args=/NODEFAULTLIB:vcruntime.lib", + "-Clink-args=/NODEFAULTLIB:msvcrt.lib", + "-Clink-args=/NODEFAULTLIB:libucrt.lib", +] + +# The backtrace code for panics in Rust is almost as large as the entire editor. +# = Huge reduction in binary size by removing all that. +[unstable] +build-std = ["std", "panic_abort"] +build-std-features = ["panic_immediate_abort"] + +# vvv The following parts are specific to official Windows builds. vvv +# (The use of internal registries, security features, etc., are mandatory.) + +# Enable shadow stacks: https://learn.microsoft.com/en-us/cpp/build/reference/cetcompat +[target.'cfg(all(target_os = "windows", any(target_arch = "x86", target_arch = "x86_64")))'] +rustflags = ["-Clink-args=/DYNAMICBASE", "-Clink-args=/CETCOMPAT"] + +[registries.Edit_PublicPackages] +index = "sparse+https://pkgs.dev.azure.com/microsoft/Dart/_packaging/Edit_PublicPackages/Cargo/index/" + +[source.crates-io] +replace-with = "Edit_PublicPackages" diff --git a/pkgs/edit/.cargo/release.toml b/pkgs/edit/.cargo/release.toml new file mode 100644 index 0000000..790fde9 --- /dev/null +++ b/pkgs/edit/.cargo/release.toml @@ -0,0 +1,23 @@ +# The following is not used by default via .cargo/config.toml, +# because `build-std-features` cannot be keyed by profile. +# This breaks the bench profile which doesn't support panic=abort. +# See: https://github.com/rust-lang/cargo/issues/11214 +# See: https://github.com/rust-lang/cargo/issues/13894 + +# Avoid linking with vcruntime140.dll by statically linking everything, +# and then explicitly linking with ucrtbase.dll dynamically. +# We do this, because vcruntime140.dll is an optional Windows component. +[target.'cfg(target_os = "windows")'] +rustflags = [ + "-Ctarget-feature=+crt-static", + "-Clink-args=/DEFAULTLIB:ucrt.lib", + "-Clink-args=/NODEFAULTLIB:vcruntime.lib", + "-Clink-args=/NODEFAULTLIB:msvcrt.lib", + "-Clink-args=/NODEFAULTLIB:libucrt.lib", +] + +# The backtrace code for panics in Rust is almost as large as the entire editor. +# = Huge reduction in binary size by removing all that. +[unstable] +build-std = ["std", "panic_abort"] +build-std-features = ["panic_immediate_abort"] diff --git a/pkgs/edit/.gitignore b/pkgs/edit/.gitignore new file mode 100644 index 0000000..c65c81a --- /dev/null +++ b/pkgs/edit/.gitignore @@ -0,0 +1,5 @@ +.idea +.vs +*.profraw +lcov.info +target diff --git a/pkgs/edit/.pipelines/release.yml b/pkgs/edit/.pipelines/release.yml new file mode 100644 index 0000000..569ca77 --- /dev/null +++ b/pkgs/edit/.pipelines/release.yml @@ -0,0 +1,168 @@ +# Documentation: https://aka.ms/obpipelines + +trigger: none + +parameters: + - name: debug + displayName: Enable debug output + type: boolean + default: false + - name: official + displayName: Whether to build Official or NonOfficial + type: string + default: NonOfficial + values: + - NonOfficial + - Official + - name: createvpack + displayName: Enable vpack creation + type: boolean + default: false + - name: buildPlatforms + type: object + default: + - x86_64-pc-windows-msvc + - aarch64-pc-windows-msvc + +variables: + system.debug: ${{parameters.debug}} + WindowsContainerImage: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + # CDP_DEFINITION_BUILD_COUNT is needed for onebranch.pipeline.version task. + # See: https://aka.ms/obpipelines/versioning + CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] + # LOAD BEARING - the vpack task fails without these + ROOT: $(Build.SourcesDirectory) + REPOROOT: $(Build.SourcesDirectory) + OUTPUTROOT: $(REPOROOT)\out + NUGET_XMLDOC_MODE: none + +resources: + repositories: + - repository: GovernedTemplates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: v2/Microsoft.${{parameters.official}}.yml@GovernedTemplates + parameters: + featureFlags: + WindowsHostVersion: + Version: 2022 + Network: R1 + platform: + name: windows_undocked + product: edit + # https://aka.ms/obpipelines/cloudvault + cloudvault: + enabled: false + # https://aka.ms/obpipelines/sdl + globalSdl: + binskim: + # > Due to some legacy reasons, 1ES PT is scanning full sources directory + # > for BinSkim tool instead of just scanning the output directory [...] + scanOutputDirectoryOnly: true + isNativeCode: true + tsa: + enabled: ${{eq(parameters.official, 'Official')}} + configFile: "$(Build.SourcesDirectory)/.pipelines/tsa.json" + stages: + # Our Build stage will build all three targets in one job, so we don't need + # to repeat most of the boilerplate work in three separate jobs. + - stage: Build + jobs: + - job: Windows + pool: + type: windows + variables: + # Binaries will go here. + # More settings at https://aka.ms/obpipelines/yaml/jobs + ob_outputDirectory: "$(Build.SourcesDirectory)/out" + # The vPack gets created from stuff in here. + # It will have a structure like: + # .../vpack/ + # - amd64/ + # - edit.exe + # - i386/ + # - edit.exe + # - arm64/ + # - edit.exe + ob_createvpack_enabled: ${{parameters.createvpack}} + ob_createvpack_vpackdirectory: "$(ob_outputDirectory)/vpack" + ob_createvpack_packagename: "windows_edit.$(Build.SourceBranchName)" + ob_createvpack_owneralias: lhecker@microsoft.com + ob_createvpack_description: Microsoft Edit + ob_createvpack_targetDestinationDirectory: "$(Destination)" + ob_createvpack_propsFile: false + ob_createvpack_provData: true + ob_createvpack_versionAs: string + ob_createvpack_version: "$(EditVersion)-$(CDP_DEFINITION_BUILD_COUNT)" + ob_createvpack_metadata: "$(Build.SourceVersion)" + ob_createvpack_topLevelRetries: 0 + ob_createvpack_failOnStdErr: true + ob_createvpack_verbose: ${{ parameters.debug }} + # For details on this cargo_target_dir setting, see: + # https://eng.ms/docs/more/rust/topics/onebranch-workaround + CARGO_TARGET_DIR: C:\cargo_target_dir + # msrustup only supports stable toolchains, but this project requires nightly. + # We were told RUSTC_BOOTSTRAP=1 is a supported workaround. + RUSTC_BOOTSTRAP: 1 + steps: + # NOTE: Step objects have ordered keys and you MUST have "task" as the first key. + # Objects with ordered keys... lol + - task: RustInstaller@1 + displayName: Install Rust toolchain + inputs: + rustVersion: ms-stable + additionalTargets: x86_64-pc-windows-msvc aarch64-pc-windows-msvc + # URL of an Azure Artifacts feed configured with a crates.io upstream. Must be within the current ADO collection. + # NOTE: Azure Artifacts support for Rust is not yet public, but it is enabled for internal ADO organizations. + # https://learn.microsoft.com/en-us/azure/devops/artifacts/how-to/set-up-upstream-sources?view=azure-devops + cratesIoFeedOverride: sparse+https://pkgs.dev.azure.com/microsoft/Dart/_packaging/Edit_PublicPackages/Cargo/index/ + # URL of an Azure Artifacts NuGet feed configured with the mscodehub Rust feed as an upstream. + # * The feed must be within the current ADO collection. + # * The CI account, usually "Project Collection Build Service (org-name)", must have at least "Collaborator" permission. + # When setting up the upstream NuGet feed, use following Azure Artifacts feed locator: + # azure-feed://mscodehub/Rust/Rust@Release + toolchainFeed: https://pkgs.dev.azure.com/microsoft/_packaging/RustTools/nuget/v3/index.json + - task: CargoAuthenticate@0 + displayName: Authenticate with Azure Artifacts + inputs: + configFile: ".cargo/release-windows-ms.toml" + # We recommend making a separate `cargo fetch` step, as some build systems perform + # fetching entirely prior to the build, and perform the build with the network disabled. + - script: cargo fetch --config .cargo/release-windows-ms.toml + displayName: Fetch crates + - ${{ each platform in parameters.buildPlatforms }}: + - script: cargo build --config .cargo/release-windows-ms.toml --frozen --release --target ${{platform}} + displayName: Build ${{platform}} Release + - task: CopyFiles@2 + displayName: Copy files to vpack (${{platform}}) + inputs: + sourceFolder: "$(CARGO_TARGET_DIR)/${{platform}}/release" + ${{ if eq(platform, 'i686-pc-windows-msvc') }}: + targetFolder: "$(ob_createvpack_vpackdirectory)/i386" + ${{ elseif eq(platform, 'x86_64-pc-windows-msvc') }}: + targetFolder: "$(ob_createvpack_vpackdirectory)/amd64" + ${{ else }}: # aarch64-pc-windows-msvc + targetFolder: "$(ob_createvpack_vpackdirectory)/arm64" + contents: | + *.exe + *.pdb + # Extract the version for `ob_createvpack_version`. + - script: |- + @echo off + for /f "tokens=3 delims=- " %%x in ('findstr /c:"version = " Cargo.toml') do ( + echo ##vso[task.setvariable variable=EditVersion]%%~x + goto :EOF + ) + displayName: "Set EditVersion" + - task: onebranch.pipeline.signing@1 + displayName: "Sign files" + inputs: + command: "sign" + signing_profile: "external_distribution" + files_to_sign: "**/edit.exe" + search_root: "$(ob_createvpack_vpackdirectory)" + use_testsign: false + in_container: true diff --git a/pkgs/edit/.pipelines/tsa.json b/pkgs/edit/.pipelines/tsa.json new file mode 100644 index 0000000..0e1f9b0 --- /dev/null +++ b/pkgs/edit/.pipelines/tsa.json @@ -0,0 +1,7 @@ +{ + "instanceUrl": "https://microsoft.visualstudio.com", + "projectName": "OS", + "areaPath": "OS\\Windows Client and Services\\WinPD\\DFX-Developer Fundamentals and Experiences\\DEFT\\SHINE\\Commandline Tooling", + "notificationAliases": ["condev@microsoft.com", "duhowett@microsoft.com"], + "template": "VSTS_Microsoft_OSGS" +} diff --git a/pkgs/edit/.vscode/launch.json b/pkgs/edit/.vscode/launch.json new file mode 100644 index 0000000..b57b8b7 --- /dev/null +++ b/pkgs/edit/.vscode/launch.json @@ -0,0 +1,28 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Debug (Windows)", + "preLaunchTask": "rust: cargo build", + "type": "cppvsdbg", + "request": "launch", + "console": "externalTerminal", + "program": "${workspaceFolder}/target/debug/edit", + "cwd": "${workspaceFolder}", + "args": [ + "${workspaceFolder}/src/bin/edit/main.rs" + ], + }, + { + "name": "Launch Debug (GDB/LLDB)", + "preLaunchTask": "rust: cargo build", + "type": "cppdbg", + "request": "launch", + "program": "${workspaceFolder}/target/debug/edit", + "cwd": "${workspaceFolder}", + "args": [ + "${workspaceFolder}/src/bin/edit/main.rs" + ], + } + ] +} diff --git a/pkgs/edit/.vscode/tasks.json b/pkgs/edit/.vscode/tasks.json new file mode 100644 index 0000000..daeaf8c --- /dev/null +++ b/pkgs/edit/.vscode/tasks.json @@ -0,0 +1,24 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "rust: cargo build", + "type": "process", + "command": "cargo", + "args": [ + "build", + "--package", + "edit", + "--features", + "debug-latency" + ], + "group": { + "kind": "build", + "isDefault": true + }, + "problemMatcher": [ + "$rustc" + ] + } + ] +} diff --git a/pkgs/edit/CODE_OF_CONDUCT.md b/pkgs/edit/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..686e5e7 --- /dev/null +++ b/pkgs/edit/CODE_OF_CONDUCT.md @@ -0,0 +1,10 @@ +# Microsoft Open Source Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns +- Employees can reach out at [aka.ms/opensource/moderation-support](https://aka.ms/opensource/moderation-support) diff --git a/pkgs/edit/CONTRIBUTING.md b/pkgs/edit/CONTRIBUTING.md new file mode 100644 index 0000000..1398694 --- /dev/null +++ b/pkgs/edit/CONTRIBUTING.md @@ -0,0 +1,49 @@ +# Contributing + +## Translation improvements + +You can find our translations in [`src/bin/edit/localization.rs`](./src/bin/edit/localization.rs). +Please feel free to open a pull request with your changes at any time. +If you'd like to discuss your changes first, please feel free to open an issue. + +## Bug reports + +If you find any bugs, we gladly accept pull requests without prior discussion. +Otherwise, you can of course always open an issue for us to look into. + +## Feature requests + +Please open a new issue for any feature requests you have in mind. +Keeping the binary size of the editor small is a priority for us and so we may need to discuss any new features first until we have support for plugins. + +## Code changes + +The project has a focus on a small binary size and sufficient (good) performance. +As such, we generally do not accept pull requests that introduce dependencies (there are always exceptions of course). +Otherwise, you can consider this project a playground for trying out any cool ideas you have. + +The overall architecture of the project can be summarized as follows: +* The underlying text buffer in `src/buffer` doesn't keep track of line breaks in the document. + This is a crucial design aspect that permeates throughout the entire codebase. + + To oversimplify, the *only* state that is kept is the current cursor position. + When the user asks to move to another line, the editor will `O(n)` seek through the underlying document until it found the corresponding number of line breaks. + * As a result, `src/simd` contains crucial `memchr2` functions to quickly find the next or previous line break (runs at up to >100GB/s). + * Furthermore, `src/unicode` implements an `Utf8Chars` iterator which transparently inserts U+FFFD replacements during iteration (runs at up to 4GB/s). + * Furthermore, `src/unicode` also implements grapheme cluster segmentation and cluster width measurement via its `MeasurementConfig` (runs at up to 600MB/s). + * If word wrap is disabled, `memchr2` is used for all navigation across lines, allowing us to breeze through 1GB large files as if they were 1MB. + * Even if word-wrap is enabled, it's still sufficiently smooth thanks to `MeasurementConfig`. This is only possible because these base functions are heavily optimized. +* `src/framebuffer.rs` implements a "framebuffer" like in video games. + It allows us to draw the UI output into an intermediate buffer first, accumulating all changes and handling things like color blending. + Then, it can compare the accumulated output with the previous frame and only send the necessary changes to the terminal. +* `src/tui.rs` implements an immediate mode UI. Its module implementation gives an overview how it works and I recommend reading it. +* `src/vt.rs` implements our VT parser. +* `src/sys` contains our platform abstractions. +* Finally, `src/bin/edit` ties everything together. + It's roughly 90% UI code and business logic. + It contains a little bit of VT logic in `setup_terminal`. + +If you have an issue with your terminal, the places of interest are the aforementioned: +* VT parser in `src/vt.rs` +* Platform specific code in `src/sys` +* And the `setup_terminal` function in `src/bin/edit/main.rs` diff --git a/pkgs/edit/Cargo.lock b/pkgs/edit/Cargo.lock new file mode 100644 index 0000000..87664f5 --- /dev/null +++ b/pkgs/edit/Cargo.lock @@ -0,0 +1,627 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.5.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + +[[package]] +name = "edit" +version = "1.0.0" +dependencies = [ + "criterion", + "libc", + "windows-sys", + "winres", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hermit-abi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" + +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "proc-macro2" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winres" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b68db261ef59e9e52806f688020631e987592bd83619edccda9c47d42cde4f6c" +dependencies = [ + "toml", +] diff --git a/pkgs/edit/Cargo.toml b/pkgs/edit/Cargo.toml new file mode 100644 index 0000000..05872fc --- /dev/null +++ b/pkgs/edit/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "edit" +version = "1.0.0" +edition = "2024" + +[[bench]] +name = "lib" +harness = false + +[features] +debug-layout = [] +debug-latency = [] + +# `opt-level = "s"` may be useful in the future as it significantly reduces binary size. +# We could then use the `#[optimize(speed)]` attribute for spot optimizations. +# Unfortunately, that attribute currently doesn't work on intrinsics such as memset. +[profile.release] +codegen-units = 1 # reduces binary size by ~2% +debug = "full" # No one needs an undebuggable release binary +lto = true # reduces binary size by ~14% +opt-level = "s" # reduces binary size by ~25% +panic = "abort" # reduces binary size by ~50% in combination with -Zbuild-std-features=panic_immediate_abort +split-debuginfo = "packed" # generates a seperate *.dwp/*.dSYM so the binary can get stripped +strip = "symbols" # See split-debuginfo - allows us to drop the size by ~65% + +[profile.bench] +codegen-units = 16 # Make compiling criterion faster (16 is the default, but profile.release sets it to 1) +lto = "thin" # Similarly, speed up linking by a ton + +[dependencies] + +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[target.'cfg(windows)'.build-dependencies] +winres = "0.1" + +[target.'cfg(windows)'.dependencies.windows-sys] +version = "0.59" +features = [ + "Win32_Globalization", + "Win32_Security", + "Win32_Storage_FileSystem", + "Win32_System_Console", + "Win32_System_Diagnostics_Debug", + "Win32_System_IO", + "Win32_System_LibraryLoader", + "Win32_System_Memory", + "Win32_System_Threading", +] + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } diff --git a/pkgs/edit/LICENSE b/pkgs/edit/LICENSE new file mode 100644 index 0000000..88040d8 --- /dev/null +++ b/pkgs/edit/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Microsoft Corporation. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pkgs/edit/README.md b/pkgs/edit/README.md new file mode 100644 index 0000000..acd0947 --- /dev/null +++ b/pkgs/edit/README.md @@ -0,0 +1,22 @@ +# ![Application Icon for Edit](./assets/edit.svg) Edit + +A simple editor for simple needs. + +This editor pays homage to the classic [MS-DOS Editor](https://en.wikipedia.org/wiki/MS-DOS_Editor), but with a modern interface and input controls similar to VS Code. The goal is to provide an accessible editor that even users largely unfamiliar with terminals can easily use. + +![Screenshot of Edit with the About dialog in the foreground](./assets/edit_hero_image.png) + +## Installation + +* Download the latest release from our [releases page](https://github.com/microsoft/edit/releases/latest) +* Extract the archive +* Copy the `edit` binary to a directory in your `PATH` +* You may delete any other files in the archive if you don't need them + +## Build Instructions + +* [Install Rust](https://www.rust-lang.org/tools/install) +* Install the nightly toolchain: `rustup install nightly` + * Alternatively, set the environment variable `RUSTC_BOOTSTRAP=1` +* Clone the repository +* For a release build, run: `cargo build --config .cargo/release.toml --release` diff --git a/pkgs/edit/SECURITY.md b/pkgs/edit/SECURITY.md new file mode 100644 index 0000000..82db58a --- /dev/null +++ b/pkgs/edit/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). + + diff --git a/pkgs/edit/assets/Microsoft_logo_(1980).svg b/pkgs/edit/assets/Microsoft_logo_(1980).svg new file mode 100644 index 0000000..473c8d5 --- /dev/null +++ b/pkgs/edit/assets/Microsoft_logo_(1980).svg @@ -0,0 +1,26 @@ + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + diff --git a/pkgs/edit/assets/edit.svg b/pkgs/edit/assets/edit.svg new file mode 100644 index 0000000..74a6242 --- /dev/null +++ b/pkgs/edit/assets/edit.svg @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pkgs/edit/assets/edit_hero_image.png b/pkgs/edit/assets/edit_hero_image.png new file mode 100644 index 0000000..87bed35 Binary files /dev/null and b/pkgs/edit/assets/edit_hero_image.png differ diff --git a/pkgs/edit/assets/microsoft.png b/pkgs/edit/assets/microsoft.png new file mode 100644 index 0000000..1092d4c Binary files /dev/null and b/pkgs/edit/assets/microsoft.png differ diff --git a/pkgs/edit/assets/microsoft.sixel b/pkgs/edit/assets/microsoft.sixel new file mode 100644 index 0000000..bf53919 --- /dev/null +++ b/pkgs/edit/assets/microsoft.sixel @@ -0,0 +1 @@ +P;1q"1;1;300;60#0;2;100;100;100#0!42?_ow{}!12?_ow{}!6?_ow{}}!5?_ow{{}}}!17~^NFbpw{}!8~!4}{wwo_!12?_oow{{{!4}!6~!4}{{wwo__!4?_ow{{}}}!23~^Nfrxw{{}}}!9~!4}{{woo_!12?_ow{}!15~^NFbpw{}!17~^NFB@-!36?_ow{}!6~!6?_ow{}!6~??w{}!7~?o{}!10~^^!10NFBpw{}!6~!8N^!9~{_!4?_o{}!8~^^!9N^^!9~{w}!8~^!18NFbx{}!9~^^!8N^^!9~}{o???ow{}!6~!11NFB@GKM!5N!10~!4NFB@-!30?_ow{}!12~_ow{}!12~??!20~FB@!15?!10~!10?r!9~???{!8~NB@!15?@FN!16~!4{!4wooo__!5?_}!8~^FB!16?@F^!8~{o!10~!9o!13?!10~-!24?_ow{}!35~??!19~x!18?!10~?CK[!4{}!9~^B??N!8~x!21?!10~N^^!18~}{o!10~!22?!29~!13?!10~-!18?_ow{}!8~^NFB@?!11~^NFB@?!10~??!10~F!9~}{wo__!12?!10~!5?@BFN^!9~}{wof^!7~}wo__!11?__o{!9~N@!7?!6@Bb!10~N!9~{o__!12?__o{}!8~F@!10~!9B!13?!10~-!12?_ow{}!8~^NFB@!7?!5~^NFB@!7?!10~??!10~??@FN^!20~??!10~!11?@BFN^!23~!7}!10~^NFB~!12}!12~^NB??BFN^!9~!10}!9~^NF@???!10~!22?!5~^NFB@-!6?_ow{}!8~^NFB@!13?FFB@!13?!10F??!10F!7?@@BB!15F??!10F!17?@BFN^!10~|zrfFF!10NFFFBB@@!5?!21FBB@!11?@BBFFNNN!10^NNNFFBB@!8?!10~!22?NFB@-_ow{}!8~^NFB@!119?@BFN^!9~}{wo!88?!10~-!7~^NFB@!131?@BFN^!7~!88?!7~^NF-~^NFB@!143?@BFN^~!88?~^NFB@\ diff --git a/pkgs/edit/benches/lib.rs b/pkgs/edit/benches/lib.rs new file mode 100644 index 0000000..1131f34 --- /dev/null +++ b/pkgs/edit/benches/lib.rs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::hint::black_box; +use std::mem; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use edit::helpers::*; +use edit::simd::MemsetSafe; +use edit::{hash, oklab, simd, unicode}; + +fn bench_hash(c: &mut Criterion) { + c.benchmark_group("hash") + .throughput(Throughput::Bytes(8)) + .bench_function(BenchmarkId::new("hash", 8), |b| { + let data = [0u8; 8]; + b.iter(|| hash::hash(0, black_box(&data))) + }) + .throughput(Throughput::Bytes(16)) + .bench_function(BenchmarkId::new("hash", 16), |b| { + let data = [0u8; 16]; + b.iter(|| hash::hash(0, black_box(&data))) + }) + .throughput(Throughput::Bytes(1024)) + .bench_function(BenchmarkId::new("hash", 1024), |b| { + let data = [0u8; 1024]; + b.iter(|| hash::hash(0, black_box(&data))) + }); +} + +fn bench_oklab(c: &mut Criterion) { + c.benchmark_group("oklab") + .bench_function("srgb_to_oklab", |b| b.iter(|| oklab::srgb_to_oklab(black_box(0xff212cbe)))) + .bench_function("oklab_blend", |b| { + b.iter(|| oklab::oklab_blend(black_box(0x7f212cbe), black_box(0x7f3aae3f))) + }); +} + +fn bench_simd_memchr2(c: &mut Criterion) { + let mut group = c.benchmark_group("simd"); + let mut buffer_u8 = [0u8; 2048]; + + for &bytes in &[8usize, 32 + 8, 64 + 8, KIBI + 8] { + group.throughput(Throughput::Bytes(bytes as u64 + 1)).bench_with_input( + BenchmarkId::new("memchr2", bytes), + &bytes, + |b, &size| { + buffer_u8.fill(b'a'); + buffer_u8[size] = b'\n'; + b.iter(|| simd::memchr2(b'\n', b'\r', black_box(&buffer_u8), 0)); + }, + ); + } +} + +fn bench_simd_memset(c: &mut Criterion) { + let mut group = c.benchmark_group("simd"); + let name = format!("memset<{}>", std::any::type_name::()); + let size = mem::size_of::(); + let mut buf: Vec = vec![Default::default(); 2048 / size]; + + for &bytes in &[8usize, 32 + 8, 64 + 8, KIBI + 8] { + group.throughput(Throughput::Bytes(bytes as u64)).bench_with_input( + BenchmarkId::new(&name, bytes), + &bytes, + |b, &bytes| { + let slice = unsafe { buf.get_unchecked_mut(..bytes / size) }; + b.iter(|| simd::memset(black_box(slice), Default::default())); + }, + ); + } +} + +fn bench_unicode(c: &mut Criterion) { + let reference = concat!( + "In the quiet twilight, dreams unfold, soft whispers of a story untold.\n", + "月明かりが静かに照らし出し、夢を見る心の奥で詩が静かに囁かれる\n", + "Stars collide in the early light of hope, echoing the silent call of the night.\n", + "夜の静寂、希望と孤独が混ざり合うその中で詩が永遠に続く\n", + ); + let buffer = reference.repeat(10); + let bytes = buffer.as_bytes(); + + c.benchmark_group("unicode::MeasurementConfig::goto_logical") + .throughput(Throughput::Bytes(bytes.len() as u64)) + .bench_function("basic", |b| { + b.iter(|| unicode::MeasurementConfig::new(&bytes).goto_logical(Point::MAX)) + }) + .bench_function("word_wrap", |b| { + b.iter(|| { + unicode::MeasurementConfig::new(black_box(&bytes)) + .with_word_wrap_column(50) + .goto_logical(Point::MAX) + }) + }); + + c.benchmark_group("unicode::Utf8Chars") + .throughput(Throughput::Bytes(bytes.len() as u64)) + .bench_function("next", |b| { + b.iter(|| { + unicode::Utf8Chars::new(bytes, 0).fold(0u32, |acc, ch| acc.wrapping_add(ch as u32)) + }) + }); +} + +fn bench(c: &mut Criterion) { + bench_hash(c); + bench_oklab(c); + bench_simd_memchr2(c); + bench_simd_memset::(c); + bench_simd_memset::(c); + bench_unicode(c); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/pkgs/edit/build.rs b/pkgs/edit/build.rs new file mode 100644 index 0000000..2e52f02 --- /dev/null +++ b/pkgs/edit/build.rs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +fn main() { + #[cfg(windows)] + if std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default() == "windows" { + winres::WindowsResource::new() + .set_manifest_file("src/bin/edit/edit.exe.manifest") + .set("FileDescription", "Microsoft Edit") + .set("LegalCopyright", "© Microsoft Corporation. All rights reserved.") + .compile() + .unwrap(); + } +} diff --git a/pkgs/edit/edit.nix b/pkgs/edit/edit.nix new file mode 100644 index 0000000..4e6684b --- /dev/null +++ b/pkgs/edit/edit.nix @@ -0,0 +1,27 @@ +{ + lib, + pkgs +}: + +let + rustPlatform = pkgs.makeRustPlatform { + cargo = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default); + rustc = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default); + }; +in + +rustPlatform.buildRustPackage (finalAttrs: { + pname = "edit"; + version = "1.0.0"; + + src = ./.; + + cargoHash = "sha256-DEzjfrXSmum/GJdYanaRDKxG4+eNPWf5echLhStxcIg="; + + meta = { + description = "We all edit."; + homepage = "https://github.com/microsoft/edit"; + license = lib.licenses.mit; + maintainers = [ ]; + }; +}) diff --git a/pkgs/edit/flake.lock b/pkgs/edit/flake.lock new file mode 100644 index 0000000..c001d69 --- /dev/null +++ b/pkgs/edit/flake.lock @@ -0,0 +1,99 @@ +{ + "nodes": { + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1747392669, + "narHash": "sha256-zky3+lndxKRu98PAwVK8kXPdg+Q1NVAhaI7YGrboKYA=", + "owner": "nix-community", + "repo": "fenix", + "rev": "c3c27e603b0d9b5aac8a16236586696338856fbb", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1747744144, + "narHash": "sha256-W7lqHp0qZiENCDwUZ5EX/lNhxjMdNapFnbErcbnP11Q=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "2795c506fe8fb7b03c36ccb51f75b6df0ab2553f", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "nixos-unstable", + "type": "indirect" + } + }, + "root": { + "inputs": { + "fenix": "fenix", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1747323949, + "narHash": "sha256-G4NwzhODScKnXqt2mEQtDFOnI0wU3L1WxsiHX3cID/0=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "f8e784353bde7cbf9a9046285c1caf41ac484ebe", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/pkgs/edit/flake.nix b/pkgs/edit/flake.nix new file mode 100644 index 0000000..a6fe65a --- /dev/null +++ b/pkgs/edit/flake.nix @@ -0,0 +1,39 @@ +{ + description = "Flake containing Edit"; + + inputs = { + fenix = { + url = "github:nix-community/fenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + flake-utils.url = "github:numtide/flake-utils"; + nixpkgs.url = "nixpkgs/nixos-unstable"; + }; + + outputs = { self, fenix, flake-utils, nixpkgs }: + flake-utils.lib.eachDefaultSystem (system: + let pkgs = nixpkgs.legacyPackages.${system}; in + { + defaultPackage = + let rustPlatform = (pkgs.makeRustPlatform { + inherit (fenix.packages.${system}.minimal) cargo rustc; + }); + in rustPlatform.buildRustPackage { + pname = "edit"; + version = "1.0.0"; + + src = ./.; + + cargoHash = "sha256-DEzjfrXSmum/GJdYanaRDKxG4+eNPWf5echLhStxcIg="; + + meta = { + description = "We all edit."; + homepage = "https://github.com/microsoft/edit"; + license = pkgs.lib.licenses.mit; + maintainers = [ ]; + }; + + }; + } + ); +} diff --git a/pkgs/edit/hi.nix b/pkgs/edit/hi.nix new file mode 100644 index 0000000..8ea921d --- /dev/null +++ b/pkgs/edit/hi.nix @@ -0,0 +1,62 @@ +{ + description = "core-lending"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable"; + flake-parts.url = "github:hercules-ci/flake-parts"; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = inputs: inputs.flake-parts.lib.mkFlake { inherit inputs; } { + systems = [ + "x86_64-linux" + "aarch64-darwin" + ]; + + perSystem = { self', pkgs, system, ... }: + let + rustVersion = "1.86.0"; + rust = pkgs.rust-bin.stable.${rustVersion}; + rustPlatform = pkgs.makeRustPlatform { + rustc = rust.minimal; + cargo = rust.minimal; + }; + cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); + in + { + _module.args.pkgs = import inputs.nixpkgs { + inherit system; + overlays = [ inputs.rust-overlay.overlays.default ]; + }; + + packages.default = rustPlatform.buildRustPackage { + pname = cargoToml.package.name; + version = cargoToml.package.version; + + src = ./.; + cargoLock = { + lockFile = ./Cargo.lock; + }; + }; + + devShells.default = pkgs.mkShell { + buildInputs = [ + pkgs.diesel-cli + pkgs.elmPackages.elm + pkgs.elmPackages.elm-language-server + pkgs.postgresql_17 + (rust.default.override { + extensions = [ "rust-analyzer" "rust-src" ]; + }) + ]; + + shellHook = '' + export LANG="en_US.UTF-8" + ''; + }; + }; + }; +} diff --git a/pkgs/edit/result b/pkgs/edit/result new file mode 120000 index 0000000..7cba96b --- /dev/null +++ b/pkgs/edit/result @@ -0,0 +1 @@ +/nix/store/inr2b18j80dc17bkaz5xm7f20g0i3s0r-edit-1.0.0 \ No newline at end of file diff --git a/pkgs/edit/rustfmt.toml b/pkgs/edit/rustfmt.toml new file mode 100644 index 0000000..90f2491 --- /dev/null +++ b/pkgs/edit/rustfmt.toml @@ -0,0 +1,7 @@ +style_edition = "2024" +use_small_heuristics = "Max" +group_imports = "StdExternalCrate" +imports_granularity = "Module" +format_code_in_doc_comments = true +newline_style = "Unix" +use_field_init_shorthand = true diff --git a/pkgs/edit/src/apperr.rs b/pkgs/edit/src/apperr.rs new file mode 100644 index 0000000..b1dd6d2 --- /dev/null +++ b/pkgs/edit/src/apperr.rs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Provides a transparent error type for edit. + +use std::{io, result}; + +use crate::sys; + +pub const APP_ICU_MISSING: Error = Error::new_app(0); + +/// Edit's transparent `Result` type. +pub type Result = result::Result; + +/// Edit's transparent `Error` type. +/// Abstracts over system and application errors. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + App(u32), + Icu(u32), + Sys(u32), +} + +impl Error { + pub const fn new_app(code: u32) -> Self { + Error::App(code) + } + + pub const fn new_icu(code: u32) -> Self { + Error::Icu(code) + } + + pub const fn new_sys(code: u32) -> Self { + Error::Sys(code) + } +} + +impl From for Error { + fn from(err: io::Error) -> Self { + sys::io_error_to_apperr(err) + } +} diff --git a/pkgs/edit/src/arena/debug.rs b/pkgs/edit/src/arena/debug.rs new file mode 100644 index 0000000..4b1ae6c --- /dev/null +++ b/pkgs/edit/src/arena/debug.rs @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#![allow(clippy::missing_safety_doc, clippy::mut_from_ref)] + +use std::alloc::{AllocError, Allocator, Layout}; +use std::mem::{self, MaybeUninit}; +use std::ptr::NonNull; + +use super::release; +use crate::apperr; + +/// A debug wrapper for [`release::Arena`]. +/// +/// The problem with [`super::ScratchArena`] is that it only "borrows" an underlying +/// [`release::Arena`]. Once the [`super::ScratchArena`] is dropped it resets the watermark +/// of the underlying [`release::Arena`], freeing all allocations done since borrowing it. +/// +/// It is completely valid for the same [`release::Arena`] to be borrowed multiple times at once, +/// *as long as* you only use the most recent borrow. Bad example: +/// ```should_panic +/// use edit::arena::scratch_arena; +/// +/// let mut scratch1 = scratch_arena(None); +/// let mut scratch2 = scratch_arena(None); +/// +/// let foo = scratch1.alloc_uninit::(); +/// +/// // This will also reset `scratch1`'s allocation. +/// drop(scratch2); +/// +/// *foo; // BOOM! ...if it wasn't for our debug wrapper. +/// ``` +/// +/// To avoid this, this wraps the real [`release::Arena`] in a "debug" one, which pretends as if every +/// instance of itself is a distinct [`release::Arena`] instance. Then we use this "debug" [`release::Arena`] +/// for [`super::ScratchArena`] which allows us to track which borrow is the most recent one. +pub enum Arena { + // Delegate is 'static, because release::Arena requires no lifetime + // annotations, and so this mere debug helper cannot use them either. + Delegated { delegate: &'static release::Arena, borrow: usize }, + Owned { arena: release::Arena }, +} + +impl Drop for Arena { + fn drop(&mut self) { + if let Arena::Delegated { delegate, borrow } = self { + let borrows = delegate.borrows.get(); + assert_eq!(*borrow, borrows); + delegate.borrows.set(borrows - 1); + } + } +} + +impl Default for Arena { + fn default() -> Self { + Self::empty() + } +} + +impl Arena { + pub const fn empty() -> Self { + Self::Owned { arena: release::Arena::empty() } + } + + pub fn new(capacity: usize) -> apperr::Result { + Ok(Self::Owned { arena: release::Arena::new(capacity)? }) + } + + pub(super) fn delegated(delegate: &release::Arena) -> Arena { + let borrow = delegate.borrows.get() + 1; + delegate.borrows.set(borrow); + Self::Delegated { delegate: unsafe { mem::transmute(delegate) }, borrow } + } + + #[inline] + pub(super) fn delegate_target(&self) -> &release::Arena { + match *self { + Arena::Delegated { delegate, borrow } => { + assert!( + borrow == delegate.borrows.get(), + "Arena already borrowed by a newer ScratchArena" + ); + delegate + } + Arena::Owned { ref arena } => arena, + } + } + + #[inline] + pub(super) fn delegate_target_unchecked(&self) -> &release::Arena { + match self { + Arena::Delegated { delegate, .. } => delegate, + Arena::Owned { arena } => arena, + } + } + + pub fn offset(&self) -> usize { + self.delegate_target().offset() + } + + pub unsafe fn reset(&self, to: usize) { + unsafe { self.delegate_target().reset(to) } + } + + pub fn alloc_uninit(&self) -> &mut MaybeUninit { + self.delegate_target().alloc_uninit() + } + + pub fn alloc_uninit_slice(&self, count: usize) -> &mut [MaybeUninit] { + self.delegate_target().alloc_uninit_slice(count) + } +} + +unsafe impl Allocator for Arena { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + self.delegate_target().alloc_raw(layout.size(), layout.align()) + } + + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + self.delegate_target().allocate_zeroed(layout) + } + + // While it is possible to shrink the tail end of the arena, it is + // not very useful given the existence of scoped scratch arenas. + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + unsafe { self.delegate_target().deallocate(ptr, layout) } + } + + unsafe fn grow( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + unsafe { self.delegate_target().grow(ptr, old_layout, new_layout) } + } + + unsafe fn grow_zeroed( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + unsafe { self.delegate_target().grow_zeroed(ptr, old_layout, new_layout) } + } + + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + unsafe { self.delegate_target().shrink(ptr, old_layout, new_layout) } + } +} diff --git a/pkgs/edit/src/arena/mod.rs b/pkgs/edit/src/arena/mod.rs new file mode 100644 index 0000000..a7099c7 --- /dev/null +++ b/pkgs/edit/src/arena/mod.rs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Arena allocators. Small and fast. + +#[cfg(debug_assertions)] +mod debug; +mod release; +mod scratch; +mod string; + +#[cfg(all(not(doc), debug_assertions))] +pub use self::debug::Arena; +#[cfg(any(doc, not(debug_assertions)))] +pub use self::release::Arena; +pub use self::scratch::{ScratchArena, init, scratch_arena}; +pub use self::string::ArenaString; diff --git a/pkgs/edit/src/arena/release.rs b/pkgs/edit/src/arena/release.rs new file mode 100644 index 0000000..dec572c --- /dev/null +++ b/pkgs/edit/src/arena/release.rs @@ -0,0 +1,278 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#![allow(clippy::mut_from_ref)] + +use std::alloc::{AllocError, Allocator, Layout}; +use std::cell::Cell; +use std::hint::cold_path; +use std::mem::MaybeUninit; +use std::ptr::{self, NonNull}; +use std::{mem, slice}; + +use crate::helpers::*; +use crate::{apperr, sys}; + +const ALLOC_CHUNK_SIZE: usize = 64 * KIBI; + +/// An arena allocator. +/// +/// If you have never used an arena allocator before, think of it as +/// allocating objects on the stack, but the stack is *really* big. +/// Each time you allocate, memory gets pushed at the end of the stack, +/// each time you deallocate, memory gets popped from the end of the stack. +/// +/// One reason you'd want to use this is obviously performance: It's very simple +/// and so it's also very fast, >10x faster than your system allocator. +/// +/// However, modern allocators such as `mimalloc` are just as fast, so why not use them? +/// Because their performance comes at the cost of binary size and we can't have that. +/// +/// The biggest benefit though is that it sometimes massively simplifies lifetime +/// and memory management. This can best be seen by this project's UI code, which +/// uses an arena to allocate a tree of UI nodes. This is infameously difficult +/// to do in Rust, but not so when you got an arena allocator: +/// All nodes have the same lifetime, so you can just use references. +/// +/// # Safety +/// +/// **Do not** push objects into the arena that require destructors. +/// Destructors are not executed. Use a pool allocator for that. +pub struct Arena { + base: NonNull, + capacity: usize, + commit: Cell, + offset: Cell, + + /// See [`super::debug`], which uses this for borrow tracking. + #[cfg(debug_assertions)] + pub(super) borrows: Cell, +} + +impl Arena { + pub const fn empty() -> Self { + Self { + base: NonNull::dangling(), + capacity: 0, + commit: Cell::new(0), + offset: Cell::new(0), + + #[cfg(debug_assertions)] + borrows: Cell::new(0), + } + } + + pub fn new(capacity: usize) -> apperr::Result { + let capacity = (capacity.max(1) + ALLOC_CHUNK_SIZE - 1) & !(ALLOC_CHUNK_SIZE - 1); + let base = unsafe { sys::virtual_reserve(capacity)? }; + + Ok(Arena { + base, + capacity, + commit: Cell::new(0), + offset: Cell::new(0), + + #[cfg(debug_assertions)] + borrows: Cell::new(0), + }) + } + + pub fn offset(&self) -> usize { + self.offset.get() + } + + /// "Deallocates" the memory in the arena down to the given offset. + /// + /// # Safety + /// + /// Obviously, this is GIGA UNSAFE. It runs no destructors and does not check + /// whether the offset is valid. You better take care when using this function. + pub unsafe fn reset(&self, to: usize) { + // Fill the deallocated memory with 0xDD to aid debugging. + if cfg!(debug_assertions) && self.offset.get() > to { + let commit = self.commit.get(); + let len = (self.offset.get() + 128).min(commit) - to; + unsafe { slice::from_raw_parts_mut(self.base.add(to).as_ptr(), len).fill(0xDD) }; + } + + self.offset.replace(to); + } + + #[inline] + pub(super) fn alloc_raw( + &self, + bytes: usize, + alignment: usize, + ) -> Result, AllocError> { + let commit = self.commit.get(); + let offset = self.offset.get(); + + let beg = (offset + alignment - 1) & !(alignment - 1); + let end = beg + bytes; + + if end > commit { + return self.alloc_raw_bump(beg, end); + } + + if cfg!(debug_assertions) { + let ptr = unsafe { self.base.add(offset) }; + let len = (end + 128).min(self.commit.get()) - offset; + unsafe { slice::from_raw_parts_mut(ptr.as_ptr(), len).fill(0xCD) }; + } + + self.offset.replace(end); + Ok(unsafe { NonNull::slice_from_raw_parts(self.base.add(beg), bytes) }) + } + + // With the code in `alloc_raw_bump()` out of the way, `alloc_raw()` compiles down to some super tight assembly. + #[cold] + fn alloc_raw_bump(&self, beg: usize, end: usize) -> Result, AllocError> { + let offset = self.offset.get(); + let commit_old = self.commit.get(); + let commit_new = (end + ALLOC_CHUNK_SIZE - 1) & !(ALLOC_CHUNK_SIZE - 1); + + if commit_new > self.capacity + || unsafe { + sys::virtual_commit(self.base.add(commit_old), commit_new - commit_old).is_err() + } + { + return Err(AllocError); + } + + if cfg!(debug_assertions) { + let ptr = unsafe { self.base.add(offset) }; + let len = (end + 128).min(self.commit.get()) - offset; + unsafe { slice::from_raw_parts_mut(ptr.as_ptr(), len).fill(0xCD) }; + } + + self.commit.replace(commit_new); + self.offset.replace(end); + Ok(unsafe { NonNull::slice_from_raw_parts(self.base.add(beg), end - beg) }) + } + + #[allow(clippy::mut_from_ref)] + pub fn alloc_uninit(&self) -> &mut MaybeUninit { + let bytes = mem::size_of::(); + let alignment = mem::align_of::(); + let ptr = self.alloc_raw(bytes, alignment).unwrap(); + unsafe { ptr.cast().as_mut() } + } + + #[allow(clippy::mut_from_ref)] + pub fn alloc_uninit_slice(&self, count: usize) -> &mut [MaybeUninit] { + let bytes = mem::size_of::() * count; + let alignment = mem::align_of::(); + let ptr = self.alloc_raw(bytes, alignment).unwrap(); + unsafe { slice::from_raw_parts_mut(ptr.cast().as_ptr(), count) } + } +} + +impl Drop for Arena { + fn drop(&mut self) { + if self.base != NonNull::dangling() { + unsafe { sys::virtual_release(self.base, self.capacity) }; + } + } +} + +impl Default for Arena { + fn default() -> Self { + Arena::empty() + } +} + +unsafe impl Allocator for Arena { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + self.alloc_raw(layout.size(), layout.align()) + } + + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + let p = self.alloc_raw(layout.size(), layout.align())?; + unsafe { p.cast::().as_ptr().write_bytes(0, p.len()) } + Ok(p) + } + + // While it is possible to shrink the tail end of the arena, it is + // not very useful given the existence of scoped scratch arenas. + unsafe fn deallocate(&self, _: NonNull, _: Layout) {} + + unsafe fn grow( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!(new_layout.size() >= old_layout.size()); + debug_assert!(new_layout.align() <= old_layout.align()); + + let new_ptr; + + // Growing the given area is possible if it is at the end of the arena. + if unsafe { ptr.add(old_layout.size()) == self.base.add(self.offset.get()) } { + new_ptr = ptr; + let delta = new_layout.size() - old_layout.size(); + // Assuming that the given ptr/length area is at the end of the arena, + // we can just push more memory to the end of the arena to grow it. + self.alloc_raw(delta, 1)?; + } else { + cold_path(); + + new_ptr = self.allocate(new_layout)?.cast(); + + // SAFETY: It's weird to me that this doesn't assert new_layout.size() >= old_layout.size(), + // but neither does the stdlib code at the time of writing. + // So, assuming that is not needed, this code is safe since it just copies the old data over. + unsafe { + ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_ptr(), old_layout.size()); + self.deallocate(ptr, old_layout); + } + } + + Ok(NonNull::slice_from_raw_parts(new_ptr, new_layout.size())) + } + + unsafe fn grow_zeroed( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + unsafe { + // SAFETY: Same as grow(). + let ptr = self.grow(ptr, old_layout, new_layout)?; + + // SAFETY: At this point, `ptr` must be valid for `new_layout.size()` bytes, + // allowing us to safely zero out the delta since `old_layout.size()`. + ptr.cast::() + .add(old_layout.size()) + .write_bytes(0, new_layout.size() - old_layout.size()); + + Ok(ptr) + } + } + + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!(new_layout.size() <= old_layout.size()); + debug_assert!(new_layout.align() <= old_layout.align()); + + let mut len = old_layout.size(); + + // Shrinking the given area is possible if it is at the end of the arena. + if unsafe { ptr.add(len) == self.base.add(self.offset.get()) } { + self.offset.set(self.offset.get() - len + new_layout.size()); + len = new_layout.size(); + } else { + debug_assert!( + false, + "Did you call shrink_to_fit()? Only the last allocation can be shrunk!" + ); + } + + Ok(NonNull::slice_from_raw_parts(ptr, len)) + } +} diff --git a/pkgs/edit/src/arena/scratch.rs b/pkgs/edit/src/arena/scratch.rs new file mode 100644 index 0000000..a719db4 --- /dev/null +++ b/pkgs/edit/src/arena/scratch.rs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::ops::Deref; + +#[cfg(debug_assertions)] +use super::debug; +use super::{Arena, release}; +use crate::apperr; +use crate::helpers::*; + +static mut S_SCRATCH: [release::Arena; 2] = + const { [release::Arena::empty(), release::Arena::empty()] }; + +/// Initialize the scratch arenas with a given capacity. +/// Call this before using [`scratch_arena`]. +pub fn init(capacity: usize) -> apperr::Result<()> { + unsafe { + for s in &mut S_SCRATCH[..] { + *s = release::Arena::new(capacity)?; + } + } + Ok(()) +} + +/// Need an arena for temporary allocations? [`scratch_arena`] got you covered. +/// Call [`scratch_arena`] and it'll return an [`Arena`] that resets when it goes out of scope. +/// +/// --- +/// +/// Most methods make just two kinds of allocations: +/// * Interior: Temporary data that can be deallocated when the function returns. +/// * Exterior: Data that is returned to the caller and must remain alive until the caller stops using it. +/// +/// Such methods only have two lifetimes, for which you consequently also only need two arenas. +/// ...even if your method calls other methods recursively! This is because the exterior allocations +/// of a callee are simply interior allocations to the caller, and so on, recursively. +/// +/// This works as long as the two arenas flip/flop between being used as interior/exterior allocator +/// along the callstack. To ensure that is the case, we use a recursion counter in debug builds. +/// +/// This approach was described among others at: +/// +/// # Safety +/// +/// If your function takes an [`Arena`] argument, you **MUST** pass it to `scratch_arena` as `Some(&arena)`. +pub fn scratch_arena(conflict: Option<&Arena>) -> ScratchArena<'static> { + unsafe { + #[cfg(debug_assertions)] + let conflict = conflict.map(|a| a.delegate_target_unchecked()); + + let index = opt_ptr_eq(conflict, Some(&S_SCRATCH[0])) as usize; + let arena = &mut S_SCRATCH[index]; + ScratchArena::new(arena) + } +} + +/// Borrows an [`Arena`] for temporary allocations. +/// +/// See [`scratch_arena`]. +#[cfg(debug_assertions)] +pub struct ScratchArena<'a> { + arena: debug::Arena, + offset: usize, + _phantom: std::marker::PhantomData<&'a ()>, +} + +#[cfg(not(debug_assertions))] +pub struct ScratchArena<'a> { + arena: &'a Arena, + offset: usize, +} + +#[cfg(debug_assertions)] +impl<'a> ScratchArena<'a> { + fn new(arena: &'a release::Arena) -> Self { + let offset = arena.offset(); + ScratchArena { arena: Arena::delegated(arena), _phantom: std::marker::PhantomData, offset } + } +} + +#[cfg(not(debug_assertions))] +impl<'a> ScratchArena<'a> { + fn new(arena: &'a release::Arena) -> Self { + let offset = arena.offset(); + ScratchArena { arena, offset } + } +} + +impl Drop for ScratchArena<'_> { + fn drop(&mut self) { + unsafe { self.arena.reset(self.offset) }; + } +} + +#[cfg(debug_assertions)] +impl Deref for ScratchArena<'_> { + type Target = debug::Arena; + + fn deref(&self) -> &Self::Target { + &self.arena + } +} + +#[cfg(not(debug_assertions))] +impl Deref for ScratchArena<'_> { + type Target = Arena; + + fn deref(&self) -> &Self::Target { + self.arena + } +} diff --git a/pkgs/edit/src/arena/string.rs b/pkgs/edit/src/arena/string.rs new file mode 100644 index 0000000..ff70ec1 --- /dev/null +++ b/pkgs/edit/src/arena/string.rs @@ -0,0 +1,281 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::fmt; +use std::ops::{Bound, Deref, DerefMut, RangeBounds}; + +use super::Arena; +use crate::helpers::*; + +/// A custom string type, because `std` lacks allocator support for [`String`]. +/// +/// To keep things simple, this one is hardcoded to [`Arena`]. +#[derive(Clone)] +pub struct ArenaString<'a> { + vec: Vec, +} + +impl<'a> ArenaString<'a> { + /// Creates a new [`ArenaString`] in the given arena. + #[must_use] + pub const fn new_in(arena: &'a Arena) -> Self { + Self { vec: Vec::new_in(arena) } + } + + #[must_use] + pub fn with_capacity_in(capacity: usize, arena: &'a Arena) -> Self { + Self { vec: Vec::with_capacity_in(capacity, arena) } + } + + /// Turns a [`str`] into an [`ArenaString`]. + #[must_use] + pub fn from_str(arena: &'a Arena, s: &str) -> Self { + let mut res = Self::new_in(arena); + res.push_str(s); + res + } + + /// It says right here that you checked if `bytes` is valid UTF-8 + /// and you are sure it is. Presto! Here's an `ArenaString`! + /// + /// # Safety + /// + /// You fool! It says "unchecked" right there. Now the house is burning. + #[inline] + #[must_use] + pub unsafe fn from_utf8_unchecked(bytes: Vec) -> Self { + Self { vec: bytes } + } + + /// Checks whether `text` contains only valid UTF-8. + /// If the entire string is valid, it returns `Ok(text)`. + /// Otherwise, it returns `Err(ArenaString)` with all invalid sequences replaced with U+FFFD. + pub fn from_utf8_lossy<'s>( + arena: &'a Arena, + text: &'s [u8], + ) -> Result<&'s str, ArenaString<'a>> { + let mut iter = text.utf8_chunks(); + let Some(mut chunk) = iter.next() else { + return Ok(""); + }; + + let valid = chunk.valid(); + if chunk.invalid().is_empty() { + debug_assert_eq!(valid.len(), text.len()); + return Ok(unsafe { str::from_utf8_unchecked(text) }); + } + + const REPLACEMENT: &str = "\u{FFFD}"; + + let mut res = Self::new_in(arena); + res.reserve(text.len()); + + loop { + res.push_str(chunk.valid()); + if !chunk.invalid().is_empty() { + res.push_str(REPLACEMENT); + } + chunk = match iter.next() { + Some(chunk) => chunk, + None => break, + }; + } + + Err(res) + } + + /// Turns a [`Vec`] into an [`ArenaString`], replacing invalid UTF-8 sequences with U+FFFD. + #[must_use] + pub fn from_utf8_lossy_owned(v: Vec) -> Self { + match Self::from_utf8_lossy(v.allocator(), &v) { + Ok(..) => unsafe { Self::from_utf8_unchecked(v) }, + Err(s) => s, + } + } + + /// It's empty. + pub fn is_empty(&self) -> bool { + self.vec.is_empty() + } + + /// It's lengthy. + pub fn len(&self) -> usize { + self.vec.len() + } + + /// It's capacatity. + pub fn capacity(&self) -> usize { + self.vec.capacity() + } + + /// It's a [`String`], now it's a [`str`]. Wow! + pub fn as_str(&self) -> &str { + unsafe { str::from_utf8_unchecked(self.vec.as_slice()) } + } + + /// It's a [`String`], now it's a [`str`]. And it's mutable! WOW! + pub fn as_mut_str(&mut self) -> &mut str { + unsafe { str::from_utf8_unchecked_mut(self.vec.as_mut_slice()) } + } + + /// Now it's bytes! + pub fn as_bytes(&self) -> &[u8] { + self.vec.as_slice() + } + + /// Returns a mutable reference to the contents of this `String`. + /// + /// # Safety + /// + /// The underlying `&mut Vec` allows writing bytes which are not valid UTF-8. + pub unsafe fn as_mut_vec(&mut self) -> &mut Vec { + &mut self.vec + } + + /// Reserves *additional* memory. For you old folks out there (totally not me), + /// this is differrent from C++'s `reserve` which reserves a total size. + pub fn reserve(&mut self, additional: usize) { + self.vec.reserve(additional) + } + + /// Just like [`ArenaString::reserve`], but it doesn't overallocate. + pub fn reserve_exact(&mut self, additional: usize) { + self.vec.reserve_exact(additional) + } + + /// Now it's small! Alarming! + /// + /// *Do not* call this unless this string is the last thing on the arena. + /// Arenas are stacks, they can't deallocate what's in the middle. + pub fn shrink_to_fit(&mut self) { + self.vec.shrink_to_fit() + } + + /// To no surprise, this clears the string. + pub fn clear(&mut self) { + self.vec.clear() + } + + /// Append some text. + pub fn push_str(&mut self, string: &str) { + self.vec.extend_from_slice(string.as_bytes()) + } + + /// Append a single character. + #[inline] + pub fn push(&mut self, ch: char) { + match ch.len_utf8() { + 1 => self.vec.push(ch as u8), + _ => self.vec.extend_from_slice(ch.encode_utf8(&mut [0; 4]).as_bytes()), + } + } + + /// Same as `push(char)` but with a specified number of character copies. + /// Shockingly absent from the standard library. + pub fn push_repeat(&mut self, ch: char, total_copies: usize) { + if total_copies == 0 { + return; + } + + let buf = unsafe { self.as_mut_vec() }; + + if ch.is_ascii() { + // Compiles down to `memset()`. + buf.extend(std::iter::repeat_n(ch as u8, total_copies)); + } else { + // Implements efficient string padding using quadratic duplication. + let mut utf8_buf = [0; 4]; + let utf8 = ch.encode_utf8(&mut utf8_buf).as_bytes(); + let initial_len = buf.len(); + let added_len = utf8.len() * total_copies; + let final_len = initial_len + added_len; + + buf.reserve(added_len); + buf.extend_from_slice(utf8); + + while buf.len() != final_len { + let end = (final_len - buf.len() + initial_len).min(buf.len()); + buf.extend_from_within(initial_len..end); + } + } + } + + /// Replaces a range of characters with a new string. + pub fn replace_range>(&mut self, range: R, replace_with: &str) { + match range.start_bound() { + Bound::Included(&n) => assert!(self.is_char_boundary(n)), + Bound::Excluded(&n) => assert!(self.is_char_boundary(n + 1)), + Bound::Unbounded => {} + }; + match range.end_bound() { + Bound::Included(&n) => assert!(self.is_char_boundary(n + 1)), + Bound::Excluded(&n) => assert!(self.is_char_boundary(n)), + Bound::Unbounded => {} + }; + unsafe { self.as_mut_vec() }.replace_range(range, replace_with.as_bytes()); + } + + /// Finds `old` in the string and replaces it with `new`. + /// Only performs one replacement. + pub fn replace_once_in_place(&mut self, old: &str, new: &str) { + if let Some(beg) = self.find(old) { + unsafe { self.as_mut_vec() }.replace_range(beg..beg + old.len(), new.as_bytes()); + } + } +} + +impl fmt::Debug for ArenaString<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +impl PartialEq<&str> for ArenaString<'_> { + fn eq(&self, other: &&str) -> bool { + self.as_str() == *other + } +} + +impl Deref for ArenaString<'_> { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.as_str() + } +} + +impl DerefMut for ArenaString<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.as_mut_str() + } +} + +impl fmt::Display for ArenaString<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl fmt::Write for ArenaString<'_> { + #[inline] + fn write_str(&mut self, s: &str) -> fmt::Result { + self.push_str(s); + Ok(()) + } + + #[inline] + fn write_char(&mut self, c: char) -> fmt::Result { + self.push(c); + Ok(()) + } +} + +#[macro_export] +macro_rules! arena_format { + ($arena:expr, $($arg:tt)*) => {{ + use std::fmt::Write as _; + let mut output = $crate::arena::ArenaString::new_in($arena); + output.write_fmt(format_args!($($arg)*)).unwrap(); + output + }} +} diff --git a/pkgs/edit/src/base64.rs b/pkgs/edit/src/base64.rs new file mode 100644 index 0000000..bce13c6 --- /dev/null +++ b/pkgs/edit/src/base64.rs @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Base64 facilities. + +use crate::arena::ArenaString; + +const CHARSET: [u8; 64] = *b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +/// One aspect of base64 is that the encoded length can be +/// calculated accurately in advance, which is what this returns. +#[inline] +pub fn encode_len(src_len: usize) -> usize { + src_len.div_ceil(3) * 4 +} + +/// Encodes the given bytes as base64 and appends them to the destination string. +pub fn encode(dst: &mut ArenaString, src: &[u8]) { + unsafe { + let mut inp = src.as_ptr(); + let mut remaining = src.len(); + let dst = dst.as_mut_vec(); + + let out_len = encode_len(src.len()); + // ... we can then use this fact to reserve space all at once. + dst.reserve(out_len); + + // SAFETY: Getting a pointer to the reserved space is only safe + // *after* calling `reserve()` as it may change the pointer. + let mut out = dst.as_mut_ptr().add(dst.len()); + + if remaining != 0 { + // Translate chunks of 3 source bytes into 4 base64-encoded bytes. + while remaining > 3 { + // SAFETY: Thanks to `remaining > 3`, reading 4 bytes at once is safe. + // This improves performance massively over a byte-by-byte approach, + // because it allows us to byte-swap the read and use simple bit-shifts below. + let val = u32::from_be((inp as *const u32).read_unaligned()); + inp = inp.add(3); + remaining -= 3; + + *out = CHARSET[(val >> 26) as usize]; + out = out.add(1); + *out = CHARSET[(val >> 20) as usize & 0x3f]; + out = out.add(1); + *out = CHARSET[(val >> 14) as usize & 0x3f]; + out = out.add(1); + *out = CHARSET[(val >> 8) as usize & 0x3f]; + out = out.add(1); + } + + // Convert the remaining 1-3 bytes. + let mut in1 = 0; + let mut in2 = 0; + + // We can simplify the following logic by assuming that there's only 1 + // byte left. If there's >1 byte left, these two '=' will be overwritten. + *out.add(3) = b'='; + *out.add(2) = b'='; + + if remaining >= 3 { + in2 = inp.add(2).read() as usize; + *out.add(3) = CHARSET[in2 & 0x3f]; + } + + if remaining >= 2 { + in1 = inp.add(1).read() as usize; + *out.add(2) = CHARSET[(in1 << 2 | in2 >> 6) & 0x3f]; + } + + let in0 = inp.add(0).read() as usize; + *out.add(1) = CHARSET[(in0 << 4 | in1 >> 4) & 0x3f]; + *out.add(0) = CHARSET[in0 >> 2]; + } + + dst.set_len(dst.len() + out_len); + } +} + +#[cfg(test)] +mod tests { + use super::encode; + use crate::arena::{Arena, ArenaString}; + + #[test] + fn test_basic() { + let arena = Arena::new(4 * 1024).unwrap(); + let enc = |s: &[u8]| { + let mut dst = ArenaString::new_in(&arena); + encode(&mut dst, s); + dst + }; + assert_eq!(enc(b""), ""); + assert_eq!(enc(b"a"), "YQ=="); + assert_eq!(enc(b"ab"), "YWI="); + assert_eq!(enc(b"abc"), "YWJj"); + assert_eq!(enc(b"abcd"), "YWJjZA=="); + assert_eq!(enc(b"abcde"), "YWJjZGU="); + assert_eq!(enc(b"abcdef"), "YWJjZGVm"); + assert_eq!(enc(b"abcdefg"), "YWJjZGVmZw=="); + assert_eq!(enc(b"abcdefgh"), "YWJjZGVmZ2g="); + assert_eq!(enc(b"abcdefghi"), "YWJjZGVmZ2hp"); + assert_eq!(enc(b"abcdefghij"), "YWJjZGVmZ2hpag=="); + assert_eq!(enc(b"abcdefghijk"), "YWJjZGVmZ2hpams="); + assert_eq!(enc(b"abcdefghijkl"), "YWJjZGVmZ2hpamts"); + assert_eq!(enc(b"abcdefghijklm"), "YWJjZGVmZ2hpamtsbQ=="); + assert_eq!(enc(b"abcdefghijklmN"), "YWJjZGVmZ2hpamtsbU4="); + assert_eq!(enc(b"abcdefghijklmNO"), "YWJjZGVmZ2hpamtsbU5P"); + assert_eq!(enc(b"abcdefghijklmNOP"), "YWJjZGVmZ2hpamtsbU5PUA=="); + assert_eq!(enc(b"abcdefghijklmNOPQ"), "YWJjZGVmZ2hpamtsbU5PUFE="); + assert_eq!(enc(b"abcdefghijklmNOPQR"), "YWJjZGVmZ2hpamtsbU5PUFFS"); + assert_eq!(enc(b"abcdefghijklmNOPQRS"), "YWJjZGVmZ2hpamtsbU5PUFFSUw=="); + assert_eq!(enc(b"abcdefghijklmNOPQRST"), "YWJjZGVmZ2hpamtsbU5PUFFSU1Q="); + assert_eq!(enc(b"abcdefghijklmNOPQRSTU"), "YWJjZGVmZ2hpamtsbU5PUFFSU1RV"); + assert_eq!(enc(b"abcdefghijklmNOPQRSTUV"), "YWJjZGVmZ2hpamtsbU5PUFFSU1RVVg=="); + assert_eq!(enc(b"abcdefghijklmNOPQRSTUVW"), "YWJjZGVmZ2hpamtsbU5PUFFSU1RVVlc="); + assert_eq!(enc(b"abcdefghijklmNOPQRSTUVWX"), "YWJjZGVmZ2hpamtsbU5PUFFSU1RVVldY"); + assert_eq!(enc(b"abcdefghijklmNOPQRSTUVWXY"), "YWJjZGVmZ2hpamtsbU5PUFFSU1RVVldYWQ=="); + assert_eq!(enc(b"abcdefghijklmNOPQRSTUVWXYZ"), "YWJjZGVmZ2hpamtsbU5PUFFSU1RVVldYWVo="); + } +} diff --git a/pkgs/edit/src/bin/edit/documents.rs b/pkgs/edit/src/bin/edit/documents.rs new file mode 100644 index 0000000..349f4f3 --- /dev/null +++ b/pkgs/edit/src/bin/edit/documents.rs @@ -0,0 +1,296 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::collections::LinkedList; +use std::ffi::OsStr; +use std::fs::File; +use std::path::{Path, PathBuf}; + +use edit::buffer::{RcTextBuffer, TextBuffer}; +use edit::helpers::{CoordType, Point}; +use edit::simd::memrchr2; +use edit::{apperr, path, sys}; + +use crate::state::DisplayablePathBuf; + +pub struct Document { + pub buffer: RcTextBuffer, + pub path: Option, + pub dir: Option, + pub filename: String, + pub file_id: Option, + pub new_file_counter: usize, +} + +impl Document { + pub fn save(&mut self, new_path: Option) -> apperr::Result<()> { + let path = new_path.as_deref().unwrap_or_else(|| self.path.as_ref().unwrap().as_path()); + let mut file = DocumentManager::open_for_writing(path)?; + + { + let mut tb = self.buffer.borrow_mut(); + tb.write_file(&mut file)?; + } + + if let Ok(id) = sys::file_id(&file) { + self.file_id = Some(id); + } + + if let Some(path) = new_path { + self.set_path(path); + } + + Ok(()) + } + + pub fn reread(&mut self, encoding: Option<&'static str>) -> apperr::Result<()> { + let path = self.path.as_ref().unwrap().as_path(); + let mut file = DocumentManager::open_for_reading(path)?; + + { + let mut tb = self.buffer.borrow_mut(); + tb.read_file(&mut file, encoding)?; + } + + if let Ok(id) = sys::file_id(&file) { + self.file_id = Some(id); + } + + Ok(()) + } + + fn set_path(&mut self, path: PathBuf) { + let filename = path.file_name().unwrap_or_default().to_string_lossy().into_owned(); + let dir = path.parent().map(ToOwned::to_owned).unwrap_or_default(); + self.filename = filename; + self.dir = Some(DisplayablePathBuf::new(dir)); + self.path = Some(path); + self.update_file_mode(); + } + + fn update_file_mode(&mut self) { + let mut tb = self.buffer.borrow_mut(); + tb.set_ruler(if self.filename == "COMMIT_EDITMSG" { 72 } else { 0 }); + } +} + +#[derive(Default)] +pub struct DocumentManager { + list: LinkedList, +} + +impl DocumentManager { + #[inline] + pub fn len(&self) -> usize { + self.list.len() + } + + #[inline] + pub fn active(&self) -> Option<&Document> { + self.list.front() + } + + #[inline] + pub fn active_mut(&mut self) -> Option<&mut Document> { + self.list.front_mut() + } + + #[inline] + pub fn update_active bool>(&mut self, mut func: F) -> bool { + let mut cursor = self.list.cursor_front_mut(); + while let Some(doc) = cursor.current() { + if func(doc) { + let list = cursor.remove_current_as_list().unwrap(); + self.list.cursor_front_mut().splice_before(list); + return true; + } + cursor.move_next(); + } + false + } + + pub fn remove_active(&mut self) { + self.list.pop_front(); + } + + pub fn add_untitled(&mut self) -> apperr::Result<&mut Document> { + let buffer = TextBuffer::new_rc(false)?; + { + let mut tb = buffer.borrow_mut(); + tb.set_margin_enabled(true); + tb.set_line_highlight_enabled(true); + } + + let mut doc = Document { + buffer, + path: None, + dir: Default::default(), + filename: Default::default(), + file_id: None, + new_file_counter: 0, + }; + self.gen_untitled_name(&mut doc); + + self.list.push_front(doc); + Ok(self.list.front_mut().unwrap()) + } + + pub fn gen_untitled_name(&self, doc: &mut Document) { + let mut new_file_counter = 0; + for doc in &self.list { + new_file_counter = new_file_counter.max(doc.new_file_counter); + } + new_file_counter += 1; + + doc.filename = format!("Untitled-{new_file_counter}.txt"); + doc.new_file_counter = new_file_counter; + } + + pub fn add_file_path(&mut self, path: &Path) -> apperr::Result<&mut Document> { + let (path, goto) = Self::parse_filename_goto(path); + let path = path::normalize(path); + + let mut file = match Self::open_for_reading(&path) { + Ok(file) => Some(file), + Err(err) if sys::apperr_is_not_found(err) => None, + Err(err) => return Err(err), + }; + + let file_id = match &file { + Some(file) => Some(sys::file_id(file)?), + None => None, + }; + + // Check if the file is already open. + if file_id.is_some() && self.update_active(|doc| doc.file_id == file_id) { + let doc = self.active_mut().unwrap(); + if let Some(goto) = goto { + doc.buffer.borrow_mut().cursor_move_to_logical(goto); + } + return Ok(doc); + } + + let buffer = TextBuffer::new_rc(false)?; + { + let mut tb = buffer.borrow_mut(); + tb.set_margin_enabled(true); + tb.set_line_highlight_enabled(true); + + if let Some(file) = &mut file { + tb.read_file(file, None)?; + + if let Some(goto) = goto + && goto != Default::default() + { + tb.cursor_move_to_logical(goto); + } + } + } + + let mut doc = Document { + buffer, + path: None, + dir: None, + filename: Default::default(), + file_id, + new_file_counter: 0, + }; + doc.set_path(path); + + self.list.push_front(doc); + Ok(self.list.front_mut().unwrap()) + } + + pub fn open_for_reading(path: &Path) -> apperr::Result { + File::open(path).map_err(apperr::Error::from) + } + + pub fn open_for_writing(path: &Path) -> apperr::Result { + File::create(path).map_err(apperr::Error::from) + } + + // Parse a filename in the form of "filename:line:char". + // Returns the position of the first colon and the line/char coordinates. + fn parse_filename_goto(path: &Path) -> (&Path, Option) { + fn parse(s: &[u8]) -> Option { + if s.is_empty() { + return None; + } + + let mut num: CoordType = 0; + for &b in s { + if !b.is_ascii_digit() { + return None; + } + let digit = (b - b'0') as CoordType; + num = num.checked_mul(10)?.checked_add(digit)?; + } + Some(num) + } + + let bytes = path.as_os_str().as_encoded_bytes(); + let colend = match memrchr2(b':', b':', bytes, bytes.len()) { + // Reject filenames that would result in an empty filename after stripping off the :line:char suffix. + // For instance, a filename like ":123:456" will not be processed by this function. + Some(colend) if colend > 0 => colend, + _ => return (path, None), + }; + + let last = match parse(&bytes[colend + 1..]) { + Some(last) => last, + None => return (path, None), + }; + let last = (last - 1).max(0); + let mut len = colend; + let mut goto = Point { x: 0, y: last }; + + if let Some(colbeg) = memrchr2(b':', b':', bytes, colend) { + // Same here: Don't allow empty filenames. + if colbeg != 0 + && let Some(first) = parse(&bytes[colbeg + 1..colend]) + { + let first = (first - 1).max(0); + len = colbeg; + goto = Point { x: last, y: first }; + } + } + + // Strip off the :line:char suffix. + let path = &bytes[..len]; + let path = unsafe { OsStr::from_encoded_bytes_unchecked(path) }; + let path = Path::new(path); + (path, Some(goto)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_last_numbers() { + fn parse(s: &str) -> (&str, Option) { + let (p, g) = DocumentManager::parse_filename_goto(Path::new(s)); + (p.to_str().unwrap(), g) + } + + assert_eq!(parse("123"), ("123", None)); + assert_eq!(parse("abc"), ("abc", None)); + assert_eq!(parse(":123"), (":123", None)); + assert_eq!(parse("abc:123"), ("abc", Some(Point { x: 0, y: 122 }))); + assert_eq!(parse("45:123"), ("45", Some(Point { x: 0, y: 122 }))); + assert_eq!(parse(":45:123"), (":45", Some(Point { x: 0, y: 122 }))); + assert_eq!(parse("abc:45:123"), ("abc", Some(Point { x: 122, y: 44 }))); + assert_eq!(parse("abc:def:123"), ("abc:def", Some(Point { x: 0, y: 122 }))); + assert_eq!(parse("1:2:3"), ("1", Some(Point { x: 2, y: 1 }))); + assert_eq!(parse("::3"), (":", Some(Point { x: 0, y: 2 }))); + assert_eq!(parse("1::3"), ("1:", Some(Point { x: 0, y: 2 }))); + assert_eq!(parse(""), ("", None)); + assert_eq!(parse(":"), (":", None)); + assert_eq!(parse("::"), ("::", None)); + assert_eq!(parse("a:1"), ("a", Some(Point { x: 0, y: 0 }))); + assert_eq!(parse("1:a"), ("1:a", None)); + assert_eq!(parse("file.txt:10"), ("file.txt", Some(Point { x: 0, y: 9 }))); + assert_eq!(parse("file.txt:10:5"), ("file.txt", Some(Point { x: 4, y: 9 }))); + } +} diff --git a/pkgs/edit/src/bin/edit/draw_editor.rs b/pkgs/edit/src/bin/edit/draw_editor.rs new file mode 100644 index 0000000..bdab25a --- /dev/null +++ b/pkgs/edit/src/bin/edit/draw_editor.rs @@ -0,0 +1,273 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use edit::framebuffer::IndexedColor; +use edit::helpers::*; +use edit::icu; +use edit::input::{kbmod, vk}; +use edit::tui::*; + +use crate::localization::*; +use crate::state::*; + +pub fn draw_editor(ctx: &mut Context, state: &mut State) { + if !matches!(state.wants_search.kind, StateSearchKind::Hidden | StateSearchKind::Disabled) { + draw_search(ctx, state); + } + + let size = ctx.size(); + // TODO: The layout code should be able to just figure out the height on its own. + let height_reduction = match state.wants_search.kind { + StateSearchKind::Search => 4, + StateSearchKind::Replace => 5, + _ => 2, + }; + + if let Some(doc) = state.documents.active() { + ctx.textarea("textarea", doc.buffer.clone()); + ctx.inherit_focus(); + } else { + ctx.block_begin("empty"); + ctx.block_end(); + } + + ctx.attr_intrinsic_size(Size { width: 0, height: size.height - height_reduction }); +} + +fn draw_search(ctx: &mut Context, state: &mut State) { + enum SearchAction { + None, + Search, + Replace, + ReplaceAll, + } + + if let Err(err) = icu::init() { + error_log_add(ctx, state, err); + state.wants_search.kind = StateSearchKind::Disabled; + return; + } + + let Some(doc) = state.documents.active() else { + state.wants_search.kind = StateSearchKind::Hidden; + return; + }; + + let mut action = SearchAction::None; + let mut focus = StateSearchKind::Hidden; + + if state.wants_search.focus { + state.wants_search.focus = false; + focus = StateSearchKind::Search; + + // If the selection is empty, focus the search input field. + // Otherwise, focus the replace input field, if it exists. + if let Some(selection) = doc.buffer.borrow_mut().extract_user_selection(false) { + state.search_needle = String::from_utf8_lossy_owned(selection); + focus = state.wants_search.kind; + } + } + + ctx.block_begin("search"); + ctx.attr_focus_well(); + ctx.attr_background_rgba(ctx.indexed(IndexedColor::White)); + ctx.attr_foreground_rgba(ctx.indexed(IndexedColor::Black)); + { + if ctx.contains_focus() && ctx.consume_shortcut(vk::ESCAPE) { + state.wants_search.kind = StateSearchKind::Hidden; + } + + ctx.table_begin("needle"); + ctx.table_set_cell_gap(Size { width: 1, height: 0 }); + { + { + ctx.table_next_row(); + ctx.label("label", loc(LocId::SearchNeedleLabel)); + + if ctx.editline("needle", &mut state.search_needle) { + action = SearchAction::Search; + } + if !state.search_success { + ctx.attr_background_rgba(ctx.indexed(IndexedColor::Red)); + ctx.attr_foreground_rgba(ctx.indexed(IndexedColor::BrightWhite)); + } + ctx.attr_intrinsic_size(Size { width: COORD_TYPE_SAFE_MAX, height: 1 }); + if focus == StateSearchKind::Search { + ctx.steal_focus(); + } + if ctx.is_focused() && ctx.consume_shortcut(vk::RETURN) { + action = SearchAction::Search; + } + } + + if state.wants_search.kind == StateSearchKind::Replace { + ctx.table_next_row(); + ctx.label("label", loc(LocId::SearchReplacementLabel)); + + ctx.editline("replacement", &mut state.search_replacement); + ctx.attr_intrinsic_size(Size { width: COORD_TYPE_SAFE_MAX, height: 1 }); + if focus == StateSearchKind::Replace { + ctx.steal_focus(); + } + if ctx.is_focused() { + if ctx.consume_shortcut(vk::RETURN) { + action = SearchAction::Replace; + } else if ctx.consume_shortcut(kbmod::CTRL_ALT | vk::RETURN) { + action = SearchAction::ReplaceAll; + } + } + } + } + ctx.table_end(); + + ctx.table_begin("options"); + ctx.table_set_cell_gap(Size { width: 2, height: 0 }); + { + ctx.table_next_row(); + + let mut change = false; + change |= ctx.checkbox( + "match-case", + loc(LocId::SearchMatchCase), + &mut state.search_options.match_case, + ); + change |= ctx.checkbox( + "whole-word", + loc(LocId::SearchWholeWord), + &mut state.search_options.whole_word, + ); + change |= ctx.checkbox( + "use-regex", + loc(LocId::SearchUseRegex), + &mut state.search_options.use_regex, + ); + if change { + action = SearchAction::Search; + state.wants_search.focus = true; + ctx.needs_rerender(); + } + + if state.wants_search.kind == StateSearchKind::Replace + && ctx.button("replace-all", loc(LocId::SearchReplaceAll)) + { + action = SearchAction::ReplaceAll; + } + + if ctx.button("close", loc(LocId::SearchClose)) { + state.wants_search.kind = StateSearchKind::Hidden; + } + } + ctx.table_end(); + } + ctx.block_end(); + + state.search_success = match action { + SearchAction::None => return, + SearchAction::Search => { + doc.buffer.borrow_mut().find_and_select(&state.search_needle, state.search_options) + } + SearchAction::Replace => doc.buffer.borrow_mut().find_and_replace( + &state.search_needle, + state.search_options, + &state.search_replacement, + ), + SearchAction::ReplaceAll => doc.buffer.borrow_mut().find_and_replace_all( + &state.search_needle, + state.search_options, + &state.search_replacement, + ), + } + .is_ok(); + + ctx.needs_rerender(); +} + +pub fn draw_handle_save(ctx: &mut Context, state: &mut State) { + if let Some(doc) = state.documents.active_mut() { + if doc.path.is_some() { + if let Err(err) = doc.save(None) { + error_log_add(ctx, state, err); + } + } else { + // No path? Show the file picker. + state.wants_file_picker = StateFilePicker::SaveAs; + state.wants_save = false; + ctx.needs_rerender(); + } + } + + state.wants_save = false; +} + +pub fn draw_handle_wants_close(ctx: &mut Context, state: &mut State) { + let Some(doc) = state.documents.active() else { + state.wants_close = false; + state.wants_exit = true; + return; + }; + + if !doc.buffer.borrow().is_dirty() { + state.documents.remove_active(); + state.wants_close = false; + ctx.needs_rerender(); + return; + } + + enum Action { + None, + Save, + Discard, + Cancel, + } + let mut action = Action::None; + + ctx.modal_begin("unsaved-changes", loc(LocId::UnsavedChangesDialogTitle)); + ctx.attr_background_rgba(ctx.indexed(IndexedColor::Red)); + ctx.attr_foreground_rgba(ctx.indexed(IndexedColor::BrightWhite)); + { + ctx.label("description", loc(LocId::UnsavedChangesDialogDescription)); + ctx.attr_padding(Rect::three(1, 2, 1)); + + ctx.table_begin("choices"); + ctx.inherit_focus(); + ctx.attr_padding(Rect::three(0, 2, 1)); + ctx.attr_position(Position::Center); + ctx.table_set_cell_gap(Size { width: 2, height: 0 }); + { + ctx.table_next_row(); + ctx.inherit_focus(); + + if ctx.button("yes", loc(LocId::UnsavedChangesDialogYes)) { + action = Action::Save; + } + ctx.inherit_focus(); + if ctx.button("no", loc(LocId::UnsavedChangesDialogNo)) { + action = Action::Discard; + } + if ctx.button("cancel", loc(LocId::Cancel)) { + action = Action::Cancel; + } + + // TODO: This should highlight the corresponding letter in the label. + if ctx.consume_shortcut(vk::S) { + action = Action::Save; + } else if ctx.consume_shortcut(vk::N) { + action = Action::Discard; + } + } + ctx.table_end(); + } + if ctx.modal_end() { + action = Action::Cancel; + } + + match action { + Action::None => return, + Action::Save => state.wants_save = true, + Action::Discard => state.documents.remove_active(), + Action::Cancel => state.wants_exit = false, + } + + state.wants_close = false; + ctx.toss_focus_up(); +} diff --git a/pkgs/edit/src/bin/edit/draw_filepicker.rs b/pkgs/edit/src/bin/edit/draw_filepicker.rs new file mode 100644 index 0000000..79733d1 --- /dev/null +++ b/pkgs/edit/src/bin/edit/draw_filepicker.rs @@ -0,0 +1,258 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::cmp::Ordering; +use std::fs; +use std::path::PathBuf; + +use edit::framebuffer::IndexedColor; +use edit::helpers::*; +use edit::input::vk; +use edit::tui::*; +use edit::{icu, path, sys}; + +use crate::localization::*; +use crate::state::*; + +pub fn draw_file_picker(ctx: &mut Context, state: &mut State) { + // The save dialog is pre-filled with the current document filename. + if state.wants_file_picker == StateFilePicker::SaveAs { + state.wants_file_picker = StateFilePicker::SaveAsShown; + + if state.file_picker_pending_name.as_os_str().is_empty() { + state.file_picker_pending_name = + state.documents.active().map_or("Untitled.txt", |doc| doc.filename.as_str()).into(); + } + } + + let width = (ctx.size().width - 20).max(10); + let height = (ctx.size().height - 10).max(10); + let mut doit = None; + let mut done = false; + + ctx.modal_begin( + "file-picker", + if state.wants_file_picker == StateFilePicker::Open { + loc(LocId::FileOpen) + } else { + loc(LocId::FileSaveAs) + }, + ); + ctx.attr_intrinsic_size(Size { width, height }); + { + let mut activated = false; + + ctx.table_begin("path"); + ctx.table_set_columns(&[0, COORD_TYPE_SAFE_MAX]); + ctx.table_set_cell_gap(Size { width: 1, height: 0 }); + ctx.attr_padding(Rect::two(1, 1)); + ctx.inherit_focus(); + { + ctx.table_next_row(); + + ctx.label("dir-label", loc(LocId::SaveAsDialogPathLabel)); + ctx.label("dir", state.file_picker_pending_dir.as_str()); + ctx.attr_overflow(Overflow::TruncateMiddle); + + ctx.table_next_row(); + ctx.inherit_focus(); + + ctx.label("name-label", loc(LocId::SaveAsDialogNameLabel)); + ctx.editline("name", &mut state.file_picker_pending_name); + ctx.inherit_focus(); + if ctx.is_focused() && ctx.consume_shortcut(vk::RETURN) { + activated = true; + } + } + ctx.table_end(); + + if state.file_picker_entries.is_none() { + draw_dialog_saveas_refresh_files(state); + } + + let files = state.file_picker_entries.as_ref().unwrap(); + + ctx.scrollarea_begin( + "directory", + Size { + width: 0, + // -1 for the label (top) + // -1 for the label (bottom) + // -1 for the editline (bottom) + height: height - 3, + }, + ); + ctx.attr_background_rgba(ctx.indexed_alpha(IndexedColor::Black, 1, 4)); + ctx.next_block_id_mixin(state.file_picker_pending_dir.as_str().len() as u64); + { + ctx.list_begin("files"); + ctx.inherit_focus(); + for entry in files { + match ctx + .list_item(state.file_picker_pending_name == entry.as_path(), entry.as_str()) + { + ListSelection::Unchanged => {} + ListSelection::Selected => { + state.file_picker_pending_name = entry.as_path().into() + } + ListSelection::Activated => activated = true, + } + ctx.attr_overflow(Overflow::TruncateMiddle); + } + ctx.list_end(); + + if ctx.contains_focus() && ctx.consume_shortcut(vk::BACK) { + state.file_picker_pending_name = "..".into(); + activated = true; + } + } + ctx.scrollarea_end(); + + if activated { + doit = draw_file_picker_update_path(state); + + // Check if the file already exists and show an overwrite warning in that case. + if state.wants_file_picker != StateFilePicker::Open + && let Some(path) = doit.as_deref() + && let Some(doc) = state.documents.active() + && let Some(file_id) = &doc.file_id + && sys::file_id_at(path).is_ok_and(|id| &id == file_id) + { + state.file_picker_overwrite_warning = doit.take(); + } + } + } + if ctx.modal_end() { + done = true; + } + + if state.file_picker_overwrite_warning.is_some() { + let mut save; + + ctx.modal_begin("overwrite", loc(LocId::FileOverwriteWarning)); + ctx.attr_background_rgba(ctx.indexed(IndexedColor::Red)); + ctx.attr_foreground_rgba(ctx.indexed(IndexedColor::BrightWhite)); + { + ctx.label("description", loc(LocId::FileOverwriteWarningDescription)); + ctx.attr_overflow(Overflow::TruncateTail); + ctx.attr_padding(Rect::three(1, 2, 1)); + + ctx.table_begin("choices"); + ctx.inherit_focus(); + ctx.attr_padding(Rect::three(0, 2, 1)); + ctx.attr_position(Position::Center); + ctx.table_set_cell_gap(Size { width: 2, height: 0 }); + { + ctx.table_next_row(); + ctx.inherit_focus(); + + save = ctx.button("yes", loc(LocId::Yes)); + ctx.inherit_focus(); + + if ctx.button("no", loc(LocId::No)) { + state.file_picker_overwrite_warning = None; + } + } + ctx.table_end(); + + save |= ctx.consume_shortcut(vk::Y); + if ctx.consume_shortcut(vk::N) { + state.file_picker_overwrite_warning = None; + } + } + if ctx.modal_end() { + state.file_picker_overwrite_warning = None; + } + + if save { + doit = state.file_picker_overwrite_warning.take(); + } + } + + if let Some(path) = doit { + let res = if state.wants_file_picker == StateFilePicker::Open { + state.documents.add_file_path(&path).map(|_| ()) + } else if let Some(doc) = state.documents.active_mut() { + doc.save(Some(path)) + } else { + Ok(()) + }; + match res { + Ok(..) => { + ctx.needs_rerender(); + done = true; + } + Err(err) => error_log_add(ctx, state, err), + } + } + + if done { + state.wants_file_picker = StateFilePicker::None; + state.file_picker_pending_name = Default::default(); + state.file_picker_entries = Default::default(); + state.file_picker_overwrite_warning = Default::default(); + } +} + +// Returns Some(path) if the path refers to a file. +fn draw_file_picker_update_path(state: &mut State) -> Option { + let path = state.file_picker_pending_dir.as_path(); + let path = path.join(&state.file_picker_pending_name); + let path = path::normalize(&path); + + let (dir, name) = if path.is_dir() { + (path.as_path(), PathBuf::new()) + } else { + let dir = path.parent().unwrap_or(&path); + let name = path.file_name().map_or(Default::default(), |s| s.into()); + (dir, name) + }; + if dir != state.file_picker_pending_dir.as_path() { + state.file_picker_pending_dir = DisplayablePathBuf::new(dir.to_path_buf()); + state.file_picker_entries = None; + } + + state.file_picker_pending_name = name; + if state.file_picker_pending_name.as_os_str().is_empty() { None } else { Some(path) } +} + +fn draw_dialog_saveas_refresh_files(state: &mut State) { + let dir = state.file_picker_pending_dir.as_path(); + let mut files = Vec::new(); + + if dir.parent().is_some() { + files.push(DisplayablePathBuf::from("..")); + } + + if let Ok(iter) = fs::read_dir(dir) { + for entry in iter.flatten() { + if let Ok(metadata) = entry.metadata() { + let mut name = entry.file_name(); + if metadata.is_dir() + || (metadata.is_symlink() + && fs::metadata(entry.path()).is_ok_and(|m| m.is_dir())) + { + name.push("/"); + } + files.push(DisplayablePathBuf::from(name)); + } + } + } + + // Sort directories first, then by name, case-insensitive. + let off = files.len().saturating_sub(1); + files[off..].sort_by(|a, b| { + let a = a.as_bytes(); + let b = b.as_bytes(); + + let a_is_dir = a.last() == Some(&b'/'); + let b_is_dir = b.last() == Some(&b'/'); + + match b_is_dir.cmp(&a_is_dir) { + Ordering::Equal => icu::compare_strings(a, b), + other => other, + } + }); + + state.file_picker_entries = Some(files); +} diff --git a/pkgs/edit/src/bin/edit/draw_menubar.rs b/pkgs/edit/src/bin/edit/draw_menubar.rs new file mode 100644 index 0000000..95176c3 --- /dev/null +++ b/pkgs/edit/src/bin/edit/draw_menubar.rs @@ -0,0 +1,161 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use edit::arena_format; +use edit::helpers::*; +use edit::input::{kbmod, vk}; +use edit::tui::*; + +use crate::localization::*; +use crate::state::*; + +pub fn draw_menubar(ctx: &mut Context, state: &mut State) { + ctx.menubar_begin(); + ctx.attr_background_rgba(state.menubar_color_bg); + ctx.attr_foreground_rgba(state.menubar_color_fg); + { + if ctx.menubar_menu_begin(loc(LocId::File), 'F') { + draw_menu_file(ctx, state); + } + if state.documents.active().is_some() && ctx.menubar_menu_begin(loc(LocId::Edit), 'E') { + draw_menu_edit(ctx, state); + } + if ctx.menubar_menu_begin(loc(LocId::View), 'V') { + draw_menu_view(ctx, state); + } + if ctx.menubar_menu_begin(loc(LocId::Help), 'H') { + draw_menu_help(ctx, state); + } + } + ctx.menubar_end(); +} + +fn draw_menu_file(ctx: &mut Context, state: &mut State) { + if ctx.menubar_menu_button(loc(LocId::FileNew), 'N', kbmod::CTRL | vk::N) { + draw_add_untitled_document(ctx, state); + } + if ctx.menubar_menu_button(loc(LocId::FileOpen), 'O', kbmod::CTRL | vk::O) { + state.wants_file_picker = StateFilePicker::Open; + } + if state.documents.active().is_some() { + if ctx.menubar_menu_button(loc(LocId::FileSave), 'S', kbmod::CTRL | vk::S) { + state.wants_save = true; + } + if ctx.menubar_menu_button(loc(LocId::FileSaveAs), 'A', vk::NULL) { + state.wants_file_picker = StateFilePicker::SaveAs; + } + } + if ctx.menubar_menu_button(loc(LocId::FileClose), 'C', kbmod::CTRL | vk::W) { + state.wants_close = true; + } + if ctx.menubar_menu_button(loc(LocId::FileExit), 'X', kbmod::CTRL | vk::Q) { + state.wants_exit = true; + } + ctx.menubar_menu_end(); +} + +fn draw_menu_edit(ctx: &mut Context, state: &mut State) { + let doc = state.documents.active().unwrap(); + let mut tb = doc.buffer.borrow_mut(); + + if ctx.menubar_menu_button(loc(LocId::EditUndo), 'U', kbmod::CTRL | vk::Z) { + tb.undo(); + ctx.needs_rerender(); + } + if ctx.menubar_menu_button(loc(LocId::EditRedo), 'R', kbmod::CTRL | vk::Y) { + tb.redo(); + ctx.needs_rerender(); + } + if ctx.menubar_menu_button(loc(LocId::EditCut), 'T', kbmod::CTRL | vk::X) { + ctx.set_clipboard(tb.extract_selection(true)); + } + if ctx.menubar_menu_button(loc(LocId::EditCopy), 'C', kbmod::CTRL | vk::C) { + ctx.set_clipboard(tb.extract_selection(false)); + } + if ctx.menubar_menu_button(loc(LocId::EditPaste), 'P', kbmod::CTRL | vk::V) { + tb.write(ctx.clipboard(), true); + ctx.needs_rerender(); + } + if state.wants_search.kind != StateSearchKind::Disabled { + if ctx.menubar_menu_button(loc(LocId::EditFind), 'F', kbmod::CTRL | vk::F) { + state.wants_search.kind = StateSearchKind::Search; + state.wants_search.focus = true; + } + if ctx.menubar_menu_button(loc(LocId::EditReplace), 'R', kbmod::CTRL | vk::R) { + state.wants_search.kind = StateSearchKind::Replace; + state.wants_search.focus = true; + } + } + ctx.menubar_menu_end(); +} + +fn draw_menu_view(ctx: &mut Context, state: &mut State) { + if ctx.menubar_menu_button(loc(LocId::ViewFocusStatusbar), 'S', vk::NULL) { + state.wants_statusbar_focus = true; + } + + if let Some(doc) = state.documents.active() { + let mut tb = doc.buffer.borrow_mut(); + let word_wrap = tb.is_word_wrap_enabled(); + + if ctx.menubar_menu_checkbox(loc(LocId::ViewWordWrap), 'W', kbmod::ALT | vk::Z, word_wrap) { + tb.set_word_wrap(!word_wrap); + ctx.needs_rerender(); + } + } + + ctx.menubar_menu_end(); +} + +fn draw_menu_help(ctx: &mut Context, state: &mut State) { + if ctx.menubar_menu_button(loc(LocId::HelpAbout), 'A', vk::NULL) { + state.wants_about = true; + } + ctx.menubar_menu_end(); +} + +pub fn draw_dialog_about(ctx: &mut Context, state: &mut State) { + ctx.modal_begin("about", loc(LocId::AboutDialogTitle)); + { + ctx.block_begin("content"); + ctx.inherit_focus(); + ctx.attr_padding(Rect::three(1, 2, 1)); + { + ctx.label("description", "Microsoft Edit"); + ctx.attr_overflow(Overflow::TruncateTail); + ctx.attr_position(Position::Center); + + ctx.label( + "version", + &arena_format!( + ctx.arena(), + "{}{}", + loc(LocId::AboutDialogVersion), + env!("CARGO_PKG_VERSION") + ), + ); + ctx.attr_overflow(Overflow::TruncateHead); + ctx.attr_position(Position::Center); + + ctx.label("copyright", "Copyright (c) Microsoft Corp 2025"); + ctx.attr_overflow(Overflow::TruncateTail); + ctx.attr_position(Position::Center); + + ctx.block_begin("choices"); + ctx.inherit_focus(); + ctx.attr_padding(Rect::three(1, 2, 0)); + ctx.attr_position(Position::Center); + { + if ctx.button("ok", loc(LocId::Ok)) { + state.wants_about = false; + } + ctx.inherit_focus(); + } + ctx.block_end(); + } + ctx.block_end(); + } + if ctx.modal_end() { + state.wants_about = false; + } +} diff --git a/pkgs/edit/src/bin/edit/draw_statusbar.rs b/pkgs/edit/src/bin/edit/draw_statusbar.rs new file mode 100644 index 0000000..ecbc5e4 --- /dev/null +++ b/pkgs/edit/src/bin/edit/draw_statusbar.rs @@ -0,0 +1,291 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use edit::framebuffer::{Attributes, IndexedColor}; +use edit::helpers::*; +use edit::input::vk; +use edit::tui::*; +use edit::{arena_format, icu}; + +use crate::localization::*; +use crate::state::*; + +pub fn draw_statusbar(ctx: &mut Context, state: &mut State) { + ctx.table_begin("statusbar"); + ctx.attr_focus_well(); + ctx.attr_background_rgba(state.menubar_color_bg); + ctx.attr_foreground_rgba(state.menubar_color_fg); + ctx.table_set_cell_gap(Size { width: 2, height: 0 }); + ctx.attr_intrinsic_size(Size { width: COORD_TYPE_SAFE_MAX, height: 1 }); + ctx.attr_padding(Rect::two(0, 1)); + + if let Some(doc) = state.documents.active() { + let mut tb = doc.buffer.borrow_mut(); + + ctx.table_next_row(); + + if ctx.button("newline", if tb.is_crlf() { "CRLF" } else { "LF" }) { + let is_crlf = tb.is_crlf(); + tb.normalize_newlines(!is_crlf); + } + if state.wants_statusbar_focus { + state.wants_statusbar_focus = false; + ctx.steal_focus(); + } + + state.wants_encoding_picker |= ctx.button("encoding", tb.encoding()); + if state.wants_encoding_picker { + if doc.path.is_some() { + ctx.block_begin("frame"); + ctx.attr_float(FloatSpec { + anchor: Anchor::Last, + gravity_x: 0.0, + gravity_y: 1.0, + offset_x: 0.0, + offset_y: 0.0, + }); + ctx.attr_padding(Rect::two(0, 1)); + ctx.attr_border(); + { + if ctx.button("reopen", loc(LocId::EncodingReopen)) { + state.wants_encoding_change = StateEncodingChange::Reopen; + } + ctx.focus_on_first_present(); + if ctx.button("convert", loc(LocId::EncodingConvert)) { + state.wants_encoding_change = StateEncodingChange::Convert; + } + } + ctx.block_end(); + } else { + // Can't reopen a file that doesn't exist. + state.wants_encoding_change = StateEncodingChange::Convert; + } + + if !ctx.contains_focus() { + state.wants_encoding_picker = false; + ctx.needs_rerender(); + } + } + + state.wants_indentation_picker |= ctx.button( + "indentation", + &arena_format!( + ctx.arena(), + "{}:{}", + loc(if tb.indent_with_tabs() { + LocId::IndentationTabs + } else { + LocId::IndentationSpaces + }), + tb.tab_size(), + ), + ); + if state.wants_indentation_picker { + ctx.table_begin("indentation-picker"); + ctx.attr_float(FloatSpec { + anchor: Anchor::Last, + gravity_x: 0.0, + gravity_y: 1.0, + offset_x: 0.0, + offset_y: 0.0, + }); + ctx.attr_border(); + ctx.attr_padding(Rect::two(0, 1)); + ctx.table_set_cell_gap(Size { width: 1, height: 0 }); + { + if ctx.consume_shortcut(vk::RETURN) { + ctx.toss_focus_up(); + } + + ctx.table_next_row(); + + ctx.list_begin("type"); + ctx.focus_on_first_present(); + ctx.attr_padding(Rect::two(0, 1)); + { + if ctx.list_item(tb.indent_with_tabs(), loc(LocId::IndentationTabs)) + != ListSelection::Unchanged + { + tb.set_indent_with_tabs(true); + ctx.needs_rerender(); + } + if ctx.list_item(!tb.indent_with_tabs(), loc(LocId::IndentationSpaces)) + != ListSelection::Unchanged + { + tb.set_indent_with_tabs(false); + ctx.needs_rerender(); + } + } + ctx.list_end(); + + ctx.list_begin("width"); + ctx.attr_padding(Rect::two(0, 2)); + { + for width in 1u8..=8 { + let ch = [b'0' + width]; + let label = unsafe { std::str::from_utf8_unchecked(&ch) }; + + if ctx.list_item(tb.tab_size() == width as CoordType, label) + != ListSelection::Unchanged + { + tb.set_tab_size(width as CoordType); + ctx.needs_rerender(); + } + } + } + ctx.list_end(); + } + ctx.table_end(); + + if !ctx.contains_focus() { + state.wants_indentation_picker = false; + ctx.needs_rerender(); + } + } + + ctx.label( + "location", + &arena_format!( + ctx.arena(), + "{}:{}", + tb.cursor_logical_pos().y + 1, + tb.cursor_logical_pos().x + 1 + ), + ); + + #[cfg(any(feature = "debug-layout", feature = "debug-latency"))] + ctx.label( + "stats", + &arena_format!(ctx.arena(), "{}/{}", tb.logical_line_count(), tb.visual_line_count(),), + ); + + if tb.is_overtype() && ctx.button("overtype", "OVR") { + tb.set_overtype(false); + ctx.needs_rerender(); + } + + if tb.is_dirty() { + ctx.label("dirty", "*"); + } + + ctx.block_begin("filename-container"); + ctx.attr_intrinsic_size(Size { width: COORD_TYPE_SAFE_MAX, height: 1 }); + { + let total = state.documents.len(); + let mut filename = doc.filename.as_str(); + let filename_buf; + + if total > 1 { + filename_buf = arena_format!(ctx.arena(), "{} + {}", filename, total - 1); + filename = &filename_buf; + } + + state.wants_document_picker |= ctx.button("filename", filename); + ctx.inherit_focus(); + ctx.attr_overflow(Overflow::TruncateMiddle); + ctx.attr_position(Position::Right); + } + ctx.block_end(); + } + + ctx.table_end(); +} + +pub fn draw_dialog_encoding_change(ctx: &mut Context, state: &mut State) { + let doc = state.documents.active_mut().unwrap(); + let reopen = state.wants_encoding_change == StateEncodingChange::Reopen; + let width = (ctx.size().width - 20).max(10); + let height = (ctx.size().height - 10).max(10); + let mut change = None; + + ctx.modal_begin( + "encode", + if reopen { loc(LocId::EncodingReopen) } else { loc(LocId::EncodingConvert) }, + ); + { + ctx.scrollarea_begin("scrollarea", Size { width, height }); + ctx.attr_background_rgba(ctx.indexed_alpha(IndexedColor::Black, 1, 4)); + ctx.inherit_focus(); + { + let encodings = icu::get_available_encodings(); + + ctx.list_begin("encodings"); + ctx.inherit_focus(); + for &encoding in encodings { + if ctx.list_item(encoding == doc.buffer.borrow().encoding(), encoding) + == ListSelection::Activated + { + change = Some(encoding); + break; + } + } + ctx.list_end(); + } + ctx.scrollarea_end(); + } + if ctx.modal_end() { + state.wants_encoding_change = StateEncodingChange::None; + } + + if let Some(encoding) = change { + if reopen && doc.path.is_some() { + let mut res = Ok(()); + if doc.buffer.borrow().is_dirty() { + res = doc.save(None); + } + if res.is_ok() { + res = doc.reread(Some(encoding)); + } + if let Err(err) = res { + error_log_add(ctx, state, err); + } + } else { + doc.buffer.borrow_mut().set_encoding(encoding); + } + + state.wants_encoding_change = StateEncodingChange::None; + ctx.needs_rerender(); + } +} + +pub fn draw_document_picker(ctx: &mut Context, state: &mut State) { + ctx.modal_begin("document-picker", ""); + { + let width = (ctx.size().width - 20).max(10); + let height = (ctx.size().height - 10).max(10); + + ctx.scrollarea_begin("scrollarea", Size { width, height }); + ctx.attr_background_rgba(ctx.indexed_alpha(IndexedColor::Black, 1, 4)); + ctx.inherit_focus(); + { + ctx.list_begin("documents"); + ctx.inherit_focus(); + + if state.documents.update_active(|doc| { + let tb = doc.buffer.borrow(); + + ctx.styled_list_item_begin(); + ctx.attr_overflow(Overflow::TruncateTail); + ctx.styled_label_add_text(if tb.is_dirty() { "* " } else { " " }); + ctx.styled_label_add_text(&doc.filename); + + if let Some(path) = &doc.dir { + ctx.styled_label_add_text(" "); + ctx.styled_label_set_attributes(Attributes::Italic); + ctx.styled_label_add_text(path.as_str()); + } + + ctx.styled_list_item_end(false) == ListSelection::Activated + }) { + state.wants_document_picker = false; + ctx.needs_rerender(); + } + + ctx.list_end(); + } + ctx.scrollarea_end(); + } + if ctx.modal_end() { + state.wants_document_picker = false; + } +} diff --git a/pkgs/edit/src/bin/edit/edit.exe.manifest b/pkgs/edit/src/bin/edit/edit.exe.manifest new file mode 100644 index 0000000..b972407 --- /dev/null +++ b/pkgs/edit/src/bin/edit/edit.exe.manifest @@ -0,0 +1,22 @@ + + + + + true + UTF-8 + SegmentHeap + + + + + + + + diff --git a/pkgs/edit/src/bin/edit/localization.rs b/pkgs/edit/src/bin/edit/localization.rs new file mode 100644 index 0000000..115f888 --- /dev/null +++ b/pkgs/edit/src/bin/edit/localization.rs @@ -0,0 +1,950 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use edit::arena::scratch_arena; +use edit::sys; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum LocId { + Ctrl, + Alt, + Shift, + + Ok, + Yes, + No, + Cancel, + Always, + + // File menu + File, + FileNew, + FileOpen, + FileSave, + FileSaveAs, + FileClose, + FileExit, + + // Edit menu + Edit, + EditUndo, + EditRedo, + EditCut, + EditCopy, + EditPaste, + EditFind, + EditReplace, + + // View menu + View, + ViewFocusStatusbar, + ViewWordWrap, + + // Help menu + Help, + HelpAbout, + + // Exit dialog + UnsavedChangesDialogTitle, + UnsavedChangesDialogDescription, + UnsavedChangesDialogYes, + UnsavedChangesDialogNo, + + // About dialog + AboutDialogTitle, + AboutDialogVersion, + + // Shown when the clipboard size exceeds the limit for OSC 52 + LargeClipboardWarningLine1, + LargeClipboardWarningLine2, + LargeClipboardWarningLine3, + SuperLargeClipboardWarning, + + // Warning dialog + WarningDialogTitle, + + // Error dialog + ErrorDialogTitle, + ErrorIcuMissing, + + SearchNeedleLabel, + SearchReplacementLabel, + SearchMatchCase, + SearchWholeWord, + SearchUseRegex, + SearchReplaceAll, + SearchClose, + + EncodingReopen, + EncodingConvert, + + IndentationTabs, + IndentationSpaces, + + SaveAsDialogPathLabel, + SaveAsDialogNameLabel, + + FileOverwriteWarning, + FileOverwriteWarningDescription, + + Count, +} + +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, PartialEq, Eq)] +enum LangId { + // Base language. It's always the first one. + en, + + // Other languages. Sorted alphabetically. + de, + es, + fr, + it, + ja, + ko, + pt_br, + ru, + zh_hans, + zh_hant, + + Count, +} + +#[rustfmt::skip] +const S_LANG_LUT: [[&str; LangId::Count as usize]; LocId::Count as usize] = [ + // Ctrl (the keyboard key) + [ + /* en */ "Ctrl", + /* de */ "Strg", + /* es */ "Ctrl", + /* fr */ "Ctrl", + /* it */ "Ctrl", + /* ja */ "Ctrl", + /* ko */ "Ctrl", + /* pt_br */ "Ctrl", + /* ru */ "Ctrl", + /* zh_hans */ "Ctrl", + /* zh_hant */ "Ctrl", + ], + // Alt (the keyboard key) + [ + /* en */ "Alt", + /* de */ "Alt", + /* es */ "Alt", + /* fr */ "Alt", + /* it */ "Alt", + /* ja */ "Alt", + /* ko */ "Alt", + /* pt_br */ "Alt", + /* ru */ "Alt", + /* zh_hans */ "Alt", + /* zh_hant */ "Alt", + ], + // Shift (the keyboard key) + [ + /* en */ "Shift", + /* de */ "Umschalt", + /* es */ "Mayús", + /* fr */ "Maj", + /* it */ "Maiusc", + /* ja */ "Shift", + /* ko */ "Shift", + /* pt_br */ "Shift", + /* ru */ "Shift", + /* zh_hans */ "Shift", + /* zh_hant */ "Shift", + ], + + // Ok (used as a common dialog button) + [ + /* en */ "Ok", + /* de */ "OK", + /* es */ "Aceptar", + /* fr */ "OK", + /* it */ "OK", + /* ja */ "OK", + /* ko */ "확인", + /* pt_br */ "OK", + /* ru */ "ОК", + /* zh_hans */ "确定", + /* zh_hant */ "確定", + ], + // Yes (used as a common dialog button) + [ + /* en */ "Yes", + /* de */ "Ja", + /* es */ "Sí", + /* fr */ "Oui", + /* it */ "Sì", + /* ja */ "はい", + /* ko */ "예", + /* pt_br */ "Sim", + /* ru */ "Да", + /* zh_hans */ "是", + /* zh_hant */ "是", + ], + // No (used as a common dialog button) + [ + /* en */ "No", + /* de */ "Nein", + /* es */ "No", + /* fr */ "Non", + /* it */ "No", + /* ja */ "いいえ", + /* ko */ "아니오", + /* pt_br */ "Não", + /* ru */ "Нет", + /* zh_hans */ "否", + /* zh_hant */ "否", + ], + // Cancel (used as a common dialog button) + [ + /* en */ "Cancel", + /* de */ "Abbrechen", + /* es */ "Cancelar", + /* fr */ "Annuler", + /* it */ "Annulla", + /* ja */ "キャンセル", + /* ko */ "취소", + /* pt_br */ "Cancelar", + /* ru */ "Отмена", + /* zh_hans */ "取消", + /* zh_hant */ "取消", + ], + // Always (used as a common dialog button) + [ + /* en */ "Always", + /* de */ "Immer", + /* es */ "Siempre", + /* fr */ "Toujours", + /* it */ "Sempre", + /* ja */ "常に", + /* ko */ "항상", + /* pt_br */ "Sempre", + /* ru */ "Всегда", + /* zh_hans */ "总是", + /* zh_hant */ "總是", + ], + + // File (a menu bar item) + [ + /* en */ "File", + /* de */ "Datei", + /* es */ "Archivo", + /* fr */ "Fichier", + /* it */ "File", + /* ja */ "ファイル", + /* ko */ "파일", + /* pt_br */ "Arquivo", + /* ru */ "Файл", + /* zh_hans */ "文件", + /* zh_hant */ "檔案", + ], + // FileNew + [ + /* en */ "New File…", + /* de */ "Neue Datei…", + /* es */ "Nuevo archivo…", + /* fr */ "Nouveau fichier…", + /* it */ "Nuovo file…", + /* ja */ "新規ファイル…", + /* ko */ "새 파일…", + /* pt_br */ "Novo arquivo…", + /* ru */ "Новый файл…", + /* zh_hans */ "新建文件…", + /* zh_hant */ "新增檔案…", + ], + // FileOpen + [ + /* en */ "Open File…", + /* de */ "Datei öffnen…", + /* es */ "Abrir archivo…", + /* fr */ "Ouvrir un fichier…", + /* it */ "Apri file…", + /* ja */ "ファイルを開く…", + /* ko */ "파일 열기…", + /* pt_br */ "Abrir arquivo…", + /* ru */ "Открыть файл…", + /* zh_hans */ "打开文件…", + /* zh_hant */ "開啟檔案…", + ], + // FileSave + [ + /* en */ "Save", + /* de */ "Speichern", + /* es */ "Guardar", + /* fr */ "Enregistrer", + /* it */ "Salva", + /* ja */ "保存", + /* ko */ "저장", + /* pt_br */ "Salvar", + /* ru */ "Сохранить", + /* zh_hans */ "保存", + /* zh_hant */ "儲存", + ], + // FileSaveAs + [ + /* en */ "Save As…", + /* de */ "Speichern unter…", + /* es */ "Guardar como…", + /* fr */ "Enregistrer sous…", + /* it */ "Salva come…", + /* ja */ "名前を付けて保存…", + /* ko */ "다른 이름으로 저장…", + /* pt_br */ "Salvar como…", + /* ru */ "Сохранить как…", + /* zh_hans */ "另存为…", + /* zh_hant */ "另存新檔…", + ], + // FileClose + [ + /* en */ "Close Editor", + /* de */ "Editor schließen", + /* es */ "Cerrar editor", + /* fr */ "Fermer l'éditeur", + /* it */ "Chiudi editor", + /* ja */ "エディターを閉じる", + /* ko */ "편집기 닫기", + /* pt_br */ "Fechar editor", + /* ru */ "Закрыть редактор", + /* zh_hans */ "关闭编辑器", + /* zh_hant */ "關閉編輯器", + ], + // FileExit + [ + /* en */ "Exit", + /* de */ "Beenden", + /* es */ "Salir", + /* fr */ "Quitter", + /* it */ "Esci", + /* ja */ "終了", + /* ko */ "종료", + /* pt_br */ "Sair", + /* ru */ "Выход", + /* zh_hans */ "退出", + /* zh_hant */ "退出", + ], + + // Edit (a menu bar item) + [ + /* en */ "Edit", + /* de */ "Bearbeiten", + /* es */ "Editar", + /* fr */ "Éditer", + /* it */ "Modifica", + /* ja */ "編集", + /* ko */ "편집", + /* pt_br */ "Editar", + /* ru */ "Правка", + /* zh_hans */ "编辑", + /* zh_hant */ "編輯", + ], + // EditUndo + [ + /* en */ "Undo", + /* de */ "Rückgängig", + /* es */ "Deshacer", + /* fr */ "Annuler", + /* it */ "Annulla", + /* ja */ "元に戻す", + /* ko */ "실행 취소", + /* pt_br */ "Desfazer", + /* ru */ "Отменить", + /* zh_hans */ "撤销", + /* zh_hant */ "復原", + ], + // EditRedo + [ + /* en */ "Redo", + /* de */ "Wiederholen", + /* es */ "Rehacer", + /* fr */ "Rétablir", + /* it */ "Ripeti", + /* ja */ "やり直し", + /* ko */ "다시 실행", + /* pt_br */ "Refazer", + /* ru */ "Повторить", + /* zh_hans */ "重做", + /* zh_hant */ "重做", + ], + // EditCut + [ + /* en */ "Cut", + /* de */ "Ausschneiden", + /* es */ "Cortar", + /* fr */ "Couper", + /* it */ "Taglia", + /* ja */ "切り取り", + /* ko */ "잘라내기", + /* pt_br */ "Cortar", + /* ru */ "Вырезать", + /* zh_hans */ "剪切", + /* zh_hant */ "剪下", + ], + // EditCopy + [ + /* en */ "Copy", + /* de */ "Kopieren", + /* es */ "Copiar", + /* fr */ "Copier", + /* it */ "Copia", + /* ja */ "コピー", + /* ko */ "복사", + /* pt_br */ "Copiar", + /* ru */ "Копировать", + /* zh_hans */ "复制", + /* zh_hant */ "複製", + ], + // EditPaste + [ + /* en */ "Paste", + /* de */ "Einfügen", + /* es */ "Pegar", + /* fr */ "Coller", + /* it */ "Incolla", + /* ja */ "貼り付け", + /* ko */ "붙여넣기", + /* pt_br */ "Colar", + /* ru */ "Вставить", + /* zh_hans */ "粘贴", + /* zh_hant */ "貼上", + ], + // EditFind + [ + /* en */ "Find", + /* de */ "Suchen", + /* es */ "Buscar", + /* fr */ "Rechercher", + /* it */ "Trova", + /* ja */ "検索", + /* ko */ "찾기", + /* pt_br */ "Encontrar", + /* ru */ "Найти", + /* zh_hans */ "查找", + /* zh_hant */ "尋找", + ], + // EditReplace + [ + /* en */ "Replace", + /* de */ "Ersetzen", + /* es */ "Reemplazar", + /* fr */ "Remplacer", + /* it */ "Sostituisci", + /* ja */ "置換", + /* ko */ "바꾸기", + /* pt_br */ "Substituir", + /* ru */ "Заменить", + /* zh_hans */ "替换", + /* zh_hant */ "取代", + ], + + // View (a menu bar item) + [ + /* en */ "View", + /* de */ "Ansicht", + /* es */ "Ver", + /* fr */ "Affichage", + /* it */ "Visualizza", + /* ja */ "表示", + /* ko */ "보기", + /* pt_br */ "Exibir", + /* ru */ "Вид", + /* zh_hans */ "视图", + /* zh_hant */ "檢視", + ], + // ViewFocusStatusbar + [ + /* en */ "Focus Statusbar", + /* de */ "Statusleiste fokussieren", + /* es */ "Enfocar barra de estado", + /* fr */ "Focus sur la barre d'état", + /* it */ "Attiva barra di stato", + /* ja */ "ステータスバーにフォーカス", + /* ko */ "상태 표시줄로 포커스 이동", + /* pt_br */ "Focar barra de status", + /* ru */ "Фокус на строку состояния", + /* zh_hans */ "聚焦状态栏", + /* zh_hant */ "聚焦狀態列", + ], + // ViewWordWrap + [ + /* en */ "Word Wrap", + /* de */ "Zeilenumbruch", + /* es */ "Ajuste de línea", + /* fr */ "Retour à la ligne", + /* it */ "A capo automatico", + /* ja */ "折り返し", + /* ko */ "자동 줄 바꿈", + /* pt_br */ "Quebra de linha", + /* ru */ "Перенос слов", + /* zh_hans */ "自动换行", + /* zh_hant */ "自動換行", + ], + + // Help (a menu bar item) + [ + /* en */ "Help", + /* de */ "Hilfe", + /* es */ "Ayuda", + /* fr */ "Aide", + /* it */ "Aiuto", + /* ja */ "ヘルプ", + /* ko */ "도움말", + /* pt_br */ "Ajuda", + /* ru */ "Помощь", + /* zh_hans */ "帮助", + /* zh_hant */ "幫助", + ], + // HelpAbout + [ + /* en */ "About", + /* de */ "Über", + /* es */ "Acerca de", + /* fr */ "À propos", + /* it */ "Informazioni", + /* ja */ "情報", + /* ko */ "정보", + /* pt_br */ "Sobre", + /* ru */ "О программе", + /* zh_hans */ "关于", + /* zh_hant */ "關於", + ], + + // UnsavedChangesDialogTitle + [ + /* en */ "Unsaved Changes", + /* de */ "Ungespeicherte Änderungen", + /* es */ "Cambios sin guardar", + /* fr */ "Modifications non enregistrées", + /* it */ "Modifiche non salvate", + /* ja */ "未保存の変更", + /* ko */ "저장되지 않은 변경 사항", + /* pt_br */ "Alterações não salvas", + /* ru */ "Несохраненные изменения", + /* zh_hans */ "未保存的更改", + /* zh_hant */ "未儲存的變更", + ], + // UnsavedChangesDialogDescription + [ + /* en */ "Do you want to save the changes you made?", + /* de */ "Möchten Sie die vorgenommenen Änderungen speichern?", + /* es */ "¿Desea guardar los cambios realizados?", + /* fr */ "Voulez-vous enregistrer les modifications apportées?", + /* it */ "Vuoi salvare le modifiche apportate?", + /* ja */ "変更内容を保存しますか？", + /* ko */ "변경한 내용을 저장하시겠습니까?", + /* pt_br */ "Deseja salvar as alterações feitas?", + /* ru */ "Вы хотите сохранить внесённые изменения?", + /* zh_hans */ "您要保存所做的更改吗？", + /* zh_hant */ "您要保存所做的變更嗎？", + ], + // UnsavedChangesDialogYes + [ + /* en */ "Save", + /* de */ "Speichern", + /* es */ "Guardar", + /* fr */ "Enregistrer", + /* it */ "Salva", + /* ja */ "保存", + /* ko */ "저장", + /* pt_br */ "Salvar", + /* ru */ "Сохранить", + /* zh_hans */ "保存", + /* zh_hant */ "儲存", + ], + // UnsavedChangesDialogNo + [ + /* en */ "Don't Save", + /* de */ "Nicht speichern", + /* es */ "No guardar", + /* fr */ "Ne pas enregistrer", + /* it */ "Non salvare", + /* ja */ "保存しない", + /* ko */ "저장 안 함", + /* pt_br */ "Não salvar", + /* ru */ "Не сохранять", + /* zh_hans */ "不保存", + /* zh_hant */ "不儲存", + ], + + // AboutDialogTitle + [ + /* en */ "About", + /* de */ "Über", + /* es */ "Acerca de", + /* fr */ "À propos", + /* it */ "Informazioni", + /* ja */ "情報", + /* ko */ "정보", + /* pt_br */ "Sobre", + /* ru */ "О программе", + /* zh_hans */ "关于", + /* zh_hant */ "關於", + ], + // AboutDialogVersion + [ + /* en */ "Version: ", + /* de */ "Version: ", + /* es */ "Versión: ", + /* fr */ "Version: ", + /* it */ "Versione: ", + /* ja */ "バージョン: ", + /* ko */ "버전: ", + /* pt_br */ "Versão: ", + /* ru */ "Версия: ", + /* zh_hans */ "版本: ", + /* zh_hant */ "版本: ", + ], + + // Shown when the clipboard size exceeds the limit for OSC 52 + // LargeClipboardWarningLine1 + [ + /* en */ "Text you copy is shared with the terminal clipboard.", + /* de */ "Der kopierte Text wird mit der Terminal-Zwischenablage geteilt.", + /* es */ "El texto que copies se comparte con el portapapeles del terminal.", + /* fr */ "Le texte que vous copiez est partagé avec le presse-papiers du terminal.", + /* it */ "Il testo copiato viene condiviso con gli appunti del terminale.", + /* ja */ "コピーしたテキストはターミナルのクリップボードと共有されます。", + /* ko */ "복사한 텍스트가 터미널 클립보드와 공유됩니다.", + /* pt_br */ "O texto copiado é compartilhado com a área de transferência do terminal.", + /* ru */ "Скопированный текст передаётся в буфер обмена терминала.", + /* zh_hans */ "你复制的文本将共享到终端剪贴板。", + /* zh_hant */ "您複製的文字將會與終端機剪貼簿分享。", + ], + // LargeClipboardWarningLine2 + [ + /* en */ "You copied {size} which may take a long time to share.", + /* de */ "Sie haben {size} kopiert, das Weitergeben könnte lange dauern.", + /* es */ "Copiaste {size}, lo que puede tardar en compartirse.", + /* fr */ "Vous avez copié {size}, ce qui peut être long à partager.", + /* it */ "Hai copiato {size}, potrebbe richiedere molto tempo per condividerlo.", + /* ja */ "{size} をコピーしました。共有に時間がかかる可能性があります。", + /* ko */ "{size}를 복사했습니다. 공유하는 데 시간이 오래 걸릴 수 있습니다.", + /* pt_br */ "Você copiou {size}, o que pode demorar para compartilhar.", + /* ru */ "Вы скопировали {size}; передача может занять много времени.", + /* zh_hans */ "你复制了 {size}，共享可能需要较长时间。", + /* zh_hant */ "您已複製 {size}，共享可能需要較長時間。", + ], + // LargeClipboardWarningLine3 + [ + /* en */ "Do you want to send it anyway?", + /* de */ "Möchten Sie es trotzdem senden?", + /* es */ "¿Desea enviarlo de todas formas?", + /* fr */ "Voulez-vous quand même l’envoyer?", + /* it */ "Vuoi inviarlo comunque?", + /* ja */ "それでも送信しますか？", + /* ko */ "그래도 전송하시겠습니까?", + /* pt_br */ "Deseja enviar mesmo assim?", + /* ru */ "Отправить в любом случае?", + /* zh_hans */ "仍要发送吗？", + /* zh_hant */ "仍要傳送嗎？", + ], + // SuperLargeClipboardWarning (as an alternative to LargeClipboardWarningLine2 and 3) + [ + /* en */ "The text you copied is too large to be shared.", + /* de */ "Der kopierte Text ist zu groß, um geteilt zu werden.", + /* es */ "El texto que copiaste es demasiado grande para compartirse.", + /* fr */ "Le texte que vous avez copié est trop volumineux pour être partagé.", + /* it */ "Il testo copiato è troppo grande per essere condiviso.", + /* ja */ "コピーしたテキストは大きすぎて共有できません。", + /* ko */ "복사한 텍스트가 너무 커서 공유할 수 없습니다.", + /* pt_br */ "O texto copiado é grande demais para ser compartilhado.", + /* ru */ "Скопированный текст слишком велик для передачи.", + /* zh_hans */ "你复制的文本过大，无法共享。", + /* zh_hant */ "您複製的文字過大，無法分享。", + ], + + // WarningDialogTitle + [ + /* en */ "Warning", + /* de */ "Warnung", + /* es */ "Advertencia", + /* fr */ "Avertissement", + /* it */ "Avviso", + /* ja */ "警告", + /* ko */ "경고", + /* pt_br */ "Aviso", + /* ru */ "Предупреждение", + /* zh_hans */ "警告", + /* zh_hant */ "警告", + ], + + // ErrorDialogTitle + [ + /* en */ "Error", + /* de */ "Fehler", + /* es */ "Error", + /* fr */ "Erreur", + /* it */ "Errore", + /* ja */ "エラー", + /* ko */ "오류", + /* pt_br */ "Erro", + /* ru */ "Ошибка", + /* zh_hans */ "错误", + /* zh_hant */ "錯誤", + ], + // ErrorIcuMissing + [ + /* en */ "This operation requires the ICU library", + /* de */ "Diese Operation erfordert die ICU-Bibliothek", + /* es */ "Esta operación requiere la biblioteca ICU", + /* fr */ "Cette opération nécessite la bibliothèque ICU", + /* it */ "Questa operazione richiede la libreria ICU", + /* ja */ "この操作にはICUライブラリが必要です", + /* ko */ "이 작업에는 ICU 라이브러리가 필요합니다", + /* pt_br */ "Esta operação requer a biblioteca ICU", + /* ru */ "Эта операция требует наличия библиотеки ICU", + /* zh_hans */ "此操作需要 ICU 库", + /* zh_hant */ "此操作需要 ICU 庫", + ], + + // SearchNeedleLabel (for input field) + [ + /* en */ "Find:", + /* de */ "Suchen:", + /* es */ "Buscar:", + /* fr */ "Rechercher:", + /* it */ "Trova:", + /* ja */ "検索:", + /* ko */ "찾기:", + /* pt_br */ "Encontrar:", + /* ru */ "Найти:", + /* zh_hans */ "查找:", + /* zh_hant */ "尋找:", + ], + // SearchReplacementLabel (for input field) + [ + /* en */ "Replace:", + /* de */ "Ersetzen:", + /* es */ "Reemplazar:", + /* fr */ "Remplacer:", + /* it */ "Sostituire:", + /* ja */ "置換:", + /* ko */ "바꾸기:", + /* pt_br */ "Substituir:", + /* ru */ "Замена:", + /* zh_hans */ "替换:", + /* zh_hant */ "替換:", + ], + // SearchMatchCase (toggle) + [ + /* en */ "Match Case", + /* de */ "Groß/Klein", + /* es */ "May/Min", + /* fr */ "Casse", + /* it */ "Maius/minus", + /* ja */ "大/小文字", + /* ko */ "대소문자", + /* pt_br */ "Maius/minus", + /* ru */ "Регистр", + /* zh_hans */ "区分大小写", + /* zh_hant */ "區分大小寫", + ], + // SearchWholeWord (toggle) + [ + /* en */ "Whole Word", + /* de */ "Ganzes Wort", + /* es */ "Palabra", + /* fr */ "Mot entier", + /* it */ "Parola", + /* ja */ "単語単位", + /* ko */ "전체 단어", + /* pt_br */ "Palavra", + /* ru */ "Слово", + /* zh_hans */ "全字匹配", + /* zh_hant */ "全字匹配", + ], + // SearchUseRegex (toggle) + [ + /* en */ "Use Regex", + /* de */ "RegEx", + /* es */ "RegEx", + /* fr */ "RegEx", + /* it */ "RegEx", + /* ja */ "正規表現", + /* ko */ "정규식", + /* pt_br */ "RegEx", + /* ru */ "RegEx", + /* zh_hans */ "正则", + /* zh_hant */ "正則", + ], + // SearchReplaceAll (button) + [ + /* en */ "Replace All", + /* de */ "Alle ersetzen", + /* es */ "Reemplazar todo", + /* fr */ "Remplacer tout", + /* it */ "Sostituisci tutto", + /* ja */ "すべて置換", + /* ko */ "모두 바꾸기", + /* pt_br */ "Substituir tudo", + /* ru */ "Заменить все", + /* zh_hans */ "全部替换", + /* zh_hant */ "全部取代", + ], + // SearchClose (button) + [ + /* en */ "Close", + /* de */ "Schließen", + /* es */ "Cerrar", + /* fr */ "Fermer", + /* it */ "Chiudi", + /* ja */ "閉じる", + /* ko */ "닫기", + /* pt_br */ "Fechar", + /* ru */ "Закрыть", + /* zh_hans */ "关闭", + /* zh_hant */ "關閉", + ], + + // EncodingReopen + [ + /* en */ "Reopen with encoding", + /* de */ "Mit Kodierung erneut öffnen", + /* es */ "Reabrir con codificación", + /* fr */ "Rouvrir avec un encodage différent", + /* it */ "Riapri con codifica", + /* ja */ "エンコーディングで再度開く", + /* ko */ "인코딩으로 다시 열기", + /* pt_br */ "Reabrir com codificação", + /* ru */ "Открыть снова с кодировкой", + /* zh_hans */ "使用编码重新打开", + /* zh_hant */ "使用編碼重新打開", + ], + // EncodingConvert + [ + /* en */ "Convert to encoding", + /* de */ "In Kodierung konvertieren", + /* es */ "Convertir a otra codificación", + /* fr */ "Convertir en encodage", + /* it */ "Converti in codifica", + /* ja */ "エンコーディングに変換", + /* ko */ "인코딩으로 변환", + /* pt_br */ "Converter para codificação", + /* ru */ "Преобразовать в кодировку", + /* zh_hans */ "转换为编码", + /* zh_hant */ "轉換為編碼", + ], + + // IndentationTabs + [ + /* en */ "Tabs", + /* de */ "Tabs", + /* es */ "Tabulaciones", + /* fr */ "Tabulations", + /* it */ "Tabulazioni", + /* ja */ "タブ", + /* ko */ "탭", + /* pt_br */ "Tabulações", + /* ru */ "Табы", + /* zh_hans */ "制表符", + /* zh_hant */ "製表符", + ], + // IndentationSpaces + [ + /* en */ "Spaces", + /* de */ "Leerzeichen", + /* es */ "Espacios", + /* fr */ "Espaces", + /* it */ "Spazi", + /* ja */ "スペース", + /* ko */ "공백", + /* pt_br */ "Espaços", + /* ru */ "Пробелы", + /* zh_hans */ "空格", + /* zh_hant */ "空格", + ], + + // SaveAsDialogPathLabel + [ + /* en */ "Folder:", + /* de */ "Ordner:", + /* es */ "Carpeta:", + /* fr */ "Dossier:", + /* it */ "Cartella:", + /* ja */ "フォルダ:", + /* ko */ "폴더:", + /* pt_br */ "Pasta:", + /* ru */ "Папка:", + /* zh_hans */ "文件夹:", + /* zh_hant */ "資料夾:", + ], + // SaveAsDialogNameLabel + [ + /* en */ "File name:", + /* de */ "Dateiname:", + /* es */ "Nombre de archivo:", + /* fr */ "Nom de fichier:", + /* it */ "Nome del file:", + /* ja */ "ファイル名:", + /* ko */ "파일 이름:", + /* pt_br */ "Nome do arquivo:", + /* ru */ "Имя файла:", + /* zh_hans */ "文件名:", + /* zh_hant */ "檔案名稱:", + ], + + // FileOverwriteWarning + [ + /* en */ "Confirm Save As", + /* de */ "Speichern unter bestätigen", + /* es */ "Confirmar Guardar como", + /* fr */ "Confirmer Enregistrer sous", + /* it */ "Conferma Salva con nome", + /* ja */ "名前を付けて保存の確認", + /* ko */ "다른 이름으로 저장 확인", + /* pt_br */ "Confirmar Salvar como", + /* ru */ "Подтвердите «Сохранить как…»", + /* zh_hans */ "确认另存为", + /* zh_hant */ "確認另存新檔", + ], + // FileOverwriteWarningDescription + [ + /* en */ "File already exists. Do you want to overwrite it?", + /* de */ "Datei existiert bereits. Möchten Sie sie überschreiben?", + /* es */ "El archivo ya existe. ¿Desea sobrescribirlo?", + /* fr */ "Le fichier existe déjà. Voulez-vous l’écraser?", + /* it */ "Il file esiste già. Vuoi sovrascriverlo?", + /* ja */ "ファイルは既に存在します。上書きしますか？", + /* ko */ "파일이 이미 존재합니다. 덮어쓰시겠습니까?", + /* pt_br */ "O arquivo já existe. Deseja sobrescrevê-lo?", + /* ru */ "Файл уже существует. Перезаписать?", + /* zh_hans */ "文件已存在。要覆盖它吗？", + /* zh_hant */ "檔案已存在。要覆蓋它嗎？", + ], +]; + +static mut S_LANG: LangId = LangId::en; + +pub fn init() { + let scratch = scratch_arena(None); + let langs = sys::preferred_languages(&scratch); + let mut lang = LangId::en; + + for l in langs { + lang = match l.as_str() { + "en" => LangId::en, + "de" => LangId::de, + "es" => LangId::es, + "fr" => LangId::fr, + "it" => LangId::it, + "ja" => LangId::ja, + "ko" => LangId::ko, + "pt-br" => LangId::pt_br, + "ru" => LangId::ru, + "zh-hant" => LangId::zh_hant, + "zh" => LangId::zh_hans, + _ => continue, + }; + break; + } + + unsafe { + S_LANG = lang; + } +} + +pub fn loc(id: LocId) -> &'static str { + S_LANG_LUT[id as usize][unsafe { S_LANG as usize }] +} diff --git a/pkgs/edit/src/bin/edit/main.rs b/pkgs/edit/src/bin/edit/main.rs new file mode 100644 index 0000000..2c84004 --- /dev/null +++ b/pkgs/edit/src/bin/edit/main.rs @@ -0,0 +1,611 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#![feature(let_chains, linked_list_cursors, os_string_truncate, string_from_utf8_lossy_owned)] + +mod documents; +mod draw_editor; +mod draw_filepicker; +mod draw_menubar; +mod draw_statusbar; +mod localization; +mod state; + +use std::borrow::Cow; +#[cfg(feature = "debug-latency")] +use std::fmt::Write; +use std::path::Path; +use std::{env, process}; + +use draw_editor::*; +use draw_filepicker::*; +use draw_menubar::*; +use draw_statusbar::*; +use edit::arena::{self, ArenaString, scratch_arena}; +use edit::framebuffer::{self, IndexedColor}; +use edit::helpers::{KIBI, MEBI, MetricFormatter, Rect, Size}; +use edit::input::{self, kbmod, vk}; +use edit::oklab::oklab_blend; +use edit::tui::*; +use edit::vt::{self, Token}; +use edit::{apperr, arena_format, base64, path, sys}; +use localization::*; +use state::*; + +#[cfg(target_pointer_width = "32")] +const SCRATCH_ARENA_CAPACITY: usize = 128 * MEBI; +#[cfg(target_pointer_width = "64")] +const SCRATCH_ARENA_CAPACITY: usize = 512 * MEBI; + +fn main() -> process::ExitCode { + if cfg!(debug_assertions) { + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + drop(RestoreModes); + drop(sys::Deinit); + hook(info); + })); + } + + match run() { + Ok(()) => process::ExitCode::SUCCESS, + Err(err) => { + sys::write_stdout(&format!("{}\r\n", FormatApperr::from(err))); + process::ExitCode::FAILURE + } + } +} + +fn run() -> apperr::Result<()> { + // Init `sys` first, as everything else may depend on its functionality (IO, function pointers, etc.). + let _sys_deinit = sys::init()?; + // Next init `arena`, so that `scratch_arena` works. `loc` depends on it. + arena::init(SCRATCH_ARENA_CAPACITY)?; + // Init the `loc` module, so that error messages are localized. + localization::init(); + + let mut state = State::new()?; + if handle_args(&mut state)? { + return Ok(()); + } + + // sys::init() will switch the terminal to raw mode which prevents the user from pressing Ctrl+C. + // Since the `read_file` call may hang for some reason, we must only call this afterwards. + // `set_modes()` will enable mouse mode which is equally annoying to switch out for users + // and so we do it afterwards, for similar reasons. + sys::switch_modes()?; + + let mut vt_parser = vt::Parser::new(); + let mut input_parser = input::Parser::new(); + let mut tui = Tui::new()?; + + let _restore = setup_terminal(&mut tui, &mut vt_parser); + + state.menubar_color_bg = oklab_blend( + tui.indexed(IndexedColor::Background), + tui.indexed_alpha(IndexedColor::BrightBlue, 1, 2), + ); + state.menubar_color_fg = tui.contrasted(state.menubar_color_bg); + let floater_bg = oklab_blend( + tui.indexed_alpha(IndexedColor::Background, 2, 3), + tui.indexed_alpha(IndexedColor::Foreground, 1, 3), + ); + let floater_fg = tui.contrasted(floater_bg); + tui.setup_modifier_translations(ModifierTranslations { + ctrl: loc(LocId::Ctrl), + alt: loc(LocId::Alt), + shift: loc(LocId::Shift), + }); + tui.set_floater_default_bg(floater_bg); + tui.set_floater_default_fg(floater_fg); + tui.set_modal_default_bg(floater_bg); + tui.set_modal_default_fg(floater_fg); + + sys::inject_window_size_into_stdin(); + + #[cfg(feature = "debug-latency")] + let mut last_latency_width = 0; + + loop { + #[cfg(feature = "debug-latency")] + let time_beg; + #[cfg(feature = "debug-latency")] + let mut passes; + + // Process a batch of input. + { + let scratch = scratch_arena(None); + let read_timeout = vt_parser.read_timeout().min(tui.read_timeout()); + let Some(input) = sys::read_stdin(&scratch, read_timeout) else { + break; + }; + + #[cfg(feature = "debug-latency")] + { + time_beg = std::time::Instant::now(); + passes = 0usize; + } + + let vt_iter = vt_parser.parse(&input); + let mut input_iter = input_parser.parse(vt_iter); + + while { + let input = input_iter.next(); + let more = input.is_some(); + let mut ctx = tui.create_context(input); + + draw(&mut ctx, &mut state); + + #[cfg(feature = "debug-latency")] + { + passes += 1; + } + + more + } {} + } + + // Continue rendering until the layout has settled. + // This can take >1 frame, if the input focus is tossed between different controls. + while tui.needs_settling() { + let mut ctx = tui.create_context(None); + + draw(&mut ctx, &mut state); + + #[cfg(feature = "debug-layout")] + { + drop(ctx); + state.buffer.buffer.copy_from_str(&tui.debug_layout()); + } + + #[cfg(feature = "debug-latency")] + { + passes += 1; + } + } + + if state.exit { + break; + } + + // Render the UI and write it to the terminal. + { + let scratch = scratch_arena(None); + let mut output = tui.render(&scratch); + + { + let filename = state.documents.active().map_or("", |d| &d.filename); + if filename != state.osc_title_filename { + write_terminal_title(&mut output, filename); + state.osc_title_filename = filename.to_string(); + } + } + + if state.osc_clipboard_send_generation == tui.clipboard_generation() { + write_osc_clipboard(&mut output, &mut state, &tui); + } + + #[cfg(feature = "debug-latency")] + { + // Print the number of passes and latency in the top right corner. + let time_end = std::time::Instant::now(); + let status = time_end - time_beg; + + let scratch_alt = scratch_arena(Some(&scratch)); + let status = arena_format!( + &scratch_alt, + "{}P {}B {:.3}μs", + passes, + output.len(), + status.as_nanos() as f64 / 1000.0 + ); + + // "μs" is 3 bytes and 2 columns. + let cols = status.len() as i32 - 3 + 2; + + // Since the status may shrink and grow, we may have to overwrite the previous one with whitespace. + let padding = (last_latency_width - cols).max(0); + + // If the `output` is already very large, + // Rust may double the size during the write below. + // Let's avoid that by reserving the needed size in advance. + output.reserve_exact(128); + + // To avoid moving the cursor, push and pop it onto the VT cursor stack. + _ = write!( + output, + "\x1b7\x1b[0;41;97m\x1b[1;{0}H{1:2$}{3}\x1b8", + tui.size().width - cols - padding + 1, + "", + padding as usize, + status + ); + + last_latency_width = cols; + } + + sys::write_stdout(&output); + } + } + + Ok(()) +} + +// Returns true if the application should exit early. +fn handle_args(state: &mut State) -> apperr::Result { + let mut cwd = env::current_dir()?; + let mut path = None; + + // The best CLI argument parser in the world. + if let Some(arg) = env::args_os().nth(1) { + if arg == "-h" || arg == "--help" || (cfg!(windows) && arg == "/?") { + print_help(); + return Ok(true); + } else if arg == "-v" || arg == "--version" { + print_version(); + return Ok(true); + } else if arg == "-" { + // We'll check for a redirected stdin no matter what, so we can just ignore "-". + } else { + let p = cwd.join(Path::new(&arg)); + let p = path::normalize(&p); + if let Some(parent) = p.parent() { + cwd = parent.to_path_buf(); + } + path = Some(p); + } + } + + if let Some(mut file) = sys::open_stdin_if_redirected() { + let doc = state.documents.add_untitled()?; + let mut tb = doc.buffer.borrow_mut(); + tb.read_file(&mut file, None)?; + tb.mark_as_dirty(); + } else if let Some(path) = path { + state.documents.add_file_path(&path)?; + } else { + state.documents.add_untitled()?; + } + + state.file_picker_pending_dir = DisplayablePathBuf::new(cwd); + Ok(false) +} + +fn print_help() { + sys::write_stdout(concat!( + "Usage: edit [OPTIONS] [FILE]\r\n", + "Options:\r\n", + " -h, --help Print this help message\r\n", + " -v, --version Print the version number\r\n", + )); +} + +fn print_version() { + sys::write_stdout(concat!("edit version ", env!("CARGO_PKG_VERSION"), "\r\n")); +} + +fn draw(ctx: &mut Context, state: &mut State) { + draw_menubar(ctx, state); + draw_editor(ctx, state); + draw_statusbar(ctx, state); + + if state.wants_close { + draw_handle_wants_close(ctx, state); + } + if state.wants_exit { + draw_handle_wants_exit(ctx, state); + } + if state.wants_file_picker != StateFilePicker::None { + draw_file_picker(ctx, state); + } + if state.wants_save { + draw_handle_save(ctx, state); + } + if state.wants_encoding_change != StateEncodingChange::None { + draw_dialog_encoding_change(ctx, state); + } + if state.wants_document_picker { + draw_document_picker(ctx, state); + } + if state.wants_about { + draw_dialog_about(ctx, state); + } + if state.osc_clipboard_seen_generation != ctx.clipboard_generation() { + draw_handle_clipboard_change(ctx, state); + } + if state.error_log_count != 0 { + draw_error_log(ctx, state); + } + + if let Some(key) = ctx.keyboard_input() { + // Shortcuts that are not handled as part of the textarea, etc. + + if key == kbmod::CTRL | vk::N { + draw_add_untitled_document(ctx, state); + } else if key == kbmod::CTRL | vk::O { + state.wants_file_picker = StateFilePicker::Open; + } else if key == kbmod::CTRL | vk::S { + state.wants_save = true; + } else if key == kbmod::CTRL_SHIFT | vk::S { + state.wants_file_picker = StateFilePicker::SaveAs; + } else if key == kbmod::CTRL | vk::W { + state.wants_close = true; + } else if key == kbmod::CTRL | vk::P { + state.wants_document_picker = true; + } else if key == kbmod::CTRL | vk::Q { + state.wants_exit = true; + } else if key == kbmod::CTRL | vk::F && state.wants_search.kind != StateSearchKind::Disabled + { + state.wants_search.kind = StateSearchKind::Search; + state.wants_search.focus = true; + } else if key == kbmod::CTRL | vk::R && state.wants_search.kind != StateSearchKind::Disabled + { + state.wants_search.kind = StateSearchKind::Replace; + state.wants_search.focus = true; + } else { + return; + } + + // All of the above shortcuts happen to require a rerender. + ctx.needs_rerender(); + ctx.set_input_consumed(); + } +} + +fn draw_handle_wants_exit(_ctx: &mut Context, state: &mut State) { + while let Some(doc) = state.documents.active() { + if doc.buffer.borrow().is_dirty() { + state.wants_close = true; + return; + } + state.documents.remove_active(); + } + + if state.documents.len() == 0 { + state.exit = true; + } +} + +#[cold] +fn write_terminal_title(output: &mut ArenaString, filename: &str) { + output.push_str("\x1b]0;"); + + if !filename.is_empty() { + output.push_str(&sanitize_control_chars(filename)); + output.push_str(" - "); + } + + output.push_str("edit\x1b\\"); +} + +const LARGE_CLIPBOARD_THRESHOLD: usize = 4 * KIBI; + +fn draw_handle_clipboard_change(ctx: &mut Context, state: &mut State) { + let generation = ctx.clipboard_generation(); + + if state.osc_clipboard_always_send || ctx.clipboard().len() < LARGE_CLIPBOARD_THRESHOLD { + state.osc_clipboard_seen_generation = generation; + state.osc_clipboard_send_generation = generation; + return; + } + + let over_limit = ctx.clipboard().len() >= SCRATCH_ARENA_CAPACITY / 4; + + ctx.modal_begin("warning", loc(LocId::WarningDialogTitle)); + { + ctx.block_begin("description"); + ctx.attr_padding(Rect::three(1, 2, 1)); + + if over_limit { + ctx.label("line1", loc(LocId::LargeClipboardWarningLine1)); + ctx.attr_position(Position::Center); + ctx.label("line2", loc(LocId::SuperLargeClipboardWarning)); + ctx.attr_position(Position::Center); + } else { + let label2 = { + let template = loc(LocId::LargeClipboardWarningLine2); + let size = arena_format!(ctx.arena(), "{}", MetricFormatter(ctx.clipboard().len())); + + let mut label = + ArenaString::with_capacity_in(template.len() + size.len(), ctx.arena()); + label.push_str(template); + label.replace_once_in_place("{size}", &size); + label + }; + + ctx.label("line1", loc(LocId::LargeClipboardWarningLine1)); + ctx.attr_position(Position::Center); + ctx.label("line2", &label2); + ctx.attr_position(Position::Center); + ctx.label("line3", loc(LocId::LargeClipboardWarningLine3)); + ctx.attr_position(Position::Center); + } + ctx.block_end(); + + ctx.table_begin("choices"); + ctx.inherit_focus(); + ctx.attr_padding(Rect::three(0, 2, 1)); + ctx.attr_position(Position::Center); + ctx.table_set_cell_gap(Size { width: 2, height: 0 }); + { + ctx.table_next_row(); + ctx.inherit_focus(); + + if over_limit { + if ctx.button("ok", loc(LocId::Ok)) { + state.osc_clipboard_seen_generation = generation; + } + ctx.inherit_focus(); + } else { + if ctx.button("always", loc(LocId::Always)) { + state.osc_clipboard_always_send = true; + state.osc_clipboard_seen_generation = generation; + state.osc_clipboard_send_generation = generation; + } + + if ctx.button("yes", loc(LocId::Yes)) { + state.osc_clipboard_seen_generation = generation; + state.osc_clipboard_send_generation = generation; + } + if ctx.clipboard().len() < 10 * LARGE_CLIPBOARD_THRESHOLD { + ctx.inherit_focus(); + } + + if ctx.button("no", loc(LocId::No)) { + state.osc_clipboard_seen_generation = generation; + } + if ctx.clipboard().len() >= 10 * LARGE_CLIPBOARD_THRESHOLD { + ctx.inherit_focus(); + } + } + } + ctx.table_end(); + } + if ctx.modal_end() { + state.osc_clipboard_seen_generation = generation; + } +} + +#[cold] +fn write_osc_clipboard(output: &mut ArenaString, state: &mut State, tui: &Tui) { + let clipboard = tui.clipboard(); + if !clipboard.is_empty() { + // Rust doubles the size of a string when it needs to grow it. + // If `clipboard` is *really* large, this may then double + // the size of the `output` from e.g. 100MB to 200MB. Not good. + // We can avoid that by reserving the needed size in advance. + output.reserve_exact(base64::encode_len(clipboard.len()) + 16); + output.push_str("\x1b]52;c;"); + base64::encode(output, clipboard); + output.push_str("\x1b\\"); + } + state.osc_clipboard_send_generation = tui.clipboard_generation().wrapping_sub(1); +} + +struct RestoreModes; + +impl Drop for RestoreModes { + fn drop(&mut self) { + // Same as in the beginning but in the reverse order. + // It also includes DECSCUSR 0 to reset the cursor style and DECTCEM to show the cursor. + sys::write_stdout("\x1b[0 q\x1b[?25h\x1b]0;\x07\x1b[?1002;1006;2004l\x1b[?1049l"); + } +} + +fn setup_terminal(tui: &mut Tui, vt_parser: &mut vt::Parser) -> RestoreModes { + sys::write_stdout(concat!( + // 1049: Alternative Screen Buffer + // I put the ASB switch in the beginning, just in case the terminal performs + // some additional state tracking beyond the modes we enable/disable. + // 1002: Cell Motion Mouse Tracking + // 1006: SGR Mouse Mode + // 2004: Bracketed Paste Mode + "\x1b[?1049h\x1b[?1002;1006;2004h", + // OSC 4 color table requests for indices 0 through 15 (base colors). + "\x1b]4;0;?;1;?;2;?;3;?;4;?;5;?;6;?;7;?\x07", + "\x1b]4;8;?;9;?;10;?;11;?;12;?;13;?;14;?;15;?\x07", + // OSC 10 and 11 queries for the current foreground and background colors. + "\x1b]10;?\x07\x1b]11;?\x07", + // CSI c reports the terminal capabilities. + // It also helps us to detect the end of the responses, because not all + // terminals support the OSC queries, but all of them support CSI c. + "\x1b[c", + )); + + let mut done = false; + let mut osc_buffer = String::new(); + let mut indexed_colors = framebuffer::DEFAULT_THEME; + let mut color_responses = 0; + + while !done { + let scratch = scratch_arena(None); + let Some(input) = sys::read_stdin(&scratch, vt_parser.read_timeout()) else { + break; + }; + + let mut vt_stream = vt_parser.parse(&input); + while let Some(token) = vt_stream.next() { + match token { + Token::Csi(state) if state.final_byte == 'c' => done = true, + Token::Osc { mut data, partial } => { + if partial { + osc_buffer.push_str(data); + continue; + } + if !osc_buffer.is_empty() { + osc_buffer.push_str(data); + data = &osc_buffer; + } + + let mut splits = data.split_terminator(';'); + + let color = match splits.next().unwrap_or("") { + // The response is `4;;rgb://`. + "4" => match splits.next().unwrap_or("").parse::() { + Ok(val) if val < 16 => &mut indexed_colors[val], + _ => continue, + }, + // The response is `10;rgb://`. + "10" => &mut indexed_colors[IndexedColor::Foreground as usize], + // The response is `11;rgb://`. + "11" => &mut indexed_colors[IndexedColor::Background as usize], + _ => continue, + }; + + let color_param = splits.next().unwrap_or(""); + if !color_param.starts_with("rgb:") { + continue; + } + + let mut iter = color_param[4..].split_terminator('/'); + let rgb_parts = [(); 3].map(|_| iter.next().unwrap_or("0")); + let mut rgb = 0; + + for part in rgb_parts { + if part.len() == 2 || part.len() == 4 { + let Ok(mut val) = usize::from_str_radix(part, 16) else { + continue; + }; + if part.len() == 4 { + // Round from 16 bits to 8 bits. + val = (val * 0xff + 0x7fff) / 0xffff; + } + rgb = (rgb >> 8) | ((val as u32) << 16); + } + } + + *color = rgb | 0xff000000; + color_responses += 1; + osc_buffer.clear(); + } + _ => {} + } + } + } + + if color_responses == indexed_colors.len() { + tui.setup_indexed_colors(indexed_colors); + } + + RestoreModes +} + +/// Strips all C0 control characters from the string an replaces them with "_". +/// +/// Jury is still out on whether this should also strip C1 control characters. +/// That requires parsing UTF8 codepoints, which is annoying. +fn sanitize_control_chars(text: &str) -> Cow<'_, str> { + if let Some(off) = text.bytes().position(|b| (..0x20).contains(&b)) { + let mut sanitized = text.to_string(); + // SAFETY: We only search for ASCII and replace it with ASCII. + let vec = unsafe { sanitized.as_bytes_mut() }; + + for i in &mut vec[off..] { + *i = if (..0x20).contains(i) { b'_' } else { *i } + } + + Cow::Owned(sanitized) + } else { + Cow::Borrowed(text) + } +} diff --git a/pkgs/edit/src/bin/edit/state.rs b/pkgs/edit/src/bin/edit/state.rs new file mode 100644 index 0000000..d23be30 --- /dev/null +++ b/pkgs/edit/src/bin/edit/state.rs @@ -0,0 +1,254 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::borrow::Cow; +use std::ffi::{OsStr, OsString}; +use std::mem; +use std::path::{Path, PathBuf}; + +use edit::buffer::TextBuffer; +use edit::framebuffer::IndexedColor; +use edit::helpers::*; +use edit::tui::*; +use edit::{apperr, buffer, icu, sys}; + +use crate::documents::DocumentManager; +use crate::localization::*; + +#[repr(transparent)] +pub struct FormatApperr(apperr::Error); + +impl From for FormatApperr { + fn from(err: apperr::Error) -> Self { + FormatApperr(err) + } +} + +impl std::fmt::Display for FormatApperr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.0 { + apperr::APP_ICU_MISSING => f.write_str(loc(LocId::ErrorIcuMissing)), + apperr::Error::App(code) => write!(f, "Unknown app error code: {code}"), + apperr::Error::Icu(code) => icu::apperr_format(f, code), + apperr::Error::Sys(code) => sys::apperr_format(f, code), + } + } +} + +pub struct DisplayablePathBuf { + value: PathBuf, + str: Cow<'static, str>, +} + +impl DisplayablePathBuf { + pub fn new(value: PathBuf) -> Self { + let str = value.to_string_lossy(); + let str = unsafe { mem::transmute::, Cow<'_, str>>(str) }; + Self { value, str } + } + + pub fn as_path(&self) -> &Path { + &self.value + } + + pub fn as_str(&self) -> &str { + &self.str + } + + pub fn as_bytes(&self) -> &[u8] { + self.value.as_os_str().as_encoded_bytes() + } +} + +impl Default for DisplayablePathBuf { + fn default() -> Self { + Self { value: Default::default(), str: Cow::Borrowed("") } + } +} + +impl Clone for DisplayablePathBuf { + fn clone(&self) -> Self { + DisplayablePathBuf::new(self.value.clone()) + } +} + +impl From for DisplayablePathBuf { + fn from(s: OsString) -> DisplayablePathBuf { + DisplayablePathBuf::new(PathBuf::from(s)) + } +} + +impl> From<&T> for DisplayablePathBuf { + fn from(s: &T) -> DisplayablePathBuf { + DisplayablePathBuf::new(PathBuf::from(s)) + } +} + +pub struct StateSearch { + pub kind: StateSearchKind, + pub focus: bool, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum StateSearchKind { + Hidden, + Disabled, + Search, + Replace, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum StateFilePicker { + None, + Open, + SaveAs, + + SaveAsShown, // Transitioned from SaveAs +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum StateEncodingChange { + None, + Convert, + Reopen, +} + +pub struct State { + pub menubar_color_bg: u32, + pub menubar_color_fg: u32, + + pub documents: DocumentManager, + + // A ring buffer of the last 10 errors. + pub error_log: [String; 10], + pub error_log_index: usize, + pub error_log_count: usize, + + pub wants_file_picker: StateFilePicker, + pub file_picker_pending_dir: DisplayablePathBuf, + pub file_picker_pending_name: PathBuf, + pub file_picker_entries: Option>, + pub file_picker_overwrite_warning: Option, // The path the warning is about. + + pub wants_search: StateSearch, + pub search_needle: String, + pub search_replacement: String, + pub search_options: buffer::SearchOptions, + pub search_success: bool, + + pub wants_save: bool, + pub wants_statusbar_focus: bool, + pub wants_encoding_picker: bool, + pub wants_encoding_change: StateEncodingChange, + pub wants_indentation_picker: bool, + pub wants_document_picker: bool, + pub wants_about: bool, + pub wants_close: bool, + pub wants_exit: bool, + + pub osc_title_filename: String, + pub osc_clipboard_seen_generation: u32, + pub osc_clipboard_send_generation: u32, + pub osc_clipboard_always_send: bool, + pub exit: bool, +} + +impl State { + pub fn new() -> apperr::Result { + let buffer = TextBuffer::new_rc(false)?; + { + let mut tb = buffer.borrow_mut(); + tb.set_margin_enabled(true); + tb.set_line_highlight_enabled(true); + } + + Ok(Self { + menubar_color_bg: 0, + menubar_color_fg: 0, + + documents: Default::default(), + + error_log: [const { String::new() }; 10], + error_log_index: 0, + error_log_count: 0, + + wants_file_picker: StateFilePicker::None, + file_picker_pending_dir: Default::default(), + file_picker_pending_name: Default::default(), + file_picker_entries: None, + file_picker_overwrite_warning: None, + + wants_search: StateSearch { kind: StateSearchKind::Hidden, focus: false }, + search_needle: Default::default(), + search_replacement: Default::default(), + search_options: Default::default(), + search_success: true, + + wants_save: false, + wants_statusbar_focus: false, + wants_encoding_picker: false, + wants_encoding_change: StateEncodingChange::None, + wants_indentation_picker: false, + wants_document_picker: false, + wants_about: false, + wants_close: false, + wants_exit: false, + + osc_title_filename: Default::default(), + osc_clipboard_seen_generation: 0, + osc_clipboard_send_generation: 0, + osc_clipboard_always_send: false, + exit: false, + }) + } +} + +pub fn draw_add_untitled_document(ctx: &mut Context, state: &mut State) { + if let Err(err) = state.documents.add_untitled() { + error_log_add(ctx, state, err); + } +} + +pub fn error_log_add(ctx: &mut Context, state: &mut State, err: apperr::Error) { + let msg = format!("{}", FormatApperr::from(err)); + if !msg.is_empty() { + state.error_log[state.error_log_index] = msg; + state.error_log_index = (state.error_log_index + 1) % state.error_log.len(); + state.error_log_count = state.error_log.len().min(state.error_log_count + 1); + ctx.needs_rerender(); + } +} + +pub fn draw_error_log(ctx: &mut Context, state: &mut State) { + ctx.modal_begin("error", loc(LocId::ErrorDialogTitle)); + ctx.attr_background_rgba(ctx.indexed(IndexedColor::Red)); + ctx.attr_foreground_rgba(ctx.indexed(IndexedColor::BrightWhite)); + { + ctx.block_begin("content"); + ctx.attr_padding(Rect::three(0, 2, 1)); + { + let off = state.error_log_index + state.error_log.len() - state.error_log_count; + + for i in 0..state.error_log_count { + let idx = (off + i) % state.error_log.len(); + let msg = &state.error_log[idx][..]; + + if !msg.is_empty() { + ctx.next_block_id_mixin(i as u64); + ctx.label("error", msg); + ctx.attr_overflow(Overflow::TruncateTail); + } + } + } + ctx.block_end(); + + if ctx.button("ok", loc(LocId::Ok)) { + state.error_log_count = 0; + } + ctx.attr_position(Position::Center); + ctx.inherit_focus(); + } + if ctx.modal_end() { + state.error_log_count = 0; + } +} diff --git a/pkgs/edit/src/buffer/gap_buffer.rs b/pkgs/edit/src/buffer/gap_buffer.rs new file mode 100644 index 0000000..58425cd --- /dev/null +++ b/pkgs/edit/src/buffer/gap_buffer.rs @@ -0,0 +1,377 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::ops::Range; +use std::ptr::{self, NonNull}; +use std::slice; + +use crate::document::{ReadableDocument, WriteableDocument}; +use crate::helpers::*; +use crate::{apperr, sys}; + +#[cfg(target_pointer_width = "32")] +const LARGE_CAPACITY: usize = 128 * MEBI; +#[cfg(target_pointer_width = "64")] +const LARGE_CAPACITY: usize = 4 * GIBI; +const LARGE_ALLOC_CHUNK: usize = 64 * KIBI; +const LARGE_GAP_CHUNK: usize = 4 * KIBI; + +const SMALL_CAPACITY: usize = 128 * KIBI; +const SMALL_ALLOC_CHUNK: usize = 256; +const SMALL_GAP_CHUNK: usize = 16; + +// TODO: Instead of having a specialization for small buffers here, +// tui.rs could also just keep a MRU set of large buffers around. +enum BackingBuffer { + VirtualMemory(NonNull, usize), + Vec(Vec), +} + +impl Drop for BackingBuffer { + fn drop(&mut self) { + unsafe { + if let BackingBuffer::VirtualMemory(ptr, reserve) = *self { + sys::virtual_release(ptr, reserve); + } + } + } +} + +/// Most people know how Vec works: It has some spare capacity at the end, +/// so that pushing into it doesn't reallocate every single time. A gap buffer +/// is the same thing, but the spare capacity can be anywhere in the buffer. +/// This variant is optimized for large buffers and uses virtual memory. +pub struct GapBuffer { + /// Pointer to the buffer. + text: NonNull, + /// Maximum size of the buffer, including gap. + reserve: usize, + /// Size of the buffer, including gap. + commit: usize, + /// Length of the stored text, NOT including gap. + text_length: usize, + /// Gap offset. + gap_off: usize, + /// Gap length. + gap_len: usize, + /// Increments every time the buffer is modified. + generation: u32, + /// If `Vec(..)`, the buffer is optimized for small amounts of text + /// and uses the standard heap. Otherwise, it uses virtual memory. + buffer: BackingBuffer, +} + +impl GapBuffer { + pub fn new(small: bool) -> apperr::Result { + let reserve; + let buffer; + let text; + + if small { + reserve = SMALL_CAPACITY; + text = NonNull::dangling(); + buffer = BackingBuffer::Vec(Vec::new()); + } else { + reserve = LARGE_CAPACITY; + text = unsafe { sys::virtual_reserve(reserve)? }; + buffer = BackingBuffer::VirtualMemory(text, reserve); + } + + Ok(Self { + text, + reserve, + commit: 0, + text_length: 0, + gap_off: 0, + gap_len: 0, + generation: 0, + buffer, + }) + } + + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.text_length + } + + pub fn generation(&self) -> u32 { + self.generation + } + + pub fn set_generation(&mut self, generation: u32) { + self.generation = generation; + } + + /// WARNING: The returned slice must not necessarily be the same length as `len` (due to OOM). + pub fn allocate_gap(&mut self, off: usize, len: usize, delete: usize) -> &mut [u8] { + // Sanitize parameters + let off = off.min(self.text_length); + let delete = delete.min(self.text_length - off); + + // Move the existing gap if it exists + if off != self.gap_off { + self.move_gap(off); + } + + // Delete the text + if delete > 0 { + self.delete_text(delete); + } + + // Enlarge the gap if needed + if len > self.gap_len { + self.enlarge_gap(len); + } + + self.generation = self.generation.wrapping_add(1); + unsafe { slice::from_raw_parts_mut(self.text.add(self.gap_off).as_ptr(), self.gap_len) } + } + + fn move_gap(&mut self, off: usize) { + if self.gap_len > 0 { + // + // v gap_off + // left: |ABCDEFGHIJKLMN OPQRSTUVWXYZ| + // |ABCDEFGHI JKLMNOPQRSTUVWXYZ| + // ^ off + // move: JKLMN + // + // v gap_off + // !left: |ABCDEFGHIJKLMN OPQRSTUVWXYZ| + // |ABCDEFGHIJKLMNOPQRS TUVWXYZ| + // ^ off + // move: OPQRS + // + let left = off < self.gap_off; + let move_src = if left { off } else { self.gap_off + self.gap_len }; + let move_dst = if left { off + self.gap_len } else { self.gap_off }; + let move_len = if left { self.gap_off - off } else { off - self.gap_off }; + + unsafe { self.text.add(move_src).copy_to(self.text.add(move_dst), move_len) }; + + if cfg!(debug_assertions) { + // Fill the moved-out bytes with 0xCD to make debugging easier. + unsafe { self.text.add(off).write_bytes(0xCD, self.gap_len) }; + } + } + + self.gap_off = off; + } + + fn delete_text(&mut self, delete: usize) { + if cfg!(debug_assertions) { + // Fill the deleted bytes with 0xCD to make debugging easier. + unsafe { self.text.add(self.gap_off + self.gap_len).write_bytes(0xCD, delete) }; + } + + self.gap_len += delete; + self.text_length -= delete; + } + + fn enlarge_gap(&mut self, len: usize) { + let gap_chunk; + let alloc_chunk; + + if matches!(self.buffer, BackingBuffer::VirtualMemory(..)) { + gap_chunk = LARGE_GAP_CHUNK; + alloc_chunk = LARGE_ALLOC_CHUNK; + } else { + gap_chunk = SMALL_GAP_CHUNK; + alloc_chunk = SMALL_ALLOC_CHUNK; + } + + let gap_len_old = self.gap_len; + let gap_len_new = (len + gap_chunk + gap_chunk - 1) & !(gap_chunk - 1); + + let bytes_old = self.commit; + let bytes_new = self.text_length + gap_len_new; + + if bytes_new > bytes_old { + let bytes_new = (bytes_new + alloc_chunk - 1) & !(alloc_chunk - 1); + + if bytes_new > self.reserve { + return; + } + + match &mut self.buffer { + BackingBuffer::VirtualMemory(ptr, _) => unsafe { + if sys::virtual_commit(ptr.add(bytes_old), bytes_new - bytes_old).is_err() { + return; + } + }, + BackingBuffer::Vec(v) => { + v.resize(bytes_new, 0); + self.text = unsafe { NonNull::new_unchecked(v.as_mut_ptr()) }; + } + } + + self.commit = bytes_new; + } + + let gap_beg = unsafe { self.text.add(self.gap_off) }; + unsafe { + ptr::copy( + gap_beg.add(gap_len_old).as_ptr(), + gap_beg.add(gap_len_new).as_ptr(), + self.text_length - self.gap_off, + ) + }; + + if cfg!(debug_assertions) { + // Fill the moved-out bytes with 0xCD to make debugging easier. + unsafe { gap_beg.add(gap_len_old).write_bytes(0xCD, gap_len_new - gap_len_old) }; + } + + self.gap_len = gap_len_new; + } + + pub fn commit_gap(&mut self, len: usize) { + assert!(len <= self.gap_len); + self.text_length += len; + self.gap_off += len; + self.gap_len -= len; + } + + pub fn replace(&mut self, range: Range, src: &[u8]) { + let gap = self.allocate_gap(range.start, src.len(), range.end.saturating_sub(range.start)); + let len = slice_copy_safe(gap, src); + self.commit_gap(len); + } + + pub fn clear(&mut self) { + self.gap_off = 0; + self.gap_len += self.text_length; + self.generation = self.generation.wrapping_add(1); + self.text_length = 0; + } + + pub fn extract_raw( + &self, + mut beg: usize, + mut end: usize, + out: &mut Vec, + mut out_off: usize, + ) { + debug_assert!(beg <= end && end <= self.text_length); + + end = end.min(self.text_length); + beg = beg.min(end); + out_off = out_off.min(out.len()); + + if beg >= end { + return; + } + + out.reserve(end - beg); + + while beg < end { + let chunk = self.read_forward(beg); + let chunk = &chunk[..chunk.len().min(end - beg)]; + out.replace_range(out_off..out_off, chunk); + beg += chunk.len(); + out_off += chunk.len(); + } + } + + /// Replaces the entire buffer contents with the given `text`. + /// The method is optimized for the case where the given `text` already matches + /// the existing contents. Returns `true` if the buffer contents were changed. + pub fn copy_from(&mut self, src: &dyn ReadableDocument) -> bool { + let mut off = 0; + + // Find the position at which the contents change. + loop { + let dst_chunk = self.read_forward(off); + let src_chunk = src.read_forward(off); + + let dst_len = dst_chunk.len(); + let src_len = src_chunk.len(); + let len = dst_len.min(src_len); + let mismatch = dst_chunk[..len] != src_chunk[..len]; + + if mismatch { + break; // The contents differ. + } + if len == 0 { + if dst_len == src_len { + return false; // Both done simultaneously. -> Done. + } + break; // One of the two is shorter. + } + + off += len; + } + + // Update the buffer starting at `off`. + loop { + let chunk = src.read_forward(off); + self.replace(off..usize::MAX, chunk); + off += chunk.len(); + + // No more data to copy -> Done. By checking this _after_ the replace() + // call, we ensure that the initial `off..usize::MAX` range is deleted. + // This fixes going from some buffer contents to being empty. + if chunk.is_empty() { + return true; + } + } + } + + /// Copies the contents of the buffer into a string. + pub fn copy_into(&self, dst: &mut dyn WriteableDocument) { + let mut beg = 0; + let mut off = 0; + + while { + let chunk = self.read_forward(off); + + // The first write will be 0..usize::MAX and effectively clear() the destination. + // Every subsequent write will be usize::MAX..usize::MAX and thus effectively append(). + dst.replace(beg..usize::MAX, chunk); + beg = usize::MAX; + + off += chunk.len(); + off < self.text_length + } {} + } +} + +impl ReadableDocument for GapBuffer { + fn read_forward(&self, off: usize) -> &[u8] { + let off = off.min(self.text_length); + let beg; + let len; + + if off < self.gap_off { + // Cursor is before the gap: We can read until the start of the gap. + beg = off; + len = self.gap_off - off; + } else { + // Cursor is after the gap: We can read until the end of the buffer. + beg = off + self.gap_len; + len = self.text_length - off; + } + + unsafe { slice::from_raw_parts(self.text.add(beg).as_ptr(), len) } + } + + fn read_backward(&self, off: usize) -> &[u8] { + let off = off.min(self.text_length); + let beg; + let len; + + if off <= self.gap_off { + // Cursor is before the gap: We can read until the beginning of the buffer. + beg = 0; + len = off; + } else { + // Cursor is after the gap: We can read until the end of the gap. + beg = self.gap_off + self.gap_len; + // The cursor_off doesn't account of the gap_len. + // (This allows us to move the gap without recalculating the cursor position.) + len = off - self.gap_off; + } + + unsafe { slice::from_raw_parts(self.text.add(beg).as_ptr(), len) } + } +} diff --git a/pkgs/edit/src/buffer/mod.rs b/pkgs/edit/src/buffer/mod.rs new file mode 100644 index 0000000..5587cae --- /dev/null +++ b/pkgs/edit/src/buffer/mod.rs @@ -0,0 +1,2418 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! A text buffer for a text editor. +//! +//! Implements a Unicode-aware, layout-aware text buffer for terminals. +//! It's based on a gap buffer. It has no line cache and instead relies +//! on the performance of the ucd module for fast text navigation. +//! +//! --- +//! +//! If the project ever outgrows a basic gap buffer (e.g. to add time travel) +//! an ideal, alternative architecture would be a piece table with immutable trees. +//! The tree nodes can be allocated on the same arena allocator as the added chunks, +//! making lifetime management fairly easy. The algorithm is described here: +//! * +//! * +//! +//! The downside is that text navigation & search takes a performance hit due to small chunks. +//! The solution to the former is to keep line caches, which further complicates the architecture. +//! There's no solution for the latter. However, there's a chance that the performance will still be sufficient. + +mod gap_buffer; +mod navigation; + +use std::borrow::Cow; +use std::cell::UnsafeCell; +use std::collections::LinkedList; +use std::fmt::Write as _; +use std::fs::File; +use std::io::{Read as _, Write as _}; +use std::mem::{self, MaybeUninit}; +use std::ops::Range; +use std::rc::Rc; +use std::str; + +use gap_buffer::GapBuffer; + +use crate::arena::{ArenaString, scratch_arena}; +use crate::cell::SemiRefCell; +use crate::document::{ReadableDocument, WriteableDocument}; +use crate::framebuffer::{Framebuffer, IndexedColor}; +use crate::helpers::*; +use crate::oklab::oklab_blend; +use crate::simd::memchr2; +use crate::unicode::{self, Cursor, MeasurementConfig}; +use crate::{apperr, icu}; + +/// The margin template is used for line numbers. +/// The max. line number we should ever expect is probably 64-bit, +/// and so this template fits 19 digits, followed by " │ ". +const MARGIN_TEMPLATE: &str = " │ "; +/// Just a bunch of whitespace you can use for turning tabs into spaces. +/// Happens to reuse MARGIN_TEMPLATE, because it has sufficient whitespace. +const TAB_WHITESPACE: &str = MARGIN_TEMPLATE; + +/// Stores statistics about the whole document. +#[derive(Copy, Clone)] +pub struct TextBufferStatistics { + logical_lines: CoordType, + visual_lines: CoordType, +} + +/// Stores the active text selection anchors. +/// +/// The two points are not sorted. Instead, `beg` refers to where the selection +/// started being made and `end` refers to the currently being updated position. +#[derive(Copy, Clone)] +struct TextBufferSelection { + beg: Point, + end: Point, +} + +/// In order to group actions into a single undo step, +/// we need to know the type of action that was performed. +/// This stores the action type. +#[derive(Copy, Clone, Eq, PartialEq)] +enum HistoryType { + Other, + Write, + Delete, +} + +/// An undo/redo entry. +struct HistoryEntry { + /// [`TextBuffer::cursor`] position before the change was made. + cursor_before: Point, + /// [`TextBuffer::selection`] before the change was made. + selection_before: Option, + /// [`TextBuffer::stats`] before the change was made. + stats_before: TextBufferStatistics, + /// [`GapBuffer::generation`] before the change was made. + generation_before: u32, + /// Logical cursor position where the change took place. + /// The position is at the start of the changed range. + cursor: Point, + /// Text that was deleted from the buffer. + deleted: Vec, + /// Text that was added to the buffer. + added: Vec, +} + +/// Caches an ICU search operation. +struct ActiveSearch { + /// The search pattern. + pattern: String, + /// The search options. + options: SearchOptions, + /// The ICU `UText` object. + text: icu::Text, + /// The ICU `URegularExpression` object. + regex: icu::Regex, + /// [`GapBuffer::generation`] when the search was created. + /// This is used to detect if we need to refresh the + /// [`ActiveSearch::regex`] object. + buffer_generation: u32, + /// [`TextBuffer::selection_generation`] when the search was + /// created. When the user manually selects text, we need to + /// refresh the [`ActiveSearch::pattern`] with it. + selection_generation: u32, + /// Stores the text buffer offset in between searches. + next_search_offset: usize, + /// If we know there were no hits, we can skip searching. + no_matches: bool, +} + +/// Options for a search operation. +#[derive(Default, Clone, Copy, Eq, PartialEq)] +pub struct SearchOptions { + /// If true, the search is case-sensitive. + pub match_case: bool, + /// If true, the search matches whole words. + pub whole_word: bool, + /// If true, the search uses regex. + pub use_regex: bool, +} + +/// Caches the start and length of the active edit line for a single edit. +/// This helps us avoid having to remeasure the buffer after an edit. +struct ActiveEditLineInfo { + /// Points to the start of the currently being edited line. + safe_start: Cursor, + /// Number of visual rows of the line that starts + /// at [`ActiveEditLineInfo::safe_start`]. + line_height_in_rows: CoordType, + /// Byte distance from the start of the line at + /// [`ActiveEditLineInfo::safe_start`] to the next line. + distance_next_line_start: usize, +} + +/// Char- or word-wise navigation? Your choice. +pub enum CursorMovement { + Grapheme, + Word, +} + +/// The result of a call to [`TextBuffer::render()`]. +pub struct RenderResult { + /// The maximum visual X position we encountered during rendering. + pub visual_pos_x_max: CoordType, +} + +/// A [`TextBuffer`] with inner mutability. +pub type TextBufferCell = SemiRefCell; + +/// A [`TextBuffer`] inside an [`Rc`]. +/// +/// We need this because the TUI system needs to borrow +/// the given text buffer(s) until after the layout process. +pub type RcTextBuffer = Rc; + +/// A text buffer for a text editor. +pub struct TextBuffer { + buffer: GapBuffer, + + undo_stack: LinkedList>, + redo_stack: LinkedList>, + last_history_type: HistoryType, + last_save_generation: u32, + + active_edit_line_info: Option, + active_edit_depth: i32, + active_edit_off: usize, + + stats: TextBufferStatistics, + cursor: Cursor, + // When scrolling significant amounts of text away from the cursor, + // rendering will naturally slow down proportionally to the distance. + // To avoid this, we cache the cursor position for rendering. + // Must be cleared on every edit or reflow. + cursor_for_rendering: Option, + selection: Option, + selection_generation: u32, + search: Option>, + + width: CoordType, + margin_width: CoordType, + margin_enabled: bool, + word_wrap_column: CoordType, + word_wrap_enabled: bool, + tab_size: CoordType, + indent_with_tabs: bool, + line_highlight_enabled: bool, + ruler: CoordType, + encoding: &'static str, + newlines_are_crlf: bool, + overtype: bool, + + wants_cursor_visibility: bool, +} + +impl TextBuffer { + /// Creates a new text buffer inside an [`Rc`]. + /// See [`TextBuffer::new()`]. + pub fn new_rc(small: bool) -> apperr::Result { + let buffer = TextBuffer::new(small)?; + Ok(Rc::new(SemiRefCell::new(buffer))) + } + + /// Creates a new text buffer. With `small` you can control + /// if the buffer is optimized for <1MiB contents. + pub fn new(small: bool) -> apperr::Result { + Ok(Self { + buffer: GapBuffer::new(small)?, + + undo_stack: LinkedList::new(), + redo_stack: LinkedList::new(), + last_history_type: HistoryType::Other, + last_save_generation: 0, + + active_edit_line_info: None, + active_edit_depth: 0, + active_edit_off: 0, + + stats: TextBufferStatistics { logical_lines: 1, visual_lines: 1 }, + cursor: Default::default(), + cursor_for_rendering: None, + selection: None, + selection_generation: 0, + search: None, + + width: 0, + margin_width: 0, + margin_enabled: false, + word_wrap_column: 0, + word_wrap_enabled: false, + tab_size: 4, + indent_with_tabs: false, + line_highlight_enabled: false, + ruler: 0, + encoding: "UTF-8", + newlines_are_crlf: cfg!(windows), // Unfortunately Windows users insist on CRLF + overtype: false, + + wants_cursor_visibility: false, + }) + } + + /// Length of the document in bytes. + pub fn text_length(&self) -> usize { + self.buffer.len() + } + + /// Number of logical lines in the document, + /// that is, lines separated by newlines. + pub fn logical_line_count(&self) -> CoordType { + self.stats.logical_lines + } + + /// Number of visual lines in the document, + /// that is, the number of lines after layout. + pub fn visual_line_count(&self) -> CoordType { + self.stats.visual_lines + } + + /// Does the buffer need to be saved? + pub fn is_dirty(&self) -> bool { + self.last_save_generation != self.buffer.generation() + } + + /// The buffer generation changes on every edit. + /// With this you can check if it has changed since + /// the last time you called this function. + pub fn generation(&self) -> u32 { + self.buffer.generation() + } + + /// Force the buffer to be dirty. + pub fn mark_as_dirty(&mut self) { + self.last_save_generation = self.buffer.generation().wrapping_sub(1); + } + + fn mark_as_clean(&mut self) { + self.last_save_generation = self.buffer.generation(); + } + + /// The encoding used during reading/writing. "UTF-8" is the default. + pub fn encoding(&self) -> &'static str { + self.encoding + } + + /// Set the encoding used during reading/writing. + pub fn set_encoding(&mut self, encoding: &'static str) { + if self.encoding != encoding { + self.encoding = encoding; + self.mark_as_dirty(); + } + } + + /// The newline type used in the document. LF or CRLF. + pub fn is_crlf(&self) -> bool { + self.newlines_are_crlf + } + + /// Changes the newline type used in the document. + /// + /// NOTE: Cannot be undone. + pub fn normalize_newlines(&mut self, crlf: bool) { + let newline: &[u8] = if crlf { b"\r\n" } else { b"\n" }; + let mut off = 0; + + let mut cursor_offset = self.cursor.offset; + let mut cursor_for_rendering_offset = + self.cursor_for_rendering.map_or(cursor_offset, |c| c.offset); + + #[cfg(debug_assertions)] + let mut adjusted_newlines = 0; + + 'outer: loop { + // Seek to the offset of the next line start. + loop { + let chunk = self.read_forward(off); + if chunk.is_empty() { + break 'outer; + } + + let (delta, line) = unicode::newlines_forward(chunk, 0, 0, 1); + off += delta; + if line == 1 { + break; + } + } + + // Get the preceding newline. + let chunk = self.read_backward(off); + let chunk_newline_len = if chunk.ends_with(b"\r\n") { 2 } else { 1 }; + let chunk_newline = &chunk[chunk.len() - chunk_newline_len..]; + + if chunk_newline != newline { + // If this newline is still before our cursor position, then it still has an effect on its offset. + // Any newline adjustments past that cursor position are irrelevant. + let delta = newline.len() as isize - chunk_newline_len as isize; + if off <= cursor_offset { + cursor_offset = cursor_offset.saturating_add_signed(delta); + #[cfg(debug_assertions)] + { + adjusted_newlines += 1; + } + } + if off <= cursor_for_rendering_offset { + cursor_for_rendering_offset = + cursor_for_rendering_offset.saturating_add_signed(delta); + } + + // Replace the newline. + off -= chunk_newline_len; + self.buffer.replace(off..off + chunk_newline_len, newline); + off += newline.len(); + } + } + + // If this fails, the cursor offset calculation above is wrong. + #[cfg(debug_assertions)] + debug_assert_eq!(adjusted_newlines, self.cursor.logical_pos.y); + + self.cursor.offset = cursor_offset; + if let Some(cursor) = &mut self.cursor_for_rendering { + cursor.offset = cursor_for_rendering_offset; + } + + self.newlines_are_crlf = crlf; + } + + /// Whether to insert or overtype text when writing. + pub fn is_overtype(&self) -> bool { + self.overtype + } + + /// Set the overtype mode. + pub fn set_overtype(&mut self, overtype: bool) { + self.overtype = overtype; + } + + /// Gets the logical cursor position, that is, + /// the position in lines and graphemes per line. + pub fn cursor_logical_pos(&self) -> Point { + self.cursor.logical_pos + } + + /// Gets the visual cursor position, that is, + /// the position in laid out rows and columns. + pub fn cursor_visual_pos(&self) -> Point { + self.cursor.visual_pos + } + + /// Gets the width of the left margin. + pub fn margin_width(&self) -> CoordType { + self.margin_width + } + + /// Is the left margin enabled? + pub fn set_margin_enabled(&mut self, enabled: bool) -> bool { + if self.margin_enabled == enabled { + false + } else { + self.margin_enabled = enabled; + self.reflow(true); + true + } + } + + /// Gets the width of the text contents for layout. + pub fn text_width(&self) -> CoordType { + self.width - self.margin_width + } + + /// Ask the TUI system to scroll the buffer and make the cursor visible. + /// + /// TODO: This function shows that [`TextBuffer`] is poorly abstracted + /// away from the TUI system. The only reason this exists is so that + /// if someone outside the TUI code enables word-wrap, the TUI code + /// recognizes this and scrolls the cursor into view. But outside of this + /// scrolling, views, etc., are all UI concerns = this should not be here. + pub fn make_cursor_visible(&mut self) { + self.wants_cursor_visibility = true; + } + + /// For the TUI code to retrieve a prior [`TextBuffer::make_cursor_visible()`] request. + pub fn take_cursor_visibility_request(&mut self) -> bool { + mem::take(&mut self.wants_cursor_visibility) + } + + /// Is word-wrap enabled? + /// + /// Technically, this is a misnomer, because it's line-wrapping. + pub fn is_word_wrap_enabled(&self) -> bool { + self.word_wrap_enabled + } + + /// Enable or disable word-wrap. + /// + /// NOTE: It's expected that the tui code calls `set_width()` sometime after this. + /// This will then trigger the actual recalculation of the cursor position. + pub fn set_word_wrap(&mut self, enabled: bool) { + if self.word_wrap_enabled != enabled { + self.word_wrap_enabled = enabled; + self.width = 0; // Force a reflow. + self.make_cursor_visible(); + } + } + + /// Set the width available for layout. + /// + /// Ideally this would be a pure UI concern, but the text buffer needs this + /// so that it can abstract away visual cursor movement such as "go a line up". + /// What would that even mean if it didn't know how wide a line is? + pub fn set_width(&mut self, width: CoordType) -> bool { + if width <= 0 || width == self.width { + false + } else { + self.width = width; + self.reflow(true); + true + } + } + + /// Set the tab width. Could be anything, but is expected to be 1-8. + pub fn tab_size(&self) -> CoordType { + self.tab_size + } + + /// Set the tab size. Clamped to 1-8. + pub fn set_tab_size(&mut self, width: CoordType) -> bool { + let width = width.clamp(1, 8); + if width == self.tab_size { + false + } else { + self.tab_size = width; + self.reflow(true); + true + } + } + + /// Returns whether tabs are used for indentation. + pub fn indent_with_tabs(&self) -> bool { + self.indent_with_tabs + } + + /// Sets whether tabs or spaces are used for indentation. + pub fn set_indent_with_tabs(&mut self, indent_with_tabs: bool) { + self.indent_with_tabs = indent_with_tabs; + } + + /// Sets whether the line the cursor is on should be highlighted. + pub fn set_line_highlight_enabled(&mut self, enabled: bool) { + self.line_highlight_enabled = enabled; + } + + /// Sets a ruler column, e.g. 80. + pub fn set_ruler(&mut self, column: CoordType) { + self.ruler = column; + } + + fn reflow(&mut self, force: bool) { + // +1 onto logical_lines, because line numbers are 1-based. + // +1 onto log10, because we want the digit width and not the actual log10. + // +3 onto log10, because we append " | " to the line numbers to form the margin. + self.margin_width = if self.margin_enabled { + self.stats.logical_lines.ilog10() as CoordType + 4 + } else { + 0 + }; + + let text_width = self.text_width(); + // 2 columns are required, because otherwise wide glyphs wouldn't ever fit. + let word_wrap_column = + if self.word_wrap_enabled && text_width >= 2 { text_width } else { 0 }; + + if force || self.word_wrap_column > word_wrap_column { + self.word_wrap_column = word_wrap_column; + + if self.cursor.offset != 0 { + self.cursor = self + .cursor_move_to_logical_internal(Default::default(), self.cursor.logical_pos); + } + + // Recalculate the line statistics. + if self.word_wrap_enabled { + let end = self.cursor_move_to_logical_internal(self.cursor, Point::MAX); + self.stats.visual_lines = end.visual_pos.y + 1; + } else { + self.stats.visual_lines = self.stats.logical_lines; + } + } + + self.cursor_for_rendering = None; + } + + /// Replaces the entire buffer contents with the given `text`. + /// Assumes that the line count doesn't change. + pub fn copy_from_str(&mut self, text: &dyn ReadableDocument) { + if self.buffer.copy_from(text) { + self.recalc_after_content_swap(); + self.cursor_move_to_logical(Point { x: CoordType::MAX, y: 0 }); + + let delete = self.buffer.len() - self.cursor.offset; + if delete != 0 { + self.buffer.allocate_gap(self.cursor.offset, 0, delete); + } + } + } + + fn recalc_after_content_swap(&mut self) { + // If the buffer was changed, nothing we previously saved can be relied upon. + self.undo_stack.clear(); + self.redo_stack.clear(); + self.last_history_type = HistoryType::Other; + self.cursor = Default::default(); + self.cursor_for_rendering = None; + self.set_selection(None); + self.search = None; + self.mark_as_clean(); + self.reflow(true); + } + + /// Copies the contents of the buffer into a string. + pub fn save_as_string(&mut self, dst: &mut dyn WriteableDocument) { + self.buffer.copy_into(dst); + self.mark_as_clean(); + } + + /// Reads a file from disk into the text buffer, detecting encoding and BOM. + pub fn read_file( + &mut self, + file: &mut File, + encoding: Option<&'static str>, + ) -> apperr::Result<()> { + let scratch = scratch_arena(None); + let mut buf = scratch.alloc_uninit().transpose(); + let mut first_chunk_len = 0; + let mut read = 0; + + // Read enough bytes to detect the BOM. + while first_chunk_len < BOM_MAX_LEN { + read = file_read_uninit(file, &mut buf[first_chunk_len..])?; + if read == 0 { + break; + } + first_chunk_len += read; + } + + if let Some(encoding) = encoding { + self.encoding = encoding; + } else { + let bom = detect_bom(unsafe { buf[..first_chunk_len].assume_init_ref() }); + self.encoding = bom.unwrap_or("UTF-8"); + } + + // TODO: Since reading the file can fail, we should ensure that we also reset the cursor here. + // I don't do it, so that `recalc_after_content_swap()` works. + self.buffer.clear(); + + let done = read == 0; + if self.encoding == "UTF-8" { + self.read_file_as_utf8(file, &mut buf, first_chunk_len, done)?; + } else { + self.read_file_with_icu(file, &mut buf, first_chunk_len, done)?; + } + + // Figure out + // * the logical line count + // * the newline type (LF or CRLF) + // * the indentation type (tabs or spaces) + { + let chunk = self.read_forward(0); + let mut offset = 0; + let mut lines = 0; + // Number of lines ending in CRLF. + let mut crlf_count = 0; + // Number of lines starting with a tab. + let mut tab_indentations = 0; + // Number of lines starting with a space. + let mut space_indentations = 0; + // Histogram of the indentation depth of lines starting with between 2 and 8 spaces. + // In other words, `space_indentation_sizes[0]` is the number of lines starting with 2 spaces. + let mut space_indentation_sizes = [0; 7]; + + loop { + // Check if the line starts with a tab. + if offset < chunk.len() && chunk[offset] == b'\t' { + tab_indentations += 1; + } else { + // Otherwise, check how many spaces the line starts with. Searching for >8 spaces + // allows us to reject lines that have more than 1 level of indentation. + let space_indentation = + chunk[offset..].iter().take(9).take_while(|&&c| c == b' ').count(); + + // We'll also reject lines starting with 1 space, because that's too fickle as a heuristic. + if (2..=8).contains(&space_indentation) { + space_indentations += 1; + + // If we encounter an indentation depth of 6, it may either be a 6-space indentation, + // two 3-space indentation or 3 2-space indentations. To make this work, we increment + // all 3 possible histogram slots. + // 2 -> 2 + // 3 -> 3 + // 4 -> 4 2 + // 5 -> 5 + // 6 -> 6 3 2 + // 7 -> 7 + // 8 -> 8 4 2 + space_indentation_sizes[space_indentation - 2] += 1; + if space_indentation & 4 != 0 { + space_indentation_sizes[0] += 1; + } + if space_indentation == 6 || space_indentation == 8 { + space_indentation_sizes[space_indentation / 2 - 2] += 1; + } + } + } + + (offset, lines) = unicode::newlines_forward(chunk, offset, lines, lines + 1); + + // Check if the preceding line ended in CRLF. + if offset >= 2 && &chunk[offset - 2..offset] == b"\r\n" { + crlf_count += 1; + } + + // We'll limit our heuristics to the first 1000 lines. + // That should hopefully be enough in practice. + if offset >= chunk.len() || lines >= 1000 { + break; + } + } + + // We'll assume CRLF if more than half of the lines end in CRLF. + let newlines_are_crlf = crlf_count >= lines / 2; + + // We'll assume tabs if there are more lines starting with tabs than with spaces. + let indent_with_tabs = tab_indentations > space_indentations; + let tab_size = if indent_with_tabs { + // Tabs will get a visual size of 4 spaces by default. + 4 + } else { + // Otherwise, we'll assume the most common indentation depth. + // If there are conflicting indentation depths, we'll prefer the maximum, because in the loop + // above we incremented the histogram slot for 2-spaces when encountering 4-spaces and so on. + let mut max = 1; + let mut tab_size = 4; + for (i, &count) in space_indentation_sizes.iter().enumerate() { + if count >= max { + max = count; + tab_size = i as CoordType + 2; + } + } + tab_size + }; + + // If the file has more than 1000 lines, figure out how many are remaining. + if offset < chunk.len() { + (_, lines) = unicode::newlines_forward(chunk, offset, lines, CoordType::MAX); + } + + // Add 1, because the last line doesn't end in a newline (it ends in the literal end). + self.stats.logical_lines = lines + 1; + self.stats.visual_lines = self.stats.logical_lines; + self.newlines_are_crlf = newlines_are_crlf; + self.indent_with_tabs = indent_with_tabs; + self.tab_size = tab_size; + } + + self.recalc_after_content_swap(); + Ok(()) + } + + fn read_file_as_utf8( + &mut self, + file: &mut File, + buf: &mut [MaybeUninit; 4 * KIBI], + first_chunk_len: usize, + done: bool, + ) -> apperr::Result<()> { + { + let mut first_chunk = unsafe { buf[..first_chunk_len].assume_init_ref() }; + if first_chunk.starts_with(b"\xEF\xBB\xBF") { + first_chunk = &first_chunk[3..]; + self.encoding = "UTF-8 BOM"; + } + + self.buffer.replace(0..0, first_chunk); + } + + if done { + return Ok(()); + } + + // If we don't have file metadata, the input may be a pipe or a socket. + // Every read will have the same size until we hit the end. + let mut chunk_size = 128 * KIBI; + let mut extra_chunk_size = 128 * KIBI; + + if let Ok(m) = file.metadata() { + // Usually the next read of size `chunk_size` will read the entire file, + // but if the size has changed for some reason, then `extra_chunk_size` + // should be large enough to read the rest of the file. + // 4KiB is not too large and not too slow. + let len = m.len() as usize; + chunk_size = len.saturating_sub(first_chunk_len); + extra_chunk_size = 4 * KIBI; + } + + loop { + let gap = self.buffer.allocate_gap(self.text_length(), chunk_size, 0); + if gap.is_empty() { + break; + } + + let read = file.read(gap)?; + if read == 0 { + break; + } + + self.buffer.commit_gap(read); + chunk_size = extra_chunk_size; + } + + Ok(()) + } + + fn read_file_with_icu( + &mut self, + file: &mut File, + buf: &mut [MaybeUninit; 4 * KIBI], + first_chunk_len: usize, + mut done: bool, + ) -> apperr::Result<()> { + let scratch = scratch_arena(None); + let pivot_buffer = scratch.alloc_uninit_slice(4 * KIBI); + let mut c = icu::Converter::new(pivot_buffer, self.encoding, "UTF-8")?; + let mut first_chunk = unsafe { buf[..first_chunk_len].assume_init_ref() }; + + while !first_chunk.is_empty() { + let off = self.text_length(); + let gap = self.buffer.allocate_gap(off, 8 * KIBI, 0); + let (input_advance, mut output_advance) = + c.convert(first_chunk, slice_as_uninit_mut(gap))?; + + // Remove the BOM from the file, if this is the first chunk. + // Our caller ensures to only call us once the BOM has been identified, + // which means that if there's a BOM it must be wholly contained in this chunk. + if off == 0 { + let written = &mut gap[..output_advance]; + if written.starts_with(b"\xEF\xBB\xBF") { + written.copy_within(3.., 0); + output_advance -= 3; + } + } + + self.buffer.commit_gap(output_advance); + first_chunk = &first_chunk[input_advance..]; + } + + let mut buf_len = 0; + + loop { + if !done { + let read = file_read_uninit(file, &mut buf[buf_len..])?; + buf_len += read; + done = read == 0; + } + + let gap = self.buffer.allocate_gap(self.text_length(), 8 * KIBI, 0); + if gap.is_empty() { + break; + } + + let read = unsafe { buf[..buf_len].assume_init_ref() }; + let (input_advance, output_advance) = c.convert(read, slice_as_uninit_mut(gap))?; + + self.buffer.commit_gap(output_advance); + + let flush = done && buf_len == 0; + buf_len -= input_advance; + buf.copy_within(input_advance.., 0); + + if flush { + break; + } + } + + Ok(()) + } + + /// Writes the text buffer contents to a file, handling BOM and encoding. + pub fn write_file(&mut self, file: &mut File) -> apperr::Result<()> { + let mut offset = 0; + + if self.encoding.starts_with("UTF-8") { + if self.encoding == "UTF-8 BOM" { + file.write_all(b"\xEF\xBB\xBF")?; + } + loop { + let chunk = self.read_forward(offset); + if chunk.is_empty() { + break; + } + file.write_all(chunk)?; + offset += chunk.len(); + } + } else { + self.write_file_with_icu(file)?; + } + + self.mark_as_clean(); + Ok(()) + } + + fn write_file_with_icu(&mut self, file: &mut File) -> apperr::Result<()> { + let scratch = scratch_arena(None); + let pivot_buffer = scratch.alloc_uninit_slice(4 * KIBI); + let buf = scratch.alloc_uninit_slice(4 * KIBI); + let mut c = icu::Converter::new(pivot_buffer, "UTF-8", self.encoding)?; + let mut offset = 0; + + // Write the BOM for the encodings we know need it. + if self.encoding.starts_with("UTF-16") + || self.encoding.starts_with("UTF-32") + || self.encoding == "GB18030" + { + let (_, output_advance) = c.convert(b"\xEF\xBB\xBF", buf)?; + let chunk = unsafe { buf[..output_advance].assume_init_ref() }; + file.write_all(chunk)?; + } + + loop { + let chunk = self.read_forward(offset); + if chunk.is_empty() { + break; + } + + let (input_advance, output_advance) = c.convert(chunk, buf)?; + let chunk = unsafe { buf[..output_advance].assume_init_ref() }; + file.write_all(chunk)?; + offset += input_advance; + } + + Ok(()) + } + + /// Returns the current selection. + pub fn has_selection(&self) -> bool { + self.selection.is_some() + } + + fn set_selection(&mut self, selection: Option) -> u32 { + self.selection = selection; + self.selection_generation = self.selection_generation.wrapping_add(1); + self.selection_generation + } + + /// Moves the cursor to `visual_pos` and updates the selection to contain it. + pub fn selection_update_visual(&mut self, visual_pos: Point) { + self.set_cursor_for_selection(self.cursor_move_to_visual_internal(self.cursor, visual_pos)); + } + + /// Moves the cursor to `logical_pos` and updates the selection to contain it. + pub fn selection_update_logical(&mut self, logical_pos: Point) { + self.set_cursor_for_selection( + self.cursor_move_to_logical_internal(self.cursor, logical_pos), + ); + } + + /// Moves the cursor by `delta` and updates the selection to contain it. + pub fn selection_update_delta(&mut self, granularity: CursorMovement, delta: CoordType) { + self.set_cursor_for_selection(self.cursor_move_delta_internal( + self.cursor, + granularity, + delta, + )); + } + + /// Select the current word. + pub fn select_word(&mut self) { + let Range { start, end } = navigation::word_select(&self.buffer, self.cursor.offset); + let beg = self.cursor_move_to_offset_internal(self.cursor, start); + let end = self.cursor_move_to_offset_internal(beg, end); + unsafe { self.set_cursor(end) }; + self.set_selection(Some(TextBufferSelection { + beg: beg.logical_pos, + end: end.logical_pos, + })); + } + + /// Select the current line. + pub fn select_line(&mut self) { + let beg = self.cursor_move_to_logical_internal( + self.cursor, + Point { x: 0, y: self.cursor.logical_pos.y }, + ); + let end = self + .cursor_move_to_logical_internal(beg, Point { x: 0, y: self.cursor.logical_pos.y + 1 }); + unsafe { self.set_cursor(end) }; + self.set_selection(Some(TextBufferSelection { + beg: beg.logical_pos, + end: end.logical_pos, + })); + } + + /// Select the entire document. + pub fn select_all(&mut self) { + let beg = Default::default(); + let end = self.cursor_move_to_logical_internal(beg, Point::MAX); + unsafe { self.set_cursor(end) }; + self.set_selection(Some(TextBufferSelection { + beg: beg.logical_pos, + end: end.logical_pos, + })); + } + + /// Starts a new selection, if there's none already. + pub fn start_selection(&mut self) { + if self.selection.is_none() { + self.set_selection(Some(TextBufferSelection { + beg: self.cursor.logical_pos, + end: self.cursor.logical_pos, + })); + } + } + + /// Destroy the current selection. + pub fn clear_selection(&mut self) -> bool { + let had_selection = self.selection.is_some(); + self.set_selection(None); + had_selection + } + + /// Find the next occurrence of the given `pattern` and select it. + pub fn find_and_select(&mut self, pattern: &str, options: SearchOptions) -> apperr::Result<()> { + if let Some(search) = &mut self.search { + let search = search.get_mut(); + // When the search input changes we must reset the search. + if search.pattern != pattern || search.options != options { + self.search = None; + } + + // When transitioning from some search to no search, we must clear the selection. + if pattern.is_empty() + && let Some(TextBufferSelection { beg, .. }) = self.selection + { + self.cursor_move_to_logical(beg); + } + } + + if pattern.is_empty() { + return Ok(()); + } + + let search = match &self.search { + Some(search) => unsafe { &mut *search.get() }, + None => { + let search = self.find_construct_search(pattern, options)?; + self.search = Some(UnsafeCell::new(search)); + unsafe { &mut *self.search.as_ref().unwrap().get() } + } + }; + + // If we previously searched through the entire document and found 0 matches, + // then we can avoid searching again. + if search.no_matches { + return Ok(()); + } + + // If the user moved the cursor since the last search, but the needle remained the same, + // we still need to move the start of the search to the new cursor position. + let next_search_offset = match self.selection { + Some(TextBufferSelection { beg, end }) => { + if self.selection_generation == search.selection_generation { + search.next_search_offset + } else { + self.cursor_move_to_logical_internal(self.cursor, beg.min(end)).offset + } + } + _ => self.cursor.offset, + }; + + self.find_select_next(search, next_search_offset, true); + Ok(()) + } + + /// Find the next occurrence of the given `pattern` and replace it with `replacement`. + pub fn find_and_replace( + &mut self, + pattern: &str, + options: SearchOptions, + replacement: &str, + ) -> apperr::Result<()> { + // Editors traditionally replace the previous search hit, not the next possible one. + if let (Some(search), Some(..)) = (&mut self.search, &self.selection) { + let search = search.get_mut(); + if search.selection_generation == self.selection_generation { + self.write(replacement.as_bytes(), true); + } + } + + self.find_and_select(pattern, options) + } + + /// Find all occurrences of the given `pattern` and replace them with `replacement`. + pub fn find_and_replace_all( + &mut self, + pattern: &str, + options: SearchOptions, + replacement: &str, + ) -> apperr::Result<()> { + let replacement = replacement.as_bytes(); + let mut search = self.find_construct_search(pattern, options)?; + let mut offset = 0; + + loop { + self.find_select_next(&mut search, offset, false); + if !self.has_selection() { + break; + } + self.write(replacement, true); + offset = self.cursor.offset; + } + + Ok(()) + } + + fn find_construct_search( + &self, + pattern: &str, + options: SearchOptions, + ) -> apperr::Result { + let sanitized_pattern = if options.whole_word && options.use_regex { + Cow::Owned(format!(r"\b(?:{pattern})\b")) + } else if options.whole_word { + let mut p = String::with_capacity(pattern.len() + 16); + p.push_str(r"\b"); + + // Escape regex special characters. + let b = unsafe { p.as_mut_vec() }; + for &byte in pattern.as_bytes() { + match byte { + b'*' | b'?' | b'+' | b'[' | b'(' | b')' | b'{' | b'}' | b'^' | b'$' | b'|' + | b'\\' | b'.' => { + b.push(b'\\'); + b.push(byte); + } + _ => b.push(byte), + } + } + + p.push_str(r"\b"); + Cow::Owned(p) + } else { + Cow::Borrowed(pattern) + }; + + let mut flags = icu::Regex::MULTILINE; + if !options.match_case { + flags |= icu::Regex::CASE_INSENSITIVE; + } + if !options.use_regex && !options.whole_word { + flags |= icu::Regex::LITERAL; + } + + // Move the start of the search to the start of the selection, + // or otherwise to the current cursor position. + + let text = unsafe { icu::Text::new(self)? }; + let regex = unsafe { icu::Regex::new(&sanitized_pattern, flags, &text)? }; + + Ok(ActiveSearch { + pattern: pattern.to_string(), + options, + text, + regex, + buffer_generation: self.buffer.generation(), + selection_generation: 0, + next_search_offset: 0, + no_matches: false, + }) + } + + fn find_select_next(&mut self, search: &mut ActiveSearch, offset: usize, wrap: bool) { + if search.buffer_generation != self.buffer.generation() { + unsafe { search.regex.set_text(&search.text) }; + search.buffer_generation = self.buffer.generation(); + } + + if search.next_search_offset != offset { + search.next_search_offset = offset; + search.regex.reset(offset); + } + + let mut hit = search.regex.next(); + + // If we hit the end of the buffer, and we know that there's something to find, + // start the search again from the beginning (= wrap around). + if wrap && hit.is_none() && search.next_search_offset != 0 { + search.next_search_offset = 0; + search.regex.reset(0); + hit = search.regex.next(); + } + + search.selection_generation = if let Some(range) = hit { + // Now the search offset is no more at the start of the buffer. + search.next_search_offset = range.end; + + let beg = self.cursor_move_to_offset_internal(self.cursor, range.start); + let end = self.cursor_move_to_offset_internal(beg, range.end); + + unsafe { self.set_cursor(end) }; + self.make_cursor_visible(); + + self.set_selection(Some(TextBufferSelection { + beg: beg.logical_pos, + end: end.logical_pos, + })) + } else { + // Avoid searching through the entire document again if we know there's nothing to find. + search.no_matches = true; + self.set_selection(None) + }; + } + + fn measurement_config(&self) -> MeasurementConfig { + MeasurementConfig::new(&self.buffer) + .with_word_wrap_column(self.word_wrap_column) + .with_tab_size(self.tab_size) + } + + fn goto_line_start(&self, cursor: Cursor, y: CoordType) -> Cursor { + let mut result = cursor; + let mut seek_to_line_start = true; + + if y > result.logical_pos.y { + while y > result.logical_pos.y { + let chunk = self.read_forward(result.offset); + if chunk.is_empty() { + break; + } + + let (delta, line) = unicode::newlines_forward(chunk, 0, result.logical_pos.y, y); + result.offset += delta; + result.logical_pos.y = line; + } + + // If we're at the end of the buffer, we could either be there because the last + // character in the buffer is genuinely a newline, or because the buffer ends in a + // line of text without trailing newline. The only way to make sure is to seek + // backwards to the line start again. But otherwise we can skip that. + seek_to_line_start = + result.offset == self.text_length() && result.offset != cursor.offset; + } + + if seek_to_line_start { + loop { + let chunk = self.read_backward(result.offset); + if chunk.is_empty() { + break; + } + + let (delta, line) = + unicode::newlines_backward(chunk, chunk.len(), result.logical_pos.y, y); + result.offset -= chunk.len() - delta; + result.logical_pos.y = line; + if delta > 0 { + break; + } + } + } + + if result.offset == cursor.offset { + return result; + } + + result.logical_pos.x = 0; + result.visual_pos.x = 0; + result.visual_pos.y = result.logical_pos.y; + result.column = 0; + result.wrap_opp = false; + + if self.word_wrap_column > 0 { + let upward = result.offset < cursor.offset; + let (top, bottom) = if upward { (result, cursor) } else { (cursor, result) }; + + let mut bottom_remeasured = + self.measurement_config().with_cursor(top).goto_logical(bottom.logical_pos); + + // The second problem is that visual positions can be ambiguous. A single logical position + // can map to two visual positions: One at the end of the preceeding line in front of + // a word wrap, and another at the start of the next line after the same word wrap. + // + // This, however, only applies if we go upwards, because only then `bottom ≅ cursor`, + // and thus only then this `bottom` is ambiguous. Otherwise, `bottom ≅ result` + // and `result` is at a line start which is never ambiguous. + if upward { + let a = bottom_remeasured.visual_pos.x; + let b = bottom.visual_pos.x; + bottom_remeasured.visual_pos.y = bottom_remeasured.visual_pos.y + + (a != 0 && b == 0) as CoordType + - (a == 0 && b != 0) as CoordType; + } + + let mut delta = bottom_remeasured.visual_pos.y - top.visual_pos.y; + if upward { + delta = -delta; + } + + result.visual_pos.y = cursor.visual_pos.y + delta; + } + + result + } + + fn cursor_move_to_offset_internal(&self, mut cursor: Cursor, offset: usize) -> Cursor { + if offset == cursor.offset { + return cursor; + } + + // goto_line_start() is fast for seeking across lines _if_ line wrapping is disabled. + // For backward seeking we have to use it either way, so we're covered there. + // This implements the forward seeking portion, if it's approx. worth doing so. + if self.word_wrap_column <= 0 && offset.saturating_sub(cursor.offset) > 1024 { + // Replacing this with a more optimal, direct memchr() loop appears + // to improve performance only marginally by another 2% or so. + // Still, it's kind of "meh" looking at how poorly this is implemented... + loop { + let next = self.goto_line_start(cursor, cursor.logical_pos.y + 1); + // Stop when we either ran past the target offset, + // or when we hit the end of the buffer and `goto_line_start` backtracked to the line start. + if next.offset > offset || next.offset <= cursor.offset { + break; + } + cursor = next; + } + } + + while offset < cursor.offset { + cursor = self.goto_line_start(cursor, cursor.logical_pos.y - 1); + } + + self.measurement_config().with_cursor(cursor).goto_offset(offset) + } + + fn cursor_move_to_logical_internal(&self, mut cursor: Cursor, pos: Point) -> Cursor { + let pos = Point { x: pos.x.max(0), y: pos.y.max(0) }; + + if pos == cursor.logical_pos { + return cursor; + } + + // goto_line_start() is the fastest way for seeking across lines. As such we always + // use it if the requested `.y` position is different. We still need to use it if the + // `.x` position is smaller, but only because `goto_logical()` cannot seek backwards. + if pos.y != cursor.logical_pos.y || pos.x < cursor.logical_pos.x { + cursor = self.goto_line_start(cursor, pos.y); + } + + self.measurement_config().with_cursor(cursor).goto_logical(pos) + } + + fn cursor_move_to_visual_internal(&self, mut cursor: Cursor, pos: Point) -> Cursor { + let pos = Point { x: pos.x.max(0), y: pos.y.max(0) }; + + if pos == cursor.visual_pos { + return cursor; + } + + if self.word_wrap_column <= 0 { + // Identical to the fast-pass in `cursor_move_to_logical_internal()`. + if pos.y != cursor.logical_pos.y || pos.x < cursor.logical_pos.x { + cursor = self.goto_line_start(cursor, pos.y); + } + } else { + // `goto_visual()` can only seek foward, so we need to seek backward here if needed. + // NOTE that this intentionally doesn't use the `Eq` trait of `Point`, because if + // `pos.y == cursor.visual_pos.y` we don't need to go to `cursor.logical_pos.y - 1`. + while pos.y < cursor.visual_pos.y { + cursor = self.goto_line_start(cursor, cursor.logical_pos.y - 1); + } + if pos.y == cursor.visual_pos.y && pos.x < cursor.visual_pos.x { + cursor = self.goto_line_start(cursor, cursor.logical_pos.y); + } + } + + self.measurement_config().with_cursor(cursor).goto_visual(pos) + } + + fn cursor_move_delta_internal( + &self, + mut cursor: Cursor, + granularity: CursorMovement, + mut delta: CoordType, + ) -> Cursor { + if delta == 0 { + return cursor; + } + + let sign = if delta > 0 { 1 } else { -1 }; + + match granularity { + CursorMovement::Grapheme => { + let start_x = if delta > 0 { 0 } else { CoordType::MAX }; + + loop { + let target_x = cursor.logical_pos.x + delta; + + cursor = self.cursor_move_to_logical_internal( + cursor, + Point { x: target_x, y: cursor.logical_pos.y }, + ); + + // We can stop if we ran out of remaining delta + // (or perhaps ran past the goal; in either case the sign would've changed), + // or if we hit the beginning or end of the buffer. + delta = target_x - cursor.logical_pos.x; + if delta.signum() != sign + || (delta < 0 && cursor.offset == 0) + || (delta > 0 && cursor.offset >= self.text_length()) + { + break; + } + + cursor = self.cursor_move_to_logical_internal( + cursor, + Point { x: start_x, y: cursor.logical_pos.y + sign }, + ); + + // We crossed a newline which counts for 1 grapheme cluster. + // So, we also need to run the same check again. + delta -= sign; + if delta.signum() != sign + || cursor.offset == 0 + || cursor.offset >= self.text_length() + { + break; + } + } + } + CursorMovement::Word => { + let doc = &self.buffer as &dyn ReadableDocument; + let mut offset = self.cursor.offset; + + while delta != 0 { + if delta < 0 { + offset = navigation::word_backward(doc, offset); + } else { + offset = navigation::word_forward(doc, offset); + } + delta -= sign; + } + + cursor = self.cursor_move_to_offset_internal(cursor, offset); + } + } + + cursor + } + + /// Moves the cursor to the given offset. + pub fn cursor_move_to_offset(&mut self, offset: usize) { + unsafe { self.set_cursor(self.cursor_move_to_offset_internal(self.cursor, offset)) } + } + + /// Moves the cursor to the given logical position. + pub fn cursor_move_to_logical(&mut self, pos: Point) { + unsafe { self.set_cursor(self.cursor_move_to_logical_internal(self.cursor, pos)) } + } + + /// Moves the cursor to the given visual position. + pub fn cursor_move_to_visual(&mut self, pos: Point) { + unsafe { self.set_cursor(self.cursor_move_to_visual_internal(self.cursor, pos)) } + } + + /// Moves the cursor by the given delta. + pub fn cursor_move_delta(&mut self, granularity: CursorMovement, delta: CoordType) { + unsafe { self.set_cursor(self.cursor_move_delta_internal(self.cursor, granularity, delta)) } + } + + /// Sets the cursor to the given position, and clears the selection. + /// + /// # Safety + /// + /// This function performs no checks that the cursor is valid. "Valid" in this case means + /// that the TextBuffer has not been modified since you received the cursor from this class. + pub unsafe fn set_cursor(&mut self, cursor: Cursor) { + self.set_cursor_internal(cursor); + self.last_history_type = HistoryType::Other; + self.set_selection(None); + } + + fn set_cursor_for_selection(&mut self, cursor: Cursor) { + let beg = match self.selection { + Some(TextBufferSelection { beg, .. }) => beg, + None => self.cursor.logical_pos, + }; + + self.set_cursor_internal(cursor); + self.last_history_type = HistoryType::Other; + + let end = self.cursor.logical_pos; + self.set_selection(if beg == end { None } else { Some(TextBufferSelection { beg, end }) }); + } + + fn set_cursor_internal(&mut self, cursor: Cursor) { + debug_assert!( + cursor.offset <= self.text_length() + && cursor.logical_pos.x >= 0 + && cursor.logical_pos.y >= 0 + && cursor.logical_pos.y <= self.stats.logical_lines + && cursor.visual_pos.x >= 0 + && (self.word_wrap_column <= 0 || cursor.visual_pos.x <= self.word_wrap_column) + && cursor.visual_pos.y >= 0 + && cursor.visual_pos.y <= self.stats.visual_lines + ); + self.cursor = cursor; + } + + /// Extracts a rectangular region of the text buffer and writes it to the framebuffer. + /// The `destination` rect is framebuffer coordinates. The extracted region within this + /// text buffer has the given `origin` and the same size as the `destination` rect. + pub fn render( + &mut self, + origin: Point, + destination: Rect, + focused: bool, + fb: &mut Framebuffer, + ) -> Option { + if destination.is_empty() { + return None; + } + + let scratch = scratch_arena(None); + let width = destination.width(); + let height = destination.height(); + let line_number_width = self.margin_width.max(3) as usize - 3; + let text_width = width - self.margin_width; + let mut visualizer_buf = [0xE2, 0x90, 0x80]; // U+2400 in UTF8 + let mut line = ArenaString::new_in(&scratch); + let mut visual_pos_x_max = 0; + + // Pick the cursor closer to the `origin.y`. + let mut cursor = { + let a = self.cursor; + let b = self.cursor_for_rendering.unwrap_or_default(); + let da = (a.visual_pos.y - origin.y).abs(); + let db = (b.visual_pos.y - origin.y).abs(); + if da < db { a } else { b } + }; + + let [selection_beg, selection_end] = match self.selection { + None => [Point::MIN, Point::MIN], + Some(TextBufferSelection { beg, end }) => minmax(beg, end), + }; + + line.reserve(width as usize * 2); + + for y in 0..height { + line.clear(); + + let visual_line = origin.y + y; + let mut cursor_beg = + self.cursor_move_to_visual_internal(cursor, Point { x: origin.x, y: visual_line }); + let cursor_end = self.cursor_move_to_visual_internal( + cursor_beg, + Point { x: origin.x + text_width, y: visual_line }, + ); + + // Accelerate the next render pass by remembering where we started off. + if y == 0 { + self.cursor_for_rendering = Some(cursor_beg); + } + + if line_number_width != 0 { + if visual_line >= self.stats.visual_lines { + // Past the end of the buffer? Place " | " in the margin. + // Since we know that we won't see line numbers greater than i64::MAX (9223372036854775807) + // any time soon, we can use a static string as the template (`MARGIN`) and slice it, + // because `line_number_width` can't possibly be larger than 19. + let off = 19 - line_number_width; + unsafe { std::hint::assert_unchecked(off < MARGIN_TEMPLATE.len()) }; + line.push_str(&MARGIN_TEMPLATE[off..]); + } else if self.word_wrap_column <= 0 || cursor_beg.logical_pos.x == 0 { + // Regular line? Place "123 | " in the margin. + _ = write!(line, "{:1$} │ ", cursor_beg.logical_pos.y + 1, line_number_width); + } else { + // Wrapped line? Place " ... | " in the margin. + let number_width = (cursor_beg.logical_pos.y + 1).ilog10() as usize + 1; + _ = write!( + line, + "{0:1$}{0:∙<2$} │ ", + "", + line_number_width - number_width, + number_width + ); + // Blending in the background color will "dim" the indicator dots. + let left = destination.left; + let top = destination.top + y; + fb.blend_fg( + Rect { + left, + top, + right: left + line_number_width as CoordType, + bottom: top + 1, + }, + fb.indexed_alpha(IndexedColor::Background, 1, 2), + ); + } + } + + // Nothing to do if the entire line is empty. + if cursor_beg.offset != cursor_end.offset { + // If we couldn't reach the left edge, we may have stopped short due to a wide glyph. + // In that case we'll try to find the next character and then compute by how many + // columns it overlaps the left edge (can be anything between 1 and 7). + if cursor_beg.visual_pos.x < origin.x { + let cursor_next = self.cursor_move_to_logical_internal( + cursor_beg, + Point { x: cursor_beg.logical_pos.x + 1, y: cursor_beg.logical_pos.y }, + ); + + if cursor_next.visual_pos.x > origin.x { + let overlap = cursor_next.visual_pos.x - origin.x; + debug_assert!((1..=7).contains(&overlap)); + line.push_str(&TAB_WHITESPACE[..overlap as usize]); + cursor_beg = cursor_next; + } + } + + fn find_control_char(text: &[u8], mut offset: usize) -> usize { + while offset < text.len() && (text[offset] >= 0x20 && text[offset] != 0x7f) { + offset += 1; + } + offset + } + + let mut global_off = cursor_beg.offset; + let mut cursor_tab = cursor_beg; + + while global_off < cursor_end.offset { + let chunk = self.read_forward(global_off); + let chunk = &chunk[..chunk.len().min(cursor_end.offset - global_off)]; + + let mut chunk_off = 0; + while chunk_off < chunk.len() { + let beg = chunk_off; + chunk_off = find_control_char(chunk, beg); + + for chunk in chunk[beg..chunk_off].utf8_chunks() { + if !chunk.valid().is_empty() { + line.push_str(chunk.valid()); + } + if !chunk.invalid().is_empty() { + line.push('\u{FFFD}'); + } + } + + while chunk_off < chunk.len() + && (chunk[chunk_off] < 0x20 || chunk[chunk_off] == 0x7f) + { + let ch = chunk[chunk_off]; + chunk_off += 1; + + if ch == b'\t' { + cursor_tab = self.cursor_move_to_offset_internal( + cursor_tab, + global_off + chunk_off - 1, + ); + let tab_size = self.tab_size - (cursor_tab.column % self.tab_size); + line.push_str(&TAB_WHITESPACE[..tab_size as usize]); + + // Since we know that we just aligned ourselves to the next tab stop, + // we can trivially process any successive tabs. + while chunk_off < chunk.len() && chunk[chunk_off] == b'\t' { + line.push_str(&TAB_WHITESPACE[..self.tab_size as usize]); + chunk_off += 1; + } + continue; + } + + visualizer_buf[2] = if ch == 0x7F { + 0xA1 // U+2421 + } else { + 0x80 | ch // 0x00..=0x1F => U+2400..=U+241F + }; + // Our manually constructed UTF8 is never going to be invalid. Trust. + line.push_str(unsafe { str::from_utf8_unchecked(&visualizer_buf) }); + } + } + + global_off += chunk.len(); + } + + visual_pos_x_max = visual_pos_x_max.max(cursor_end.visual_pos.x); + } + + fb.replace_text(destination.top + y, destination.left, destination.right, &line); + + // Draw the selection on this line, if any. + // FYI: `cursor_beg.visual_pos.y == visual_line` is necessary as the `visual_line` + // may be past the end of the document, and so it may not receive a highlight. + if cursor_beg.visual_pos.y == visual_line + && selection_beg <= cursor_end.logical_pos + && selection_end >= cursor_beg.logical_pos + { + // By default, we assume the entire line is selected. + let mut beg = 0; + let mut end = COORD_TYPE_SAFE_MAX; + let mut cursor = cursor_beg; + + // The start of the selection is within this line. We need to update selection_beg. + if selection_beg <= cursor_end.logical_pos + && selection_beg >= cursor_beg.logical_pos + { + cursor = self.cursor_move_to_logical_internal(cursor, selection_beg); + beg = cursor.visual_pos.x; + } + + // The end of the selection is within this line. We need to update selection_end. + if selection_end <= cursor_end.logical_pos + && selection_end >= cursor_beg.logical_pos + { + cursor = self.cursor_move_to_logical_internal(cursor, selection_end); + end = cursor.visual_pos.x; + } + + beg = beg.max(origin.x); + end = end.min(origin.x + text_width); + + let left = destination.left + self.margin_width - origin.x; + let top = destination.top + y; + let rect = Rect { left: left + beg, top, right: left + end, bottom: top + 1 }; + + let mut bg = oklab_blend( + fb.indexed(IndexedColor::Foreground), + fb.indexed_alpha(IndexedColor::BrightBlue, 1, 2), + ); + if !focused { + bg = oklab_blend(bg, fb.indexed_alpha(IndexedColor::Background, 1, 2)) + }; + let fg = fb.contrasted(bg); + fb.blend_bg(rect, bg); + fb.blend_fg(rect, fg); + } + + cursor = cursor_end; + } + + // Colorize the margin that we wrote above. + if self.margin_width > 0 { + let margin = Rect { + left: destination.left, + top: destination.top, + right: destination.left + self.margin_width, + bottom: destination.bottom, + }; + fb.blend_fg(margin, 0x7f3f3f3f); + } + + if self.ruler > 0 { + let left = destination.left + self.margin_width + (self.ruler - origin.x).max(0); + let right = destination.right; + if left < right { + fb.blend_bg( + Rect { left, top: destination.top, right, bottom: destination.bottom }, + fb.indexed_alpha(IndexedColor::BrightRed, 1, 4), + ); + } + } + + if focused { + let mut x = self.cursor.visual_pos.x; + let mut y = self.cursor.visual_pos.y; + + if self.word_wrap_column > 0 && x >= self.word_wrap_column { + // The line the cursor is on wraps exactly on the word wrap column which + // means the cursor is invisible. We need to move it to the next line. + x = 0; + y += 1; + } + + // Move the cursor into screen space. + x += destination.left - origin.x + self.margin_width; + y += destination.top - origin.y; + + let cursor = Point { x, y }; + let text = Rect { + left: destination.left + self.margin_width, + top: destination.top, + right: destination.right, + bottom: destination.bottom, + }; + + if text.contains(cursor) { + fb.set_cursor(cursor, self.overtype); + + if self.line_highlight_enabled && selection_beg >= selection_end { + fb.blend_bg( + Rect { + left: destination.left, + top: cursor.y, + right: destination.right, + bottom: cursor.y + 1, + }, + 0x50282828, + ); + } + } + } + + Some(RenderResult { visual_pos_x_max }) + } + + /// Inserts `text` at the current cursor position. + /// + /// If there's a current selection, it will be replaced. + /// The selection is cleared after the call. + pub fn write(&mut self, text: &[u8], raw: bool) { + if text.is_empty() { + return; + } + + if let Some((beg, end)) = self.selection_range_internal(false) { + self.edit_begin(HistoryType::Write, beg); + self.edit_delete(end); + self.set_selection(None); + } + if self.active_edit_depth <= 0 { + self.edit_begin(HistoryType::Write, self.cursor); + } + + let mut offset = 0; + let scratch = scratch_arena(None); + let mut newline_buffer = ArenaString::new_in(&scratch); + + loop { + // Can't use `unicode::newlines_forward` because bracketed paste uses CR instead of LF/CRLF. + let offset_next = memchr2(b'\r', b'\n', text, offset); + let line = &text[offset..offset_next]; + let column_before = self.cursor.logical_pos.x; + + // Write the contents of the line into the buffer. + let mut line_off = 0; + while line_off < line.len() { + // Split the line into chunks of non-tabs and tabs. + let mut plain = line; + if !raw && !self.indent_with_tabs { + let end = memchr2(b'\t', b'\t', line, line_off); + plain = &line[line_off..end]; + } + + // Non-tabs are written as-is, because the outer loop already handles newline translation. + self.edit_write(plain); + line_off += plain.len(); + + // Now replace tabs with spaces. + while line_off < line.len() && line[line_off] == b'\t' { + let spaces = self.tab_size - (self.cursor.column % self.tab_size); + let spaces = &TAB_WHITESPACE.as_bytes()[..spaces as usize]; + self.edit_write(spaces); + line_off += 1; + } + } + + if !raw && self.overtype { + let delete = self.cursor.logical_pos.x - column_before; + let end = self.cursor_move_to_logical_internal( + self.cursor, + Point { x: self.cursor.logical_pos.x + delete, y: self.cursor.logical_pos.y }, + ); + self.edit_delete(end); + } + + offset += line.len(); + if offset >= text.len() { + break; + } + + // First, write the newline. + newline_buffer.clear(); + newline_buffer.push_str(if self.newlines_are_crlf { "\r\n" } else { "\n" }); + + if !raw { + // We'll give the next line the same indentation as the previous one. + // This block figures out how much that is. We can't reuse that value, + // because " a\n a\n" should give the 3rd line a total indentation of 4. + // Assuming your terminal has bracketed paste, this won't be a concern though. + // (If it doesn't, use a different terminal.) + let tab_size = self.tab_size as usize; + let line_beg = self.goto_line_start(self.cursor, self.cursor.logical_pos.y); + let limit = self.cursor.offset; + let mut off = line_beg.offset; + let mut newline_indentation = 0usize; + + 'outer: while off < limit { + let chunk = self.read_forward(off); + let chunk = &chunk[..chunk.len().min(limit - off)]; + + for &c in chunk { + if c == b' ' { + newline_indentation += 1; + } else if c == b'\t' { + newline_indentation += tab_size - (newline_indentation % tab_size); + } else { + break 'outer; + } + } + + off += chunk.len(); + } + + // If tabs are enabled, add as many tabs as we can. + if self.indent_with_tabs { + let tab_count = newline_indentation / tab_size; + newline_buffer.push_repeat('\t', tab_count); + newline_indentation -= tab_count * tab_size; + } + + // If tabs are disabled, or if the indentation wasn't a multiple of the tab size, + // add spaces to make up the difference. + newline_buffer.push_repeat(' ', newline_indentation); + } + + self.edit_write(newline_buffer.as_bytes()); + + // Skip one CR/LF/CRLF. + if offset >= text.len() { + break; + } + if text[offset] == b'\r' { + offset += 1; + } + if offset >= text.len() { + break; + } + if text[offset] == b'\n' { + offset += 1; + } + if offset >= text.len() { + break; + } + } + + self.edit_end(); + } + + /// Deletes 1 grapheme cluster from the buffer. + /// `cursor_movements` is expected to be -1 for backspace and 1 for delete. + /// If there's a current selection, it will be deleted and `cursor_movements` ignored. + /// The selection is cleared after the call. + /// Deletes characters from the buffer based on a delta from the cursor. + pub fn delete(&mut self, granularity: CursorMovement, delta: CoordType) { + debug_assert!(delta == -1 || delta == 1); + + let mut beg; + let mut end; + + if let Some(r) = self.selection_range_internal(false) { + (beg, end) = r; + } else { + if (delta == -1 && self.cursor.offset == 0) + || (delta == 1 && self.cursor.offset >= self.text_length()) + { + // Nothing to delete. + return; + } + + beg = self.cursor; + end = self.cursor_move_delta_internal(beg, granularity, delta); + if beg.offset == end.offset { + return; + } + if beg.offset > end.offset { + mem::swap(&mut beg, &mut end); + } + } + + self.edit_begin(HistoryType::Delete, beg); + self.edit_delete(end); + self.edit_end(); + + self.set_selection(None); + } + + /// Returns the logical position of the first character on this line. + /// Return `.x == 0` if there are no non-whitespace characters. + pub fn indent_end_logical_pos(&self) -> Point { + let cursor = self.goto_line_start(self.cursor, self.cursor.logical_pos.y); + let mut chars = 0; + let mut offset = cursor.offset; + + 'outer: loop { + let chunk = self.read_forward(offset); + if chunk.is_empty() { + break; + } + + for &c in chunk { + if c == b'\n' || c == b'\r' || (c != b' ' && c != b'\t') { + break 'outer; + } + chars += 1; + } + + offset += chunk.len(); + } + + Point { x: chars, y: cursor.logical_pos.y } + } + + /// Unindents the current selection or line. + /// + /// TODO: This function is ripe for some optimizations: + /// * Instead of replacing the entire selection, + /// it should unindent each line directly (as if multiple cursors had been used). + /// * The cursor movement at the end is rather costly, but at least without word wrap + /// it should be possible to calculate it directly from the removed amount. + pub fn unindent(&mut self) { + let mut selection_beg = self.cursor.logical_pos; + let mut selection_end = selection_beg; + + if let Some(TextBufferSelection { beg, end }) = self.selection { + selection_beg = beg; + selection_end = end; + } + + let [beg, end] = minmax(selection_beg, selection_end); + let beg = self.cursor_move_to_logical_internal(self.cursor, Point { x: 0, y: beg.y }); + let end = self.cursor_move_to_logical_internal(beg, Point { x: CoordType::MAX, y: end.y }); + + let mut replacement = Vec::new(); + self.buffer.extract_raw(beg.offset, end.offset, &mut replacement, 0); + + let initial_len = replacement.len(); + let mut offset = 0; + let mut y = beg.logical_pos.y; + + loop { + let mut remove = 0; + + if replacement[offset] == b'\t' { + remove = 1; + } else { + while remove < self.tab_size as usize + && offset + remove < replacement.len() + && replacement[offset + remove] == b' ' + { + remove += 1; + } + } + + if remove > 0 { + replacement.drain(offset..offset + remove); + } + + if y == selection_beg.y { + selection_beg.x -= remove as CoordType; + } + if y == selection_end.y { + selection_end.x -= remove as CoordType; + } + + (offset, y) = unicode::newlines_forward(&replacement, offset, y, y + 1); + if offset >= replacement.len() { + break; + } + } + + if replacement.len() == initial_len { + // Nothing to do. + return; + } + + self.edit_begin(HistoryType::Other, beg); + self.edit_delete(end); + self.edit_write(&replacement); + self.edit_end(); + + if let Some(TextBufferSelection { beg, end }) = &mut self.selection { + *beg = selection_beg; + *end = selection_end; + } + + self.set_cursor_internal(self.cursor_move_to_logical_internal(self.cursor, selection_end)); + } + + /// Extracts the contents of the current selection. + /// May optionally delete it, if requested. This is meant to be used for Ctrl+X. + pub fn extract_selection(&mut self, delete: bool) -> Vec { + let Some((beg, end)) = self.selection_range_internal(true) else { + return Vec::new(); + }; + + let mut out = Vec::new(); + self.buffer.extract_raw(beg.offset, end.offset, &mut out, 0); + + if delete && !out.is_empty() { + self.edit_begin(HistoryType::Delete, beg); + self.edit_delete(end); + self.edit_end(); + self.set_selection(None); + } + + out + } + + /// Extracts the contents of the current selection the user made. + /// This differs from [`TextBuffer::extract_selection()`] in that + /// it does nothing if the selection was made by searching. + pub fn extract_user_selection(&mut self, delete: bool) -> Option> { + if !self.has_selection() { + return None; + } + + if let Some(search) = &self.search { + let search = unsafe { &*search.get() }; + if search.selection_generation == self.selection_generation { + return None; + } + } + + Some(self.extract_selection(delete)) + } + + /// Returns the current selection anchors, or `None` if there + /// is no selection. The returned logical positions are sorted. + pub fn selection_range(&self) -> Option<(Cursor, Cursor)> { + self.selection_range_internal(false) + } + + /// Returns the current selection anchors. + /// + /// If there's no selection and `line_fallback` is `true`, + /// the start/end of the current line are returned. + /// This is meant to be used for Ctrl+C / Ctrl+X. + fn selection_range_internal(&self, line_fallback: bool) -> Option<(Cursor, Cursor)> { + let [beg, end] = match self.selection { + None if !line_fallback => return None, + None => [ + Point { x: 0, y: self.cursor.logical_pos.y }, + Point { x: 0, y: self.cursor.logical_pos.y + 1 }, + ], + Some(TextBufferSelection { beg, end }) => minmax(beg, end), + }; + + let beg = self.cursor_move_to_logical_internal(self.cursor, beg); + let end = self.cursor_move_to_logical_internal(beg, end); + + if beg.offset < end.offset { Some((beg, end)) } else { None } + } + + /// Starts a new edit operation. + /// This is used for tracking the undo/redo history. + fn edit_begin(&mut self, history_type: HistoryType, cursor: Cursor) { + self.active_edit_depth += 1; + if self.active_edit_depth > 1 { + return; + } + + let cursor_before = self.cursor; + self.set_cursor_internal(cursor); + + // If both the last and this are a Write/Delete operation, we skip allocating a new undo history item. + if history_type != self.last_history_type + || !matches!(history_type, HistoryType::Write | HistoryType::Delete) + { + self.redo_stack.clear(); + while self.undo_stack.len() > 1000 { + self.undo_stack.pop_front(); + } + + self.last_history_type = history_type; + self.undo_stack.push_back(SemiRefCell::new(HistoryEntry { + cursor_before: cursor_before.logical_pos, + selection_before: self.selection, + stats_before: self.stats, + generation_before: self.buffer.generation(), + cursor: cursor.logical_pos, + deleted: Vec::new(), + added: Vec::new(), + })); + } + + self.active_edit_off = cursor.offset; + + // If word-wrap is enabled, the visual layout of all logical lines affected by the write + // may have changed. This includes even text before the insertion point up to the line + // start, because this write may have joined with a word before the initial cursor. + // See other uses of `word_wrap_cursor_next_line` in this function. + if self.word_wrap_column > 0 { + let safe_start = self.goto_line_start(cursor, cursor.logical_pos.y); + let next_line = self.cursor_move_to_logical_internal( + cursor, + Point { x: 0, y: cursor.logical_pos.y + 1 }, + ); + self.active_edit_line_info = Some(ActiveEditLineInfo { + safe_start, + line_height_in_rows: next_line.visual_pos.y - safe_start.visual_pos.y, + distance_next_line_start: next_line.offset - cursor.offset, + }); + } + } + + /// Writes `text` into the buffer at the current cursor position. + /// It records the change in the undo stack. + fn edit_write(&mut self, text: &[u8]) { + let logical_y_before = self.cursor.logical_pos.y; + + // Copy the written portion into the undo entry. + { + let mut undo = self.undo_stack.back_mut().unwrap().borrow_mut(); + undo.added.extend_from_slice(text); + } + + // Write! + self.buffer.replace(self.active_edit_off..self.active_edit_off, text); + + // Move self.cursor to the end of the newly written text. Can't use `self.set_cursor_internal`, + // because we're still in the progress of recalculating the line stats. + self.active_edit_off += text.len(); + self.cursor = self.cursor_move_to_offset_internal(self.cursor, self.active_edit_off); + self.stats.logical_lines += self.cursor.logical_pos.y - logical_y_before; + } + + /// Deletes the text between the current cursor position and `to`. + /// It records the change in the undo stack. + fn edit_delete(&mut self, to: Cursor) { + debug_assert!(to.offset >= self.active_edit_off); + + let logical_y_before = self.cursor.logical_pos.y; + let off = self.active_edit_off; + let mut out_off = usize::MAX; + + let mut undo = self.undo_stack.back_mut().unwrap().borrow_mut(); + if self.cursor.logical_pos < undo.cursor { + out_off = 0; // Prepend the deleted portion. + undo.cursor = self.cursor.logical_pos; // Note the start of the deleted portion. + } + + // Copy the deleted portion into the undo entry. + let deleted = &mut undo.deleted; + self.buffer.extract_raw(off, to.offset, deleted, out_off); + + // Delete the portion from the buffer by enlarging the gap. + let count = to.offset - off; + self.buffer.allocate_gap(off, 0, count); + + self.stats.logical_lines += logical_y_before - to.logical_pos.y; + } + + /// Finalizes the current edit operation + /// and recalculates the line statistics. + fn edit_end(&mut self) { + self.active_edit_depth -= 1; + assert!(self.active_edit_depth >= 0); + if self.active_edit_depth > 0 { + return; + } + + #[cfg(debug_assertions)] + { + let entry = self.undo_stack.back_mut().unwrap().borrow_mut(); + debug_assert!(!entry.deleted.is_empty() || !entry.added.is_empty()); + } + + if let Some(info) = self.active_edit_line_info.take() { + let deleted_count = self.undo_stack.back_mut().unwrap().borrow_mut().deleted.len(); + let target = self.cursor.logical_pos; + + // From our safe position we can measure the actual visual position of the cursor. + self.set_cursor_internal(self.cursor_move_to_logical_internal(info.safe_start, target)); + + // If content is added at the insertion position, that's not a problem: + // We can just remeasure the height of this one line and calculate the delta. + // `deleted_count` is 0 in this case. + // + // The problem is when content is deleted, because it may affect lines + // beyond the end of the `next_line`. In that case we have to measure + // the entire buffer contents until the end to compute `self.stats.visual_lines`. + if deleted_count < info.distance_next_line_start { + // Now we can measure how many more visual rows this logical line spans. + let next_line = self + .cursor_move_to_logical_internal(self.cursor, Point { x: 0, y: target.y + 1 }); + let lines_before = info.line_height_in_rows; + let lines_after = next_line.visual_pos.y - info.safe_start.visual_pos.y; + self.stats.visual_lines += lines_after - lines_before; + } else { + let end = self.cursor_move_to_logical_internal(self.cursor, Point::MAX); + self.stats.visual_lines = end.visual_pos.y + 1; + } + } else { + // If word-wrap is disabled the visual line count always matches the logical one. + self.stats.visual_lines = self.stats.logical_lines; + } + + self.search = None; + + // Also takes care of clearing `cursor_for_rendering`. + self.reflow(false); + } + + /// Undo the last edit operation. + pub fn undo(&mut self) { + self.undo_redo(true); + } + + /// Redo the last undo operation. + pub fn redo(&mut self) { + self.undo_redo(false); + } + + fn undo_redo(&mut self, undo: bool) { + // Transfer the last entry from the undo stack to the redo stack or vice versa. + { + let (from, to) = if undo { + (&mut self.undo_stack, &mut self.redo_stack) + } else { + (&mut self.redo_stack, &mut self.undo_stack) + }; + + let Some(list) = from.cursor_back_mut().remove_current_as_list() else { + return; + }; + + to.cursor_back_mut().splice_after(list); + } + + let change = { + let to = if undo { &self.redo_stack } else { &self.undo_stack }; + to.back().unwrap() + }; + + // Move to the point where the modification took place. + let cursor = self.cursor_move_to_logical_internal(self.cursor, change.borrow().cursor); + + let safe_cursor = if self.word_wrap_column > 0 { + // If word-wrap is enabled, we need to move the cursor to the beginning of the line. + // This is because the undo/redo operation may have changed the visual position of the cursor. + self.goto_line_start(cursor, cursor.logical_pos.y) + } else { + cursor + }; + + { + let buffer_generation = self.buffer.generation(); + let mut change = change.borrow_mut(); + let change = &mut *change; + + // Undo: Whatever was deleted is now added and vice versa. + mem::swap(&mut change.deleted, &mut change.added); + + // Delete the inserted portion. + self.buffer.allocate_gap(cursor.offset, 0, change.deleted.len()); + + // Reinsert the deleted portion. + { + let added = &change.added[..]; + let mut beg = 0; + let mut offset = cursor.offset; + + while beg < added.len() { + let (end, line) = unicode::newlines_forward(added, beg, 0, 1); + let has_newline = line != 0; + let link = &added[beg..end]; + let line = unicode::strip_newline(link); + let mut written; + + { + let gap = self.buffer.allocate_gap(offset, line.len() + 2, 0); + written = slice_copy_safe(gap, line); + + if has_newline { + if self.newlines_are_crlf && written < gap.len() { + gap[written] = b'\r'; + written += 1; + } + if written < gap.len() { + gap[written] = b'\n'; + written += 1; + } + } + + self.buffer.commit_gap(written); + } + + beg = end; + offset += written; + } + } + + // Restore the previous line statistics. + mem::swap(&mut self.stats, &mut change.stats_before); + + // Restore the previous selection. + mem::swap(&mut self.selection, &mut change.selection_before); + + // Pretend as if the buffer was never modified. + self.buffer.set_generation(change.generation_before); + change.generation_before = buffer_generation; + + // Restore the previous cursor. + let cursor_before = + self.cursor_move_to_logical_internal(safe_cursor, change.cursor_before); + change.cursor_before = self.cursor.logical_pos; + // Can't use `set_cursor_internal` here, because we haven't updated the line stats yet. + self.cursor = cursor_before; + + if self.undo_stack.is_empty() { + self.last_history_type = HistoryType::Other; + } + } + + // Also takes care of clearing `cursor_for_rendering`. + self.reflow(false); + } + + /// For interfacing with ICU. + pub(crate) fn read_backward(&self, off: usize) -> &[u8] { + self.buffer.read_backward(off) + } + + /// For interfacing with ICU. + pub fn read_forward(&self, off: usize) -> &[u8] { + self.buffer.read_forward(off) + } +} + +pub enum Bom { + None, + UTF8, + UTF16LE, + UTF16BE, + UTF32LE, + UTF32BE, + GB18030, +} + +const BOM_MAX_LEN: usize = 4; + +fn detect_bom(bytes: &[u8]) -> Option<&'static str> { + if bytes.len() >= 4 { + if bytes.starts_with(b"\xFF\xFE\x00\x00") { + return Some("UTF-32LE"); + } + if bytes.starts_with(b"\x00\x00\xFE\xFF") { + return Some("UTF-32BE"); + } + if bytes.starts_with(b"\x84\x31\x95\x33") { + return Some("GB18030"); + } + } + if bytes.len() >= 3 && bytes.starts_with(b"\xEF\xBB\xBF") { + return Some("UTF-8"); + } + if bytes.len() >= 2 { + if bytes.starts_with(b"\xFF\xFE") { + return Some("UTF-16LE"); + } + if bytes.starts_with(b"\xFE\xFF") { + return Some("UTF-16BE"); + } + } + None +} diff --git a/pkgs/edit/src/buffer/navigation.rs b/pkgs/edit/src/buffer/navigation.rs new file mode 100644 index 0000000..a3c6aa8 --- /dev/null +++ b/pkgs/edit/src/buffer/navigation.rs @@ -0,0 +1,290 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::ops::Range; + +use crate::document::ReadableDocument; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum CharClass { + Whitespace, + Newline, + Separator, + Word, +} + +const fn construct_classifier(seperators: &[u8]) -> [CharClass; 256] { + let mut classifier = [CharClass::Word; 256]; + + classifier[b' ' as usize] = CharClass::Whitespace; + classifier[b'\t' as usize] = CharClass::Whitespace; + classifier[b'\n' as usize] = CharClass::Newline; + classifier[b'\r' as usize] = CharClass::Newline; + + let mut i = 0; + let len = seperators.len(); + while i < len { + let ch = seperators[i]; + assert!(ch < 128, "Only ASCII separators are supported."); + classifier[ch as usize] = CharClass::Separator; + i += 1; + } + + classifier +} + +const WORD_CLASSIFIER: [CharClass; 256] = + construct_classifier(br#"`~!@#$%^&*()-=+[{]}\|;:'",.<>/?"#); + +/// Finds the next word boundary given a document cursor offset. +/// Returns the offset of the next word boundary. +pub fn word_forward(doc: &dyn ReadableDocument, offset: usize) -> usize { + word_navigation(WordForward { doc, offset, chunk: &[], chunk_off: 0 }) +} + +/// The backward version of `word_forward`. +pub fn word_backward(doc: &dyn ReadableDocument, offset: usize) -> usize { + word_navigation(WordBackward { doc, offset, chunk: &[], chunk_off: 0 }) +} + +/// Word navigation implementation. Matches the behavior of VS Code. +fn word_navigation(mut nav: T) -> usize { + // First, fill `self.chunk` with at least 1 grapheme. + nav.read(); + + // Skip one newline, if any. + nav.skip_newline(); + + // Skip any whitespace. + nav.skip_class(CharClass::Whitespace); + + // Skip one word or seperator and take note of the class. + let class = nav.peek(CharClass::Whitespace); + if matches!(class, CharClass::Separator | CharClass::Word) { + nav.next(); + + let off = nav.offset(); + + // Continue skipping the same class. + nav.skip_class(class); + + // If the class was a separator and we only moved one character, + // continue skipping characters of the word class. + if off == nav.offset() && class == CharClass::Separator { + nav.skip_class(CharClass::Word); + } + } + + nav.offset() +} + +trait WordNavigation { + fn read(&mut self); + fn skip_newline(&mut self); + fn skip_class(&mut self, class: CharClass); + fn peek(&self, default: CharClass) -> CharClass; + fn next(&mut self); + fn offset(&self) -> usize; +} + +struct WordForward<'a> { + doc: &'a dyn ReadableDocument, + offset: usize, + chunk: &'a [u8], + chunk_off: usize, +} + +impl WordNavigation for WordForward<'_> { + fn read(&mut self) { + self.chunk = self.doc.read_forward(self.offset); + self.chunk_off = 0; + } + + fn skip_newline(&mut self) { + // We can rely on the fact that the document does not split graphemes across chunks. + // = If there's a newline it's wholly contained in this chunk. + // Unlike with `WordBackward`, we can't check for CR and LF separately as only a CR followed + // by a LF is a newline. A lone CR in the document is just a regular control character. + self.chunk_off += match self.chunk.get(self.chunk_off) { + Some(&b'\n') => 1, + Some(&b'\r') if self.chunk.get(self.chunk_off + 1) == Some(&b'\n') => 2, + _ => 0, + } + } + + fn skip_class(&mut self, class: CharClass) { + while !self.chunk.is_empty() { + while self.chunk_off < self.chunk.len() { + if WORD_CLASSIFIER[self.chunk[self.chunk_off] as usize] != class { + return; + } + self.chunk_off += 1; + } + + self.offset += self.chunk.len(); + self.chunk = self.doc.read_forward(self.offset); + self.chunk_off = 0; + } + } + + fn peek(&self, default: CharClass) -> CharClass { + if self.chunk_off < self.chunk.len() { + WORD_CLASSIFIER[self.chunk[self.chunk_off] as usize] + } else { + default + } + } + + fn next(&mut self) { + self.chunk_off += 1; + } + + fn offset(&self) -> usize { + self.offset + self.chunk_off + } +} + +struct WordBackward<'a> { + doc: &'a dyn ReadableDocument, + offset: usize, + chunk: &'a [u8], + chunk_off: usize, +} + +impl WordNavigation for WordBackward<'_> { + fn read(&mut self) { + self.chunk = self.doc.read_backward(self.offset); + self.chunk_off = self.chunk.len(); + } + + fn skip_newline(&mut self) { + // We can rely on the fact that the document does not split graphemes across chunks. + // = If there's a newline it's wholly contained in this chunk. + if self.chunk_off > 0 && self.chunk[self.chunk_off - 1] == b'\n' { + self.chunk_off -= 1; + } + if self.chunk_off > 0 && self.chunk[self.chunk_off - 1] == b'\r' { + self.chunk_off -= 1; + } + } + + fn skip_class(&mut self, class: CharClass) { + while !self.chunk.is_empty() { + while self.chunk_off > 0 { + if WORD_CLASSIFIER[self.chunk[self.chunk_off - 1] as usize] != class { + return; + } + self.chunk_off -= 1; + } + + self.offset -= self.chunk.len(); + self.chunk = self.doc.read_backward(self.offset); + self.chunk_off = self.chunk.len(); + } + } + + fn peek(&self, default: CharClass) -> CharClass { + if self.chunk_off > 0 { + WORD_CLASSIFIER[self.chunk[self.chunk_off - 1] as usize] + } else { + default + } + } + + fn next(&mut self) { + self.chunk_off -= 1; + } + + fn offset(&self) -> usize { + self.offset - self.chunk.len() + self.chunk_off + } +} + +/// Returns the offset range of the "word" at the given offset. +/// Does not cross newlines. Works similar to VS Code. +pub fn word_select(doc: &dyn ReadableDocument, offset: usize) -> Range { + let mut beg = offset; + let mut end = offset; + let mut class = CharClass::Newline; + + let mut chunk = doc.read_forward(end); + if !chunk.is_empty() { + // Not at the end of the document? Great! + // We default to using the next char as the class, because in terminals + // the cursor is usually always to the left of the cell you clicked on. + class = WORD_CLASSIFIER[chunk[0] as usize]; + + let mut chunk_off = 0; + + // Select the word, unless we hit a newline. + if class != CharClass::Newline { + loop { + chunk_off += 1; + end += 1; + + if chunk_off >= chunk.len() { + chunk = doc.read_forward(end); + chunk_off = 0; + if chunk.is_empty() { + break; + } + } + + if WORD_CLASSIFIER[chunk[chunk_off] as usize] != class { + break; + } + } + } + } + + let mut chunk = doc.read_backward(beg); + if !chunk.is_empty() { + let mut chunk_off = chunk.len(); + + // If we failed to determine the class, because we hit the end of the document + // or a newline, we fall back to using the previous character, of course. + if class == CharClass::Newline { + class = WORD_CLASSIFIER[chunk[chunk_off - 1] as usize]; + } + + // Select the word, unless we hit a newline. + if class != CharClass::Newline { + loop { + if WORD_CLASSIFIER[chunk[chunk_off - 1] as usize] != class { + break; + } + + chunk_off -= 1; + beg -= 1; + + if chunk_off == 0 { + chunk = doc.read_backward(beg); + chunk_off = chunk.len(); + if chunk.is_empty() { + break; + } + } + } + } + } + + beg..end +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_word_navigation() { + assert_eq!(word_forward(&"Hello World".as_bytes(), 0), 5); + assert_eq!(word_forward(&"Hello,World".as_bytes(), 0), 5); + assert_eq!(word_forward(&" Hello".as_bytes(), 0), 8); + assert_eq!(word_forward(&"\n\nHello".as_bytes(), 0), 1); + + assert_eq!(word_backward(&"Hello World".as_bytes(), 11), 6); + assert_eq!(word_backward(&"Hello,World".as_bytes(), 10), 6); + assert_eq!(word_backward(&"Hello ".as_bytes(), 7), 0); + assert_eq!(word_backward(&"Hello\n\n".as_bytes(), 7), 6); + } +} diff --git a/pkgs/edit/src/cell.rs b/pkgs/edit/src/cell.rs new file mode 100644 index 0000000..0e29039 --- /dev/null +++ b/pkgs/edit/src/cell.rs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! [`std::cell::RefCell`], but without runtime checks in release builds. + +#[cfg(debug_assertions)] +pub use debug::*; +#[cfg(not(debug_assertions))] +pub use release::*; + +#[allow(unused)] +#[cfg(debug_assertions)] +mod debug { + pub type SemiRefCell = std::cell::RefCell; + pub type Ref<'b, T> = std::cell::Ref<'b, T>; + pub type RefMut<'b, T> = std::cell::RefMut<'b, T>; +} + +#[cfg(not(debug_assertions))] +mod release { + #[derive(Default)] + #[repr(transparent)] + pub struct SemiRefCell(std::cell::UnsafeCell); + + impl SemiRefCell { + #[inline(always)] + pub const fn new(value: T) -> Self { + Self(std::cell::UnsafeCell::new(value)) + } + + #[inline(always)] + pub const fn as_ptr(&self) -> *mut T { + self.0.get() + } + + #[inline(always)] + pub const fn borrow(&self) -> Ref<'_, T> { + Ref(unsafe { &*self.0.get() }) + } + + #[inline(always)] + pub const fn borrow_mut(&self) -> RefMut<'_, T> { + RefMut(unsafe { &mut *self.0.get() }) + } + } + + #[repr(transparent)] + pub struct Ref<'b, T>(&'b T); + + impl<'b, T> Ref<'b, T> { + #[inline(always)] + pub fn clone(orig: &Ref<'b, T>) -> Ref<'b, T> { + Ref(orig.0) + } + } + + impl<'b, T> std::ops::Deref for Ref<'b, T> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.0 + } + } + + #[repr(transparent)] + pub struct RefMut<'b, T>(&'b mut T); + + impl<'b, T> std::ops::Deref for RefMut<'b, T> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.0 + } + } + + impl<'b, T> std::ops::DerefMut for RefMut<'b, T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + self.0 + } + } +} diff --git a/pkgs/edit/src/document.rs b/pkgs/edit/src/document.rs new file mode 100644 index 0000000..0ae9eb8 --- /dev/null +++ b/pkgs/edit/src/document.rs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Abstractions over reading/writing arbitrary text containers. + +use std::ffi::OsString; +use std::mem; +use std::ops::Range; +use std::path::PathBuf; + +use crate::arena::{ArenaString, scratch_arena}; +use crate::helpers::ReplaceRange as _; + +/// An abstraction over reading from text containers. +pub trait ReadableDocument { + /// Read some bytes starting at (including) the given absolute offset. + /// + /// # Warning + /// + /// * Be lenient on inputs: + /// * The given offset may be out of bounds and you MUST clamp it. + /// * You should not assume that offsets are at grapheme cluster boundaries. + /// * Be strict on outputs: + /// * You MUST NOT break grapheme clusters across chunks. + /// * You MUST NOT return an empty slice unless the offset is at or beyond the end. + fn read_forward(&self, off: usize) -> &[u8]; + + /// Read some bytes before (but not including) the given absolute offset. + /// + /// # Warning + /// + /// * Be lenient on inputs: + /// * The given offset may be out of bounds and you MUST clamp it. + /// * You should not assume that offsets are at grapheme cluster boundaries. + /// * Be strict on outputs: + /// * You MUST NOT break grapheme clusters across chunks. + /// * You MUST NOT return an empty slice unless the offset is zero. + fn read_backward(&self, off: usize) -> &[u8]; +} + +/// An abstraction over writing to text containers. +pub trait WriteableDocument: ReadableDocument { + /// Replace the given range with the given bytes. + /// + /// # Warning + /// + /// * The given range may be out of bounds and you MUST clamp it. + /// * The replacement may not be valid UTF8. + fn replace(&mut self, range: Range, replacement: &[u8]); +} + +impl ReadableDocument for &[u8] { + fn read_forward(&self, off: usize) -> &[u8] { + let s = *self; + &s[off.min(s.len())..] + } + + fn read_backward(&self, off: usize) -> &[u8] { + let s = *self; + &s[..off.min(s.len())] + } +} + +impl ReadableDocument for String { + fn read_forward(&self, off: usize) -> &[u8] { + let s = self.as_bytes(); + &s[off.min(s.len())..] + } + + fn read_backward(&self, off: usize) -> &[u8] { + let s = self.as_bytes(); + &s[..off.min(s.len())] + } +} + +impl WriteableDocument for String { + fn replace(&mut self, range: Range, replacement: &[u8]) { + // `replacement` is not guaranteed to be valid UTF-8, so we need to sanitize it. + let scratch = scratch_arena(None); + let utf8 = ArenaString::from_utf8_lossy(&scratch, replacement); + let src = match &utf8 { + Ok(s) => s, + Err(s) => s.as_str(), + }; + + // SAFETY: `range` is guaranteed to be on codepoint boundaries. + unsafe { self.as_mut_vec() }.replace_range(range, src.as_bytes()); + } +} + +impl ReadableDocument for PathBuf { + fn read_forward(&self, off: usize) -> &[u8] { + let s = self.as_os_str().as_encoded_bytes(); + &s[off.min(s.len())..] + } + + fn read_backward(&self, off: usize) -> &[u8] { + let s = self.as_os_str().as_encoded_bytes(); + &s[..off.min(s.len())] + } +} + +impl WriteableDocument for PathBuf { + fn replace(&mut self, range: Range, replacement: &[u8]) { + let mut vec = mem::take(self).into_os_string().into_encoded_bytes(); + vec.replace_range(range, replacement); + *self = unsafe { PathBuf::from(OsString::from_encoded_bytes_unchecked(vec)) }; + } +} diff --git a/pkgs/edit/src/framebuffer.rs b/pkgs/edit/src/framebuffer.rs new file mode 100644 index 0000000..71031b3 --- /dev/null +++ b/pkgs/edit/src/framebuffer.rs @@ -0,0 +1,888 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! A shoddy framebuffer for terminal applications. + +use std::cell::Cell; +use std::fmt::Write; +use std::ops::{BitOr, BitXor}; +use std::ptr; +use std::slice::ChunksExact; + +use crate::arena::{Arena, ArenaString}; +use crate::helpers::{CoordType, Point, Rect, Size}; +use crate::oklab::{oklab_blend, srgb_to_oklab}; +use crate::simd::{MemsetSafe, memset}; +use crate::unicode::MeasurementConfig; + +// Same constants as used in the PCG family of RNGs. +#[cfg(target_pointer_width = "32")] +const HASH_MULTIPLIER: usize = 747796405; // https://doi.org/10.1090/S0025-5718-99-00996-5, Table 5 +#[cfg(target_pointer_width = "64")] +const HASH_MULTIPLIER: usize = 6364136223846793005; // Knuth's MMIX multiplier + +/// The size of our cache table. 1<<8 = 256. +const CACHE_TABLE_LOG2_SIZE: usize = 8; +const CACHE_TABLE_SIZE: usize = 1 << CACHE_TABLE_LOG2_SIZE; +/// To index into the cache table, we use `color * HASH_MULTIPLIER` as the hash. +/// Since the multiplication "shifts" the bits up, we don't just mask the lowest +/// 8 bits out, but rather shift 56 bits down to get the best bits from the top. +const CACHE_TABLE_SHIFT: usize = usize::BITS as usize - CACHE_TABLE_LOG2_SIZE; + +/// Standard 16 VT & default foreground/background colors. +#[derive(Clone, Copy)] +pub enum IndexedColor { + Black, + Red, + Green, + Yellow, + Blue, + Magenta, + Cyan, + White, + BrightBlack, + BrightRed, + BrightGreen, + BrightYellow, + BrightBlue, + BrightMagenta, + BrightCyan, + BrightWhite, + + Background, + Foreground, +} + +/// Number of indices used by [`IndexedColor`]. +pub const INDEXED_COLORS_COUNT: usize = 18; + +/// Fallback theme. Matches Windows Terminal's Ottosson theme. +pub const DEFAULT_THEME: [u32; INDEXED_COLORS_COUNT] = [ + 0xff000000, 0xff212cbe, 0xff3aae3f, 0xff4a9abe, 0xffbe4d20, 0xffbe54bb, 0xffb2a700, 0xffbebebe, + 0xff808080, 0xff303eff, 0xff51ea58, 0xff44c9ff, 0xffff6a2f, 0xffff74fc, 0xfff0e100, 0xffffffff, + 0xff000000, 0xffffffff, +]; + +/// A shoddy framebuffer for terminal applications. +/// +/// The idea is that you create a [`Framebuffer`], draw a bunch of text and +/// colors into it, and it takes care of figuring out what changed since the +/// last rendering and sending the differences as VT to the terminal. +/// +/// This is an improvement over how many other terminal applications work, +/// as they fail to accurately track what changed. If you watch the output +/// of `vim` for instance, you'll notice that it redraws unrelated parts of +/// the screen all the time. +pub struct Framebuffer { + /// Store the color palette. + indexed_colors: [u32; INDEXED_COLORS_COUNT], + /// Front and back buffers. Indexed by `frame_counter & 1`. + buffers: [Buffer; 2], + /// The current frame counter. Increments on every `flip` call. + frame_counter: usize, + /// The colors used for `contrast()`. It stores the default colors + /// of the palette as [dark, light], unless the palette is recognized + /// as a light them, in which case it swaps them. + auto_colors: [u32; 2], + /// A cache table for previously contrasted colors. + /// See: + contrast_colors: [Cell<(u32, u32)>; CACHE_TABLE_SIZE], + background_fill: u32, + foreground_fill: u32, +} + +impl Framebuffer { + /// Creates a new framebuffer. + pub fn new() -> Self { + Self { + indexed_colors: DEFAULT_THEME, + buffers: Default::default(), + frame_counter: 0, + auto_colors: [ + DEFAULT_THEME[IndexedColor::Black as usize], + DEFAULT_THEME[IndexedColor::White as usize], + ], + contrast_colors: [const { Cell::new((0, 0)) }; CACHE_TABLE_SIZE], + background_fill: DEFAULT_THEME[IndexedColor::Background as usize], + foreground_fill: DEFAULT_THEME[IndexedColor::Foreground as usize], + } + } + + /// Sets the base color palette. + /// + /// If you call this method, [`Framebuffer`] expects that you + /// successfully detect the light/dark mode of the terminal. + pub fn set_indexed_colors(&mut self, colors: [u32; INDEXED_COLORS_COUNT]) { + self.indexed_colors = colors; + self.background_fill = 0; + self.foreground_fill = 0; + + self.auto_colors = [ + self.indexed_colors[IndexedColor::Black as usize], + self.indexed_colors[IndexedColor::BrightWhite as usize], + ]; + if !Self::is_dark(self.auto_colors[0]) { + self.auto_colors.swap(0, 1); + } + } + + /// Begins a new frame with the given `size`. + pub fn flip(&mut self, size: Size) { + if size != self.buffers[0].bg_bitmap.size { + for buffer in &mut self.buffers { + buffer.text = LineBuffer::new(size); + buffer.bg_bitmap = Bitmap::new(size); + buffer.fg_bitmap = Bitmap::new(size); + buffer.attributes = AttributeBuffer::new(size); + } + + let front = &mut self.buffers[self.frame_counter & 1]; + // Trigger a full redraw. (Yes, it's a hack.) + front.fg_bitmap.fill(1); + // Trigger a cursor update as well, just to be sure. + front.cursor = Cursor::new_invalid(); + } + + self.frame_counter = self.frame_counter.wrapping_add(1); + + let back = &mut self.buffers[self.frame_counter & 1]; + + back.text.fill_whitespace(); + back.bg_bitmap.fill(self.background_fill); + back.fg_bitmap.fill(self.foreground_fill); + back.attributes.reset(); + back.cursor = Cursor::new_disabled(); + } + + /// Replaces text contents in a single line of the framebuffer. + /// All coordinates are in viewport coordinates. + /// Assumes that control characters have been replaced or escaped. + pub fn replace_text( + &mut self, + y: CoordType, + origin_x: CoordType, + clip_right: CoordType, + text: &str, + ) { + let back = &mut self.buffers[self.frame_counter & 1]; + back.text.replace_text(y, origin_x, clip_right, text) + } + + /// Draws a scrollbar in the given `track` rectangle. + /// + /// Not entirely sure why I put it here instead of elsewhere. + /// + /// # Parameters + /// + /// * `clip_rect`: Clips the rendering to this rectangle. + /// This is relevant when you have scrollareas inside scrollareas. + /// * `track`: The rectangle in which to draw the scrollbar. + /// In absolute viewport coordinates. + /// * `content_offset`: The current offset of the scrollarea. + /// * `content_height`: The height of the scrollarea content. + pub fn draw_scrollbar( + &mut self, + clip_rect: Rect, + track: Rect, + content_offset: CoordType, + content_height: CoordType, + ) -> CoordType { + let track_clipped = track.intersect(clip_rect); + if track_clipped.is_empty() { + return 0; + } + + let viewport_height = track.height(); + // The content height is at least the viewport height. + let content_height = content_height.max(viewport_height); + + // No need to draw a scrollbar if the content fits in the viewport. + let content_offset_max = content_height - viewport_height; + if content_offset_max == 0 { + return 0; + } + + // The content offset must be at least one viewport height from the bottom. + // You don't want to scroll past the end after all... + let content_offset = content_offset.clamp(0, content_offset_max); + + // In order to increase the visual resolution of the scrollbar, + // we'll use 1/8th blocks to represent the thumb. + // First, scale the offsets to get that 1/8th resolution. + let viewport_height = viewport_height as i64 * 8; + let content_offset_max = content_offset_max as i64 * 8; + let content_offset = content_offset as i64 * 8; + let content_height = content_height as i64 * 8; + + // The proportional thumb height (0-1) is the fraction of viewport and + // content height. The taller the content, the smaller the thumb: + // = viewport_height / content_height + // We then scale that to the viewport height to get the height in 1/8th units. + // = viewport_height * viewport_height / content_height + // We add content_height/2 to round the integer division, which results in a numerator of: + // = viewport_height * viewport_height + content_height / 2 + let numerator = viewport_height * viewport_height + content_height / 2; + let thumb_height = numerator / content_height; + // Ensure the thumb has a minimum size of 1 row. + let thumb_height = thumb_height.max(8); + + // The proportional thumb top position (0-1) is: + // = content_offset / content_offset_max + // The maximum thumb top position is the viewport height minus the thumb height: + // = viewport_height - thumb_height + // To get the thumb top position in 1/8th units, we multiply both: + // = (viewport_height - thumb_height) * content_offset / content_offset_max + // We add content_offset_max/2 to round the integer division, which results in a numerator of: + // = (viewport_height - thumb_height) * content_offset + content_offset_max / 2 + let numerator = (viewport_height - thumb_height) * content_offset + content_offset_max / 2; + let thumb_top = numerator / content_offset_max; + // The thumb bottom position is the thumb top position plus the thumb height. + let thumb_bottom = thumb_top + thumb_height; + + // Shift to absolute coordinates. + let thumb_top = thumb_top + track.top as i64 * 8; + let thumb_bottom = thumb_bottom + track.top as i64 * 8; + + // Clamp to the visible area. + let thumb_top = thumb_top.max(track_clipped.top as i64 * 8); + let thumb_bottom = thumb_bottom.min(track_clipped.bottom as i64 * 8); + + // Calculate the height of the top/bottom cell of the thumb. + let top_fract = (thumb_top % 8) as CoordType; + let bottom_fract = (thumb_bottom % 8) as CoordType; + + // Shift to absolute coordinates. + let thumb_top = ((thumb_top + 7) / 8) as CoordType; + let thumb_bottom = (thumb_bottom / 8) as CoordType; + + self.blend_bg(track_clipped, self.indexed(IndexedColor::BrightBlack)); + self.blend_fg(track_clipped, self.indexed(IndexedColor::BrightWhite)); + + // Draw the full blocks. + for y in thumb_top..thumb_bottom { + self.replace_text(y, track_clipped.left, track_clipped.right, "█"); + } + + // Draw the top/bottom cell of the thumb. + // U+2581 to U+2588, 1/8th block to 8/8th block elements glyphs: ▁▂▃▄▅▆▇█ + // In UTF8: E2 96 81 to E2 96 88 + let mut fract_buf = [0xE2, 0x96, 0x88]; + if top_fract != 0 { + fract_buf[2] = (0x88 - top_fract) as u8; + self.replace_text(thumb_top - 1, track_clipped.left, track_clipped.right, unsafe { + std::str::from_utf8_unchecked(&fract_buf) + }); + } + if bottom_fract != 0 { + fract_buf[2] = (0x88 - bottom_fract) as u8; + self.replace_text(thumb_bottom, track_clipped.left, track_clipped.right, unsafe { + std::str::from_utf8_unchecked(&fract_buf) + }); + let rect = Rect { + left: track_clipped.left, + top: thumb_bottom, + right: track_clipped.right, + bottom: thumb_bottom + 1, + }; + self.blend_bg(rect, self.indexed(IndexedColor::BrightWhite)); + self.blend_fg(rect, self.indexed(IndexedColor::BrightBlack)); + } + + ((thumb_height + 4) / 8) as CoordType + } + + #[inline] + pub fn indexed(&self, index: IndexedColor) -> u32 { + self.indexed_colors[index as usize] + } + + /// Returns a color from the palette. + /// + /// To facilitate constant folding by the compiler, + /// alpha is given as a fraction (`numerator` / `denominator`). + #[inline] + pub fn indexed_alpha(&self, index: IndexedColor, numerator: u32, denominator: u32) -> u32 { + let c = self.indexed_colors[index as usize]; + let a = 255 * numerator / denominator; + let r = (((c >> 16) & 0xFF) * numerator) / denominator; + let g = (((c >> 8) & 0xFF) * numerator) / denominator; + let b = ((c & 0xFF) * numerator) / denominator; + a << 24 | r << 16 | g << 8 | b + } + + /// Returns a color opposite to the brightness of the given `color`. + pub fn contrasted(&self, color: u32) -> u32 { + let idx = (color as usize).wrapping_mul(HASH_MULTIPLIER) >> CACHE_TABLE_SHIFT; + let slot = self.contrast_colors[idx].get(); + if slot.0 == color { slot.1 } else { self.contrasted_slow(color) } + } + + #[cold] + fn contrasted_slow(&self, color: u32) -> u32 { + let idx = (color as usize).wrapping_mul(HASH_MULTIPLIER) >> CACHE_TABLE_SHIFT; + let contrast = self.auto_colors[Self::is_dark(color) as usize]; + self.contrast_colors[idx].set((color, contrast)); + contrast + } + + fn is_dark(color: u32) -> bool { + srgb_to_oklab(color).l < 0.5 + } + + /// Blends the given sRGB color onto the background bitmap. + /// + /// TODO: The current approach blends foreground/background independently, + /// but ideally `blend_bg` with semi-transparent dark should also darken text below it. + pub fn blend_bg(&mut self, target: Rect, bg: u32) { + let back = &mut self.buffers[self.frame_counter & 1]; + back.bg_bitmap.blend(target, bg); + } + + /// Blends the given sRGB color onto the foreground bitmap. + /// + /// TODO: The current approach blends foreground/background independently, + /// but ideally `blend_fg` should blend with the background color below it. + pub fn blend_fg(&mut self, target: Rect, fg: u32) { + let back = &mut self.buffers[self.frame_counter & 1]; + back.fg_bitmap.blend(target, fg); + } + + /// Reverses the foreground and background colors in the given rectangle. + pub fn reverse(&mut self, target: Rect) { + let back = &mut self.buffers[self.frame_counter & 1]; + + let target = target.intersect(back.bg_bitmap.size.as_rect()); + if target.is_empty() { + return; + } + + let top = target.top as usize; + let bottom = target.bottom as usize; + let left = target.left as usize; + let right = target.right as usize; + let stride = back.bg_bitmap.size.width as usize; + + for y in top..bottom { + let beg = y * stride + left; + let end = y * stride + right; + let bg = &mut back.bg_bitmap.data[beg..end]; + let fg = &mut back.fg_bitmap.data[beg..end]; + bg.swap_with_slice(fg); + } + } + + /// Replaces VT attributes in the given rectangle. + pub fn replace_attr(&mut self, target: Rect, mask: Attributes, attr: Attributes) { + let back = &mut self.buffers[self.frame_counter & 1]; + back.attributes.replace(target, mask, attr); + } + + /// Sets the current visible cursor position and type. + /// + /// Call this when focus is inside an editable area and you want to show the cursor. + pub fn set_cursor(&mut self, pos: Point, overtype: bool) { + let back = &mut self.buffers[self.frame_counter & 1]; + back.cursor.pos = pos; + back.cursor.overtype = overtype; + } + + /// Renders the framebuffer contents accumulated since the + /// last call to `flip()` and returns them serialized as VT. + pub fn render<'a>(&mut self, arena: &'a Arena) -> ArenaString<'a> { + let idx = self.frame_counter & 1; + // Borrows the front/back buffers without letting Rust know that we have a reference to self. + // SAFETY: Well this is certainly correct, but whether Rust and its strict rules likes it is another question. + let (back, front) = unsafe { + let ptr = self.buffers.as_mut_ptr(); + let back = &mut *ptr.add(idx); + let front = &*ptr.add(1 - idx); + (back, front) + }; + + let mut front_lines = front.text.lines.iter(); // hahaha + let mut front_bgs = front.bg_bitmap.iter(); + let mut front_fgs = front.fg_bitmap.iter(); + let mut front_attrs = front.attributes.iter(); + + let mut back_lines = back.text.lines.iter(); + let mut back_bgs = back.bg_bitmap.iter(); + let mut back_fgs = back.fg_bitmap.iter(); + let mut back_attrs = back.attributes.iter(); + + let mut result = ArenaString::new_in(arena); + let mut last_bg = u64::MAX; + let mut last_fg = u64::MAX; + let mut last_attr = Attributes::None; + + for y in 0..front.text.size.height { + // SAFETY: The only thing that changes the size of these containers, + // is the reset() method and it always resets front/back to the same size. + let front_line = unsafe { front_lines.next().unwrap_unchecked() }; + let front_bg = unsafe { front_bgs.next().unwrap_unchecked() }; + let front_fg = unsafe { front_fgs.next().unwrap_unchecked() }; + let front_attr = unsafe { front_attrs.next().unwrap_unchecked() }; + + let back_line = unsafe { back_lines.next().unwrap_unchecked() }; + let back_bg = unsafe { back_bgs.next().unwrap_unchecked() }; + let back_fg = unsafe { back_fgs.next().unwrap_unchecked() }; + let back_attr = unsafe { back_attrs.next().unwrap_unchecked() }; + + // TODO: Ideally, we should properly diff the contents and so if + // only parts of a line change, we should only update those parts. + if front_line == back_line + && front_bg == back_bg + && front_fg == back_fg + && front_attr == back_attr + { + continue; + } + + let line_bytes = back_line.as_bytes(); + let mut cfg = MeasurementConfig::new(&line_bytes); + let mut chunk_end = 0; + + if result.is_empty() { + result.push_str("\x1b[m"); + } + _ = write!(result, "\x1b[{};1H", y + 1); + + while { + let bg = back_bg[chunk_end]; + let fg = back_fg[chunk_end]; + let attr = back_attr[chunk_end]; + + // Chunk into runs of the same color. + while { + chunk_end += 1; + chunk_end < back_bg.len() + && back_bg[chunk_end] == bg + && back_fg[chunk_end] == fg + && back_attr[chunk_end] == attr + } {} + + if last_bg != bg as u64 { + last_bg = bg as u64; + self.format_color(&mut result, false, bg); + } + + if last_fg != fg as u64 { + last_fg = fg as u64; + self.format_color(&mut result, true, fg); + } + + if last_attr != attr { + let diff = last_attr ^ attr; + if diff.is(Attributes::Italic) { + if attr.is(Attributes::Italic) { + result.push_str("\x1b[3m"); + } else { + result.push_str("\x1b[23m"); + } + } + if diff.is(Attributes::Underlined) { + if attr.is(Attributes::Underlined) { + result.push_str("\x1b[4m"); + } else { + result.push_str("\x1b[24m"); + } + } + last_attr = attr; + } + + let beg = cfg.cursor().offset; + let end = cfg.goto_visual(Point { x: chunk_end as CoordType, y: 0 }).offset; + result.push_str(&back_line[beg..end]); + + chunk_end < back_bg.len() + } {} + } + + // If the cursor has changed since the last frame we naturally need to update it, + // but this also applies if the code above wrote to the screen, + // as it uses CUP sequences to reposition the cursor for writing. + if !result.is_empty() || back.cursor != front.cursor { + if back.cursor.pos.x >= 0 && back.cursor.pos.y >= 0 { + // CUP to the cursor position. + // DECSCUSR to set the cursor style. + // DECTCEM to show the cursor. + _ = write!( + result, + "\x1b[{};{}H\x1b[{} q\x1b[?25h", + back.cursor.pos.y + 1, + back.cursor.pos.x + 1, + if back.cursor.overtype { 1 } else { 5 } + ); + } else { + // DECTCEM to hide the cursor. + result.push_str("\x1b[?25l"); + } + } + + result + } + + fn format_color(&self, dst: &mut ArenaString, fg: bool, mut color: u32) { + let typ = if fg { '3' } else { '4' }; + + // Some terminals support transparent backgrounds which are used + // if the default background color is active (CSI 49 m). + // + // If [`Framebuffer::set_indexed_colors`] was never called, we assume + // that the terminal doesn't support transparency and initialize the + // background bitmap with the `DEFAULT_THEME` default background color. + // Otherwise, we assume that the terminal supports transparency + // and initialize it with 0x00000000 (transparent). + // + // We also apply this to the foreground color, because it compresses + // the output slightly and ensures that we keep "default foreground" + // and "color that happens to be default foreground" separate. + // (This also applies to the background color by the way.) + if color == 0 { + _ = write!(dst, "\x1b[{typ}9m"); + return; + } + + if (color & 0xff000000) != 0xff000000 { + let idx = if fg { IndexedColor::Foreground } else { IndexedColor::Background }; + let dst = self.indexed(idx); + color = oklab_blend(dst, color); + } + + let r = color & 0xff; + let g = (color >> 8) & 0xff; + let b = (color >> 16) & 0xff; + _ = write!(dst, "\x1b[{typ}8;2;{r};{g};{b}m"); + } +} + +#[derive(Default)] +struct Buffer { + text: LineBuffer, + bg_bitmap: Bitmap, + fg_bitmap: Bitmap, + attributes: AttributeBuffer, + cursor: Cursor, +} + +/// A buffer for the text contents of the framebuffer. +#[derive(Default)] +struct LineBuffer { + lines: Vec, + size: Size, +} + +impl LineBuffer { + fn new(size: Size) -> Self { + Self { lines: vec![String::new(); size.height as usize], size } + } + + fn fill_whitespace(&mut self) { + let width = self.size.width as usize; + for l in &mut self.lines { + l.clear(); + l.reserve(width + width / 2); + + let buf = unsafe { l.as_mut_vec() }; + // Compiles down to `memset()`. + buf.extend(std::iter::repeat_n(b' ', width)); + } + } + + /// Replaces text contents in a single line of the framebuffer. + /// All coordinates are in viewport coordinates. + /// Assumes that control characters have been replaced or escaped. + fn replace_text( + &mut self, + y: CoordType, + origin_x: CoordType, + clip_right: CoordType, + text: &str, + ) { + let Some(line) = self.lines.get_mut(y as usize) else { + return; + }; + + let bytes = text.as_bytes(); + let clip_right = clip_right.clamp(0, self.size.width); + let layout_width = clip_right - origin_x; + + // Can't insert text that can't fit or is empty. + if layout_width <= 0 || bytes.is_empty() { + return; + } + + let mut cfg = MeasurementConfig::new(&bytes); + + // Check if the text intersects with the left edge of the framebuffer + // and figure out the parts that are inside. + let mut left = origin_x; + if left < 0 { + let mut cursor = cfg.goto_visual(Point { x: -left, y: 0 }); + + if left + cursor.visual_pos.x < 0 && cursor.offset < text.len() { + // `-left` must've intersected a wide glyph and since goto_visual stops _before_ reaching the target, + // we stoped before the wide glyph and thus must step forward to the next glyph. + cursor = cfg.goto_logical(Point { x: cursor.logical_pos.x + 1, y: 0 }); + } + + left += cursor.visual_pos.x; + } + + // If the text still starts outside the framebuffer, we must've ran out of text above. + // Otherwise, if it starts outside the right edge to begin with, we can't insert it anyway. + if left < 0 || left >= clip_right { + return; + } + + // Measure the width of the new text (= `res_new.visual_target.x`). + let beg_off = cfg.cursor().offset; + let end = cfg.goto_visual(Point { x: layout_width, y: 0 }); + + // Figure out at which byte offset the new text gets inserted. + let right = left + end.visual_pos.x; + let line_bytes = line.as_bytes(); + let mut cfg_old = MeasurementConfig::new(&line_bytes); + let res_old_beg = cfg_old.goto_visual(Point { x: left, y: 0 }); + let mut res_old_end = cfg_old.goto_visual(Point { x: right, y: 0 }); + + // Since the goto functions will always stop short of the target position, + // we need to manually step beyond it if we intersect with a wide glyph. + if res_old_end.visual_pos.x < right { + res_old_end = cfg_old.goto_logical(Point { x: res_old_end.logical_pos.x + 1, y: 0 }); + } + + // If we intersect a wide glyph, we need to pad the new text with spaces. + let src = &text[beg_off..end.offset]; + let overlap_beg = (left - res_old_beg.visual_pos.x).max(0) as usize; + let overlap_end = (res_old_end.visual_pos.x - right).max(0) as usize; + let total_add = src.len() + overlap_beg + overlap_end; + let total_del = res_old_end.offset - res_old_beg.offset; + + // This is basically a hand-written version of `Vec::splice()`, + // but for strings under the assumption that all inputs are valid. + // It also takes care of `overlap_beg` and `overlap_end` by inserting spaces. + unsafe { + // SAFETY: Our ucd code only returns valid UTF-8 offsets. + // If it didn't that'd be a priority -9000 bug for any text editor. + // And apart from that, all inputs are &str (= UTF8). + let dst = line.as_mut_vec(); + + let dst_len = dst.len(); + let src_len = src.len(); + + // Make room for the new elements. NOTE that this must be done before + // we call as_mut_ptr, or else we risk accessing a stale pointer. + // We only need to reserve as much as the string actually grows by. + dst.reserve(total_add.saturating_sub(total_del)); + + // Move the pointer to the start of the insert. + let mut ptr = dst.as_mut_ptr().add(res_old_beg.offset); + + // Move the tail end of the string by `total_add - total_del`-many bytes. + // This both effectively deletes the old text and makes room for the new text. + if total_add != total_del { + // Move the tail of the vector to make room for the new elements. + ptr::copy( + ptr.add(total_del), + ptr.add(total_add), + dst_len - total_del - res_old_beg.offset, + ); + } + + // Pad left. + for _ in 0..overlap_beg { + ptr.write(b' '); + ptr = ptr.add(1); + } + + // Copy the new elements into the vector. + ptr::copy_nonoverlapping(src.as_ptr(), ptr, src_len); + ptr = ptr.add(src_len); + + // Pad right. + for _ in 0..overlap_end { + ptr.write(b' '); + ptr = ptr.add(1); + } + + // Update the length of the vector. + dst.set_len(dst_len - total_del + total_add); + } + } +} + +/// An sRGB bitmap. +#[derive(Default)] +struct Bitmap { + data: Vec, + size: Size, +} + +impl Bitmap { + fn new(size: Size) -> Self { + Self { data: vec![0; (size.width * size.height) as usize], size } + } + + fn fill(&mut self, color: u32) { + memset(&mut self.data, color); + } + + /// Blends the given sRGB color onto the bitmap. + /// + /// This uses the `oklab` color space for blending so the + /// resulting colors may look different from what you'd expect. + fn blend(&mut self, target: Rect, color: u32) { + if (color & 0xff000000) == 0x00000000 { + return; + } + + let target = target.intersect(self.size.as_rect()); + if target.is_empty() { + return; + } + + let top = target.top as usize; + let bottom = target.bottom as usize; + let left = target.left as usize; + let right = target.right as usize; + let stride = self.size.width as usize; + + for y in top..bottom { + let beg = y * stride + left; + let end = y * stride + right; + let data = &mut self.data[beg..end]; + + if (color & 0xff000000) == 0xff000000 { + memset(data, color); + } else { + let end = data.len(); + let mut off = 0; + + while { + let c = data[off]; + + // Chunk into runs of the same color, so that we only call alpha_blend once per run. + let chunk_beg = off; + while { + off += 1; + off < end && data[off] == c + } {} + let chunk_end = off; + + let c = oklab_blend(c, color); + memset(&mut data[chunk_beg..chunk_end], c); + + off < end + } {} + } + } + } + + /// Iterates over each row in the bitmap. + fn iter(&self) -> ChunksExact { + self.data.chunks_exact(self.size.width as usize) + } +} + +/// A bitfield for VT text attributes. +/// +/// It being a bitfield allows for simple diffing. +#[repr(transparent)] +#[derive(Default, Clone, Copy, PartialEq, Eq)] +pub struct Attributes(u8); + +#[allow(non_upper_case_globals)] +impl Attributes { + pub const None: Attributes = Attributes(0); + pub const Italic: Attributes = Attributes(0b1); + pub const Underlined: Attributes = Attributes(0b10); + pub const All: Attributes = Attributes(0b11); + + pub const fn is(self, attr: Attributes) -> bool { + (self.0 & attr.0) == attr.0 + } +} + +unsafe impl MemsetSafe for Attributes {} + +impl BitOr for Attributes { + type Output = Attributes; + + fn bitor(self, rhs: Self) -> Self::Output { + Attributes(self.0 | rhs.0) + } +} + +impl BitXor for Attributes { + type Output = Attributes; + + fn bitxor(self, rhs: Self) -> Self::Output { + Attributes(self.0 ^ rhs.0) + } +} + +/// Stores VT attributes for the framebuffer. +#[derive(Default)] +struct AttributeBuffer { + data: Vec, + size: Size, +} + +impl AttributeBuffer { + fn new(size: Size) -> Self { + Self { data: vec![Default::default(); (size.width * size.height) as usize], size } + } + + fn reset(&mut self) { + memset(&mut self.data, Default::default()); + } + + fn replace(&mut self, target: Rect, mask: Attributes, attr: Attributes) { + let target = target.intersect(self.size.as_rect()); + if target.is_empty() { + return; + } + + let top = target.top as usize; + let bottom = target.bottom as usize; + let left = target.left as usize; + let right = target.right as usize; + let stride = self.size.width as usize; + + for y in top..bottom { + let beg = y * stride + left; + let end = y * stride + right; + let dst = &mut self.data[beg..end]; + + if mask == Attributes::All { + memset(dst, attr); + } else { + for a in dst { + *a = Attributes(a.0 & !mask.0 | attr.0); + } + } + } + } + + /// Iterates over each row in the bitmap. + fn iter(&self) -> ChunksExact { + self.data.chunks_exact(self.size.width as usize) + } +} + +/// Stores cursor position and type for the framebuffer. +#[derive(Default, PartialEq, Eq)] +struct Cursor { + pos: Point, + overtype: bool, +} + +impl Cursor { + const fn new_invalid() -> Self { + Self { pos: Point::MIN, overtype: false } + } + + const fn new_disabled() -> Self { + Self { pos: Point { x: -1, y: -1 }, overtype: false } + } +} diff --git a/pkgs/edit/src/fuzzy.rs b/pkgs/edit/src/fuzzy.rs new file mode 100644 index 0000000..997ccda --- /dev/null +++ b/pkgs/edit/src/fuzzy.rs @@ -0,0 +1,221 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Fuzzy search algorithm based on the one used in VS Code (`/src/vs/base/common/fuzzyScorer.ts`). +//! Other algorithms exist, such as Sublime Text's, or the one used in `fzf`, +//! but I figured that this one is what lots of people may be familiar with. + +use std::vec; + +use crate::arena::{Arena, scratch_arena}; +use crate::icu; + +const NO_MATCH: i32 = 0; + +pub fn score_fuzzy<'a>( + arena: &'a Arena, + haystack: &str, + needle: &str, + allow_non_contiguous_matches: bool, +) -> (i32, Vec) { + if haystack.is_empty() || needle.is_empty() { + // return early if target or query are empty + return (NO_MATCH, Vec::new_in(arena)); + } + + let scratch = scratch_arena(Some(arena)); + let target = map_chars(&scratch, haystack); + let query = map_chars(&scratch, needle); + + if target.len() < query.len() { + // impossible for query to be contained in target + return (NO_MATCH, Vec::new_in(arena)); + } + + let target_lower = icu::fold_case(&scratch, haystack); + let query_lower = icu::fold_case(&scratch, needle); + let target_lower = map_chars(&scratch, &target_lower); + let query_lower = map_chars(&scratch, &query_lower); + + let area = query.len() * target.len(); + let mut scores = vec::from_elem_in(0, area, &*scratch); + let mut matches = vec::from_elem_in(0, area, &*scratch); + + // + // Build Scorer Matrix: + // + // The matrix is composed of query q and target t. For each index we score + // q[i] with t[i] and compare that with the previous score. If the score is + // equal or larger, we keep the match. In addition to the score, we also keep + // the length of the consecutive matches to use as boost for the score. + // + // t a r g e t + // q + // u + // e + // r + // y + // + for query_index in 0..query.len() { + let query_index_offset = query_index * target.len(); + let query_index_previous_offset = + if query_index > 0 { (query_index - 1) * target.len() } else { 0 }; + + for target_index in 0..target.len() { + let current_index = query_index_offset + target_index; + let diag_index = if query_index > 0 && target_index > 0 { + query_index_previous_offset + target_index - 1 + } else { + 0 + }; + let left_score = if target_index > 0 { scores[current_index - 1] } else { 0 }; + let diag_score = + if query_index > 0 && target_index > 0 { scores[diag_index] } else { 0 }; + let matches_sequence_len = + if query_index > 0 && target_index > 0 { matches[diag_index] } else { 0 }; + + // If we are not matching on the first query character any more, we only produce a + // score if we had a score previously for the last query index (by looking at the diagScore). + // This makes sure that the query always matches in sequence on the target. For example + // given a target of "ede" and a query of "de", we would otherwise produce a wrong high score + // for query[1] ("e") matching on target[0] ("e") because of the "beginning of word" boost. + let score = if diag_score == 0 && query_index != 0 { + 0 + } else { + compute_char_score( + query[query_index], + query_lower[query_index], + if target_index != 0 { Some(target[target_index - 1]) } else { None }, + target[target_index], + target_lower[target_index], + matches_sequence_len, + ) + }; + + // We have a score and its equal or larger than the left score + // Match: sequence continues growing from previous diag value + // Score: increases by diag score value + let is_valid_score = score != 0 && diag_score + score >= left_score; + if is_valid_score + && ( + // We don't need to check if it's contiguous if we allow non-contiguous matches + allow_non_contiguous_matches || + // We must be looking for a contiguous match. + // Looking at an index higher than 0 in the query means we must have already + // found out this is contiguous otherwise there wouldn't have been a score + query_index > 0 || + // lastly check if the query is completely contiguous at this index in the target + target_lower[target_index..].starts_with(&query_lower) + ) + { + matches[current_index] = matches_sequence_len + 1; + scores[current_index] = diag_score + score; + } else { + // We either have no score or the score is lower than the left score + // Match: reset to 0 + // Score: pick up from left hand side + matches[current_index] = NO_MATCH; + scores[current_index] = left_score; + } + } + } + + // Restore Positions (starting from bottom right of matrix) + let mut positions = Vec::new_in(arena); + + if !query.is_empty() && !target.is_empty() { + let mut query_index = query.len() - 1; + let mut target_index = target.len() - 1; + + loop { + let current_index = query_index * target.len() + target_index; + if matches[current_index] == NO_MATCH { + if target_index == 0 { + break; + } + target_index -= 1; // go left + } else { + positions.push(target_index); + + // go up and left + if query_index == 0 || target_index == 0 { + break; + } + query_index -= 1; + target_index -= 1; + } + } + + positions.reverse(); + } + + (scores[area - 1], positions) +} + +fn compute_char_score( + query: char, + query_lower: char, + target_prev: Option, + target_curr: char, + target_curr_lower: char, + matches_sequence_len: i32, +) -> i32 { + let mut score = 0; + + if !consider_as_equal(query_lower, target_curr_lower) { + return score; // no match of characters + } + + // Character match bonus + score += 1; + + // Consecutive match bonus + if matches_sequence_len > 0 { + score += matches_sequence_len * 5; + } + + // Same case bonus + if query == target_curr { + score += 1; + } + + if let Some(target_prev) = target_prev { + // After separator bonus + let separator_bonus = score_separator_at_pos(target_prev); + if separator_bonus > 0 { + score += separator_bonus; + } + // Inside word upper case bonus (camel case). We only give this bonus if we're not in a contiguous sequence. + // For example: + // NPE => NullPointerException = boost + // HTTP => HTTP = not boost + else if target_curr != target_curr_lower && matches_sequence_len == 0 { + score += 2; + } + } else { + // Start of word bonus + score += 8; + } + + score +} + +fn consider_as_equal(a: char, b: char) -> bool { + // Special case path separators: ignore platform differences + a == b || a == '/' || a == '\\' && b == '/' || b == '\\' +} + +fn score_separator_at_pos(ch: char) -> i32 { + match ch { + '/' | '\\' => 5, // prefer path separators... + '_' | '-' | '.' | ' ' | '\'' | '"' | ':' => 4, // ...over other separators + _ => 0, + } +} + +fn map_chars<'a>(arena: &'a Arena, s: &str) -> Vec { + let mut chars = Vec::with_capacity_in(s.len(), arena); + chars.extend(s.chars()); + chars.shrink_to_fit(); + chars +} diff --git a/pkgs/edit/src/hash.rs b/pkgs/edit/src/hash.rs new file mode 100644 index 0000000..0634529 --- /dev/null +++ b/pkgs/edit/src/hash.rs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Provides fast, non-cryptographic hash functions. + +/// The venerable wyhash hash function. +/// +/// It's fast, has good statistical properties, and is in the public domain. +/// See: +/// If you visit the link, you'll find that it was superseded by "rapidhash", +/// but that's not particularly interesting for this project. rapidhash results +/// in way larger assembly and isn't faster when hashing small amounts of data. +pub fn hash(mut seed: u64, data: &[u8]) -> u64 { + unsafe { + const S0: u64 = 0xa0761d6478bd642f; + const S1: u64 = 0xe7037ed1a0b428db; + const S2: u64 = 0x8ebc6af09c88c6e3; + const S3: u64 = 0x589965cc75374cc3; + + let len = data.len(); + let mut p = data.as_ptr(); + let a; + let b; + + seed ^= S0; + + if len <= 16 { + if len >= 4 { + a = (wyr4(p) << 32) | wyr4(p.add((len >> 3) << 2)); + b = (wyr4(p.add(len - 4)) << 32) | wyr4(p.add(len - 4 - ((len >> 3) << 2))); + } else if len > 0 { + a = wyr3(p, len); + b = 0; + } else { + a = 0; + b = 0; + } + } else { + let mut i = len; + if i > 48 { + let mut seed1 = seed; + let mut seed2 = seed; + while { + seed = wymix(wyr8(p) ^ S1, wyr8(p.add(8)) ^ seed); + seed1 = wymix(wyr8(p.add(16)) ^ S2, wyr8(p.add(24)) ^ seed1); + seed2 = wymix(wyr8(p.add(32)) ^ S3, wyr8(p.add(40)) ^ seed2); + p = p.add(48); + i -= 48; + i > 48 + } {} + seed ^= seed1 ^ seed2; + } + while i > 16 { + seed = wymix(wyr8(p) ^ S1, wyr8(p.add(8)) ^ seed); + i -= 16; + p = p.add(16); + } + a = wyr8(p.offset(i as isize - 16)); + b = wyr8(p.offset(i as isize - 8)); + } + + wymix(S1 ^ (len as u64), wymix(a ^ S1, b ^ seed)) + } +} + +unsafe fn wyr3(p: *const u8, k: usize) -> u64 { + let p0 = unsafe { p.read() as u64 }; + let p1 = unsafe { p.add(k >> 1).read() as u64 }; + let p2 = unsafe { p.add(k - 1).read() as u64 }; + (p0 << 16) | (p1 << 8) | p2 +} + +unsafe fn wyr4(p: *const u8) -> u64 { + unsafe { (p as *const u32).read_unaligned() as u64 } +} + +unsafe fn wyr8(p: *const u8) -> u64 { + unsafe { (p as *const u64).read_unaligned() } +} + +// This is a weak mix function on its own. It may be worth considering +// replacing external uses of this function with a stronger one. +// On the other hand, it's very fast. +pub fn wymix(lhs: u64, rhs: u64) -> u64 { + let lhs = lhs as u128; + let rhs = rhs as u128; + let r = lhs * rhs; + (r >> 64) as u64 ^ (r as u64) +} + +pub fn hash_str(seed: u64, s: &str) -> u64 { + hash(seed, s.as_bytes()) +} diff --git a/pkgs/edit/src/helpers.rs b/pkgs/edit/src/helpers.rs new file mode 100644 index 0000000..569218d --- /dev/null +++ b/pkgs/edit/src/helpers.rs @@ -0,0 +1,277 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Random assortment of helpers I didn't know where to put. + +use std::alloc::Allocator; +use std::cmp::Ordering; +use std::io::Read; +use std::mem::{self, MaybeUninit}; +use std::ops::{Bound, Range, RangeBounds}; +use std::{fmt, ptr, slice, str}; + +use crate::apperr; + +pub const KILO: usize = 1000; +pub const MEGA: usize = 1000 * 1000; +pub const GIGA: usize = 1000 * 1000 * 1000; + +pub const KIBI: usize = 1024; +pub const MEBI: usize = 1024 * 1024; +pub const GIBI: usize = 1024 * 1024 * 1024; + +pub struct MetricFormatter(pub T); + +impl fmt::Display for MetricFormatter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut value = self.0; + let mut suffix = "B"; + if value >= GIGA { + value /= GIGA; + suffix = "GB"; + } else if value >= MEGA { + value /= MEGA; + suffix = "MB"; + } else if value >= KILO { + value /= KILO; + suffix = "kB"; + } + write!(f, "{value}{suffix}") + } +} + +/// A viewport coordinate type used throughout the application. +pub type CoordType = i32; + +/// To avoid overflow issues because you're adding two [`CoordType::MAX`] values together, +/// you can use [`COORD_TYPE_SAFE_MIN`] and [`COORD_TYPE_SAFE_MAX`]. +pub const COORD_TYPE_SAFE_MAX: CoordType = 32767; + +/// See [`COORD_TYPE_SAFE_MAX`]. +pub const COORD_TYPE_SAFE_MIN: CoordType = -32767 - 1; + +/// A 2D point. Uses [`CoordType`]. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub struct Point { + pub x: CoordType, + pub y: CoordType, +} + +impl Point { + pub const MIN: Point = Point { x: CoordType::MIN, y: CoordType::MIN }; + pub const MAX: Point = Point { x: CoordType::MAX, y: CoordType::MAX }; +} + +impl PartialOrd for Point { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Point { + fn cmp(&self, other: &Self) -> Ordering { + match self.y.cmp(&other.y) { + Ordering::Equal => self.x.cmp(&other.x), + ord => ord, + } + } +} + +/// A 2D size. Uses [`CoordType`]. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub struct Size { + pub width: CoordType, + pub height: CoordType, +} + +impl Size { + pub fn as_rect(&self) -> Rect { + Rect { left: 0, top: 0, right: self.width, bottom: self.height } + } +} + +/// A 2D rectangle. Uses [`CoordType`]. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub struct Rect { + pub left: CoordType, + pub top: CoordType, + pub right: CoordType, + pub bottom: CoordType, +} + +impl Rect { + /// Mimics CSS's `padding` property where `padding: a` is `a a a a`. + pub fn one(value: CoordType) -> Self { + Self { left: value, top: value, right: value, bottom: value } + } + + /// Mimics CSS's `padding` property where `padding: a b` is `a b a b`, + /// and `a` is top/bottom and `b` is left/right. + pub fn two(top_bottom: CoordType, left_right: CoordType) -> Self { + Self { left: left_right, top: top_bottom, right: left_right, bottom: top_bottom } + } + + /// Mimics CSS's `padding` property where `padding: a b c` is `a b c b`, + /// and `a` is top, `b` is left/right, and `c` is bottom. + pub fn three(top: CoordType, left_right: CoordType, bottom: CoordType) -> Self { + Self { left: left_right, top, right: left_right, bottom } + } + + /// Is the rectangle empty? + pub fn is_empty(&self) -> bool { + self.left >= self.right || self.top >= self.bottom + } + + /// Width of the rectangle. + pub fn width(&self) -> CoordType { + self.right - self.left + } + + /// Height of the rectangle. + pub fn height(&self) -> CoordType { + self.bottom - self.top + } + + /// Check if it contains a point. + pub fn contains(&self, point: Point) -> bool { + point.x >= self.left && point.x < self.right && point.y >= self.top && point.y < self.bottom + } + + /// Intersect two rectangles. + pub fn intersect(&self, rhs: Self) -> Self { + let l = self.left.max(rhs.left); + let t = self.top.max(rhs.top); + let r = self.right.min(rhs.right); + let b = self.bottom.min(rhs.bottom); + + // Ensure that the size is non-negative. This avoids bugs, + // because some height/width is negative all of a sudden. + let r = l.max(r); + let b = t.max(b); + + Rect { left: l, top: t, right: r, bottom: b } + } +} + +/// [`std::cmp::minmax`] is unstable, as per usual. +pub fn minmax(v1: T, v2: T) -> [T; 2] +where + T: Ord, +{ + if v2 < v1 { [v2, v1] } else { [v1, v2] } +} + +#[inline(always)] +#[allow(clippy::ptr_eq)] +fn opt_ptr(a: Option<&T>) -> *const T { + unsafe { mem::transmute(a) } +} + +/// Surprisingly, there's no way in Rust to do a `ptr::eq` on `Option<&T>`. +/// Uses `unsafe` so that the debug performance isn't too bad. +#[inline(always)] +#[allow(clippy::ptr_eq)] +pub fn opt_ptr_eq(a: Option<&T>, b: Option<&T>) -> bool { + opt_ptr(a) == opt_ptr(b) +} + +/// Creates a `&str` from a pointer and a length. +/// Exists, because `std::str::from_raw_parts` is unstable, par for the course. +/// +/// # Safety +/// +/// The given data must be valid UTF-8. +/// The given data must outlive the returned reference. +#[inline] +#[must_use] +pub const unsafe fn str_from_raw_parts<'a>(ptr: *const u8, len: usize) -> &'a str { + unsafe { str::from_utf8_unchecked(slice::from_raw_parts(ptr, len)) } +} + +/// [`<[T]>::copy_from_slice`] panics if the two slices have different lengths. +/// This one just returns the copied amount. +pub fn slice_copy_safe(dst: &mut [T], src: &[T]) -> usize { + let len = src.len().min(dst.len()); + unsafe { ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), len) }; + len +} + +/// [`Vec::splice`] results in really bad assembly. +/// This doesn't. Don't use [`Vec::splice`]. +pub trait ReplaceRange { + fn replace_range>(&mut self, range: R, src: &[T]); +} + +impl ReplaceRange for Vec { + fn replace_range>(&mut self, range: R, src: &[T]) { + let start = match range.start_bound() { + Bound::Included(&start) => start, + Bound::Excluded(start) => start + 1, + Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + Bound::Included(end) => end + 1, + Bound::Excluded(&end) => end, + Bound::Unbounded => usize::MAX, + }; + vec_replace_impl(self, start..end, src); + } +} + +fn vec_replace_impl(dst: &mut Vec, range: Range, src: &[T]) { + unsafe { + let dst_len = dst.len(); + let src_len = src.len(); + let off = range.start.min(dst_len); + let del_len = range.end.saturating_sub(off).min(dst_len - off); + + if del_len == 0 && src_len == 0 { + return; // nothing to do + } + + let tail_len = dst_len - off - del_len; + let new_len = dst_len - del_len + src_len; + + if src_len > del_len { + dst.reserve(src_len - del_len); + } + + // NOTE: drop_in_place() is not needed here, because T is constrained to Copy. + + // SAFETY: as_mut_ptr() must called after reserve() to ensure that the pointer is valid. + let ptr = dst.as_mut_ptr().add(off); + + // Shift the tail. + if tail_len > 0 && src_len != del_len { + ptr::copy(ptr.add(del_len), ptr.add(src_len), tail_len); + } + + // Copy in the replacement. + ptr::copy_nonoverlapping(src.as_ptr(), ptr, src_len); + dst.set_len(new_len); + } +} + +/// [`Read`] but with [`MaybeUninit`] buffers. +pub fn file_read_uninit( + file: &mut T, + buf: &mut [MaybeUninit], +) -> apperr::Result { + unsafe { + let buf_slice = slice::from_raw_parts_mut(buf.as_mut_ptr() as *mut u8, buf.len()); + let n = file.read(buf_slice)?; + Ok(n) + } +} + +/// Turns a [`&[u8]`] into a [`&[MaybeUninit]`]. +#[inline(always)] +pub const fn slice_as_uninit_ref(slice: &[T]) -> &[MaybeUninit] { + unsafe { slice::from_raw_parts(slice.as_ptr() as *const MaybeUninit, slice.len()) } +} + +/// Turns a [`&mut [T]`] into a [`&mut [MaybeUninit]`]. +#[inline(always)] +pub const fn slice_as_uninit_mut(slice: &mut [T]) -> &mut [MaybeUninit] { + unsafe { slice::from_raw_parts_mut(slice.as_mut_ptr() as *mut MaybeUninit, slice.len()) } +} diff --git a/pkgs/edit/src/icu.rs b/pkgs/edit/src/icu.rs new file mode 100644 index 0000000..0829970 --- /dev/null +++ b/pkgs/edit/src/icu.rs @@ -0,0 +1,1242 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Bindings to the ICU library. + +use std::cmp::Ordering; +use std::ffi::CStr; +use std::mem; +use std::mem::MaybeUninit; +use std::ops::Range; +use std::ptr::{null, null_mut}; + +use crate::arena::{Arena, ArenaString, scratch_arena}; +use crate::buffer::TextBuffer; +use crate::unicode::Utf8Chars; +use crate::{apperr, arena_format, sys}; + +static mut ENCODINGS: Vec<&'static str> = Vec::new(); + +/// Returns a list of encodings ICU supports. +pub fn get_available_encodings() -> &'static [&'static str] { + // OnceCell for people that want to put it into a static. + #[allow(static_mut_refs)] + unsafe { + if ENCODINGS.is_empty() { + if let Ok(f) = init_if_needed() { + let mut n = 0; + loop { + let name = (f.ucnv_getAvailableName)(n); + if name.is_null() { + break; + } + ENCODINGS.push(CStr::from_ptr(name).to_str().unwrap_unchecked()); + n += 1; + } + } + + if ENCODINGS.is_empty() { + ENCODINGS.push("UTF-8"); + } + } + &ENCODINGS + } +} + +/// Formats the given ICU error code into a human-readable string. +pub fn apperr_format(f: &mut std::fmt::Formatter<'_>, code: u32) -> std::fmt::Result { + fn format(code: u32) -> &'static str { + let Ok(f) = init_if_needed() else { + return ""; + }; + + let status = icu_ffi::UErrorCode::new(code); + let ptr = unsafe { (f.u_errorName)(status) }; + if ptr.is_null() { + return ""; + } + + let str = unsafe { CStr::from_ptr(ptr) }; + str.to_str().unwrap_or("") + } + + let msg = format(code); + if !msg.is_empty() { + write!(f, "ICU Error: {msg}") + } else { + write!(f, "ICU Error: {code:#08x}") + } +} + +/// Converts between two encodings using ICU. +pub struct Converter<'pivot> { + source: *mut icu_ffi::UConverter, + target: *mut icu_ffi::UConverter, + pivot_buffer: &'pivot mut [MaybeUninit], + pivot_source: *mut u16, + pivot_target: *mut u16, + reset: bool, +} + +impl Drop for Converter<'_> { + fn drop(&mut self) { + let f = assume_loaded(); + unsafe { (f.ucnv_close)(self.source) }; + unsafe { (f.ucnv_close)(self.target) }; + } +} + +impl<'pivot> Converter<'pivot> { + /// Constructs a new `Converter` instance. + /// + /// # Parameters + /// + /// * `pivot_buffer`: A buffer used to cache partial conversions. + /// Don't make it too small. + /// * `source_encoding`: The source encoding name (e.g., "UTF-8"). + /// * `target_encoding`: The target encoding name (e.g., "UTF-16"). + pub fn new( + pivot_buffer: &'pivot mut [MaybeUninit], + source_encoding: &str, + target_encoding: &str, + ) -> apperr::Result { + let f = init_if_needed()?; + + let arena = scratch_arena(None); + let source_encoding = Self::append_nul(&arena, source_encoding); + let target_encoding = Self::append_nul(&arena, target_encoding); + + let mut status = icu_ffi::U_ZERO_ERROR; + let source = unsafe { (f.ucnv_open)(source_encoding.as_ptr(), &mut status) }; + let target = unsafe { (f.ucnv_open)(target_encoding.as_ptr(), &mut status) }; + if status.is_failure() { + if !source.is_null() { + unsafe { (f.ucnv_close)(source) }; + } + if !target.is_null() { + unsafe { (f.ucnv_close)(target) }; + } + return Err(status.as_error()); + } + + let pivot_source = pivot_buffer.as_mut_ptr() as *mut u16; + let pivot_target = unsafe { pivot_source.add(pivot_buffer.len()) }; + + Ok(Self { source, target, pivot_buffer, pivot_source, pivot_target, reset: true }) + } + + fn append_nul<'a>(arena: &'a Arena, input: &str) -> ArenaString<'a> { + arena_format!(arena, "{}\0", input) + } + + /// Performs one step of the encoding conversion. + /// + /// # Parameters + /// + /// * `input`: The input buffer to convert from. + /// It should be in the `source_encoding` that was previously specified. + /// * `output`: The output buffer to convert to. + /// It should be in the `target_encoding` that was previously specified. + /// + /// # Returns + /// + /// A tuple containing: + /// 1. The number of bytes read from the input buffer. + /// 2. The number of bytes written to the output buffer. + pub fn convert( + &mut self, + input: &[u8], + output: &mut [MaybeUninit], + ) -> apperr::Result<(usize, usize)> { + let f = assume_loaded(); + + let input_beg = input.as_ptr(); + let input_end = unsafe { input_beg.add(input.len()) }; + let mut input_ptr = input_beg; + + let output_beg = output.as_mut_ptr() as *mut u8; + let output_end = unsafe { output_beg.add(output.len()) }; + let mut output_ptr = output_beg; + + let pivot_beg = self.pivot_buffer.as_mut_ptr() as *mut u16; + let pivot_end = unsafe { pivot_beg.add(self.pivot_buffer.len()) }; + + let flush = input.is_empty(); + let mut status = icu_ffi::U_ZERO_ERROR; + + unsafe { + (f.ucnv_convertEx)( + /* target_cnv */ self.target, + /* source_cnv */ self.source, + /* target */ &mut output_ptr, + /* target_limit */ output_end, + /* source */ &mut input_ptr, + /* source_limit */ input_end, + /* pivot_start */ pivot_beg, + /* pivot_source */ &mut self.pivot_source, + /* pivot_target */ &mut self.pivot_target, + /* pivot_limit */ pivot_end, + /* reset */ self.reset, + /* flush */ flush, + /* status */ &mut status, + ); + } + + self.reset = false; + if status.is_failure() && status != icu_ffi::U_BUFFER_OVERFLOW_ERROR { + return Err(status.as_error()); + } + + let input_advance = unsafe { input_ptr.offset_from(input_beg) as usize }; + let output_advance = unsafe { output_ptr.offset_from(output_beg) as usize }; + Ok((input_advance, output_advance)) + } +} + +// In benchmarking, I found that the performance does not really change much by changing this value. +// I picked 64 because it seemed like a reasonable lower bound. +const CACHE_SIZE: usize = 64; + +/// Caches a chunk of TextBuffer contents (UTF-8) in UTF-16 format. +struct Cache { + /// The translated text. Contains [`Cache::utf16_len`]-many valid items. + utf16: [u16; CACHE_SIZE], + /// For each character in [`Cache::utf16`] this stores the offset in the [`TextBuffer`], + /// relative to the start offset stored in `native_beg`. + /// This has the same length as [`Cache::utf16`]. + utf16_to_utf8_offsets: [u16; CACHE_SIZE], + /// `utf8_to_utf16_offsets[native_offset - native_beg]` will tell you which character in + /// [`Cache::utf16`] maps to the given `native_offset` in the underlying [`TextBuffer`]. + /// Contains `native_end - native_beg`-many valid items. + utf8_to_utf16_offsets: [u16; CACHE_SIZE], + + /// The number of valid items in [`Cache::utf16`]. + utf16_len: usize, + /// Offset of the first non-ASCII character. + /// Less than or equal to [`Cache::utf16_len`]. + native_indexing_limit: usize, + + /// The range of UTF-8 text in the [`TextBuffer`] that this chunk covers. + utf8_range: Range, +} + +struct DoubleCache { + cache: [Cache; 2], + /// You can consider this a 1 bit index into `cache`. + mru: bool, +} + +/// A wrapper around ICU's `UText` struct. +/// +/// In our case its only purpose is to adapt a [`TextBuffer`] for ICU. +/// +/// # Safety +/// +/// Warning! No lifetime tracking is done here. +/// I initially did it properly with a PhantomData marker for the TextBuffer +/// lifetime, but it was a pain so now I don't. Not a big deal in our case. +pub struct Text(&'static mut icu_ffi::UText); + +impl Drop for Text { + fn drop(&mut self) { + let f = assume_loaded(); + unsafe { (f.utext_close)(self.0) }; + } +} + +impl Text { + /// Constructs an ICU `UText` instance from a [`TextBuffer`]. + /// + /// # Safety + /// + /// The caller must ensure that the given [`TextBuffer`] + /// outlives the returned `Text` instance. + pub unsafe fn new(tb: &TextBuffer) -> apperr::Result { + let f = init_if_needed()?; + + let mut status = icu_ffi::U_ZERO_ERROR; + let ptr = + unsafe { (f.utext_setup)(null_mut(), size_of::() as i32, &mut status) }; + if status.is_failure() { + return Err(status.as_error()); + } + + const FUNCS: icu_ffi::UTextFuncs = icu_ffi::UTextFuncs { + table_size: size_of::() as i32, + reserved1: 0, + reserved2: 0, + reserved3: 0, + clone: Some(utext_clone), + native_length: Some(utext_native_length), + access: Some(utext_access), + extract: None, + replace: None, + copy: None, + map_offset_to_native: Some(utext_map_offset_to_native), + map_native_index_to_utf16: Some(utext_map_native_index_to_utf16), + close: None, + spare1: None, + spare2: None, + spare3: None, + }; + + let ut = unsafe { &mut *ptr }; + ut.p_funcs = &FUNCS; + ut.context = tb as *const TextBuffer as *mut _; + ut.a = tb.generation() as i64; + + // ICU unfortunately expects a `UText` instance to have valid contents after construction. + utext_access(ut, 0, true); + + Ok(Self(ut)) + } +} + +fn text_buffer_from_utext<'a>(ut: &icu_ffi::UText) -> &'a TextBuffer { + unsafe { &*(ut.context as *const TextBuffer) } +} + +fn double_cache_from_utext<'a>(ut: &icu_ffi::UText) -> &'a mut DoubleCache { + unsafe { &mut *(ut.p_extra as *mut DoubleCache) } +} + +extern "C" fn utext_clone( + dest: *mut icu_ffi::UText, + src: &icu_ffi::UText, + deep: bool, + status: &mut icu_ffi::UErrorCode, +) -> *mut icu_ffi::UText { + if status.is_failure() { + return null_mut(); + } + + if deep { + *status = icu_ffi::U_UNSUPPORTED_ERROR; + return null_mut(); + } + + let f = assume_loaded(); + let ut_ptr = unsafe { (f.utext_setup)(dest, size_of::() as i32, status) }; + if status.is_failure() { + return null_mut(); + } + + unsafe { + let ut = &mut *ut_ptr; + let src_double_cache = double_cache_from_utext(src); + let dst_double_cache = double_cache_from_utext(ut); + let src_cache = &src_double_cache.cache[src_double_cache.mru as usize]; + let dst_cache = &mut dst_double_cache.cache[dst_double_cache.mru as usize]; + + ut.provider_properties = src.provider_properties; + ut.chunk_native_limit = src.chunk_native_limit; + ut.native_indexing_limit = src.native_indexing_limit; + ut.chunk_native_start = src.chunk_native_start; + ut.chunk_offset = src.chunk_offset; + ut.chunk_length = src.chunk_length; + ut.chunk_contents = dst_cache.utf16.as_ptr(); + ut.p_funcs = src.p_funcs; + ut.context = src.context; + ut.a = src.a; + + // I wonder if it would make sense to use a Cow here. But probably not. + std::ptr::copy_nonoverlapping(src_cache, dst_cache, 1); + } + + ut_ptr +} + +extern "C" fn utext_native_length(ut: &mut icu_ffi::UText) -> i64 { + let tb = text_buffer_from_utext(ut); + tb.text_length() as i64 +} + +extern "C" fn utext_access(ut: &mut icu_ffi::UText, native_index: i64, forward: bool) -> bool { + if let Some(cache) = utext_access_impl(ut, native_index, forward) { + let native_off = native_index as usize - cache.utf8_range.start; + ut.chunk_contents = cache.utf16.as_ptr(); + ut.chunk_length = cache.utf16_len as i32; + ut.chunk_offset = cache.utf8_to_utf16_offsets[native_off] as i32; + ut.chunk_native_start = cache.utf8_range.start as i64; + ut.chunk_native_limit = cache.utf8_range.end as i64; + ut.native_indexing_limit = cache.native_indexing_limit as i32; + true + } else { + false + } +} + +fn utext_access_impl<'a>( + ut: &mut icu_ffi::UText, + native_index: i64, + forward: bool, +) -> Option<&'a mut Cache> { + let tb = text_buffer_from_utext(ut); + let mut index_contained = native_index; + + if !forward { + index_contained -= 1; + } + if index_contained < 0 || index_contained as usize >= tb.text_length() { + return None; + } + + let index_contained = index_contained as usize; + let native_index = native_index as usize; + let double_cache = double_cache_from_utext(ut); + let dirty = ut.a != tb.generation() as i64; + + if dirty { + // The text buffer contents have changed. + // Invalidate both caches so that future calls don't mistakenly use them + // when they enter the for loop in the else branch below (`dirty == false`). + double_cache.cache[0].utf16_len = 0; + double_cache.cache[1].utf16_len = 0; + double_cache.cache[0].utf8_range = 0..0; + double_cache.cache[1].utf8_range = 0..0; + ut.a = tb.generation() as i64; + } else { + // Check if one of the caches already contains the requested range. + for (i, cache) in double_cache.cache.iter_mut().enumerate() { + if cache.utf8_range.contains(&index_contained) { + double_cache.mru = i != 0; + return Some(cache); + } + } + } + + // Turn the least recently used cache into the most recently used one. + let double_cache = double_cache_from_utext(ut); + double_cache.mru = !double_cache.mru; + let cache = &mut double_cache.cache[double_cache.mru as usize]; + + // In order to safely fit any UTF-8 character into our cache, + // we must assume the worst case of a 4-byte long encoding. + const UTF16_LEN_LIMIT: usize = CACHE_SIZE - 4; + let utf8_len_limit; + let native_start; + + if forward { + utf8_len_limit = (tb.text_length() - native_index).min(UTF16_LEN_LIMIT); + native_start = native_index; + } else { + // The worst case ratio for UTF-8 to UTF-16 is 1:1, when the text is ASCII. + // This allows us to safely subtract the UTF-16 buffer size + // and assume that whatever we read as UTF-8 will fit. + // TODO: Test what happens if you have lots of invalid UTF-8 text blow up to U+FFFD. + utf8_len_limit = native_index.min(UTF16_LEN_LIMIT); + + // Since simply subtracting an offset may end up in the middle of a codepoint sequence, + // we must align the offset to the next codepoint boundary. + // Here we skip trail bytes until we find a lead. + let mut beg = native_index - utf8_len_limit; + let chunk = tb.read_forward(beg); + for &c in chunk { + if c & 0b1100_0000 != 0b1000_0000 { + break; + } + beg += 1; + } + + native_start = beg; + } + + // Translate the given range from UTF-8 to UTF-16. + // NOTE: This code makes the assumption that the `native_index` is always + // at UTF-8 codepoint boundaries which technically isn't guaranteed. + let mut utf16_len = 0; + let mut utf8_len = 0; + let mut ascii_len = 0; + 'outer: loop { + let initial_utf8_len = utf8_len; + let chunk = tb.read_forward(native_start + utf8_len); + if chunk.is_empty() { + break; + } + + let mut it = Utf8Chars::new(chunk, 0); + + // If we've only seen ASCII so far we can fast-pass the UTF-16 translation, + // because we can just widen from u8 -> u16. + if utf16_len == ascii_len { + let haystack = &chunk[..chunk.len().min(utf8_len_limit - ascii_len)]; + + // When it comes to performance, and the search space is small (which it is here), + // it's always a good idea to keep the loops small and tight... + let len = haystack.iter().position(|&c| c >= 0x80).unwrap_or(haystack.len()); + + // ...In this case it allows the compiler to vectorize this loop and double + // the performance. Luckily, llvm doesn't unroll the loop, which is great, + // because `len` will always be a relatively small number. + for &c in &chunk[..len] { + unsafe { + *cache.utf16.get_unchecked_mut(ascii_len) = c as u16; + *cache.utf16_to_utf8_offsets.get_unchecked_mut(ascii_len) = ascii_len as u16; + *cache.utf8_to_utf16_offsets.get_unchecked_mut(ascii_len) = ascii_len as u16; + } + ascii_len += 1; + } + + utf16_len += len; + utf8_len += len; + it.seek(len); + if ascii_len >= UTF16_LEN_LIMIT { + break; + } + } + + loop { + let Some(c) = it.next() else { + break; + }; + + // Thanks to our `if utf16_len >= UTF16_LEN_LIMIT` check, + // we can safely assume that this will fit. + unsafe { + let utf8_len_beg = utf8_len; + let utf8_len_end = initial_utf8_len + it.offset(); + + while utf8_len < utf8_len_end { + *cache.utf8_to_utf16_offsets.get_unchecked_mut(utf8_len) = utf16_len as u16; + utf8_len += 1; + } + + if c <= '\u{FFFF}' { + *cache.utf16.get_unchecked_mut(utf16_len) = c as u16; + *cache.utf16_to_utf8_offsets.get_unchecked_mut(utf16_len) = utf8_len_beg as u16; + utf16_len += 1; + } else { + let c = c as u32 - 0x10000; + let b = utf8_len_beg as u16; + *cache.utf16.get_unchecked_mut(utf16_len) = (c >> 10) as u16 | 0xD800; + *cache.utf16.get_unchecked_mut(utf16_len + 1) = (c & 0x3FF) as u16 | 0xDC00; + *cache.utf16_to_utf8_offsets.get_unchecked_mut(utf16_len) = b; + *cache.utf16_to_utf8_offsets.get_unchecked_mut(utf16_len + 1) = b; + utf16_len += 2; + } + } + + if utf16_len >= UTF16_LEN_LIMIT || utf8_len >= utf8_len_limit { + break 'outer; + } + } + } + + // Allow for looking up past-the-end indices via + // `utext_map_offset_to_native` and `utext_map_native_index_to_utf16`. + cache.utf16_to_utf8_offsets[utf16_len] = utf8_len as u16; + cache.utf8_to_utf16_offsets[utf8_len] = utf16_len as u16; + + let native_limit = native_start + utf8_len; + cache.utf16_len = utf16_len; + // If parts of the UTF-8 chunk are ASCII, we can tell ICU that it doesn't need to call + // utext_map_offset_to_native. For some reason, uregex calls that function *a lot*, + // literally half the CPU time is spent on it. + cache.native_indexing_limit = ascii_len; + cache.utf8_range = native_start..native_limit; + Some(cache) +} + +extern "C" fn utext_map_offset_to_native(ut: &icu_ffi::UText) -> i64 { + debug_assert!((0..=ut.chunk_length).contains(&ut.chunk_offset)); + + let double_cache = double_cache_from_utext(ut); + let cache = &double_cache.cache[double_cache.mru as usize]; + let off_rel = cache.utf16_to_utf8_offsets[ut.chunk_offset as usize]; + let off_abs = cache.utf8_range.start + off_rel as usize; + off_abs as i64 +} + +extern "C" fn utext_map_native_index_to_utf16(ut: &icu_ffi::UText, native_index: i64) -> i32 { + debug_assert!((ut.chunk_native_start..=ut.chunk_native_limit).contains(&native_index)); + + let double_cache = double_cache_from_utext(ut); + let cache = &double_cache.cache[double_cache.mru as usize]; + let off_rel = cache.utf8_to_utf16_offsets[(native_index - ut.chunk_native_start) as usize]; + off_rel as i32 +} + +/// A wrapper around ICU's `URegularExpression` struct. +/// +/// # Safety +/// +/// Warning! No lifetime tracking is done here. +pub struct Regex(&'static mut icu_ffi::URegularExpression); + +impl Drop for Regex { + fn drop(&mut self) { + let f = assume_loaded(); + unsafe { (f.uregex_close)(self.0) }; + } +} + +impl Regex { + /// Enable case-insensitive matching. + pub const CASE_INSENSITIVE: i32 = icu_ffi::UREGEX_CASE_INSENSITIVE; + + /// If set, ^ and $ match the start and end of each line. + /// Otherwise, they match the start and end of the entire string. + pub const MULTILINE: i32 = icu_ffi::UREGEX_MULTILINE; + + /// Treat the given pattern as a literal string. + pub const LITERAL: i32 = icu_ffi::UREGEX_LITERAL; + + /// Constructs a regex, plain and simple. Read `uregex_open` docs. + /// + /// # Safety + /// + /// The caller must ensure that the given `Text` outlives the returned `Regex` instance. + pub unsafe fn new(pattern: &str, flags: i32, text: &Text) -> apperr::Result { + let f = init_if_needed()?; + unsafe { + let scratch = scratch_arena(None); + let mut utf16 = Vec::new_in(&*scratch); + let mut status = icu_ffi::U_ZERO_ERROR; + + utf16.extend(pattern.encode_utf16()); + + let ptr = (f.uregex_open)( + utf16.as_ptr(), + utf16.len() as i32, + icu_ffi::UREGEX_MULTILINE | icu_ffi::UREGEX_ERROR_ON_UNKNOWN_ESCAPES | flags, + None, + &mut status, + ); + // ICU describes the time unit as being dependent on CPU performance + // and "typically [in] the order of milliseconds", but this claim seems + // highly outdated. On my CPU from 2021, a limit of 4096 equals roughly 600ms. + (f.uregex_setTimeLimit)(ptr, 4096, &mut status); + (f.uregex_setUText)(ptr, text.0 as *const _ as *mut _, &mut status); + if status.is_failure() { + return Err(status.as_error()); + } + + Ok(Self(&mut *ptr)) + } + } + + /// Updates the regex pattern with the given text. + /// If the text contents have changed, you can pass the same text as you used + /// initially and it'll trigger ICU to reload the text and invalidate its caches. + /// + /// # Safety + /// + /// The caller must ensure that the given `Text` outlives the `Regex` instance. + pub unsafe fn set_text(&mut self, text: &Text) { + let f = assume_loaded(); + let mut status = icu_ffi::U_ZERO_ERROR; + unsafe { (f.uregex_setUText)(self.0, text.0 as *const _ as *mut _, &mut status) }; + } + + /// Sets the regex to the absolute offset in the underlying text. + pub fn reset(&mut self, index: usize) { + let f = assume_loaded(); + let mut status = icu_ffi::U_ZERO_ERROR; + unsafe { (f.uregex_reset64)(self.0, index as i64, &mut status) }; + } +} + +impl Iterator for Regex { + type Item = Range; + + fn next(&mut self) -> Option { + let f = assume_loaded(); + + let mut status = icu_ffi::U_ZERO_ERROR; + let ok = unsafe { (f.uregex_findNext)(self.0, &mut status) }; + if !ok { + return None; + } + + let start = unsafe { (f.uregex_start64)(self.0, 0, &mut status) }; + let end = unsafe { (f.uregex_end64)(self.0, 0, &mut status) }; + if status.is_failure() { + return None; + } + + let start = start.max(0); + let end = end.max(start); + Some(start as usize..end as usize) + } +} + +static mut ROOT_COLLATOR: Option<*mut icu_ffi::UCollator> = None; + +/// Compares two UTF-8 strings for sorting using ICU's collation algorithm. +pub fn compare_strings(a: &[u8], b: &[u8]) -> Ordering { + // OnceCell for people that want to put it into a static. + #[allow(static_mut_refs)] + let coll = unsafe { + if ROOT_COLLATOR.is_none() { + ROOT_COLLATOR = Some(if let Ok(f) = init_if_needed() { + let mut status = icu_ffi::U_ZERO_ERROR; + (f.ucol_open)(c"".as_ptr(), &mut status) + } else { + null_mut() + }); + } + ROOT_COLLATOR.unwrap_unchecked() + }; + + if coll.is_null() { + compare_strings_ascii(a, b) + } else { + let f = assume_loaded(); + let mut status = icu_ffi::U_ZERO_ERROR; + let res = unsafe { + (f.ucol_strcollUTF8)( + coll, + a.as_ptr(), + a.len() as i32, + b.as_ptr(), + b.len() as i32, + &mut status, + ) + }; + + match res { + icu_ffi::UCollationResult::UCOL_EQUAL => Ordering::Equal, + icu_ffi::UCollationResult::UCOL_GREATER => Ordering::Greater, + icu_ffi::UCollationResult::UCOL_LESS => Ordering::Less, + } + } +} + +/// Unicode collation via `ucol_strcollUTF8`, now for ASCII! +fn compare_strings_ascii(a: &[u8], b: &[u8]) -> Ordering { + let mut iter = a.iter().zip(b.iter()); + + // Low weight: Find the first character which differs. + // + // Remember that result in case all remaining characters are + // case-insensitive equal, because then we use that as a fallback. + while let Some((&a, &b)) = iter.next() { + if a != b { + let mut order = a.cmp(&b); + let la = a.to_ascii_lowercase(); + let lb = b.to_ascii_lowercase(); + + if la == lb { + // High weight: Find the first character which + // differs case-insensitively. + for (a, b) in iter { + let la = a.to_ascii_lowercase(); + let lb = b.to_ascii_lowercase(); + + if la != lb { + order = la.cmp(&lb); + break; + } + } + } + + return order; + } + } + + // Fallback: The shorter string wins. + a.len().cmp(&b.len()) +} + +static mut ROOT_CASEMAP: Option<*mut icu_ffi::UCaseMap> = None; + +/// Converts the given UTF-8 string to lower case. +/// +/// Case folding differs from lower case in that the output is primarily useful +/// to machines for comparisons. It's like applying Unicode normalization. +pub fn fold_case<'a>(arena: &'a Arena, input: &str) -> ArenaString<'a> { + // OnceCell for people that want to put it into a static. + #[allow(static_mut_refs)] + let casemap = unsafe { + if ROOT_CASEMAP.is_none() { + ROOT_CASEMAP = Some(if let Ok(f) = init_if_needed() { + let mut status = icu_ffi::U_ZERO_ERROR; + (f.ucasemap_open)(null(), 0, &mut status) + } else { + null_mut() + }) + } + ROOT_CASEMAP.unwrap_unchecked() + }; + + if !casemap.is_null() { + let f = assume_loaded(); + let mut status = icu_ffi::U_ZERO_ERROR; + let mut output = Vec::new_in(arena); + let mut output_len; + + // First, guess the output length: + // TODO: What's a good heuristic here? + { + output.reserve_exact(input.len() + 16); + let output = output.spare_capacity_mut(); + output_len = unsafe { + (f.ucasemap_utf8FoldCase)( + casemap, + output.as_mut_ptr() as *mut _, + output.len() as i32, + input.as_ptr() as *const _, + input.len() as i32, + &mut status, + ) + }; + } + + // If that failed to fit, retry with the correct length. + if status == icu_ffi::U_BUFFER_OVERFLOW_ERROR && output_len > 0 { + output.reserve_exact(output_len as usize); + let output = output.spare_capacity_mut(); + output_len = unsafe { + (f.ucasemap_utf8FoldCase)( + casemap, + output.as_mut_ptr() as *mut _, + output.len() as i32, + input.as_ptr() as *const _, + input.len() as i32, + &mut status, + ) + }; + } + + if status.is_success() && output_len > 0 { + unsafe { + output.set_len(output_len as usize); + } + return unsafe { ArenaString::from_utf8_unchecked(output) }; + } + } + + let mut result = ArenaString::from_str(arena, input); + for b in unsafe { result.as_bytes_mut() } { + b.make_ascii_lowercase(); + } + result +} + +// WARNING: +// The order of the fields MUST match the order of strings in the following two arrays. +#[allow(non_snake_case)] +#[repr(C)] +struct LibraryFunctions { + // LIBICUUC_PROC_NAMES + u_errorName: icu_ffi::u_errorName, + ucnv_getAvailableName: icu_ffi::ucnv_getAvailableName, + ucnv_open: icu_ffi::ucnv_open, + ucnv_close: icu_ffi::ucnv_close, + ucnv_convertEx: icu_ffi::ucnv_convertEx, + ucasemap_open: icu_ffi::ucasemap_open, + ucasemap_utf8FoldCase: icu_ffi::ucasemap_utf8FoldCase, + utext_setup: icu_ffi::utext_setup, + utext_close: icu_ffi::utext_close, + + // LIBICUI18N_PROC_NAMES + uregex_open: icu_ffi::uregex_open, + uregex_close: icu_ffi::uregex_close, + uregex_setTimeLimit: icu_ffi::uregex_setTimeLimit, + uregex_setUText: icu_ffi::uregex_setUText, + uregex_reset64: icu_ffi::uregex_reset64, + uregex_findNext: icu_ffi::uregex_findNext, + uregex_start64: icu_ffi::uregex_start64, + uregex_end64: icu_ffi::uregex_end64, + ucol_open: icu_ffi::ucol_open, + ucol_strcollUTF8: icu_ffi::ucol_strcollUTF8, +} + +const LIBICUUC_PROC_NAMES: [&CStr; 9] = [ + // Found in libicuuc.so on UNIX, icuuc.dll/icu.dll on Windows. + c"u_errorName", + c"ucnv_getAvailableName", + c"ucnv_open", + c"ucnv_close", + c"ucnv_convertEx", + c"ucasemap_open", + c"ucasemap_utf8FoldCase", + c"utext_setup", + c"utext_close", +]; + +const LIBICUI18N_PROC_NAMES: [&CStr; 10] = [ + // Found in libicui18n.so on UNIX, icuin.dll/icu.dll on Windows. + c"uregex_open", + c"uregex_close", + c"uregex_setTimeLimit", + c"uregex_setUText", + c"uregex_reset64", + c"uregex_findNext", + c"uregex_start64", + c"uregex_end64", + c"ucol_open", + c"ucol_strcollUTF8", +]; + +enum LibraryFunctionsState { + Uninitialized, + Failed, + Loaded(LibraryFunctions), +} + +static mut LIBRARY_FUNCTIONS: LibraryFunctionsState = LibraryFunctionsState::Uninitialized; + +pub fn init() -> apperr::Result<()> { + init_if_needed()?; + Ok(()) +} + +#[allow(static_mut_refs)] +fn init_if_needed() -> apperr::Result<&'static LibraryFunctions> { + #[cold] + fn load() { + unsafe { + LIBRARY_FUNCTIONS = LibraryFunctionsState::Failed; + + let Ok(libicuuc) = sys::load_libicuuc() else { + return; + }; + let Ok(libicui18n) = sys::load_libicui18n() else { + return; + }; + + type TransparentFunction = unsafe extern "C" fn() -> *const (); + + // OH NO I'M DOING A BAD THING + // + // If this assertion hits, you either forgot to update `LIBRARY_PROC_NAMES` + // or you're on a platform where `dlsym` behaves different from classic UNIX and Windows. + // + // This code assumes that we can treat the `LibraryFunctions` struct containing various different function + // pointers as an array of `TransparentFunction` pointers. In C, this works on any platform that supports + // POSIX `dlsym` or equivalent, but I suspect Rust is once again being extra about it. In any case, that's + // still better than loading every function one by one, just to blow up our binary size for no reason. + const _: () = assert!( + mem::size_of::() + == mem::size_of::() + * (LIBICUUC_PROC_NAMES.len() + LIBICUI18N_PROC_NAMES.len()) + ); + + let mut funcs = MaybeUninit::::uninit(); + let mut ptr = funcs.as_mut_ptr() as *mut TransparentFunction; + + #[cfg(unix)] + let scratch_outer = scratch_arena(None); + #[cfg(unix)] + let suffix = sys::icu_proc_suffix(&scratch_outer, libicuuc); + + for (handle, names) in + [(libicuuc, &LIBICUUC_PROC_NAMES[..]), (libicui18n, &LIBICUI18N_PROC_NAMES[..])] + { + for name in names { + #[cfg(unix)] + let scratch = scratch_arena(Some(&scratch_outer)); + #[cfg(unix)] + let name = &sys::add_icu_proc_suffix(&scratch, name, &suffix); + + let Ok(func) = sys::get_proc_address(handle, name) else { + debug_assert!( + false, + "Failed to load ICU function: {}", + name.to_string_lossy() + ); + return; + }; + + ptr.write(func); + ptr = ptr.add(1); + } + } + + LIBRARY_FUNCTIONS = LibraryFunctionsState::Loaded(funcs.assume_init()); + } + } + + unsafe { + if matches!(&LIBRARY_FUNCTIONS, LibraryFunctionsState::Uninitialized) { + load(); + } + } + + match unsafe { &LIBRARY_FUNCTIONS } { + LibraryFunctionsState::Loaded(f) => Ok(f), + _ => Err(apperr::APP_ICU_MISSING), + } +} + +#[allow(static_mut_refs)] +fn assume_loaded() -> &'static LibraryFunctions { + match unsafe { &LIBRARY_FUNCTIONS } { + LibraryFunctionsState::Loaded(f) => f, + _ => unreachable!(), + } +} + +mod icu_ffi { + #![allow(dead_code, non_camel_case_types)] + + use std::ffi::{c_char, c_int, c_void}; + + use crate::apperr; + + #[derive(Copy, Clone, Eq, PartialEq)] + #[repr(transparent)] + pub struct UErrorCode(c_int); + + impl UErrorCode { + pub const fn new(code: u32) -> Self { + Self(code as c_int) + } + + pub fn is_success(&self) -> bool { + self.0 <= 0 + } + + pub fn is_failure(&self) -> bool { + self.0 > 0 + } + + pub fn as_error(&self) -> apperr::Error { + debug_assert!(self.0 > 0); + apperr::Error::new_icu(self.0 as u32) + } + } + + pub const U_ZERO_ERROR: UErrorCode = UErrorCode(0); + pub const U_BUFFER_OVERFLOW_ERROR: UErrorCode = UErrorCode(15); + pub const U_UNSUPPORTED_ERROR: UErrorCode = UErrorCode(16); + + pub type u_errorName = unsafe extern "C" fn(code: UErrorCode) -> *const c_char; + + pub struct UConverter; + + pub type ucnv_getAvailableName = unsafe extern "C" fn(n: i32) -> *mut c_char; + + pub type ucnv_open = + unsafe extern "C" fn(converter_name: *const u8, status: &mut UErrorCode) -> *mut UConverter; + + pub type ucnv_close = unsafe extern "C" fn(converter: *mut UConverter); + + pub type ucnv_convertEx = unsafe extern "C" fn( + target_cnv: *mut UConverter, + source_cnv: *mut UConverter, + target: *mut *mut u8, + target_limit: *const u8, + source: *mut *const u8, + source_limit: *const u8, + pivot_start: *mut u16, + pivot_source: *mut *mut u16, + pivot_target: *mut *mut u16, + pivot_limit: *const u16, + reset: bool, + flush: bool, + status: &mut UErrorCode, + ); + + pub struct UCaseMap; + + pub type ucasemap_open = unsafe extern "C" fn( + locale: *const c_char, + options: u32, + status: &mut UErrorCode, + ) -> *mut UCaseMap; + + pub type ucasemap_utf8FoldCase = unsafe extern "C" fn( + csm: *const UCaseMap, + dest: *mut c_char, + dest_capacity: i32, + src: *const c_char, + src_length: i32, + status: &mut UErrorCode, + ) -> i32; + + #[repr(C)] + pub enum UCollationResult { + UCOL_EQUAL = 0, + UCOL_GREATER = 1, + UCOL_LESS = -1, + } + + #[repr(C)] + pub struct UCollator; + + pub type ucol_open = + unsafe extern "C" fn(loc: *const c_char, status: &mut UErrorCode) -> *mut UCollator; + + pub type ucol_strcollUTF8 = unsafe extern "C" fn( + coll: *mut UCollator, + source: *const u8, + source_length: i32, + target: *const u8, + target_length: i32, + status: &mut UErrorCode, + ) -> UCollationResult; + + // UText callback functions + pub type UTextClone = unsafe extern "C" fn( + dest: *mut UText, + src: &UText, + deep: bool, + status: &mut UErrorCode, + ) -> *mut UText; + pub type UTextNativeLength = unsafe extern "C" fn(ut: &mut UText) -> i64; + pub type UTextAccess = + unsafe extern "C" fn(ut: &mut UText, native_index: i64, forward: bool) -> bool; + pub type UTextExtract = unsafe extern "C" fn( + ut: &mut UText, + native_start: i64, + native_limit: i64, + dest: *mut u16, + dest_capacity: i32, + status: &mut UErrorCode, + ) -> i32; + pub type UTextReplace = unsafe extern "C" fn( + ut: &mut UText, + native_start: i64, + native_limit: i64, + replacement_text: *const u16, + replacement_length: i32, + status: &mut UErrorCode, + ) -> i32; + pub type UTextCopy = unsafe extern "C" fn( + ut: &mut UText, + native_start: i64, + native_limit: i64, + native_dest: i64, + move_text: bool, + status: &mut UErrorCode, + ); + pub type UTextMapOffsetToNative = unsafe extern "C" fn(ut: &UText) -> i64; + pub type UTextMapNativeIndexToUTF16 = + unsafe extern "C" fn(ut: &UText, native_index: i64) -> i32; + pub type UTextClose = unsafe extern "C" fn(ut: &mut UText); + + #[repr(C)] + pub struct UTextFuncs { + pub table_size: i32, + pub reserved1: i32, + pub reserved2: i32, + pub reserved3: i32, + pub clone: Option, + pub native_length: Option, + pub access: Option, + pub extract: Option, + pub replace: Option, + pub copy: Option, + pub map_offset_to_native: Option, + pub map_native_index_to_utf16: Option, + pub close: Option, + pub spare1: Option, + pub spare2: Option, + pub spare3: Option, + } + + #[repr(C)] + pub struct UText { + pub magic: u32, + pub flags: i32, + pub provider_properties: i32, + pub size_of_struct: i32, + pub chunk_native_limit: i64, + pub extra_size: i32, + pub native_indexing_limit: i32, + pub chunk_native_start: i64, + pub chunk_offset: i32, + pub chunk_length: i32, + pub chunk_contents: *const u16, + pub p_funcs: &'static UTextFuncs, + pub p_extra: *mut c_void, + pub context: *mut c_void, + pub p: *mut c_void, + pub q: *mut c_void, + pub r: *mut c_void, + pub priv_p: *mut c_void, + pub a: i64, + pub b: i32, + pub c: i32, + pub priv_a: i64, + pub priv_b: i32, + pub priv_c: i32, + } + + pub const UTEXT_MAGIC: u32 = 0x345ad82c; + pub const UTEXT_PROVIDER_LENGTH_IS_EXPENSIVE: i32 = 1; + pub const UTEXT_PROVIDER_STABLE_CHUNKS: i32 = 2; + pub const UTEXT_PROVIDER_WRITABLE: i32 = 3; + pub const UTEXT_PROVIDER_HAS_META_DATA: i32 = 4; + pub const UTEXT_PROVIDER_OWNS_TEXT: i32 = 5; + + pub type utext_setup = unsafe extern "C" fn( + ut: *mut UText, + extra_space: i32, + status: &mut UErrorCode, + ) -> *mut UText; + pub type utext_close = unsafe extern "C" fn(ut: *mut UText) -> *mut UText; + + #[repr(C)] + pub struct UParseError { + pub line: i32, + pub offset: i32, + pub pre_context: [u16; 16], + pub post_context: [u16; 16], + } + + #[repr(C)] + pub struct URegularExpression; + + pub const UREGEX_UNIX_LINES: i32 = 1; + pub const UREGEX_CASE_INSENSITIVE: i32 = 2; + pub const UREGEX_COMMENTS: i32 = 4; + pub const UREGEX_MULTILINE: i32 = 8; + pub const UREGEX_LITERAL: i32 = 16; + pub const UREGEX_DOTALL: i32 = 32; + pub const UREGEX_UWORD: i32 = 256; + pub const UREGEX_ERROR_ON_UNKNOWN_ESCAPES: i32 = 512; + + pub type uregex_open = unsafe extern "C" fn( + pattern: *const u16, + pattern_length: i32, + flags: i32, + pe: Option<&mut UParseError>, + status: &mut UErrorCode, + ) -> *mut URegularExpression; + pub type uregex_close = unsafe extern "C" fn(regexp: *mut URegularExpression); + pub type uregex_setTimeLimit = + unsafe extern "C" fn(regexp: *mut URegularExpression, limit: i32, status: &mut UErrorCode); + pub type uregex_setUText = unsafe extern "C" fn( + regexp: *mut URegularExpression, + text: *mut UText, + status: &mut UErrorCode, + ); + pub type uregex_reset64 = + unsafe extern "C" fn(regexp: *mut URegularExpression, index: i64, status: &mut UErrorCode); + pub type uregex_findNext = + unsafe extern "C" fn(regexp: *mut URegularExpression, status: &mut UErrorCode) -> bool; + pub type uregex_start64 = unsafe extern "C" fn( + regexp: *mut URegularExpression, + group_num: i32, + status: &mut UErrorCode, + ) -> i64; + pub type uregex_end64 = unsafe extern "C" fn( + regexp: *mut URegularExpression, + group_num: i32, + status: &mut UErrorCode, + ) -> i64; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_compare_strings_ascii() { + // Empty strings + assert_eq!(compare_strings_ascii(b"", b""), Ordering::Equal); + // Equal strings + assert_eq!(compare_strings_ascii(b"hello", b"hello"), Ordering::Equal); + // Different lengths + assert_eq!(compare_strings_ascii(b"abc", b"abcd"), Ordering::Less); + assert_eq!(compare_strings_ascii(b"abcd", b"abc"), Ordering::Greater); + // Same chars, different cases - 1st char wins + assert_eq!(compare_strings_ascii(b"AbC", b"aBc"), Ordering::Less); + // Different chars, different cases - 2nd char wins, because it differs + assert_eq!(compare_strings_ascii(b"hallo", b"Hello"), Ordering::Less); + assert_eq!(compare_strings_ascii(b"Hello", b"hallo"), Ordering::Greater); + } +} diff --git a/pkgs/edit/src/input.rs b/pkgs/edit/src/input.rs new file mode 100644 index 0000000..88a32ac --- /dev/null +++ b/pkgs/edit/src/input.rs @@ -0,0 +1,577 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Parses VT sequences into input events. +//! +//! In the future this allows us to take apart the application and +//! support input schemes that aren't VT, such as UEFI, or GUI. + +use crate::helpers::{CoordType, Point, Size}; +use crate::vt; + +/// Represents a key/modifier combination. +/// +/// TODO: Is this a good idea? I did it to allow typing `kbmod::CTRL | vk::A`. +/// The reason it's an awkard u32 and not a struct is to hopefully make ABIs easier later. +/// Of course you could just translate on the ABI boundary, but my hope is that this +/// design lets me realize some restrictions early on that I can't foresee yet. +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct InputKey(u32); + +impl InputKey { + pub(crate) const fn new(v: u32) -> Self { + Self(v) + } + + pub(crate) const fn from_ascii(ch: char) -> Option { + if ch == ' ' || (ch >= '0' && ch <= '9') { + Some(Self(ch as u32)) + } else if ch >= 'a' && ch <= 'z' { + Some(Self(ch as u32 & !0x20)) // Shift a-z to A-Z + } else if ch >= 'A' && ch <= 'Z' { + Some(Self(kbmod::SHIFT.0 | ch as u32)) + } else { + None + } + } + + pub(crate) const fn value(&self) -> u32 { + self.0 + } + + pub(crate) const fn key(&self) -> InputKey { + InputKey(self.0 & 0x00FFFFFF) + } + + pub(crate) const fn modifiers(&self) -> InputKeyMod { + InputKeyMod(self.0 & 0xFF000000) + } + + pub(crate) const fn modifiers_contains(&self, modifier: InputKeyMod) -> bool { + (self.0 & modifier.0) != 0 + } + + pub(crate) const fn with_modifiers(&self, modifiers: InputKeyMod) -> InputKey { + InputKey(self.0 | modifiers.0) + } +} + +/// A keyboard modifier. Ctrl/Alt/Shift. +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct InputKeyMod(u32); + +impl InputKeyMod { + const fn new(v: u32) -> Self { + Self(v) + } + + pub(crate) const fn contains(&self, modifier: InputKeyMod) -> bool { + (self.0 & modifier.0) != 0 + } +} + +impl std::ops::BitOr for InputKey { + type Output = InputKey; + + fn bitor(self, rhs: InputKeyMod) -> InputKey { + InputKey(self.0 | rhs.0) + } +} + +impl std::ops::BitOr for InputKeyMod { + type Output = InputKey; + + fn bitor(self, rhs: InputKey) -> InputKey { + InputKey(self.0 | rhs.0) + } +} + +impl std::ops::BitOrAssign for InputKeyMod { + fn bitor_assign(&mut self, rhs: Self) { + self.0 |= rhs.0; + } +} + +/// Keyboard keys. +/// +/// The codes defined here match the VK_* constants on Windows. +/// It's a convenient way to handle keyboard input, even on other platforms. +pub mod vk { + use super::InputKey; + + pub const NULL: InputKey = InputKey::new('\0' as u32); + pub const BACK: InputKey = InputKey::new(0x08); + pub const TAB: InputKey = InputKey::new('\t' as u32); + pub const RETURN: InputKey = InputKey::new('\r' as u32); + pub const ESCAPE: InputKey = InputKey::new(0x1B); + pub const SPACE: InputKey = InputKey::new(' ' as u32); + pub const PRIOR: InputKey = InputKey::new(0x21); + pub const NEXT: InputKey = InputKey::new(0x22); + + pub const END: InputKey = InputKey::new(0x23); + pub const HOME: InputKey = InputKey::new(0x24); + + pub const LEFT: InputKey = InputKey::new(0x25); + pub const UP: InputKey = InputKey::new(0x26); + pub const RIGHT: InputKey = InputKey::new(0x27); + pub const DOWN: InputKey = InputKey::new(0x28); + + pub const INSERT: InputKey = InputKey::new(0x2D); + pub const DELETE: InputKey = InputKey::new(0x2E); + + pub const N0: InputKey = InputKey::new('0' as u32); + pub const N1: InputKey = InputKey::new('1' as u32); + pub const N2: InputKey = InputKey::new('2' as u32); + pub const N3: InputKey = InputKey::new('3' as u32); + pub const N4: InputKey = InputKey::new('4' as u32); + pub const N5: InputKey = InputKey::new('5' as u32); + pub const N6: InputKey = InputKey::new('6' as u32); + pub const N7: InputKey = InputKey::new('7' as u32); + pub const N8: InputKey = InputKey::new('8' as u32); + pub const N9: InputKey = InputKey::new('9' as u32); + + pub const A: InputKey = InputKey::new('A' as u32); + pub const B: InputKey = InputKey::new('B' as u32); + pub const C: InputKey = InputKey::new('C' as u32); + pub const D: InputKey = InputKey::new('D' as u32); + pub const E: InputKey = InputKey::new('E' as u32); + pub const F: InputKey = InputKey::new('F' as u32); + pub const G: InputKey = InputKey::new('G' as u32); + pub const H: InputKey = InputKey::new('H' as u32); + pub const I: InputKey = InputKey::new('I' as u32); + pub const J: InputKey = InputKey::new('J' as u32); + pub const K: InputKey = InputKey::new('K' as u32); + pub const L: InputKey = InputKey::new('L' as u32); + pub const M: InputKey = InputKey::new('M' as u32); + pub const N: InputKey = InputKey::new('N' as u32); + pub const O: InputKey = InputKey::new('O' as u32); + pub const P: InputKey = InputKey::new('P' as u32); + pub const Q: InputKey = InputKey::new('Q' as u32); + pub const R: InputKey = InputKey::new('R' as u32); + pub const S: InputKey = InputKey::new('S' as u32); + pub const T: InputKey = InputKey::new('T' as u32); + pub const U: InputKey = InputKey::new('U' as u32); + pub const V: InputKey = InputKey::new('V' as u32); + pub const W: InputKey = InputKey::new('W' as u32); + pub const X: InputKey = InputKey::new('X' as u32); + pub const Y: InputKey = InputKey::new('Y' as u32); + pub const Z: InputKey = InputKey::new('Z' as u32); + + pub const NUMPAD0: InputKey = InputKey::new(0x60); + pub const NUMPAD1: InputKey = InputKey::new(0x61); + pub const NUMPAD2: InputKey = InputKey::new(0x62); + pub const NUMPAD3: InputKey = InputKey::new(0x63); + pub const NUMPAD4: InputKey = InputKey::new(0x64); + pub const NUMPAD5: InputKey = InputKey::new(0x65); + pub const NUMPAD6: InputKey = InputKey::new(0x66); + pub const NUMPAD7: InputKey = InputKey::new(0x67); + pub const NUMPAD8: InputKey = InputKey::new(0x68); + pub const NUMPAD9: InputKey = InputKey::new(0x69); + pub const MULTIPLY: InputKey = InputKey::new(0x6A); + pub const ADD: InputKey = InputKey::new(0x6B); + pub const SEPARATOR: InputKey = InputKey::new(0x6C); + pub const SUBTRACT: InputKey = InputKey::new(0x6D); + pub const DECIMAL: InputKey = InputKey::new(0x6E); + pub const DIVIDE: InputKey = InputKey::new(0x6F); + + pub const F1: InputKey = InputKey::new(0x70); + pub const F2: InputKey = InputKey::new(0x71); + pub const F3: InputKey = InputKey::new(0x72); + pub const F4: InputKey = InputKey::new(0x73); + pub const F5: InputKey = InputKey::new(0x74); + pub const F6: InputKey = InputKey::new(0x75); + pub const F7: InputKey = InputKey::new(0x76); + pub const F8: InputKey = InputKey::new(0x77); + pub const F9: InputKey = InputKey::new(0x78); + pub const F10: InputKey = InputKey::new(0x79); + pub const F11: InputKey = InputKey::new(0x7A); + pub const F12: InputKey = InputKey::new(0x7B); + pub const F13: InputKey = InputKey::new(0x7C); + pub const F14: InputKey = InputKey::new(0x7D); + pub const F15: InputKey = InputKey::new(0x7E); + pub const F16: InputKey = InputKey::new(0x7F); + pub const F17: InputKey = InputKey::new(0x80); + pub const F18: InputKey = InputKey::new(0x81); + pub const F19: InputKey = InputKey::new(0x82); + pub const F20: InputKey = InputKey::new(0x83); + pub const F21: InputKey = InputKey::new(0x84); + pub const F22: InputKey = InputKey::new(0x85); + pub const F23: InputKey = InputKey::new(0x86); + pub const F24: InputKey = InputKey::new(0x87); +} + +/// Keyboard modifiers. +pub mod kbmod { + use super::InputKeyMod; + + pub const NONE: InputKeyMod = InputKeyMod::new(0x00000000); + pub const CTRL: InputKeyMod = InputKeyMod::new(0x01000000); + pub const ALT: InputKeyMod = InputKeyMod::new(0x02000000); + pub const SHIFT: InputKeyMod = InputKeyMod::new(0x04000000); + + pub const CTRL_ALT: InputKeyMod = InputKeyMod::new(0x03000000); + pub const CTRL_SHIFT: InputKeyMod = InputKeyMod::new(0x05000000); + pub const ALT_SHIFT: InputKeyMod = InputKeyMod::new(0x06000000); + pub const CTRL_ALT_SHIFT: InputKeyMod = InputKeyMod::new(0x07000000); +} + +/// Text input. +/// +/// "Keyboard" input is also "text" input and vice versa. +/// It differs in that text input can also be Unicode. +#[derive(Clone, Copy)] +pub struct InputText<'a> { + pub text: &'a str, + pub bracketed: bool, +} + +/// Mouse input state. Up/Down, Left/Right, etc. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default)] +pub enum InputMouseState { + #[default] + None, + + // These 3 carry their state between frames. + Left, + Middle, + Right, + + // These 2 get reset to None on the next frame. + Release, + Scroll, +} + +/// Mouse input. +#[derive(Clone, Copy)] +pub struct InputMouse { + /// The state of the mouse.Up/Down, Left/Right, etc. + pub state: InputMouseState, + /// Any keyboard modifiers that are held down. + pub modifiers: InputKeyMod, + /// Position of the mouse in the viewport. + pub position: Point, + /// Scroll delta. + pub scroll: Point, +} + +/// Primary result type of the parser. +pub enum Input<'input> { + /// Window resize event. + Resize(Size), + /// Text input. + /// + /// Note that [`Input::Keyboard`] events can also be text. + Text(InputText<'input>), + /// Keyboard input. + Keyboard(InputKey), + /// Mouse input. + Mouse(InputMouse), +} + +/// Parses VT sequences into input events. +pub struct Parser { + bracketed_paste: bool, + x10_mouse_want: bool, + x10_mouse_buf: [u8; 3], + x10_mouse_len: usize, +} + +impl Parser { + /// Creates a new parser that turns VT sequences into input events. + /// + /// Keep the instance alive for the lifetime of the input stream. + pub fn new() -> Self { + Self { + bracketed_paste: false, + x10_mouse_want: false, + x10_mouse_buf: [0; 3], + x10_mouse_len: 0, + } + } + + /// Takes an [`vt::Stream`] and returns a [`Stream`] + /// that turns VT sequences into input events. + pub fn parse<'parser, 'vt, 'input>( + &'parser mut self, + stream: vt::Stream<'vt, 'input>, + ) -> Stream<'parser, 'vt, 'input> { + Stream { parser: self, stream } + } +} + +/// An iterator that parses VT sequences into input events. +/// +/// Can't implement [`Iterator`], because this is a "lending iterator". +pub struct Stream<'parser, 'vt, 'input> { + parser: &'parser mut Parser, + stream: vt::Stream<'vt, 'input>, +} + +impl<'input> Stream<'_, '_, 'input> { + #[allow(clippy::should_implement_trait)] + pub fn next(&mut self) -> Option> { + loop { + if self.parser.bracketed_paste { + return self.handle_bracketed_paste(); + } + + if self.parser.x10_mouse_want { + return self.parse_x10_mouse_coordinates(); + } + + match self.stream.next()? { + vt::Token::Text(text) => { + return Some(Input::Text(InputText { text, bracketed: false })); + } + vt::Token::Ctrl(ch) => match ch { + '\0' | '\t' | '\r' => return Some(Input::Keyboard(InputKey::new(ch as u32))), + '\n' => return Some(Input::Keyboard(kbmod::CTRL | vk::RETURN)), + ..='\x1a' => { + // Shift control code to A-Z + let key = ch as u32 | 0x40; + return Some(Input::Keyboard(kbmod::CTRL | InputKey::new(key))); + } + '\x7f' => return Some(Input::Keyboard(vk::BACK)), + _ => {} + }, + vt::Token::Esc(ch) => { + match ch { + '\0' => return Some(Input::Keyboard(vk::ESCAPE)), + '\n' => return Some(Input::Keyboard(kbmod::CTRL_ALT | vk::RETURN)), + ' '..='~' => { + let ch = ch as u32; + let key = ch & !0x20; // Shift a-z to A-Z + let modifiers = + if (ch & 0x20) != 0 { kbmod::ALT } else { kbmod::ALT_SHIFT }; + return Some(Input::Keyboard(modifiers | InputKey::new(key))); + } + _ => {} + } + } + vt::Token::SS3(ch) => { + if ('P'..='S').contains(&ch) { + let key = vk::F1.value() + ch as u32 - 'P' as u32; + return Some(Input::Keyboard(InputKey::new(key))); + } + } + vt::Token::Csi(csi) => { + match csi.final_byte { + 'A'..='H' => { + const LUT: [u8; 8] = [ + vk::UP.value() as u8, // A + vk::DOWN.value() as u8, // B + vk::RIGHT.value() as u8, // C + vk::LEFT.value() as u8, // D + 0, // E + vk::END.value() as u8, // F + 0, // G + vk::HOME.value() as u8, // H + ]; + let vk = LUT[csi.final_byte as usize - 'A' as usize]; + if vk != 0 { + return Some(Input::Keyboard( + InputKey::new(vk as u32) | Self::parse_modifiers(csi), + )); + } + } + 'Z' => return Some(Input::Keyboard(kbmod::SHIFT | vk::TAB)), + '~' => { + const LUT: [u8; 35] = [ + 0, + vk::HOME.value() as u8, // 1 + vk::INSERT.value() as u8, // 2 + vk::DELETE.value() as u8, // 3 + vk::END.value() as u8, // 4 + vk::PRIOR.value() as u8, // 5 + vk::NEXT.value() as u8, // 6 + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + vk::F5.value() as u8, // 15 + 0, + vk::F6.value() as u8, // 17 + vk::F7.value() as u8, // 18 + vk::F8.value() as u8, // 19 + vk::F9.value() as u8, // 20 + vk::F10.value() as u8, // 21 + 0, + vk::F11.value() as u8, // 23 + vk::F12.value() as u8, // 24 + vk::F13.value() as u8, // 25 + vk::F14.value() as u8, // 26 + 0, + vk::F15.value() as u8, // 28 + vk::F16.value() as u8, // 29 + 0, + vk::F17.value() as u8, // 31 + vk::F18.value() as u8, // 32 + vk::F19.value() as u8, // 33 + vk::F20.value() as u8, // 34 + ]; + const LUT_LEN: u16 = LUT.len() as u16; + + match csi.params[0] { + 0..LUT_LEN => { + let vk = LUT[csi.params[0] as usize]; + if vk != 0 { + return Some(Input::Keyboard( + InputKey::new(vk as u32) | Self::parse_modifiers(csi), + )); + } + } + 200 => self.parser.bracketed_paste = true, + _ => {} + } + } + 'm' | 'M' if csi.private_byte == '<' => { + let btn = csi.params[0]; + let mut mouse = InputMouse { + state: InputMouseState::None, + modifiers: kbmod::NONE, + position: Default::default(), + scroll: Default::default(), + }; + + mouse.state = InputMouseState::None; + if (btn & 0x40) != 0 { + mouse.state = InputMouseState::Scroll; + mouse.scroll.y += if (btn & 0x01) != 0 { 3 } else { -3 }; + } else if csi.final_byte == 'M' { + const STATES: [InputMouseState; 4] = [ + InputMouseState::Left, + InputMouseState::Middle, + InputMouseState::Right, + InputMouseState::None, + ]; + mouse.state = STATES[(btn as usize) & 0x03]; + } + + mouse.modifiers = kbmod::NONE; + mouse.modifiers |= + if (btn & 0x04) != 0 { kbmod::SHIFT } else { kbmod::NONE }; + mouse.modifiers |= + if (btn & 0x08) != 0 { kbmod::ALT } else { kbmod::NONE }; + mouse.modifiers |= + if (btn & 0x10f) != 0 { kbmod::CTRL } else { kbmod::NONE }; + + mouse.position.x = csi.params[1] as CoordType - 1; + mouse.position.y = csi.params[2] as CoordType - 1; + return Some(Input::Mouse(mouse)); + } + 'M' if csi.param_count == 0 => { + self.parser.x10_mouse_want = true; + } + 't' if csi.params[0] == 8 => { + // Window Size + let width = (csi.params[2] as CoordType).clamp(1, 32767); + let height = (csi.params[1] as CoordType).clamp(1, 32767); + return Some(Input::Resize(Size { width, height })); + } + _ => {} + } + } + _ => {} + } + } + } + + /// Once we encounter the start of a bracketed paste + /// we seek to the end of the paste in this function. + /// + /// A bracketed paste is basically: + /// ```text + /// [201~ lots of text [201~ + /// ``` + /// + /// That text inbetween is then expected to be taken literally. + /// It can inbetween be anything though, including other escape sequences. + /// This is the reason why this is a separate method. + #[cold] + fn handle_bracketed_paste(&mut self) -> Option> { + let beg = self.stream.offset(); + let mut end = beg; + + while let Some(token) = self.stream.next() { + if let vt::Token::Csi(csi) = token + && csi.final_byte == '~' + && csi.params[0] == 201 + { + self.parser.bracketed_paste = false; + break; + } + end = self.stream.offset(); + } + + if end != beg { + let input = self.stream.input(); + Some(Input::Text(InputText { text: &input[beg..end], bracketed: true })) + } else { + None + } + } + + /// Implements the X10 mouse protocol via `CSI M CbCxCy`. + /// + /// You want to send numeric mouse coordinates. + /// You have CSI sequences with numeric parameters. + /// So, of course you put the coordinates as shifted ASCII characters after + /// the end of the sequence. Limited coordinate range and complicated parsing! + /// This is so puzzling to me. The existence of this function makes me unhappy. + #[cold] + fn parse_x10_mouse_coordinates(&mut self) -> Option> { + self.parser.x10_mouse_len += + self.stream.read(&mut self.parser.x10_mouse_buf[self.parser.x10_mouse_len..]); + if self.parser.x10_mouse_len < 3 { + return None; + } + + let button = self.parser.x10_mouse_buf[0] & 0b11; + let modifier = self.parser.x10_mouse_buf[0] & 0b11100; + let x = self.parser.x10_mouse_buf[1] as CoordType - 0x21; + let y = self.parser.x10_mouse_buf[2] as CoordType - 0x21; + let action = match button { + 0 => InputMouseState::Left, + 1 => InputMouseState::Middle, + 2 => InputMouseState::Right, + _ => InputMouseState::None, + }; + let modifiers = match modifier { + 4 => kbmod::SHIFT, + 8 => kbmod::ALT, + 16 => kbmod::CTRL, + _ => kbmod::NONE, + }; + + self.parser.x10_mouse_want = false; + self.parser.x10_mouse_len = 0; + + Some(Input::Mouse(InputMouse { + state: action, + modifiers, + position: Point { x, y }, + scroll: Default::default(), + })) + } + + fn parse_modifiers(csi: &vt::Csi) -> InputKeyMod { + let mut modifiers = kbmod::NONE; + let p1 = csi.params[1].saturating_sub(1); + if (p1 & 0x01) != 0 { + modifiers |= kbmod::SHIFT; + } + if (p1 & 0x02) != 0 { + modifiers |= kbmod::ALT; + } + if (p1 & 0x04) != 0 { + modifiers |= kbmod::CTRL; + } + modifiers + } +} diff --git a/pkgs/edit/src/lib.rs b/pkgs/edit/src/lib.rs new file mode 100644 index 0000000..ed0b978 --- /dev/null +++ b/pkgs/edit/src/lib.rs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#![feature( + allocator_api, + breakpoint, + cold_path, + inherent_str_constructors, + let_chains, + linked_list_cursors, + maybe_uninit_fill, + maybe_uninit_slice, + maybe_uninit_uninit_array_transpose, + os_string_truncate +)] +#![allow(clippy::missing_transmute_annotations, clippy::new_without_default, stable_features)] + +#[macro_use] +pub mod arena; + +pub mod apperr; +pub mod base64; +pub mod buffer; +pub mod cell; +pub mod document; +pub mod framebuffer; +pub mod hash; +pub mod helpers; +pub mod icu; +pub mod input; +pub mod oklab; +pub mod path; +pub mod simd; +pub mod sys; +pub mod tui; +pub mod unicode; +pub mod vt; diff --git a/pkgs/edit/src/oklab.rs b/pkgs/edit/src/oklab.rs new file mode 100644 index 0000000..5af68bf --- /dev/null +++ b/pkgs/edit/src/oklab.rs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Oklab colorspace conversions. +//! +//! Implements Oklab as defined at: + +#![allow(clippy::excessive_precision)] + +/// An Oklab color with alpha. +pub struct Lab { + pub l: f32, + pub a: f32, + pub b: f32, + pub alpha: f32, +} + +/// Converts a 32-bit sRGB color to Oklab. +pub fn srgb_to_oklab(color: u32) -> Lab { + let r = SRGB_TO_RGB_LUT[(color & 0xff) as usize]; + let g = SRGB_TO_RGB_LUT[((color >> 8) & 0xff) as usize]; + let b = SRGB_TO_RGB_LUT[((color >> 16) & 0xff) as usize]; + let alpha = (color >> 24) as f32 * (1.0 / 255.0); + + let l = 0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b; + let m = 0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b; + let s = 0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b; + + let l_ = cbrtf_est(l); + let m_ = cbrtf_est(m); + let s_ = cbrtf_est(s); + + Lab { + l: 0.2104542553 * l_ + 0.7936177850 * m_ - 0.0040720468 * s_, + a: 1.9779984951 * l_ - 2.4285922050 * m_ + 0.4505937099 * s_, + b: 0.0259040371 * l_ + 0.7827717662 * m_ - 0.8086757660 * s_, + alpha, + } +} + +/// Converts an Oklab color to a 32-bit sRGB color. +pub fn oklab_to_srgb(c: Lab) -> u32 { + let l_ = c.l + 0.3963377774 * c.a + 0.2158037573 * c.b; + let m_ = c.l - 0.1055613458 * c.a - 0.0638541728 * c.b; + let s_ = c.l - 0.0894841775 * c.a - 1.2914855480 * c.b; + + let l = l_ * l_ * l_; + let m = m_ * m_ * m_; + let s = s_ * s_ * s_; + + let r = 4.0767416621 * l - 3.3077115913 * m + 0.2309699292 * s; + let g = -1.2684380046 * l + 2.6097574011 * m - 0.3413193965 * s; + let b = -0.0041960863 * l - 0.7034186147 * m + 1.7076147010 * s; + + let r = r.clamp(0.0, 1.0); + let g = g.clamp(0.0, 1.0); + let b = b.clamp(0.0, 1.0); + let alpha = c.alpha.clamp(0.0, 1.0); + + let r = linear_to_srgb(r); + let g = linear_to_srgb(g); + let b = linear_to_srgb(b); + let a = (alpha * 255.0) as u32; + + r | (g << 8) | (b << 16) | (a << 24) +} + +/// Blends two 32-bit sRGB colors in the Oklab color space. +pub fn oklab_blend(dst: u32, src: u32) -> u32 { + let dst = srgb_to_oklab(dst); + let src = srgb_to_oklab(src); + + let inv_a = 1.0 - src.alpha; + let l = src.l + dst.l * inv_a; + let a = src.a + dst.a * inv_a; + let b = src.b + dst.b * inv_a; + let alpha = src.alpha + dst.alpha * inv_a; + + oklab_to_srgb(Lab { l, a, b, alpha }) +} + +fn linear_to_srgb(c: f32) -> u32 { + (if c > 0.0031308 { + 255.0 * 1.055 * c.powf(1.0 / 2.4) - 255.0 * 0.055 + } else { + 255.0 * 12.92 * c + }) as u32 +} + +#[inline] +fn cbrtf_est(a: f32) -> f32 { + // http://metamerist.com/cbrt/cbrt.htm showed a great estimator for the cube root: + // f32_as_uint32_t / 3 + 709921077 + // It's similar to the well known "fast inverse square root" trick. + // Lots of numbers around 709921077 perform at least equally well to 709921077, + // and it is unknown how and why 709921077 was chosen specifically. + let u: u32 = f32::to_bits(a); // evil f32ing point bit level hacking + let u = u / 3 + 709921077; // what the fuck? + let x: f32 = f32::from_bits(u); + + // One round of Newton's method. It follows the Wikipedia article at + // https://en.wikipedia.org/wiki/Cube_root#Numerical_methods + // For `a`s in the range between 0 and 1, this results in a maximum error of + // less than 6.7e-4f, which is not good, but good enough for us, because + // we're not an image editor. The benefit is that it's really fast. + (1.0 / 3.0) * (a / (x * x) + (x + x)) // 1st iteration +} + +#[rustfmt::skip] +#[allow(clippy::excessive_precision)] +const SRGB_TO_RGB_LUT: [f32; 256] = [ + 0.0000000000, 0.0003035270, 0.0006070540, 0.0009105810, 0.0012141080, 0.0015176350, 0.0018211619, 0.0021246888, 0.0024282159, 0.0027317430, 0.0030352699, 0.0033465356, 0.0036765069, 0.0040247170, 0.0043914421, 0.0047769533, + 0.0051815170, 0.0056053917, 0.0060488326, 0.0065120910, 0.0069954102, 0.0074990317, 0.0080231922, 0.0085681248, 0.0091340570, 0.0097212177, 0.0103298230, 0.0109600937, 0.0116122449, 0.0122864870, 0.0129830306, 0.0137020806, + 0.0144438436, 0.0152085144, 0.0159962922, 0.0168073755, 0.0176419523, 0.0185002182, 0.0193823613, 0.0202885624, 0.0212190095, 0.0221738834, 0.0231533647, 0.0241576303, 0.0251868572, 0.0262412224, 0.0273208916, 0.0284260381, + 0.0295568332, 0.0307134409, 0.0318960287, 0.0331047624, 0.0343398079, 0.0356013142, 0.0368894450, 0.0382043645, 0.0395462364, 0.0409151986, 0.0423114114, 0.0437350273, 0.0451862030, 0.0466650836, 0.0481718220, 0.0497065634, + 0.0512694679, 0.0528606549, 0.0544802807, 0.0561284944, 0.0578054339, 0.0595112406, 0.0612460710, 0.0630100295, 0.0648032799, 0.0666259527, 0.0684781820, 0.0703601092, 0.0722718611, 0.0742135793, 0.0761853904, 0.0781874284, + 0.0802198276, 0.0822827145, 0.0843762159, 0.0865004659, 0.0886556059, 0.0908417329, 0.0930589810, 0.0953074843, 0.0975873619, 0.0998987406, 0.1022417471, 0.1046164930, 0.1070231125, 0.1094617173, 0.1119324341, 0.1144353822, + 0.1169706732, 0.1195384338, 0.1221387982, 0.1247718409, 0.1274376959, 0.1301364899, 0.1328683347, 0.1356333494, 0.1384316236, 0.1412633061, 0.1441284865, 0.1470272839, 0.1499598026, 0.1529261619, 0.1559264660, 0.1589608639, + 0.1620294005, 0.1651322246, 0.1682693958, 0.1714410931, 0.1746473908, 0.1778884083, 0.1811642349, 0.1844749898, 0.1878207624, 0.1912016720, 0.1946178079, 0.1980693042, 0.2015562356, 0.2050787061, 0.2086368501, 0.2122307271, + 0.2158605307, 0.2195262313, 0.2232279778, 0.2269658893, 0.2307400703, 0.2345506549, 0.2383976579, 0.2422811985, 0.2462013960, 0.2501583695, 0.2541521788, 0.2581829131, 0.2622507215, 0.2663556635, 0.2704978585, 0.2746773660, + 0.2788943350, 0.2831487954, 0.2874408960, 0.2917706966, 0.2961383164, 0.3005438447, 0.3049873710, 0.3094689548, 0.3139887452, 0.3185468316, 0.3231432438, 0.3277781308, 0.3324515820, 0.3371636569, 0.3419144452, 0.3467040956, + 0.3515326977, 0.3564002514, 0.3613068759, 0.3662526906, 0.3712377846, 0.3762622178, 0.3813261092, 0.3864295185, 0.3915725648, 0.3967553079, 0.4019778669, 0.4072403014, 0.4125427008, 0.4178851545, 0.4232677519, 0.4286905527, + 0.4341537058, 0.4396572411, 0.4452012479, 0.4507858455, 0.4564110637, 0.4620770514, 0.4677838385, 0.4735315442, 0.4793202281, 0.4851499796, 0.4910208881, 0.4969330430, 0.5028865933, 0.5088814497, 0.5149177909, 0.5209956765, + 0.5271152258, 0.5332764983, 0.5394796133, 0.5457245708, 0.5520114899, 0.5583404899, 0.5647116303, 0.5711249113, 0.5775805116, 0.5840784907, 0.5906189084, 0.5972018838, 0.6038274169, 0.6104956269, 0.6172066331, 0.6239604354, + 0.6307572126, 0.6375969648, 0.6444797516, 0.6514056921, 0.6583748460, 0.6653873324, 0.6724432111, 0.6795425415, 0.6866854429, 0.6938719153, 0.7011020184, 0.7083759308, 0.7156936526, 0.7230552435, 0.7304608822, 0.7379105687, + 0.7454043627, 0.7529423237, 0.7605246305, 0.7681512833, 0.7758223414, 0.7835379243, 0.7912980318, 0.7991028428, 0.8069523573, 0.8148466945, 0.8227858543, 0.8307699561, 0.8387991190, 0.8468732834, 0.8549926877, 0.8631572723, + 0.8713672161, 0.8796223402, 0.8879231811, 0.8962693810, 0.9046613574, 0.9130986929, 0.9215820432, 0.9301108718, 0.9386858940, 0.9473065734, 0.9559735060, 0.9646862745, 0.9734454751, 0.9822505713, 0.9911022186, 1.0000000000, +]; diff --git a/pkgs/edit/src/path.rs b/pkgs/edit/src/path.rs new file mode 100644 index 0000000..45ba6ac --- /dev/null +++ b/pkgs/edit/src/path.rs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Path related helpers. + +use std::ffi::OsStr; +use std::path::{Component, MAIN_SEPARATOR_STR, Path, PathBuf}; + +/// Normalizes a given path by removing redundant components. +/// The given path must be absolute (e.g. by joining it with the current working directory). +pub fn normalize(path: &Path) -> PathBuf { + debug_assert!(path.is_absolute()); + + let mut res = PathBuf::with_capacity(path.as_os_str().len()); + let mut root_len = 0; + + for component in path.components() { + match component { + Component::Prefix(p) => res.push(p.as_os_str()), + Component::RootDir => { + res.push(OsStr::new(MAIN_SEPARATOR_STR)); + root_len = res.as_os_str().len(); + } + Component::CurDir => {} + Component::ParentDir => { + // Get the length up to the parent directory + if let Some(len) = res + .parent() + .map(|p| p.as_os_str().as_encoded_bytes().len()) + // Ensure we don't pop the root directory + && len >= root_len + { + res.as_mut_os_string().truncate(len); + } + } + Component::Normal(p) => res.push(p), + } + } + + res +} + +#[cfg(test)] +mod tests { + use std::ffi::OsString; + use std::path::Path; + + use super::*; + + fn norm(s: &str) -> OsString { + normalize(Path::new(s)).into_os_string() + } + + #[cfg(unix)] + #[test] + fn test_unix() { + assert_eq!(norm("/a/b/c"), "/a/b/c"); + assert_eq!(norm("/a/b/c/"), "/a/b/c"); + assert_eq!(norm("/a/./b"), "/a/b"); + assert_eq!(norm("/a/b/../c"), "/a/c"); + assert_eq!(norm("/../../a"), "/a"); + assert_eq!(norm("/../"), "/"); + assert_eq!(norm("/a//b/c"), "/a/b/c"); + assert_eq!(norm("/a/b/c/../../../../d"), "/d"); + assert_eq!(norm("//"), "/"); + } + + #[cfg(windows)] + #[test] + fn test_windows() { + assert_eq!(norm(r"C:\a\b\c"), r"C:\a\b\c"); + assert_eq!(norm(r"C:\a\b\c\"), r"C:\a\b\c"); + assert_eq!(norm(r"C:\a\.\b"), r"C:\a\b"); + assert_eq!(norm(r"C:\a\b\..\c"), r"C:\a\c"); + assert_eq!(norm(r"C:\..\..\a"), r"C:\a"); + assert_eq!(norm(r"C:\..\"), r"C:\"); + assert_eq!(norm(r"C:\a\\b\c"), r"C:\a\b\c"); + assert_eq!(norm(r"C:/a\b/c"), r"C:\a\b\c"); + assert_eq!(norm(r"C:\a\b\c\..\..\..\..\d"), r"C:\d"); + assert_eq!(norm(r"\\server\share\path"), r"\\server\share\path"); + } +} diff --git a/pkgs/edit/src/simd/memchr2.rs b/pkgs/edit/src/simd/memchr2.rs new file mode 100644 index 0000000..be2980e --- /dev/null +++ b/pkgs/edit/src/simd/memchr2.rs @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! `memchr`, but with two needles. + +use std::ptr; + +use super::distance; + +/// `memchr`, but with two needles. +/// +/// Returns the index of the first occurrence of either needle in the +/// `haystack`. If no needle is found, `haystack.len()` is returned. +/// `offset` specifies the index to start searching from. +pub fn memchr2(needle1: u8, needle2: u8, haystack: &[u8], offset: usize) -> usize { + unsafe { + let beg = haystack.as_ptr(); + let end = beg.add(haystack.len()); + let it = beg.add(offset.min(haystack.len())); + let it = memchr2_raw(needle1, needle2, it, end); + distance(it, beg) + } +} + +unsafe fn memchr2_raw(needle1: u8, needle2: u8, beg: *const u8, end: *const u8) -> *const u8 { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + return unsafe { MEMCHR2_DISPATCH(needle1, needle2, beg, end) }; + + #[cfg(target_arch = "aarch64")] + return unsafe { memchr2_neon(needle1, needle2, beg, end) }; +} + +unsafe fn memchr2_fallback( + needle1: u8, + needle2: u8, + mut beg: *const u8, + end: *const u8, +) -> *const u8 { + unsafe { + while !ptr::eq(beg, end) { + let ch = *beg; + if ch == needle1 || ch == needle2 { + break; + } + beg = beg.add(1); + } + beg + } +} + +// In order to make `memchr2_raw` slim and fast, we use a function pointer that updates +// itself to the correct implementation on the first call. This reduces binary size. +// It would also reduce branches if we had >2 implementations (a jump still needs to be predicted). +// NOTE that this ONLY works if Control Flow Guard is disabled on Windows. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +static mut MEMCHR2_DISPATCH: unsafe fn( + needle1: u8, + needle2: u8, + beg: *const u8, + end: *const u8, +) -> *const u8 = memchr2_dispatch; + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +unsafe fn memchr2_dispatch(needle1: u8, needle2: u8, beg: *const u8, end: *const u8) -> *const u8 { + let func = if is_x86_feature_detected!("avx2") { memchr2_avx2 } else { memchr2_fallback }; + unsafe { MEMCHR2_DISPATCH = func }; + unsafe { func(needle1, needle2, beg, end) } +} + +// FWIW, I found that adding support for AVX512 was not useful at the time, +// as it only marginally improved file load performance by <5%. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "avx2")] +unsafe fn memchr2_avx2(needle1: u8, needle2: u8, mut beg: *const u8, end: *const u8) -> *const u8 { + unsafe { + #[cfg(target_arch = "x86")] + use std::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use std::arch::x86_64::*; + + let n1 = _mm256_set1_epi8(needle1 as i8); + let n2 = _mm256_set1_epi8(needle2 as i8); + let mut remaining = distance(end, beg); + + while remaining >= 32 { + let v = _mm256_loadu_si256(beg as *const _); + let a = _mm256_cmpeq_epi8(v, n1); + let b = _mm256_cmpeq_epi8(v, n2); + let c = _mm256_or_si256(a, b); + let m = _mm256_movemask_epi8(c) as u32; + + if m != 0 { + return beg.add(m.trailing_zeros() as usize); + } + + beg = beg.add(32); + remaining -= 32; + } + + memchr2_fallback(needle1, needle2, beg, end) + } +} + +#[cfg(target_arch = "aarch64")] +unsafe fn memchr2_neon(needle1: u8, needle2: u8, mut beg: *const u8, end: *const u8) -> *const u8 { + unsafe { + use std::arch::aarch64::*; + + if distance(end, beg) >= 16 { + let n1 = vdupq_n_u8(needle1); + let n2 = vdupq_n_u8(needle2); + + loop { + let v = vld1q_u8(beg as *const _); + let a = vceqq_u8(v, n1); + let b = vceqq_u8(v, n2); + let c = vorrq_u8(a, b); + + // https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon + let m = vreinterpretq_u16_u8(c); + let m = vshrn_n_u16(m, 4); + let m = vreinterpret_u64_u8(m); + let m = vget_lane_u64(m, 0); + + if m != 0 { + return beg.add(m.trailing_zeros() as usize >> 2); + } + + beg = beg.add(16); + if distance(end, beg) < 16 { + break; + } + } + } + + memchr2_fallback(needle1, needle2, beg, end) + } +} + +#[cfg(test)] +mod tests { + use std::slice; + + use super::*; + use crate::sys; + + #[test] + fn test_empty() { + assert_eq!(memchr2(b'a', b'b', b"", 0), 0); + } + + #[test] + fn test_basic() { + let haystack = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + let haystack = &haystack[..43]; + + assert_eq!(memchr2(b'a', b'z', haystack, 0), 0); + assert_eq!(memchr2(b'p', b'q', haystack, 0), 15); + assert_eq!(memchr2(b'Q', b'Z', haystack, 0), 42); + assert_eq!(memchr2(b'0', b'9', haystack, 0), haystack.len()); + } + + // Test that it doesn't match before/after the start offset respectively. + #[test] + fn test_with_offset() { + let haystack = b"abcdefghabcdefghabcdefghabcdefghabcdefgh"; + + assert_eq!(memchr2(b'a', b'b', haystack, 0), 0); + assert_eq!(memchr2(b'a', b'b', haystack, 1), 1); + assert_eq!(memchr2(b'a', b'b', haystack, 2), 8); + assert_eq!(memchr2(b'a', b'b', haystack, 9), 9); + assert_eq!(memchr2(b'a', b'b', haystack, 16), 16); + assert_eq!(memchr2(b'a', b'b', haystack, 41), 40); + } + + // Test memory access safety at page boundaries. + // The test is a success if it doesn't segfault. + #[test] + fn test_page_boundary() { + let page = unsafe { + let page_size = 4096; + + // 3 pages: uncommitted, committed, uncommitted + let ptr = sys::virtual_reserve(page_size * 3).unwrap(); + sys::virtual_commit(ptr.add(page_size), page_size).unwrap(); + slice::from_raw_parts_mut(ptr.add(page_size).as_ptr(), page_size) + }; + + page.fill(b'a'); + + // Test if it seeks beyond the page boundary. + assert_eq!(memchr2(b'\0', b'\0', &page[page.len() - 40..], 0), 40); + // Test if it seeks before the page boundary for the masked/partial load. + assert_eq!(memchr2(b'\0', b'\0', &page[..10], 0), 10); + } +} diff --git a/pkgs/edit/src/simd/memrchr2.rs b/pkgs/edit/src/simd/memrchr2.rs new file mode 100644 index 0000000..26a5a55 --- /dev/null +++ b/pkgs/edit/src/simd/memrchr2.rs @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! `memchr`, but with two needles. + +use std::ptr; + +use super::distance; + +/// `memchr`, but with two needles. +/// +/// If no needle is found, 0 is returned. +/// Unlike `memchr2` (or `memrchr`), an offset PAST the hit is returned. +/// This is because this function is primarily used for +/// `ucd::newlines_backward`, which needs exactly that. +pub fn memrchr2(needle1: u8, needle2: u8, haystack: &[u8], offset: usize) -> Option { + unsafe { + let beg = haystack.as_ptr(); + let it = beg.add(offset.min(haystack.len())); + let it = memrchr2_raw(needle1, needle2, beg, it); + if it.is_null() { None } else { Some(distance(it, beg)) } + } +} + +unsafe fn memrchr2_raw(needle1: u8, needle2: u8, beg: *const u8, end: *const u8) -> *const u8 { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + return unsafe { MEMRCHR2_DISPATCH(needle1, needle2, beg, end) }; + + #[cfg(target_arch = "aarch64")] + return unsafe { memrchr2_neon(needle1, needle2, beg, end) }; + + #[allow(unreachable_code)] + return unsafe { memrchr2_fallback(needle1, needle2, beg, end) }; +} + +unsafe fn memrchr2_fallback( + needle1: u8, + needle2: u8, + beg: *const u8, + mut end: *const u8, +) -> *const u8 { + unsafe { + while !ptr::eq(end, beg) { + end = end.sub(1); + let ch = *end; + if ch == needle1 || needle2 == ch { + return end; + } + } + ptr::null() + } +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +static mut MEMRCHR2_DISPATCH: unsafe fn( + needle1: u8, + needle2: u8, + beg: *const u8, + end: *const u8, +) -> *const u8 = memrchr2_dispatch; + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +unsafe fn memrchr2_dispatch(needle1: u8, needle2: u8, beg: *const u8, end: *const u8) -> *const u8 { + let func = if is_x86_feature_detected!("avx2") { memrchr2_avx2 } else { memrchr2_fallback }; + unsafe { MEMRCHR2_DISPATCH = func }; + unsafe { func(needle1, needle2, beg, end) } +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "avx2")] +unsafe fn memrchr2_avx2(needle1: u8, needle2: u8, beg: *const u8, mut end: *const u8) -> *const u8 { + unsafe { + #[cfg(target_arch = "x86")] + use std::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use std::arch::x86_64::*; + + if distance(end, beg) >= 32 { + let n1 = _mm256_set1_epi8(needle1 as i8); + let n2 = _mm256_set1_epi8(needle2 as i8); + + loop { + end = end.sub(32); + + let v = _mm256_loadu_si256(end as *const _); + let a = _mm256_cmpeq_epi8(v, n1); + let b = _mm256_cmpeq_epi8(v, n2); + let c = _mm256_or_si256(a, b); + let m = _mm256_movemask_epi8(c) as u32; + + if m != 0 { + return end.add(31 - m.leading_zeros() as usize); + } + + if distance(end, beg) < 32 { + break; + } + } + } + + memrchr2_fallback(needle1, needle2, beg, end) + } +} + +#[cfg(target_arch = "aarch64")] +unsafe fn memrchr2_neon(needle1: u8, needle2: u8, beg: *const u8, mut end: *const u8) -> *const u8 { + unsafe { + use std::arch::aarch64::*; + + if distance(end, beg) >= 16 { + let n1 = vdupq_n_u8(needle1); + let n2 = vdupq_n_u8(needle2); + + loop { + end = end.sub(16); + + let v = vld1q_u8(end as *const _); + let a = vceqq_u8(v, n1); + let b = vceqq_u8(v, n2); + let c = vorrq_u8(a, b); + + // https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon + let m = vreinterpretq_u16_u8(c); + let m = vshrn_n_u16(m, 4); + let m = vreinterpret_u64_u8(m); + let m = vget_lane_u64(m, 0); + + if m != 0 { + return end.add(15 - (m.leading_zeros() as usize >> 2)); + } + + if distance(end, beg) < 16 { + break; + } + } + } + + memrchr2_fallback(needle1, needle2, beg, end) + } +} + +#[cfg(test)] +mod tests { + use std::slice; + + use super::*; + use crate::sys; + + #[test] + fn test_empty() { + assert_eq!(memrchr2(b'a', b'b', b"", 0), None); + } + + #[test] + fn test_basic() { + let haystack = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + let haystack = &haystack[..43]; + + assert_eq!(memrchr2(b'Q', b'P', haystack, 43), Some(42)); + assert_eq!(memrchr2(b'p', b'o', haystack, 43), Some(15)); + assert_eq!(memrchr2(b'a', b'b', haystack, 43), Some(1)); + assert_eq!(memrchr2(b'0', b'9', haystack, 43), None); + } + + // Test that it doesn't match before/after the start offset respectively. + #[test] + fn test_with_offset() { + let haystack = b"abcdefghabcdefghabcdefghabcdefghabcdefgh"; + + assert_eq!(memrchr2(b'h', b'g', haystack, 40), Some(39)); + assert_eq!(memrchr2(b'h', b'g', haystack, 39), Some(38)); + assert_eq!(memrchr2(b'a', b'b', haystack, 9), Some(8)); + assert_eq!(memrchr2(b'a', b'b', haystack, 1), Some(0)); + assert_eq!(memrchr2(b'a', b'b', haystack, 0), None); + } + + // Test memory access safety at page boundaries. + // The test is a success if it doesn't segfault. + #[test] + fn test_page_boundary() { + let page = unsafe { + let page_size = 4096; + + // 3 pages: uncommitted, committed, uncommitted + let ptr = sys::virtual_reserve(page_size * 3).unwrap(); + sys::virtual_commit(ptr.add(page_size), page_size).unwrap(); + slice::from_raw_parts_mut(ptr.add(page_size).as_ptr(), page_size) + }; + + page.fill(b'a'); + + // Same as above, but for memrchr2 (hence reversed). + assert_eq!(memrchr2(b'\0', b'\0', &page[page.len() - 10..], 10), None); + assert_eq!(memrchr2(b'\0', b'\0', &page[..40], 40), None); + } +} diff --git a/pkgs/edit/src/simd/memset.rs b/pkgs/edit/src/simd/memset.rs new file mode 100644 index 0000000..028b66b --- /dev/null +++ b/pkgs/edit/src/simd/memset.rs @@ -0,0 +1,345 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! `memchr` for arbitrary sizes (1/2/4/8 bytes). +//! +//! Clang calls the C `memset` function only for byte-sized types (or 0 fills). +//! We however need to fill other types as well. For that, clang generates +//! SIMD loops under higher optimization levels. With `-Os` however, it only +//! generates a trivial loop which is too slow for our needs. +//! +//! This implementation uses SWAR to only have a single implementation for all +//! 4 sizes: By duplicating smaller types into a larger `u64` register we can +//! treat all sizes as if they were `u64`. The only thing we need to take care +//! of is the tail end of the array, which needs to write 0-7 additional bytes. + +use std::mem; + +use super::distance; + +/// A marker trait for types that are safe to `memset`. +/// +/// # Safety +/// +/// Just like with C's `memset`, bad things happen +/// if you use this with non-trivial types. +pub unsafe trait MemsetSafe: Copy {} + +unsafe impl MemsetSafe for u8 {} +unsafe impl MemsetSafe for u16 {} +unsafe impl MemsetSafe for u32 {} +unsafe impl MemsetSafe for u64 {} +unsafe impl MemsetSafe for usize {} + +unsafe impl MemsetSafe for i8 {} +unsafe impl MemsetSafe for i16 {} +unsafe impl MemsetSafe for i32 {} +unsafe impl MemsetSafe for i64 {} +unsafe impl MemsetSafe for isize {} + +/// Fills a slice with the given value. +#[inline] +pub fn memset(dst: &mut [T], val: T) { + unsafe { + match mem::size_of::() { + 1 => { + // LLVM will compile this to a call to `memset`, + // which hopefully should be better optimized than my code. + let beg = dst.as_mut_ptr(); + let val = mem::transmute_copy::<_, u8>(&val); + beg.write_bytes(val, dst.len()); + } + 2 => { + let beg = dst.as_mut_ptr(); + let end = beg.add(dst.len()); + let val = mem::transmute_copy::<_, u16>(&val); + memset_raw(beg as *mut u8, end as *mut u8, val as u64 * 0x0001000100010001); + } + 4 => { + let beg = dst.as_mut_ptr(); + let end = beg.add(dst.len()); + let val = mem::transmute_copy::<_, u32>(&val); + memset_raw(beg as *mut u8, end as *mut u8, val as u64 * 0x0000000100000001); + } + 8 => { + let beg = dst.as_mut_ptr(); + let end = beg.add(dst.len()); + let val = mem::transmute_copy::<_, u64>(&val); + memset_raw(beg as *mut u8, end as *mut u8, val); + } + _ => unreachable!(), + } + } +} + +#[inline] +fn memset_raw(beg: *mut u8, end: *mut u8, val: u64) { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + return unsafe { MEMSET_DISPATCH(beg, end, val) }; + + #[cfg(target_arch = "aarch64")] + return unsafe { memset_neon(beg, end, val) }; +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +static mut MEMSET_DISPATCH: unsafe fn(beg: *mut u8, end: *mut u8, val: u64) = memset_dispatch; + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +fn memset_dispatch(beg: *mut u8, end: *mut u8, val: u64) { + let func = if is_x86_feature_detected!("avx2") { memset_avx2 } else { memset_sse2 }; + unsafe { MEMSET_DISPATCH = func }; + unsafe { func(beg, end, val) } +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "sse2")] +unsafe fn memset_sse2(mut beg: *mut u8, end: *mut u8, val: u64) { + unsafe { + #[cfg(target_arch = "x86")] + use std::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use std::arch::x86_64::*; + + let mut remaining = distance(end, beg); + + if remaining >= 16 { + let fill = _mm_set1_epi64x(val as i64); + + while remaining >= 32 { + _mm_storeu_si128(beg as *mut _, fill); + _mm_storeu_si128(beg.add(16) as *mut _, fill); + + beg = beg.add(32); + remaining -= 32; + } + + if remaining >= 16 { + // 16-31 bytes remaining + _mm_storeu_si128(beg as *mut _, fill); + _mm_storeu_si128(end.sub(16) as *mut _, fill); + return; + } + } + + if remaining >= 8 { + // 8-15 bytes remaining + (beg as *mut u64).write_unaligned(val); + (end.sub(8) as *mut u64).write_unaligned(val); + } else if remaining >= 4 { + // 4-7 bytes remaining + (beg as *mut u32).write_unaligned(val as u32); + (end.sub(4) as *mut u32).write_unaligned(val as u32); + } else if remaining >= 2 { + // 2-3 bytes remaining + (beg as *mut u16).write_unaligned(val as u16); + (end.sub(2) as *mut u16).write_unaligned(val as u16); + } else if remaining >= 1 { + // 1 byte remaining + beg.write(val as u8); + } + } +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "avx2")] +fn memset_avx2(mut beg: *mut u8, end: *mut u8, val: u64) { + unsafe { + #[cfg(target_arch = "x86")] + use std::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use std::arch::x86_64::*; + use std::hint::black_box; + + let mut remaining = distance(end, beg); + + if remaining >= 128 { + let fill = _mm256_set1_epi64x(val as i64); + + loop { + _mm256_storeu_si256(beg as *mut _, fill); + _mm256_storeu_si256(beg.add(32) as *mut _, fill); + _mm256_storeu_si256(beg.add(64) as *mut _, fill); + _mm256_storeu_si256(beg.add(96) as *mut _, fill); + + beg = beg.add(128); + remaining -= 128; + if remaining < 128 { + break; + } + } + } + + if remaining >= 16 { + let fill = _mm_set1_epi64x(val as i64); + + loop { + // LLVM is _very_ eager to unroll loops. In the absence of an unroll attribute, black_box does the job. + // Note that this must not be applied to the intrinsic parameters, as they're otherwise misoptimized. + #[allow(clippy::unit_arg)] + black_box(_mm_storeu_si128(beg as *mut _, fill)); + + beg = beg.add(16); + remaining -= 16; + if remaining < 16 { + break; + } + } + } + + // `remaining` is between 0 and 15 at this point. + // By overlapping the stores we can write all of them in at most 2 stores. This approach + // can be seen in various libraries, such as wyhash which uses it for loading data in `wyr3`. + if remaining >= 8 { + // 8-15 bytes + (beg as *mut u64).write_unaligned(val); + (end.sub(8) as *mut u64).write_unaligned(val); + } else if remaining >= 4 { + // 4-7 bytes + (beg as *mut u32).write_unaligned(val as u32); + (end.sub(4) as *mut u32).write_unaligned(val as u32); + } else if remaining >= 2 { + // 2-3 bytes + (beg as *mut u16).write_unaligned(val as u16); + (end.sub(2) as *mut u16).write_unaligned(val as u16); + } else if remaining >= 1 { + // 1 byte + beg.write(val as u8); + } + } +} + +#[cfg(target_arch = "aarch64")] +unsafe fn memset_neon(mut beg: *mut u8, end: *mut u8, val: u64) { + unsafe { + use std::arch::aarch64::*; + let mut remaining = distance(end, beg); + + if remaining >= 32 { + let fill = vdupq_n_u64(val); + + loop { + // Compiles to a single `stp` instruction. + vst1q_u64(beg as *mut _, fill); + vst1q_u64(beg.add(16) as *mut _, fill); + + beg = beg.add(32); + remaining -= 32; + if remaining < 32 { + break; + } + } + } + + if remaining >= 16 { + // 16-31 bytes remaining + let fill = vdupq_n_u64(val); + vst1q_u64(beg as *mut _, fill); + vst1q_u64(end.sub(16) as *mut _, fill); + } else if remaining >= 8 { + // 8-15 bytes remaining + (beg as *mut u64).write_unaligned(val); + (end.sub(8) as *mut u64).write_unaligned(val); + } else if remaining >= 4 { + // 4-7 bytes remaining + (beg as *mut u32).write_unaligned(val as u32); + (end.sub(4) as *mut u32).write_unaligned(val as u32); + } else if remaining >= 2 { + // 2-3 bytes remaining + (beg as *mut u16).write_unaligned(val as u16); + (end.sub(2) as *mut u16).write_unaligned(val as u16); + } else if remaining >= 1 { + // 1 byte remaining + beg.write(val as u8); + } + } +} + +#[cfg(test)] +mod tests { + use std::fmt; + use std::ops::Not; + + use super::*; + + fn check_memset(val: T, len: usize) + where + T: MemsetSafe + Not + PartialEq + fmt::Debug, + { + let mut buf = vec![!val; len]; + memset(&mut buf, val); + assert!(buf.iter().all(|&x| x == val)); + } + + #[test] + fn test_memset_empty() { + check_memset(0u8, 0); + check_memset(0u16, 0); + check_memset(0u32, 0); + check_memset(0u64, 0); + } + + #[test] + fn test_memset_single() { + check_memset(0u8, 1); + check_memset(0xFFu8, 1); + check_memset(0xABu16, 1); + check_memset(0x12345678u32, 1); + check_memset(0xDEADBEEFu64, 1); + } + + #[test] + fn test_memset_small() { + for &len in &[2, 3, 4, 5, 7, 8, 9] { + check_memset(0xAAu8, len); + check_memset(0xBEEFu16, len); + check_memset(0xCAFEBABEu32, len); + check_memset(0x1234567890ABCDEFu64, len); + } + } + + #[test] + fn test_memset_large() { + check_memset(0u8, 1000); + check_memset(0xFFu8, 1024); + check_memset(0xBEEFu16, 512); + check_memset(0xCAFEBABEu32, 256); + check_memset(0x1234567890ABCDEFu64, 128); + } + + #[test] + fn test_memset_various_values() { + check_memset(0u8, 17); + check_memset(0x7Fu8, 17); + check_memset(0x8001u16, 17); + check_memset(0xFFFFFFFFu32, 17); + check_memset(0x8000000000000001u64, 17); + } + + #[test] + fn test_memset_signed_types() { + check_memset(-1i8, 8); + check_memset(-2i16, 8); + check_memset(-3i32, 8); + check_memset(-4i64, 8); + check_memset(-5isize, 8); + } + + #[test] + fn test_memset_usize_isize() { + check_memset(0usize, 4); + check_memset(usize::MAX, 4); + check_memset(0isize, 4); + check_memset(isize::MIN, 4); + } + + #[test] + fn test_memset_alignment() { + // Check that memset works for slices not aligned to 8 bytes + let mut buf = [0u8; 15]; + for offset in 0..8 { + let slice = &mut buf[offset..(offset + 7)]; + memset(slice, 0x5A); + assert!(slice.iter().all(|&x| x == 0x5A)); + } + } +} diff --git a/pkgs/edit/src/simd/mod.rs b/pkgs/edit/src/simd/mod.rs new file mode 100644 index 0000000..79414d8 --- /dev/null +++ b/pkgs/edit/src/simd/mod.rs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Provides various high-throughput utilities. + +mod memchr2; +mod memrchr2; +mod memset; + +pub use memchr2::*; +pub use memrchr2::*; +pub use memset::*; + +// Can be replaced with `sub_ptr` once it's stabilized. +#[inline(always)] +unsafe fn distance(hi: *const T, lo: *const T) -> usize { + unsafe { usize::try_from(hi.offset_from(lo)).unwrap_unchecked() } +} diff --git a/pkgs/edit/src/sys/mod.rs b/pkgs/edit/src/sys/mod.rs new file mode 100644 index 0000000..f5305c0 --- /dev/null +++ b/pkgs/edit/src/sys/mod.rs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Platform abstractions. + +use std::fs::File; +use std::path::Path; + +use crate::apperr; + +#[cfg(unix)] +mod unix; +#[cfg(windows)] +mod windows; + +#[cfg(not(windows))] +pub use std::fs::canonicalize; + +#[cfg(unix)] +pub use unix::*; +#[cfg(windows)] +pub use windows::*; + +pub fn file_id_at(path: &Path) -> apperr::Result { + let file = File::open(path)?; + let file_id = file_id(&file)?; + Ok(file_id) +} diff --git a/pkgs/edit/src/sys/unix.rs b/pkgs/edit/src/sys/unix.rs new file mode 100644 index 0000000..be80c59 --- /dev/null +++ b/pkgs/edit/src/sys/unix.rs @@ -0,0 +1,569 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Unix-specific platform code. +//! +//! Read the `windows` module for reference. +//! TODO: This reminds me that the sys API should probably be a trait. + +use std::ffi::{CStr, c_int, c_void}; +use std::fs::{self, File}; +use std::mem::{self, MaybeUninit}; +use std::os::fd::{AsRawFd as _, FromRawFd as _}; +use std::ptr::{self, NonNull, null, null_mut}; +use std::{thread, time}; + +use crate::arena::{Arena, ArenaString, scratch_arena}; +use crate::helpers::*; +use crate::{apperr, arena_format}; + +struct State { + stdin: libc::c_int, + stdin_flags: libc::c_int, + stdout: libc::c_int, + stdout_initial_termios: Option, + inject_resize: bool, + // Buffer for incomplete UTF-8 sequences (max 4 bytes needed) + utf8_buf: [u8; 4], + utf8_len: usize, +} + +static mut STATE: State = State { + stdin: libc::STDIN_FILENO, + stdin_flags: 0, + stdout: libc::STDOUT_FILENO, + stdout_initial_termios: None, + inject_resize: false, + utf8_buf: [0; 4], + utf8_len: 0, +}; + +extern "C" fn sigwinch_handler(_: libc::c_int) { + unsafe { + STATE.inject_resize = true; + } +} + +pub fn init() -> apperr::Result { + unsafe { + // Reopen stdin if it's redirected (= piped input). + if libc::isatty(STATE.stdin) == 0 { + STATE.stdin = check_int_return(libc::open(c"/dev/tty".as_ptr(), libc::O_RDONLY))?; + } + + // Store the stdin flags so we can more easily toggle `O_NONBLOCK` later on. + STATE.stdin_flags = check_int_return(libc::fcntl(STATE.stdin, libc::F_GETFL))?; + + Ok(Deinit) + } +} + +pub struct Deinit; + +impl Drop for Deinit { + fn drop(&mut self) { + unsafe { + #[allow(static_mut_refs)] + if let Some(termios) = STATE.stdout_initial_termios.take() { + // Restore the original terminal modes. + libc::tcsetattr(STATE.stdout, libc::TCSANOW, &termios); + } + } + } +} + +pub fn switch_modes() -> apperr::Result<()> { + unsafe { + // Set STATE.inject_resize to true whenever we get a SIGWINCH. + let mut sigwinch_action: libc::sigaction = mem::zeroed(); + sigwinch_action.sa_sigaction = sigwinch_handler as libc::sighandler_t; + check_int_return(libc::sigaction(libc::SIGWINCH, &sigwinch_action, null_mut()))?; + + // Get the original terminal modes so we can disable raw mode on exit. + let mut termios = MaybeUninit::::uninit(); + check_int_return(libc::tcgetattr(STATE.stdin, termios.as_mut_ptr()))?; + let mut termios = termios.assume_init(); + STATE.stdout_initial_termios = Some(termios); + + termios.c_iflag &= !( + // When neither IGNBRK... + libc::IGNBRK + // ...nor BRKINT are set, a BREAK reads as a null byte ('\0'), ... + | libc::BRKINT + // ...except when PARMRK is set, in which case it reads as the sequence \377 \0 \0. + | libc::PARMRK + // Disable input parity checking. + | libc::INPCK + // Disable stripping of eighth bit. + | libc::ISTRIP + // Disable mapping of NL to CR on input. + | libc::INLCR + // Disable ignoring CR on input. + | libc::IGNCR + // Disable mapping of CR to NL on input. + | libc::ICRNL + // Disable software flow control. + | libc::IXON + ); + // Disable output processing. + termios.c_oflag &= !libc::OPOST; + termios.c_cflag &= !( + // Reset character size mask. + libc::CSIZE + // Disable parity generation. + | libc::PARENB + ); + // Set character size back to 8 bits. + termios.c_cflag |= libc::CS8; + termios.c_lflag &= !( + // Disable signal generation (SIGINT, SIGTSTP, SIGQUIT). + libc::ISIG + // Disable canonical mode (line buffering). + | libc::ICANON + // Disable echoing of input characters. + | libc::ECHO + // Disable echoing of NL. + | libc::ECHONL + // Disable extended input processing (e.g. Ctrl-V). + | libc::IEXTEN + ); + + // Set the terminal to raw mode. + termios.c_lflag &= !(libc::ICANON | libc::ECHO); + check_int_return(libc::tcsetattr(STATE.stdin, libc::TCSANOW, &termios))?; + + Ok(()) + } +} + +pub fn inject_window_size_into_stdin() { + unsafe { + STATE.inject_resize = true; + } +} + +fn get_window_size() -> (u16, u16) { + let mut winsz: libc::winsize = unsafe { mem::zeroed() }; + + for attempt in 1.. { + let ret = unsafe { libc::ioctl(STATE.stdout, libc::TIOCGWINSZ, &raw mut winsz) }; + if ret == -1 || (winsz.ws_col != 0 && winsz.ws_row != 0) { + break; + } + + if attempt == 10 { + winsz.ws_col = 80; + winsz.ws_row = 24; + break; + } + + // Some terminals are bad emulators and don't report TIOCGWINSZ immediately. + thread::sleep(time::Duration::from_millis(10 * attempt)); + } + + (winsz.ws_col, winsz.ws_row) +} + +/// Reads from stdin. +/// +/// Returns `None` if there was an error reading from stdin. +/// Returns `Some("")` if the given timeout was reached. +/// Otherwise, it returns the read, non-empty string. +pub fn read_stdin(arena: &Arena, mut timeout: time::Duration) -> Option> { + unsafe { + if STATE.inject_resize { + timeout = time::Duration::ZERO; + } + + let read_poll = timeout != time::Duration::MAX; + let mut buf = Vec::new_in(arena); + + // We don't know if the input is valid UTF8, so we first use a Vec and then + // later turn it into UTF8 using `from_utf8_lossy_owned`. + // It is important that we allocate the buffer with an explicit capacity, + // because we later use `spare_capacity_mut` to access it. + buf.reserve(4 * KIBI); + + // We got some leftover broken UTF8 from a previous read? Prepend it. + if STATE.utf8_len != 0 { + STATE.utf8_len = 0; + buf.extend_from_slice(&STATE.utf8_buf[..STATE.utf8_len]); + } + + loop { + if timeout != time::Duration::MAX { + let beg = time::Instant::now(); + + let mut pollfd = libc::pollfd { fd: STATE.stdin, events: libc::POLLIN, revents: 0 }; + let ts = libc::timespec { + tv_sec: timeout.as_secs() as libc::time_t, + tv_nsec: timeout.subsec_nanos() as libc::c_long, + }; + let ret = libc::ppoll(&mut pollfd, 1, &ts, null()); + if ret < 0 { + return None; // Error? Let's assume it's an EOF. + } + if ret == 0 { + break; // Timeout? We can stop reading. + } + + timeout = timeout.saturating_sub(beg.elapsed()); + }; + + // If we're asked for a non-blocking read we need + // to manipulate `O_NONBLOCK` and vice versa. + set_tty_nonblocking(read_poll); + + // Read from stdin. + let spare = buf.spare_capacity_mut(); + let ret = libc::read(STATE.stdin, spare.as_mut_ptr() as *mut _, spare.len()); + if ret > 0 { + buf.set_len(buf.len() + ret as usize); + break; + } + if ret == 0 { + return None; // EOF + } + if ret < 0 { + match *libc::__errno_location() { + libc::EINTR if STATE.inject_resize => break, + libc::EAGAIN if timeout == time::Duration::ZERO => break, + libc::EINTR | libc::EAGAIN => {} + _ => return None, + } + } + } + + if !buf.is_empty() { + // We only need to check the last 3 bytes for UTF-8 continuation bytes, + // because we should be able to assume that any 4 byte sequence is complete. + let lim = buf.len().saturating_sub(3); + let mut off = buf.len() - 1; + + // Find the start of the last potentially incomplete UTF-8 sequence. + while off > lim && buf[off] & 0b1100_0000 == 0b1000_0000 { + off -= 1; + } + + let seq_len = match buf[off] { + b if b & 0b1000_0000 == 0 => 1, + b if b & 0b1110_0000 == 0b1100_0000 => 2, + b if b & 0b1111_0000 == 0b1110_0000 => 3, + b if b & 0b1111_1000 == 0b1111_0000 => 4, + // If the lead byte we found isn't actually one, we don't cache it. + // `from_utf8_lossy_owned` will replace it with U+FFFD. + _ => 0, + }; + + // Cache incomplete sequence if any. + if off + seq_len > buf.len() { + STATE.utf8_len = buf.len() - off; + STATE.utf8_buf[..STATE.utf8_len].copy_from_slice(&buf[off..]); + buf.truncate(off); + } + } + + let mut result = ArenaString::from_utf8_lossy_owned(buf); + + // We received a SIGWINCH? Add a fake window size sequence for our input parser. + // I prepend it so that on startup, the TUI system gets first initialized with a size. + if STATE.inject_resize { + STATE.inject_resize = false; + let (w, h) = get_window_size(); + if w > 0 && h > 0 { + let scratch = scratch_arena(Some(arena)); + let seq = arena_format!(&scratch, "\x1b[8;{h};{w}t"); + result.replace_range(0..0, &seq); + } + } + + result.shrink_to_fit(); + Some(result) + } +} + +pub fn write_stdout(text: &str) { + if text.is_empty() { + return; + } + + // If we don't set the TTY to blocking mode, + // the write will potentially fail with EAGAIN. + set_tty_nonblocking(false); + + let buf = text.as_bytes(); + let mut written = 0; + + while written < buf.len() { + let w = &buf[written..]; + let w = &buf[..w.len().min(GIBI)]; + let n = unsafe { libc::write(STATE.stdout, w.as_ptr() as *const _, w.len()) }; + + if n >= 0 { + written += n as usize; + continue; + } + + let err = unsafe { *libc::__errno_location() }; + if err != libc::EINTR { + return; + } + } +} + +/// Sets/Resets `O_NONBLOCK` on the TTY handle. +/// +/// Note that setting this flag applies to both stdin and stdout, because the +/// TTY is a bidirectional device and both handles refer to the same thing. +fn set_tty_nonblocking(nonblock: bool) { + unsafe { + let is_nonblock = (STATE.stdin_flags & libc::O_NONBLOCK) != 0; + if is_nonblock != nonblock { + STATE.stdin_flags ^= libc::O_NONBLOCK; + let _ = libc::fcntl(STATE.stdin, libc::F_SETFL, STATE.stdin_flags); + } + } +} + +pub fn open_stdin_if_redirected() -> Option { + unsafe { + // Did we reopen stdin during `init()`? + if STATE.stdin != libc::STDIN_FILENO { + Some(File::from_raw_fd(libc::STDIN_FILENO)) + } else { + None + } + } +} + +#[derive(Clone, PartialEq, Eq)] +pub struct FileId { + st_dev: libc::dev_t, + st_ino: libc::ino_t, +} + +/// Returns a unique identifier for the given file. +pub fn file_id(file: &File) -> apperr::Result { + unsafe { + let mut stat = MaybeUninit::::uninit(); + check_int_return(libc::fstat(file.as_raw_fd(), stat.as_mut_ptr()))?; + let stat = stat.assume_init(); + Ok(FileId { st_dev: stat.st_dev, st_ino: stat.st_ino }) + } +} + +/// Reserves a virtual memory region of the given size. +/// To commit the memory, use `virtual_commit`. +/// To release the memory, use `virtual_release`. +/// +/// # Safety +/// +/// This function is unsafe because it uses raw pointers. +/// Don't forget to release the memory when you're done with it or you'll leak it. +pub unsafe fn virtual_reserve(size: usize) -> apperr::Result> { + unsafe { + let ptr = libc::mmap( + null_mut(), + size, + libc::PROT_NONE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ); + if ptr.is_null() || ptr::eq(ptr, libc::MAP_FAILED) { + Err(errno_to_apperr(libc::ENOMEM)) + } else { + Ok(NonNull::new_unchecked(ptr as *mut u8)) + } + } +} + +/// Releases a virtual memory region of the given size. +/// +/// # Safety +/// +/// This function is unsafe because it uses raw pointers. +/// Make sure to only pass pointers acquired from `virtual_reserve`. +pub unsafe fn virtual_release(base: NonNull, size: usize) { + unsafe { + libc::munmap(base.cast().as_ptr(), size); + } +} + +/// Commits a virtual memory region of the given size. +/// +/// # Safety +/// +/// This function is unsafe because it uses raw pointers. +/// Make sure to only pass pointers acquired from `virtual_reserve` +/// and to pass a size less than or equal to the size passed to `virtual_reserve`. +pub unsafe fn virtual_commit(base: NonNull, size: usize) -> apperr::Result<()> { + unsafe { + let status = libc::mprotect(base.cast().as_ptr(), size, libc::PROT_READ | libc::PROT_WRITE); + if status != 0 { Err(errno_to_apperr(libc::ENOMEM)) } else { Ok(()) } + } +} + +unsafe fn load_library(name: &CStr) -> apperr::Result> { + unsafe { + NonNull::new(libc::dlopen(name.as_ptr(), libc::RTLD_LAZY)) + .ok_or_else(|| errno_to_apperr(libc::ELIBACC)) + } +} + +/// Loads a function from a dynamic library. +/// +/// # Safety +/// +/// This function is highly unsafe as it requires you to know the exact type +/// of the function you're loading. No type checks whatsoever are performed. +// +// It'd be nice to constrain T to std::marker::FnPtr, but that's unstable. +pub unsafe fn get_proc_address(handle: NonNull, name: &CStr) -> apperr::Result { + unsafe { + let sym = libc::dlsym(handle.as_ptr(), name.as_ptr()); + if sym.is_null() { + Err(errno_to_apperr(libc::ELIBACC)) + } else { + Ok(mem::transmute_copy(&sym)) + } + } +} + +pub fn load_libicuuc() -> apperr::Result> { + unsafe { load_library(c"libicuuc.so") } +} + +pub fn load_libicui18n() -> apperr::Result> { + unsafe { load_library(c"libicui18n.so") } +} + +/// ICU, by default, adds the major version as a suffix to each exported symbol. +/// They also recommend to disable this for system-level installations (`runConfigureICU Linux --disable-renaming`), +/// but I found that many (most?) Linux distributions don't do this for some reason. +/// This function returns the suffix, if any. +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub fn icu_proc_suffix(arena: &Arena, handle: NonNull) -> ArenaString<'_> { + unsafe { + type T = *const c_void; + + let mut res = ArenaString::new_in(arena); + + // Check if the ICU library is using unversioned symbols. + // Return an empty suffix in that case. + if get_proc_address::(handle, c"u_errorName").is_ok() { + return res; + } + + // In the versions (63-76) and distributions (Arch/Debian) I tested, + // this symbol seems to be always present. This allows us to call `dladdr`. + // It's the `UCaseMap::~UCaseMap()` destructor which for some reason isn't + // in a namespace. Thank you ICU maintainers for this oversight. + let proc = match get_proc_address::(handle, c"_ZN8UCaseMapD1Ev") { + Ok(proc) => proc, + Err(_) => return res, + }; + + // `dladdr` is specific to GNU's libc unfortunately. + let mut info: libc::Dl_info = mem::zeroed(); + let ret = libc::dladdr(proc, &mut info); + if ret == 0 { + return res; + } + + // The library path is in `info.dli_fname`. + let path = match CStr::from_ptr(info.dli_fname).to_str() { + Ok(name) => name, + Err(_) => return res, + }; + + let path = match fs::read_link(path) { + Ok(path) => path, + Err(_) => path.into(), + }; + + // I'm going to assume it's something like "libicuuc.so.76.1". + let path = path.into_os_string(); + let path = path.to_string_lossy(); + let suffix_start = match path.rfind(".so.") { + Some(pos) => pos + 4, + None => return res, + }; + let version = &path[suffix_start..]; + let version_end = version.find('.').unwrap_or(version.len()); + let version = &version[..version_end]; + + res.push('_'); + res.push_str(version); + res + } +} + +pub fn add_icu_proc_suffix<'a, 'b, 'r>(arena: &'a Arena, name: &'b CStr, suffix: &str) -> &'r CStr +where + 'a: 'r, + 'b: 'r, +{ + if suffix.is_empty() { + name + } else { + // SAFETY: In this particualar case we know that the string + // is valid UTF-8, because it comes from icu.rs. + let name = unsafe { name.to_str().unwrap_unchecked() }; + + let mut res = ArenaString::new_in(arena); + res.reserve(name.len() + suffix.len() + 1); + res.push_str(name); + res.push_str(suffix); + res.push('\0'); + + let bytes: &'a [u8] = unsafe { mem::transmute(res.as_bytes()) }; + unsafe { CStr::from_bytes_with_nul_unchecked(bytes) } + } +} + +pub fn preferred_languages(arena: &Arena) -> Vec, &Arena> { + let mut locales = Vec::new_in(arena); + + for key in ["LANGUAGE", "LC_ALL", "LANG"] { + if let Ok(val) = std::env::var(key) { + locales.extend( + val.split(':').filter(|s| !s.is_empty()).map(|s| ArenaString::from_str(arena, s)), + ); + break; + } + } + + locales +} + +#[inline] +pub(crate) fn io_error_to_apperr(err: std::io::Error) -> apperr::Error { + errno_to_apperr(err.raw_os_error().unwrap_or(0)) +} + +pub fn apperr_format(f: &mut std::fmt::Formatter<'_>, code: u32) -> std::fmt::Result { + write!(f, "Error {code}")?; + + unsafe { + let ptr = libc::strerror(code as i32); + if !ptr.is_null() { + let msg = CStr::from_ptr(ptr).to_string_lossy(); + write!(f, ": {msg}")?; + } + } + + Ok(()) +} + +pub fn apperr_is_not_found(err: apperr::Error) -> bool { + err == errno_to_apperr(libc::ENOENT) +} + +const fn errno_to_apperr(no: c_int) -> apperr::Error { + apperr::Error::new_sys(if no < 0 { 0 } else { no as u32 }) +} + +fn check_int_return(ret: libc::c_int) -> apperr::Result { + if ret < 0 { Err(errno_to_apperr(unsafe { *libc::__errno_location() })) } else { Ok(ret) } +} diff --git a/pkgs/edit/src/sys/windows.rs b/pkgs/edit/src/sys/windows.rs new file mode 100644 index 0000000..952ca47 --- /dev/null +++ b/pkgs/edit/src/sys/windows.rs @@ -0,0 +1,680 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::ffi::{CStr, OsString, c_void}; +use std::fmt::Write as _; +use std::fs::{self, File}; +use std::mem::MaybeUninit; +use std::os::windows::io::{AsRawHandle as _, FromRawHandle}; +use std::path::{Path, PathBuf}; +use std::ptr::{self, NonNull, null, null_mut}; +use std::{mem, time}; + +use windows_sys::Win32::Storage::FileSystem; +use windows_sys::Win32::System::Diagnostics::Debug; +use windows_sys::Win32::System::{Console, IO, LibraryLoader, Memory, Threading}; +use windows_sys::Win32::{Foundation, Globalization}; +use windows_sys::w; + +use crate::apperr; +use crate::arena::{Arena, ArenaString, scratch_arena}; +use crate::helpers::*; + +type ReadConsoleInputExW = unsafe extern "system" fn( + h_console_input: Foundation::HANDLE, + lp_buffer: *mut Console::INPUT_RECORD, + n_length: u32, + lp_number_of_events_read: *mut u32, + w_flags: u16, +) -> Foundation::BOOL; + +unsafe extern "system" fn read_console_input_ex_placeholder( + _: Foundation::HANDLE, + _: *mut Console::INPUT_RECORD, + _: u32, + _: *mut u32, + _: u16, +) -> Foundation::BOOL { + panic!(); +} + +const CONSOLE_READ_NOWAIT: u16 = 0x0002; + +const INVALID_CONSOLE_MODE: u32 = u32::MAX; + +struct State { + read_console_input_ex: ReadConsoleInputExW, + stdin: Foundation::HANDLE, + stdout: Foundation::HANDLE, + stdin_cp_old: u32, + stdout_cp_old: u32, + stdin_mode_old: u32, + stdout_mode_old: u32, + leading_surrogate: u16, + inject_resize: bool, + wants_exit: bool, +} + +static mut STATE: State = State { + read_console_input_ex: read_console_input_ex_placeholder, + stdin: null_mut(), + stdout: null_mut(), + stdin_cp_old: 0, + stdout_cp_old: 0, + stdin_mode_old: INVALID_CONSOLE_MODE, + stdout_mode_old: INVALID_CONSOLE_MODE, + leading_surrogate: 0, + inject_resize: false, + wants_exit: false, +}; + +extern "system" fn console_ctrl_handler(_ctrl_type: u32) -> Foundation::BOOL { + unsafe { + STATE.wants_exit = true; + IO::CancelIoEx(STATE.stdin, null()); + } + 1 +} + +/// Initializes the platform-specific state. +pub fn init() -> apperr::Result { + unsafe { + // Get the stdin and stdout handles first, so that if this function fails, + // we at least got something to use for `write_stdout`. + STATE.stdin = Console::GetStdHandle(Console::STD_INPUT_HANDLE); + STATE.stdout = Console::GetStdHandle(Console::STD_OUTPUT_HANDLE); + + // Reopen stdin if it's redirected (= piped input). + if !ptr::eq(STATE.stdin, Foundation::INVALID_HANDLE_VALUE) + && matches!( + FileSystem::GetFileType(STATE.stdin), + FileSystem::FILE_TYPE_DISK | FileSystem::FILE_TYPE_PIPE + ) + { + STATE.stdin = FileSystem::CreateFileW( + w!("CONIN$"), + Foundation::GENERIC_READ | Foundation::GENERIC_WRITE, + FileSystem::FILE_SHARE_READ | FileSystem::FILE_SHARE_WRITE, + null_mut(), + FileSystem::OPEN_EXISTING, + 0, + null_mut(), + ); + } + + if ptr::eq(STATE.stdin, Foundation::INVALID_HANDLE_VALUE) + || ptr::eq(STATE.stdout, Foundation::INVALID_HANDLE_VALUE) + { + return Err(get_last_error()); + } + + unsafe fn load_read_func(module: *const u16) -> apperr::Result { + unsafe { get_module(module).and_then(|m| get_proc_address(m, c"ReadConsoleInputExW")) } + } + + // `kernel32.dll` doesn't exist on OneCore variants of Windows. + // NOTE: `kernelbase.dll` is NOT a stable API to rely on. In our case it's the best option though. + // + // This is written as two nested `match` statements so that we can return the error from the first + // `load_read_func` call if it fails. The kernel32.dll lookup may contain some valid information, + // while the kernelbase.dll lookup may not, since it's not a stable API. + STATE.read_console_input_ex = match load_read_func(w!("kernel32.dll")) { + Ok(func) => func, + Err(err) => match load_read_func(w!("kernelbase.dll")) { + Ok(func) => func, + Err(_) => return Err(err), + }, + }; + + Ok(Deinit) + } +} + +pub struct Deinit; + +impl Drop for Deinit { + fn drop(&mut self) { + unsafe { + if STATE.stdin_cp_old != 0 { + Console::SetConsoleCP(STATE.stdin_cp_old); + STATE.stdin_cp_old = 0; + } + if STATE.stdout_cp_old != 0 { + Console::SetConsoleOutputCP(STATE.stdout_cp_old); + STATE.stdout_cp_old = 0; + } + if STATE.stdin_mode_old != INVALID_CONSOLE_MODE { + Console::SetConsoleMode(STATE.stdin, STATE.stdin_mode_old); + STATE.stdin_mode_old = INVALID_CONSOLE_MODE; + } + if STATE.stdout_mode_old != INVALID_CONSOLE_MODE { + Console::SetConsoleMode(STATE.stdout, STATE.stdout_mode_old); + STATE.stdout_mode_old = INVALID_CONSOLE_MODE; + } + } + } +} + +/// Switches the terminal into raw mode, etc. +pub fn switch_modes() -> apperr::Result<()> { + unsafe { + check_bool_return(Console::SetConsoleCtrlHandler(Some(console_ctrl_handler), 1))?; + + STATE.stdin_cp_old = Console::GetConsoleCP(); + STATE.stdout_cp_old = Console::GetConsoleOutputCP(); + check_bool_return(Console::GetConsoleMode(STATE.stdin, &raw mut STATE.stdin_mode_old))?; + check_bool_return(Console::GetConsoleMode(STATE.stdout, &raw mut STATE.stdout_mode_old))?; + + check_bool_return(Console::SetConsoleCP(Globalization::CP_UTF8))?; + check_bool_return(Console::SetConsoleOutputCP(Globalization::CP_UTF8))?; + check_bool_return(Console::SetConsoleMode( + STATE.stdin, + Console::ENABLE_WINDOW_INPUT + | Console::ENABLE_EXTENDED_FLAGS + | Console::ENABLE_VIRTUAL_TERMINAL_INPUT, + ))?; + check_bool_return(Console::SetConsoleMode( + STATE.stdout, + Console::ENABLE_PROCESSED_OUTPUT + | Console::ENABLE_WRAP_AT_EOL_OUTPUT + | Console::ENABLE_VIRTUAL_TERMINAL_PROCESSING + | Console::DISABLE_NEWLINE_AUTO_RETURN, + ))?; + + Ok(()) + } +} + +/// During startup we need to get the window size from the terminal. +/// Because I didn't want to type a bunch of code, this function tells +/// [`read_stdin`] to inject a fake sequence, which gets picked up by +/// the input parser and provided to the TUI code. +pub fn inject_window_size_into_stdin() { + unsafe { + STATE.inject_resize = true; + } +} + +fn get_console_size() -> Option { + unsafe { + let mut info: Console::CONSOLE_SCREEN_BUFFER_INFOEX = mem::zeroed(); + info.cbSize = mem::size_of::() as u32; + if Console::GetConsoleScreenBufferInfoEx(STATE.stdout, &mut info) == 0 { + return None; + } + + let w = (info.srWindow.Right - info.srWindow.Left + 1).max(1) as CoordType; + let h = (info.srWindow.Bottom - info.srWindow.Top + 1).max(1) as CoordType; + Some(Size { width: w, height: h }) + } +} + +/// Reads from stdin. +/// +/// # Returns +/// +/// * `None` if there was an error reading from stdin. +/// * `Some("")` if the given timeout was reached. +/// * Otherwise, it returns the read, non-empty string. +pub fn read_stdin(arena: &Arena, mut timeout: time::Duration) -> Option> { + let scratch = scratch_arena(Some(arena)); + + // On startup we're asked to inject a window size so that the UI system can layout the elements. + // --> Inject a fake sequence for our input parser. + let mut resize_event = None; + if unsafe { STATE.inject_resize } { + unsafe { STATE.inject_resize = false }; + timeout = time::Duration::ZERO; + resize_event = get_console_size(); + } + + let read_poll = timeout != time::Duration::MAX; // there is a timeout -> don't block in read() + let input_buf = scratch.alloc_uninit_slice(4 * KIBI); + let mut input_buf_cap = input_buf.len(); + let utf16_buf = scratch.alloc_uninit_slice(4 * KIBI); + let mut utf16_buf_len = 0; + + // If there was a leftover leading surrogate from the last read, we prepend it to the buffer. + if unsafe { STATE.leading_surrogate } != 0 { + utf16_buf[0] = MaybeUninit::new(unsafe { STATE.leading_surrogate }); + utf16_buf_len = 1; + input_buf_cap -= 1; + unsafe { STATE.leading_surrogate = 0 }; + } + + // Read until there's either a timeout or we have something to process. + loop { + if timeout != time::Duration::MAX { + let beg = time::Instant::now(); + + match unsafe { Threading::WaitForSingleObject(STATE.stdin, timeout.as_millis() as u32) } + { + // Ready to read? Continue with reading below. + Foundation::WAIT_OBJECT_0 => {} + // Timeout? Skip reading entirely. + Foundation::WAIT_TIMEOUT => break, + // Error? Tell the caller stdin is broken. + _ => return None, + } + + timeout = timeout.saturating_sub(beg.elapsed()); + } + + // Read from stdin. + let input = unsafe { + // If we had a `inject_resize`, we don't want to block indefinitely for other pending input on startup, + // but are still interested in any other pending input that may be waiting for us. + let flags = if read_poll { CONSOLE_READ_NOWAIT } else { 0 }; + let mut read = 0; + let ok = (STATE.read_console_input_ex)( + STATE.stdin, + input_buf[0].as_mut_ptr(), + input_buf_cap as u32, + &mut read, + flags, + ); + if ok == 0 || STATE.wants_exit { + return None; + } + input_buf[..read as usize].assume_init_ref() + }; + + // Convert Win32 input records into UTF16. + for inp in input { + match inp.EventType as u32 { + Console::KEY_EVENT => { + let event = unsafe { &inp.Event.KeyEvent }; + let ch = unsafe { event.uChar.UnicodeChar }; + if event.bKeyDown != 0 && ch != 0 { + utf16_buf[utf16_buf_len] = MaybeUninit::new(ch); + utf16_buf_len += 1; + } + } + Console::WINDOW_BUFFER_SIZE_EVENT => { + let event = unsafe { &inp.Event.WindowBufferSizeEvent }; + let w = event.dwSize.X as CoordType; + let h = event.dwSize.Y as CoordType; + // Windows is prone to sending broken/useless `WINDOW_BUFFER_SIZE_EVENT`s. + // E.g. starting conhost will emit 3 in a row. Skip rendering in that case. + if w > 0 && h > 0 { + resize_event = Some(Size { width: w, height: h }); + } + } + _ => {} + } + } + + if resize_event.is_some() || utf16_buf_len != 0 { + break; + } + } + + const RESIZE_EVENT_FMT_MAX_LEN: usize = 16; // "\x1b[8;65535;65535t" + let resize_event_len = if resize_event.is_some() { RESIZE_EVENT_FMT_MAX_LEN } else { 0 }; + // +1 to account for a potential `STATE.leading_surrogate`. + let utf8_max_len = (utf16_buf_len + 1) * 3; + let mut text = ArenaString::new_in(arena); + text.reserve(utf8_max_len + resize_event_len); + + // Now prepend our previously extracted resize event. + if let Some(resize_event) = resize_event { + // If I read xterm's documentation correctly, CSI 18 t reports the window size in characters. + // CSI 8 ; height ; width t is the response. Of course, we didn't send the request, + // but we can use this fake response to trigger the editor to resize itself. + _ = write!(text, "\x1b[8;{};{}t", resize_event.height, resize_event.width); + } + + // If the input ends with a lone lead surrogate, we need to remember it for the next read. + if utf16_buf_len > 0 { + unsafe { + let last_char = utf16_buf[utf16_buf_len - 1].assume_init(); + if (0xD800..0xDC00).contains(&last_char) { + STATE.leading_surrogate = last_char; + utf16_buf_len -= 1; + } + } + } + + // Convert the remaining input to UTF8, the sane encoding. + if utf16_buf_len > 0 { + unsafe { + let vec = text.as_mut_vec(); + let spare = vec.spare_capacity_mut(); + + let len = Globalization::WideCharToMultiByte( + Globalization::CP_UTF8, + 0, + utf16_buf[0].as_ptr(), + utf16_buf_len as i32, + spare.as_mut_ptr() as *mut _, + spare.len() as i32, + null(), + null_mut(), + ); + + if len > 0 { + vec.set_len(vec.len() + len as usize); + } + } + } + + text.shrink_to_fit(); + Some(text) +} + +/// Writes a string to stdout. +/// +/// Use this instead of `print!` or `println!` to avoid +/// the overhead of Rust's stdio handling. Don't need that. +pub fn write_stdout(text: &str) { + unsafe { + let mut offset = 0; + + while offset < text.len() { + let ptr = text.as_ptr().add(offset); + let write = (text.len() - offset).min(GIBI) as u32; + let mut written = 0; + let ok = FileSystem::WriteFile(STATE.stdout, ptr, write, &mut written, null_mut()); + offset += written as usize; + if ok == 0 || written == 0 { + break; + } + } + } +} + +/// Check if the stdin handle is redirected to a file, etc. +/// +/// # Returns +/// +/// * `Some(file)` if stdin is redirected. +/// * Otherwise, `None`. +pub fn open_stdin_if_redirected() -> Option { + unsafe { + let handle = Console::GetStdHandle(Console::STD_INPUT_HANDLE); + // Did we reopen stdin during `init()`? + if !std::ptr::eq(STATE.stdin, handle) { Some(File::from_raw_handle(handle)) } else { None } + } +} + +/// A unique identifier for a file. +#[derive(Clone)] +#[repr(transparent)] +pub struct FileId(FileSystem::FILE_ID_INFO); + +impl PartialEq for FileId { + fn eq(&self, other: &Self) -> bool { + // Lowers to an efficient word-wise comparison. + const SIZE: usize = std::mem::size_of::(); + let a: &[u8; SIZE] = unsafe { mem::transmute(&self.0) }; + let b: &[u8; SIZE] = unsafe { mem::transmute(&other.0) }; + a == b + } +} + +impl Eq for FileId {} + +/// Returns a unique identifier for the given file. +pub fn file_id(file: &File) -> apperr::Result { + unsafe { + let mut info = MaybeUninit::::uninit(); + check_bool_return(FileSystem::GetFileInformationByHandleEx( + file.as_raw_handle(), + FileSystem::FileIdInfo, + info.as_mut_ptr() as *mut _, + mem::size_of::() as u32, + ))?; + Ok(FileId(info.assume_init())) + } +} + +/// Canonicalizes the given path. +/// +/// This differs from [`fs::canonicalize`] in that it strips the `\\?\` UNC +/// prefix on Windows. This is because it's confusing/ugly when displaying it. +pub fn canonicalize(path: &Path) -> std::io::Result { + let mut path = fs::canonicalize(path)?; + let path = path.as_mut_os_string(); + let mut path = mem::take(path).into_encoded_bytes(); + + if path.len() > 6 && &path[0..4] == br"\\?\" && path[4].is_ascii_uppercase() && path[5] == b':' + { + path.drain(0..4); + } + + let path = unsafe { OsString::from_encoded_bytes_unchecked(path) }; + let path = PathBuf::from(path); + Ok(path) +} + +/// Reserves a virtual memory region of the given size. +/// To commit the memory, use [`virtual_commit`]. +/// To release the memory, use [`virtual_release`]. +/// +/// # Safety +/// +/// This function is unsafe because it uses raw pointers. +/// Don't forget to release the memory when you're done with it or you'll leak it. +pub unsafe fn virtual_reserve(size: usize) -> apperr::Result> { + unsafe { + #[allow(unused_assignments, unused_mut)] + let mut base = null_mut(); + + // In debug builds, we use fixed addresses to aid in debugging. + // Makes it possible to immediately tell which address space a pointer belongs to. + #[cfg(all(debug_assertions, not(target_pointer_width = "32")))] + { + static mut S_BASE_GEN: usize = 0x0000100000000000; // 16 TiB + S_BASE_GEN += 0x0000001000000000; // 64 GiB + base = S_BASE_GEN as *mut _; + } + + check_ptr_return(Memory::VirtualAlloc( + base, + size, + Memory::MEM_RESERVE, + Memory::PAGE_READWRITE, + ) as *mut u8) + } +} + +/// Releases a virtual memory region of the given size. +/// +/// # Safety +/// +/// This function is unsafe because it uses raw pointers. +/// Make sure to only pass pointers acquired from [`virtual_reserve`]. +pub unsafe fn virtual_release(base: NonNull, size: usize) { + unsafe { + Memory::VirtualFree(base.as_ptr() as *mut _, size, Memory::MEM_RELEASE); + } +} + +/// Commits a virtual memory region of the given size. +/// +/// # Safety +/// +/// This function is unsafe because it uses raw pointers. +/// Make sure to only pass pointers acquired from [`virtual_reserve`] +/// and to pass a size less than or equal to the size passed to [`virtual_reserve`]. +pub unsafe fn virtual_commit(base: NonNull, size: usize) -> apperr::Result<()> { + unsafe { + check_ptr_return(Memory::VirtualAlloc( + base.as_ptr() as *mut _, + size, + Memory::MEM_COMMIT, + Memory::PAGE_READWRITE, + )) + .map(|_| ()) + } +} + +unsafe fn get_module(name: *const u16) -> apperr::Result> { + unsafe { check_ptr_return(LibraryLoader::GetModuleHandleW(name)) } +} + +unsafe fn load_library(name: *const u16) -> apperr::Result> { + unsafe { + check_ptr_return(LibraryLoader::LoadLibraryExW( + name, + null_mut(), + LibraryLoader::LOAD_LIBRARY_SEARCH_SYSTEM32, + )) + } +} + +/// Loads a function from a dynamic library. +/// +/// # Safety +/// +/// This function is highly unsafe as it requires you to know the exact type +/// of the function you're loading. No type checks whatsoever are performed. +// +// It'd be nice to constrain T to std::marker::FnPtr, but that's unstable. +pub unsafe fn get_proc_address(handle: NonNull, name: &CStr) -> apperr::Result { + unsafe { + let ptr = LibraryLoader::GetProcAddress(handle.as_ptr(), name.as_ptr() as *const u8); + if let Some(ptr) = ptr { Ok(mem::transmute_copy(&ptr)) } else { Err(get_last_error()) } + } +} + +/// Loads the "common" portion of ICU4C. +pub fn load_libicuuc() -> apperr::Result> { + unsafe { load_library(w!("icuuc.dll")) } +} + +/// Loads the internationalization portion of ICU4C. +pub fn load_libicui18n() -> apperr::Result> { + unsafe { load_library(w!("icuin.dll")) } +} + +/// Returns a list of preferred languages for the current user. +pub fn preferred_languages(arena: &Arena) -> Vec { + // If the GetUserPreferredUILanguages() don't fit into 512 characters, + // honestly, just give up. How many languages do you realistically need? + const LEN: usize = 512; + + let scratch = scratch_arena(Some(arena)); + let mut res = Vec::new_in(arena); + + // Get the list of preferred languages via `GetUserPreferredUILanguages`. + let langs = unsafe { + let buf = scratch.alloc_uninit_slice(LEN); + let mut len = buf.len() as u32; + let mut num = 0; + + let ok = Globalization::GetUserPreferredUILanguages( + Globalization::MUI_LANGUAGE_NAME, + &mut num, + buf[0].as_mut_ptr(), + &mut len, + ); + + if ok == 0 || num == 0 { + len = 0; + } + + // Drop the terminating double-null character. + len = len.saturating_sub(1); + + buf[..len as usize].assume_init_ref() + }; + + // Convert UTF16 to UTF8. + let mut langs = wide_to_utf8(&scratch, langs); + + // Turn "de-DE" into "de-de" for easier comparisons. + langs.make_ascii_lowercase(); + + // Split the null-delimited string into individual chunks + // and copy them into the given arena. + res.extend( + langs + .split_terminator('\0') + .filter(|s| !s.is_empty()) + .map(|s| ArenaString::from_str(arena, s)), + ); + res +} + +fn wide_to_utf8<'a>(arena: &'a Arena, wide: &[u16]) -> ArenaString<'a> { + let mut res = ArenaString::new_in(arena); + res.reserve(wide.len() * 3); + + let len = unsafe { + Globalization::WideCharToMultiByte( + Globalization::CP_UTF8, + 0, + wide.as_ptr(), + wide.len() as i32, + res.as_mut_ptr() as *mut _, + res.capacity() as i32, + null(), + null_mut(), + ) + }; + if len > 0 { + unsafe { res.as_mut_vec().set_len(len as usize) }; + } + + res.shrink_to_fit(); + res +} + +#[cold] +fn get_last_error() -> apperr::Error { + unsafe { gle_to_apperr(Foundation::GetLastError()) } +} + +#[inline] +const fn gle_to_apperr(gle: u32) -> apperr::Error { + apperr::Error::new_sys(if gle == 0 { 0x8000FFFF } else { 0x80070000 | gle }) +} + +#[inline] +pub(crate) fn io_error_to_apperr(err: std::io::Error) -> apperr::Error { + gle_to_apperr(err.raw_os_error().unwrap_or(0) as u32) +} + +/// Formats a platform error code into a human-readable string. +pub fn apperr_format(f: &mut std::fmt::Formatter<'_>, code: u32) -> std::fmt::Result { + unsafe { + let mut ptr: *mut u8 = null_mut(); + let len = Debug::FormatMessageA( + Debug::FORMAT_MESSAGE_ALLOCATE_BUFFER + | Debug::FORMAT_MESSAGE_FROM_SYSTEM + | Debug::FORMAT_MESSAGE_IGNORE_INSERTS, + null(), + code, + 0, + &mut ptr as *mut *mut _ as *mut _, + 0, + null_mut(), + ); + + write!(f, "Error {code:#08x}")?; + + if len > 0 { + let msg = str_from_raw_parts(ptr, len as usize); + let msg = msg.trim_ascii(); + let msg = msg.replace(['\r', '\n'], " "); + write!(f, ": {msg}")?; + Foundation::LocalFree(ptr as *mut _); + } + + Ok(()) + } +} + +/// Checks if the given error is a "file not found" error. +pub fn apperr_is_not_found(err: apperr::Error) -> bool { + err == gle_to_apperr(Foundation::ERROR_FILE_NOT_FOUND) +} + +fn check_bool_return(ret: Foundation::BOOL) -> apperr::Result<()> { + if ret == 0 { Err(get_last_error()) } else { Ok(()) } +} + +fn check_ptr_return(ret: *mut T) -> apperr::Result> { + NonNull::new(ret).ok_or_else(get_last_error) +} diff --git a/pkgs/edit/src/tui.rs b/pkgs/edit/src/tui.rs new file mode 100644 index 0000000..e2c22ff --- /dev/null +++ b/pkgs/edit/src/tui.rs @@ -0,0 +1,3853 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! An immediate mode UI framework for terminals. +//! +//! # Why immediate mode? +//! +//! This uses an "immediate mode" design, similar to [ImGui](https://github.com/ocornut/imgui). +//! The reason for this is that I expect the UI needs for any terminal application to be +//! fairly minimal, and for that purpose an immediate mode design is much simpler to use. +//! +//! So what's "immediate mode"? The primary alternative is called "retained mode". +//! The diference is that when you create a button in this framework in one frame, +//! and you stop telling this framework in the next frame, the button will vanish. +//! When you use a regular retained mode UI framework, you create the button once, +//! set up callbacks for when it is clicked, and then stop worrying about it. +//! +//! The downside of immediate mode is that your UI code _may_ become cluttered. +//! The upside however is that that you cannot leak UI elements, you don't need to +//! worry about lifetimes nor callbacks, and that simple UIs are simple to write. +//! +//! More importantly though, the primary reason for this is that the +//! lack of callbacks means we can use this design across a plain C ABI, +//! which we'll need once plugins come into play. GTK's `g_signal_connect` +//! shows that the alternative can be rather cumbersome. +//! +//! # Design overview +//! +//! While this file is fairly lengthy, the overall algorithm is simple. +//! On the first frame ever: +//! * Prepare an empty `arena_next`. +//! * Parse the incoming [`input::Input`] which should be a resize event. +//! * Create a new [`Context`] instance and give it the caller. +//! * Now the caller will draw their UI with the [`Context`] by calling the +//! various [`Context`] UI methods, such as [`Context::block_begin()`] and +//! [`Context::block_end()`]. These two are the basis which all other UI +//! elements are built upon by the way. Each UI element that is created gets +//! allocated onto `arena_next` and inserted into the UI tree. +//! That tree works exactly like the DOM tree in HTML: Each node in the tree +//! has a parent, children, and siblings. The tree layout at the end is then +//! a direct mirror of the code "layout" that created it. +//! * Once the caller is done and drops the [`Context`], it'll secretly call +//! `report_context_completion`. This causes a number of things: +//! * The DOM tree that was built is stored in `prev_tree`. +//! * A hashmap of all nodes is built and stored in `prev_node_map`. +//! * `arena_next` is swapped with `arena_prev`. +//! * Each UI node is measured and laid out. +//! * Now the caller is expected to repeat this process with a [`None`] +//! input event until [`Tui::needs_settling()`] returns false. +//! This is necessary, because when [`Context::button()`] returns `true` +//! in one frame, it may change the state in the caller's code +//! and require another frame to be drawn. +//! * Finally a call to [`Tui::render()`] will render the UI tree into the +//! framebuffer and return VT output. +//! +//! On every subsequent frame the process is similar, but one crucial element +//! of any immediate mode UI framework is added: +//! Now when the caller draws their UI, the various [`Context`] UI elements +//! have access to `prev_node_map` and the previously built UI tree. +//! This allows the UI framework to reuse the previously computed layout for +//! hit tests, caching scroll offsets, and so on. +//! +//! In the end it looks very similar: +//! * Prepare an empty `arena_next`. +//! * Parse the incoming [`input::Input`]... +//! * **BUT** now we can hit-test mouse clicks onto the previously built +//! UI tree. This way we can delegate focus on left mouse clicks. +//! * Create a new [`Context`] instance and give it the caller. +//! * The caller draws their UI with the [`Context`]... +//! * **BUT** we can preserve the UI state across frames. +//! * Continue rendering until [`Tui::needs_settling()`] returns false. +//! * And the final call to [`Tui::render()`]. +//! +//! # Classnames and node IDs +//! +//! So how do we find which node from the previous tree correlates to the +//! current node? Each node needs to be constructed with a "classname". +//! The classname is hashed with the parent node ID as the seed. This derived +//! hash is then used as the new child node ID. Under the assumption that the +//! collision likelihood of the hash function is low, this serves as true IDs. +//! +//! This has the nice added property that finding a node with the same ID +//! guarantees that all of the parent nodes must have equivalent IDs as well. +//! This turns "is the focus anywhere inside this subtree" into an O(1) check. +//! +//! The reason "classnames" are used is because I was hoping to add theming +//! in the future with a syntax similar to CSS (simplified, however). +//! +//! # Example +//! +//! ``` +//! use edit::helpers::Size; +//! use edit::input::Input; +//! use edit::tui::*; +//! use edit::{arena, arena_format}; +//! +//! struct State { +//! counter: i32, +//! } +//! +//! fn main() { +//! arena::init(128 * 1024 * 1024).unwrap(); +//! +//! // Create a `Tui` instance which holds state across frames. +//! let mut tui = Tui::new().unwrap(); +//! let mut state = State { counter: 0 }; +//! let input = Input::Resize(Size { width: 80, height: 24 }); +//! +//! // Pass the input to the TUI. +//! { +//! let mut ctx = tui.create_context(Some(input)); +//! draw(&mut ctx, &mut state); +//! } +//! +//! // Continue until the layout has settled. +//! while tui.needs_settling() { +//! let mut ctx = tui.create_context(None); +//! draw(&mut ctx, &mut state); +//! } +//! +//! // Render the output. +//! let scratch = arena::scratch_arena(None); +//! let output = tui.render(&*scratch); +//! println!("{}", output); +//! } +//! +//! fn draw(ctx: &mut Context, state: &mut State) { +//! ctx.table_begin("classname"); +//! { +//! ctx.table_next_row(); +//! +//! // Thanks to the lack of callbacks, we can use a primitive +//! // if condition here, as well as in any potential C code. +//! if ctx.button("button", "Click me!") { +//! state.counter += 1; +//! } +//! +//! // Similarly, formatting and showing labels is straightforward. +//! // It's impossible to forget updating the label this way. +//! ctx.label("label", &arena_format!(ctx.arena(), "Counter: {}", state.counter)); +//! } +//! ctx.table_end(); +//! } +//! ``` + +use std::arch::breakpoint; +#[cfg(debug_assertions)] +use std::collections::HashSet; +use std::fmt::Write as _; +use std::{iter, mem, ptr, time}; + +use crate::arena::{Arena, ArenaString, scratch_arena}; +use crate::buffer::{CursorMovement, RcTextBuffer, TextBuffer, TextBufferCell}; +use crate::cell::*; +use crate::document::WriteableDocument; +use crate::framebuffer::{Attributes, Framebuffer, INDEXED_COLORS_COUNT, IndexedColor}; +use crate::hash::*; +use crate::helpers::*; +use crate::input::{InputKeyMod, kbmod, vk}; +use crate::{apperr, arena_format, input, unicode}; + +const ROOT_ID: u64 = 0x14057B7EF767814F; // Knuth's MMIX constant +const SHIFT_TAB: InputKey = vk::TAB.with_modifiers(kbmod::SHIFT); + +type Input<'input> = input::Input<'input>; +type InputKey = input::InputKey; +type InputMouseState = input::InputMouseState; +type InputText<'input> = input::InputText<'input>; + +/// Since [`TextBuffer`] creation and management is expensive, +/// we cache instances of them for reuse between frames. +/// This is used for [`Context::editline()`]. +struct CachedTextBuffer { + node_id: u64, + editor: RcTextBuffer, + seen: bool, +} + +/// Since [`Context::editline()`] and [`Context::textarea()`] +/// do almost the same thing, this abstracts over the two. +enum TextBufferPayload<'a> { + Editline(&'a mut dyn WriteableDocument), + Textarea(RcTextBuffer), +} + +/// In order for the TUI to show the correct Ctrl/Alt/Shift +/// translations, this struct lets you set them. +pub struct ModifierTranslations { + pub ctrl: &'static str, + pub alt: &'static str, + pub shift: &'static str, +} + +/// Controls to which node the floater is anchored. +#[derive(Default, Clone, Copy, PartialEq, Eq)] +pub enum Anchor { + /// The floater is attached relative to the node created last. + #[default] + Last, + /// The floater is attached relative to the current node (= parent of new nodes). + Parent, + /// The floater is attached relative to the root node (= usually the viewport). + Root, +} + +/// Controls the position of the floater. See [`Context::attr_float`]. +#[derive(Default)] +pub struct FloatSpec { + /// Controls to which node the floater is anchored. + pub anchor: Anchor, + // Specifies the origin of the container relative to the container size. [0, 1] + pub gravity_x: f32, + pub gravity_y: f32, + // Specifies an offset from the origin in cells. + pub offset_x: f32, + pub offset_y: f32, +} + +/// Informs you about the change that was made to the list selection. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ListSelection { + /// The selection wasn't changed. + Unchanged, + /// The selection was changed to the current list item. + Selected, + /// The selection was changed to the current list item + /// *and* the item was also activated (Enter or Double-click). + Activated, +} + +/// Controls the position of a node relative to its parent. +#[derive(Default)] +pub enum Position { + /// The child is stretched to fill the parent. + #[default] + Stretch, + /// The child is positioned at the left edge of the parent. + Left, + /// The child is positioned at the center of the parent. + Center, + /// The child is positioned at the right edge of the parent. + Right, +} + +/// Controls the text overflow behavior of a label +/// when the text doesn't fit the container. +#[derive(Default, Clone, Copy, PartialEq, Eq)] +pub enum Overflow { + /// Text is simply cut off when it doesn't fit. + #[default] + Clip, + /// An ellipsis is shown at the end of the text. + TruncateHead, + /// An ellipsis is shown in the middle of the text. + TruncateMiddle, + /// An ellipsis is shown at the beginning of the text. + TruncateTail, +} + +/// There's two types of lifetimes the TUI code needs to manage: +/// * Across frames +/// * Per frame +/// +/// [`Tui`] manages the first one. It's also the entrypoint for +/// everything else you may want to do. +pub struct Tui { + /// Arena used for the previous frame. + arena_prev: Arena, + /// Arena used for the current frame. + arena_next: Arena, + /// The UI tree built in the previous frame. + /// This refers to memory in `arena_prev`. + prev_tree: Tree<'static>, + /// A hashmap of all nodes built in the previous frame. + /// This refers to memory in `arena_prev`. + prev_node_map: NodeMap<'static>, + /// The framebuffer used for rendering. + framebuffer: Framebuffer, + + modifier_translations: ModifierTranslations, + floater_default_bg: u32, + floater_default_fg: u32, + modal_default_bg: u32, + modal_default_fg: u32, + + /// Last known terminal size. + /// + /// This lives here instead of [`Context`], because we need to + /// track the state across frames and input events. + /// This also applies to the remaining members in this block below. + size: Size, + /// Last known mouse position. + mouse_position: Point, + /// Between mouse down and up, the position where the mouse was pressed. + /// Otherwise, this contains Point::MIN. + mouse_down_position: Point, + /// Node ID of the node that was clicked on. + /// Used for tracking drag targets. + left_mouse_down_target: u64, + /// Timestamp of the last mouse up event. + /// Used for tracking double/triple clicks. + mouse_up_timestamp: std::time::Instant, + /// The current mouse state. + mouse_state: InputMouseState, + /// Whether the mouse is currently being dragged. + mouse_is_drag: bool, + /// The number of clicks that have happened in a row. + /// Gets reset when the mouse was released for a while. + mouse_click_counter: CoordType, + /// The path to the node that was clicked on. + mouse_down_node_path: Vec, + /// The position of the first click in a double/triple click series. + first_click_position: Point, + /// The node ID of the node that was first clicked on + /// in a double/triple click series. + first_click_target: u64, + + /// Path to the currently focused node. + focused_node_path: Vec, + /// Contains the last element in [`Tui::focused_node_path`]. + /// This way we can track if the focus changed, because then we + /// need to scroll the node into view if it's within a scrollarea. + focused_node_for_scrolling: u64, + + /// A list of cached text buffers used for [`Context::editline()`]. + cached_text_buffers: Vec, + + /// The clipboard contents. + clipboard: Vec, + /// A counter that is incremented every time the clipboard changes. + /// Allows for tracking clipboard changes without comparing contents. + clipboard_generation: u32, + + settling_have: i32, + settling_want: i32, + read_timeout: time::Duration, +} + +impl Tui { + /// Creates a new [`Tui`] instance for storing state across frames. + pub fn new() -> apperr::Result { + let arena_prev = Arena::new(128 * MEBI)?; + let arena_next = Arena::new(128 * MEBI)?; + // SAFETY: Since `prev_tree` refers to `arena_prev`/`arena_next`, from its POV the lifetime + // is `'static`, requiring us to use `transmute` to circumvent the borrow checker. + let prev_tree = Tree::new(unsafe { mem::transmute::<&Arena, &Arena>(&arena_next) }); + + let mut tui = Self { + arena_prev, + arena_next, + prev_tree, + prev_node_map: Default::default(), + framebuffer: Framebuffer::new(), + + modifier_translations: ModifierTranslations { + ctrl: "Ctrl", + alt: "Alt", + shift: "Shift", + }, + floater_default_bg: 0, + floater_default_fg: 0, + modal_default_bg: 0, + modal_default_fg: 0, + + size: Size { width: 0, height: 0 }, + mouse_position: Point::MIN, + mouse_down_position: Point::MIN, + left_mouse_down_target: 0, + mouse_up_timestamp: std::time::Instant::now(), + mouse_state: InputMouseState::None, + mouse_is_drag: false, + mouse_click_counter: 0, + mouse_down_node_path: Vec::with_capacity(16), + first_click_position: Point::MIN, + first_click_target: 0, + + focused_node_path: Vec::with_capacity(16), + focused_node_for_scrolling: ROOT_ID, + + cached_text_buffers: Vec::with_capacity(16), + + clipboard: Vec::new(), + clipboard_generation: 0, + + settling_have: 0, + settling_want: 0, + read_timeout: time::Duration::MAX, + }; + Self::clean_node_path(&mut tui.mouse_down_node_path); + Self::clean_node_path(&mut tui.focused_node_path); + Ok(tui) + } + + /// Sets up the framebuffer's color palette. + pub fn setup_indexed_colors(&mut self, colors: [u32; INDEXED_COLORS_COUNT]) { + self.framebuffer.set_indexed_colors(colors); + } + + /// Set up translations for Ctrl/Alt/Shift modifiers. + pub fn setup_modifier_translations(&mut self, translations: ModifierTranslations) { + self.modifier_translations = translations; + } + + /// Set the default background color for floaters (dropdowns, etc.). + pub fn set_floater_default_bg(&mut self, color: u32) { + self.floater_default_bg = color; + } + + /// Set the default foreground color for floaters (dropdowns, etc.). + pub fn set_floater_default_fg(&mut self, color: u32) { + self.floater_default_fg = color; + } + + /// Set the default background color for modals. + pub fn set_modal_default_bg(&mut self, color: u32) { + self.modal_default_bg = color; + } + + /// Set the default foreground color for modals. + pub fn set_modal_default_fg(&mut self, color: u32) { + self.modal_default_fg = color; + } + + /// If the TUI is currently running animations, etc., + /// this will return a timeout smaller than [`time::Duration::MAX`]. + pub fn read_timeout(&mut self) -> time::Duration { + mem::replace(&mut self.read_timeout, time::Duration::MAX) + } + + /// Returns the viewport size. + pub fn size(&self) -> Size { + // We don't use the size stored in the framebuffer, because until + // `render()` is called, the framebuffer will use a stale size. + self.size + } + + /// Returns an indexed color from the framebuffer. + #[inline] + pub fn indexed(&self, index: IndexedColor) -> u32 { + self.framebuffer.indexed(index) + } + + /// Returns an indexed color from the framebuffer with the given alpha. + /// See [`Framebuffer::indexed_alpha()`]. + #[inline] + pub fn indexed_alpha(&self, index: IndexedColor, numerator: u32, denominator: u32) -> u32 { + self.framebuffer.indexed_alpha(index, numerator, denominator) + } + + /// Returns a color in contrast with the given color. + /// See [`Framebuffer::contrasted()`]. + pub fn contrasted(&self, color: u32) -> u32 { + self.framebuffer.contrasted(color) + } + + /// Returns the current clipboard contents. + pub fn clipboard(&self) -> &[u8] { + &self.clipboard + } + + /// Returns the current clipboard generation. + /// The generation changes every time the clipboard contents change. + /// This allows you to track clipboard changes. + pub fn clipboard_generation(&self) -> u32 { + self.clipboard_generation + } + + /// Starts a new frame and returns a [`Context`] for it. + pub fn create_context<'a, 'input>( + &'a mut self, + input: Option>, + ) -> Context<'a, 'input> { + // SAFETY: Since we have a unique `&mut self`, nothing is holding onto `arena_prev`, + // which will become `arena_next` and get reset. It's safe to reset and reuse its memory. + mem::swap(&mut self.arena_prev, &mut self.arena_next); + unsafe { self.arena_next.reset(0) }; + + // In the input handler below we transformed a mouse up into a release event. + // Now, a frame later, we must reset it back to none, to stop it from triggering things. + // Same for Scroll events. + if self.mouse_state > InputMouseState::Right { + self.mouse_down_position = Point::MIN; + self.mouse_down_node_path.clear(); + self.left_mouse_down_target = 0; + self.mouse_state = InputMouseState::None; + self.mouse_is_drag = false; + } + + if self.scroll_to_focused() { + self.needs_more_settling(); + } + + let now = std::time::Instant::now(); + let mut input_text = None; + let mut input_keyboard = None; + let mut input_mouse_modifiers = kbmod::NONE; + let mut input_mouse_click = 0; + let mut input_scroll_delta = Point { x: 0, y: 0 }; + let input_consumed = self.needs_settling(); + + match input { + None => {} + Some(Input::Resize(resize)) => { + assert!(resize.width > 0 && resize.height > 0); + assert!(resize.width < 32768 && resize.height < 32768); + self.size = resize; + } + Some(Input::Text(text)) => { + input_text = Some(text); + // TODO: the .len()==1 check causes us to ignore keyboard inputs that are faster than we process them. + // For instance, imagine the user presses "A" twice and we happen to read it in a single chunk. + // This causes us to ignore the keyboard input here. We need a way to inform the caller over + // how much of the input text we actually processed in a single frame. Or perhaps we could use + // the needs_settling logic? + if !text.bracketed && text.text.len() == 1 { + let ch = text.text.as_bytes()[0]; + input_keyboard = InputKey::from_ascii(ch as char) + } + } + Some(Input::Keyboard(keyboard)) => { + input_keyboard = Some(keyboard); + } + Some(Input::Mouse(mouse)) => { + let mut next_state = mouse.state; + let next_position = mouse.position; + let next_scroll = mouse.scroll; + let mouse_down = self.mouse_state == InputMouseState::None + && next_state != InputMouseState::None; + let mouse_up = self.mouse_state != InputMouseState::None + && next_state == InputMouseState::None; + let is_drag = self.mouse_state == InputMouseState::Left + && next_state == InputMouseState::Left + && next_position != self.mouse_position; + + let mut hovered_node = None; // Needed for `mouse_down` + let mut focused_node = None; // Needed for `mouse_down` and `is_click` + if mouse_down || mouse_up { + for root in self.prev_tree.iterate_roots() { + Tree::visit_all(root, root, true, |node| { + let n = node.borrow(); + if !n.outer_clipped.contains(next_position) { + // Skip the entire sub-tree, because it doesn't contain the cursor. + return VisitControl::SkipChildren; + } + hovered_node = Some(node); + if n.attributes.focusable { + focused_node = Some(node); + } + VisitControl::Continue + }); + } + } + + if mouse_down { + // Transition from no mouse input to some mouse input --> Record the mouse down position. + Self::build_node_path(hovered_node, &mut self.mouse_down_node_path); + + // On left-mouse-down we change focus. + let mut target = 0; + if next_state == InputMouseState::Left { + target = focused_node.map_or(0, |n| n.borrow().id); + Self::build_node_path(focused_node, &mut self.focused_node_path); + self.needs_more_settling(); // See `needs_more_settling()`. + } + + // Double-/Triple-/Etc.-clicks are triggered on mouse-down, + // unlike the first initial click, which is triggered on mouse-up. + if self.mouse_click_counter != 0 { + if self.first_click_target != target + || self.first_click_position != next_position + || (now - self.mouse_up_timestamp) + > std::time::Duration::from_millis(500) + { + // If the cursor moved / the focus changed in between, or if the user did a slow click, + // we reset the click counter. On mouse-up it'll transition to a regular click. + self.mouse_click_counter = 0; + self.first_click_position = Point::MIN; + self.first_click_target = 0; + } else { + self.mouse_click_counter += 1; + input_mouse_click = self.mouse_click_counter; + }; + } + + // Gets reset at the start of this function. + self.left_mouse_down_target = target; + self.mouse_down_position = next_position; + } else if mouse_up { + // Transition from some mouse input to no mouse input --> The mouse button was released. + next_state = InputMouseState::Release; + + let target = focused_node.map_or(0, |n| n.borrow().id); + + if self.left_mouse_down_target == 0 || self.left_mouse_down_target != target { + // If `left_mouse_down_target == 0`, then it wasn't a left-click, in which case + // the target gets reset. Same, if the focus changed in between any clicks. + self.mouse_click_counter = 0; + self.first_click_position = Point::MIN; + self.first_click_target = 0; + } else if self.mouse_click_counter == 0 { + // No focus change, and no previous clicks? This is an initial, regular click. + self.mouse_click_counter = 1; + self.first_click_position = self.mouse_down_position; + self.first_click_target = target; + input_mouse_click = 1; + } + + self.mouse_up_timestamp = now; + } else if is_drag { + self.mouse_is_drag = true; + } + + input_mouse_modifiers = mouse.modifiers; + input_scroll_delta = next_scroll; + self.mouse_position = next_position; + self.mouse_state = next_state; + } + } + + if !input_consumed { + // Every time there's input, we naturally need to re-render at least once. + self.settling_have = 0; + self.settling_want = 1; + } + + // TODO: There should be a way to do this without unsafe. + // Allocating from the arena borrows the arena, and so allocating the tree here borrows self. + // This conflicts with us passing a mutable reference to `self` into the struct below. + let tree = Tree::new(unsafe { mem::transmute::<&Arena, &Arena>(&self.arena_next) }); + + Context { + tui: self, + + input_text, + input_keyboard, + input_mouse_modifiers, + input_mouse_click, + input_scroll_delta, + input_consumed, + + tree, + last_modal: None, + next_block_id_mixin: 0, + needs_settling: false, + + #[cfg(debug_assertions)] + seen_ids: HashSet::new(), + } + } + + fn report_context_completion<'a>(&'a mut self, ctx: &mut Context<'a, '_>) { + // If this hits, you forgot to block_end() somewhere. The best way to figure + // out where is to do a binary search of commenting out code in main.rs. + debug_assert!(ctx.tree.current_node.borrow().stack_parent.is_none()); + + // Ensure that focus doesn't escape the active modal. + if let Some(node) = ctx.last_modal + && !self.is_subtree_focused(&node.borrow()) + { + ctx.steal_focus_for(node); + } + + // If nodes have appeared or disappeared, we need to re-render. + // Same, if the focus has changed (= changes the highlight color, etc.). + let mut needs_settling = ctx.needs_settling; + needs_settling |= self.prev_tree.checksum != ctx.tree.checksum; + + // Adopt the new tree and recalculate the node hashmap. + // + // SAFETY: The memory used by the tree is owned by the `self.arena_next` right now. + // Stealing the tree here thus doesn't need to copy any memory unless someone resets the arena. + // (The arena is reset in `reset()` above.) + unsafe { + self.prev_tree = mem::transmute_copy(&ctx.tree); + self.prev_node_map = NodeMap::new(mem::transmute(&self.arena_next), &self.prev_tree); + } + + let mut focus_path_pop_min = 0; + // If the user pressed Escape, we move the focus to a parent node. + if !ctx.input_consumed && ctx.consume_shortcut(vk::ESCAPE) { + focus_path_pop_min = 1; + } + + // Remove any unknown nodes from the focus path. + // It's important that we do this after the tree has been swapped out, + // so that pop_focusable_node() has access to the newest version of the tree. + let focus_path_changed = self.pop_focusable_node(focus_path_pop_min); + needs_settling |= focus_path_changed; + + // If some elements went away and the focus path changed above, we ignore tab presses. + // It may otherwise lead to weird situations where focus moves unexpectedly. + if !focus_path_changed + && !ctx.input_consumed + && let Some(input) = ctx.input_keyboard + { + needs_settling |= self.move_focus(input); + } + + // `needs_more_settling()` depends on the current value + // of `settling_have` and so we increment it first. + self.settling_have += 1; + + if needs_settling { + self.needs_more_settling(); + } + + // Remove cached text editors that are no longer in use. + self.cached_text_buffers.retain(|c| c.seen); + + for root in Tree::iterate_siblings(Some(self.prev_tree.root_first)) { + let mut root = root.borrow_mut(); + root.compute_intrinsic_size(); + } + + let viewport = self.size.as_rect(); + + for root in Tree::iterate_siblings(Some(self.prev_tree.root_first)) { + let mut root = root.borrow_mut(); + let root = &mut *root; + + if let Some(float) = &root.attributes.float { + let mut x = 0; + let mut y = 0; + + if let Some(node) = root.parent { + let node = node.borrow(); + x = node.outer.left; + y = node.outer.top; + } + + let size = root.intrinsic_to_outer(); + + x += (float.offset_x - float.gravity_x * size.width as f32) as CoordType; + y += (float.offset_y - float.gravity_y * size.height as f32) as CoordType; + + root.outer.left = x; + root.outer.top = y; + root.outer.right = x + size.width; + root.outer.bottom = y + size.height; + root.outer = root.outer.intersect(viewport); + } else { + root.outer = viewport; + } + + root.inner = root.outer_to_inner(root.outer); + root.outer_clipped = root.outer; + root.inner_clipped = root.inner; + + let outer = root.outer; + root.layout_children(outer); + } + } + + fn build_node_path(node: Option<&NodeCell>, path: &mut Vec) { + path.clear(); + if let Some(mut node) = node { + loop { + let n = node.borrow(); + path.push(n.id); + node = match n.parent { + Some(parent) => parent, + None => break, + }; + } + path.reverse(); + } else { + path.push(ROOT_ID); + } + } + + fn clean_node_path(path: &mut Vec) { + Self::build_node_path(None, path); + } + + /// After you finished processing all input, continue redrawing your UI until this returns false. + pub fn needs_settling(&mut self) -> bool { + self.settling_have <= self.settling_want + } + + fn needs_more_settling(&mut self) { + // If the focus has changed, the new node may need to be re-rendered. + // Same, every time we encounter a previously unknown node via `get_prev_node`, + // because that means it likely failed to get crucial information such as the layout size. + if cfg!(debug_assertions) && self.settling_have == 15 { + breakpoint(); + } + self.settling_want = (self.settling_have + 1).min(20); + } + + /// Renders the last frame into the framebuffer and returns the VT output. + pub fn render<'a>(&mut self, arena: &'a Arena) -> ArenaString<'a> { + self.framebuffer.flip(self.size); + for child in self.prev_tree.iterate_roots() { + let mut child = child.borrow_mut(); + self.render_node(&mut child); + } + self.framebuffer.render(arena) + } + + /// Recursively renders each node and its children. + #[allow(clippy::only_used_in_recursion)] + fn render_node(&mut self, node: &mut Node) { + let outer_clipped = node.outer_clipped; + if outer_clipped.is_empty() { + return; + } + + let scratch = scratch_arena(None); + + if node.attributes.bordered { + // ┌────┐ + { + let mut fill = ArenaString::new_in(&scratch); + fill.push('┌'); + fill.push_repeat('─', (outer_clipped.right - outer_clipped.left - 2) as usize); + fill.push('┐'); + self.framebuffer.replace_text( + outer_clipped.top, + outer_clipped.left, + outer_clipped.right, + &fill, + ); + } + + // │ │ + { + let mut fill = ArenaString::new_in(&scratch); + fill.push('│'); + fill.push_repeat(' ', (outer_clipped.right - outer_clipped.left - 2) as usize); + fill.push('│'); + + for y in outer_clipped.top + 1..outer_clipped.bottom - 1 { + self.framebuffer.replace_text( + y, + outer_clipped.left, + outer_clipped.right, + &fill, + ); + } + } + + // └────┘ + { + let mut fill = ArenaString::new_in(&scratch); + fill.push('└'); + fill.push_repeat('─', (outer_clipped.right - outer_clipped.left - 2) as usize); + fill.push('┘'); + self.framebuffer.replace_text( + outer_clipped.bottom - 1, + outer_clipped.left, + outer_clipped.right, + &fill, + ); + } + } + + if node.attributes.float.is_some() && node.attributes.bg & 0xff000000 == 0xff000000 { + if !node.attributes.bordered { + let mut fill = ArenaString::new_in(&scratch); + fill.push_repeat(' ', (outer_clipped.right - outer_clipped.left) as usize); + + for y in outer_clipped.top..outer_clipped.bottom { + self.framebuffer.replace_text( + y, + outer_clipped.left, + outer_clipped.right, + &fill, + ); + } + } + + self.framebuffer.replace_attr(outer_clipped, Attributes::All, Attributes::None); + } + + self.framebuffer.blend_bg(outer_clipped, node.attributes.bg); + self.framebuffer.blend_fg(outer_clipped, node.attributes.fg); + + if node.attributes.reverse { + self.framebuffer.reverse(outer_clipped); + } + + let inner = node.inner; + let inner_clipped = node.inner_clipped; + if inner_clipped.is_empty() { + return; + } + + match &mut node.content { + NodeContent::Modal(title) => { + if !title.is_empty() { + self.framebuffer.replace_text( + node.outer.top, + node.outer.left + 2, + node.outer.right - 1, + title, + ); + } + } + NodeContent::Text(content) => self.render_styled_text( + inner, + node.intrinsic_size.width, + &content.text, + &content.chunks, + content.overflow, + ), + NodeContent::Textarea(tc) => { + let mut tb = tc.buffer.borrow_mut(); + let mut destination = Rect { + left: inner_clipped.left, + top: inner_clipped.top, + right: inner_clipped.right, + bottom: inner_clipped.bottom, + }; + + if !tc.single_line { + // Account for the scrollbar. + destination.right -= 1; + } + + if let Some(res) = + tb.render(tc.scroll_offset, destination, tc.has_focus, &mut self.framebuffer) + { + tc.scroll_offset_x_max = res.visual_pos_x_max; + } + + if !tc.single_line { + // Render the scrollbar. + let track = Rect { + left: inner_clipped.right - 1, + top: inner_clipped.top, + right: inner_clipped.right, + bottom: inner_clipped.bottom, + }; + tc.thumb_height = self.framebuffer.draw_scrollbar( + inner_clipped, + track, + tc.scroll_offset.y, + tb.visual_line_count() + inner.height() - 1, + ); + } + } + NodeContent::Scrollarea(sc) => { + let content = node.children.first.unwrap().borrow(); + let track = Rect { + left: inner.right, + top: inner.top, + right: inner.right + 1, + bottom: inner.bottom, + }; + sc.thumb_height = self.framebuffer.draw_scrollbar( + outer_clipped, + track, + sc.scroll_offset.y, + content.intrinsic_size.height, + ); + } + _ => {} + } + + for child in Tree::iterate_siblings(node.children.first) { + let mut child = child.borrow_mut(); + self.render_node(&mut child); + } + } + + fn render_styled_text( + &mut self, + target: Rect, + actual_width: CoordType, + text: &str, + chunks: &[StyledTextChunk], + overflow: Overflow, + ) { + let target_width = target.width(); + // The section of `text` that is skipped by the ellipsis. + let mut skipped = 0..0; + // The number of columns skipped by the ellipsis. + let mut skipped_cols = 0; + + if overflow == Overflow::Clip || target_width >= actual_width { + self.framebuffer.replace_text(target.top, target.left, target.right, text); + } else { + let bytes = text.as_bytes(); + let mut cfg = unicode::MeasurementConfig::new(&bytes); + + match overflow { + Overflow::Clip => unreachable!(), + Overflow::TruncateHead => { + let beg = cfg.goto_visual(Point { x: actual_width - target_width + 1, y: 0 }); + skipped = 0..beg.offset; + skipped_cols = beg.visual_pos.x - 1; + } + Overflow::TruncateMiddle => { + let mid_beg_x = (target_width - 1) / 2; + let mid_end_x = actual_width - target_width / 2; + let beg = cfg.goto_visual(Point { x: mid_beg_x, y: 0 }); + let end = cfg.goto_visual(Point { x: mid_end_x, y: 0 }); + skipped = beg.offset..end.offset; + skipped_cols = end.visual_pos.x - beg.visual_pos.x - 1; + } + Overflow::TruncateTail => { + let end = cfg.goto_visual(Point { x: target_width - 1, y: 0 }); + skipped_cols = actual_width - end.visual_pos.x - 1; + skipped = end.offset..text.len(); + } + } + + let scratch = scratch_arena(None); + + let mut modified = ArenaString::new_in(&scratch); + modified.reserve(text.len() + 3); + modified.push_str(&text[..skipped.start]); + modified.push('…'); + modified.push_str(&text[skipped.end..]); + + self.framebuffer.replace_text(target.top, target.left, target.right, &modified); + } + + if !chunks.is_empty() { + let bytes = text.as_bytes(); + let mut cfg = unicode::MeasurementConfig::new(&bytes).with_cursor(unicode::Cursor { + visual_pos: Point { x: target.left, y: 0 }, + ..Default::default() + }); + + let mut iter = chunks.iter().peekable(); + + while let Some(chunk) = iter.next() { + let beg = chunk.offset; + let end = iter.peek().map_or(text.len(), |c| c.offset); + + if beg >= skipped.start && end <= skipped.end { + // Chunk is fully inside the text skipped by the ellipsis. + // We don't need to render it at all. + continue; + } + + if beg < skipped.start { + let beg = cfg.goto_offset(beg).visual_pos.x; + let end = cfg.goto_offset(end.min(skipped.start)).visual_pos.x; + let rect = + Rect { left: beg, top: target.top, right: end, bottom: target.bottom }; + self.framebuffer.blend_fg(rect, chunk.fg); + self.framebuffer.replace_attr(rect, chunk.attr, chunk.attr); + } + + if end > skipped.end { + let beg = cfg.goto_offset(beg.max(skipped.end)).visual_pos.x - skipped_cols; + let end = cfg.goto_offset(end).visual_pos.x - skipped_cols; + let rect = + Rect { left: beg, top: target.top, right: end, bottom: target.bottom }; + self.framebuffer.blend_fg(rect, chunk.fg); + self.framebuffer.replace_attr(rect, chunk.attr, chunk.attr); + } + } + } + } + + /// Outputs a debug string of the layout and focus tree. + pub fn debug_layout<'a>(&mut self, arena: &'a Arena) -> ArenaString<'a> { + let mut result = ArenaString::new_in(arena); + result.push_str("general:\r\n- focus_path:\r\n"); + + for &id in &self.focused_node_path { + _ = write!(result, " - {id:016x}\r\n"); + } + + result.push_str("\r\ntree:\r\n"); + + for root in self.prev_tree.iterate_roots() { + Tree::visit_all(root, root, true, |node| { + let node = node.borrow(); + let depth = node.depth; + result.push_repeat(' ', depth * 2); + _ = write!(result, "- id: {:016x}\r\n", node.id); + + result.push_repeat(' ', depth * 2); + _ = write!(result, " classname: {}\r\n", node.classname); + + if depth == 0 + && let Some(parent) = node.parent + { + let parent = parent.borrow(); + result.push_repeat(' ', depth * 2); + _ = write!(result, " parent: {:016x}\r\n", parent.id); + } + + result.push_repeat(' ', depth * 2); + _ = write!( + result, + " intrinsic: {{{}, {}}}\r\n", + node.intrinsic_size.width, node.intrinsic_size.height + ); + + result.push_repeat(' ', depth * 2); + _ = write!( + result, + " outer: {{{}, {}, {}, {}}}\r\n", + node.outer.left, node.outer.top, node.outer.right, node.outer.bottom + ); + + result.push_repeat(' ', depth * 2); + _ = write!( + result, + " inner: {{{}, {}, {}, {}}}\r\n", + node.inner.left, node.inner.top, node.inner.right, node.inner.bottom + ); + + if node.attributes.bordered { + result.push_repeat(' ', depth * 2); + result.push_str(" bordered: true\r\n"); + } + + if node.attributes.bg != 0 { + result.push_repeat(' ', depth * 2); + _ = write!(result, " bg: #{:08x}\r\n", node.attributes.bg); + } + + if node.attributes.fg != 0 { + result.push_repeat(' ', depth * 2); + _ = write!(result, " fg: #{:08x}\r\n", node.attributes.fg); + } + + if self.is_node_focused(node.id) { + result.push_repeat(' ', depth * 2); + result.push_str(" focused: true\r\n"); + } + + match &node.content { + NodeContent::Text(content) => { + result.push_repeat(' ', depth * 2); + _ = write!(result, " text: \"{}\"\r\n", &content.text); + } + NodeContent::Textarea(content) => { + let tb = content.buffer.borrow(); + let tb = &*tb; + result.push_repeat(' ', depth * 2); + _ = write!(result, " textarea: {tb:p}\r\n"); + } + NodeContent::Scrollarea(..) => { + result.push_repeat(' ', depth * 2); + result.push_str(" scrollable: true\r\n"); + } + _ => {} + } + + VisitControl::Continue + }); + } + + result + } + + fn was_mouse_down_on_node(&self, id: u64) -> bool { + self.mouse_down_node_path.last() == Some(&id) + } + + fn was_mouse_down_on_subtree(&self, node: &Node) -> bool { + self.mouse_down_node_path.get(node.depth) == Some(&node.id) + } + + fn is_node_focused(&self, id: u64) -> bool { + // We construct the focused_node_path always with at least 1 element (the root id). + unsafe { *self.focused_node_path.last().unwrap_unchecked() == id } + } + + fn is_subtree_focused(&self, node: &Node) -> bool { + self.focused_node_path.get(node.depth) == Some(&node.id) + } + + fn is_subtree_focused_alt(&self, id: u64, depth: usize) -> bool { + self.focused_node_path.get(depth) == Some(&id) + } + + fn pop_focusable_node(&mut self, pop_minimum: usize) -> bool { + let last_before = self.focused_node_path.last().cloned().unwrap_or(0); + + // Remove `pop_minimum`-many nodes from the end of the focus path. + let path = &self.focused_node_path[..]; + let path = &path[..path.len().saturating_sub(pop_minimum)]; + let mut len = 0; + + for (i, &id) in path.iter().enumerate() { + // Truncate the path so that it only contains nodes that still exist. + let Some(node) = self.prev_node_map.get(id) else { + break; + }; + + let n = node.borrow(); + // If the caller requested upward movement, pop out of the current focus void, if any. + // This is kind of janky, to be fair. + if pop_minimum != 0 && n.attributes.focus_void { + break; + } + + // Skip over those that aren't focusable. + if n.attributes.focusable { + // At this point `n.depth == i` should be true, + // but I kind of don't want to rely on that. + len = i + 1; + } + } + + self.focused_node_path.truncate(len); + + // If it's empty now, push `ROOT_ID` because there must always be >=1 element. + if self.focused_node_path.is_empty() { + self.focused_node_path.push(ROOT_ID); + } + + // Return true if the focus path changed. + let last_after = self.focused_node_path.last().cloned().unwrap_or(0); + last_before != last_after + } + + // TODO: Move this into `block_end()` and run it whenever the block is a `focus_well`. + // It makes no sense otherwise that all input handling occurs in the controls, except for this. + fn move_focus(&mut self, input: InputKey) -> bool { + if !matches!(input, vk::TAB | SHIFT_TAB | vk::UP | vk::DOWN | vk::LEFT | vk::RIGHT) { + return false; + } + + let focused_id = self.focused_node_path.last().cloned().unwrap_or(0); + let Some(focused) = self.prev_node_map.get(focused_id) else { + debug_assert!(false); // The caller should've cleaned up the focus path. + return false; + }; + + let mut focused_start = focused; + let mut root = focused; + + // Figure out if we're inside a focus void (a container that doesn't + // allow tabbing inside), and in that case, toss the focus to it. + // + // Also, figure out the container within which the focus must be contained. + // This way, tab/shift-tab only moves within the same window. + // The ROOT_ID node has no parent, and the others have a float attribute. + // If the root is the focused node, it should of course not move upward. + loop { + let root_node = root.borrow(); + if root_node.attributes.focus_well { + break; + } + if root_node.attributes.focus_void { + focused_start = root; + } + root = match root_node.parent { + Some(parent) => parent, + None => break, + } + } + + let forward; + let min_depth; + match input { + SHIFT_TAB | vk::TAB => { + forward = input == vk::TAB; + min_depth = usize::MAX; + } + vk::UP | vk::DOWN => { + forward = input == vk::DOWN; + min_depth = usize::MAX; + } + vk::LEFT | vk::RIGHT => { + // Find the cell within a row within a table that we're in. + // To do so we'll use a circular buffer of the last 3 nodes while we travel up. + let mut buf = [None; 3]; + let mut idx = buf.len() - 1; + let mut node = focused_start; + + loop { + idx = (idx + 1) % buf.len(); + buf[idx] = Some(node); + if let NodeContent::Table(..) = &node.borrow().content { + break; + } + if ptr::eq(node, root) { + return false; + } + node = match node.borrow().parent { + Some(parent) => parent, + None => return false, + } + } + + // The current `idx` points to the table. + // The last item is the row. + // The 2nd to last item is the cell. + let Some(row) = buf[(idx + 3 - 1) % buf.len()] else { + return false; + }; + let Some(cell) = buf[(idx + 3 - 2) % buf.len()] else { + return false; + }; + + root = row; + focused_start = cell; + forward = input == vk::RIGHT; + min_depth = root.borrow().depth; + } + _ => return false, + } + + let mut focused_next = focused_start; + Tree::visit_all(root, focused_start, forward, |node| { + let n = node.borrow(); + if ptr::eq(node, root) { + VisitControl::Continue + } else if n.attributes.focusable && !ptr::eq(node, focused_start) { + focused_next = node; + VisitControl::Stop + } else if n.attributes.focus_void || n.depth >= min_depth { + VisitControl::SkipChildren + } else { + VisitControl::Continue + } + }); + + if ptr::eq(focused_next, focused_start) { + false + } else { + Tui::build_node_path(Some(focused_next), &mut self.focused_node_path); + true + } + } + + // Scroll the focused node(s) into view inside scrollviews + fn scroll_to_focused(&mut self) -> bool { + let focused_id = self.focused_node_path.last().cloned().unwrap_or(0); + if self.focused_node_for_scrolling == focused_id { + return false; + } + + let Some(node) = self.prev_node_map.get(focused_id) else { + // Node not found because we're using the old layout tree. + // Retry in the next rendering loop. + return true; + }; + + let mut node = node.borrow_mut(); + let mut scroll_to = node.outer; + + while node.parent.is_some() && node.attributes.float.is_none() { + let n = &mut *node; + if let NodeContent::Scrollarea(sc) = &mut n.content { + let off_y = sc.scroll_offset.y.max(0); + let mut y = off_y; + y = y.min(scroll_to.top - n.inner.top + off_y); + y = y.max(scroll_to.bottom - n.inner.bottom + off_y); + sc.scroll_offset.y = y; + scroll_to = n.outer; + } + node = node.parent.unwrap().borrow_mut(); + } + + self.focused_node_for_scrolling = focused_id; + true + } +} + +/// Context is a temporary object that is created for each frame. +/// Its primary purpose is to build a UI tree. +pub struct Context<'a, 'input> { + tui: &'a mut Tui, + + /// Current text input, if any. + input_text: Option>, + /// Current keyboard input, if any. + input_keyboard: Option, + input_mouse_modifiers: InputKeyMod, + input_mouse_click: CoordType, + /// By how much the mouse wheel was scrolled since the last frame. + input_scroll_delta: Point, + input_consumed: bool, + + tree: Tree<'a>, + last_modal: Option<&'a NodeCell<'a>>, + next_block_id_mixin: u64, + needs_settling: bool, + + #[cfg(debug_assertions)] + seen_ids: HashSet, +} + +impl<'a> Drop for Context<'a, '_> { + fn drop(&mut self) { + let tui: &'a mut Tui = unsafe { mem::transmute(&mut *self.tui) }; + tui.report_context_completion(self); + } +} + +impl<'a> Context<'a, '_> { + /// Get an arena for temporary allocations such as for [`arena_format`]. + pub fn arena(&self) -> &'a Arena { + // TODO: + // `Context` borrows `Tui` for lifetime 'a, so `self.tui` should be `&'a Tui`, right? + // And if I do `&self.tui.arena` then that should be 'a too, right? + // Searching for and failing to find a workaround for this was _very_ annoying. + // + // SAFETY: Both the returned reference and its allocations outlive &self. + unsafe { mem::transmute::<&'_ Arena, &'a Arena>(&self.tui.arena_next) } + } + + /// Returns the viewport size. + pub fn size(&self) -> Size { + self.tui.size() + } + + /// Returns an indexed color from the framebuffer. + #[inline] + pub fn indexed(&self, index: IndexedColor) -> u32 { + self.tui.framebuffer.indexed(index) + } + + /// Returns an indexed color from the framebuffer with the given alpha. + /// See [`Framebuffer::indexed_alpha()`]. + #[inline] + pub fn indexed_alpha(&self, index: IndexedColor, numerator: u32, denominator: u32) -> u32 { + self.tui.framebuffer.indexed_alpha(index, numerator, denominator) + } + + /// Returns a color in contrast with the given color. + /// See [`Framebuffer::contrasted()`]. + pub fn contrasted(&self, color: u32) -> u32 { + self.tui.framebuffer.contrasted(color) + } + + /// Returns the current clipboard contents. + pub fn clipboard(&self) -> &[u8] { + self.tui.clipboard() + } + + /// Returns the current clipboard generation. + /// The generation changes every time the clipboard contents change. + /// This allows you to track clipboard changes. + pub fn clipboard_generation(&self) -> u32 { + self.tui.clipboard_generation() + } + + /// Sets the clipboard contents. + pub fn set_clipboard(&mut self, data: Vec) { + if !data.is_empty() { + self.tui.clipboard = data; + self.tui.clipboard_generation = self.tui.clipboard_generation.wrapping_add(1); + self.needs_rerender(); + } + } + + /// Tell the UI framework that your state changed and you need another layout pass. + pub fn needs_rerender(&mut self) { + // If this hits, the call stack is responsible is trying to deadlock you. + debug_assert!(self.tui.settling_have < 15); + self.needs_settling = true; + } + + /// Begins a generic UI block (container) with a unique ID derived from the given `classname`. + pub fn block_begin(&mut self, classname: &'static str) { + let parent = self.tree.current_node; + + let mut id = hash_str(parent.borrow().id, classname); + if self.next_block_id_mixin != 0 { + id = hash(id, &self.next_block_id_mixin.to_ne_bytes()); + self.next_block_id_mixin = 0; + } + + // If this hits, you have tried to create a block with the same ID as a previous one + // somewhere up this call stack. Change the classname, or use next_block_id_mixin(). + // TODO: HashMap + #[cfg(debug_assertions)] + if !self.seen_ids.insert(id) { + panic!("Duplicate node ID: {id:x}"); + } + + let node = Tree::alloc_node(self.arena()); + { + let mut n = node.borrow_mut(); + n.id = id; + n.classname = classname; + } + + self.tree.push_child(node); + } + + /// Ends the current UI block, returning to its parent container. + pub fn block_end(&mut self) { + self.tree.pop_stack(); + } + + /// Mixes in an extra value to the next UI block's ID for uniqueness. + /// Use this when you build a list of items with the same classname. + pub fn next_block_id_mixin(&mut self, id: u64) { + self.next_block_id_mixin = id; + } + + fn attr_focusable(&mut self) { + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.attributes.focusable = true; + } + + /// If this is the first time the current node is being drawn, + /// it'll steal the active focus. + pub fn focus_on_first_present(&mut self) { + let steal = { + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.attributes.focusable = true; + self.tui.prev_node_map.get(last_node.id).is_none() + }; + if steal { + self.steal_focus(); + } + } + + /// Steals the focus unconditionally. + pub fn steal_focus(&mut self) { + self.steal_focus_for(self.tree.last_node); + } + + fn steal_focus_for(&mut self, node: &NodeCell<'a>) { + if !self.tui.is_node_focused(node.borrow().id) { + Tui::build_node_path(Some(node), &mut self.tui.focused_node_path); + self.needs_rerender(); + } + } + + /// If the current node owns the focus, it'll be given to the parent. + pub fn toss_focus_up(&mut self) { + if self.tui.pop_focusable_node(1) { + self.needs_rerender(); + } + } + + /// If the parent node owns the focus, it'll be given to the current node. + pub fn inherit_focus(&mut self) { + let mut last_node = self.tree.last_node.borrow_mut(); + let Some(parent) = last_node.parent else { + return; + }; + + last_node.attributes.focusable = true; + + // Mark the parent as focusable, so that if the user presses Escape, + // and `block_end` bubbles the focus up the tree, it'll stop on our parent, + // which will then focus us on the next iteration. + let mut parent = parent.borrow_mut(); + parent.attributes.focusable = true; + + if self.tui.is_node_focused(parent.id) { + self.needs_rerender(); + self.tui.focused_node_path.push(last_node.id); + } + } + + /// Causes keyboard focus to be unable to escape this node and its children. + /// It's a "well" because if the focus is inside it, it can't escape. + pub fn attr_focus_well(&mut self) { + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.attributes.focus_well = true; + } + + /// Explicitly sets the intrinsic size of the current node. + /// The intrinsic size is the size the node ideally wants to be. + pub fn attr_intrinsic_size(&mut self, size: Size) { + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.intrinsic_size = size; + last_node.intrinsic_size_set = true; + } + + /// Turns the current node into a floating node, + /// like a popup, modal or a tooltip. + pub fn attr_float(&mut self, spec: FloatSpec) { + let last_node = self.tree.last_node; + let anchor = { + let ln = last_node.borrow(); + match spec.anchor { + Anchor::Last if ln.siblings.prev.is_some() => ln.siblings.prev, + Anchor::Last | Anchor::Parent => ln.parent, + // By not giving such floats a parent, they get the same origin as the original root node, + // but they also gain their own "root id" in the tree. That way, their focus path is totally unique, + // which means that we can easily check if a modal is open by calling `is_focused()` on the original root. + Anchor::Root => None, + } + }; + + self.tree.move_node_to_root(last_node, anchor); + + let mut ln = last_node.borrow_mut(); + ln.attributes.focus_well = true; + ln.attributes.float = Some(FloatAttributes { + gravity_x: spec.gravity_x.clamp(0.0, 1.0), + gravity_y: spec.gravity_y.clamp(0.0, 1.0), + offset_x: spec.offset_x, + offset_y: spec.offset_y, + }); + ln.attributes.bg = self.tui.floater_default_bg; + ln.attributes.fg = self.tui.floater_default_fg; + } + + /// Gives the current node a border. + pub fn attr_border(&mut self) { + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.attributes.bordered = true; + } + + /// Sets the current node's position inside the parent. + pub fn attr_position(&mut self, align: Position) { + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.attributes.position = align; + } + + /// Assigns padding to the current node. + pub fn attr_padding(&mut self, padding: Rect) { + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.attributes.padding = Self::normalize_rect(padding); + } + + fn normalize_rect(rect: Rect) -> Rect { + Rect { + left: rect.left.max(0), + top: rect.top.max(0), + right: rect.right.max(0), + bottom: rect.bottom.max(0), + } + } + + /// Assigns a sRGB background color to the current node. + pub fn attr_background_rgba(&mut self, bg: u32) { + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.attributes.bg = bg; + } + + /// Assigns a sRGB foreground color to the current node. + pub fn attr_foreground_rgba(&mut self, fg: u32) { + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.attributes.fg = fg; + } + + /// Applies reverse-video to the current node: + /// Background and foreground colors are swapped. + pub fn attr_reverse(&mut self) { + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.attributes.reverse = true; + } + + /// Checks if the current keyboard input matches the given shortcut, + /// consumes it if it is and returns true in that case. + pub fn consume_shortcut(&mut self, shortcut: InputKey) -> bool { + if !self.input_consumed && self.input_keyboard == Some(shortcut) { + self.set_input_consumed(); + true + } else { + false + } + } + + /// Returns current keyboard input, if any. + /// Returns None if the input was already consumed. + pub fn keyboard_input(&self) -> Option { + if self.input_consumed { None } else { self.input_keyboard } + } + + #[inline] + pub fn set_input_consumed(&mut self) { + debug_assert!(!self.input_consumed); + self.set_input_consumed_unchecked(); + } + + #[inline] + fn set_input_consumed_unchecked(&mut self) { + self.input_consumed = true; + } + + /// Returns whether the mouse was pressed down on the current node. + pub fn was_mouse_down(&mut self) -> bool { + let last_node = self.tree.last_node.borrow(); + self.tui.was_mouse_down_on_node(last_node.id) + } + + /// Returns whether the mouse was pressed down on the current node's subtree. + pub fn contains_mouse_down(&mut self) -> bool { + let last_node = self.tree.last_node.borrow(); + self.tui.was_mouse_down_on_subtree(&last_node) + } + + /// Returns whether the current node is focused. + pub fn is_focused(&mut self) -> bool { + let last_node = self.tree.last_node.borrow(); + self.tui.is_node_focused(last_node.id) + } + + /// Returns whether the current node's subtree is focused. + pub fn contains_focus(&mut self) -> bool { + let last_node = self.tree.last_node.borrow(); + self.tui.is_subtree_focused(&last_node) + } + + /// Begins a modal window. Call [`Context::modal_end()`]. + pub fn modal_begin(&mut self, classname: &'static str, title: &str) { + self.block_begin(classname); + self.attr_float(FloatSpec { anchor: Anchor::Root, ..Default::default() }); + self.attr_intrinsic_size(Size { width: self.tui.size.width, height: self.tui.size.height }); + self.attr_background_rgba(self.indexed_alpha(IndexedColor::Background, 1, 2)); + self.attr_foreground_rgba(self.indexed_alpha(IndexedColor::Background, 1, 2)); + self.attr_focus_well(); + + self.block_begin("window"); + self.attr_float(FloatSpec { + anchor: Anchor::Last, + gravity_x: 0.5, + gravity_y: 0.5, + offset_x: self.tui.size.width as f32 * 0.5, + offset_y: self.tui.size.height as f32 * 0.5, + }); + self.attr_border(); + self.attr_background_rgba(self.tui.modal_default_bg); + self.attr_foreground_rgba(self.tui.modal_default_fg); + self.inherit_focus(); + self.focus_on_first_present(); + + let mut last_node = self.tree.last_node.borrow_mut(); + let title = if title.is_empty() { + ArenaString::new_in(self.arena()) + } else { + arena_format!(self.arena(), " {} ", title) + }; + last_node.content = NodeContent::Modal(title); + self.last_modal = Some(self.tree.last_node); + } + + /// Ends the current modal window block. + /// Returns true if the user pressed Escape (a request to close). + pub fn modal_end(&mut self) -> bool { + self.block_end(); + self.block_end(); + + // Consume the input unconditionally, so that the root (the "main window") + // doesn't accidentally receive any input via `consume_shortcut()`. + if self.contains_focus() { + let exit = !self.input_consumed && self.input_keyboard == Some(vk::ESCAPE); + self.set_input_consumed_unchecked(); + exit + } else { + false + } + } + + /// Begins a table block. Call [`Context::table_end()`]. + /// Tables are the primary way to create a grid layout, + /// and to layout controls on a single row (= a table with 1 row). + pub fn table_begin(&mut self, classname: &'static str) { + self.block_begin(classname); + + let mut last_node = self.tree.last_node.borrow_mut(); + last_node.content = NodeContent::Table(TableContent { + columns: Vec::new_in(self.arena()), + cell_gap: Default::default(), + }); + } + + /// Assigns widths to the columns of the current table. + /// By default, the table will left-align all columns. + pub fn table_set_columns(&mut self, columns: &[CoordType]) { + let mut last_node = self.tree.last_node.borrow_mut(); + if let NodeContent::Table(spec) = &mut last_node.content { + spec.columns.clear(); + spec.columns.extend_from_slice(columns); + } else { + debug_assert!(false); + } + } + + /// Assigns the gap between cells in the current table. + pub fn table_set_cell_gap(&mut self, cell_gap: Size) { + let mut last_node = self.tree.last_node.borrow_mut(); + if let NodeContent::Table(spec) = &mut last_node.content { + spec.cell_gap = cell_gap; + } else { + debug_assert!(false); + } + } + + /// Starts the next row in the current table. + pub fn table_next_row(&mut self) { + { + let current_node = self.tree.current_node.borrow(); + + // If this is the first call to table_next_row() inside a new table, the + // current_node will refer to the table. Otherwise, it'll refer to the current row. + if !matches!(current_node.content, NodeContent::Table(_)) { + let Some(parent) = current_node.parent else { + return; + }; + + let parent = parent.borrow(); + // Neither the current nor its parent nodes are a table? + // You definitely called this outside of a table block. + debug_assert!(matches!(parent.content, NodeContent::Table(_))); + + self.block_end(); + self.next_block_id_mixin(parent.child_count as u64); + } + } + + self.block_begin("row"); + } + + /// Ends the current table block. + pub fn table_end(&mut self) { + let current_node = self.tree.current_node.borrow(); + + // If this is the first call to table_next_row() inside a new table, the + // current_node will refer to the table. Otherwise, it'll refer to the current row. + if !matches!(current_node.content, NodeContent::Table(_)) { + self.block_end(); + } + + self.block_end(); // table + } + + /// Creates a simple text label. + pub fn label(&mut self, classname: &'static str, text: &str) { + self.styled_label_begin(classname); + self.styled_label_add_text(text); + self.styled_label_end(); + } + + /// Creates a styled text label. + /// + /// # Example + /// ``` + /// use edit::framebuffer::IndexedColor; + /// use edit::tui::Context; + /// + /// fn draw(ctx: &mut Context) { + /// ctx.styled_label_begin("label"); + /// // Shows "Hello" in the inherited foreground color. + /// ctx.styled_label_add_text("Hello"); + /// // Shows ", World!" next to "Hello" in red. + /// ctx.styled_label_set_foreground(ctx.indexed(IndexedColor::Red)); + /// ctx.styled_label_add_text(", World!"); + /// } + /// ``` + pub fn styled_label_begin(&mut self, classname: &'static str) { + self.block_begin(classname); + self.tree.last_node.borrow_mut().content = NodeContent::Text(TextContent { + text: ArenaString::new_in(self.arena()), + chunks: Vec::with_capacity_in(4, self.arena()), + overflow: Overflow::Clip, + }); + } + + /// Changes the active pencil color of the current label. + pub fn styled_label_set_foreground(&mut self, fg: u32) { + let mut node = self.tree.last_node.borrow_mut(); + let NodeContent::Text(content) = &mut node.content else { + unreachable!(); + }; + + let last = content.chunks.last().unwrap_or(&INVALID_STYLED_TEXT_CHUNK); + if last.offset != content.text.len() && last.fg != fg { + content.chunks.push(StyledTextChunk { + offset: content.text.len(), + fg, + attr: last.attr, + }); + } + } + + /// Changes the active pencil attributes of the current label. + pub fn styled_label_set_attributes(&mut self, attr: Attributes) { + let mut node = self.tree.last_node.borrow_mut(); + let NodeContent::Text(content) = &mut node.content else { + unreachable!(); + }; + + let last = content.chunks.last().unwrap_or(&INVALID_STYLED_TEXT_CHUNK); + if last.offset != content.text.len() && last.attr != attr { + content.chunks.push(StyledTextChunk { offset: content.text.len(), fg: last.fg, attr }); + } + } + + /// Adds text to the current label. + pub fn styled_label_add_text(&mut self, text: &str) { + let mut node = self.tree.last_node.borrow_mut(); + let NodeContent::Text(content) = &mut node.content else { + unreachable!(); + }; + + content.text.push_str(text); + } + + /// Ends the current label block. + pub fn styled_label_end(&mut self) { + { + let mut last_node = self.tree.last_node.borrow_mut(); + let NodeContent::Text(content) = &last_node.content else { + return; + }; + + let cursor = unicode::MeasurementConfig::new(&content.text.as_bytes()) + .goto_visual(Point { x: CoordType::MAX, y: 0 }); + last_node.intrinsic_size.width = cursor.visual_pos.x; + last_node.intrinsic_size.height = 1; + last_node.intrinsic_size_set = true; + } + + self.block_end(); + } + + /// Sets the overflow behavior of the current label. + pub fn attr_overflow(&mut self, overflow: Overflow) { + let mut last_node = self.tree.last_node.borrow_mut(); + let NodeContent::Text(content) = &mut last_node.content else { + return; + }; + + content.overflow = overflow; + } + + /// Creates a button with the given text. + /// Returns true if the button was activated. + pub fn button(&mut self, classname: &'static str, text: &str) -> bool { + self.styled_label_begin(classname); + self.attr_focusable(); + if self.is_focused() { + self.attr_reverse(); + } + self.styled_label_add_text("["); + self.styled_label_add_text(text); + self.styled_label_add_text("]"); + self.styled_label_end(); + + self.button_activated() + } + + /// Creates a checkbox with the given text. + /// Returns true if the checkbox was activated. + pub fn checkbox(&mut self, classname: &'static str, text: &str, checked: &mut bool) -> bool { + self.styled_label_begin(classname); + self.attr_focusable(); + if self.is_focused() { + self.attr_reverse(); + } + self.styled_label_add_text(if *checked { "[▣ " } else { "[☐ " }); + self.styled_label_add_text(text); + self.styled_label_add_text("]"); + self.styled_label_end(); + + let activated = self.button_activated(); + if activated { + *checked = !*checked; + } + activated + } + + fn button_activated(&mut self) -> bool { + if !self.input_consumed + && ((self.input_mouse_click != 0 && self.contains_mouse_down()) + || self.input_keyboard == Some(vk::RETURN) + || self.input_keyboard == Some(vk::SPACE)) + && self.is_focused() + { + self.set_input_consumed(); + true + } else { + false + } + } + + /// Creates a text input field. + /// Returns true if the text contents changed. + pub fn editline<'s, 'b: 's>( + &'s mut self, + classname: &'static str, + text: &'b mut dyn WriteableDocument, + ) -> bool { + self.textarea_internal(classname, TextBufferPayload::Editline(text)) + } + + /// Creates a text area. + pub fn textarea(&mut self, classname: &'static str, tb: RcTextBuffer) { + self.textarea_internal(classname, TextBufferPayload::Textarea(tb)); + } + + fn textarea_internal(&mut self, classname: &'static str, payload: TextBufferPayload) -> bool { + self.block_begin(classname); + self.block_end(); + + let mut node = self.tree.last_node.borrow_mut(); + let node = &mut *node; + let single_line = match &payload { + TextBufferPayload::Editline(_) => true, + TextBufferPayload::Textarea(_) => false, + }; + + let buffer = { + let buffers = &mut self.tui.cached_text_buffers; + + let cached = match buffers.iter_mut().find(|t| t.node_id == node.id) { + Some(cached) => { + if let TextBufferPayload::Textarea(tb) = &payload { + cached.editor = tb.clone(); + }; + cached.seen = true; + cached + } + None => { + // If the node is not in the cache, we need to create a new one. + buffers.push(CachedTextBuffer { + node_id: node.id, + editor: match &payload { + TextBufferPayload::Editline(_) => TextBuffer::new_rc(true).unwrap(), + TextBufferPayload::Textarea(tb) => tb.clone(), + }, + seen: true, + }); + buffers.last_mut().unwrap() + } + }; + + // SAFETY: *Assuming* that there are no duplicate node IDs in the tree that + // would cause this cache slot to be overwritten, then this operation is safe. + // The text buffer cache will keep the buffer alive for us long enough. + unsafe { mem::transmute(&*cached.editor) } + }; + + node.content = NodeContent::Textarea(TextareaContent { + buffer, + scroll_offset: Default::default(), + scroll_offset_y_drag_start: CoordType::MIN, + scroll_offset_x_max: 0, + thumb_height: 0, + preferred_column: 0, + single_line, + has_focus: self.tui.is_node_focused(node.id), + }); + + let content = match node.content { + NodeContent::Textarea(ref mut content) => content, + _ => unreachable!(), + }; + + if let TextBufferPayload::Editline(text) = &payload { + content.buffer.borrow_mut().copy_from_str(*text); + } + + if let Some(node_prev) = self.tui.prev_node_map.get(node.id) { + let node_prev = node_prev.borrow(); + if let NodeContent::Textarea(content_prev) = &node_prev.content { + content.scroll_offset = content_prev.scroll_offset; + content.scroll_offset_y_drag_start = content_prev.scroll_offset_y_drag_start; + content.scroll_offset_x_max = content_prev.scroll_offset_x_max; + content.thumb_height = content_prev.thumb_height; + content.preferred_column = content_prev.preferred_column; + + let mut text_width = node_prev.inner.width(); + if !single_line { + // Subtract -1 to account for the scrollbar. + text_width -= 1; + } + + let mut make_cursor_visible; + { + let mut tb = content.buffer.borrow_mut(); + make_cursor_visible = tb.take_cursor_visibility_request(); + make_cursor_visible |= tb.set_width(text_width); + } + + make_cursor_visible |= self.textarea_handle_input(content, &node_prev, single_line); + + if make_cursor_visible { + self.textarea_make_cursor_visible(content, &node_prev); + } + } else { + debug_assert!(false); + } + } + + let dirty; + { + let mut tb = content.buffer.borrow_mut(); + dirty = tb.is_dirty(); + if dirty && let TextBufferPayload::Editline(text) = payload { + tb.save_as_string(text); + } + } + + self.textarea_adjust_scroll_offset(content); + + if single_line { + node.attributes.fg = self.indexed(IndexedColor::Foreground); + node.attributes.bg = self.indexed(IndexedColor::Background); + if !content.has_focus { + node.attributes.fg = self.contrasted(node.attributes.bg); + node.attributes.bg = self.indexed_alpha(IndexedColor::Background, 1, 2); + } + } + + node.attributes.focusable = true; + node.intrinsic_size.height = content.buffer.borrow().visual_line_count(); + node.intrinsic_size_set = true; + + dirty + } + + fn textarea_handle_input( + &mut self, + tc: &mut TextareaContent, + node_prev: &Node, + single_line: bool, + ) -> bool { + if self.input_consumed { + return false; + } + + let mut tb = tc.buffer.borrow_mut(); + let tb = &mut *tb; + let mut make_cursor_visible = false; + + if self.tui.mouse_state != InputMouseState::None + && self.tui.was_mouse_down_on_node(node_prev.id) + { + // Scrolling works even if the node isn't focused. + if self.tui.mouse_state == InputMouseState::Scroll { + tc.scroll_offset.x += self.input_scroll_delta.x; + tc.scroll_offset.y += self.input_scroll_delta.y; + self.set_input_consumed(); + } else if self.tui.is_node_focused(node_prev.id) { + let mouse = self.tui.mouse_position; + let inner = node_prev.inner; + let text_rect = Rect { + left: inner.left + tb.margin_width(), + top: inner.top, + right: inner.right - !single_line as CoordType, + bottom: inner.bottom, + }; + let track_rect = Rect { + left: text_rect.right, + top: inner.top, + right: inner.right, + bottom: inner.bottom, + }; + let pos = Point { + x: mouse.x - inner.left - tb.margin_width() + tc.scroll_offset.x, + y: mouse.y - inner.top + tc.scroll_offset.y, + }; + + if text_rect.contains(self.tui.mouse_down_position) { + if self.tui.mouse_is_drag { + tb.selection_update_visual(pos); + tc.preferred_column = tb.cursor_visual_pos().x; + + let height = inner.height(); + + // If the editor is only 1 line tall we can't possibly scroll up or down. + if height >= 2 { + fn calc(min: CoordType, max: CoordType, mouse: CoordType) -> CoordType { + // Otherwise, the scroll zone is up to 3 lines at the top/bottom. + let zone_height = ((max - min) / 2).min(3); + + // The .y positions where the scroll zones begin: + // Mouse coordinates above top and below bottom respectively. + let scroll_min = min + zone_height; + let scroll_max = max - zone_height - 1; + + // Calculate the delta for scrolling up or down. + let delta_min = (mouse - scroll_min).clamp(-zone_height, 0); + let delta_max = (mouse - scroll_max).clamp(0, zone_height); + + // If I didn't mess up my logic here, only one of the two values can possibly be !=0. + let idx = 3 + delta_min + delta_max; + + const SPEEDS: [CoordType; 7] = [-9, -3, -1, 0, 1, 3, 9]; + let idx = idx.clamp(0, SPEEDS.len() as CoordType) as usize; + SPEEDS[idx] + } + + let delta_x = calc(text_rect.left, text_rect.right, mouse.x); + let delta_y = calc(text_rect.top, text_rect.bottom, mouse.y); + + tc.scroll_offset.x += delta_x; + tc.scroll_offset.y += delta_y; + + if delta_x != 0 || delta_y != 0 { + self.tui.read_timeout = time::Duration::from_millis(25); + } + } + } else { + match self.input_mouse_click { + 5.. => {} + 4 => tb.select_all(), + 3 => tb.select_line(), + 2 => tb.select_word(), + _ => match self.tui.mouse_state { + InputMouseState::Left => { + if self.input_mouse_modifiers.contains(kbmod::SHIFT) { + // TODO: Untested because Windows Terminal surprisingly doesn't support Shift+Click. + tb.selection_update_visual(pos); + } else { + tb.cursor_move_to_visual(pos); + } + tc.preferred_column = tb.cursor_visual_pos().x; + make_cursor_visible = true; + } + _ => return false, + }, + } + } + } else if track_rect.contains(self.tui.mouse_down_position) { + if self.tui.mouse_state == InputMouseState::Release { + tc.scroll_offset_y_drag_start = CoordType::MIN; + } else if self.tui.mouse_is_drag { + if tc.scroll_offset_y_drag_start == CoordType::MIN { + tc.scroll_offset_y_drag_start = tc.scroll_offset.y; + } + + // The textarea supports 1 height worth of "scrolling beyond the end". + // `track_height` is the same as the viewport height. + let scrollable_height = tb.visual_line_count() - 1; + + if scrollable_height > 0 { + let trackable = track_rect.height() - tc.thumb_height; + let delta_y = mouse.y - self.tui.mouse_down_position.y; + tc.scroll_offset.y = tc.scroll_offset_y_drag_start + + ((delta_y * scrollable_height) / trackable); + } + } + } + + self.set_input_consumed(); + } + + return make_cursor_visible; + } + + if !tc.has_focus { + return false; + } + + let mut write: &[u8] = b""; + let mut write_raw = false; + + if let Some(input) = &self.input_text { + write = input.text.as_bytes(); + write_raw = input.bracketed; + tc.preferred_column = tb.cursor_visual_pos().x; + make_cursor_visible = true; + } else if let Some(input) = &self.input_keyboard { + let key = input.key(); + let modifiers = input.modifiers(); + + make_cursor_visible = true; + + match key { + vk::BACK => { + let granularity = if modifiers == kbmod::CTRL { + CursorMovement::Word + } else { + CursorMovement::Grapheme + }; + tb.delete(granularity, -1); + } + vk::TAB => { + if single_line { + // If this is just a simple input field, don't consume Tab (= early return). + return false; + } + if modifiers == kbmod::SHIFT { + tb.unindent(); + } else { + write = b"\t"; + } + } + vk::RETURN => { + if single_line { + // If this is just a simple input field, don't consume Enter (= early return). + return false; + } + write = b"\n"; + } + vk::ESCAPE => { + // If there was a selection, clear it and show the cursor (= fallthrough). + if !tb.clear_selection() { + if single_line { + // If this is just a simple input field, don't consume the escape key + // (early return) and don't show the cursor (= return false). + return false; + } + + // If this is a textarea, don't show the cursor if + // the escape key was pressed and nothing happened. + make_cursor_visible = false; + } + } + vk::PRIOR => { + let height = node_prev.inner.height() - 1; + + // If the cursor was already on the first line, + // move it to the start of the buffer. + if tb.cursor_visual_pos().y == 0 { + tc.preferred_column = 0; + } + + if modifiers == kbmod::SHIFT { + tb.selection_update_visual(Point { + x: tc.preferred_column, + y: tb.cursor_visual_pos().y - height, + }); + } else { + tb.cursor_move_to_visual(Point { + x: tc.preferred_column, + y: tb.cursor_visual_pos().y - height, + }); + } + } + vk::NEXT => { + let height = node_prev.inner.height() - 1; + + // If the cursor was already on the last line, + // move it to the end of the buffer. + if tb.cursor_visual_pos().y >= tb.visual_line_count() - 1 { + tc.preferred_column = CoordType::MAX; + } + + if modifiers == kbmod::SHIFT { + tb.selection_update_visual(Point { + x: tc.preferred_column, + y: tb.cursor_visual_pos().y + height, + }); + } else { + tb.cursor_move_to_visual(Point { + x: tc.preferred_column, + y: tb.cursor_visual_pos().y + height, + }); + } + + if tc.preferred_column == CoordType::MAX { + tc.preferred_column = tb.cursor_visual_pos().x; + } + } + vk::END => { + let logical_before = tb.cursor_logical_pos(); + let destination = if modifiers.contains(kbmod::CTRL) { + Point::MAX + } else { + Point { x: CoordType::MAX, y: tb.cursor_visual_pos().y } + }; + + if modifiers.contains(kbmod::SHIFT) { + tb.selection_update_visual(destination); + } else { + tb.cursor_move_to_visual(destination); + } + + if !modifiers.contains(kbmod::CTRL) { + let logical_after = tb.cursor_logical_pos(); + + // If word-wrap is enabled and the user presses End the first time, + // it moves to the start of the visual line. The second time they + // press it, it moves to the start of the logical line. + if tb.is_word_wrap_enabled() && logical_after == logical_before { + if modifiers == kbmod::SHIFT { + tb.selection_update_logical(Point { + x: CoordType::MAX, + y: tb.cursor_logical_pos().y, + }); + } else { + tb.cursor_move_to_logical(Point { + x: CoordType::MAX, + y: tb.cursor_logical_pos().y, + }); + } + } + } + } + vk::HOME => { + let logical_before = tb.cursor_logical_pos(); + let destination = if modifiers.contains(kbmod::CTRL) { + Default::default() + } else { + Point { x: 0, y: tb.cursor_visual_pos().y } + }; + + if modifiers.contains(kbmod::SHIFT) { + tb.selection_update_visual(destination); + } else { + tb.cursor_move_to_visual(destination); + } + + if !modifiers.contains(kbmod::CTRL) { + let mut logical_after = tb.cursor_logical_pos(); + + // If word-wrap is enabled and the user presses Home the first time, + // it moves to the start of the visual line. The second time they + // press it, it moves to the start of the logical line. + if tb.is_word_wrap_enabled() && logical_after == logical_before { + if modifiers == kbmod::SHIFT { + tb.selection_update_logical(Point { + x: 0, + y: tb.cursor_logical_pos().y, + }); + } else { + tb.cursor_move_to_logical(Point { + x: 0, + y: tb.cursor_logical_pos().y, + }); + } + logical_after = tb.cursor_logical_pos(); + } + + // If the line has some indentation and the user pressed Home, + // the first time it'll stop at the indentation. The second time + // they press it, it'll move to the true start of the line. + let indent_end = tb.indent_end_logical_pos(); + if logical_after.x == 0 && logical_before > indent_end { + if modifiers == kbmod::SHIFT { + tb.selection_update_logical(indent_end); + } else { + tb.cursor_move_to_logical(indent_end); + } + } + } + } + vk::LEFT => { + let granularity = if modifiers.contains(kbmod::CTRL) { + CursorMovement::Word + } else { + CursorMovement::Grapheme + }; + if modifiers.contains(kbmod::SHIFT) { + tb.selection_update_delta(granularity, -1); + } else if let Some((beg, _)) = tb.selection_range() { + unsafe { tb.set_cursor(beg) }; + } else { + tb.cursor_move_delta(granularity, -1); + } + } + vk::UP => { + match modifiers { + kbmod::NONE => { + let mut x = tc.preferred_column; + let mut y = tb.cursor_visual_pos().y - 1; + + // If there's a selection we put the cursor above it. + if let Some((beg, _)) = tb.selection_range() { + x = beg.visual_pos.x; + y = beg.visual_pos.y - 1; + tc.preferred_column = x; + } + + // If the cursor was already on the first line, + // move it to the start of the buffer. + if y < 0 { + x = 0; + tc.preferred_column = 0; + } + + tb.cursor_move_to_visual(Point { x, y }); + } + kbmod::CTRL => { + tc.scroll_offset.y -= 1; + make_cursor_visible = false; + } + kbmod::SHIFT => { + // If the cursor was already on the first line, + // move it to the start of the buffer. + if tb.cursor_visual_pos().y == 0 { + tc.preferred_column = 0; + } + + tb.selection_update_visual(Point { + x: tc.preferred_column, + y: tb.cursor_visual_pos().y - 1, + }); + } + kbmod::CTRL_ALT => { + // TODO: Add cursor above + } + _ => return false, + } + } + vk::RIGHT => { + let granularity = if modifiers.contains(kbmod::CTRL) { + CursorMovement::Word + } else { + CursorMovement::Grapheme + }; + if modifiers.contains(kbmod::SHIFT) { + tb.selection_update_delta(granularity, 1); + } else if let Some((_, end)) = tb.selection_range() { + unsafe { tb.set_cursor(end) }; + } else { + tb.cursor_move_delta(granularity, 1); + } + } + vk::DOWN => match modifiers { + kbmod::NONE => { + let mut x = tc.preferred_column; + let mut y = tb.cursor_visual_pos().y + 1; + + // If there's a selection we put the cursor below it. + if let Some((_, end)) = tb.selection_range() { + x = end.visual_pos.x; + y = end.visual_pos.y + 1; + tc.preferred_column = x; + } + + // If the cursor was already on the last line, + // move it to the end of the buffer. + if y >= tb.visual_line_count() { + x = CoordType::MAX; + } + + tb.cursor_move_to_visual(Point { x, y }); + + // If we fell into the `if y >= tb.get_visual_line_count()` above, we wanted to + // update the `preferred_column` but didn't know yet what it was. Now we know! + if x == CoordType::MAX { + tc.preferred_column = tb.cursor_visual_pos().x; + } + } + kbmod::CTRL => { + tc.scroll_offset.y += 1; + make_cursor_visible = false; + } + kbmod::SHIFT => { + // If the cursor was already on the last line, + // move it to the end of the buffer. + if tb.cursor_visual_pos().y >= tb.visual_line_count() - 1 { + tc.preferred_column = CoordType::MAX; + } + + tb.selection_update_visual(Point { + x: tc.preferred_column, + y: tb.cursor_visual_pos().y + 1, + }); + + if tc.preferred_column == CoordType::MAX { + tc.preferred_column = tb.cursor_visual_pos().x; + } + } + kbmod::CTRL_ALT => { + // TODO: Add cursor above + } + _ => return false, + }, + vk::INSERT => match modifiers { + kbmod::SHIFT => { + write = &self.tui.clipboard; + write_raw = true; + } + kbmod::CTRL => self.set_clipboard(tb.extract_selection(false)), + _ => tb.set_overtype(!tb.is_overtype()), + }, + vk::DELETE => match modifiers { + kbmod::SHIFT => self.set_clipboard(tb.extract_selection(true)), + kbmod::CTRL => tb.delete(CursorMovement::Word, 1), + _ => tb.delete(CursorMovement::Grapheme, 1), + }, + vk::A => match modifiers { + kbmod::CTRL => tb.select_all(), + _ => return false, + }, + vk::H => match modifiers { + kbmod::CTRL => tb.delete(CursorMovement::Word, -1), + _ => return false, + }, + vk::X => match modifiers { + kbmod::CTRL => self.set_clipboard(tb.extract_selection(true)), + _ => return false, + }, + vk::C => match modifiers { + kbmod::CTRL => self.set_clipboard(tb.extract_selection(false)), + _ => return false, + }, + vk::V => match modifiers { + kbmod::CTRL => { + write = &self.tui.clipboard; + write_raw = true; + } + _ => return false, + }, + vk::Y => match modifiers { + kbmod::CTRL => tb.redo(), + _ => return false, + }, + vk::Z => match modifiers { + kbmod::CTRL => tb.undo(), + kbmod::CTRL_SHIFT => tb.redo(), + kbmod::ALT => tb.set_word_wrap(!tb.is_word_wrap_enabled()), + _ => return false, + }, + _ => return false, + } + + if !matches!(key, vk::PRIOR | vk::NEXT | vk::UP | vk::DOWN) { + tc.preferred_column = tb.cursor_visual_pos().x; + } + } else { + return false; + } + + if single_line && !write.is_empty() { + let (end, _) = unicode::newlines_forward(write, 0, 0, 1); + write = unicode::strip_newline(&write[..end]); + } + if !write.is_empty() { + tb.write(write, write_raw); + } + + self.set_input_consumed(); + make_cursor_visible + } + + fn textarea_make_cursor_visible(&self, tc: &mut TextareaContent, node_prev: &Node) { + let tb = tc.buffer.borrow(); + let mut scroll_x = tc.scroll_offset.x; + let mut scroll_y = tc.scroll_offset.y; + + let text_width = tb.text_width(); + let cursor_x = tb.cursor_visual_pos().x; + scroll_x = scroll_x.min(cursor_x - 10); + scroll_x = scroll_x.max(cursor_x - text_width + 10); + + let viewport_height = node_prev.inner.height(); + let cursor_y = tb.cursor_visual_pos().y; + // Scroll up if the cursor is above the visible area. + scroll_y = scroll_y.min(cursor_y); + // Scroll down if the cursor is below the visible area. + scroll_y = scroll_y.max(cursor_y - viewport_height + 1); + + tc.scroll_offset.x = scroll_x; + tc.scroll_offset.y = scroll_y; + } + + fn textarea_adjust_scroll_offset(&self, tc: &mut TextareaContent) { + let tb = tc.buffer.borrow(); + let mut scroll_x = tc.scroll_offset.x; + let mut scroll_y = tc.scroll_offset.y; + + scroll_x = scroll_x.min(tc.scroll_offset_x_max.max(tb.cursor_visual_pos().x) - 10); + scroll_x = scroll_x.max(0); + scroll_y = scroll_y.clamp(0, tb.visual_line_count() - 1); + + if tb.is_word_wrap_enabled() { + scroll_x = 0; + } + + tc.scroll_offset.x = scroll_x; + tc.scroll_offset.y = scroll_y; + } + + /// Creates a scrollable area. + pub fn scrollarea_begin(&mut self, classname: &'static str, intrinsic_size: Size) { + self.block_begin(classname); + + let container_node = self.tree.last_node; + { + let mut container = self.tree.last_node.borrow_mut(); + container.content = NodeContent::Scrollarea(ScrollareaContent { + scroll_offset: Point::MIN, + scroll_offset_y_drag_start: CoordType::MIN, + thumb_height: 0, + }); + + if intrinsic_size.width > 0 || intrinsic_size.height > 0 { + container.intrinsic_size.width = intrinsic_size.width.max(0); + container.intrinsic_size.height = intrinsic_size.height.max(0); + container.intrinsic_size_set = true; + } + } + + self.block_begin("content"); + self.inherit_focus(); + + // Ensure that attribute modifications apply to the outer container. + self.tree.last_node = container_node; + } + + /// Scrolls the current scrollable area to the given position. + pub fn scrollarea_scroll_to(&mut self, pos: Point) { + let mut container = self.tree.last_node.borrow_mut(); + if let NodeContent::Scrollarea(sc) = &mut container.content { + sc.scroll_offset = pos; + } else { + debug_assert!(false); + } + } + + /// Ends the current scrollarea block. + pub fn scrollarea_end(&mut self) { + self.block_end(); // content block + self.block_end(); // outer container + + let mut container = self.tree.last_node.borrow_mut(); + let container_id = container.id; + let container_depth = container.depth; + let Some(prev_container) = self.tui.prev_node_map.get(container_id) else { + return; + }; + + let prev_container = prev_container.borrow(); + let NodeContent::Scrollarea(sc) = &mut container.content else { + unreachable!(); + }; + + if sc.scroll_offset == Point::MIN + && let NodeContent::Scrollarea(sc_prev) = &prev_container.content + { + *sc = sc_prev.clone(); + } + + if !self.input_consumed { + if self.tui.mouse_state != InputMouseState::None { + let container_rect = prev_container.inner; + + match self.tui.mouse_state { + InputMouseState::Left => { + if self.tui.mouse_is_drag { + // We don't need to look up the previous track node, + // since it has a fixed size based on the container size. + let track_rect = Rect { + left: container_rect.right, + top: container_rect.top, + right: container_rect.right + 1, + bottom: container_rect.bottom, + }; + if track_rect.contains(self.tui.mouse_down_position) { + if sc.scroll_offset_y_drag_start == CoordType::MIN { + sc.scroll_offset_y_drag_start = sc.scroll_offset.y; + } + + let content = prev_container.children.first.unwrap().borrow(); + let content_rect = content.inner; + let content_height = content_rect.height(); + let track_height = track_rect.height(); + let scrollable_height = content_height - track_height; + + if scrollable_height > 0 { + let trackable = track_height - sc.thumb_height; + let delta_y = + self.tui.mouse_position.y - self.tui.mouse_down_position.y; + sc.scroll_offset.y = sc.scroll_offset_y_drag_start + + ((delta_y * scrollable_height) / trackable); + } + + self.set_input_consumed(); + } + } + } + InputMouseState::Release => { + sc.scroll_offset_y_drag_start = CoordType::MIN; + } + InputMouseState::Scroll => { + if container_rect.contains(self.tui.mouse_position) { + sc.scroll_offset.x += self.input_scroll_delta.x; + sc.scroll_offset.y += self.input_scroll_delta.y; + self.set_input_consumed(); + } + } + _ => {} + } + } else if self.tui.is_subtree_focused_alt(container_id, container_depth) + && let Some(key) = self.input_keyboard + { + match key { + vk::PRIOR => sc.scroll_offset.y -= prev_container.inner_clipped.height(), + vk::NEXT => sc.scroll_offset.y += prev_container.inner_clipped.height(), + vk::END => sc.scroll_offset.y = CoordType::MAX, + vk::HOME => sc.scroll_offset.y = 0, + _ => return, + } + self.set_input_consumed(); + } + } + } + + /// Creates a list where exactly one item is selected. + pub fn list_begin(&mut self, classname: &'static str) { + self.block_begin(classname); + self.attr_focusable(); + + let mut last_node = self.tree.last_node.borrow_mut(); + let content = self + .tui + .prev_node_map + .get(last_node.id) + .and_then(|node| match &node.borrow().content { + NodeContent::List(content) => { + Some(ListContent { selected: content.selected, selected_node: None }) + } + _ => None, + }) + .unwrap_or(ListContent { selected: 0, selected_node: None }); + + last_node.attributes.focus_void = true; + last_node.content = NodeContent::List(content); + } + + /// Creates a list item with the given text. + pub fn list_item(&mut self, select: bool, text: &str) -> ListSelection { + self.styled_list_item_begin(); + self.styled_label_add_text(text); + self.styled_list_item_end(select) + } + + /// Creates a list item consisting of a styled label. + /// See [`Context::styled_label_begin`]. + pub fn styled_list_item_begin(&mut self) { + let list = self.tree.current_node; + let idx = list.borrow().child_count; + + self.next_block_id_mixin(idx as u64); + self.styled_label_begin("item"); + self.styled_label_add_text(" "); + self.attr_focusable(); + } + + /// Ends the current styled list item. + pub fn styled_list_item_end(&mut self, select: bool) -> ListSelection { + self.styled_label_end(); + + let list = self.tree.current_node; + + let selected_before; + let selected_now; + let focused; + { + let mut list = list.borrow_mut(); + let content = match &mut list.content { + NodeContent::List(content) => content, + _ => unreachable!(), + }; + + let item = self.tree.last_node.borrow(); + let item_id = item.id; + selected_before = content.selected == item_id; + focused = self.is_focused(); + + // Inherit the default selection & Click changes selection + selected_now = selected_before || (select && content.selected == 0) || focused; + + // Note down the selected node for keyboard navigation. + if selected_now { + content.selected_node = Some(self.tree.last_node); + if !selected_before { + content.selected = item_id; + self.needs_rerender(); + } + } + } + + // Clicking an item activates it + let clicked = + !self.input_consumed && (self.input_mouse_click == 2 && self.was_mouse_down()); + // Pressing Enter on a selected item activates it as well + let entered = focused + && selected_before + && !self.input_consumed + && matches!(self.input_keyboard, Some(vk::RETURN)); + let activated = clicked || entered; + if activated { + self.set_input_consumed(); + } + + if selected_before && activated { + ListSelection::Activated + } else if selected_now && !selected_before { + ListSelection::Selected + } else { + ListSelection::Unchanged + } + } + + /// Ends the current list block. + pub fn list_end(&mut self) { + self.block_end(); + + let contains_focus; + let selected_now; + let mut selected_next; + { + let list = self.tree.last_node.borrow(); + + contains_focus = self.tui.is_subtree_focused(&list); + selected_now = match &list.content { + NodeContent::List(content) => content.selected_node, + _ => unreachable!(), + }; + selected_next = match selected_now.or(list.children.first) { + Some(node) => node, + None => return, + }; + } + + if contains_focus + && !self.input_consumed + && let Some(key) = self.input_keyboard + && let Some(selected_now) = selected_now + { + let list = self.tree.last_node.borrow(); + + if let Some(prev_container) = self.tui.prev_node_map.get(list.id) { + let mut consumed = true; + + match key { + vk::PRIOR => { + selected_next = selected_now; + for _ in 0..prev_container.borrow().inner_clipped.height() - 1 { + let node = selected_next.borrow(); + selected_next = match node.siblings.prev { + Some(node) => node, + None => break, + }; + } + } + vk::NEXT => { + selected_next = selected_now; + for _ in 0..prev_container.borrow().inner_clipped.height() - 1 { + let node = selected_next.borrow(); + selected_next = match node.siblings.next { + Some(node) => node, + None => break, + }; + } + } + vk::END => { + selected_next = list.children.last.unwrap_or(selected_next); + } + vk::HOME => { + selected_next = list.children.first.unwrap_or(selected_next); + } + vk::UP => { + selected_next = selected_now + .borrow() + .siblings + .prev + .or(list.children.last) + .unwrap_or(selected_next); + } + vk::DOWN => { + selected_next = selected_now + .borrow() + .siblings + .next + .or(list.children.first) + .unwrap_or(selected_next); + } + _ => consumed = false, + } + + if consumed { + self.set_input_consumed(); + } + } + } + + // Now that we know which item is selected we can mark it as such. + if !opt_ptr_eq(selected_now, Some(selected_next)) + && let NodeContent::List(content) = &mut self.tree.last_node.borrow_mut().content + { + content.selected_node = Some(selected_next); + } + + // Now that we know which item is selected we can mark it as such. + if let NodeContent::Text(content) = &mut selected_next.borrow_mut().content { + unsafe { + content.text.as_bytes_mut()[0] = b'>'; + } + } + + // If the list has focus, we also delegate focus to the selected item and colorize it. + if contains_focus { + { + let mut node = selected_next.borrow_mut(); + node.attributes.bg = self.indexed(IndexedColor::Green); + node.attributes.fg = self.contrasted(self.indexed(IndexedColor::Green)); + } + self.steal_focus_for(selected_next); + } + } + + /// Creates a menubar, to be shown at the top of the screen. + pub fn menubar_begin(&mut self) { + self.table_begin("menubar"); + self.attr_focus_well(); + self.table_next_row(); + } + + /// Appends a menu to the current menubar. + /// + /// Returns true if the menu is open. Continue appending items to it in that case. + pub fn menubar_menu_begin(&mut self, text: &str, accelerator: char) -> bool { + let mixin = self.tree.current_node.borrow().child_count as u64; + self.next_block_id_mixin(mixin); + + self.menubar_label(text, accelerator, None); + self.attr_focusable(); + self.attr_padding(Rect::two(0, 1)); + + let contains_focus = self.contains_focus(); + let keyboard_focus = !contains_focus + && self.consume_shortcut(kbmod::ALT | InputKey::new(accelerator as u32)); + + if contains_focus || keyboard_focus { + self.attr_background_rgba(self.tui.floater_default_bg); + self.attr_foreground_rgba(self.tui.floater_default_fg); + + if self.is_focused() { + self.attr_background_rgba(self.indexed(IndexedColor::Green)); + self.attr_foreground_rgba(self.contrasted(self.indexed(IndexedColor::Green))); + } + + self.next_block_id_mixin(mixin); + self.table_begin("flyout"); + self.attr_float(FloatSpec { + anchor: Anchor::Last, + gravity_x: 0.0, + gravity_y: 0.0, + offset_x: 0.0, + offset_y: 1.0, + }); + self.attr_border(); + self.attr_focus_well(); + + if keyboard_focus { + self.steal_focus(); + } + + true + } else { + false + } + } + + /// Appends a button to the current menu. + pub fn menubar_menu_button( + &mut self, + text: &str, + accelerator: char, + shortcut: InputKey, + ) -> bool { + self.menubar_menu_checkbox(text, accelerator, shortcut, false) + } + + /// Appends a checkbox to the current menu. + /// Returns true if the checkbox was activated. + pub fn menubar_menu_checkbox( + &mut self, + text: &str, + accelerator: char, + shortcut: InputKey, + checked: bool, + ) -> bool { + self.table_next_row(); + self.attr_focusable(); + + // First menu item? Steal focus. + if self.tree.current_node.borrow_mut().siblings.prev.is_none() { + self.inherit_focus(); + } + + if self.is_focused() { + self.attr_background_rgba(self.indexed(IndexedColor::Green)); + self.attr_foreground_rgba(self.contrasted(self.indexed(IndexedColor::Green))); + } + + let clicked = + self.button_activated() || self.consume_shortcut(InputKey::new(accelerator as u32)); + + self.menubar_label(text, accelerator, Some(checked)); + self.menubar_shortcut(shortcut); + + if clicked { + // TODO: This should reassign the previous focused path. + self.needs_rerender(); + Tui::clean_node_path(&mut self.tui.focused_node_path); + } + + clicked + } + + /// Ends the current menu. + pub fn menubar_menu_end(&mut self) { + self.table_end(); + + if !self.input_consumed + && let Some(key) = self.input_keyboard + && matches!(key, vk::ESCAPE | vk::UP | vk::DOWN | vk::LEFT | vk::RIGHT) + { + if matches!(key, vk::UP | vk::DOWN) { + let ln = self.tree.last_node.borrow(); + if self.tui.is_node_focused(ln.parent.map_or(0, |n| n.borrow().id)) { + let selected_next = + if key == vk::UP { ln.children.last } else { ln.children.first }; + if let Some(selected_next) = selected_next { + self.steal_focus_for(selected_next); + self.set_input_consumed(); + } + } + } else if self.contains_focus() { + if key == vk::ESCAPE { + // TODO: This should reassign the previous focused path. + self.needs_rerender(); + self.set_input_consumed(); + Tui::clean_node_path(&mut self.tui.focused_node_path); + } else if !self.is_focused() { + self.tui.pop_focusable_node(2); + } + } + } + } + + /// Ends the current menubar. + pub fn menubar_end(&mut self) { + self.table_end(); + } + + fn menubar_label(&mut self, text: &str, accelerator: char, checked: Option) { + if !accelerator.is_ascii_uppercase() { + self.label("label", text); + return; + } + + let mut off = text.len(); + + for (i, c) in text.bytes().enumerate() { + // Perfect match (uppercase character) --> stop + if c as char == accelerator { + off = i; + break; + } + // Inexact match (lowercase character) --> use first hit + if (c & !0x20) as char == accelerator && off == text.len() { + off = i; + } + } + + self.styled_label_begin("label"); + if let Some(checked) = checked { + self.styled_label_add_text(if checked { "▣ " } else { " " }); + } + + if off < text.len() { + // Add an underline to the accelerator. + self.styled_label_add_text(&text[..off]); + self.styled_label_set_attributes(Attributes::Underlined); + self.styled_label_add_text(&text[off..off + 1]); + self.styled_label_set_attributes(Attributes::None); + self.styled_label_add_text(&text[off + 1..]); + } else { + // Add the accelerator in parentheses and underline it. + let ch = accelerator as u8; + self.styled_label_add_text(text); + self.styled_label_add_text("("); + self.styled_label_set_attributes(Attributes::Underlined); + self.styled_label_add_text(unsafe { str_from_raw_parts(&ch, 1) }); + self.styled_label_set_attributes(Attributes::None); + self.styled_label_add_text(")"); + } + + self.styled_label_end(); + self.attr_padding(Rect { left: 0, top: 0, right: 2, bottom: 0 }); + } + + fn menubar_shortcut(&mut self, shortcut: InputKey) { + let shortcut_letter = shortcut.value() as u8 as char; + if shortcut_letter.is_ascii_uppercase() { + let mut shortcut_text = ArenaString::new_in(self.arena()); + if shortcut.modifiers_contains(kbmod::CTRL) { + shortcut_text.push_str(self.tui.modifier_translations.ctrl); + shortcut_text.push('+'); + } + if shortcut.modifiers_contains(kbmod::ALT) { + shortcut_text.push_str(self.tui.modifier_translations.alt); + shortcut_text.push('+'); + } + if shortcut.modifiers_contains(kbmod::SHIFT) { + shortcut_text.push_str(self.tui.modifier_translations.shift); + shortcut_text.push('+'); + } + shortcut_text.push(shortcut_letter); + + self.label("shortcut", &shortcut_text); + } else { + self.block_begin("shortcut"); + self.block_end(); + } + self.attr_padding(Rect { left: 0, top: 0, right: 2, bottom: 0 }); + } +} + +/// See [`Tree::visit_all`]. +#[derive(Clone, Copy)] +enum VisitControl { + Continue, + SkipChildren, + Stop, +} + +/// Stores the root of the "DOM" tree of the UI. +struct Tree<'a> { + tail: &'a NodeCell<'a>, + root_first: &'a NodeCell<'a>, + root_last: &'a NodeCell<'a>, + last_node: &'a NodeCell<'a>, + current_node: &'a NodeCell<'a>, + + count: usize, + checksum: u64, +} + +impl<'a> Tree<'a> { + /// Creates a new tree inside the given arena. + /// A single root node is added for the main contents. + fn new(arena: &'a Arena) -> Self { + let root = Self::alloc_node(arena); + { + let mut r = root.borrow_mut(); + r.id = ROOT_ID; + r.classname = "root"; + r.attributes.focusable = true; + r.attributes.focus_well = true; + } + Self { + tail: root, + root_first: root, + root_last: root, + last_node: root, + current_node: root, + count: 1, + checksum: ROOT_ID, + } + } + + fn alloc_node(arena: &'a Arena) -> &'a NodeCell<'a> { + arena.alloc_uninit().write(Default::default()) + } + + /// Appends a child node to the current node. + fn push_child(&mut self, node: &'a NodeCell<'a>) { + let mut n = node.borrow_mut(); + n.parent = Some(self.current_node); + n.stack_parent = Some(self.current_node); + + { + let mut p = self.current_node.borrow_mut(); + n.siblings.prev = p.children.last; + n.depth = p.depth + 1; + + if let Some(child_last) = p.children.last { + let mut child_last = child_last.borrow_mut(); + child_last.siblings.next = Some(node); + } + if p.children.first.is_none() { + p.children.first = Some(node); + } + p.children.last = Some(node); + p.child_count += 1; + } + + n.prev = Some(self.tail); + { + let mut tail = self.tail.borrow_mut(); + tail.next = Some(node); + } + self.tail = node; + + self.last_node = node; + self.current_node = node; + self.count += 1; + // wymix is weak, but both checksum and node.id are proper random, so... it's not *that* bad. + self.checksum = wymix(self.checksum, n.id); + } + + /// Removes the current node from its parent and appends it as a new root. + /// Used for [`Context::attr_float`]. + fn move_node_to_root(&mut self, node: &'a NodeCell<'a>, anchor: Option<&'a NodeCell<'a>>) { + let mut n = node.borrow_mut(); + let Some(parent) = n.parent else { + return; + }; + + if let Some(sibling_prev) = n.siblings.prev { + let mut sibling_prev = sibling_prev.borrow_mut(); + sibling_prev.siblings.next = n.siblings.next; + } + if let Some(sibling_next) = n.siblings.next { + let mut sibling_next = sibling_next.borrow_mut(); + sibling_next.siblings.prev = n.siblings.prev; + } + + { + let mut p = parent.borrow_mut(); + if opt_ptr_eq(p.children.first, Some(node)) { + p.children.first = n.siblings.next; + } + if opt_ptr_eq(p.children.last, Some(node)) { + p.children.last = n.siblings.prev; + } + p.child_count -= 1; + } + + n.parent = anchor; + n.depth = anchor.map_or(0, |n| n.borrow().depth + 1); + n.siblings.prev = Some(self.root_last); + n.siblings.next = None; + + self.root_last.borrow_mut().siblings.next = Some(node); + self.root_last = node; + } + + /// Completes the current node and moves focus to the parent. + fn pop_stack(&mut self) { + let current_node = self.current_node.borrow(); + let stack_parent = current_node.stack_parent.unwrap(); + self.last_node = self.current_node; + self.current_node = stack_parent; + } + + fn iterate_siblings( + mut node: Option<&'a NodeCell<'a>>, + ) -> impl Iterator> + use<'a> { + iter::from_fn(move || { + let n = node?; + node = n.borrow().siblings.next; + Some(n) + }) + } + + fn iterate_roots(&self) -> impl Iterator> + use<'a> { + Self::iterate_siblings(Some(self.root_first)) + } + + /// Visits all nodes under and including `root` in depth order. + /// Starts with node `start`. + /// + /// WARNING: Breaks in hilarious ways if `start` is not within `root`. + fn visit_all) -> VisitControl>( + root: &'a NodeCell<'a>, + start: &'a NodeCell<'a>, + forward: bool, + mut cb: T, + ) { + let root_depth = root.borrow().depth; + let mut node = start; + let children_idx = if forward { NodeChildren::FIRST } else { NodeChildren::LAST }; + let siblings_idx = if forward { NodeSiblings::NEXT } else { NodeSiblings::PREV }; + + while { + 'traverse: { + match cb(node) { + VisitControl::Continue => { + // Depth first search: It has a child? Go there. + if let Some(child) = node.borrow().children.get(children_idx) { + node = child; + break 'traverse; + } + } + VisitControl::SkipChildren => {} + VisitControl::Stop => return, + } + + loop { + // If we hit the root while going up, we restart the traversal at + // `root` going down again until we hit `start` again. + let n = node.borrow(); + if n.depth <= root_depth { + break 'traverse; + } + + // Go to the parent's next sibling. --> Next subtree. + if let Some(sibling) = n.siblings.get(siblings_idx) { + node = sibling; + break; + } + + // Out of children? Go back to the parent. + node = n.parent.unwrap(); + } + } + + // We're done once we wrapped around to the `start`. + !ptr::eq(node, start) + } {} + } +} + +/// A hashmap of node IDs to nodes. +/// +/// This map uses a simple open addressing scheme with linear probing. +/// It's fast, simple, and sufficient for the small number of nodes we have. +struct NodeMap<'a> { + slots: &'a [Option<&'a NodeCell<'a>>], + shift: usize, + mask: u64, +} + +impl Default for NodeMap<'static> { + fn default() -> Self { + Self { slots: &[None, None], shift: 63, mask: 0 } + } +} + +impl<'a> NodeMap<'a> { + /// Creates a new node map for the given tree. + fn new(arena: &'a Arena, tree: &Tree<'a>) -> Self { + // Since we aren't expected to have millions of nodes, + // we allocate 4x the number of slots for a 25% fill factor. + let width = (4 * tree.count + 1).ilog2().max(1) as usize; + let slots = 1 << width; + let shift = 64 - width; + let mask = (slots - 1) as u64; + + let slots = arena.alloc_uninit_slice(slots).write_filled(None); + let mut node = tree.root_first; + + loop { + let n = node.borrow(); + let mut slot = n.id >> shift; + + loop { + if slots[slot as usize].is_none() { + slots[slot as usize] = Some(node); + break; + } + slot = (slot + 1) & mask; + } + + node = match n.next { + Some(node) => node, + None => break, + }; + } + + Self { slots, shift, mask } + } + + /// Gets a node by its ID. + fn get(&mut self, id: u64) -> Option<&'a NodeCell<'a>> { + let shift = self.shift; + let mask = self.mask; + let mut slot = id >> shift; + + loop { + let node = self.slots[slot as usize]?; + if node.borrow().id == id { + return Some(node); + } + slot = (slot + 1) & mask; + } + } +} + +struct FloatAttributes { + // Specifies the origin of the container relative to the container size. [0, 1] + gravity_x: f32, + gravity_y: f32, + // Specifies an offset from the origin in cells. + offset_x: f32, + offset_y: f32, +} + +/// NOTE: Must not contain items that require drop(). +#[derive(Default)] +struct NodeAttributes { + float: Option, + position: Position, + padding: Rect, + bg: u32, + fg: u32, + reverse: bool, + bordered: bool, + focusable: bool, + focus_well: bool, // Prevents focus from leaving via Tab + focus_void: bool, // Prevents focus from entering via Tab +} + +/// NOTE: Must not contain items that require drop(). +struct ListContent<'a> { + selected: u64, + // Points to the Node that holds this ListContent instance, if any>. + selected_node: Option<&'a NodeCell<'a>>, +} + +/// NOTE: Must not contain items that require drop(). +struct TableContent<'a> { + columns: Vec, + cell_gap: Size, +} + +/// NOTE: Must not contain items that require drop(). +struct StyledTextChunk { + offset: usize, + fg: u32, + attr: Attributes, +} + +const INVALID_STYLED_TEXT_CHUNK: StyledTextChunk = + StyledTextChunk { offset: usize::MAX, fg: 0, attr: Attributes::None }; + +/// NOTE: Must not contain items that require drop(). +struct TextContent<'a> { + text: ArenaString<'a>, + chunks: Vec, + overflow: Overflow, +} + +/// NOTE: Must not contain items that require drop(). +struct TextareaContent<'a> { + buffer: &'a TextBufferCell, + + // Carries over between frames. + scroll_offset: Point, + scroll_offset_y_drag_start: CoordType, + scroll_offset_x_max: CoordType, + thumb_height: CoordType, + preferred_column: CoordType, + + single_line: bool, + has_focus: bool, +} + +/// NOTE: Must not contain items that require drop(). +#[derive(Clone)] +struct ScrollareaContent { + scroll_offset: Point, + scroll_offset_y_drag_start: CoordType, + thumb_height: CoordType, +} + +/// NOTE: Must not contain items that require drop(). +#[derive(Default)] +enum NodeContent<'a> { + #[default] + None, + List(ListContent<'a>), + Modal(ArenaString<'a>), // title + Table(TableContent<'a>), + Text(TextContent<'a>), + Textarea(TextareaContent<'a>), + Scrollarea(ScrollareaContent), +} + +/// NOTE: Must not contain items that require drop(). +#[derive(Default)] +struct NodeSiblings<'a> { + prev: Option<&'a NodeCell<'a>>, + next: Option<&'a NodeCell<'a>>, +} + +impl<'a> NodeSiblings<'a> { + const PREV: usize = 0; + const NEXT: usize = 1; + + fn get(&self, off: usize) -> Option<&'a NodeCell<'a>> { + match off & 1 { + 0 => self.prev, + 1 => self.next, + _ => unreachable!(), + } + } +} + +/// NOTE: Must not contain items that require drop(). +#[derive(Default)] +struct NodeChildren<'a> { + first: Option<&'a NodeCell<'a>>, + last: Option<&'a NodeCell<'a>>, +} + +impl<'a> NodeChildren<'a> { + const FIRST: usize = 0; + const LAST: usize = 1; + + fn get(&self, off: usize) -> Option<&'a NodeCell<'a>> { + match off & 1 { + 0 => self.first, + 1 => self.last, + _ => unreachable!(), + } + } +} + +type NodeCell<'a> = SemiRefCell>; + +/// A node in the UI tree. +/// +/// NOTE: Must not contain items that require drop(). +#[derive(Default)] +struct Node<'a> { + prev: Option<&'a NodeCell<'a>>, + next: Option<&'a NodeCell<'a>>, + stack_parent: Option<&'a NodeCell<'a>>, + + id: u64, + classname: &'static str, + parent: Option<&'a NodeCell<'a>>, + depth: usize, + siblings: NodeSiblings<'a>, + children: NodeChildren<'a>, + child_count: usize, + + attributes: NodeAttributes, + content: NodeContent<'a>, + + intrinsic_size: Size, + intrinsic_size_set: bool, + outer: Rect, // in screen-space, calculated during layout + inner: Rect, // in screen-space, calculated during layout + outer_clipped: Rect, // in screen-space, calculated during layout, restricted to the viewport + inner_clipped: Rect, // in screen-space, calculated during layout, restricted to the viewport +} + +impl Node<'_> { + /// Given an outer rectangle (including padding and borders) of this node, + /// this returns the inner rectangle (excluding padding and borders). + fn outer_to_inner(&self, mut outer: Rect) -> Rect { + let l = self.attributes.bordered; + let t = self.attributes.bordered; + let r = self.attributes.bordered || matches!(self.content, NodeContent::Scrollarea(..)); + let b = self.attributes.bordered; + + outer.left += self.attributes.padding.left + l as CoordType; + outer.top += self.attributes.padding.top + t as CoordType; + outer.right -= self.attributes.padding.right + r as CoordType; + outer.bottom -= self.attributes.padding.bottom + b as CoordType; + outer + } + + /// Given an intrinsic size (excluding padding and borders) of this node, + /// this returns the outer size (including padding and borders). + fn intrinsic_to_outer(&self) -> Size { + let l = self.attributes.bordered; + let t = self.attributes.bordered; + let r = self.attributes.bordered || matches!(self.content, NodeContent::Scrollarea(..)); + let b = self.attributes.bordered; + + let mut size = self.intrinsic_size; + size.width += self.attributes.padding.left + + self.attributes.padding.right + + l as CoordType + + r as CoordType; + size.height += self.attributes.padding.top + + self.attributes.padding.bottom + + t as CoordType + + b as CoordType; + size + } + + /// Computes the intrinsic size of this node and its children. + fn compute_intrinsic_size(&mut self) { + match &mut self.content { + NodeContent::Table(spec) => { + // Calculate each row's height and the maximum width of each of its columns. + for row in Tree::iterate_siblings(self.children.first) { + let mut row = row.borrow_mut(); + let mut row_height = 0; + + for (column, cell) in Tree::iterate_siblings(row.children.first).enumerate() { + let mut cell = cell.borrow_mut(); + cell.compute_intrinsic_size(); + + let size = cell.intrinsic_to_outer(); + + // If the spec.columns[] value is positive, it's an absolute width. + // Otherwise, it's a fraction of the remaining space. + // + // TODO: The latter is computed incorrectly. + // Example: If the items are "a","b","c" then the intrinsic widths are [1,1,1]. + // If the column spec is [0,-3,-1], then this code assigns an intrinsic row + // width of 3, but it should be 5 (1+1+3), because the spec says that the + // last column (flexible 1/1) must be 3 times as wide as the 2nd one (1/3rd). + // It's not a big deal yet, because such functionality isn't needed just yet. + if column >= spec.columns.len() { + spec.columns.push(0); + } + spec.columns[column] = spec.columns[column].max(size.width); + + row_height = row_height.max(size.height); + } + + row.intrinsic_size.height = row_height; + } + + // Assuming each column has the width of the widest cell in that column, + // calculate the total width of the table. + let total_gap_width = + spec.cell_gap.width * spec.columns.len().saturating_sub(1) as CoordType; + let total_inner_width = spec.columns.iter().sum::() + total_gap_width; + let mut total_width = 0; + let mut total_height = 0; + + // Assign the total width to each row. + for row in Tree::iterate_siblings(self.children.first) { + let mut row = row.borrow_mut(); + row.intrinsic_size.width = total_inner_width; + row.intrinsic_size_set = true; + + let size = row.intrinsic_to_outer(); + total_width = total_width.max(size.width); + total_height += size.height; + } + + let total_gap_height = + spec.cell_gap.height * self.child_count.saturating_sub(1) as CoordType; + total_height += total_gap_height; + + // Assign the total width/height to the table. + if !self.intrinsic_size_set { + self.intrinsic_size.width = total_width; + self.intrinsic_size.height = total_height; + self.intrinsic_size_set = true; + } + } + _ => { + let mut max_width = 0; + let mut total_height = 0; + + for child in Tree::iterate_siblings(self.children.first) { + let mut child = child.borrow_mut(); + child.compute_intrinsic_size(); + + let size = child.intrinsic_to_outer(); + max_width = max_width.max(size.width); + total_height += size.height; + } + + if !self.intrinsic_size_set { + self.intrinsic_size.width = max_width; + self.intrinsic_size.height = total_height; + self.intrinsic_size_set = true; + } + } + } + } + + /// Lays out the children of this node. + /// The clip rect restricts "rendering" to a certain area (the viewport). + fn layout_children(&mut self, clip: Rect) { + if self.children.first.is_none() || self.inner.is_empty() { + return; + } + + match &mut self.content { + NodeContent::Table(spec) => { + let width = self.inner.right - self.inner.left; + let mut x = self.inner.left; + let mut y = self.inner.top; + + for row in Tree::iterate_siblings(self.children.first) { + let mut row = row.borrow_mut(); + let mut size = row.intrinsic_to_outer(); + size.width = width; + row.outer.left = x; + row.outer.top = y; + row.outer.right = x + size.width; + row.outer.bottom = y + size.height; + row.outer = row.outer.intersect(self.inner); + row.inner = row.outer_to_inner(row.outer); + row.outer_clipped = row.outer.intersect(clip); + row.inner_clipped = row.inner.intersect(clip); + + let mut row_height = 0; + + for (column, cell) in Tree::iterate_siblings(row.children.first).enumerate() { + let mut cell = cell.borrow_mut(); + let mut size = cell.intrinsic_to_outer(); + size.width = spec.columns[column]; + cell.outer.left = x; + cell.outer.top = y; + cell.outer.right = x + size.width; + cell.outer.bottom = y + size.height; + cell.outer = cell.outer.intersect(self.inner); + cell.inner = cell.outer_to_inner(cell.outer); + cell.outer_clipped = cell.outer.intersect(clip); + cell.inner_clipped = cell.inner.intersect(clip); + + x += size.width + spec.cell_gap.width; + row_height = row_height.max(size.height); + + cell.layout_children(clip); + } + + x = self.inner.left; + y += row_height + spec.cell_gap.height; + } + } + NodeContent::Scrollarea(sc) => { + let mut content = self.children.first.unwrap().borrow_mut(); + + // content available viewport size (-1 for the track) + let sx = self.inner.right - self.inner.left; + let sy = self.inner.bottom - self.inner.top; + // actual content size + let cx = sx; + let cy = content.intrinsic_size.height.max(sy); + // scroll offset + let ox = 0; + let oy = sc.scroll_offset.y.clamp(0, cy - sy); + + sc.scroll_offset.x = ox; + sc.scroll_offset.y = oy; + + content.outer.left = self.inner.left - ox; + content.outer.top = self.inner.top - oy; + content.outer.right = content.outer.left + cx; + content.outer.bottom = content.outer.top + cy; + content.inner = content.outer_to_inner(content.outer); + content.outer_clipped = content.outer.intersect(self.inner_clipped); + content.inner_clipped = content.inner.intersect(self.inner_clipped); + + let clip = content.inner_clipped; + content.layout_children(clip); + } + _ => { + let width = self.inner.right - self.inner.left; + let x = self.inner.left; + let mut y = self.inner.top; + + for child in Tree::iterate_siblings(self.children.first) { + let mut child = child.borrow_mut(); + let size = child.intrinsic_to_outer(); + let remaining = (width - size.width).max(0); + + child.outer.left = x + match child.attributes.position { + Position::Stretch | Position::Left => 0, + Position::Center => remaining / 2, + Position::Right => remaining, + }; + child.outer.right = child.outer.left + + match child.attributes.position { + Position::Stretch => width, + _ => size.width, + }; + child.outer.top = y; + child.outer.bottom = y + size.height; + + child.outer = child.outer.intersect(self.inner); + child.inner = child.outer_to_inner(child.outer); + child.outer_clipped = child.outer.intersect(clip); + child.inner_clipped = child.inner.intersect(clip); + + y += size.height; + } + + for child in Tree::iterate_siblings(self.children.first) { + let mut child = child.borrow_mut(); + child.layout_children(clip); + } + } + } + } +} diff --git a/pkgs/edit/src/unicode/measurement.rs b/pkgs/edit/src/unicode/measurement.rs new file mode 100644 index 0000000..f495d14 --- /dev/null +++ b/pkgs/edit/src/unicode/measurement.rs @@ -0,0 +1,1186 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::hint::cold_path; + +use super::Utf8Chars; +use super::tables::*; +use crate::document::ReadableDocument; +use crate::helpers::{CoordType, Point}; +use crate::simd::{memchr2, memrchr2}; + +/// Stores a position inside a [`ReadableDocument`]. +/// +/// The cursor tracks both the absolute byte-offset, +/// as well as the position in terminal-related coordinates. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub struct Cursor { + /// Offset in bytes within the buffer. + pub offset: usize, + /// Position in the buffer in lines (.y) and grapheme clusters (.x). + /// + /// Line wrapping has NO influence on this. + pub logical_pos: Point, + /// Position in the buffer in laid out rows (.y) and columns (.x). + /// + /// Line wrapping has an influence on this. + pub visual_pos: Point, + /// Horizontal position in visual columns. + /// + /// Line wrapping has NO influence on this and if word wrap is disabled, + /// it's identical to `visual_pos.x`. This is useful for calculating tab widths. + pub column: CoordType, + /// When `measure_forward` hits the `word_wrap_column`, the question is: + /// Was there a wrap opportunity on this line? Because if there wasn't, + /// a hard-wrap is required, otherwise the word that is being layouted is + /// moved to the next line. This boolean carries this state between calls. + pub wrap_opp: bool, +} + +/// Your entrypoint to navigating inside a [`ReadableDocument`]. +#[derive(Clone)] +pub struct MeasurementConfig<'doc> { + buffer: &'doc dyn ReadableDocument, + tab_size: CoordType, + word_wrap_column: CoordType, + cursor: Cursor, +} + +impl<'doc> MeasurementConfig<'doc> { + /// Creates a new [`MeasurementConfig`] for the given document. + pub fn new(buffer: &'doc dyn ReadableDocument) -> Self { + Self { buffer, tab_size: 8, word_wrap_column: 0, cursor: Default::default() } + } + + /// Sets the tab size. + /// + /// Defaults to 8, because that's what a tab in terminals evaluates to. + pub fn with_tab_size(mut self, tab_size: CoordType) -> Self { + self.tab_size = tab_size.max(1); + self + } + + /// You want word wrap? Set it here! + /// + /// Defaults to 0, which means no word wrap. + pub fn with_word_wrap_column(mut self, word_wrap_column: CoordType) -> Self { + self.word_wrap_column = word_wrap_column; + self + } + + /// Sets the initial cursor to the given position. + /// + /// WARNING: While the code doesn't panic if the cursor is invalid, + /// the results will obviously be complete garbage. + pub fn with_cursor(mut self, cursor: Cursor) -> Self { + self.cursor = cursor; + self + } + + /// Navigates **forward** to the given absolute offset. + /// + /// # Returns + /// + /// The cursor position after the navigation. + pub fn goto_offset(&mut self, offset: usize) -> Cursor { + self.cursor = Self::measure_forward( + self.tab_size, + self.word_wrap_column, + offset, + Point::MAX, + Point::MAX, + self.cursor, + self.buffer, + ); + self.cursor + } + + /// Navigates **forward** to the given logical position. + /// + /// Logical positions are in lines and grapheme clusters. + /// + /// # Returns + /// + /// The cursor position after the navigation. + pub fn goto_logical(&mut self, logical_target: Point) -> Cursor { + self.cursor = Self::measure_forward( + self.tab_size, + self.word_wrap_column, + usize::MAX, + logical_target, + Point::MAX, + self.cursor, + self.buffer, + ); + self.cursor + } + + /// Navigates **forward** to the given visual position. + /// + /// Visual positions are in laid out rows and columns. + /// + /// # Returns + /// + /// The cursor position after the navigation. + pub fn goto_visual(&mut self, visual_target: Point) -> Cursor { + self.cursor = Self::measure_forward( + self.tab_size, + self.word_wrap_column, + usize::MAX, + Point::MAX, + visual_target, + self.cursor, + self.buffer, + ); + self.cursor + } + + /// Returns the current cursor position. + pub fn cursor(&self) -> Cursor { + self.cursor + } + + // NOTE that going to a visual target can result in ambiguous results, + // where going to an identical logical target will yield a different result. + // + // Imagine if you have a `word_wrap_column` of 6 and there's "Hello World" on the line: + // `goto_logical` will return a `visual_pos` of {0,1}, while `goto_visual` returns {6,0}. + // This is because from a logical POV, if the wrap location equals the wrap column, + // the wrap exists on both lines and it'll default to wrapping. `goto_visual` however will always + // try to return a Y position that matches the requested position, so that Home/End works properly. + fn measure_forward( + tab_size: CoordType, + word_wrap_column: CoordType, + offset_target: usize, + logical_target: Point, + visual_target: Point, + cursor: Cursor, + buffer: &dyn ReadableDocument, + ) -> Cursor { + if cursor.offset >= offset_target + || cursor.logical_pos >= logical_target + || cursor.visual_pos >= visual_target + { + return cursor; + } + + let mut offset = cursor.offset; + let mut logical_pos_x = cursor.logical_pos.x; + let mut logical_pos_y = cursor.logical_pos.y; + let mut visual_pos_x = cursor.visual_pos.x; + let mut visual_pos_y = cursor.visual_pos.y; + let mut column = cursor.column; + + let mut logical_target_x = Self::calc_target_x(logical_target, logical_pos_y); + let mut visual_target_x = Self::calc_target_x(visual_target, visual_pos_y); + + // wrap_opp = Wrap Opportunity + // These store the position and column of the last wrap opportunity. If `word_wrap_column` is + // zero (word wrap disabled), all grapheme clusters are a wrap opportunity, because none are. + let mut wrap_opp = cursor.wrap_opp; + let mut wrap_opp_offset = offset; + let mut wrap_opp_logical_pos_x = logical_pos_x; + let mut wrap_opp_visual_pos_x = visual_pos_x; + let mut wrap_opp_column = column; + + let mut chunk_iter = Utf8Chars::new(b"", 0); + let mut chunk_range = offset..offset; + let mut props_next_cluster = ucd_start_of_text_properties(); + + loop { + // Have we reached the target already? Stop. + if offset >= offset_target + || logical_pos_x >= logical_target_x + || visual_pos_x >= visual_target_x + { + break; + } + + let props_current_cluster = props_next_cluster; + let mut props_last_char; + let mut offset_next_cluster; + let mut state = 0; + let mut width = 0; + + // Since we want to measure the width of the current cluster, + // by necessity we need to seek to the next cluster. + // We'll then reuse the offset and properties of the next cluster in + // the next iteration of the this (outer) loop (`props_next_cluster`). + loop { + if !chunk_iter.has_next() { + cold_path(); + chunk_iter = Utf8Chars::new(buffer.read_forward(chunk_range.end), 0); + chunk_range = chunk_range.end..chunk_range.end + chunk_iter.len(); + } + + // Since this loop seeks ahead to the next cluster, and since `chunk_iter` + // records the offset of the next character after the returned one, we need + // to save the offset of the previous `chunk_iter` before calling `next()`. + // Similar applies to the width. + props_last_char = props_next_cluster; + offset_next_cluster = chunk_range.start + chunk_iter.offset(); + width += ucd_grapheme_cluster_character_width(props_next_cluster) as CoordType; + + // The `Document::read_forward` interface promises us that it will not split + // grapheme clusters across chunks. Therefore, we can safely break here. + let ch = match chunk_iter.next() { + Some(ch) => ch, + None => break, + }; + + // Get the properties of the next cluster. + props_next_cluster = ucd_grapheme_cluster_lookup(ch); + state = ucd_grapheme_cluster_joins(state, props_last_char, props_next_cluster); + + // Stop if the next character does not join. + if ucd_grapheme_cluster_joins_done(state) { + break; + } + } + + if offset_next_cluster == offset { + // No advance and the iterator is empty? End of text reached. + if chunk_iter.is_empty() { + break; + } + // Ignore the first iteration when processing the start-of-text. + continue; + } + + // The max. width of a terminal cell is 2. + width = width.min(2); + + // Tabs require special handling because they can have a variable width. + if props_last_char == ucd_tab_properties() { + // SAFETY: `tab_size` is clamped to >= 1 in `with_tab_size`. + // This assert ensures that Rust doesn't insert panicking null checks. + unsafe { std::hint::assert_unchecked(tab_size >= 1) }; + width = tab_size - (column % tab_size); + } + + // Hard wrap: Both the logical and visual position advance by one line. + if props_last_char == ucd_linefeed_properties() { + cold_path(); + + wrap_opp = false; + + // Don't cross the newline if the target is on this line but we haven't reached it. + // E.g. if the callers asks for column 100 on a 10 column line, + // we'll return with the cursor set to column 10. + if logical_pos_y >= logical_target.y || visual_pos_y >= visual_target.y { + break; + } + + offset = offset_next_cluster; + logical_pos_x = 0; + logical_pos_y += 1; + visual_pos_x = 0; + visual_pos_y += 1; + column = 0; + + logical_target_x = Self::calc_target_x(logical_target, logical_pos_y); + visual_target_x = Self::calc_target_x(visual_target, visual_pos_y); + continue; + } + + // Avoid advancing past the visual target, because `width` can be greater than 1. + if visual_pos_x + width > visual_target_x { + break; + } + + // Since this code above may need to revert to a previous `wrap_opp_*`, + // it must be done before advancing / checking for `ucd_line_break_joins`. + if word_wrap_column > 0 && visual_pos_x + width > word_wrap_column { + if !wrap_opp { + // Otherwise, the lack of a wrap opportunity means that a single word + // is wider than the word wrap column. We need to force-break the word. + // This is similar to the above, but "bar" gets written at column 0. + wrap_opp_offset = offset; + wrap_opp_logical_pos_x = logical_pos_x; + wrap_opp_visual_pos_x = visual_pos_x; + wrap_opp_column = column; + visual_pos_x = 0; + } else { + // If we had a wrap opportunity on this line, we can move all + // characters since then to the next line without stopping this loop: + // +---------+ +---------+ +---------+ + // | foo| -> | | -> | | + // | | |foo | |foobar | + // +---------+ +---------+ +---------+ + // We don't actually move "foo", but rather just change where "bar" goes. + // Since this function doesn't copy text, the end result is the same. + visual_pos_x -= wrap_opp_visual_pos_x; + } + + wrap_opp = false; + visual_pos_y += 1; + visual_target_x = Self::calc_target_x(visual_target, visual_pos_y); + + if visual_pos_x == visual_target_x { + break; + } + + // Imagine the word is "hello" and on the "o" we notice it wraps. + // If the target however was the "e", then we must revert back to "h" and search for it. + if visual_pos_x > visual_target_x { + cold_path(); + + offset = wrap_opp_offset; + logical_pos_x = wrap_opp_logical_pos_x; + visual_pos_x = 0; + column = wrap_opp_column; + + chunk_iter.seek(chunk_iter.len()); + chunk_range = offset..offset; + props_next_cluster = ucd_start_of_text_properties(); + continue; + } + } + + offset = offset_next_cluster; + logical_pos_x += 1; + visual_pos_x += width; + column += width; + + if word_wrap_column > 0 + && !ucd_line_break_joins(props_current_cluster, props_next_cluster) + { + wrap_opp = true; + wrap_opp_offset = offset; + wrap_opp_logical_pos_x = logical_pos_x; + wrap_opp_visual_pos_x = visual_pos_x; + wrap_opp_column = column; + } + } + + // If we're here, we hit our target. Now the only question is: + // Is the word we're currently on so wide that it will be wrapped further down the document? + if word_wrap_column > 0 { + if !wrap_opp { + // If the current layouted line had no wrap opportunities, it means we had an input + // such as "fooooooooooooooooooooo" at a `word_wrap_column` of e.g. 10. The word + // didn't fit and the lack of a `wrap_opp` indicates we must force a hard wrap. + // Thankfully, if we reach this point, that was already done by the code above. + } else if wrap_opp_logical_pos_x != logical_pos_x && visual_pos_y <= visual_target.y { + // Imagine the string "foo bar" with a word wrap column of 6. If I ask for the cursor at + // `logical_pos={5,0}`, then the code above exited while reaching the target. + // At this point, this function doesn't know yet that after the "b" there's "ar" + // which causes a word wrap, and causes the final visual position to be {1,1}. + // This code thus seeks ahead and checks if the current word will wrap or not. + // Of course we only need to do this if the cursor isn't on a wrap opportunity already. + + // The loop below should not modify the target we already found. + let mut visual_pos_x_lookahead = visual_pos_x; + + loop { + let props_current_cluster = props_next_cluster; + let mut props_last_char; + let mut offset_next_cluster; + let mut state = 0; + let mut width = 0; + + // Since we want to measure the width of the current cluster, + // by necessity we need to seek to the next cluster. + // We'll then reuse the offset and properties of the next cluster in + // the next iteration of the this (outer) loop (`props_next_cluster`). + loop { + if !chunk_iter.has_next() { + cold_path(); + chunk_iter = Utf8Chars::new(buffer.read_forward(chunk_range.end), 0); + chunk_range = chunk_range.end..chunk_range.end + chunk_iter.len(); + } + + // Since this loop seeks ahead to the next cluster, and since `chunk_iter` + // records the offset of the next character after the returned one, we need + // to save the offset of the previous `chunk_iter` before calling `next()`. + // Similar applies to the width. + props_last_char = props_next_cluster; + offset_next_cluster = chunk_range.start + chunk_iter.offset(); + width += + ucd_grapheme_cluster_character_width(props_next_cluster) as CoordType; + + // The `Document::read_forward` interface promises us that it will not split + // grapheme clusters across chunks. Therefore, we can safely break here. + let ch = match chunk_iter.next() { + Some(ch) => ch, + None => break, + }; + + // Get the properties of the next cluster. + props_next_cluster = ucd_grapheme_cluster_lookup(ch); + state = + ucd_grapheme_cluster_joins(state, props_last_char, props_next_cluster); + + // Stop if the next character does not join. + if ucd_grapheme_cluster_joins_done(state) { + break; + } + } + + if offset_next_cluster == offset { + // No advance and the iterator is empty? End of text reached. + if chunk_iter.is_empty() { + break; + } + // Ignore the first iteration when processing the start-of-text. + continue; + } + + // The max. width of a terminal cell is 2. + width = width.min(2); + + // Tabs require special handling because they can have a variable width. + if props_last_char == ucd_tab_properties() { + // SAFETY: `tab_size` is clamped to >= 1 in `with_tab_size`. + // This assert ensures that Rust doesn't insert panicking null checks. + unsafe { std::hint::assert_unchecked(tab_size >= 1) }; + width = tab_size - (column % tab_size); + } + + // Hard wrap: Both the logical and visual position advance by one line. + if props_last_char == ucd_linefeed_properties() { + break; + } + + visual_pos_x_lookahead += width; + + if visual_pos_x_lookahead > word_wrap_column { + visual_pos_x -= wrap_opp_visual_pos_x; + visual_pos_y += 1; + break; + } else if !ucd_line_break_joins(props_current_cluster, props_next_cluster) { + break; + } + } + } + + if visual_pos_y > visual_target.y { + // Imagine the string "foo bar" with a word wrap column of 6. If I ask for the cursor at + // `visual_pos={100,0}`, the code above exited early after wrapping without reaching the target. + // Since I asked for the last character on the first line, we must wrap back up the last wrap + offset = wrap_opp_offset; + logical_pos_x = wrap_opp_logical_pos_x; + visual_pos_x = wrap_opp_visual_pos_x; + visual_pos_y = visual_target.y; + column = wrap_opp_column; + wrap_opp = true; + } + } + + Cursor { + offset, + logical_pos: Point { x: logical_pos_x, y: logical_pos_y }, + visual_pos: Point { x: visual_pos_x, y: visual_pos_y }, + column, + wrap_opp, + } + } + + #[inline] + fn calc_target_x(target: Point, pos_y: CoordType) -> CoordType { + match pos_y.cmp(&target.y) { + std::cmp::Ordering::Less => CoordType::MAX, + std::cmp::Ordering::Equal => target.x, + std::cmp::Ordering::Greater => 0, + } + } +} + +/// Seeks forward to to the given line start. +/// +/// If given a piece of `text`, and assuming you're currently at `offset` which +/// is on the logical line `line`, this will seek forward until the logical line +/// `line_stop` is reached. For instance, if `line` is 0 and `line_stop` is 2, +/// it'll seek forward past 2 line feeds. +/// +/// This function always stops exactly past a line feed +/// and thus returns a position at the start of a line. +/// +/// # Warning +/// +/// If the end of `text` is hit before reaching `line_stop`, the function +/// will return an offset of `text.len()`, not at the start of a line. +/// +/// # Parameters +/// +/// * `text`: The text to search in. +/// * `offset`: The offset to start searching from. +/// * `line`: The current line. +/// * `line_stop`: The line to stop at. +/// +/// # Returns +/// +/// A tuple consisting of: +/// * The new offset. +/// * The line number that was reached. +pub fn newlines_forward( + text: &[u8], + mut offset: usize, + mut line: CoordType, + line_stop: CoordType, +) -> (usize, CoordType) { + // Leaving the cursor at the beginning of the current line when the limit + // is 0 makes this function behave identical to ucd_newlines_backward. + if line >= line_stop { + return newlines_backward(text, offset, line, line_stop); + } + + let len = text.len(); + offset = offset.min(len); + + loop { + // TODO: This code could be optimized by replacing memchr with manual line counting. + // + // If `line_stop` is very far away, we could accumulate newline counts horizontally + // in a AVX2 register (= 32 u8 slots). Then, every 256 bytes we compute the horizontal + // sum via `_mm256_sad_epu8` yielding us the newline count in the last block. + // + // We could also just use `_mm256_sad_epu8` on each fetch as-is. + offset = memchr2(b'\n', b'\n', text, offset); + if offset >= len { + break; + } + + offset += 1; + line += 1; + if line >= line_stop { + break; + } + } + + (offset, line) +} + +/// Seeks backward to the given line start. +/// +/// See [`newlines_forward`] for details. +/// This function does almost the same thing, but in reverse. +/// +/// # Warning +/// +/// In addition to the notes in [`newlines_forward`]: +/// +/// No matter what parameters are given, [`newlines_backward`] only returns an +/// offset at the start of a line. Put differently, even if `line == line_stop`, +/// it'll seek backward to the line start. +pub fn newlines_backward( + text: &[u8], + mut offset: usize, + mut line: CoordType, + line_stop: CoordType, +) -> (usize, CoordType) { + offset = offset.min(text.len()); + + loop { + offset = match memrchr2(b'\n', b'\n', text, offset) { + Some(i) => i, + None => return (0, line), + }; + if line <= line_stop { + // +1: Past the newline, at the start of the current line. + return (offset + 1, line); + } + line -= 1; + } +} + +/// Returns an offset past a newline. +/// +/// If `offset` is right in front of a newline, +/// this will return the offset past said newline. +pub fn skip_newline(text: &[u8], mut offset: usize) -> usize { + if offset >= text.len() { + return offset; + } + if text[offset] == b'\r' { + offset += 1; + } + if offset >= text.len() { + return offset; + } + if text[offset] == b'\n' { + offset += 1; + } + offset +} + +/// Strips a trailing newline from the given text. +pub fn strip_newline(mut text: &[u8]) -> &[u8] { + // Rust generates surprisingly tight assembly for this. + if text.last() == Some(&b'\n') { + text = &text[..text.len() - 1]; + } + if text.last() == Some(&b'\r') { + text = &text[..text.len() - 1]; + } + text +} + +#[cfg(test)] +mod test { + use super::*; + + struct ChunkedDoc<'a>(&'a [&'a [u8]]); + + impl ReadableDocument for ChunkedDoc<'_> { + fn read_forward(&self, mut off: usize) -> &[u8] { + for chunk in self.0 { + if off < chunk.len() { + return &chunk[off..]; + } + off -= chunk.len(); + } + &[] + } + + fn read_backward(&self, mut off: usize) -> &[u8] { + for chunk in self.0.iter().rev() { + if off < chunk.len() { + return &chunk[..chunk.len() - off]; + } + off -= chunk.len(); + } + &[] + } + } + + #[test] + fn test_measure_forward_newline_start() { + let cursor = + MeasurementConfig::new(&"foo\nbar".as_bytes()).goto_visual(Point { x: 0, y: 1 }); + assert_eq!( + cursor, + Cursor { + offset: 4, + logical_pos: Point { x: 0, y: 1 }, + visual_pos: Point { x: 0, y: 1 }, + column: 0, + wrap_opp: false, + } + ); + } + + #[test] + fn test_measure_forward_clipped_wide_char() { + let cursor = MeasurementConfig::new(&"a😶‍🌫️b".as_bytes()).goto_visual(Point { x: 2, y: 0 }); + assert_eq!( + cursor, + Cursor { + offset: 1, + logical_pos: Point { x: 1, y: 0 }, + visual_pos: Point { x: 1, y: 0 }, + column: 1, + wrap_opp: false, + } + ); + } + + #[test] + fn test_measure_forward_word_wrap() { + // |foo␣ | + // |bar␣ | + // |baz | + let text = "foo bar \nbaz".as_bytes(); + + // Does hitting a logical target wrap the visual position along with the word? + let mut cfg = MeasurementConfig::new(&text).with_word_wrap_column(6); + let cursor = cfg.goto_logical(Point { x: 5, y: 0 }); + assert_eq!( + cursor, + Cursor { + offset: 5, + logical_pos: Point { x: 5, y: 0 }, + visual_pos: Point { x: 1, y: 1 }, + column: 5, + wrap_opp: true, + } + ); + + // Does hitting the visual target within a word reset the hit back to the end of the visual line? + let mut cfg = MeasurementConfig::new(&text).with_word_wrap_column(6); + let cursor = cfg.goto_visual(Point { x: CoordType::MAX, y: 0 }); + assert_eq!( + cursor, + Cursor { + offset: 4, + logical_pos: Point { x: 4, y: 0 }, + visual_pos: Point { x: 4, y: 0 }, + column: 4, + wrap_opp: true, + } + ); + + // Does hitting the same target but with a non-zero starting position result in the same outcome? + let mut cfg = MeasurementConfig::new(&text).with_word_wrap_column(6).with_cursor(Cursor { + offset: 1, + logical_pos: Point { x: 1, y: 0 }, + visual_pos: Point { x: 1, y: 0 }, + column: 1, + wrap_opp: false, + }); + let cursor = cfg.goto_visual(Point { x: 5, y: 0 }); + assert_eq!( + cursor, + Cursor { + offset: 4, + logical_pos: Point { x: 4, y: 0 }, + visual_pos: Point { x: 4, y: 0 }, + column: 4, + wrap_opp: true, + } + ); + + let cursor = cfg.goto_visual(Point { x: 0, y: 1 }); + assert_eq!( + cursor, + Cursor { + offset: 4, + logical_pos: Point { x: 4, y: 0 }, + visual_pos: Point { x: 0, y: 1 }, + column: 4, + wrap_opp: false, + } + ); + + let cursor = cfg.goto_visual(Point { x: 5, y: 1 }); + assert_eq!( + cursor, + Cursor { + offset: 8, + logical_pos: Point { x: 8, y: 0 }, + visual_pos: Point { x: 4, y: 1 }, + column: 8, + wrap_opp: false, + } + ); + + let cursor = cfg.goto_visual(Point { x: 0, y: 2 }); + assert_eq!( + cursor, + Cursor { + offset: 9, + logical_pos: Point { x: 0, y: 1 }, + visual_pos: Point { x: 0, y: 2 }, + column: 0, + wrap_opp: false, + } + ); + + let cursor = cfg.goto_visual(Point { x: 5, y: 2 }); + assert_eq!( + cursor, + Cursor { + offset: 12, + logical_pos: Point { x: 3, y: 1 }, + visual_pos: Point { x: 3, y: 2 }, + column: 3, + wrap_opp: false, + } + ); + } + + #[test] + fn test_measure_forward_tabs() { + let text = "a\tb\tc".as_bytes(); + let cursor = + MeasurementConfig::new(&text).with_tab_size(4).goto_visual(Point { x: 4, y: 0 }); + assert_eq!( + cursor, + Cursor { + offset: 2, + logical_pos: Point { x: 2, y: 0 }, + visual_pos: Point { x: 4, y: 0 }, + column: 4, + wrap_opp: false, + } + ); + } + + #[test] + fn test_measure_forward_chunk_boundaries() { + let chunks = [ + "Hello".as_bytes(), + "\u{1F469}\u{1F3FB}".as_bytes(), // 8 bytes, 2 columns + "World".as_bytes(), + ]; + let doc = ChunkedDoc(&chunks); + let cursor = MeasurementConfig::new(&doc).goto_visual(Point { x: 5 + 2 + 3, y: 0 }); + assert_eq!(cursor.offset, 5 + 8 + 3); + assert_eq!(cursor.logical_pos, Point { x: 5 + 1 + 3, y: 0 }); + } + + #[test] + fn test_exact_wrap() { + // |foo_ | + // |bar. | + // |abc | + let chunks = ["foo ".as_bytes(), "bar".as_bytes(), ".\n".as_bytes(), "abc".as_bytes()]; + let doc = ChunkedDoc(&chunks); + let mut cfg = MeasurementConfig::new(&doc).with_word_wrap_column(7); + let max = CoordType::MAX; + + let end0 = cfg.goto_visual(Point { x: 7, y: 0 }); + assert_eq!( + end0, + Cursor { + offset: 4, + logical_pos: Point { x: 4, y: 0 }, + visual_pos: Point { x: 4, y: 0 }, + column: 4, + wrap_opp: true, + } + ); + + let beg1 = cfg.goto_visual(Point { x: 0, y: 1 }); + assert_eq!( + beg1, + Cursor { + offset: 4, + logical_pos: Point { x: 4, y: 0 }, + visual_pos: Point { x: 0, y: 1 }, + column: 4, + wrap_opp: false, + } + ); + + let end1 = cfg.goto_visual(Point { x: max, y: 1 }); + assert_eq!( + end1, + Cursor { + offset: 8, + logical_pos: Point { x: 8, y: 0 }, + visual_pos: Point { x: 4, y: 1 }, + column: 8, + wrap_opp: false, + } + ); + + let beg2 = cfg.goto_visual(Point { x: 0, y: 2 }); + assert_eq!( + beg2, + Cursor { + offset: 9, + logical_pos: Point { x: 0, y: 1 }, + visual_pos: Point { x: 0, y: 2 }, + column: 0, + wrap_opp: false, + } + ); + + let end2 = cfg.goto_visual(Point { x: max, y: 2 }); + assert_eq!( + end2, + Cursor { + offset: 12, + logical_pos: Point { x: 3, y: 1 }, + visual_pos: Point { x: 3, y: 2 }, + column: 3, + wrap_opp: false, + } + ); + } + + #[test] + fn test_force_wrap() { + // |//_ | + // |aaaaaaaa| + // |aaaa | + let bytes = "// aaaaaaaaaaaa".as_bytes(); + let mut cfg = MeasurementConfig::new(&bytes).with_word_wrap_column(8); + let max = CoordType::MAX; + + // At the end of "// " there should be a wrap. + let end0 = cfg.goto_visual(Point { x: max, y: 0 }); + assert_eq!( + end0, + Cursor { + offset: 3, + logical_pos: Point { x: 3, y: 0 }, + visual_pos: Point { x: 3, y: 0 }, + column: 3, + wrap_opp: true, + } + ); + + // Test if the ambiguous visual position at the wrap location doesn't change the offset. + let beg0 = cfg.goto_visual(Point { x: 0, y: 1 }); + assert_eq!( + beg0, + Cursor { + offset: 3, + logical_pos: Point { x: 3, y: 0 }, + visual_pos: Point { x: 0, y: 1 }, + column: 3, + wrap_opp: false, + } + ); + + // Test if navigating inside the wrapped line doesn't cause further wrapping. + // + // This step of the test is important, as it ensures that the following force-wrap works, + // even if 1 of the 8 "a"s was already processed. + let beg0_off1 = cfg.goto_logical(Point { x: 4, y: 0 }); + assert_eq!( + beg0_off1, + Cursor { + offset: 4, + logical_pos: Point { x: 4, y: 0 }, + visual_pos: Point { x: 1, y: 1 }, + column: 4, + wrap_opp: false, + } + ); + + // Test if the force-wrap applies at the end of the first 8 "a"s. + let end1 = cfg.goto_visual(Point { x: max, y: 1 }); + assert_eq!( + end1, + Cursor { + offset: 11, + logical_pos: Point { x: 11, y: 0 }, + visual_pos: Point { x: 8, y: 1 }, + column: 11, + wrap_opp: true, + } + ); + + // Test if the remaining 4 "a"s are properly layouted. + let end2 = cfg.goto_visual(Point { x: max, y: 2 }); + assert_eq!( + end2, + Cursor { + offset: 15, + logical_pos: Point { x: 15, y: 0 }, + visual_pos: Point { x: 4, y: 2 }, + column: 15, + wrap_opp: false, + } + ); + } + + #[test] + fn test_force_wrap_wide() { + // These Yijing Hexagram Symbols form no word wrap opportunities. + let text = "䷀䷁䷂䷃䷄䷅䷆䷇䷈䷉"; + let expected = ["䷀䷁", "䷂䷃", "䷄䷅", "䷆䷇", "䷈䷉"]; + let bytes = text.as_bytes(); + let mut cfg = MeasurementConfig::new(&bytes).with_word_wrap_column(5); + + for (y, &expected) in expected.iter().enumerate() { + let y = y as CoordType; + // In order for `goto_visual()` to hit columnn 0 after a word wrap, + // it MUST be able to go back by 1 grapheme, which is what this tests. + let beg = cfg.goto_visual(Point { x: 0, y }); + let end = cfg.goto_visual(Point { x: 5, y }); + let actual = &text[beg.offset..end.offset]; + assert_eq!(actual, expected); + } + } + + // Similar to the `test_force_wrap` test, but here we vertically descend + // down the document without ever touching the first or last column. + // I found that this finds curious bugs at times. + #[test] + fn test_force_wrap_column() { + // |//_ | + // |aaaaaaaa| + // |aaaa | + let bytes = "// aaaaaaaaaaaa".as_bytes(); + let mut cfg = MeasurementConfig::new(&bytes).with_word_wrap_column(8); + + // At the end of "// " there should be a wrap. + let end0 = cfg.goto_visual(Point { x: CoordType::MAX, y: 0 }); + assert_eq!( + end0, + Cursor { + offset: 3, + logical_pos: Point { x: 3, y: 0 }, + visual_pos: Point { x: 3, y: 0 }, + column: 3, + wrap_opp: true, + } + ); + + let mid1 = cfg.goto_visual(Point { x: end0.visual_pos.x, y: 1 }); + assert_eq!( + mid1, + Cursor { + offset: 6, + logical_pos: Point { x: 6, y: 0 }, + visual_pos: Point { x: 3, y: 1 }, + column: 6, + wrap_opp: false, + } + ); + + let mid2 = cfg.goto_visual(Point { x: end0.visual_pos.x, y: 2 }); + assert_eq!( + mid2, + Cursor { + offset: 14, + logical_pos: Point { x: 14, y: 0 }, + visual_pos: Point { x: 3, y: 2 }, + column: 14, + wrap_opp: false, + } + ); + } + + #[test] + fn test_any_wrap() { + // |//_-----| + // |------- | + let bytes = "// ------------".as_bytes(); + let mut cfg = MeasurementConfig::new(&bytes).with_word_wrap_column(8); + let max = CoordType::MAX; + + let end0 = cfg.goto_visual(Point { x: max, y: 0 }); + assert_eq!( + end0, + Cursor { + offset: 8, + logical_pos: Point { x: 8, y: 0 }, + visual_pos: Point { x: 8, y: 0 }, + column: 8, + wrap_opp: true, + } + ); + + let end1 = cfg.goto_visual(Point { x: max, y: 1 }); + assert_eq!( + end1, + Cursor { + offset: 15, + logical_pos: Point { x: 15, y: 0 }, + visual_pos: Point { x: 7, y: 1 }, + column: 15, + wrap_opp: true, + } + ); + } + + #[test] + fn test_any_wrap_wide() { + // These Japanese characters form word wrap opportunity between each character. + let text = "零一二三四五六七八九"; + let expected = ["零一", "二三", "四五", "六七", "八九"]; + let bytes = text.as_bytes(); + let mut cfg = MeasurementConfig::new(&bytes).with_word_wrap_column(5); + + for (y, &expected) in expected.iter().enumerate() { + let y = y as CoordType; + // In order for `goto_visual()` to hit columnn 0 after a word wrap, + // it MUST be able to go back by 1 grapheme, which is what this tests. + let beg = cfg.goto_visual(Point { x: 0, y }); + let end = cfg.goto_visual(Point { x: 5, y }); + let actual = &text[beg.offset..end.offset]; + assert_eq!(actual, expected); + } + } + + #[test] + fn test_wrap_tab() { + // |foo_ | <- 1 space + // |____b | <- 1 tab, 1 space + let text = "foo \t b"; + let bytes = text.as_bytes(); + let mut cfg = MeasurementConfig::new(&bytes).with_word_wrap_column(8).with_tab_size(4); + let max = CoordType::MAX; + + let end0 = cfg.goto_visual(Point { x: max, y: 0 }); + assert_eq!( + end0, + Cursor { + offset: 4, + logical_pos: Point { x: 4, y: 0 }, + visual_pos: Point { x: 4, y: 0 }, + column: 4, + wrap_opp: true, + }, + ); + + let beg1 = cfg.goto_visual(Point { x: 0, y: 1 }); + assert_eq!( + beg1, + Cursor { + offset: 4, + logical_pos: Point { x: 4, y: 0 }, + visual_pos: Point { x: 0, y: 1 }, + column: 4, + wrap_opp: false, + }, + ); + + let end1 = cfg.goto_visual(Point { x: max, y: 1 }); + assert_eq!( + end1, + Cursor { + offset: 7, + logical_pos: Point { x: 7, y: 0 }, + visual_pos: Point { x: 6, y: 1 }, + column: 10, + wrap_opp: true, + }, + ); + } + + #[test] + fn test_crlf() { + let text = "a\r\nbcd\r\ne".as_bytes(); + let cursor = MeasurementConfig::new(&text).goto_visual(Point { x: CoordType::MAX, y: 1 }); + assert_eq!( + cursor, + Cursor { + offset: 6, + logical_pos: Point { x: 3, y: 1 }, + visual_pos: Point { x: 3, y: 1 }, + column: 3, + wrap_opp: false, + } + ); + } + + #[test] + fn test_wrapped_cursor_can_seek_backward() { + let bytes = "hello world".as_bytes(); + let mut cfg = MeasurementConfig::new(&bytes).with_word_wrap_column(10); + + // When the word wrap at column 10 hits, the cursor will be at the end of the word "world" (between l and d). + // This tests if the algorithm is capable of going back to the start of the word and find the actual target. + let cursor = cfg.goto_visual(Point { x: 2, y: 1 }); + assert_eq!( + cursor, + Cursor { + offset: 8, + logical_pos: Point { x: 8, y: 0 }, + visual_pos: Point { x: 2, y: 1 }, + column: 8, + wrap_opp: false, + } + ); + } + + #[test] + fn test_newlines_and_strip() { + // Offset line 0: 0 + // Offset line 1: 6 + // Offset line 2: 13 + // Offset line 3: 18 + let text = "line1\nline2\r\nline3".as_bytes(); + + assert_eq!(newlines_forward(text, 0, 0, 2), (13, 2)); + assert_eq!(newlines_forward(text, 0, 0, 0), (0, 0)); + assert_eq!(newlines_forward(text, 100, 2, 100), (18, 2)); + + assert_eq!(newlines_backward(text, 18, 2, 1), (6, 1)); + assert_eq!(newlines_backward(text, 18, 2, 0), (0, 0)); + assert_eq!(newlines_backward(text, 100, 2, 1), (6, 1)); + } + + #[test] + fn test_strip_newline() { + assert_eq!(strip_newline(b"hello\n"), b"hello"); + assert_eq!(strip_newline(b"hello\r\n"), b"hello"); + assert_eq!(strip_newline(b"hello"), b"hello"); + } +} diff --git a/pkgs/edit/src/unicode/mod.rs b/pkgs/edit/src/unicode/mod.rs new file mode 100644 index 0000000..20cf301 --- /dev/null +++ b/pkgs/edit/src/unicode/mod.rs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Everything related to Unicode lives here. + +mod measurement; +mod tables; +mod utf8; + +pub use measurement::*; +pub use utf8::*; diff --git a/pkgs/edit/src/unicode/tables.rs b/pkgs/edit/src/unicode/tables.rs new file mode 100644 index 0000000..dc2ffa4 --- /dev/null +++ b/pkgs/edit/src/unicode/tables.rs @@ -0,0 +1,1109 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// BEGIN: Generated by grapheme-table-gen on 2025-03-31T16:50:08Z, from Unicode 16.0.0, with --lang=rust --extended --no-ambiguous --line-breaks, 16950 bytes +#[rustfmt::skip] +const STAGE0: [u16; 544] = [ + 0x0000, 0x0040, 0x007f, 0x00bf, 0x00ff, 0x013f, 0x017f, 0x0194, 0x0194, 0x01a6, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, + 0x0194, 0x0194, 0x0194, 0x0194, 0x01e6, 0x0226, 0x024a, 0x024b, 0x024c, 0x0246, 0x0255, 0x0295, 0x0295, 0x0295, 0x0295, 0x02cd, + 0x030d, 0x034d, 0x038d, 0x03cd, 0x040d, 0x0438, 0x0478, 0x049b, 0x04bc, 0x0295, 0x0295, 0x0295, 0x04f4, 0x0534, 0x0194, 0x0194, + 0x0574, 0x05b4, 0x0295, 0x0295, 0x0295, 0x05dd, 0x061d, 0x063d, 0x0295, 0x0663, 0x06a3, 0x06e3, 0x0723, 0x0763, 0x07a3, 0x07e3, + 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, + 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0823, + 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, + 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0194, 0x0823, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0863, 0x0873, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, + 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, 0x0295, +]; +#[rustfmt::skip] +const STAGE1: [u16; 2227] = [ + 0x0000, 0x0008, 0x0010, 0x0018, 0x0020, 0x0028, 0x0030, 0x0030, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x003d, 0x0036, 0x0045, 0x0045, 0x004a, 0x0052, 0x005a, 0x0062, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x006a, 0x0036, 0x0036, 0x0036, 0x0036, 0x006e, 0x0073, 0x0036, 0x007a, 0x007f, 0x0087, 0x008d, 0x0095, 0x0036, 0x009d, 0x00a5, 0x0036, 0x0036, 0x00aa, 0x00b2, 0x00b9, 0x00be, 0x00c4, 0x0036, 0x0036, 0x00cb, 0x00d3, 0x00d9, + 0x00e1, 0x00e8, 0x00f0, 0x00f8, 0x00fd, 0x0036, 0x0105, 0x010d, 0x0115, 0x011b, 0x0123, 0x012b, 0x0133, 0x0139, 0x0141, 0x0149, 0x0151, 0x0157, 0x015f, 0x0167, 0x016f, 0x0175, 0x017d, 0x0185, 0x018d, 0x0193, 0x019b, 0x01a3, 0x01ab, 0x01b3, 0x01bb, 0x01c2, 0x01ca, 0x01d0, 0x01d8, 0x01e0, 0x01e8, 0x01ee, 0x01f6, 0x01fe, 0x0206, 0x020c, 0x0214, 0x021c, 0x0224, 0x022b, 0x0233, 0x023b, 0x0241, 0x0245, 0x024d, 0x0241, 0x0241, 0x0254, 0x025c, 0x0241, 0x0264, 0x026c, 0x0070, 0x0274, 0x027c, 0x0283, 0x028b, 0x0241, + 0x0292, 0x029a, 0x02a2, 0x02aa, 0x0036, 0x02b2, 0x0036, 0x02ba, 0x02ba, 0x02ba, 0x02c2, 0x02c2, 0x02c8, 0x02ca, 0x02ca, 0x0036, 0x0036, 0x02d2, 0x0036, 0x02da, 0x02de, 0x02e6, 0x0036, 0x02ec, 0x0036, 0x02f2, 0x02fa, 0x0302, 0x0036, 0x0036, 0x030a, 0x0312, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x031a, 0x0036, 0x0036, 0x0322, 0x032a, 0x0332, 0x033a, 0x0342, 0x0241, 0x0347, 0x034f, 0x0357, 0x035f, + 0x0036, 0x0036, 0x0367, 0x036f, 0x0375, 0x0036, 0x0379, 0x0037, 0x0381, 0x0389, 0x0241, 0x0241, 0x0241, 0x038d, 0x0036, 0x0395, 0x0241, 0x039d, 0x03a5, 0x03ad, 0x03b4, 0x03b9, 0x0241, 0x03c1, 0x03c4, 0x03cc, 0x03d4, 0x03dc, 0x03e4, 0x0241, 0x03ec, 0x0036, 0x03f3, 0x03fb, 0x0402, 0x00f8, 0x040a, 0x0412, 0x041a, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0422, 0x0426, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x030a, 0x0036, 0x042e, 0x0436, 0x0036, 0x043e, 0x0442, 0x044a, 0x0452, + 0x045a, 0x0462, 0x046a, 0x0472, 0x047a, 0x0482, 0x0486, 0x048e, 0x0496, 0x049d, 0x04a5, 0x04ac, 0x04b3, 0x04b7, 0x0036, 0x04bf, 0x04c7, 0x04cf, 0x04d7, 0x04df, 0x04e6, 0x0036, 0x04ee, 0x04f4, 0x04fb, 0x0036, 0x0036, 0x0501, 0x0036, 0x0506, 0x050c, 0x0036, 0x0513, 0x051b, 0x0241, 0x0241, 0x0241, 0x0523, 0x0524, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x052c, 0x0534, 0x053c, 0x0544, 0x054c, 0x0554, 0x055c, 0x0564, 0x056c, 0x0574, 0x057c, 0x0584, 0x058b, 0x0592, 0x059a, 0x05a0, 0x05a8, 0x05b0, 0x05b7, 0x0036, + 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x05bb, 0x0036, 0x0036, 0x05c3, 0x0036, 0x05ca, 0x05d1, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x05d9, 0x0036, 0x05e1, 0x05e8, 0x05ee, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x05f4, 0x0036, 0x02b2, 0x0036, 0x05fc, 0x0604, 0x060c, 0x060c, 0x0045, 0x0614, 0x061c, 0x0624, 0x0241, 0x062c, 0x0633, 0x0633, 0x0636, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x063e, 0x0644, 0x064c, + 0x0654, 0x065c, 0x0664, 0x066c, 0x0674, 0x0664, 0x067c, 0x0684, 0x0688, 0x0633, 0x0633, 0x068d, 0x0633, 0x0633, 0x0694, 0x069c, 0x0633, 0x06a4, 0x0633, 0x06a8, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x06b0, 0x06b0, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x06b8, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x06be, 0x0633, 0x06c5, 0x0402, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x06ca, 0x06d2, 0x0036, 0x06da, 0x06e2, 0x0036, 0x0036, 0x06ea, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x06f2, 0x06fa, 0x0702, 0x070a, 0x0036, 0x0712, 0x071a, 0x071d, 0x0724, 0x072c, 0x00d3, 0x0734, 0x073b, 0x0743, 0x074b, 0x074f, 0x0757, 0x075f, 0x0241, 0x0766, 0x076e, 0x0776, 0x0241, 0x077e, 0x0786, 0x078e, 0x0796, 0x079e, + 0x0036, 0x07a3, 0x0036, 0x0036, 0x0036, 0x07ab, 0x07b3, 0x07b4, 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07b9, 0x07b3, 0x07b4, 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07b9, 0x07b3, 0x07b4, 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07b9, 0x07b3, 0x07b4, 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07b9, 0x07b3, 0x07b4, 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07b9, 0x07b3, 0x07b4, 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07b9, 0x07b3, 0x07b4, 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07b9, 0x07b3, 0x07b4, 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07b9, 0x07b3, 0x07b4, + 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07b9, 0x07b3, 0x07b4, 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07b9, 0x07b3, 0x07b4, 0x07b5, 0x07b6, 0x07b7, 0x07b8, 0x07c0, 0x07c7, 0x07ca, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x07d2, 0x07da, 0x07e2, 0x0036, 0x0036, 0x0036, 0x07ea, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x07ef, 0x0036, 0x0036, 0x05e9, 0x0036, 0x07f7, 0x07fb, 0x0803, 0x080b, 0x0812, + 0x081a, 0x0036, 0x0036, 0x0036, 0x0820, 0x0828, 0x0830, 0x0838, 0x0840, 0x0845, 0x084d, 0x0855, 0x085d, 0x006f, 0x0865, 0x086d, 0x0241, 0x0036, 0x0036, 0x0036, 0x07e4, 0x0875, 0x0878, 0x0036, 0x0036, 0x087e, 0x0240, 0x0886, 0x088a, 0x0241, 0x0241, 0x0241, 0x0241, 0x0892, 0x0036, 0x0895, 0x089d, 0x0036, 0x08a3, 0x00f8, 0x08a7, 0x08af, 0x0036, 0x08b7, 0x0241, 0x0036, 0x0036, 0x0036, 0x0036, 0x0436, 0x0363, 0x08bf, 0x08c5, 0x0036, 0x08ca, 0x0036, 0x08d1, 0x08d5, 0x08da, 0x0036, 0x08e2, 0x0036, 0x0036, 0x0036, + 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0604, 0x0379, 0x08e5, 0x0061, 0x08ed, 0x0241, 0x0241, 0x08f5, 0x08f8, 0x0900, 0x0036, 0x0037, 0x0908, 0x0241, 0x0910, 0x0917, 0x091f, 0x0241, 0x0241, 0x0036, 0x0927, 0x05e9, 0x0036, 0x092f, 0x0936, 0x093e, 0x0036, 0x0036, 0x0241, 0x0036, 0x0946, 0x0036, 0x094e, 0x0438, 0x0956, 0x095c, 0x0964, 0x0241, 0x0241, 0x0036, 0x0036, 0x096c, 0x0241, 0x0036, 0x07e6, 0x0036, 0x0974, 0x0036, 0x097b, 0x00d3, 0x0983, 0x098a, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, + 0x0037, 0x0036, 0x0992, 0x099a, 0x099c, 0x0036, 0x08ca, 0x09a4, 0x0886, 0x09ac, 0x0886, 0x08e4, 0x0604, 0x09b4, 0x09b6, 0x09bd, 0x09c4, 0x03dc, 0x09cc, 0x09d4, 0x09da, 0x09e2, 0x09e9, 0x09f1, 0x09f5, 0x03dc, 0x09fd, 0x0a05, 0x0a0d, 0x005e, 0x0a15, 0x0a1d, 0x0241, 0x0a25, 0x0a2d, 0x0063, 0x0a35, 0x0a3d, 0x0a3f, 0x0a47, 0x0a4f, 0x0241, 0x0a55, 0x0a5d, 0x0a65, 0x0036, 0x0a6d, 0x0a75, 0x0a7d, 0x0036, 0x0a85, 0x0a8d, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0036, 0x0a95, 0x0a9d, 0x0241, 0x0036, 0x0aa5, 0x0aad, + 0x0ab5, 0x0036, 0x0abd, 0x0ac5, 0x0acc, 0x0acd, 0x0ad5, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0036, 0x0add, 0x0241, 0x0241, 0x0241, 0x0036, 0x0036, 0x0ae5, 0x0241, 0x0aed, 0x0af5, 0x0241, 0x0241, 0x05eb, 0x0afd, 0x0b05, 0x0b0d, 0x0b11, 0x0b19, 0x0036, 0x0b20, 0x0b28, 0x0036, 0x0367, 0x0b30, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0036, 0x0b38, 0x0b40, 0x0b45, 0x0b4d, 0x0b54, 0x0b59, 0x0b5f, 0x0241, 0x0241, 0x0b67, 0x0b6b, 0x0b73, 0x0b7b, 0x0b81, 0x0b89, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0241, 0x0b8d, 0x0b95, 0x0b98, 0x0ba0, 0x0241, 0x0241, 0x0ba7, 0x0baf, 0x0bb7, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0302, 0x0241, 0x0241, 0x0241, 0x0036, 0x0036, 0x0036, 0x0bbf, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0bc7, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0886, 0x0036, 0x0036, 0x07e6, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, + 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0bcf, 0x0036, 0x0bd7, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0bda, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0be1, 0x0be9, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, + 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x07e4, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0bf1, 0x0036, 0x0036, 0x0036, 0x0bf8, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0bfa, 0x0c02, 0x0241, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, + 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0367, 0x0037, 0x0c0a, 0x0036, 0x0037, 0x0363, 0x0c0f, 0x0036, 0x0c17, 0x0c1e, 0x0c26, 0x08e3, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0036, 0x0c2e, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0036, 0x0036, 0x0c36, 0x0241, 0x0241, 0x0241, 0x0036, 0x0036, 0x0c3e, 0x0c43, 0x0c49, 0x0241, 0x0241, 0x0c51, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0635, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x06b0, 0x0c59, 0x0c5f, 0x0c67, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0c6b, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0c73, 0x0c78, 0x0c7f, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0634, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0036, 0x0036, 0x0036, 0x0c87, 0x0c8c, 0x0c94, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0c9c, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x08e2, 0x0241, 0x0241, 0x0045, 0x0ca4, 0x0cab, 0x0036, 0x0036, 0x0036, 0x0bc7, 0x0241, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0379, 0x0036, 0x0cb2, 0x0036, 0x0cb9, 0x0cc1, 0x0cc7, 0x0036, 0x051b, 0x0036, 0x0036, 0x0ccf, 0x0241, 0x0241, 0x0241, 0x08e2, 0x08e2, 0x06b0, 0x06b0, 0x0cd7, 0x0cdf, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0036, 0x0036, 0x043e, 0x0036, 0x0ce7, 0x0cef, 0x0cf7, 0x0036, 0x0cfe, 0x0cf9, 0x0d06, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0d0d, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0d12, 0x0d16, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0045, 0x0d1e, 0x0045, 0x0d25, 0x0d2c, 0x0d34, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0037, 0x0d3b, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0d43, 0x0d4b, 0x0036, 0x0d50, 0x0d55, 0x0241, 0x0241, 0x0241, 0x0036, 0x0d5d, 0x0d65, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0886, 0x0d6d, 0x0036, 0x0d75, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0886, 0x0d7d, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0886, 0x0d85, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0d8d, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0036, 0x0d95, 0x0241, 0x0036, 0x0036, 0x0d9d, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, + 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0da5, 0x0036, 0x0daa, 0x0241, 0x0241, 0x0db2, 0x0436, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0cf7, 0x0dba, 0x0dc2, 0x0dca, 0x0dd2, 0x0dda, 0x0241, 0x0b34, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0241, 0x0de2, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de9, 0x0de4, 0x0df1, 0x0df6, 0x0241, 0x0dfc, 0x0e04, 0x0e0b, 0x0de4, 0x0e12, 0x0e1a, 0x0e21, 0x0e29, 0x0e31, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0e39, 0x0e41, 0x0e39, 0x0e47, 0x0e4f, + 0x0e57, 0x0e5f, 0x0e67, 0x0e39, 0x0e6f, 0x0e77, 0x0e39, 0x0e39, 0x0e7f, 0x0e39, 0x0e84, 0x0e8c, 0x0e93, 0x0e9b, 0x0ea1, 0x0ea8, 0x0de2, 0x0eae, 0x0eb5, 0x0e39, 0x0e39, 0x0ebc, 0x0ec0, 0x0e39, 0x0e39, 0x0ec8, 0x0ed0, 0x0036, 0x0036, 0x0036, 0x0ed8, 0x0036, 0x0036, 0x0ee0, 0x0ee8, 0x0ef0, 0x0036, 0x0ef6, 0x0036, 0x0efe, 0x0f03, 0x0f0b, 0x0f0c, 0x0f14, 0x0f17, 0x0f1e, 0x0e39, 0x0e39, 0x0e39, 0x0e39, 0x0e39, 0x0f26, 0x0f26, 0x0f29, 0x0f2e, 0x0f36, 0x0e39, 0x0f3d, 0x0f45, 0x0036, 0x0036, 0x0036, 0x0036, 0x0031, + 0x0036, 0x0036, 0x0c9c, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0de4, 0x0f4c, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, + 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0633, 0x0f54, 0x0f5c, 0x0045, 0x0045, 0x0045, 0x0020, 0x0020, 0x0020, 0x0020, 0x0045, 0x0045, 0x0045, 0x0045, 0x0045, 0x0045, 0x0045, 0x0f64, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, +]; +#[rustfmt::skip] +const STAGE2: [u16; 3948] = [ + 0x0000, 0x0004, 0x0008, 0x000c, 0x0010, 0x0014, 0x0018, 0x001c, + 0x0020, 0x0024, 0x0028, 0x002c, 0x0030, 0x0034, 0x0038, 0x003c, + 0x0040, 0x0044, 0x0048, 0x004c, 0x0050, 0x0054, 0x0058, 0x005c, + 0x0060, 0x0064, 0x0068, 0x006c, 0x0070, 0x0074, 0x0078, 0x007c, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0080, 0x0083, 0x0086, 0x008a, 0x008e, 0x0092, 0x0094, 0x0098, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x009c, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x009c, 0x00a0, + 0x00a4, 0x00a5, 0x0040, 0x00a9, 0x00ad, 0x00b1, 0x00b1, 0x00b1, + 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x00b2, 0x00b1, 0x00b1, + 0x00b1, 0x00b5, 0x00b6, 0x00b1, 0x00b1, 0x00b1, 0x0040, 0x0040, + 0x00ba, 0x00bc, 0x00a9, 0x0040, 0x009c, 0x00bf, 0x0040, 0x0040, + 0x0040, 0x0040, 0x00c1, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x00c4, 0x00b1, 0x00c7, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x00a5, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x009c, 0x00a5, 0x0040, 0x0040, 0x00ca, 0x00cd, 0x00d1, 0x00b1, + 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x00d3, 0x00c6, + 0x00d6, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, 0x009c, + 0x00aa, 0x0040, 0x0093, 0x00a9, 0x00a9, 0x00da, 0x00dc, 0x00df, + 0x003a, 0x00b1, 0x00b1, 0x00e3, 0x00e5, 0x0040, 0x0040, 0x00c4, + 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x0030, 0x0030, 0x00e9, + 0x00ec, 0x00f0, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00f4, + 0x00b1, 0x00f7, 0x00b1, 0x00fa, 0x00fd, 0x00c7, 0x0030, 0x0030, + 0x0101, 0x0040, 0x0040, 0x0040, 0x0103, 0x00ef, 0x0040, 0x0040, + 0x0040, 0x0040, 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x0107, 0x00a5, + 0x0040, 0x0040, 0x0040, 0x0040, 0x00c5, 0x00b1, 0x00b1, 0x010b, + 0x00a9, 0x00a9, 0x00a9, 0x0030, 0x0030, 0x0101, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x00c4, 0x00b1, 0x00b1, 0x0040, 0x010f, + 0x0112, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00c5, 0x0116, + 0x00b1, 0x0118, 0x0118, 0x011a, 0x0040, 0x0040, 0x0040, 0x009c, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0118, 0x00ab, + 0x0040, 0x0040, 0x009c, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, + 0x009c, 0x011e, 0x0120, 0x00b1, 0x00b1, 0x0040, 0x0040, 0x00c5, + 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x0123, 0x00b1, 0x00b1, + 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x0126, 0x0040, 0x0040, + 0x0040, 0x0040, 0x012a, 0x012b, 0x012b, 0x012b, 0x012b, 0x012b, + 0x012b, 0x012d, 0x0131, 0x0134, 0x00b1, 0x0137, 0x013a, 0x0118, + 0x00b1, 0x012b, 0x012b, 0x00c5, 0x013e, 0x0030, 0x0030, 0x0040, + 0x0040, 0x012b, 0x012b, 0x0142, 0x00a5, 0x0040, 0x0146, 0x0146, + 0x012a, 0x012b, 0x012b, 0x014a, 0x012b, 0x014d, 0x0150, 0x0152, + 0x0131, 0x0134, 0x0156, 0x0159, 0x015c, 0x00a9, 0x015f, 0x00a9, + 0x014c, 0x00c5, 0x0163, 0x0030, 0x0030, 0x0167, 0x0040, 0x016b, + 0x016f, 0x0172, 0x00a5, 0x009c, 0x00aa, 0x0146, 0x0040, 0x0040, + 0x0040, 0x00bf, 0x0040, 0x00bf, 0x00c0, 0x00a7, 0x0176, 0x0179, + 0x0120, 0x017b, 0x011a, 0x0155, 0x00a9, 0x00a5, 0x017f, 0x00a9, + 0x0163, 0x0030, 0x0030, 0x00c7, 0x0183, 0x00a9, 0x00a9, 0x0172, + 0x00a5, 0x0040, 0x00c1, 0x00c1, 0x012a, 0x012b, 0x012b, 0x014a, + 0x012b, 0x014a, 0x0186, 0x0152, 0x0131, 0x0134, 0x0108, 0x018a, + 0x018d, 0x0093, 0x00a9, 0x00a9, 0x00a9, 0x00c5, 0x0163, 0x0030, + 0x0030, 0x0191, 0x00a9, 0x0194, 0x00b1, 0x0198, 0x00a5, 0x0040, + 0x0146, 0x0146, 0x012a, 0x012b, 0x012b, 0x014a, 0x012b, 0x014a, + 0x0186, 0x0152, 0x019c, 0x0134, 0x0156, 0x0159, 0x018d, 0x00a9, + 0x0172, 0x00a9, 0x014c, 0x00c5, 0x0163, 0x0030, 0x0030, 0x01a0, + 0x0040, 0x00a9, 0x00a9, 0x017c, 0x00a5, 0x009c, 0x00ba, 0x00bf, + 0x00a7, 0x00c0, 0x00bf, 0x00aa, 0x0093, 0x009c, 0x00ba, 0x0040, + 0x0040, 0x00a7, 0x01a4, 0x01a8, 0x01a4, 0x01aa, 0x01ad, 0x0093, + 0x015f, 0x00a9, 0x00a9, 0x0163, 0x0030, 0x0030, 0x0040, 0x0040, + 0x01b1, 0x00a9, 0x0137, 0x00f0, 0x0040, 0x00bf, 0x00bf, 0x012a, + 0x012b, 0x012b, 0x014a, 0x012b, 0x012b, 0x012b, 0x0152, 0x00fd, + 0x0137, 0x01b5, 0x0171, 0x01b8, 0x00a9, 0x01bb, 0x01bf, 0x01c2, + 0x00c5, 0x0163, 0x0030, 0x0030, 0x00a9, 0x00a1, 0x0040, 0x0040, + 0x0142, 0x01c6, 0x0040, 0x00bf, 0x00bf, 0x0040, 0x0040, 0x0040, + 0x00bf, 0x0040, 0x0040, 0x00a5, 0x00a7, 0x019c, 0x01ca, 0x01cd, + 0x01aa, 0x011a, 0x00a9, 0x01d1, 0x00a9, 0x00c0, 0x00c5, 0x0163, + 0x0030, 0x0030, 0x01d4, 0x00a9, 0x00a9, 0x00a9, 0x0136, 0x0040, + 0x0040, 0x00bf, 0x00bf, 0x012a, 0x012b, 0x012b, 0x012b, 0x012b, + 0x012b, 0x012b, 0x012c, 0x0131, 0x0134, 0x0176, 0x01aa, 0x01d7, + 0x00a9, 0x01c7, 0x0040, 0x0040, 0x00c5, 0x0163, 0x0030, 0x0030, + 0x0040, 0x0040, 0x01da, 0x0040, 0x0198, 0x00a5, 0x0040, 0x0040, + 0x0040, 0x009c, 0x00ba, 0x0040, 0x0040, 0x0040, 0x0040, 0x00c1, + 0x0040, 0x0040, 0x01c2, 0x0040, 0x009c, 0x0154, 0x015f, 0x0133, + 0x01de, 0x01ca, 0x01ca, 0x00a9, 0x0163, 0x0030, 0x0030, 0x01a4, + 0x0093, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x01df, 0x00b1, 0x0107, 0x01e3, 0x00a9, 0x0120, 0x00b1, + 0x01e7, 0x0030, 0x0030, 0x01eb, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x01df, 0x00b1, 0x00b1, 0x01ef, 0x00a9, 0x00a9, 0x00b1, 0x0107, + 0x0030, 0x0030, 0x01f3, 0x00a9, 0x01f7, 0x01fa, 0x01fe, 0x0202, + 0x0204, 0x003f, 0x00c7, 0x0040, 0x0030, 0x0030, 0x0101, 0x0040, + 0x0040, 0x0208, 0x020a, 0x020c, 0x0040, 0x0040, 0x0040, 0x0093, + 0x00d1, 0x00b1, 0x00b1, 0x0210, 0x00b1, 0x00d4, 0x0040, 0x0118, + 0x00b1, 0x00b1, 0x00d1, 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x00b1, + 0x00b1, 0x00b1, 0x0214, 0x0040, 0x00ee, 0x0040, 0x00bf, 0x0218, + 0x0040, 0x021c, 0x00a9, 0x00a9, 0x00a9, 0x00d1, 0x0220, 0x00b1, + 0x0172, 0x0179, 0x0030, 0x0030, 0x01eb, 0x0040, 0x00a9, 0x01a4, + 0x011a, 0x0121, 0x01ef, 0x00a9, 0x00a9, 0x00a9, 0x00d1, 0x01ef, + 0x00a9, 0x00a9, 0x0154, 0x0179, 0x00a9, 0x0155, 0x0030, 0x0030, + 0x01f3, 0x0155, 0x0040, 0x00c1, 0x00a9, 0x01c2, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0224, 0x0224, 0x0224, 0x0224, 0x0224, 0x0224, + 0x0224, 0x0224, 0x0228, 0x0228, 0x0228, 0x0228, 0x0228, 0x0228, + 0x0228, 0x0228, 0x022c, 0x022c, 0x022c, 0x022c, 0x022c, 0x022c, + 0x022c, 0x022c, 0x0040, 0x0040, 0x00bf, 0x00a7, 0x0040, 0x009c, + 0x00bf, 0x00a7, 0x0040, 0x0040, 0x00bf, 0x00a7, 0x0040, 0x0040, + 0x0040, 0x0040, 0x00bf, 0x00a7, 0x0040, 0x009c, 0x00bf, 0x00a7, + 0x0040, 0x0040, 0x0040, 0x009c, 0x0040, 0x0040, 0x0040, 0x0040, + 0x00bf, 0x00a7, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x009c, 0x00d1, 0x0230, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0093, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x00a7, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00a7, + 0x0040, 0x00a7, 0x0231, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0231, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0058, 0x0235, 0x0040, 0x0040, 0x0239, 0x023c, 0x0040, 0x0040, + 0x0093, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, 0x00c5, 0x0240, + 0x00a9, 0x00aa, 0x0040, 0x0040, 0x0040, 0x0040, 0x00c5, 0x0244, + 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, 0x00c5, 0x00a9, + 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, 0x00bf, 0x0248, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0173, 0x00b1, 0x0136, 0x01ca, + 0x01a6, 0x0134, 0x00b1, 0x00b1, 0x024c, 0x0250, 0x0155, 0x0030, + 0x0030, 0x01f3, 0x00a9, 0x0040, 0x0040, 0x00a7, 0x00a9, 0x0254, + 0x0258, 0x025c, 0x025f, 0x0030, 0x0030, 0x01f3, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0093, 0x00a9, 0x0040, + 0x00c6, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0183, + 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00a7, 0x00a9, + 0x00a9, 0x0126, 0x0263, 0x0137, 0x00a9, 0x01a6, 0x01ca, 0x0134, + 0x00a9, 0x0093, 0x00e7, 0x0030, 0x0030, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x0030, 0x0030, 0x0267, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x00c4, 0x0199, 0x00ba, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x026a, 0x00b1, 0x0107, 0x01de, 0x00d1, 0x00b1, + 0x0137, 0x0263, 0x00b1, 0x00b1, 0x017b, 0x0030, 0x0030, 0x01f3, + 0x00a9, 0x0030, 0x0030, 0x01f3, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x0107, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00b1, 0x01e2, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x0220, 0x0126, 0x0137, 0x01a6, 0x01e2, 0x00a9, 0x026e, + 0x00a9, 0x00a9, 0x026e, 0x0272, 0x0275, 0x0276, 0x0277, 0x00b1, + 0x00b1, 0x0276, 0x0276, 0x0272, 0x0127, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x027b, 0x0136, 0x0173, 0x00c7, + 0x0030, 0x0030, 0x0101, 0x0040, 0x00a9, 0x027f, 0x0136, 0x0282, + 0x0136, 0x00a9, 0x00a9, 0x0040, 0x01ca, 0x01ca, 0x00b1, 0x00b1, + 0x0133, 0x0286, 0x0289, 0x0030, 0x0030, 0x01f3, 0x00a5, 0x0030, + 0x0030, 0x0101, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x023a, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x009c, 0x00a5, 0x0040, 0x0040, 0x00a9, 0x00a9, 0x01e7, 0x00b1, + 0x00b1, 0x00b1, 0x0220, 0x00b1, 0x00f0, 0x00ef, 0x0040, 0x028d, + 0x0291, 0x00a9, 0x00b1, 0x00b1, 0x00b1, 0x0295, 0x00b1, 0x00b1, + 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x0296, 0x0040, 0x00a7, + 0x0040, 0x00a7, 0x0040, 0x0040, 0x00ac, 0x00ac, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00a7, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x00bf, 0x0040, 0x0040, 0x0040, 0x00ba, + 0x0040, 0x00a5, 0x0040, 0x0040, 0x0040, 0x0040, 0x00ba, 0x00bf, + 0x0040, 0x029a, 0x0289, 0x029e, 0x02a2, 0x02a6, 0x02a0, 0x00aa, + 0x02aa, 0x02aa, 0x00ba, 0x02ae, 0x02b2, 0x02b4, 0x02b8, 0x02b8, + 0x02bc, 0x02c0, 0x0040, 0x02c4, 0x02c7, 0x0040, 0x0040, 0x02c9, + 0x0289, 0x02cd, 0x02d1, 0x02d4, 0x00b1, 0x00b1, 0x00a7, 0x00a5, + 0x0040, 0x02d8, 0x0093, 0x00a5, 0x0040, 0x02d8, 0x0040, 0x0040, + 0x0040, 0x0093, 0x02dc, 0x02dd, 0x02dc, 0x02dc, 0x02dc, 0x02de, + 0x02dd, 0x02de, 0x02e0, 0x02dc, 0x02dc, 0x02dc, 0x00b1, 0x00b1, + 0x00b1, 0x00b1, 0x01ef, 0x00a9, 0x00a9, 0x00a9, 0x02e4, 0x00bf, + 0x01da, 0x0040, 0x009c, 0x02e8, 0x0040, 0x0040, 0x02eb, 0x0040, + 0x009c, 0x0040, 0x0040, 0x0040, 0x02ee, 0x0040, 0x0040, 0x0040, + 0x0040, 0x00a9, 0x00a9, 0x00a9, 0x00aa, 0x00a9, 0x00a9, 0x00a9, + 0x0040, 0x00a9, 0x00a9, 0x00ba, 0x0040, 0x0040, 0x00bf, 0x00a9, + 0x00a9, 0x02f2, 0x02f4, 0x0040, 0x0040, 0x02f7, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x00c1, 0x00a5, 0x0040, 0x0040, 0x01c2, + 0x009c, 0x00c0, 0x009c, 0x02fa, 0x00bf, 0x00c1, 0x0093, 0x00c0, + 0x017f, 0x00a9, 0x00ac, 0x0040, 0x00a9, 0x0040, 0x00ba, 0x0040, + 0x0040, 0x00a5, 0x00a5, 0x00c1, 0x0040, 0x0040, 0x0040, 0x00ba, + 0x00a9, 0x00a7, 0x00a7, 0x0040, 0x0040, 0x0040, 0x0040, 0x00a7, + 0x00a7, 0x0040, 0x0040, 0x0040, 0x00bf, 0x00bf, 0x0040, 0x00bf, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x009c, 0x0040, 0x0040, + 0x0040, 0x02fe, 0x0040, 0x0040, 0x0040, 0x0040, 0x0302, 0x0040, + 0x00c1, 0x0040, 0x0306, 0x0040, 0x0040, 0x030a, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x030e, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x030f, 0x0040, 0x0040, 0x0040, 0x0040, 0x0313, 0x0316, + 0x031a, 0x0040, 0x031e, 0x0040, 0x0040, 0x00a7, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x009c, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x0322, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00aa, 0x00ab, 0x00a9, 0x0325, 0x0040, + 0x00a7, 0x0329, 0x0040, 0x00ba, 0x032b, 0x00a7, 0x00c0, 0x00a7, + 0x00ba, 0x0040, 0x0040, 0x0040, 0x00a7, 0x00ba, 0x0040, 0x009c, + 0x0040, 0x0040, 0x030f, 0x032f, 0x0333, 0x0337, 0x033a, 0x033c, + 0x031e, 0x0340, 0x0344, 0x0333, 0x0348, 0x0348, 0x0348, 0x0348, + 0x034c, 0x034c, 0x0350, 0x0348, 0x0354, 0x0348, 0x034c, 0x034c, + 0x034c, 0x0348, 0x0348, 0x0348, 0x0358, 0x0358, 0x035c, 0x0358, + 0x0348, 0x0348, 0x0348, 0x0317, 0x0348, 0x0327, 0x0360, 0x0362, + 0x0349, 0x0348, 0x0348, 0x033c, 0x0366, 0x0348, 0x034a, 0x0348, + 0x0348, 0x0348, 0x0348, 0x0369, 0x0333, 0x036a, 0x036d, 0x0370, + 0x0373, 0x0377, 0x036c, 0x037b, 0x0335, 0x0348, 0x037f, 0x02f2, + 0x0382, 0x0386, 0x0389, 0x038c, 0x0333, 0x038f, 0x0393, 0x0346, + 0x031e, 0x0397, 0x0040, 0x02ee, 0x0040, 0x039b, 0x0040, 0x030f, + 0x030e, 0x0040, 0x0040, 0x039f, 0x0040, 0x03a3, 0x03a6, 0x03a9, + 0x03ad, 0x03b0, 0x03b3, 0x0347, 0x0302, 0x0302, 0x0302, 0x03b7, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0313, 0x0040, 0x0040, + 0x02ee, 0x0040, 0x0040, 0x0040, 0x039b, 0x0040, 0x0040, 0x03a6, + 0x0040, 0x03bb, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x03be, 0x0302, 0x0302, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0327, 0x0040, 0x0040, 0x0058, 0x03c1, 0x03c1, 0x03c1, 0x03c1, + 0x03c1, 0x03c5, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0302, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0304, 0x0040, 0x03c8, 0x0040, 0x0040, 0x0040, 0x0040, 0x03a6, + 0x039b, 0x0040, 0x0040, 0x0040, 0x0040, 0x039b, 0x03cc, 0x00ba, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00ba, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x00c1, 0x0040, 0x0040, 0x0040, 0x00c4, + 0x00c7, 0x00a9, 0x03cf, 0x03d2, 0x0040, 0x0040, 0x00a9, 0x00aa, + 0x03d5, 0x00a9, 0x00a9, 0x0120, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x009c, 0x00a9, 0x00a9, 0x0040, 0x009c, 0x0040, 0x009c, + 0x0040, 0x009c, 0x0040, 0x009c, 0x03b0, 0x03b0, 0x03b0, 0x03d9, + 0x0289, 0x03db, 0x03df, 0x03e3, 0x03e7, 0x0302, 0x03e9, 0x03eb, + 0x03db, 0x0231, 0x00a7, 0x03ef, 0x03f3, 0x0289, 0x03ef, 0x03f1, + 0x003c, 0x03f7, 0x03f9, 0x03fd, 0x0401, 0x0401, 0x0401, 0x0401, + 0x0401, 0x0401, 0x0403, 0x0401, 0x0401, 0x0401, 0x0401, 0x0401, + 0x0401, 0x0401, 0x0401, 0x00a9, 0x00a9, 0x00a9, 0x0401, 0x0401, + 0x0401, 0x0401, 0x0401, 0x0406, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x0401, 0x0401, 0x0401, 0x0401, 0x040a, 0x040d, 0x0411, 0x0411, + 0x0413, 0x0411, 0x0411, 0x0417, 0x0401, 0x0401, 0x041b, 0x041d, + 0x0421, 0x0424, 0x0426, 0x0429, 0x042d, 0x042f, 0x0424, 0x0401, + 0x0401, 0x0401, 0x0401, 0x0401, 0x0422, 0x0401, 0x0401, 0x0401, + 0x0401, 0x0401, 0x0401, 0x0401, 0x0422, 0x042f, 0x0401, 0x0423, + 0x0401, 0x0431, 0x0434, 0x0437, 0x043b, 0x042f, 0x0424, 0x0401, + 0x0401, 0x0401, 0x0401, 0x0401, 0x0422, 0x042f, 0x0401, 0x0423, + 0x0401, 0x043d, 0x0426, 0x0441, 0x00a9, 0x0400, 0x0401, 0x0401, + 0x0401, 0x0401, 0x0401, 0x0401, 0x0400, 0x0401, 0x0401, 0x0401, + 0x0402, 0x0401, 0x0401, 0x0401, 0x0401, 0x0406, 0x00a9, 0x0445, + 0x0449, 0x0449, 0x0449, 0x0449, 0x0401, 0x0401, 0x0401, 0x0401, + 0x0401, 0x0401, 0x0401, 0x0402, 0x0401, 0x0401, 0x00a9, 0x00a9, + 0x0401, 0x0401, 0x0401, 0x0401, 0x0401, 0x044d, 0x044f, 0x0401, + 0x0362, 0x0362, 0x0362, 0x0362, 0x0362, 0x0362, 0x0362, 0x0362, + 0x0401, 0x0401, 0x0401, 0x0401, 0x0401, 0x040d, 0x0401, 0x0401, + 0x0401, 0x0444, 0x0401, 0x0401, 0x0401, 0x0401, 0x0402, 0x00a9, + 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, 0x0453, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0030, 0x0030, 0x0101, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, 0x00c4, 0x01e7, 0x00b1, + 0x00b1, 0x00c7, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x00c5, 0x0040, 0x0040, 0x0040, 0x0040, 0x0457, 0x0289, + 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, 0x00a7, 0x00c1, 0x00a5, + 0x0040, 0x0093, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00ba, 0x0040, + 0x0040, 0x0040, 0x00ee, 0x00ee, 0x00c4, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x01c7, 0x045b, 0x0040, 0x01ef, 0x0040, 0x0040, + 0x045f, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0463, + 0x00a9, 0x00a9, 0x0467, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x01ca, 0x01ca, 0x01ca, 0x011a, 0x00a9, 0x026e, + 0x0030, 0x0030, 0x01f3, 0x00a9, 0x00b1, 0x00b1, 0x00b1, 0x00b1, + 0x00c7, 0x0040, 0x0040, 0x046b, 0x0040, 0x00c5, 0x00b1, 0x024a, + 0x0040, 0x0040, 0x0040, 0x0040, 0x00c4, 0x00b1, 0x00b1, 0x0136, + 0x00a9, 0x00a9, 0x00aa, 0x0224, 0x0224, 0x0224, 0x0224, 0x0224, + 0x0224, 0x0224, 0x046f, 0x0126, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x0120, 0x0133, 0x0136, 0x0136, 0x0473, + 0x0474, 0x0274, 0x0478, 0x00a9, 0x00a9, 0x00a9, 0x047c, 0x00a9, + 0x0155, 0x00a9, 0x00a9, 0x0030, 0x0030, 0x01f3, 0x00a9, 0x00a9, + 0x00d1, 0x0126, 0x045b, 0x0179, 0x00a9, 0x00a9, 0x028a, 0x0289, + 0x0289, 0x0240, 0x00a9, 0x00a9, 0x00a9, 0x0272, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x01ef, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x0171, 0x017b, 0x01ef, 0x0121, 0x0155, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, + 0x01c7, 0x0136, 0x023c, 0x0480, 0x00a9, 0x00a9, 0x00a5, 0x009c, + 0x00a5, 0x009c, 0x00a5, 0x009c, 0x00a9, 0x00a9, 0x0040, 0x009c, + 0x0040, 0x009c, 0x0040, 0x0040, 0x0040, 0x0040, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0040, 0x01c7, 0x01a7, 0x0484, 0x01ad, 0x0030, + 0x0030, 0x01f3, 0x00a9, 0x0488, 0x0489, 0x0489, 0x0489, 0x0489, + 0x0489, 0x0489, 0x0488, 0x0489, 0x0489, 0x0489, 0x0489, 0x0489, + 0x0489, 0x00a9, 0x00a9, 0x00a9, 0x0228, 0x0228, 0x0228, 0x0228, + 0x048d, 0x0490, 0x022c, 0x022c, 0x022c, 0x022c, 0x022c, 0x022c, + 0x022c, 0x00a9, 0x0040, 0x009c, 0x00a9, 0x00a9, 0x00aa, 0x0040, + 0x00a9, 0x0182, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x009c, + 0x0040, 0x017f, 0x00c1, 0x00bf, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x009c, 0x00a9, 0x00a9, 0x00a9, 0x00aa, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0494, 0x0040, + 0x0040, 0x00a9, 0x00aa, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0498, 0x00b1, 0x00b1, 0x00b1, 0x049c, 0x04a0, + 0x04a3, 0x04a7, 0x00a9, 0x04ab, 0x04ad, 0x04ac, 0x04ae, 0x0401, + 0x0410, 0x04b2, 0x04b2, 0x04b6, 0x04ba, 0x0401, 0x04be, 0x04c2, + 0x0410, 0x0412, 0x0401, 0x0402, 0x04c6, 0x00a9, 0x0040, 0x00bf, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x04ca, + 0x04ce, 0x04d2, 0x0413, 0x04d6, 0x0401, 0x0401, 0x04d9, 0x04dd, + 0x0401, 0x0401, 0x0401, 0x0401, 0x0401, 0x0401, 0x04e1, 0x04d7, + 0x0401, 0x0401, 0x0401, 0x0401, 0x0401, 0x0401, 0x04e1, 0x04e5, + 0x04e9, 0x04ec, 0x00a9, 0x00a9, 0x04ef, 0x0276, 0x0276, 0x0276, + 0x0276, 0x0276, 0x0276, 0x0276, 0x04f1, 0x0276, 0x0276, 0x0276, + 0x0276, 0x0276, 0x0276, 0x0276, 0x04f5, 0x047c, 0x0276, 0x047c, + 0x0276, 0x047c, 0x0276, 0x047c, 0x04f7, 0x04fb, 0x04fe, 0x0040, + 0x009c, 0x0000, 0x0000, 0x02b3, 0x00a9, 0x0040, 0x009c, 0x0040, + 0x0040, 0x0040, 0x0040, 0x009c, 0x00c1, 0x0040, 0x0040, 0x0040, + 0x00a7, 0x0040, 0x0040, 0x0040, 0x00a7, 0x0502, 0x00aa, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00aa, 0x0040, 0x0040, + 0x0040, 0x009c, 0x0040, 0x0040, 0x0040, 0x0093, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0506, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0093, 0x00a9, 0x00a9, 0x00a9, 0x00f0, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x00a9, 0x00a9, 0x00a5, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x00c5, 0x0107, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x050a, 0x0040, + 0x00a9, 0x0040, 0x0040, 0x0231, 0x00a7, 0x00a9, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x00a9, 0x00a9, 0x0040, 0x0040, + 0x0040, 0x0040, 0x00a9, 0x00a9, 0x00aa, 0x0040, 0x0040, 0x009c, + 0x0040, 0x009c, 0x00c1, 0x0040, 0x0040, 0x0040, 0x00c1, 0x0040, + 0x00c1, 0x0093, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x0040, 0x00bf, 0x0040, 0x009c, 0x00a9, 0x0040, 0x00a7, 0x00bf, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00c1, 0x0093, 0x0146, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x050a, 0x0040, 0x0040, + 0x00a9, 0x00aa, 0x0040, 0x0040, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x0040, 0x0040, 0x0040, 0x0040, 0x009c, 0x00a7, 0x00aa, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0286, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00a7, 0x00aa, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00a9, 0x0040, 0x0118, + 0x01bb, 0x00a9, 0x00b1, 0x0040, 0x00a5, 0x00a5, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x00a7, 0x0107, 0x0120, 0x0040, 0x0040, + 0x0093, 0x00a9, 0x0289, 0x0289, 0x0093, 0x00a9, 0x0040, 0x050e, + 0x00aa, 0x0040, 0x0289, 0x0512, 0x00a9, 0x00a9, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x00a7, 0x0288, 0x0289, 0x0040, 0x0040, + 0x0040, 0x0040, 0x009c, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, + 0x00a7, 0x00a9, 0x00a5, 0x0093, 0x00a9, 0x00a9, 0x00a5, 0x0040, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0093, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, + 0x009c, 0x00a9, 0x00ba, 0x0040, 0x00b1, 0x00a9, 0x00a9, 0x0030, + 0x0030, 0x01f3, 0x00a9, 0x0040, 0x00a7, 0x00d1, 0x0516, 0x0040, + 0x0040, 0x0040, 0x0040, 0x00a7, 0x00a9, 0x00ba, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0519, 0x051c, 0x00a7, 0x00a9, + 0x00a9, 0x00a9, 0x00ba, 0x0093, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00b1, 0x0040, 0x00c5, 0x00b1, 0x00b1, + 0x00f0, 0x0040, 0x00a7, 0x00a9, 0x00c5, 0x00c7, 0x00a7, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x026b, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00b1, 0x00b1, 0x00d2, 0x0275, + 0x04f6, 0x047c, 0x0276, 0x0276, 0x0276, 0x04f6, 0x00a9, 0x00a9, + 0x017b, 0x01ef, 0x00a9, 0x051e, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0263, 0x0126, 0x0290, 0x0522, 0x01ed, 0x00a9, 0x00a9, 0x0526, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0093, 0x00a9, 0x0030, 0x0030, + 0x01f3, 0x00a9, 0x01e7, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x00c4, 0x00b1, 0x0134, 0x00b1, 0x052a, 0x0030, + 0x0030, 0x0289, 0x052e, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x0040, 0x00c4, 0x029a, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x0040, 0x01c7, 0x0133, 0x00b1, 0x0126, 0x0530, 0x023b, 0x0534, + 0x019c, 0x0030, 0x0030, 0x0538, 0x02cd, 0x00a5, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0093, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x0263, 0x0136, 0x0220, 0x03db, 0x053c, 0x0506, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x009c, 0x00bf, + 0x00c1, 0x0040, 0x0040, 0x0040, 0x00c1, 0x0040, 0x0040, 0x053f, + 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, 0x0263, 0x00b1, 0x0107, + 0x00a9, 0x0030, 0x0030, 0x01f3, 0x00a9, 0x0136, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0120, 0x0543, 0x0137, + 0x0159, 0x0159, 0x0545, 0x00a9, 0x015f, 0x00a9, 0x047a, 0x01a4, + 0x0121, 0x00b1, 0x01ef, 0x00b1, 0x01ef, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x0547, 0x0263, 0x00b1, 0x01e0, 0x054b, 0x01cb, + 0x01a6, 0x054f, 0x0552, 0x04f7, 0x00a9, 0x01bb, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x01c9, 0x00b1, 0x00b1, 0x0133, 0x012f, 0x0239, + 0x03ef, 0x0030, 0x0030, 0x01eb, 0x0182, 0x00a7, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0263, 0x00b1, 0x0281, 0x0263, 0x0220, 0x0040, 0x00a9, + 0x00a9, 0x0030, 0x0030, 0x01f3, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x01c7, 0x0133, 0x011a, 0x01ca, 0x0173, 0x0556, 0x055a, 0x02cd, + 0x0289, 0x0289, 0x0289, 0x0040, 0x011a, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0263, 0x00b1, 0x0126, 0x0282, 0x055e, 0x0093, 0x00a9, + 0x00a9, 0x0030, 0x0030, 0x01f3, 0x00a9, 0x0562, 0x0562, 0x0562, + 0x00a0, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x00c4, + 0x01a7, 0x00b1, 0x0173, 0x00a7, 0x00a9, 0x0030, 0x0030, 0x01f3, + 0x00a9, 0x0030, 0x0030, 0x0030, 0x0030, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x021f, 0x0121, 0x0173, 0x00b1, + 0x00a9, 0x0030, 0x0030, 0x01f3, 0x0502, 0x0040, 0x0040, 0x0040, + 0x0263, 0x00b1, 0x00b1, 0x0290, 0x00a9, 0x0030, 0x0030, 0x0101, + 0x0040, 0x009c, 0x00a9, 0x00a9, 0x00aa, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x01ca, 0x01a9, 0x0566, 0x0569, 0x056d, 0x0502, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x0040, 0x01c9, 0x00b1, 0x0121, 0x01ca, 0x0299, 0x01e2, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0118, 0x00b1, 0x01e7, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00c4, 0x00b1, 0x0570, + 0x0573, 0x02cd, 0x0577, 0x00a9, 0x00a9, 0x0118, 0x0126, 0x0134, + 0x0040, 0x057b, 0x057d, 0x00b1, 0x00b1, 0x0126, 0x024a, 0x0560, + 0x0581, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0562, 0x0562, 0x0585, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x00a7, 0x00a9, 0x00a9, 0x00a9, 0x0030, 0x0030, 0x01f3, 0x00a9, + 0x0040, 0x0040, 0x00bf, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x01c7, 0x00b1, 0x0107, 0x00b1, 0x0173, 0x02cd, 0x0589, 0x00a9, + 0x00a9, 0x0030, 0x0030, 0x0101, 0x0040, 0x0040, 0x0040, 0x0093, + 0x058d, 0x0040, 0x0040, 0x0040, 0x0040, 0x0121, 0x00b1, 0x00b1, + 0x00b1, 0x0591, 0x00b1, 0x0220, 0x0179, 0x00a9, 0x00a9, 0x0040, + 0x009c, 0x00c1, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0118, + 0x0107, 0x0154, 0x0108, 0x00b1, 0x0593, 0x00a9, 0x00a9, 0x0030, + 0x0030, 0x01f3, 0x00a9, 0x0040, 0x00c1, 0x00bf, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x01c8, 0x01cb, 0x0596, 0x0282, 0x0093, + 0x00a9, 0x0030, 0x0030, 0x01f3, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x059a, 0x0484, 0x03d5, 0x00a9, 0x059d, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0133, 0x0107, 0x01a4, + 0x05a1, 0x0275, 0x0276, 0x0276, 0x00a9, 0x00a9, 0x0154, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x0093, 0x00a9, 0x00a9, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00df, 0x0498, + 0x0040, 0x0040, 0x0040, 0x00a7, 0x00a9, 0x00a9, 0x0286, 0x0040, + 0x0040, 0x0040, 0x009c, 0x0289, 0x03d5, 0x00a9, 0x00a9, 0x0040, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x05a5, 0x05a8, 0x05aa, + 0x03be, 0x0304, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x05ad, 0x0040, 0x0040, 0x0040, 0x0058, 0x00b5, 0x05b1, 0x05b5, + 0x05b9, 0x00f0, 0x00c4, 0x00b1, 0x00b1, 0x00b1, 0x011a, 0x00a9, + 0x00a9, 0x0040, 0x0040, 0x0040, 0x03be, 0x0040, 0x0040, 0x0040, + 0x0040, 0x009c, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x0121, 0x00b1, 0x00b1, 0x0136, 0x0134, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x0030, 0x0030, 0x01f3, 0x026e, 0x0040, 0x0040, + 0x0040, 0x0040, 0x00a7, 0x00b1, 0x051c, 0x00a9, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0040, 0x00b1, 0x00d2, 0x023c, 0x0040, 0x05bd, + 0x00a9, 0x00a9, 0x0030, 0x0030, 0x05c1, 0x0040, 0x00c1, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x00a9, 0x00a5, 0x05c4, 0x05c4, + 0x05c7, 0x023a, 0x0030, 0x0030, 0x01f3, 0x00a9, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0239, 0x0518, 0x00a9, 0x0040, 0x0040, + 0x009c, 0x0120, 0x01c9, 0x01ca, 0x01ca, 0x01ca, 0x01ca, 0x01ca, + 0x01ca, 0x01ca, 0x01ca, 0x00a9, 0x0120, 0x01e7, 0x0040, 0x0040, + 0x0040, 0x05cb, 0x05cf, 0x00a9, 0x00a9, 0x05d3, 0x00a9, 0x00a9, + 0x00a9, 0x0362, 0x0362, 0x0362, 0x0362, 0x0362, 0x05d7, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x05d9, 0x0401, + 0x0401, 0x0444, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0362, + 0x05db, 0x0362, 0x05d6, 0x0402, 0x00a9, 0x00a9, 0x00a9, 0x05df, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x05e3, 0x05e6, 0x00a9, 0x00a9, + 0x0449, 0x00a9, 0x00a9, 0x0401, 0x0401, 0x0401, 0x0401, 0x0040, + 0x0040, 0x009c, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0093, 0x00a9, + 0x0040, 0x0040, 0x00a7, 0x05ea, 0x00b1, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0030, 0x0030, 0x01f3, 0x00a9, 0x00b1, 0x00b1, 0x00b1, 0x011a, + 0x00b1, 0x00b1, 0x00b1, 0x00b1, 0x0107, 0x00a9, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0040, 0x009c, 0x00a5, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x0132, 0x00c7, 0x01c9, 0x0263, 0x00b1, 0x00b1, + 0x00b1, 0x01e7, 0x0118, 0x00b1, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x00c5, 0x00c7, 0x0040, 0x0040, 0x0040, 0x0040, 0x00c5, + 0x010b, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0362, + 0x0362, 0x0362, 0x0362, 0x0362, 0x05dc, 0x00a9, 0x00a9, 0x0362, + 0x0362, 0x0362, 0x0362, 0x0362, 0x05ee, 0x0093, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00bf, 0x00ab, + 0x00c0, 0x00a5, 0x00bf, 0x0040, 0x0040, 0x00c1, 0x00a5, 0x0040, + 0x00a5, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x00c1, + 0x009c, 0x00a5, 0x0040, 0x00bf, 0x0040, 0x00bf, 0x0040, 0x017f, + 0x00ba, 0x0040, 0x00bf, 0x0040, 0x0040, 0x0040, 0x00a7, 0x0040, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0163, 0x0030, 0x0030, + 0x0030, 0x0030, 0x0030, 0x0030, 0x0030, 0x0030, 0x00b1, 0x00b1, + 0x00b1, 0x00b1, 0x00b1, 0x01e7, 0x00c4, 0x00b1, 0x00b1, 0x00b1, + 0x00f0, 0x0040, 0x00ef, 0x0040, 0x0040, 0x05f2, 0x03ef, 0x00a9, + 0x00a9, 0x00a9, 0x0120, 0x00b1, 0x00d1, 0x00b1, 0x00b1, 0x00b1, + 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00a5, 0x009c, 0x00a9, 0x00a9, + 0x00a9, 0x00a9, 0x00a9, 0x00b1, 0x0107, 0x00b1, 0x00b1, 0x00b1, + 0x00b1, 0x017b, 0x00b1, 0x0108, 0x0171, 0x0107, 0x00a9, 0x0040, + 0x0040, 0x0040, 0x0040, 0x00a7, 0x00a9, 0x00a9, 0x00a9, 0x00a9, + 0x0120, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x0093, 0x00b1, 0x01e7, 0x0040, 0x00a7, 0x0030, 0x0030, 0x01f3, + 0x00ba, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x016f, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x00b1, 0x0030, 0x0030, 0x01f3, 0x01e3, 0x0040, 0x0040, 0x0040, + 0x00b1, 0x0030, 0x0030, 0x01f3, 0x00a9, 0x0040, 0x0040, 0x0040, + 0x00c5, 0x05f6, 0x0030, 0x0267, 0x00aa, 0x0040, 0x009c, 0x0040, + 0x00c0, 0x0040, 0x0040, 0x0040, 0x009c, 0x0040, 0x0146, 0x0040, + 0x0040, 0x00b1, 0x0107, 0x00a9, 0x00a9, 0x0040, 0x00b1, 0x01e7, + 0x00a9, 0x0030, 0x0030, 0x01f3, 0x05fa, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x00a5, 0x0040, 0x0040, 0x0040, 0x0498, 0x0498, 0x0093, + 0x00a9, 0x00a9, 0x00a5, 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, + 0x0040, 0x0040, 0x00c0, 0x0146, 0x00a5, 0x0040, 0x009c, 0x0040, + 0x00ac, 0x00a9, 0x00ab, 0x00aa, 0x00ac, 0x00a5, 0x00c0, 0x0146, + 0x00ac, 0x00ac, 0x00c0, 0x0146, 0x009c, 0x0040, 0x009c, 0x0040, + 0x00a5, 0x017f, 0x0040, 0x0040, 0x00c1, 0x0040, 0x0040, 0x0040, + 0x0040, 0x00a9, 0x00a5, 0x00a5, 0x00c1, 0x0040, 0x0040, 0x0040, + 0x0040, 0x00a9, 0x0333, 0x05fe, 0x0333, 0x0333, 0x0333, 0x0333, + 0x0333, 0x0333, 0x0333, 0x0333, 0x05ff, 0x0333, 0x0333, 0x0333, + 0x0333, 0x00a9, 0x00a9, 0x00a9, 0x0603, 0x00a9, 0x00a9, 0x00a9, + 0x00a9, 0x0607, 0x00a9, 0x00a9, 0x00a9, 0x00a9, 0x00ba, 0x0348, + 0x060b, 0x00a9, 0x00a9, 0x060d, 0x00a9, 0x00a9, 0x00a9, 0x0611, + 0x0614, 0x0615, 0x0616, 0x00a9, 0x00a9, 0x00a9, 0x0619, 0x0333, + 0x0333, 0x0333, 0x0333, 0x061b, 0x061d, 0x061d, 0x061d, 0x061d, + 0x061d, 0x061d, 0x0621, 0x0333, 0x0333, 0x0333, 0x0401, 0x0401, + 0x044e, 0x0401, 0x0401, 0x0401, 0x044d, 0x0625, 0x0627, 0x0628, + 0x0333, 0x0401, 0x0401, 0x062b, 0x0333, 0x036a, 0x0333, 0x0333, + 0x0333, 0x0627, 0x036a, 0x0333, 0x0333, 0x0333, 0x0333, 0x0333, + 0x0333, 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, + 0x0627, 0x05fe, 0x0333, 0x0333, 0x062e, 0x0627, 0x0384, 0x0627, + 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x0631, 0x0627, + 0x0627, 0x0627, 0x0627, 0x0627, 0x0333, 0x0333, 0x0635, 0x0627, + 0x0627, 0x0627, 0x0627, 0x0627, 0x0639, 0x0627, 0x063b, 0x0627, + 0x0627, 0x062f, 0x05ff, 0x0627, 0x0333, 0x0333, 0x0333, 0x0627, + 0x0627, 0x0627, 0x0627, 0x05fe, 0x05fe, 0x063c, 0x063f, 0x0627, + 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x062f, 0x0631, + 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x0643, + 0x063b, 0x0627, 0x0646, 0x0639, 0x0627, 0x0627, 0x0627, 0x0627, + 0x0627, 0x0627, 0x0627, 0x031a, 0x034c, 0x0649, 0x0627, 0x0627, + 0x0627, 0x0646, 0x034c, 0x034c, 0x063b, 0x0627, 0x0627, 0x0647, + 0x034c, 0x034c, 0x064d, 0x0040, 0x02f6, 0x0651, 0x062f, 0x0627, + 0x0627, 0x0627, 0x0627, 0x0333, 0x0333, 0x0333, 0x0333, 0x0655, + 0x0333, 0x0333, 0x0333, 0x0333, 0x0333, 0x0383, 0x0333, 0x0333, + 0x0333, 0x0333, 0x0333, 0x0348, 0x0348, 0x0333, 0x0333, 0x0333, + 0x0333, 0x0333, 0x0348, 0x0651, 0x0627, 0x0627, 0x0627, 0x0627, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x03ae, 0x0659, 0x0040, + 0x0627, 0x036a, 0x0333, 0x05fe, 0x062f, 0x062e, 0x0333, 0x0627, + 0x0333, 0x0333, 0x05ff, 0x05fe, 0x0333, 0x0627, 0x0627, 0x05fe, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0333, 0x0333, 0x0333, + 0x0040, 0x0040, 0x0040, 0x0040, 0x0040, 0x0332, 0x0333, 0x0333, + 0x0627, 0x0627, 0x0627, 0x0333, 0x05fe, 0x0333, 0x0333, 0x0333, + 0x0040, 0x0040, 0x0040, 0x02f2, 0x0040, 0x0040, 0x0040, 0x0040, + 0x02f2, 0x02f2, 0x0040, 0x0040, 0x02f0, 0x02f2, 0x0040, 0x0040, + 0x02f2, 0x02f2, 0x0040, 0x0040, 0x0040, 0x0040, 0x02f0, 0x0348, + 0x0348, 0x0348, 0x02f2, 0x033c, 0x02f2, 0x02f2, 0x02f2, 0x02f2, + 0x02f2, 0x02f2, 0x02f2, 0x02f2, 0x0040, 0x0040, 0x0040, 0x0627, + 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x065d, 0x0627, 0x065e, + 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x0627, 0x0348, 0x0348, + 0x0348, 0x0348, 0x0348, 0x0348, 0x0348, 0x0348, 0x0333, 0x0333, + 0x0333, 0x0333, 0x0627, 0x0627, 0x0627, 0x05fe, 0x0627, 0x0627, + 0x036a, 0x05ff, 0x0627, 0x0627, 0x0627, 0x0627, 0x062f, 0x0333, + 0x0393, 0x0627, 0x0627, 0x0627, 0x031a, 0x0627, 0x0627, 0x036a, + 0x0333, 0x0627, 0x0627, 0x05fe, 0x0333, 0x0333, 0x0333, 0x0333, + 0x0333, 0x0333, 0x0333, 0x0662, 0x0401, 0x0401, 0x0401, 0x0401, + 0x0401, 0x0401, 0x0401, 0x0406, 0x0666, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x00b1, 0x00b1, 0x00b1, 0x00b1, + 0x0000, 0x0000, 0x0000, 0x0000, +]; +#[rustfmt::skip] +const STAGE3: [u16; 1642] = [ + 0x0803, 0x0803, 0x0803, 0x0803, + 0x0803, 0x0803, 0x0803, 0x0803, + 0x0803, 0x0963, 0x0802, 0x0803, + 0x0803, 0x0801, 0x0803, 0x0803, + 0x0803, 0x0803, 0x0803, 0x0803, + 0x0803, 0x0803, 0x0803, 0x0803, + 0x0803, 0x0803, 0x0803, 0x0803, + 0x0803, 0x0803, 0x0803, 0x0803, + 0x0900, 0x0ac0, 0x0c00, 0x0d80, + 0x0d00, 0x0cc0, 0x0d80, 0x0c00, + 0x0bc0, 0x0a80, 0x0d80, 0x0d00, + 0x0c40, 0x09c0, 0x0c40, 0x0d40, + 0x0c80, 0x0c80, 0x0c80, 0x0c80, + 0x0c80, 0x0c80, 0x0c80, 0x0c80, + 0x0c80, 0x0c80, 0x0c40, 0x0c40, + 0x0d80, 0x0d80, 0x0d80, 0x0ac0, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0bc0, + 0x0d00, 0x0a80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0d80, + 0x0d80, 0x0d80, 0x0d80, 0x0bc0, + 0x0940, 0x0a00, 0x0d80, 0x0803, + 0x08c0, 0x0bc0, 0x0cc0, 0x0d00, + 0x0d00, 0x0d80, 0x0800, 0x0d8e, + 0x0800, 0x0c00, 0x0d80, 0x0944, + 0x0d8e, 0x0d80, 0x0cc0, 0x0d00, + 0x0800, 0x0800, 0x0980, 0x0d80, + 0x0800, 0x0800, 0x0800, 0x0c00, + 0x0800, 0x0800, 0x0800, 0x0bc0, + 0x0d80, 0x0d80, 0x0d80, 0x0800, + 0x0980, 0x0800, 0x0800, 0x0800, + 0x0980, 0x0800, 0x0d80, 0x0d80, + 0x0d80, 0x0800, 0x0800, 0x0800, + 0x0800, 0x0d80, 0x0800, 0x0d80, + 0x0980, 0x0004, 0x0004, 0x0004, + 0x0004, 0x00c4, 0x00c4, 0x00c4, + 0x00c4, 0x0004, 0x0800, 0x0800, + 0x0d80, 0x0d80, 0x0c40, 0x0d80, + 0x0800, 0x0d80, 0x0d80, 0x0800, + 0x0d80, 0x0d80, 0x0d80, 0x0004, + 0x0004, 0x0d80, 0x0d80, 0x0c40, + 0x0940, 0x0800, 0x0d80, 0x0d80, + 0x0d00, 0x0800, 0x0004, 0x0004, + 0x0004, 0x0940, 0x0004, 0x0004, + 0x0ac0, 0x0004, 0x0486, 0x0486, + 0x0486, 0x0486, 0x0d80, 0x0d80, + 0x0cc0, 0x0cc0, 0x0cc0, 0x0004, + 0x0004, 0x0004, 0x0ac0, 0x0ac0, + 0x0ac0, 0x0c80, 0x0c80, 0x0cc0, + 0x0c80, 0x0d80, 0x0d80, 0x0d80, + 0x0004, 0x0d80, 0x0d80, 0x0d80, + 0x0ac0, 0x0d80, 0x0004, 0x0004, + 0x0486, 0x0d80, 0x0004, 0x0d80, + 0x0d80, 0x0004, 0x0d80, 0x0004, + 0x0004, 0x0c80, 0x0c80, 0x0d80, + 0x0d80, 0x0800, 0x0586, 0x0004, + 0x0004, 0x0004, 0x0800, 0x0004, + 0x0d80, 0x0800, 0x0800, 0x0c40, + 0x0ac0, 0x0d80, 0x0800, 0x0004, + 0x0d00, 0x0d00, 0x0004, 0x0004, + 0x0d80, 0x0004, 0x0004, 0x0004, + 0x0800, 0x0800, 0x0486, 0x0486, + 0x0800, 0x0800, 0x0800, 0x0004, + 0x0004, 0x0486, 0x0004, 0x0004, + 0x0004, 0x0804, 0x0d80, 0x0d8d, + 0x0d8d, 0x0d8d, 0x0d8d, 0x0004, + 0x0804, 0x0004, 0x0d80, 0x0804, + 0x0804, 0x0004, 0x0004, 0x0004, + 0x0804, 0x0804, 0x0804, 0x000c, + 0x0804, 0x0804, 0x0940, 0x0940, + 0x0c80, 0x0c80, 0x0d80, 0x0004, + 0x0804, 0x0804, 0x0d80, 0x0800, + 0x0800, 0x0d80, 0x0d8d, 0x0800, + 0x0d8d, 0x0d8d, 0x0800, 0x0d8d, + 0x0800, 0x0800, 0x0d8d, 0x0d8d, + 0x0800, 0x0800, 0x0004, 0x0800, + 0x0800, 0x0804, 0x0800, 0x0800, + 0x0804, 0x000c, 0x0d80, 0x0800, + 0x0800, 0x0800, 0x0804, 0x0800, + 0x0800, 0x0c80, 0x0c80, 0x0d8d, + 0x0d8d, 0x0cc0, 0x0cc0, 0x0d80, + 0x0cc0, 0x0d80, 0x0d00, 0x0d80, + 0x0d80, 0x0004, 0x0800, 0x0004, + 0x0004, 0x0804, 0x0004, 0x0800, + 0x0804, 0x0804, 0x0004, 0x0004, + 0x0800, 0x0800, 0x0004, 0x0d80, + 0x0800, 0x0d80, 0x0800, 0x0d80, + 0x0004, 0x0d80, 0x0800, 0x0d8d, + 0x0d8d, 0x0d8d, 0x0004, 0x0804, + 0x0800, 0x0804, 0x000c, 0x0800, + 0x0800, 0x0d80, 0x0d00, 0x0800, + 0x0800, 0x0d8d, 0x0004, 0x0004, + 0x0800, 0x0004, 0x0804, 0x0804, + 0x0004, 0x0d80, 0x0804, 0x0004, + 0x0d80, 0x0d8d, 0x0d80, 0x0d80, + 0x0800, 0x0800, 0x0804, 0x0804, + 0x0004, 0x0804, 0x0804, 0x0800, + 0x0804, 0x0804, 0x0004, 0x0800, + 0x0800, 0x0d80, 0x0d00, 0x0d80, + 0x0800, 0x0804, 0x0800, 0x0004, + 0x0004, 0x000c, 0x0800, 0x0800, + 0x0004, 0x0004, 0x0800, 0x0d8d, + 0x0d8d, 0x0d8d, 0x0800, 0x0d80, + 0x0800, 0x0800, 0x0980, 0x0d80, + 0x0d80, 0x0d80, 0x0804, 0x0804, + 0x0804, 0x0804, 0x0800, 0x0004, + 0x0804, 0x0800, 0x0804, 0x0804, + 0x0800, 0x0d80, 0x0d80, 0x0804, + 0x000c, 0x0d86, 0x0d80, 0x0cc0, + 0x0d80, 0x0d80, 0x0004, 0x0800, + 0x0004, 0x0800, 0x0804, 0x0800, + 0x0800, 0x0800, 0x0d00, 0x0004, + 0x0004, 0x0004, 0x0d80, 0x0c80, + 0x0c80, 0x0940, 0x0940, 0x0004, + 0x0800, 0x0800, 0x0800, 0x0c80, + 0x0c80, 0x0800, 0x0800, 0x0d80, + 0x0980, 0x0980, 0x0980, 0x0d80, + 0x0980, 0x0980, 0x08c0, 0x0980, + 0x0980, 0x0940, 0x08c0, 0x0ac0, + 0x0ac0, 0x0ac0, 0x08c0, 0x0d80, + 0x0940, 0x0004, 0x0d80, 0x0004, + 0x0bc0, 0x0a00, 0x0804, 0x0804, + 0x0004, 0x0004, 0x0004, 0x0944, + 0x0004, 0x0800, 0x0940, 0x0940, + 0x0980, 0x0980, 0x0940, 0x0980, + 0x0d80, 0x08c0, 0x08c0, 0x0800, + 0x0004, 0x0804, 0x0004, 0x0004, + 0x1007, 0x1007, 0x1007, 0x1007, + 0x0808, 0x0808, 0x0808, 0x0808, + 0x0809, 0x0809, 0x0809, 0x0809, + 0x0d80, 0x0940, 0x0d80, 0x0d80, + 0x0d80, 0x0a00, 0x0800, 0x0800, + 0x0800, 0x0d80, 0x0d80, 0x0d80, + 0x0940, 0x0940, 0x0d80, 0x0d80, + 0x0004, 0x0804, 0x0800, 0x0800, + 0x0804, 0x0940, 0x0940, 0x0800, + 0x0d80, 0x0800, 0x0004, 0x0004, + 0x0940, 0x0940, 0x0b40, 0x0800, + 0x0940, 0x0d80, 0x0940, 0x0d00, + 0x0d80, 0x0d80, 0x0ac0, 0x0ac0, + 0x0940, 0x0940, 0x0980, 0x0d80, + 0x0ac0, 0x0ac0, 0x0d80, 0x0004, + 0x0004, 0x00c4, 0x0004, 0x0804, + 0x0804, 0x0804, 0x0004, 0x0c80, + 0x0c80, 0x0c80, 0x0800, 0x0804, + 0x0004, 0x0804, 0x0800, 0x0800, + 0x0940, 0x0940, 0x0dc0, 0x0940, + 0x0940, 0x0940, 0x0dc0, 0x0dc0, + 0x0dc0, 0x0dc0, 0x0004, 0x0d80, + 0x0804, 0x0004, 0x0004, 0x0800, + 0x0800, 0x0004, 0x0804, 0x0004, + 0x0804, 0x0004, 0x0800, 0x0800, + 0x0800, 0x0940, 0x0940, 0x0940, + 0x0940, 0x0004, 0x0d80, 0x0d80, + 0x0804, 0x0004, 0x0004, 0x0d80, + 0x0800, 0x0004, 0x00c4, 0x0004, + 0x0004, 0x0004, 0x0d80, 0x0980, + 0x0d80, 0x0800, 0x0940, 0x0940, + 0x0940, 0x08c0, 0x0940, 0x0940, + 0x0940, 0x0084, 0x0004, 0x000f, + 0x0004, 0x0004, 0x0c00, 0x0c00, + 0x0bc0, 0x0c00, 0x0b00, 0x0b00, + 0x0b00, 0x0940, 0x0803, 0x0803, + 0x0004, 0x0004, 0x0004, 0x08c0, + 0x0cc0, 0x0cc0, 0x0cc0, 0x0cc0, + 0x0d80, 0x0c00, 0x0c00, 0x0800, + 0x0b4e, 0x0b40, 0x0d80, 0x0d80, + 0x0c40, 0x0bc0, 0x0a00, 0x0b40, + 0x0b4e, 0x0d80, 0x0d80, 0x0940, + 0x0cc0, 0x0d80, 0x0940, 0x0940, + 0x0940, 0x0044, 0x0584, 0x0584, + 0x0584, 0x0803, 0x0004, 0x0004, + 0x0d80, 0x0bc0, 0x0a00, 0x0800, + 0x0d00, 0x0d00, 0x0d00, 0x0d00, + 0x0cc0, 0x0d00, 0x0d00, 0x0d00, + 0x0d80, 0x0d80, 0x0d80, 0x0cc0, + 0x0d80, 0x0d80, 0x0d00, 0x0d80, + 0x0800, 0x080e, 0x0d80, 0x0d8e, + 0x0d80, 0x0d80, 0x080e, 0x080e, + 0x080e, 0x080e, 0x0d80, 0x0d80, + 0x0d8e, 0x0d8e, 0x0d80, 0x0800, + 0x0d00, 0x0d00, 0x0d80, 0x0d80, + 0x0d80, 0x0b00, 0x0bc0, 0x0a00, + 0x0bc0, 0x0a00, 0x0d80, 0x0d80, + 0x15ce, 0x15ce, 0x0d8e, 0x1380, + 0x1200, 0x0d80, 0x0d8e, 0x0d80, + 0x0d80, 0x0d80, 0x0d8e, 0x0d80, + 0x158e, 0x158e, 0x158e, 0x0d8e, + 0x0d8e, 0x0d8e, 0x15ce, 0x0dce, + 0x0dce, 0x15ce, 0x0d8e, 0x0d8e, + 0x0d8e, 0x0d80, 0x0800, 0x0800, + 0x080e, 0x0800, 0x0800, 0x0d8e, + 0x0d8e, 0x0d80, 0x0d80, 0x080e, + 0x0800, 0x0d80, 0x0d80, 0x0d8e, + 0x158e, 0x158e, 0x0d80, 0x0dce, + 0x0dce, 0x0dce, 0x0dce, 0x0d8e, + 0x080e, 0x0800, 0x0d8e, 0x080e, + 0x0d8e, 0x0d8e, 0x080e, 0x080e, + 0x15ce, 0x15ce, 0x080e, 0x080e, + 0x0dce, 0x0d8e, 0x0dce, 0x0dce, + 0x0d8e, 0x0d8e, 0x0d8e, 0x0d8e, + 0x158e, 0x158e, 0x158e, 0x158e, + 0x0d8e, 0x0dce, 0x0dce, 0x0dce, + 0x080e, 0x0d8e, 0x080e, 0x0d8e, + 0x080e, 0x080e, 0x0d8e, 0x080e, + 0x0dce, 0x080e, 0x080e, 0x0d8e, + 0x0d80, 0x0d80, 0x1580, 0x1580, + 0x1580, 0x1580, 0x0d8e, 0x158e, + 0x0d8e, 0x0d8e, 0x15ce, 0x15ce, + 0x0dce, 0x0dce, 0x080e, 0x080e, + 0x080e, 0x0dce, 0x158e, 0x0dce, + 0x0dce, 0x080e, 0x0dce, 0x15ce, + 0x080e, 0x080e, 0x080e, 0x0dce, + 0x080e, 0x080e, 0x0dce, 0x080e, + 0x080e, 0x15ce, 0x080e, 0x0dce, + 0x15ce, 0x15ce, 0x0dce, 0x15ce, + 0x080e, 0x0dce, 0x0dce, 0x15ce, + 0x080e, 0x15ce, 0x0dce, 0x0dce, + 0x158e, 0x0d80, 0x0d80, 0x0dce, + 0x0dce, 0x15ce, 0x15ce, 0x0d8e, + 0x0d80, 0x0d8e, 0x0d80, 0x158e, + 0x0d80, 0x0d80, 0x0d80, 0x0d8e, + 0x0d80, 0x0d80, 0x0d8e, 0x158e, + 0x0d80, 0x158e, 0x0d80, 0x0d80, + 0x0d80, 0x158e, 0x158e, 0x0d80, + 0x100e, 0x0d80, 0x0d80, 0x0d80, + 0x0c00, 0x0c00, 0x0c00, 0x0c00, + 0x0d80, 0x0ac0, 0x0ace, 0x0bc0, + 0x0a00, 0x0800, 0x0800, 0x0d80, + 0x0bc0, 0x0a00, 0x0d80, 0x0d80, + 0x0bc0, 0x0a00, 0x0bc0, 0x0a00, + 0x0bc0, 0x0a00, 0x0d80, 0x0d80, + 0x0d80, 0x0d8e, 0x0d8e, 0x0d8e, + 0x0d80, 0x100e, 0x0800, 0x0800, + 0x0ac0, 0x0940, 0x0940, 0x0d80, + 0x0ac0, 0x0940, 0x0800, 0x0800, + 0x0800, 0x0c00, 0x0c00, 0x0940, + 0x0940, 0x0d80, 0x0940, 0x0bc0, + 0x0940, 0x0d80, 0x0d80, 0x0c00, + 0x0c00, 0x0d80, 0x0d80, 0x0c00, + 0x0c00, 0x0bc0, 0x0a00, 0x0940, + 0x0940, 0x0ac0, 0x0d80, 0x0940, + 0x0940, 0x0940, 0x0d80, 0x0940, + 0x0940, 0x0bc0, 0x0940, 0x0ac0, + 0x0bc0, 0x0a80, 0x0bc0, 0x0a80, + 0x0bc0, 0x0a80, 0x0940, 0x0800, + 0x0800, 0x15c0, 0x15c0, 0x15c0, + 0x15c0, 0x0800, 0x15c0, 0x15c0, + 0x0800, 0x0800, 0x1140, 0x1200, + 0x1200, 0x15c0, 0x1340, 0x15c0, + 0x15c0, 0x1380, 0x1200, 0x1380, + 0x1200, 0x15c0, 0x15c0, 0x1340, + 0x1380, 0x1200, 0x1200, 0x15c0, + 0x15c0, 0x0004, 0x0004, 0x1004, + 0x1004, 0x15ce, 0x15c0, 0x15c0, + 0x15c0, 0x1000, 0x15c0, 0x15c0, + 0x15c0, 0x1340, 0x15ce, 0x15c0, + 0x0dc0, 0x0800, 0x1000, 0x15c0, + 0x1000, 0x15c0, 0x1000, 0x1000, + 0x0800, 0x0004, 0x0004, 0x1340, + 0x1340, 0x1340, 0x15c0, 0x1340, + 0x1000, 0x15c0, 0x1000, 0x1000, + 0x15c0, 0x1000, 0x1340, 0x1340, + 0x15c0, 0x0800, 0x0800, 0x0800, + 0x15c0, 0x1000, 0x1000, 0x1000, + 0x1000, 0x15c0, 0x15c0, 0x15c0, + 0x15ce, 0x15c0, 0x15c0, 0x0d80, + 0x0940, 0x0ac0, 0x0940, 0x0004, + 0x0004, 0x0d80, 0x0940, 0x0804, + 0x0004, 0x0004, 0x0804, 0x0cc0, + 0x0d80, 0x0800, 0x0800, 0x0980, + 0x0980, 0x0ac0, 0x0ac0, 0x0804, + 0x0804, 0x0d80, 0x0d80, 0x0980, + 0x0d80, 0x0d80, 0x0004, 0x1007, + 0x0800, 0x0800, 0x0800, 0x0804, + 0x0dc0, 0x0dc0, 0x0dc0, 0x0940, + 0x0dc0, 0x0dc0, 0x0800, 0x0940, + 0x0800, 0x0800, 0x0dc0, 0x0dc0, + 0x0d80, 0x0804, 0x0004, 0x0800, + 0x0004, 0x0804, 0x0804, 0x0940, + 0x100a, 0x100b, 0x100b, 0x100b, + 0x100b, 0x0808, 0x0808, 0x0808, + 0x0800, 0x0800, 0x0800, 0x0809, + 0x0d80, 0x0d80, 0x0a00, 0x0bc0, + 0x0cc0, 0x0d80, 0x0d80, 0x0d80, + 0x0004, 0x0004, 0x0004, 0x1004, + 0x1200, 0x1200, 0x1200, 0x1340, + 0x12c0, 0x12c0, 0x1380, 0x1200, + 0x1300, 0x0800, 0x0800, 0x00c4, + 0x0004, 0x00c4, 0x0004, 0x00c4, + 0x00c4, 0x0004, 0x1200, 0x1380, + 0x1200, 0x1380, 0x1200, 0x15c0, + 0x15c0, 0x1380, 0x1200, 0x15c0, + 0x15c0, 0x15c0, 0x1200, 0x15c0, + 0x1200, 0x0800, 0x1340, 0x1340, + 0x12c0, 0x12c0, 0x15c0, 0x1500, + 0x14c0, 0x15c0, 0x0d80, 0x0800, + 0x0800, 0x0044, 0x0800, 0x12c0, + 0x15c0, 0x15c0, 0x1500, 0x14c0, + 0x15c0, 0x15c0, 0x1200, 0x15c0, + 0x1200, 0x15c0, 0x15c0, 0x1340, + 0x1340, 0x15c0, 0x15c0, 0x15c0, + 0x12c0, 0x15c0, 0x15c0, 0x15c0, + 0x1380, 0x15c0, 0x1200, 0x15c0, + 0x1380, 0x1200, 0x0a00, 0x0b80, + 0x0a00, 0x0b40, 0x0dc0, 0x0800, + 0x0dc0, 0x0dc0, 0x0dc0, 0x0b44, + 0x0b44, 0x0dc0, 0x0dc0, 0x0dc0, + 0x0800, 0x0800, 0x0800, 0x14c0, + 0x1500, 0x15c0, 0x15c0, 0x1500, + 0x1500, 0x0800, 0x0940, 0x0940, + 0x0940, 0x0800, 0x0d80, 0x0004, + 0x0800, 0x0800, 0x0d80, 0x0d80, + 0x0800, 0x0940, 0x0d80, 0x0004, + 0x0004, 0x0800, 0x0940, 0x0940, + 0x0b00, 0x0800, 0x0004, 0x0004, + 0x0940, 0x0d80, 0x0d80, 0x0800, + 0x0004, 0x0940, 0x0800, 0x0800, + 0x0800, 0x00c4, 0x0d80, 0x0486, + 0x0940, 0x0940, 0x0800, 0x0486, + 0x0800, 0x0800, 0x0004, 0x0800, + 0x0c80, 0x0c80, 0x0d80, 0x0804, + 0x0804, 0x0d80, 0x0d86, 0x0d86, + 0x0940, 0x0004, 0x0004, 0x0004, + 0x0c80, 0x0c80, 0x0d80, 0x0980, + 0x0940, 0x0d80, 0x0004, 0x0d80, + 0x0940, 0x0800, 0x0800, 0x0004, + 0x0940, 0x0804, 0x0804, 0x0800, + 0x0800, 0x0800, 0x0dc0, 0x0800, + 0x0804, 0x0800, 0x0804, 0x0004, + 0x0806, 0x0004, 0x0dc0, 0x0dc0, + 0x0800, 0x0dc0, 0x0004, 0x0980, + 0x0940, 0x0940, 0x0ac0, 0x0ac0, + 0x0d80, 0x0d80, 0x0004, 0x0940, + 0x0940, 0x0d80, 0x0980, 0x0980, + 0x0980, 0x0980, 0x0804, 0x0800, + 0x0800, 0x0004, 0x0804, 0x0004, + 0x0806, 0x0804, 0x0806, 0x0804, + 0x0004, 0x0804, 0x0d86, 0x0004, + 0x0004, 0x0004, 0x0980, 0x0940, + 0x0980, 0x0d80, 0x0004, 0x0d86, + 0x0d86, 0x0d86, 0x0d86, 0x0004, + 0x0004, 0x0980, 0x0940, 0x0940, + 0x0800, 0x0980, 0x0980, 0x0800, + 0x0800, 0x0940, 0x0940, 0x0800, + 0x0800, 0x0980, 0x0ac0, 0x0d80, + 0x0d80, 0x0800, 0x0804, 0x0004, + 0x0004, 0x0d86, 0x0004, 0x0004, + 0x0800, 0x0804, 0x0800, 0x0800, + 0x0940, 0x0004, 0x0004, 0x0806, + 0x0804, 0x0004, 0x0804, 0x0004, + 0x0940, 0x0bc0, 0x0bc0, 0x0bc0, + 0x0a00, 0x0a00, 0x0d80, 0x0d80, + 0x0a00, 0x0d80, 0x0bc0, 0x0a00, + 0x0a00, 0x00c4, 0x00c4, 0x00c4, + 0x03c4, 0x0204, 0x00c4, 0x00c4, + 0x00c4, 0x03c4, 0x0204, 0x03c4, + 0x0204, 0x0940, 0x0d80, 0x0800, + 0x0800, 0x0c80, 0x0c80, 0x0800, + 0x0d80, 0x0d80, 0x0d80, 0x0d88, + 0x0d88, 0x0d88, 0x0d80, 0x1340, + 0x1340, 0x1340, 0x1340, 0x00c4, + 0x0800, 0x0800, 0x0800, 0x1004, + 0x1004, 0x0800, 0x0800, 0x1580, + 0x1580, 0x0800, 0x0800, 0x0800, + 0x1580, 0x1580, 0x1580, 0x0800, + 0x0800, 0x1000, 0x0800, 0x1000, + 0x1000, 0x1000, 0x0800, 0x1000, + 0x0800, 0x0800, 0x0d80, 0x0004, + 0x0004, 0x0940, 0x1580, 0x1580, + 0x1580, 0x0d80, 0x0004, 0x0d80, + 0x0d80, 0x0940, 0x0d80, 0x0c80, + 0x0c80, 0x0c80, 0x0800, 0x0800, + 0x0bc0, 0x0bc0, 0x15ce, 0x0dce, + 0x0dce, 0x0dce, 0x15ce, 0x0800, + 0x0d8e, 0x0d8e, 0x0d8e, 0x0800, + 0x0800, 0x0d80, 0x0d8e, 0x080e, + 0x080e, 0x0800, 0x0800, 0x080e, + 0x080e, 0x0800, 0x0800, 0x100e, + 0x0800, 0x100e, 0x100e, 0x100e, + 0x100e, 0x0800, 0x0d8e, 0x0dce, + 0x0dce, 0x0805, 0x0805, 0x0805, + 0x0805, 0x15c0, 0x15ce, 0x15ce, + 0x0dce, 0x15c0, 0x15c0, 0x15ce, + 0x15ce, 0x15ce, 0x15ce, 0x15c0, + 0x0dce, 0x0dce, 0x0dce, 0x15ce, + 0x15ce, 0x15ce, 0x0dce, 0x15ce, + 0x15ce, 0x0d8e, 0x0d8e, 0x0dce, + 0x0dce, 0x15ce, 0x158e, 0x158e, + 0x15ce, 0x15ce, 0x15ce, 0x15c4, + 0x15c4, 0x15c4, 0x15c4, 0x158e, + 0x15ce, 0x158e, 0x15ce, 0x15ce, + 0x15ce, 0x158e, 0x158e, 0x158e, + 0x15ce, 0x158e, 0x158e, 0x0d80, + 0x0d80, 0x0d8e, 0x0d8e, 0x0dce, + 0x15ce, 0x0dce, 0x0dce, 0x15ce, + 0x0dce, 0x0c00, 0x0b40, 0x0b40, + 0x0b40, 0x15ce, 0x15ce, 0x15ce, + 0x0dc0, 0x15ce, 0x0dce, 0x0dce, + 0x0800, 0x0800, 0x0803, 0x0004, + 0x0803, 0x0803, +]; +#[rustfmt::skip] +const GRAPHEME_JOIN_RULES: [[u32; 16]; 2] = [ + [ + 0b00111100111111111111110011111111, + 0b11111111111111111111111111001111, + 0b11111111111111111111111111111111, + 0b11111111111111111111111111111111, + 0b00111100111111111111110011111111, + 0b00111100111111111111010011111111, + 0b00000000000000000000000011111100, + 0b00111100000011000011110011111111, + 0b00111100111100001111110011111111, + 0b00111100111100111111110011111111, + 0b00111100111100001111110011111111, + 0b00111100111100111111110011111111, + 0b00110000111111111111110011111111, + 0b00111100111111111111110011111111, + 0b00111100111111111111110011111111, + 0b00001100111111111111110011111111, + ], + [ + 0b00111100111111111111110011111111, + 0b11111111111111111111111111001111, + 0b11111111111111111111111111111111, + 0b11111111111111111111111111111111, + 0b00111100111111111111110011111111, + 0b00111100111111111111110011111111, + 0b00000000000000000000000011111100, + 0b00111100000011000011110011111111, + 0b00111100111100001111110011111111, + 0b00111100111100111111110011111111, + 0b00111100111100001111110011111111, + 0b00111100111100111111110011111111, + 0b00110000111111111111110011111111, + 0b00111100111111111111110011111111, + 0b00111100111111111111110011111111, + 0b00001100111111111111110011111111, + ], +]; +#[rustfmt::skip] +const LINE_BREAK_JOIN_RULES: [u32; 25] = [ + 0b00000000001000110011111110111010, + 0b00000000111111111111111111111111, + 0b00000000000000000000000000010000, + 0b00000000111111111111111111111111, + 0b00000000001000110000111100010010, + 0b00000000001000110011111110110010, + 0b00000000111111111111111111111111, + 0b00000000001001110011111100110010, + 0b00000000001110110011111110111010, + 0b00000000001110110011111110111010, + 0b00000000011111110011111110111010, + 0b00000000001000110011111110111010, + 0b00000000001000110011111110111010, + 0b00000000001000110011111110111010, + 0b00000000111111111111111111111111, + 0b00000000111111111111111111111111, + 0b00000000111111111111111111111111, + 0b00000000011111110011111110111010, + 0b00000000011111111011111110111010, + 0b00000000011001111111111110111010, + 0b00000000111001111111111110111010, + 0b00000000001111110011111110111010, + 0b00000000011111111011111110111010, + 0b00000000001010110011111110111010, + 0b00000000000000000000000000000000, +]; +#[inline(always)] +pub fn ucd_grapheme_cluster_lookup(cp: char) -> usize { + unsafe { + let cp = cp as usize; + if cp < 0x80 { + return STAGE3[cp] as usize; + } + let s = *STAGE0.get_unchecked(cp >> 11) as usize; + let s = *STAGE1.get_unchecked(s + ((cp >> 5) & 63)) as usize; + let s = *STAGE2.get_unchecked(s + ((cp >> 2) & 7)) as usize; + *STAGE3.get_unchecked(s + (cp & 3)) as usize + } +} +#[inline(always)] +pub fn ucd_grapheme_cluster_joins(state: u32, lead: usize, trail: usize) -> u32 { + unsafe { + let l = lead & 31; + let t = trail & 31; + let s = GRAPHEME_JOIN_RULES.get_unchecked(state as usize); + (s[l] >> (t * 2)) & 3 + } +} +#[inline(always)] +pub fn ucd_grapheme_cluster_joins_done(state: u32) -> bool { + state == 3 +} +#[inline(always)] +pub fn ucd_grapheme_cluster_character_width(val: usize) -> usize { + val >> 11 +} +#[inline(always)] +pub fn ucd_line_break_joins(lead: usize, trail: usize) -> bool { + unsafe { + let l = (lead >> 6) & 31; + let t = (trail >> 6) & 31; + let s = *LINE_BREAK_JOIN_RULES.get_unchecked(l); + ((s >> t) & 1) != 0 + } +} +#[inline(always)] +pub fn ucd_start_of_text_properties() -> usize { + 0x603 +} +#[inline(always)] +pub fn ucd_tab_properties() -> usize { + 0x963 +} +#[inline(always)] +pub fn ucd_linefeed_properties() -> usize { + 0x802 +} +// END: Generated by grapheme-table-gen diff --git a/pkgs/edit/src/unicode/utf8.rs b/pkgs/edit/src/unicode/utf8.rs new file mode 100644 index 0000000..7ad05dd --- /dev/null +++ b/pkgs/edit/src/unicode/utf8.rs @@ -0,0 +1,278 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::{hint, iter}; + +/// An iterator over UTF-8 encoded characters. +/// +/// This differs from [`std::str::Chars`] in that it works on unsanitized +/// byte slices and transparently replaces invalid UTF-8 sequences with U+FFFD. +/// +/// This follows ICU's bitmask approach for `U8_NEXT_OR_FFFD` relatively +/// closely. This is important for compatibility, because it implements the +/// WHATWG recommendation for UTF8 error recovery. It's also helpful, because +/// the excellent folks at ICU have probably spent a lot of time optimizing it. +#[derive(Clone, Copy)] +pub struct Utf8Chars<'a> { + source: &'a [u8], + offset: usize, +} + +impl<'a> Utf8Chars<'a> { + /// Creates a new `Utf8Chars` iterator starting at the given `offset`. + pub fn new(source: &'a [u8], offset: usize) -> Self { + Self { source, offset } + } + + /// Returns the byte slice this iterator was created with. + pub fn source(&self) -> &'a [u8] { + self.source + } + + /// Checks if the source is empty. + pub fn is_empty(&self) -> bool { + self.source.is_empty() + } + + /// Returns the length of the source. + pub fn len(&self) -> usize { + self.source.len() + } + + /// Returns the current offset in the byte slice. + /// + /// This will be past the last returned character. + pub fn offset(&self) -> usize { + self.offset + } + + /// Sets the offset to continue iterating from. + pub fn seek(&mut self, offset: usize) { + self.offset = offset; + } + + /// Returns true if `next` will return another character. + pub fn has_next(&self) -> bool { + self.offset < self.source.len() + } + + // I found that on mixed 50/50 English/Non-English text, + // performance actually suffers when this gets inlined. + #[cold] + fn next_slow(&mut self, c: u8) -> char { + if self.offset >= self.source.len() { + return Self::fffd(); + } + + let mut cp = c as u32; + + if cp < 0xE0 { + // UTF8-2 = %xC2-DF UTF8-tail + + if cp < 0xC2 { + return Self::fffd(); + } + + // The lead byte is 110xxxxx + // -> Strip off the 110 prefix + cp &= !0xE0; + } else if cp < 0xF0 { + // UTF8-3 = + // %xE0 %xA0-BF UTF8-tail + // %xE1-EC UTF8-tail UTF8-tail + // %xED %x80-9F UTF8-tail + // %xEE-EF UTF8-tail UTF8-tail + + // This is a pretty neat approach seen in ICU4C, because it's a 1:1 translation of the RFC. + // I don't understand why others don't do the same thing. It's rather performant. + const BITS_80_9F: u8 = 1 << 0b100; // 0x80-9F, aka 0b100xxxxx + const BITS_A0_BF: u8 = 1 << 0b101; // 0xA0-BF, aka 0b101xxxxx + const BITS_BOTH: u8 = BITS_80_9F | BITS_A0_BF; + const LEAD_TRAIL1_BITS: [u8; 16] = [ + // v-- lead byte + BITS_A0_BF, // 0xE0 + BITS_BOTH, // 0xE1 + BITS_BOTH, // 0xE2 + BITS_BOTH, // 0xE3 + BITS_BOTH, // 0xE4 + BITS_BOTH, // 0xE5 + BITS_BOTH, // 0xE6 + BITS_BOTH, // 0xE7 + BITS_BOTH, // 0xE8 + BITS_BOTH, // 0xE9 + BITS_BOTH, // 0xEA + BITS_BOTH, // 0xEB + BITS_BOTH, // 0xEC + BITS_80_9F, // 0xED + BITS_BOTH, // 0xEE + BITS_BOTH, // 0xEF + ]; + + // The lead byte is 1110xxxx + // -> Strip off the 1110 prefix + cp &= !0xF0; + + let t = self.source[self.offset] as u32; + if LEAD_TRAIL1_BITS[cp as usize] & (1 << (t >> 5)) == 0 { + return Self::fffd(); + } + cp = (cp << 6) | (t & 0x3F); + + self.offset += 1; + if self.offset >= self.source.len() { + return Self::fffd(); + } + } else { + // UTF8-4 = + // %xF0 %x90-BF UTF8-tail UTF8-tail + // %xF1-F3 UTF8-tail UTF8-tail UTF8-tail + // %xF4 %x80-8F UTF8-tail UTF8-tail + + // This is similar to the above, but with the indices flipped: + // The trail byte is the index and the lead byte mask is the value. + // This is because the split at 0x90 requires more bits than fit into an u8. + const TRAIL1_LEAD_BITS: [u8; 16] = [ + // --------- 0xF4 lead + // | ... + // | +---- 0xF0 lead + // v v + 0b_00000, // + 0b_00000, // + 0b_00000, // + 0b_00000, // + 0b_00000, // + 0b_00000, // + 0b_00000, // trail bytes: + 0b_00000, // + 0b_11110, // 0x80-8F -> 0x80-8F can be preceded by 0xF1-F4 + 0b_01111, // 0x90-9F -v + 0b_01111, // 0xA0-AF -> 0x90-BF can be preceded by 0xF0-F3 + 0b_01111, // 0xB0-BF -^ + 0b_00000, // + 0b_00000, // + 0b_00000, // + 0b_00000, // + ]; + + // The lead byte *may* be 11110xxx, but could also be e.g. 11111xxx. + // -> Only strip off the 1111 prefix + cp &= !0xF0; + + // Now we can verify if it's actually <= 0xF4. + // Curiously, this if condition does a lot of heavy lifting for + // performance (+13%). I think it's just a coincidence though. + if cp > 4 { + return Self::fffd(); + } + + let t = self.source[self.offset] as u32; + if TRAIL1_LEAD_BITS[(t >> 4) as usize] & (1 << cp) == 0 { + return Self::fffd(); + } + cp = (cp << 6) | (t & 0x3F); + + self.offset += 1; + if self.offset >= self.source.len() { + return Self::fffd(); + } + + // UTF8-tail = %x80-BF + let t = (self.source[self.offset] as u32).wrapping_sub(0x80); + if t > 0x3F { + return Self::fffd(); + } + cp = (cp << 6) | t; + + self.offset += 1; + if self.offset >= self.source.len() { + return Self::fffd(); + } + } + + // SAFETY: All branches above check for `if self.offset >= self.source.len()` + // one way or another. This is here because the compiler doesn't get it otherwise. + unsafe { hint::assert_unchecked(self.offset < self.source.len()) }; + + // UTF8-tail = %x80-BF + let t = (self.source[self.offset] as u32).wrapping_sub(0x80); + if t > 0x3F { + return Self::fffd(); + } + cp = (cp << 6) | t; + + self.offset += 1; + + // SAFETY: If `cp` wasn't a valid codepoint, we already returned U+FFFD above. + #[allow(clippy::transmute_int_to_char)] + unsafe { + char::from_u32_unchecked(cp) + } + } + + // This simultaneously serves as a `cold_path` marker. + // It improves performance by ~5% and reduces code size. + #[cold] + #[inline(always)] + fn fffd() -> char { + '\u{FFFD}' + } +} + +impl Iterator for Utf8Chars<'_> { + type Item = char; + + #[inline] + fn next(&mut self) -> Option { + if self.offset >= self.source.len() { + return None; + } + + let c = self.source[self.offset]; + self.offset += 1; + + // Fast-passing ASCII allows this function to be trivially inlined everywhere, + // as the full decoder is a little too large for that. + if (c & 0x80) == 0 { + // UTF8-1 = %x00-7F + Some(c as char) + } else { + // Weirdly enough, adding a hint here to assert that `next_slow` + // only returns codepoints >= 0x80 makes `ucd` ~5% slower. + Some(self.next_slow(c)) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + // Lower bound: All remaining bytes are 4-byte sequences. + // Upper bound: All remaining bytes are ASCII. + let remaining = self.source.len() - self.offset; + (remaining / 4, Some(remaining)) + } +} + +impl iter::FusedIterator for Utf8Chars<'_> {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_broken_utf8() { + let source = [b'a', 0xED, 0xA0, 0x80, b'b']; + let mut chars = Utf8Chars::new(&source, 0); + let mut offset = 0; + for chunk in source.utf8_chunks() { + for ch in chunk.valid().chars() { + offset += ch.len_utf8(); + assert_eq!(chars.next(), Some(ch)); + assert_eq!(chars.offset(), offset); + } + if !chunk.invalid().is_empty() { + offset += chunk.invalid().len(); + assert_eq!(chars.next(), Some('\u{FFFD}')); + assert_eq!(chars.offset(), offset); + } + } + } +} diff --git a/pkgs/edit/src/vt.rs b/pkgs/edit/src/vt.rs new file mode 100644 index 0000000..31701c0 --- /dev/null +++ b/pkgs/edit/src/vt.rs @@ -0,0 +1,339 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Our VT parser. + +use std::{mem, time}; + +use crate::simd::memchr2; + +/// The parser produces these tokens. +pub enum Token<'parser, 'input> { + /// A bunch of text. Doesn't contain any control characters. + Text(&'input str), + /// A single control character, like backspace or return. + Ctrl(char), + /// We encountered `ESC x` and this contains `x`. + Esc(char), + /// We encountered `ESC O x` and this contains `x`. + SS3(char), + /// A CSI sequence started with `ESC [`. + /// + /// They are the most common escape sequences. See [`Csi`]. + Csi(&'parser Csi), + /// An OSC sequence started with `ESC ]`. + /// + /// The sequence may be split up into multiple tokens if the input + /// is given in chunks. This is indicated by the `partial` field. + Osc { data: &'input str, partial: bool }, + /// An DCS sequence started with `ESC P`. + /// + /// The sequence may be split up into multiple tokens if the input + /// is given in chunks. This is indicated by the `partial` field. + Dcs { data: &'input str, partial: bool }, +} + +/// Stores the state of the parser. +#[derive(Clone, Copy)] +enum State { + Ground, + Esc, + Ss3, + Csi, + Osc, + Dcs, + OscEsc, + DcsEsc, +} + +/// A single CSI sequence, parsed for your convenience. +pub struct Csi { + /// The parameters of the CSI sequence. + pub params: [u16; 32], + /// The number of parameters stored in [`Csi::params`]. + pub param_count: usize, + /// The private byte, if any. `0` if none. + /// + /// The private byte is the first character right after the + /// `ESC [` sequence. It is usually a `?` or `<`. + pub private_byte: char, + /// The final byte of the CSI sequence. + /// + /// This is the last character of the sequence, e.g. `m` or `H`. + pub final_byte: char, +} + +pub struct Parser { + state: State, + // Csi is not part of State, because it allows us + // to more quickly erase and reuse the struct. + csi: Csi, +} + +impl Parser { + pub fn new() -> Self { + Self { + state: State::Ground, + csi: Csi { params: [0; 32], param_count: 0, private_byte: '\0', final_byte: '\0' }, + } + } + + /// Suggests a timeout for the next call to `read()`. + /// + /// We need this because of the ambiguouity of whether a trailing + /// escape character in an input is starting another escape sequence or + /// is just the result of the user literally pressing the Escape key. + pub fn read_timeout(&mut self) -> std::time::Duration { + match self.state { + // 100ms is a upper ceiling for a responsive feel. This uses half that, + // under the assumption that a really slow terminal needs equal amounts + // of time for I and O. Realistically though, this could be much lower. + State::Esc => time::Duration::from_millis(50), + _ => time::Duration::MAX, + } + } + + /// Parses the given input into VT sequences. + /// + /// You should call this function even if your `read()` + /// had a timeout (pass an empty string in that case). + pub fn parse<'parser, 'input>( + &'parser mut self, + input: &'input str, + ) -> Stream<'parser, 'input> { + Stream { parser: self, input, off: 0 } + } +} + +/// An iterator that parses VT sequences into [`Token`]s. +/// +/// Can't implement [`Iterator`], because this is a "lending iterator". +pub struct Stream<'parser, 'input> { + parser: &'parser mut Parser, + input: &'input str, + off: usize, +} + +impl<'parser, 'input> Stream<'parser, 'input> { + /// Returns the input that is being parsed. + pub fn input(&self) -> &'input str { + self.input + } + + /// Returns the current parser offset. + pub fn offset(&self) -> usize { + self.off + } + + /// Reads and consumes raw bytes from the input. + pub fn read(&mut self, dst: &mut [u8]) -> usize { + let bytes = self.input.as_bytes(); + let off = self.off.min(bytes.len()); + let len = dst.len().min(bytes.len() - off); + dst[..len].copy_from_slice(&bytes[off..off + len]); + self.off += len; + len + } + + /// Parses the next VT sequence from the previously given input. + #[allow(clippy::should_implement_trait)] + pub fn next(&mut self) -> Option> { + // I don't know how to tell Rust that `self.parser` and its lifetime + // `'parser` outlives `self`, and at this point I don't care. + let parser = unsafe { mem::transmute::<_, &'parser mut Parser>(&mut *self.parser) }; + let input = self.input; + let bytes = input.as_bytes(); + + // If the previous input ended with an escape character, `read_timeout()` + // returned `Some(..)` timeout, and if the caller did everything correctly + // and there was indeed a timeout, we should be called with an empty + // input. In that case we'll return the escape as its own token. + if input.is_empty() && matches!(parser.state, State::Esc) { + parser.state = State::Ground; + return Some(Token::Esc('\0')); + } + + while self.off < bytes.len() { + match parser.state { + State::Ground => match bytes[self.off] { + 0x1b => { + parser.state = State::Esc; + self.off += 1; + } + c @ (0x00..0x20 | 0x7f) => { + self.off += 1; + return Some(Token::Ctrl(c as char)); + } + _ => { + let beg = self.off; + while { + self.off += 1; + self.off < bytes.len() + && bytes[self.off] >= 0x20 + && bytes[self.off] != 0x7f + } {} + return Some(Token::Text(&input[beg..self.off])); + } + }, + State::Esc => { + let c = bytes[self.off]; + self.off += 1; + match c { + b'[' => { + parser.state = State::Csi; + parser.csi.private_byte = '\0'; + parser.csi.final_byte = '\0'; + while parser.csi.param_count > 0 { + parser.csi.param_count -= 1; + parser.csi.params[parser.csi.param_count] = 0; + } + } + b']' => { + parser.state = State::Osc; + } + b'O' => { + parser.state = State::Ss3; + } + b'P' => { + parser.state = State::Dcs; + } + c => { + parser.state = State::Ground; + return Some(Token::Esc(c as char)); + } + } + } + State::Ss3 => { + parser.state = State::Ground; + let c = bytes[self.off]; + self.off += 1; + return Some(Token::SS3(c as char)); + } + State::Csi => { + loop { + // If we still have slots left, parse the parameter. + if parser.csi.param_count < parser.csi.params.len() { + let dst = &mut parser.csi.params[parser.csi.param_count]; + while self.off < bytes.len() && bytes[self.off].is_ascii_digit() { + let add = bytes[self.off] as u32 - b'0' as u32; + let value = *dst as u32 * 10 + add; + *dst = value.min(u16::MAX as u32) as u16; + self.off += 1; + } + } else { + // ...otherwise, skip the parameters until we find the final byte. + while self.off < bytes.len() && bytes[self.off].is_ascii_digit() { + self.off += 1; + } + } + + // Encountered the end of the input before finding the final byte. + if self.off >= bytes.len() { + return None; + } + + let c = bytes[self.off]; + self.off += 1; + + match c { + 0x40..=0x7e => { + parser.state = State::Ground; + parser.csi.final_byte = c as char; + if parser.csi.param_count != 0 || parser.csi.params[0] != 0 { + parser.csi.param_count += 1; + } + return Some(Token::Csi(&parser.csi as &'parser Csi)); + } + b';' => parser.csi.param_count += 1, + b'<'..=b'?' => parser.csi.private_byte = c as char, + _ => {} + } + } + } + State::Osc | State::Dcs => { + let beg = self.off; + let mut data; + let mut partial; + + loop { + // Find any indication for the end of the OSC/DCS sequence. + self.off = memchr2(b'\x07', b'\x1b', bytes, self.off); + + data = &input[beg..self.off]; + partial = self.off >= bytes.len(); + + // Encountered the end of the input before finding the terminator. + if partial { + break; + } + + let c = bytes[self.off]; + self.off += 1; + + if c == 0x1b { + // It's only a string terminator if it's followed by \. + // We're at the end so we're saving the state and will continue next time. + if self.off >= bytes.len() { + parser.state = match parser.state { + State::Osc => State::OscEsc, + _ => State::DcsEsc, + }; + partial = true; + break; + } + + // False alarm: Not a string terminator. + if bytes[self.off] != b'\\' { + continue; + } + + self.off += 1; + } + + break; + } + + let state = parser.state; + if !partial { + parser.state = State::Ground; + } + return match state { + State::Osc => Some(Token::Osc { data, partial }), + _ => Some(Token::Dcs { data, partial }), + }; + } + State::OscEsc | State::DcsEsc => { + // We were processing an OSC/DCS sequence and the last byte was an escape character. + // It's only a string terminator if it's followed by \ (= "\x1b\\"). + if bytes[self.off] == b'\\' { + // It was indeed a string terminator and we can now tell the caller about it. + let state = parser.state; + + // Consume the terminator (one byte in the previous input and this byte). + parser.state = State::Ground; + self.off += 1; + + return match state { + State::OscEsc => Some(Token::Osc { data: "", partial: false }), + _ => Some(Token::Dcs { data: "", partial: false }), + }; + } else { + // False alarm: Not a string terminator. + // We'll return the escape character as a separate token. + // Processing will continue from the current state (`bytes[self.off]`). + parser.state = match parser.state { + State::OscEsc => State::Osc, + _ => State::Dcs, + }; + return match parser.state { + State::Osc => Some(Token::Osc { data: "\x1b", partial: true }), + _ => Some(Token::Dcs { data: "\x1b", partial: true }), + }; + } + } + } + } + + None + } +} diff --git a/pkgs/edit/tools/grapheme-table-gen/Cargo.lock b/pkgs/edit/tools/grapheme-table-gen/Cargo.lock new file mode 100644 index 0000000..f063166 --- /dev/null +++ b/pkgs/edit/tools/grapheme-table-gen/Cargo.lock @@ -0,0 +1,380 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "cc" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "grapheme-table-gen" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "indoc", + "pico-args", + "rayon", + "roxmltree", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "js-sys" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "pico-args" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "wasm-bindgen" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/pkgs/edit/tools/grapheme-table-gen/Cargo.toml b/pkgs/edit/tools/grapheme-table-gen/Cargo.toml new file mode 100644 index 0000000..46cf013 --- /dev/null +++ b/pkgs/edit/tools/grapheme-table-gen/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "grapheme-table-gen" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.95" +chrono = "0.4.39" +indoc = "2.0.5" +pico-args = { version = "0.5.0", features = ["eq-separator"] } +rayon = "1.10.0" +roxmltree = { version = "0.20.0", default-features = false, features = ["std"] } diff --git a/pkgs/edit/tools/grapheme-table-gen/README.md b/pkgs/edit/tools/grapheme-table-gen/README.md new file mode 100644 index 0000000..48d3a37 --- /dev/null +++ b/pkgs/edit/tools/grapheme-table-gen/README.md @@ -0,0 +1,15 @@ +# Grapheme Table Generator + +This tool processes Unicode Character Database (UCD) XML files to generate efficient, multi-stage trie lookup tables for properties relevant to terminal applications: +* Grapheme cluster breaking rules +* Line breaking rules (optional) +* Character width properties + +## Usage + +* Download [ucd.nounihan.grouped.zip](https://www.unicode.org/Public/UCD/latest/ucdxml/ucd.nounihan.grouped.zip) +* Run some equivalent of: + ```sh + grapheme-table-gen --lang=rust --extended --no-ambiguous --line-breaks path/to/ucd.nounihan.grouped.xml + ``` +* Place the result in `src/unicode/tables.rs` diff --git a/pkgs/edit/tools/grapheme-table-gen/src/main.rs b/pkgs/edit/tools/grapheme-table-gen/src/main.rs new file mode 100644 index 0000000..6d4defa --- /dev/null +++ b/pkgs/edit/tools/grapheme-table-gen/src/main.rs @@ -0,0 +1,1043 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +mod rules; + +use std::collections::HashMap; +use std::fmt::Write as FmtWrite; +use std::io::Write as IoWrite; +use std::ops::RangeInclusive; +use std::path::PathBuf; + +use anyhow::{Context, bail}; +use indoc::writedoc; +use rayon::prelude::*; + +use crate::rules::{JOIN_RULES_GRAPHEME_CLUSTER, JOIN_RULES_LINE_BREAK}; + +// `CharacterWidth` is 2 bits. +#[derive(Clone, Copy, PartialEq, Eq)] +enum CharacterWidth { + ZeroWidth, + Narrow, + Wide, + Ambiguous, +} + +// `ClusterBreak` is 4 bits without `StartOfText`, 5 bits with it. +// NOTE: The order of these items must match JOIN_RULES_GRAPHEME_CLUSTER. +#[derive(Clone, Copy, PartialEq, Eq)] +#[allow(clippy::upper_case_acronyms)] +enum ClusterBreak { + Other, // GB999 + CR, // GB3, GB4, GB5 + LF, // GB3, GB4, GB5 + Control, // GB4, GB5 + Extend, // GB9, GB9a -- includes SpacingMark + RI, // GB12, GB13 + Prepend, // GB9b + HangulL, // GB6, GB7, GB8 + HangulV, // GB6, GB7, GB8 + HangulT, // GB6, GB7, GB8 + HangulLV, // GB6, GB7, GB8 + HangulLVT, // GB6, GB7, GB8 + InCBLinker, // GB9c + InCBConsonant, // GB9c + ExtPic, // GB11 + ZWJ, // GB9, GB11 +} + +// Extended information for each `ClusterBreak` via --extended. +// Currently only used for storing the subtype "tab" for `ClusterBreak::Control`. +// As such, this is 1 bit. +#[derive(Clone, Copy, PartialEq, Eq)] +enum ClusterBreakExt { + ControlTab = 1, +} + +// `LineBreak` is 5 bits. +// NOTE: The order of these items must match JOIN_RULES_LINE_BREAK. +#[derive(Clone, Copy, PartialEq, Eq)] +#[allow(non_camel_case_types)] +enum LineBreak { + Other, // Anything else + + // Non-tailorable Line Breaking Classes + WordJoiner, // WJ + ZeroWidthSpace, // ZW + Glue, // GL + Space, // SP + + // Break Opportunities + BreakAfter, // BA + BreakBefore, // BB + Hyphen, // HY + + // Characters Prohibiting Certain Breaks + ClosePunctuation, // CL + CloseParenthesis_EA, // CP, East Asian + CloseParenthesis_NotEA, // CP, not East Asian + Exclamation, // EX + Inseparable, // IN + Nonstarter, // NS + OpenPunctuation_EA, // OP, East Asian + OpenPunctuation_NotEA, // OP, not East Asian + Quotation, // QU + + // Numeric Context + InfixNumericSeparator, // IS + Numeric, // NU + PostfixNumeric, // PO + PrefixNumeric, // PR + SymbolsAllowingBreakAfter, // SY + + // Other Characters + Alphabetic, // AL & HL + Ideographic, // ID & EB & EM + + StartOfText, // LB2 (optional via --extended) +} + +#[repr(transparent)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +struct TrieType(u32); + +impl TrieType { + fn new(packing: &BitPacking, cb: ClusterBreak, lb: LineBreak, cw: CharacterWidth) -> Self { + let cb = cb as u32; + let lb = lb as u32; + let cw = cw as u32; + assert!(cb <= packing.mask_cluster_break); + assert!(lb <= packing.mask_line_break); + assert!(cw <= packing.mask_character_width); + + let cb = cb << packing.shift_cluster_break; + let lb = lb << packing.shift_line_break; + let cw = cw << packing.shift_character_width; + Self(cb | lb | cw) + } + + fn change_cluster_break_ext(&mut self, packing: &BitPacking, cbe: ClusterBreakExt) { + let mask = packing.mask_cluster_break_ext; + let shift = packing.shift_cluster_break_ext; + + let cbe = cbe as u32; + assert!(cbe <= mask); + + self.0 = (self.0 & !(mask << shift)) | (cbe << shift); + } + + fn change_width(&mut self, packing: &BitPacking, cw: CharacterWidth) { + let mask = packing.mask_character_width; + let shift = packing.shift_character_width; + + let cw = cw as u32; + assert!(cw <= mask); + + self.0 = (self.0 & !(mask << shift)) | (cw << shift); + } + + fn value(&self) -> u32 { + self.0 + } +} + +#[derive(Default)] +struct BitPacking { + mask_cluster_break: u32, + mask_cluster_break_ext: u32, + mask_line_break: u32, + mask_character_width: u32, + + shift_cluster_break: u32, + shift_cluster_break_ext: u32, + shift_line_break: u32, + shift_character_width: u32, +} + +impl BitPacking { + fn new(line_breaks: bool, extended: bool) -> Self { + let cb_width: u32 = if extended { 5 } else { 4 }; + let cb_ext_width: u32 = if extended { 1 } else { 0 }; + let lb_width: u32 = if line_breaks { 5 } else { 0 }; + let cw_width: u32 = 3; + + Self { + mask_cluster_break: (1 << cb_width) - 1, + mask_cluster_break_ext: (1 << cb_ext_width) - 1, + mask_line_break: (1 << lb_width) - 1, + mask_character_width: (1 << cw_width) - 1, + + shift_cluster_break: 0, + shift_cluster_break_ext: cb_width, + shift_line_break: cb_width + cb_ext_width, + shift_character_width: cb_width + cb_ext_width + lb_width, + } + } +} + +#[derive(Default)] +struct Ucd { + description: String, + values: Vec, + packing: BitPacking, +} + +#[derive(Clone, Default)] +struct Stage { + values: Vec, + index: usize, + shift: usize, + mask: usize, + bits: usize, +} + +#[derive(Clone, Default)] +struct Trie { + stages: Vec, + total_size: usize, +} + +#[derive(Clone, Copy, Default)] +enum Language { + #[default] + C, + Rust, +} + +#[derive(Default)] +struct Output { + arg_lang: Language, + arg_extended: bool, + arg_no_ambiguous: bool, + arg_line_breaks: bool, + + ucd: Ucd, + trie: Trie, + rules_gc: Vec>, + rules_lb: Vec, + total_size: usize, +} + +impl Output { + fn args(&self) -> String { + let mut buf = String::new(); + match self.arg_lang { + Language::C => buf.push_str("--lang=c"), + Language::Rust => buf.push_str("--lang=rust"), + } + if self.arg_extended { + buf.push_str(" --extended") + } + if self.arg_no_ambiguous { + buf.push_str(" --no-ambiguous") + } + if self.arg_line_breaks { + buf.push_str(" --line-breaks") + } + buf + } +} + +const HELP: &str = "\ +Usage: grapheme-table-gen [options...] + -h, --help Prints help information + --lang= Output language (default: c) + --extended Expose a start-of-text property for kickstarting the segmentation + Expose tab and linefeed as grapheme cluster properties + --no-ambiguous Treat all ambiguous characters as narrow + --line-breaks Store and expose line break information +"; + +fn main() -> anyhow::Result<()> { + let mut args = pico_args::Arguments::from_env(); + if args.contains(["-h", "--help"]) { + eprint!("{}", HELP); + return Ok(()); + } + + let mut out = Output { + arg_lang: args.value_from_fn("--lang", |arg| match arg { + "c" => Ok(Language::C), + "rust" => Ok(Language::Rust), + l => bail!("invalid language: \"{}\"", l), + })?, + arg_extended: args.contains("--extended"), + arg_no_ambiguous: args.contains("--no-ambiguous"), + arg_line_breaks: args.contains("--line-breaks"), + ..Default::default() + }; + let arg_input = args.free_from_os_str(|s| -> Result { Ok(s.into()) })?; + let arg_remaining = args.finish(); + if !arg_remaining.is_empty() { + bail!("unrecognized arguments: {:?}", arg_remaining); + } + + let input = std::fs::read_to_string(arg_input)?; + let doc = roxmltree::Document::parse(&input)?; + out.ucd = extract_values_from_ucd(&doc, &out)?; + + // Find the best trie configuration over the given block sizes (2^2 - 2^8) and stages (4). + // More stages = Less size. The trajectory roughly follows a+b*c^stages, where c < 1. + // 4 still gives ~30% savings over 3 stages and going beyond 5 gives diminishing returns (<10%). + out.trie = build_best_trie(&out.ucd.values, 2, 8, 4); + + // The joinRules above has 2 bits per value. This packs it into 32-bit integers to save space. + out.rules_gc = JOIN_RULES_GRAPHEME_CLUSTER + .iter() + .map(|t| { + let rules_gc_len = if out.arg_extended { t.len() } else { 16 }; + t[..rules_gc_len].iter().map(|row| prepare_rules_row(row, 2, 3)).collect() + }) + .collect(); + + // Same for line breaks, but in 2D. + let rules_lb_len = if out.arg_extended { JOIN_RULES_LINE_BREAK.len() } else { 24 }; + out.rules_lb = JOIN_RULES_LINE_BREAK[..rules_lb_len] + .iter() + .map(|row| prepare_rules_row(row, 1, 0)) + .collect(); + + // Each rules item has the same length. Each item is 32 bits = 4 bytes. + out.total_size = out.trie.total_size + out.rules_gc.len() * out.rules_gc[0].len() * 4; + if out.arg_line_breaks { + out.total_size += out.rules_lb.len() * 4; + } + + // Run a quick sanity check to ensure that the trie works as expected. + for (cp, expected) in out.ucd.values.iter().enumerate() { + let mut actual = 0; + for s in &out.trie.stages { + actual = s.values[actual as usize + ((cp >> s.shift) & s.mask)]; + } + assert_eq!(expected.value(), actual, "trie sanity check failed for U+{:04X}", cp); + } + for (cp, &expected) in out.ucd.values[..0x80].iter().enumerate() { + let last = out.trie.stages.last().unwrap(); + let actual = last.values[cp]; + assert_eq!( + expected.value(), + actual, + "trie sanity check failed for direct ASCII mapping of U+{:04X}", + cp + ); + } + + let buf = match out.arg_lang { + Language::C => generate_c(out), + Language::Rust => generate_rust(out), + }; + + std::io::stdout().write_all(buf.as_bytes())?; + Ok(()) +} + +fn generate_c(out: Output) -> String { + let mut buf = String::new(); + + _ = writedoc!( + buf, + " + // BEGIN: Generated by grapheme-table-gen on {}, from {}, with {}, {} bytes + // clang-format off + ", + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + out.ucd.description, + out.args(), + out.total_size, + ); + + for stage in &out.trie.stages { + let mut width = 16; + if stage.index != 0 { + width = stage.mask + 1; + } + + _ = write!(buf, "static const uint{}_t s_stage{}[] = {{", stage.bits, stage.index); + for (j, &value) in stage.values.iter().enumerate() { + if j % width == 0 { + buf.push_str("\n "); + } + _ = write!(buf, " 0x{:01$x},", value, stage.bits / 4); + } + buf.push_str("\n};\n"); + } + + _ = writeln!( + buf, + "static const uint32_t s_grapheme_cluster_join_rules[{}][{}] = {{", + out.rules_gc.len(), + out.rules_gc[0].len() + ); + for table in &out.rules_gc { + buf.push_str(" {\n"); + for &r in table { + _ = writeln!(buf, " 0b{:032b},", r); + } + buf.push_str(" },\n"); + } + buf.push_str("};\n"); + + if out.arg_line_breaks { + _ = writeln!( + buf, + "static const uint32_t s_line_break_join_rules[{}] = {{", + out.rules_lb.len() + ); + for r in &out.rules_lb { + _ = writeln!(buf, " 0b{r:032b},"); + } + buf.push_str("};\n"); + } + + _ = writedoc!( + buf, + " + inline int ucd_grapheme_cluster_lookup(const uint32_t cp) + {{ + if (cp < 0x80) {{ + return s_stage{}[cp]; + }} + ", + out.trie.stages.len() - 1, + ); + for stage in &out.trie.stages { + if stage.index == 0 { + _ = writeln!( + buf, + " const uint{}_t s0 = s_stage0[cp >> {}];", + stage.bits, stage.shift, + ); + } else { + _ = writeln!( + buf, + " const uint{}_t s{} = s_stage{}[s{} + ((cp >> {}) & {})];", + stage.bits, + stage.index, + stage.index, + stage.index - 1, + stage.shift, + stage.mask, + ); + } + } + _ = writedoc!( + buf, + " + return s{}; + }} + ", + out.trie.stages.len() - 1, + ); + + _ = writedoc!( + buf, + " + inline int ucd_grapheme_cluster_joins(const int state, const int lead, const int trail) + {{ + const int l = lead & {0}; + const int t = trail & {0}; + return (s_grapheme_cluster_join_rules[state][l] >> (t * 2)) & 3; + }} + inline bool ucd_grapheme_cluster_joins_done(const int state) + {{ + return state == 3; + }} + inline int ucd_grapheme_cluster_character_width(const int val) + {{ + return val >> {1}; + }} + ", + out.ucd.packing.mask_cluster_break, + out.ucd.packing.shift_character_width, + ); + + if out.arg_line_breaks { + _ = writedoc!( + buf, + " + inline bool ucd_line_break_joins(const int lead, const int trail) + {{ + const int l = (lead >> {0}) & {1}; + const int t = (trail >> {0}) & {1}; + return (s_line_break_join_rules[l] >> t) & 1; + }} + ", + out.ucd.packing.shift_line_break, + out.ucd.packing.mask_line_break, + ); + } + + if out.arg_extended { + _ = writedoc!( + buf, + " + inline int ucd_start_of_text_properties() + {{ + return {:#x}; + }} + inline int ucd_tab_properties() + {{ + return {:#x}; + }} + inline int ucd_linefeed_properties() + {{ + return {:#x}; + }} + ", + TrieType::new( + &out.ucd.packing, + // Control behaves identical to SOT (start of text) in a way, + // as it doesn't join with any surrounding character. + ClusterBreak::Control, + LineBreak::StartOfText, + CharacterWidth::ZeroWidth, + ) + .value(), + out.ucd.values['\t' as usize].value(), + out.ucd.values['\n' as usize].value(), + ); + } + + buf.push_str("// clang-format on\n// END: Generated by grapheme-table-gen\n"); + buf +} + +fn generate_rust(out: Output) -> String { + let mut buf = String::new(); + + _ = writeln!( + buf, + "// BEGIN: Generated by grapheme-table-gen on {}, from {}, with {}, {} bytes", + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + out.ucd.description, + out.args(), + out.total_size, + ); + + for stage in &out.trie.stages { + let mut width = 16; + if stage.index != 0 { + width = stage.mask + 1; + } + + _ = write!( + buf, + "#[rustfmt::skip]\nconst STAGE{}: [u{}; {}] = [", + stage.index, + stage.bits, + stage.values.len(), + ); + for (j, &value) in stage.values.iter().enumerate() { + if j % width == 0 { + buf.push_str("\n "); + } + _ = write!(buf, " 0x{:01$x},", value, stage.bits / 4); + } + buf.push_str("\n];\n"); + } + + _ = writeln!( + buf, + "#[rustfmt::skip]\nconst GRAPHEME_JOIN_RULES: [[u32; {}]; {}] = [", + out.rules_gc[0].len(), + out.rules_gc.len(), + ); + for table in &out.rules_gc { + buf.push_str(" [\n"); + for &r in table { + _ = writeln!(buf, " 0b{:032b},", r); + } + buf.push_str(" ],\n"); + } + buf.push_str("];\n"); + + if out.arg_line_breaks { + _ = writeln!( + buf, + "#[rustfmt::skip]\nconst LINE_BREAK_JOIN_RULES: [u32; {}] = [", + out.rules_lb.len(), + ); + for r in &out.rules_lb { + _ = writeln!(buf, " 0b{r:032b},"); + } + buf.push_str("];\n"); + } + + _ = writedoc!( + buf, + " + #[inline(always)] + pub fn ucd_grapheme_cluster_lookup(cp: char) -> usize {{ + unsafe {{ + let cp = cp as usize; + if cp < 0x80 {{ + return STAGE{}[cp] as usize; + }} + ", + out.trie.stages.len() - 1, + ); + for stage in &out.trie.stages { + if stage.index == 0 { + _ = writeln!( + buf, + " let s = *STAGE{}.get_unchecked(cp >> {}) as usize;", + stage.index, stage.shift, + ); + } else if stage.index != out.trie.stages.len() - 1 { + _ = writeln!( + buf, + " let s = *STAGE{}.get_unchecked(s + ((cp >> {}) & {})) as usize;", + stage.index, stage.shift, stage.mask, + ); + } else { + _ = writeln!( + buf, + " *STAGE{}.get_unchecked(s + (cp & {})) as usize", + stage.index, stage.mask, + ); + } + } + _ = writedoc!( + buf, + " + }} + }} + ", + ); + + _ = writedoc!( + buf, + " + #[inline(always)] + pub fn ucd_grapheme_cluster_joins(state: u32, lead: usize, trail: usize) -> u32 {{ + unsafe {{ + let l = lead & {0}; + let t = trail & {0}; + let s = GRAPHEME_JOIN_RULES.get_unchecked(state as usize); + (s[l] >> (t * 2)) & 3 + }} + }} + #[inline(always)] + pub fn ucd_grapheme_cluster_joins_done(state: u32) -> bool {{ + state == 3 + }} + #[inline(always)] + pub fn ucd_grapheme_cluster_character_width(val: usize) -> usize {{ + val >> {1} + }} + ", + out.ucd.packing.mask_cluster_break, + out.ucd.packing.shift_character_width, + ); + + if out.arg_line_breaks { + _ = writedoc!( + buf, + " + #[inline(always)] + pub fn ucd_line_break_joins(lead: usize, trail: usize) -> bool {{ + unsafe {{ + let l = (lead >> {0}) & {1}; + let t = (trail >> {0}) & {1}; + let s = *LINE_BREAK_JOIN_RULES.get_unchecked(l); + ((s >> t) & 1) != 0 + }} + }} + ", + out.ucd.packing.shift_line_break, + out.ucd.packing.mask_line_break, + ); + } + + if out.arg_extended { + _ = writedoc!( + buf, + " + #[inline(always)] + pub fn ucd_start_of_text_properties() -> usize {{ + {:#x} + }} + #[inline(always)] + pub fn ucd_tab_properties() -> usize {{ + {:#x} + }} + #[inline(always)] + pub fn ucd_linefeed_properties() -> usize {{ + {:#x} + }} + ", + TrieType::new( + &out.ucd.packing, + // Control behaves identical to SOT (start of text) in a way, + // as it doesn't join with any surrounding character. + ClusterBreak::Control, + LineBreak::StartOfText, + CharacterWidth::ZeroWidth, + ) + .value(), + out.ucd.values['\t' as usize].value(), + out.ucd.values['\n' as usize].value(), + ); + } + + buf.push_str("// END: Generated by grapheme-table-gen\n"); + buf +} + +fn extract_values_from_ucd(doc: &roxmltree::Document, out: &Output) -> anyhow::Result { + let packing = BitPacking::new(out.arg_line_breaks, out.arg_extended); + let ambiguous_value = + if out.arg_no_ambiguous { CharacterWidth::Narrow } else { CharacterWidth::Ambiguous }; + + let mut values = + vec![ + TrieType::new(&packing, ClusterBreak::Other, LineBreak::Other, CharacterWidth::Narrow,); + 1114112 + ]; + + let ns = "http://www.unicode.org/ns/2003/ucd/1.0"; + let root = doc.root_element(); + let description = root + .children() + .find(|n| n.has_tag_name((ns, "description"))) + .context("missing ucd description")?; + let repertoire = root + .children() + .find(|n| n.has_tag_name((ns, "repertoire"))) + .context("missing ucd repertoire")?; + let description = description.text().unwrap_or_default().to_string(); + + for group in repertoire.children().filter(|n| n.is_element()) { + const DEFAULT_ATTRIBUTES: UcdAttributes = UcdAttributes { + general_category: "", + line_break: "", + grapheme_cluster_break: "", + indic_conjunct_break: "", + extended_pictographic: "", + east_asian: "", + }; + let group_attributes = extract_attributes(&group, &DEFAULT_ATTRIBUTES); + + for char in group.children().filter(|n| n.is_element()) { + let char_attributes = extract_attributes(&char, &group_attributes); + let range = extract_range(&char); + + let mut cb = match char_attributes.grapheme_cluster_break { + "XX" => ClusterBreak::Other, // Anything else + // We ignore GB3 which demands that CR × LF do not break apart, because + // * these control characters won't normally reach our text storage + // * otherwise we're in a raw write mode and historically conhost stores them in separate cells + "CR" => ClusterBreak::CR, // Carriage Return + "LF" => ClusterBreak::LF, // Line Feed + "CN" => ClusterBreak::Control, // Control + "EX" | "SM" => ClusterBreak::Extend, // Extend, SpacingMark + "PP" => ClusterBreak::Prepend, // Prepend + "ZWJ" => ClusterBreak::ZWJ, // Zero Width Joiner + "RI" => ClusterBreak::RI, // Regional Indicator + "L" => ClusterBreak::HangulL, // Hangul Syllable Type L + "V" => ClusterBreak::HangulV, // Hangul Syllable Type V + "T" => ClusterBreak::HangulT, // Hangul Syllable Type T + "LV" => ClusterBreak::HangulLV, // Hangul Syllable Type LV + "LVT" => ClusterBreak::HangulLVT, // Hangul Syllable Type LVT + _ => bail!( + "Unrecognized GCB {:?} for U+{:04X} to U+{:04X}", + char_attributes.grapheme_cluster_break, + range.start(), + range.end() + ), + }; + + if char_attributes.extended_pictographic == "Y" { + // Currently every single Extended_Pictographic codepoint happens to be GCB=XX. + // This is fantastic for us because it means we can stuff it into the ClusterBreak enum + // and treat it as an alias of EXTEND, but with the special GB11 properties. + if cb != ClusterBreak::Other { + bail!( + "Unexpected GCB {:?} with ExtPict=Y for U+{:04X} to U+{:04X}", + char_attributes.grapheme_cluster_break, + range.start(), + range.end() + ); + } + + cb = ClusterBreak::ExtPic; + } + + cb = match char_attributes.indic_conjunct_break { + "None" | "Extend" => cb, + "Linker" => ClusterBreak::InCBLinker, + "Consonant" => ClusterBreak::InCBConsonant, + _ => bail!( + "Unrecognized InCB {:?} for U+{:04X} to U+{:04X}", + char_attributes.indic_conjunct_break, + range.start(), + range.end() + ), + }; + + let mut cw = match char_attributes.east_asian { + "N" | "Na" | "H" => CharacterWidth::Narrow, // Half-width, Narrow, Neutral + "F" | "W" => CharacterWidth::Wide, // Wide, Full-width + "A" => ambiguous_value, // Ambiguous + _ => bail!( + "Unrecognized ea {:?} for U+{:04X} to U+{:04X}", + char_attributes.east_asian, + range.start(), + range.end() + ), + }; + + // There's no "ea" attribute for "zero width" so we need to do that ourselves. This matches: + // Me: Mark, enclosing + // Mn: Mark, non-spacing + // Cf: Control, format + match char_attributes.general_category { + "Cf" if cb == ClusterBreak::Control => { + // A significant portion of Cf characters are not just gc=Cf (= commonly considered zero-width), + // but also GCB=CN (= does not join). This is a bit of a problem for terminals, + // because they don't support zero-width graphemes, as zero-width columns can't exist. + // So, we turn all of them into Extend, which is roughly how wcswidth() would treat them. + cb = ClusterBreak::Extend; + cw = CharacterWidth::ZeroWidth; + } + "Me" | "Mn" | "Cf" => { + cw = CharacterWidth::ZeroWidth; + } + _ => {} + }; + + let lb = if out.arg_line_breaks { + let lb_ea = matches!(char_attributes.east_asian, "F" | "W" | "H"); + match char_attributes.line_break { + "WJ" => LineBreak::WordJoiner, + "ZW" => LineBreak::ZeroWidthSpace, + "GL" => LineBreak::Glue, + "SP" => LineBreak::Space, + + "BA" => LineBreak::BreakAfter, + "BB" => LineBreak::BreakBefore, + "HY" => LineBreak::Hyphen, + + "CL" => LineBreak::ClosePunctuation, + "CP" if lb_ea => LineBreak::CloseParenthesis_EA, + "CP" => LineBreak::CloseParenthesis_NotEA, + "EX" => LineBreak::Exclamation, + "IN" => LineBreak::Inseparable, + "NS" => LineBreak::Nonstarter, + "OP" if lb_ea => LineBreak::OpenPunctuation_EA, + "OP" => LineBreak::OpenPunctuation_NotEA, + "QU" => LineBreak::Quotation, + + "IS" => LineBreak::InfixNumericSeparator, + "NU" => LineBreak::Numeric, + "PO" => LineBreak::PostfixNumeric, + "PR" => LineBreak::PrefixNumeric, + "SY" => LineBreak::SymbolsAllowingBreakAfter, + + "AL" | "HL" => LineBreak::Alphabetic, + "ID" | "EB" | "EM" => LineBreak::Ideographic, + + _ => LineBreak::Other, + } + } else { + LineBreak::Other + }; + + values[range].fill(TrieType::new(&packing, cb, lb, cw)); + } + } + + if out.arg_extended { + values['\t' as usize].change_cluster_break_ext(&packing, ClusterBreakExt::ControlTab); + } + + // U+00AD: Soft Hyphen + // A soft hyphen is a hint that a word break is allowed at that position. + // By default, the glyph is supposed to be invisible, and only if + // a word break occurs, the text renderer should display a hyphen. + // A terminal does not support computerized typesetting, but unlike the other + // gc=Cf cases we give it a Narrow width, because that matches wcswidth(). + values[0x00AD].change_width(&packing, CharacterWidth::Narrow); + + // U+2500 to U+257F: Box Drawing block + // U+2580 to U+259F: Block Elements block + // By default, CharacterWidth.Ambiguous, but by convention .Narrow in terminals. + // + // Most of these characters are LineBreak.Other, but some are actually LineBreak.Alphabetic. + // But to us this doesn't really matter much, because it doesn't make much sense anyway that + // a light double dash is "alphabetic" while a light triple dash is not. + values[0x2500..=0x259F].fill(TrieType::new( + &packing, + ClusterBreak::Other, + LineBreak::Other, + CharacterWidth::Narrow, + )); + + // U+FE0F Variation Selector-16 is used to turn unqualified Emojis into qualified ones. + // By convention, this turns them from being ambiguous width (= narrow) into wide ones. + // We achieve this here by explicitly giving this codepoint a wide width. + // Later down below we'll clamp width back to <= 2. + // + // U+FE0F actually has a LineBreak property of CM (Combining Mark), + // but for us that's equivalent to Other. + values[0xFE0F].change_width(&packing, CharacterWidth::Wide); + + Ok(Ucd { description, values, packing }) +} + +struct UcdAttributes<'a> { + general_category: &'a str, + line_break: &'a str, + grapheme_cluster_break: &'a str, + indic_conjunct_break: &'a str, + extended_pictographic: &'a str, + east_asian: &'a str, +} + +fn extract_attributes<'a>( + node: &'a roxmltree::Node, + default: &'a UcdAttributes, +) -> UcdAttributes<'a> { + UcdAttributes { + general_category: node.attribute("gc").unwrap_or(default.general_category), + line_break: node.attribute("lb").unwrap_or(default.line_break), + grapheme_cluster_break: node.attribute("GCB").unwrap_or(default.grapheme_cluster_break), + indic_conjunct_break: node.attribute("InCB").unwrap_or(default.indic_conjunct_break), + extended_pictographic: node.attribute("ExtPict").unwrap_or(default.extended_pictographic), + east_asian: node.attribute("ea").unwrap_or(default.east_asian), + } +} + +fn extract_range(node: &roxmltree::Node) -> RangeInclusive { + let (first, last) = match node.attribute("cp") { + Some(val) => { + let cp = usize::from_str_radix(val, 16).unwrap(); + (cp, cp) + } + None => ( + usize::from_str_radix(node.attribute("first-cp").unwrap_or("0"), 16).unwrap(), + usize::from_str_radix(node.attribute("last-cp").unwrap_or("0"), 16).unwrap(), + ), + }; + first..=last +} + +fn build_best_trie( + uncompressed: &[TrieType], + min_shift: usize, + max_shift: usize, + stages: usize, +) -> Trie { + let depth = stages - 1; + let delta = max_shift - min_shift + 1; + let total = delta.pow(depth as u32); + + let mut tasks = Vec::new(); + for i in 0..total { + let mut shifts = vec![0; depth]; + let mut index = i; + for s in &mut shifts { + *s = min_shift + (index % delta); + index /= delta; + } + tasks.push(shifts); + } + + tasks + .par_iter() + .map(|shifts| build_trie(uncompressed.to_vec(), shifts)) + .min_by_key(|t| t.total_size) + .unwrap() +} + +fn build_trie(uncompressed: Vec, shifts: &[usize]) -> Trie { + // Fun fact: Rust optimizes the into_iter/collect into a no-op. Neat! + let mut uncompressed: Vec = uncompressed.into_iter().map(|c| c.value()).collect(); + let mut cumulative_shift = 0; + let mut stages = Vec::new(); + + for (stage, &shift) in shifts.iter().enumerate() { + let chunk_size = 1 << shift; + let mut cache = HashMap::new(); + let mut compressed = Vec::new(); + let mut offsets = Vec::new(); + let mut off = 0; + + while off < uncompressed.len() { + let chunk = &uncompressed[off..off + chunk_size.min(uncompressed.len() - off)]; + + let offset = if stage == 0 && off < 0x80 { + // The first stage (well, really the last stage - the one which contains the values instead of indices) + // contains a direct 1:1 mapping for all ASCII codepoints as they're most common in IT environments. + compressed.extend_from_slice(chunk); + (compressed.len() - chunk.len()) as u32 + } else { + *cache.entry(chunk).or_insert_with(|| { + if let Some(existing) = find_existing(&compressed, chunk) { + existing as u32 + } else { + let overlap = measure_overlap(&compressed, chunk); + compressed.extend_from_slice(&chunk[overlap..]); + (compressed.len() - chunk.len()) as u32 + } + }) + }; + + offsets.push(offset); + off += chunk.len(); + } + + stages.push(Stage { + values: compressed, + index: shifts.len() - stages.len(), + shift: cumulative_shift, + mask: chunk_size - 1, + bits: 0, + }); + + uncompressed = offsets; + cumulative_shift += shift; + } + + stages.push(Stage { + values: uncompressed, + index: 0, + shift: cumulative_shift, + mask: usize::MAX, + bits: 0, + }); + + stages.reverse(); + + for stage in stages.iter_mut() { + let max_val = stage.values.iter().max().cloned().unwrap_or(0); + stage.bits = match max_val { + 0..0x100 => 8, + 0x100..0x10000 => 16, + _ => 32, + }; + } + + let total_size: usize = stages.iter().map(|stage| (stage.bits / 8) * stage.values.len()).sum(); + + Trie { stages, total_size } +} + +fn find_existing(haystack: &[u32], needle: &[u32]) -> Option { + haystack.windows(needle.len()).position(|window| window == needle) +} + +fn measure_overlap(prev: &[u32], next: &[u32]) -> usize { + (0..prev.len().min(next.len())) + .rev() + .find(|&i| prev[prev.len() - i..] == next[..i]) + .unwrap_or(0) +} + +fn prepare_rules_row(row: &[i32], bit_width: usize, non_joiner_value: i32) -> u32 { + row.iter().enumerate().fold(0u32, |acc, (trail, &value)| { + let value = if value < 0 { non_joiner_value } else { value }; + acc | ((value as u32) << (trail * bit_width)) + }) +} diff --git a/pkgs/edit/tools/grapheme-table-gen/src/rules.rs b/pkgs/edit/tools/grapheme-table-gen/src/rules.rs new file mode 100644 index 0000000..7ddadb3 --- /dev/null +++ b/pkgs/edit/tools/grapheme-table-gen/src/rules.rs @@ -0,0 +1,288 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Used as an indicator in our rules for ÷ ("does not join"). +// Underscore is one of the few characters that are permitted as an identifier, +// are monospace in most fonts and also visually distinct from the digits. +const X: i32 = -1; + +// The following rules are based on the Grapheme Cluster Boundaries section of Unicode Standard Annex #29, +// but slightly modified to allow for use with a plain MxN lookup table. +// +// Break at the start and end of text, unless the text is empty. +// GB1: ~ sot ÷ Any +// GB2: ~ Any ÷ eot +// Handled by our ucd_* functions. +// +// Do not break between a CR and LF. Otherwise, break before and after controls. +// GB3: ✓ CR × LF +// GB4: ✓ (Control | CR | LF) ÷ +// GB5: ✓ ÷ (Control | CR | LF) +// +// Do not break Hangul syllable or other conjoining sequences. +// GB6: ✓ L × (L | V | LV | LVT) +// GB7: ✓ (LV | V) × (V | T) +// GB8: ✓ (LVT | T) × T +// +// Do not break before extending characters or ZWJ. +// GB9: ✓ × (Extend | ZWJ) +// +// Do not break before SpacingMarks, or after Prepend characters. +// GB9a: ✓ × SpacingMark +// GB9b: ✓ Prepend × +// +// Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker. +// GB9c: ~ \p{InCB=Linker} × \p{InCB=Consonant} +// × \p{InCB=Linker} +// modified from +// \p{InCB=Consonant} [ \p{InCB=Extend} \p{InCB=Linker} ]* \p{InCB=Linker} [ \p{InCB=Extend} \p{InCB=Linker} ]* × \p{InCB=Consonant} +// because this has almost the same effect from what I can tell for most text, and greatly simplifies our design. +// +// Do not break within emoji modifier sequences or emoji zwj sequences. +// GB11: ~ ZWJ × \p{Extended_Pictographic} modified from \p{Extended_Pictographic} Extend* ZWJ × \p{Extended_Pictographic} +// because this allows us to use LUTs, while working for most valid text. +// +// Do not break within emoji flag sequences. That is, do not break between regional indicator (RI) symbols if there is an odd number of RI characters before the break point. +// GB12: ~ sot (RI RI)* RI × RI +// GB13: ~ [^RI] (RI RI)* RI × RI +// the lookup table we generate supports RIs via something akin to RI ÷ RI × RI ÷ RI, but the corresponding +// grapheme cluster algorithm doesn't count them. It would need to be updated to recognize and special-case RIs. +// +// Otherwise, break everywhere. +// GB999: ✓ Any ÷ Any +// +// This is a great reference for the resulting table: +// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakTest.html +#[rustfmt::skip] +pub const JOIN_RULES_GRAPHEME_CLUSTER: [[[i32; 16]; 16]; 2] = [ + // Base table + [ + /* ↓ leading → trailing codepoint */ + /* | Other | CR | LF | Control | Extend | RI | Prepend | HangulL | HangulV | HangulT | HangulLV | HangulLVT | InCBLinker | InCBConsonant | ExtPic | ZWJ | */ + /* Other | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* CR | */ [X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */], + /* LF | */ [X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */], + /* Control | */ [X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */], + /* Extend | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* RI | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, 1 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* Prepend | */ [0 /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */], + /* HangulL | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */, 0 /* | */, X /* | */, 0 /* | */, 0 /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* HangulV | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* HangulT | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* HangulLV | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* HangulLVT | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* InCBLinker | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, 0 /* | */, X /* | */, 0 /* | */], + /* InCBConsonant | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* ExtPic | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* ZWJ | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, 0 /* | */, 0 /* | */], + ], + // Once we have encountered a Regional Indicator pair we'll enter this table. + // It's a copy of the base table, but instead of RI × RI, we're RI ÷ RI. + [ + /* ↓ leading → trailing codepoint */ + /* | Other | CR | LF | Control | Extend | RI | Prepend | HangulL | HangulV | HangulT | HangulLV | HangulLVT | InCBLinker | InCBConsonant | ExtPic | ZWJ | */ + /* Other | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* CR | */ [X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */], + /* LF | */ [X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */], + /* Control | */ [X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */], + /* Extend | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* RI | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* Prepend | */ [0 /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */, 0 /* | */], + /* HangulL | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */, 0 /* | */, X /* | */, 0 /* | */, 0 /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* HangulV | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* HangulT | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* HangulLV | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* HangulLVT | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* InCBLinker | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, 0 /* | */, X /* | */, 0 /* | */], + /* InCBConsonant | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* ExtPic | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, 0 /* | */], + /* ZWJ | */ [X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 0 /* | */, X /* | */, 0 /* | */, 0 /* | */], + ], +]; + +// The following rules are based on Unicode Standard Annex #14: Line Breaking Properties, +// but heavily modified to allow for use with lookup tables. +// +// TODO: I should go through this and cross check: https://www.unicode.org/Public/draft/ucd/auxiliary/LineBreakTest.html +// +// NOTE: If you convert these rules into a lookup table, you must apply them in reverse order. +// This is because the rules are ordered from most to least important (e.g. LB8 overrides LB18). +// +// Resolve line breaking classes: +// LB1: Assign a line breaking class [...]. +// ✗ Unicode does that for us via the "lb" attribute. +// +// Start and end of text: +// LB2: Never break at the start of text. +// ~ Functionality not needed. +// LB3: Always break at the end of text. +// ~ Functionality not needed. +// +// Mandatory breaks: +// LB4: Always break after hard line breaks. +// ~ Handled by our ucd_* functions. +// LB5: Treat CR followed by LF, as well as CR, LF, and NL as hard line breaks. +// ~ Handled by our ucd_* functions. +// LB6: Do not break before hard line breaks. +// ~ Handled by our ucd_* functions. +// +// Explicit breaks and non-breaks: +// LB7: Do not break before spaces or zero width space. +// ✓ × SP +// ✓ × ZW +// LB8: Break before any character following a zero-width space, even if one or more spaces intervene. +// ~ ZW ÷ modified from ZW SP* ÷ because it's not worth being this anal about accuracy here. +// LB8a: Do not break after a zero width joiner. +// ~ Our ucd_* functions never break within grapheme clusters. +// +// Combining marks: +// LB9: Do not break a combining character sequence; treat it as if it has the line breaking class of the base character in all of the following rules. Treat ZWJ as if it were CM. +// ~ Our ucd_* functions never break within grapheme clusters. +// LB10: Treat any remaining combining mark or ZWJ as AL. +// ✗ To be honest, I'm not entirely sure, I understand the implications of this rule. +// +// Word joiner: +// LB11: Do not break before or after Word joiner and related characters. +// ✓ × WJ +// ✓ WJ × +// +// Non-breaking characters: +// LB12: Do not break after NBSP and related characters. +// ✓ GL × +// LB12a: Do not break before NBSP and related characters, except after spaces and hyphens. +// ✓ [^SP BA HY] × GL +// +// Opening and closing: +// LB13: Do not break before ']' or '!' or '/', even after spaces. +// ✓ × CL +// ✓ × CP +// ✓ × EX +// ✓ × SY +// LB14: Do not break after '[', even after spaces. +// ~ OP × modified from OP SP* × just because it's simpler. It would be nice to address this. +// LB15a: Do not break after an unresolved initial punctuation that lies at the start of the line, after a space, after opening punctuation, or after an unresolved quotation mark, even after spaces. +// ✗ Not implemented. Seemed too complex for little gain? +// LB15b: Do not break before an unresolved final punctuation that lies at the end of the line, before a space, before a prohibited break, or before an unresolved quotation mark, even after spaces. +// ✗ Not implemented. Seemed too complex for little gain? +// LB15c: Break before a decimal mark that follows a space, for instance, in 'subtract .5'. +// ~ SP ÷ IS modified from SP ÷ IS NU because this fits neatly with LB15d. +// LB15d: Otherwise, do not break before ';', ',', or '.', even after spaces. +// ✓ × IS +// LB16: Do not break between closing punctuation and a nonstarter (lb=NS), even with intervening spaces. +// ✗ Not implemented. Could be useful in the future, but its usefulness seemed limited to me. +// LB17: Do not break within '——', even with intervening spaces. +// ✗ Not implemented. Terminal applications nor code use em-dashes much anyway. +// +// Spaces: +// LB18: Break after spaces. +// ✓ SP ÷ +// +// Special case rules: +// LB19: Do not break before non-initial unresolved quotation marks, such as ' ” ' or ' " ', nor after non-final unresolved quotation marks, such as ' “ ' or ' " '. +// ~ × QU modified from × [ QU - \p{Pi} ] +// ~ QU × modified from [ QU - \p{Pf} ] × +// We implement the Unicode 16.0 instead of 16.1 rules, because it's simpler and allows us to use a LUT. +// LB19a: Unless surrounded by East Asian characters, do not break either side of any unresolved quotation marks. +// ✗ [^$EastAsian] × QU +// ✗ × QU ( [^$EastAsian] | eot ) +// ✗ QU × [^$EastAsian] +// ✗ ( sot | [^$EastAsian] ) QU × +// Same as LB19. +// LB20: Break before and after unresolved CB. +// ✗ We break by default. Unicode inline objects are super irrelevant in a terminal in either case. +// LB20a: Do not break after a word-initial hyphen. +// ✗ Not implemented. Seemed not worth the hassle as the window will almost always be >1 char wide. +// LB21: Do not break before hyphen-minus, other hyphens, fixed-width spaces, small kana, and other non-starters, or after acute accents. +// ✓ × BA +// ✓ × HY +// ✓ × NS +// ✓ BB × +// ✗ Added HY ÷ HY, because of the following note in TR14: +// > If used as hyphen, it acts like U+2010 HYPHEN, which has line break class BA. +// LB21a: Do not break after the hyphen in Hebrew + Hyphen + non-Hebrew. +// ✗ Not implemented. Perhaps in the future. +// LB21b: Do not break between Solidus and Hebrew letters. +// ✗ Not implemented. Perhaps in the future. +// LB22: Do not break before ellipses. +// ✓ × IN +// +// Numbers: +// LB23: Do not break between digits and letters. +// ✓ (AL | HL) × NU +// ✓ NU × (AL | HL) +// LB23a: Do not break between numeric prefixes and ideographs, or between ideographs and numeric postfixes. +// ✓ PR × (ID | EB | EM) +// ✓ (ID | EB | EM) × PO +// LB24: Do not break between numeric prefix/postfix and letters, or between letters and prefix/postfix. +// ✓ (PR | PO) × (AL | HL) +// ✓ (AL | HL) × (PR | PO) +// LB25: Do not break numbers: +// ~ CL × PO modified from NU ( SY | IS )* CL × PO +// ~ CP × PO modified from NU ( SY | IS )* CP × PO +// ~ CL × PR modified from NU ( SY | IS )* CL × PR +// ~ CP × PR modified from NU ( SY | IS )* CP × PR +// ~ ( NU | SY | IS ) × PO modified from NU ( SY | IS )* × PO +// ~ ( NU | SY | IS ) × PR modified from NU ( SY | IS )* × PR +// ~ PO × OP modified from PO × OP NU +// ~ PO × OP modified from PO × OP IS NU +// ✓ PO × NU +// ~ PR × OP modified from PR × OP NU +// ~ PR × OP modified from PR × OP IS NU +// ✓ PR × NU +// ✓ HY × NU +// ✓ IS × NU +// ~ ( NU | SY | IS ) × NU modified from NU ( SY | IS )* × NU +// Most were simplified because the cases this additionally allows don't matter much here. +// +// Korean syllable blocks +// LB26: Do not break a Korean syllable. +// ✗ Our ucd_* functions never break within grapheme clusters. +// LB27: Treat a Korean Syllable Block the same as ID. +// ✗ Our ucd_* functions never break within grapheme clusters. +// +// Finally, join alphabetic letters into words and break everything else. +// LB28: Do not break between alphabetics ("at"). +// ✓ (AL | HL) × (AL | HL) +// LB28a: Do not break inside the orthographic syllables of Brahmic scripts. +// ✗ Our ucd_* functions never break within grapheme clusters. +// LB29: Do not break between numeric punctuation and alphabetics ("e.g."). +// ✓ IS × (AL | HL) +// LB30: Do not break between letters, numbers, or ordinary symbols and opening or closing parentheses. +// ✓ (AL | HL | NU) × [OP-$EastAsian] +// ✓ [CP-$EastAsian] × (AL | HL | NU) +// LB30a: Break between two regional indicator symbols if and only if there are an even number of regional indicators preceding the position of the break. +// ✗ Our ucd_* functions never break within grapheme clusters. +// LB30b: Do not break between an emoji base (or potential emoji) and an emoji modifier. +// ✗ Our ucd_* functions never break within grapheme clusters. +// LB31: Break everywhere else. +// ✗ Our default behavior. +#[rustfmt::skip] +pub const JOIN_RULES_LINE_BREAK: [[i32; 24]; 25] = [ + /* ↓ leading → trailing codepoint */ + /* | Other | WordJoiner | ZeroWidthSpace | Glue | Space | BreakAfter | BreakBefore | Hyphen | ClosePunctuation | CloseParenthesis_EA | CloseParenthesis_NotEA | Exclamation | Inseparable | Nonstarter | OpenPunctuation_EA | OpenPunctuation_NotEA | Quotation | InfixNumericSeparator | Numeric | PostfixNumeric | PrefixNumeric | SymbolsAllowingBreakAfter | Alphabetic | Ideographic | */ + /* Other | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, X /* | */, 1 /* | */, X /* | */, X /* | */], + /* WordJoiner | */ [1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */], + /* ZeroWidthSpace | */ [X /* | */, X /* | */, X /* | */, X /* | */, 1 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */], + /* Glue | */ [1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */], + /* Space | */ [X /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, X /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, X /* | */, 1 /* | */, X /* | */, X /* | */], + /* BreakAfter | */ [X /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, X /* | */, 1 /* | */, X /* | */, X /* | */], + /* BreakBefore | */ [1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */], + /* Hyphen | */ [X /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, X /* | */, X /* | */], + /* ClosePunctuation | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */], + /* CloseParenthesis_EA | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */], + /* CloseParenthesis_NotEA | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */], + /* Exclamation | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, X /* | */, 1 /* | */, X /* | */, X /* | */], + /* Inseparable | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, X /* | */, 1 /* | */, X /* | */, X /* | */], + /* Nonstarter | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, X /* | */, 1 /* | */, X /* | */, X /* | */], + /* OpenPunctuation_EA | */ [1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */], + /* OpenPunctuation_NotEA | */ [1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */], + /* Quotation | */ [1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */], + /* InfixNumericSeparator | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */], + /* Numeric | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */], + /* PostfixNumeric | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */], + /* PrefixNumeric | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */], + /* SymbolsAllowingBreakAfter | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */], + /* Alphabetic | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */], + /* Ideographic | */ [X /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, 1 /* | */, X /* | */, X /* | */, 1 /* | */, 1 /* | */, X /* | */, 1 /* | */, X /* | */, 1 /* | */, X /* | */, X /* | */], + /* StartOfText | */ [X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */, X /* | */], +];