diff --git a/test/Dockerfile b/test/Dockerfile deleted file mode 100644 index 910df5d91..000000000 --- a/test/Dockerfile +++ /dev/null @@ -1,66 +0,0 @@ -#-------------------------------------------------------------------- -# Alacritty since it has no build in Ubuntu -#-------------------------------------------------------------------- -FROM rust AS alacritty - -RUN apt-get update && apt-get install -y \ - cmake pkg-config libfreetype6-dev libfontconfig1-dev \ - libxcb-xfixes0-dev python3 git - -RUN git clone https://github.com/alacritty/alacritty.git /tmp/alacritty - -WORKDIR /tmp/alacritty - -RUN git checkout tags/$(git describe --tags $(git rev-list --tags --max-count=1)) - -RUN RUSTFLAGS='-C link-arg=-s' cargo build --release - -#-------------------------------------------------------------------- -# Wraptest -#-------------------------------------------------------------------- -FROM ubuntu:22.04 AS wraptest - -RUN apt-get update && apt-get install -y \ - build-essential git - -RUN git clone https://github.com/mattiase/wraptest.git /tmp/wraptest - -WORKDIR /tmp/wraptest - -RUN gcc -o wraptest wraptest.c - -#-------------------------------------------------------------------- -# Main Runner -#-------------------------------------------------------------------- -# Note: we used to use Alpine, but we need to use an OS that is more -# glibc friendly because on Linux we still require glibc (active todo to -# support musl). -FROM ubuntu:22.04 - -# Base deps -RUN apt-get update && apt-get install -y \ - fonts-inconsolata \ - i3 \ - imagemagick \ - libgl1-mesa-dev \ - libxcursor1 \ - patchelf \ - software-properties-common \ - xdotool \ - xvfb \ - vttest - -# Terminals -RUN apt-get install -y \ - xterm - -RUN rm -rf /var/lib/apt/lists/* - -COPY --from=alacritty /tmp/alacritty/target/release/alacritty /usr/bin/alacritty -COPY --from=wraptest /tmp/wraptest/wraptest /usr/bin/wraptest - -COPY ./run.sh /entrypoint.sh - -COPY ./ghostty /usr/bin/ghostty - -ENTRYPOINT ["/bin/bash"] diff --git a/test/README.md b/test/README.md index 8b59aa062..2ce7c2ae9 100644 --- a/test/README.md +++ b/test/README.md @@ -1,40 +1,4 @@ -# Acceptance Testing +# Test Utilities -This directory contains an acceptance test suite for ghostty. This works -by running the terminal emulator within a windowing environment, capturing a -screenshot, and comparing results. We use this to visually verify that -all rendering conforms to what we expect. - -This test suite can also execute alternate terminal emulators so that we -can easily compare outputs between them. - -## Running a Single Test - -To run a single test, use the `run-host.sh` script. This must be executed -from this directory. Example: - -```shell-session -$ ./run-host.sh --exec xterm --case /src/cases/vttest/launch.sh -``` - -The `--case` flag uses `/src` as the root for this directory. - -The `--update` flag can be used to update the screenshot in place. This -should be used to gather a new screenshot. If you want to compare to the old -screenshot, copy the old one or use git to revert. - -## Running the Full Suite - -**Warning:** This can take a long time and isn't recommended. The CI -environment automatically runs the full test suite and is the recommended -approach. - -To run the full test suite against all terminal emulators, use the -`run-all.sh` script. This optionally takes an `--exec` parameter to run -the full test suite against only a single terminal emulator. - -## Modifying the `ghostty` Binary - -This test suite expects the `ghostty` binary to be in _this directory_. -You can manually copy it into place. Each time you modify the binary, you -must rebuild the Docker image. +This directory contains anything related to testing Ghostty that doesn't +fit within the standard Zig testing framework alongside the source. diff --git a/test/cases/vttest/1_1.sh b/test/cases/vttest/1_1.sh deleted file mode 100755 index daf02f411..000000000 --- a/test/cases/vttest/1_1.sh +++ /dev/null @@ -1,7 +0,0 @@ -function test_do { - xdotool type "vttest" - xdotool key Return - sleep 1 - xdotool type "1" - xdotool key Return -} diff --git a/test/cases/vttest/1_1.sh.alacritty.png b/test/cases/vttest/1_1.sh.alacritty.png deleted file mode 100644 index 0be3ead8f..000000000 Binary files a/test/cases/vttest/1_1.sh.alacritty.png and /dev/null differ diff --git a/test/cases/vttest/1_1.sh.ghostty.png b/test/cases/vttest/1_1.sh.ghostty.png deleted file mode 100644 index 7afe135ec..000000000 Binary files a/test/cases/vttest/1_1.sh.ghostty.png and /dev/null differ diff --git a/test/cases/vttest/1_1.sh.xterm.png b/test/cases/vttest/1_1.sh.xterm.png deleted file mode 100644 index 3253846ad..000000000 Binary files a/test/cases/vttest/1_1.sh.xterm.png and /dev/null differ diff --git a/test/cases/vttest/1_2.sh b/test/cases/vttest/1_2.sh deleted file mode 100644 index d55a6587b..000000000 --- a/test/cases/vttest/1_2.sh +++ /dev/null @@ -1,9 +0,0 @@ -function test_do { - xdotool type "vttest" - xdotool key Return - sleep 1 - xdotool type "1" - xdotool key Return - sleep 0.5 - xdotool key Return -} diff --git a/test/cases/vttest/1_2.sh.alacritty.png b/test/cases/vttest/1_2.sh.alacritty.png deleted file mode 100644 index 47d289ea5..000000000 Binary files a/test/cases/vttest/1_2.sh.alacritty.png and /dev/null differ diff --git a/test/cases/vttest/1_2.sh.ghostty.png b/test/cases/vttest/1_2.sh.ghostty.png deleted file mode 100644 index ed43c7e44..000000000 Binary files a/test/cases/vttest/1_2.sh.ghostty.png and /dev/null differ diff --git a/test/cases/vttest/1_2.sh.xterm.png b/test/cases/vttest/1_2.sh.xterm.png deleted file mode 100644 index 97437fe7e..000000000 Binary files a/test/cases/vttest/1_2.sh.xterm.png and /dev/null differ diff --git a/test/cases/vttest/1_3.sh b/test/cases/vttest/1_3.sh deleted file mode 100644 index 16689a43c..000000000 --- a/test/cases/vttest/1_3.sh +++ /dev/null @@ -1,11 +0,0 @@ -function test_do { - xdotool type "vttest" - xdotool key Return - sleep 1 - xdotool type "1" - xdotool key Return - sleep 0.5 - xdotool key Return - sleep 0.5 - xdotool key Return -} diff --git a/test/cases/vttest/1_3.sh.alacritty.png b/test/cases/vttest/1_3.sh.alacritty.png deleted file mode 100644 index 88a0ee527..000000000 Binary files a/test/cases/vttest/1_3.sh.alacritty.png and /dev/null differ diff --git a/test/cases/vttest/1_3.sh.ghostty.png b/test/cases/vttest/1_3.sh.ghostty.png deleted file mode 100644 index 871b944de..000000000 Binary files a/test/cases/vttest/1_3.sh.ghostty.png and /dev/null differ diff --git a/test/cases/vttest/1_3.sh.xterm.png b/test/cases/vttest/1_3.sh.xterm.png deleted file mode 100644 index d0da962a9..000000000 Binary files a/test/cases/vttest/1_3.sh.xterm.png and /dev/null differ diff --git a/test/cases/vttest/1_4.sh b/test/cases/vttest/1_4.sh deleted file mode 100644 index 7bc120636..000000000 --- a/test/cases/vttest/1_4.sh +++ /dev/null @@ -1,13 +0,0 @@ -function test_do { - xdotool type "vttest" - xdotool key Return - sleep 1 - xdotool type "1" - xdotool key Return - sleep 0.5 - xdotool key Return - sleep 0.5 - xdotool key Return - sleep 0.5 - xdotool key Return -} diff --git a/test/cases/vttest/1_4.sh.alacritty.png b/test/cases/vttest/1_4.sh.alacritty.png deleted file mode 100644 index b829aed4e..000000000 Binary files a/test/cases/vttest/1_4.sh.alacritty.png and /dev/null differ diff --git a/test/cases/vttest/1_4.sh.ghostty.png b/test/cases/vttest/1_4.sh.ghostty.png deleted file mode 100644 index f7a67c23f..000000000 Binary files a/test/cases/vttest/1_4.sh.ghostty.png and /dev/null differ diff --git a/test/cases/vttest/1_4.sh.xterm.png b/test/cases/vttest/1_4.sh.xterm.png deleted file mode 100644 index 421949860..000000000 Binary files a/test/cases/vttest/1_4.sh.xterm.png and /dev/null differ diff --git a/test/cases/vttest/1_5.sh b/test/cases/vttest/1_5.sh deleted file mode 100644 index 0a840300e..000000000 --- a/test/cases/vttest/1_5.sh +++ /dev/null @@ -1,15 +0,0 @@ -function test_do { - xdotool type "vttest" - xdotool key Return - sleep 1 - xdotool type "1" - xdotool key Return - sleep 0.5 - xdotool key Return - sleep 0.5 - xdotool key Return - sleep 0.5 - xdotool key Return - sleep 0.5 - xdotool key Return -} diff --git a/test/cases/vttest/1_5.sh.alacritty.png b/test/cases/vttest/1_5.sh.alacritty.png deleted file mode 100644 index cdbd58743..000000000 Binary files a/test/cases/vttest/1_5.sh.alacritty.png and /dev/null differ diff --git a/test/cases/vttest/1_5.sh.ghostty.png b/test/cases/vttest/1_5.sh.ghostty.png deleted file mode 100644 index c04f47d97..000000000 Binary files a/test/cases/vttest/1_5.sh.ghostty.png and /dev/null differ diff --git a/test/cases/vttest/1_5.sh.xterm.png b/test/cases/vttest/1_5.sh.xterm.png deleted file mode 100644 index 3453ac923..000000000 Binary files a/test/cases/vttest/1_5.sh.xterm.png and /dev/null differ diff --git a/test/cases/vttest/1_6.sh b/test/cases/vttest/1_6.sh deleted file mode 100644 index 9249c9bdf..000000000 --- a/test/cases/vttest/1_6.sh +++ /dev/null @@ -1,17 +0,0 @@ -function test_do { - xdotool type "vttest" - xdotool key Return - sleep 1 - xdotool type "1" - xdotool key Return - sleep 0.5 - xdotool key Return - sleep 0.5 - xdotool key Return - sleep 0.5 - xdotool key Return - sleep 0.5 - xdotool key Return - sleep 0.5 - xdotool key Return -} diff --git a/test/cases/vttest/1_6.sh.alacritty.png b/test/cases/vttest/1_6.sh.alacritty.png deleted file mode 100644 index 08cdbd52a..000000000 Binary files a/test/cases/vttest/1_6.sh.alacritty.png and /dev/null differ diff --git a/test/cases/vttest/1_6.sh.ghostty.png b/test/cases/vttest/1_6.sh.ghostty.png deleted file mode 100644 index 672b8eb25..000000000 Binary files a/test/cases/vttest/1_6.sh.ghostty.png and /dev/null differ diff --git a/test/cases/vttest/1_6.sh.xterm.png b/test/cases/vttest/1_6.sh.xterm.png deleted file mode 100644 index 43bdcb09e..000000000 Binary files a/test/cases/vttest/1_6.sh.xterm.png and /dev/null differ diff --git a/test/cases/vttest/launch.sh b/test/cases/vttest/launch.sh deleted file mode 100644 index 5cda9b087..000000000 --- a/test/cases/vttest/launch.sh +++ /dev/null @@ -1,4 +0,0 @@ -function test_do { - xdotool type "vttest" - xdotool key Return -} diff --git a/test/cases/vttest/launch.sh.alacritty.png b/test/cases/vttest/launch.sh.alacritty.png deleted file mode 100644 index 5575a2551..000000000 Binary files a/test/cases/vttest/launch.sh.alacritty.png and /dev/null differ diff --git a/test/cases/vttest/launch.sh.ghostty.png b/test/cases/vttest/launch.sh.ghostty.png deleted file mode 100644 index 283b20daf..000000000 Binary files a/test/cases/vttest/launch.sh.ghostty.png and /dev/null differ diff --git a/test/cases/vttest/launch.sh.xterm.png b/test/cases/vttest/launch.sh.xterm.png deleted file mode 100644 index 399e7089b..000000000 Binary files a/test/cases/vttest/launch.sh.xterm.png and /dev/null differ diff --git a/test/cases/wraptest.sh b/test/cases/wraptest.sh deleted file mode 100644 index 60fc36c22..000000000 --- a/test/cases/wraptest.sh +++ /dev/null @@ -1,4 +0,0 @@ -function test_do { - xdotool type "wraptest" - xdotool key Return -} diff --git a/test/cases/wraptest.sh.alacritty.png b/test/cases/wraptest.sh.alacritty.png deleted file mode 100644 index cc6001140..000000000 Binary files a/test/cases/wraptest.sh.alacritty.png and /dev/null differ diff --git a/test/cases/wraptest.sh.ghostty.png b/test/cases/wraptest.sh.ghostty.png deleted file mode 100644 index fff26df9a..000000000 Binary files a/test/cases/wraptest.sh.ghostty.png and /dev/null differ diff --git a/test/cases/wraptest.sh.xterm.png b/test/cases/wraptest.sh.xterm.png deleted file mode 100644 index 267d207b4..000000000 Binary files a/test/cases/wraptest.sh.xterm.png and /dev/null differ diff --git a/test/run-all.sh b/test/run-all.sh deleted file mode 100755 index d4a785a44..000000000 --- a/test/run-all.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# -# Run all of the test cases. All test cases are found by traversing -# the "cases" directory, finding all shell files, and executing the -# "./run-host.sh" command for each. - -DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) - -# We always copy the bin in case it was rebuilt -cp ${DIR}/../zig-out/bin/ghostty ${DIR}/ - -# Unix shortcut to just execute ./run-host for each one. We can do -# this less esoterically if we ever wanted. -find ${DIR}/cases \ - -type f \ - -name '*.sh' | \ - sort | \ - parallel \ - --will-cite \ - ${DIR}/run-host.sh \ - --case '{}' \ - --rewrite-abs-path \ - "$@" diff --git a/test/run-host.sh b/test/run-host.sh deleted file mode 100755 index da9dbe2e5..000000000 --- a/test/run-host.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -# -# This runs a single test case from the host (not from Docker itself). The -# arguments are the same as run.sh but this wraps it in docker. - -DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) -IMAGE=$(docker build --file ${DIR}/Dockerfile -q ${DIR}) - -docker run \ - --init \ - --rm \ - -v ${DIR}:/src \ - --entrypoint "xvfb-run" \ - $IMAGE \ - --server-args="-screen 0, 1600x900x24" \ - /entrypoint.sh "$@" diff --git a/test/run.sh b/test/run.sh deleted file mode 100755 index db05ede76..000000000 --- a/test/run.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env bash -# -# This script runs a given test case and captures a screenshot. This script -# expects to run in the Docker image so it just captures the full screen rather -# than a specific window. -# -# This script also compares the output to the expected value. The expected -# value is the case file with ".png" appended. If the "--update" flag is -# appended, the test case is updated. - -#-------------------------------------------------------------------- -# Helpers - -function has_func() { - declare -f -F $1 > /dev/null - return $? -} - -# Colors -export BOLD="\\e[1m" -export RED="\\e[1;31m" -export GREEN="\\e[1;32m" -export YELLOW="\\e[1;33m" -export WHITE="\\e[1;37m" -export RESET="\\e[0;39m" - -#-------------------------------------------------------------------- -# Flag parsing - -ARG_REWRITE=0 -ARG_UPDATE=0 -while [[ "$#" -gt 0 ]]; do - case $1 in - -e|--exec) ARG_EXEC="$2"; shift ;; - -c|--case) ARG_CASE="$2"; shift ;; - -o|--output) ARG_OUT="$2"; shift ;; - -u|--update) ARG_UPDATE=1 ;; - --rewrite-abs-path) ARG_REWRITE=1 ;; - *) echo "Unknown parameter passed: $1"; exit 1 ;; - esac - shift -done - -# Rewrite the path to be valid for us. This regex can be fooled in many ways -# but its good enough for my PC (mitchellh) and CI. Contributors feel free -# to harden it. -if [ "$ARG_REWRITE" -eq 1 ]; then - ARG_CASE=$(echo $ARG_CASE | sed -e 's/.*cases/\/src\/cases/') -fi - -# If we're updating, then just update the file in-place -GOLDEN_OUT="${ARG_CASE}.${ARG_EXEC}.png" -ARG_OUT="${ARG_CASE}.${ARG_EXEC}.actual.png" -if [ "$ARG_UPDATE" -eq 1 ]; then ARG_OUT=$GOLDEN_OUT; fi - -bad=0 -if [ -z "$ARG_EXEC" ]; then bad=1; fi -if [ -z "$ARG_CASE" ]; then bad=1; fi -if [ -z "$ARG_OUT" ]; then bad=1; fi -if [ $bad -ne 0 ]; then - echo "Usage: run.sh --exec --case --output " - exit 1 -fi - -# Load our test case -# shellcheck disable=SC1090 -source ${ARG_CASE} -if ! has_func "test_do"; then - echo "Test case is invalid." - exit 1 -fi - -# NOTE: This is a huge hack right now. -if [ "$ARG_EXEC" = "ghostty" ]; then - ARG_EXEC="/tmp/ghostty"; - - # Copy so we don't read/write race when running in parallel - cp /src/ghostty ${ARG_EXEC} - - # We build in Nix (maybe). To be sure, we replace the interpreter so - # it doesn't point to a Nix path. If we don't build in Nix, this should - # still be safe. - patchelf --set-interpreter /lib/ld-linux-"$(uname -m)".so.1 ${ARG_EXEC} -fi - -#-------------------------------------------------------------------- -# Some terminals require XDG be properly setup. We create a new -# set of XDG directories for this. -export XDG_BASE_DIR="/work/xdg" -export XDG_RUNTIME_DIR="${XDG_BASE_DIR}/runtime" -mkdir -p ${XDG_BASE_DIR} ${XDG_RUNTIME_DIR} -chmod 0700 $XDG_RUNTIME_DIR - -# Configure i3 -cat <${XDG_BASE_DIR}/i3.cfg -# i3 config file (v4) - -exec ${ARG_EXEC} - -bar { - mode invisible -} -EOF - -#-------------------------------------------------------------------- - -printf "${RESET}${BOLD}[$(basename $ARG_EXEC)]${RESET} $ARG_CASE ... ${RESET}" - -# Start up the program under test by launching i3. We use i3 so we can -# more carefully control the window settings, test resizing, etc. -WM_LOG="${XDG_BASE_DIR}/wm.log" -i3 -c ${XDG_BASE_DIR}/i3.cfg >${WM_LOG} 2>&1 & - -# Wait for startup -# TODO: we can probably use xdotool or wmctrl or something to detect if any -# windows actually launched and make error handling here better. -sleep 2 - -# Run our test case (should be defined in test case file) -test_do - -# Sleep a second to let it render -sleep 1 - -# Uncomment this and use run-host.sh to get logs of the terminal emulator -# cat $WM_LOG - -import -window root ${ARG_OUT} - -DIFF=$(compare -metric AE ${ARG_OUT} ${GOLDEN_OUT} null: 2>&1) -if [ $? -eq 2 ] ; then - printf "${RED}ERROR${RESET}\n" - exit 1 -else - if [ $DIFF -gt 0 ]; then - printf "${RED}Fail (Diff: ${WHITE}${DIFF}${RED})${RESET}\n" - exit 1 - else - printf "${GREEN}Pass (Diff: ${WHITE}${DIFF}${GREEN})${RESET}\n" - fi -fi -