ComputerSurge/lazygit
3
0
Fork 0
Code Pull Requests 1 Activity

remove snapshot approach for new integration tests

Browse Source
This commit is contained in:
Jesse Duffield
2022-12-20 22:40:14 +11:00
parent bc4ace8357
commit e3c6738535
656 changed files with 20 additions and 1613 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/integration/README.md
View File

@@ -63,26 +63,22 @@ You can pass the KEY_PRESS_DELAY env var to the test runner in order to set a de
### Running tests in VSCode
If you've opened an integration test file in your editor you can run that file by bringing up the command panel with `cmd+shift+p` and typing 'run task', then selecting the test task you want to run
If you've opened an integration test file in your editor you can run that file by bringing up the command panel with `cmd+shift+p` and typing 'run task', then selecting the test task you want to run
![image](https://user-images.githubusercontent.com/8456633/201500427-b86e129f-5f35-4d55-b7bd-fff5d8e4a04e.png)
![image](https://user-images.githubusercontent.com/8456633/201500431-903deb8c-c210-4054-8514-ab7088c7a839.png)
The test will run in a VSCode terminal:
![image](https://user-images.githubusercontent.com/8456633/201500446-b87abf11-9653-438f-8a9a-e0bf8abdb7ee.png)
### Snapshots
At the moment (this is subject to change) each test has a snapshot repo created after running for the first time. These snapshots live in `test/integration_new`, in folders named 'expected' (alongside the 'actual' folders which contain the resulting repo from the last test run). Whenever you run a test, the resultant repo will be compared against the snapshot repo and if they're different, you'll be asked whether you want to update the snapshot. If you want to update a snapshot without being prompted you can pass MODE=update to the test runner.
### Sandbox mode
Say you want to do a manual test of how lazygit handles merge-conflicts, but you can't be bothered actually finding a way to create merge conflicts in a repo. To make your life easier, you can simply run a merge-conflicts test in sandbox mode, meaning the setup step is run for you, and then instead of the test driving the lazygit session, you're allowed to drive it yourself.
To run a test in sandbox mode you can press 's' on a test in the test TUI or in the test runner pass MODE=sandbox or the --sandbox argument.
To run a test in sandbox mode you can press 's' on a test in the test TUI or in the test runner pass the --sandbox argument.
## Migration process
At the time of writing, most tests are created under an old approach, where you would record yourself in a lazygit session and then the test would replay the keybindings with the same timestamps. This old approach is great for writing tests quickly, but is much harder to maintain. It has to rely entirely on snapshots to determining if a test passes or fails, and can't do assertions along the way. It's also harder to grok what's the intention behind certain actions that take place within the test (e.g. was the recorder intentionally switching to another panel or was that just a misclick?).
At the time of writing, most tests are created under an old approach, where you would record yourself in a lazygit session and then the test would replay the keybindings with the same timestamps. This old approach is great for writing tests quickly, but is much harder to maintain. It has to rely on snapshots to determining if a test passes or fails, and can't do assertions along the way. It's also harder to grok what's the intention behind certain actions that take place within the test (e.g. was the recorder intentionally switching to another panel or was that just a misclick?).
At the moment, all the deprecated test code lives in pkg/integration/deprecated. Hopefully in the very near future we migrate everything across so that we don't need to maintain two systems.
@@ -92,6 +88,6 @@ We should never write any new tests under the old method, and if a given test br
go run pkg/integration/deprecated/cmd/tui/main.go
```
The tests in the old format live in test/integration. In the old format, test definitions are co-located with the snapshots. The setup step is done in a `setup.sh` shell script and the `recording.json` file contains the recorded keypresses to be replayed during the test.
The tests in the old format live in test/integration. In the old format, test definitions are co-located with snapshots. The setup step is done in a `setup.sh` shell script and the `recording.json` file contains the recorded keypresses to be replayed during the test.
If you have rewritten an integration test under the new pattern, be sure to delete the old integration test directory.

25
pkg/integration/clients/cli.go
View File

@@ -29,19 +29,12 @@ func RunCLI(testNames []string, slow bool, sandbox bool) {
keyPressDelay = SLOW_KEY_PRESS_DELAY
}
var mode components.Mode
if sandbox {
mode = components.SANDBOX
} else {
mode = getModeFromEnv()
}
err := components.RunTests(
getTestsToRun(testNames),
log.Printf,
runCmdInTerminal,
runAndPrintFatalError,
mode,
sandbox,
keyPressDelay,
1,
)
@@ -95,22 +88,6 @@ func runCmdInTerminal(cmd *exec.Cmd) error {
return cmd.Run()
}
func getModeFromEnv() components.Mode {
switch os.Getenv("MODE") {
case "", "ask":
return components.ASK_TO_UPDATE_SNAPSHOT
case "check":
return components.CHECK_SNAPSHOT
case "update":
return components.UPDATE_SNAPSHOT
case "sandbox":
return components.SANDBOX
default:
log.Fatalf("unknown test mode: %s, must be one of [ask, check, update, sandbox]", os.Getenv("MODE"))
panic("unreachable")
}
}
func tryConvert(numStr string, defaultVal int) int {
num, err := strconv.Atoi(numStr)
if err != nil {

2
pkg/integration/clients/go_test.go
View File

@@ -46,7 +46,7 @@ func TestIntegration(t *testing.T) {
assert.NoError(t, err)
})
},
components.CHECK_SNAPSHOT,
false,
0,
// allowing two attempts at the test. If a test fails intermittently,
// there may be a concurrency issue that we need to resolve.

14
pkg/integration/clients/tui.go
View File

@@ -82,7 +82,7 @@ func RunTUI() {
return nil
}
suspendAndRunTest(currentTest, components.SANDBOX, 0)
suspendAndRunTest(currentTest, true, 0)
return nil
}); err != nil {
@@ -95,7 +95,7 @@ func RunTUI() {
return nil
}
suspendAndRunTest(currentTest, components.ASK_TO_UPDATE_SNAPSHOT, 0)
suspendAndRunTest(currentTest, false, 0)
return nil
}); err != nil {
@@ -108,7 +108,7 @@ func RunTUI() {
return nil
}
suspendAndRunTest(currentTest, components.ASK_TO_UPDATE_SNAPSHOT, SLOW_KEY_PRESS_DELAY)
suspendAndRunTest(currentTest, false, SLOW_KEY_PRESS_DELAY)
return nil
}); err != nil {
@@ -268,12 +268,12 @@ func (self *app) wrapEditor(f func(v *gocui.View, key gocui.Key, ch rune, mod go
}
}
func suspendAndRunTest(test *components.IntegrationTest, mode components.Mode, keyPressDelay int) {
func suspendAndRunTest(test *components.IntegrationTest, sandbox bool, keyPressDelay int) {
if err := gocui.Screen.Suspend(); err != nil {
panic(err)
}
runTuiTest(test, mode, keyPressDelay)
runTuiTest(test, sandbox, keyPressDelay)
fmt.Fprintf(os.Stdout, "\n%s", style.FgGreen.Sprint("press enter to return"))
fmt.Scanln() // wait for enter press
@@ -367,13 +367,13 @@ func quit(g *gocui.Gui, v *gocui.View) error {
return gocui.ErrQuit
}
func runTuiTest(test *components.IntegrationTest, mode components.Mode, keyPressDelay int) {
func runTuiTest(test *components.IntegrationTest, sandbox bool, keyPressDelay int) {
err := components.RunTests(
[]*components.IntegrationTest{test},
log.Printf,
runCmdInTerminal,
runAndPrintError,
mode,
sandbox,
keyPressDelay,
1,
)

31
pkg/integration/components/runner.go
View File

@@ -18,29 +18,12 @@ const (
SANDBOX_ENV_VAR = "SANDBOX"
)
type Mode int
const (
// Default: if a snapshot test fails, the we'll be asked whether we want to update it
ASK_TO_UPDATE_SNAPSHOT Mode = iota
// fails the test if the snapshots don't match
CHECK_SNAPSHOT
// runs the test and updates the snapshot
UPDATE_SNAPSHOT
// This just makes use of the setup step of the test to get you into
// a lazygit session. Then you'll be able to do whatever you want. Useful
// when you want to test certain things without needing to manually set
// up the situation yourself.
// fails the test if the snapshots don't match
SANDBOX
)
func RunTests(
tests []*IntegrationTest,
logf func(format string, formatArgs ...interface{}),
runCmd func(cmd *exec.Cmd) error,
testWrapper func(test *IntegrationTest, f func() error),
mode Mode,
sandbox bool,
keyPressDelay int,
maxAttempts int,
) error {
@@ -65,7 +48,7 @@ func RunTests(
)
for i := 0; i < maxAttempts; i++ {
err := runTest(test, paths, projectRootDir, logf, runCmd, mode, keyPressDelay)
err := runTest(test, paths, projectRootDir, logf, runCmd, sandbox, keyPressDelay)
if err != nil {
if i == maxAttempts-1 {
return err
@@ -89,7 +72,7 @@ func runTest(
projectRootDir string,
logf func(format string, formatArgs ...interface{}),
runCmd func(cmd *exec.Cmd) error,
mode Mode,
sandbox bool,
keyPressDelay int,
) error {
if test.Skip() {
@@ -103,7 +86,7 @@ func runTest(
return err
}
cmd, err := getLazygitCommand(test, paths, projectRootDir, mode, keyPressDelay)
cmd, err := getLazygitCommand(test, paths, projectRootDir, sandbox, keyPressDelay)
if err != nil {
return err
}
@@ -113,7 +96,7 @@ func runTest(
return err
}
return HandleSnapshots(paths, logf, test, mode)
return nil
}
func prepareTestDir(
@@ -151,7 +134,7 @@ func createFixture(test *IntegrationTest, paths Paths) error {
return nil
}
func getLazygitCommand(test *IntegrationTest, paths Paths, rootDir string, mode Mode, keyPressDelay int) (*exec.Cmd, error) {
func getLazygitCommand(test *IntegrationTest, paths Paths, rootDir string, sandbox bool, keyPressDelay int) (*exec.Cmd, error) {
osCommand := oscommands.NewDummyOSCommand()
templateConfigDir := filepath.Join(rootDir, "test", "default_test_config")
@@ -170,7 +153,7 @@ func getLazygitCommand(test *IntegrationTest, paths Paths, rootDir string, mode
cmdObj := osCommand.Cmd.New(cmdStr)
cmdObj.AddEnvVars(fmt.Sprintf("%s=%s", TEST_NAME_ENV_VAR, test.Name()))
if mode == SANDBOX {
if sandbox {
cmdObj.AddEnvVars(fmt.Sprintf("%s=%s", "SANDBOX", "true"))
}

372
pkg/integration/components/snapshot.go
View File

@@ -1,372 +0,0 @@
package components
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/jesseduffield/generics/slices"
"github.com/jesseduffield/lazygit/pkg/commands/oscommands"
"github.com/stretchr/testify/assert"
)
// This creates and compares integration test snapshots.
type (
logf func(format string, formatArgs ...interface{})
)
func HandleSnapshots(paths Paths, logf logf, test *IntegrationTest, mode Mode) error {
return NewSnapshotter(paths, logf, test, mode).
handleSnapshots()
}
type Snapshotter struct {
paths Paths
logf logf
test *IntegrationTest
mode Mode
}
func NewSnapshotter(
paths Paths,
logf logf,
test *IntegrationTest,
mode Mode,
) *Snapshotter {
return &Snapshotter{
paths: paths,
logf: logf,
test: test,
mode: mode,
}
}
func (self *Snapshotter) handleSnapshots() error {
switch self.mode {
case UPDATE_SNAPSHOT:
return self.handleUpdate()
case CHECK_SNAPSHOT:
return self.handleCheck()
case ASK_TO_UPDATE_SNAPSHOT:
return self.handleAskToUpdate()
case SANDBOX:
self.logf("Sandbox session exited")
}
return nil
}
func (self *Snapshotter) handleUpdate() error {
if err := self.updateSnapshot(); err != nil {
return err
}
self.logf("Test passed: %s", self.test.Name())
return nil
}
func (self *Snapshotter) handleCheck() error {
self.logf("Comparing snapshots")
if err := self.compareSnapshots(); err != nil {
return err
}
self.logf("Test passed: %s", self.test.Name())
return nil
}
func (self *Snapshotter) handleAskToUpdate() error {
if _, err := os.Stat(self.paths.Expected()); os.IsNotExist(err) {
if err := self.updateSnapshot(); err != nil {
return err
}
self.logf("No existing snapshot found for %s. Created snapshot.", self.test.Name())
return nil
}
self.logf("Comparing snapshots...")
if err := self.compareSnapshots(); err != nil {
self.logf("%s", err)
// prompt user whether to update the snapshot (Y/N)
if promptUserToUpdateSnapshot() {
if err := self.updateSnapshot(); err != nil {
return err
}
self.logf("Snapshot updated: %s", self.test.Name())
} else {
return err
}
}
self.logf("Test passed: %s", self.test.Name())
return nil
}
func (self *Snapshotter) updateSnapshot() error {
// create/update snapshot
err := oscommands.CopyDir(self.paths.Actual(), self.paths.Expected())
if err != nil {
return err
}
if err := renameSpecialPaths(self.paths.Expected()); err != nil {
return err
}
return nil
}
func (self *Snapshotter) compareSnapshots() error {
// there are a couple of reasons we're not generating the snapshot in expectedDir directly:
// Firstly we don't want to have to revert our .git file back to .git_keep.
// Secondly, the act of calling git commands like 'git status' actually changes the index
// for some reason, and we don't want to leave your lazygit working tree dirty as a result.
expectedDirCopy := filepath.Join(os.TempDir(), "expected_dir_test", self.test.Name())
err := oscommands.CopyDir(self.paths.Expected(), expectedDirCopy)
if err != nil {
return err
}
defer func() {
err := os.RemoveAll(expectedDirCopy)
if err != nil {
panic(err)
}
}()
if err := restoreSpecialPaths(expectedDirCopy); err != nil {
return err
}
err = validateSameRepos(expectedDirCopy, self.paths.Actual())
if err != nil {
return err
}
// iterate through each repo in the expected dir and comparet to the corresponding repo in the actual dir
expectedFiles, err := ioutil.ReadDir(expectedDirCopy)
if err != nil {
return err
}
for _, f := range expectedFiles {
if !f.IsDir() {
return errors.New("unexpected file (as opposed to directory) in integration test 'expected' directory")
}
// get corresponding file name from actual dir
actualRepoPath := filepath.Join(self.paths.Actual(), f.Name())
expectedRepoPath := filepath.Join(expectedDirCopy, f.Name())
actualRepo, expectedRepo, err := generateSnapshots(actualRepoPath, expectedRepoPath)
if err != nil {
return err
}
if expectedRepo != actualRepo {
// get the log file and print it
bytes, err := os.ReadFile(filepath.Join(self.paths.Config(), "development.log"))
if err != nil {
return err
}
self.logf("%s", string(bytes))
return errors.New(getDiff(f.Name(), expectedRepo, actualRepo))
}
}
return nil
}
func promptUserToUpdateSnapshot() bool {
fmt.Println("Test failed. Update snapshot? (y/n)")
var input string
fmt.Scanln(&input)
return input == "y"
}
func generateSnapshots(actualDir string, expectedDir string) (string, string, error) {
actual, err := generateSnapshot(actualDir)
if err != nil {
return "", "", err
}
expected, err := generateSnapshot(expectedDir)
if err != nil {
return "", "", err
}
return actual, expected, nil
}
// note that we don't actually store this snapshot in the lazygit repo.
// Instead we store the whole expected git repo of our test, so that
// we can easily change what we want to compare without needing to regenerate
// snapshots for each test.
func generateSnapshot(dir string) (string, error) {
osCommand := oscommands.NewDummyOSCommand()
_, err := os.Stat(filepath.Join(dir, ".git"))
if err != nil {
return "git directory not found", nil
}
snapshot := ""
cmdStrs := []string{
`remote show -n origin`, // remote branches
// TODO: find a way to bring this back without breaking tests
// `ls-remote origin`,
`status`, // file tree
`log --pretty=%B|%an|%ae -p -1`, // log
`tag -n`, // tags
`stash list`, // stash
`submodule foreach 'git status'`, // submodule status
`submodule foreach 'git log --pretty=%B -p -1'`, // submodule log
`submodule foreach 'git tag -n'`, // submodule tags
`submodule foreach 'git stash list'`, // submodule stash
}
for _, cmdStr := range cmdStrs {
// ignoring error for now. If there's an error it could be that there are no results
output, _ := osCommand.Cmd.New(fmt.Sprintf("git -C %s %s", dir, cmdStr)).RunWithOutput()
snapshot += fmt.Sprintf("git %s:\n%s\n", cmdStr, output)
}
snapshot += "files in repo:\n"
err = filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
if f.IsDir() {
if f.Name() == ".git" {
return filepath.SkipDir
}
return nil
}
bytes, err := os.ReadFile(path)
if err != nil {
return err
}
relativePath, err := filepath.Rel(dir, path)
if err != nil {
return err
}
snapshot += fmt.Sprintf("path: %s\ncontent:\n%s\n", relativePath, string(bytes))
return nil
})
if err != nil {
return "", err
}
return snapshot, nil
}
func getPathsToRename(dir string, needle string, contains string) []string {
pathsToRename := []string{}
err := filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
if f.Name() == needle && (contains == "" || strings.Contains(path, contains)) {
pathsToRename = append(pathsToRename, path)
}
return nil
})
if err != nil {
panic(err)
}
return pathsToRename
}
var specialPathMappings = []struct{ original, new, contains string }{
// git refuses to track .git or .gitmodules in subdirectories so we need to rename them
{".git", ".git_keep", ""},
{".gitmodules", ".gitmodules_keep", ""},
// we also need git to ignore the contents of our test gitignore files so that
// we actually commit files that are ignored within the test.
{".gitignore", "lg_ignore_file", ""},
// this is the .git/info/exclude file. We're being a little more specific here
// so that we don't accidentally mess with some other file named 'exclude' in the test.
{"exclude", "lg_exclude_file", ".git/info/exclude"},
}
func renameSpecialPaths(dir string) error {
for _, specialPath := range specialPathMappings {
for _, path := range getPathsToRename(dir, specialPath.original, specialPath.contains) {
err := os.Rename(path, filepath.Join(filepath.Dir(path), specialPath.new))
if err != nil {
return err
}
}
}
return nil
}
func restoreSpecialPaths(dir string) error {
for _, specialPath := range specialPathMappings {
for _, path := range getPathsToRename(dir, specialPath.new, specialPath.contains) {
err := os.Rename(path, filepath.Join(filepath.Dir(path), specialPath.original))
if err != nil {
return err
}
}
}
return nil
}
// validates that the actual and expected dirs have the same repo names (doesn't actually check the contents of the repos)
func validateSameRepos(expectedDir string, actualDir string) error {
// iterate through each repo in the expected dir and compare to the corresponding repo in the actual dir
expectedFiles, err := ioutil.ReadDir(expectedDir)
if err != nil {
return err
}
var actualFiles []os.FileInfo
actualFiles, err = ioutil.ReadDir(actualDir)
if err != nil {
return err
}
expectedFileNames := slices.Map(expectedFiles, getFileName)
actualFileNames := slices.Map(actualFiles, getFileName)
if !slices.Equal(expectedFileNames, actualFileNames) {
return fmt.Errorf("expected and actual repo dirs do not match: expected: %s, actual: %s", expectedFileNames, actualFileNames)
}
return nil
}
func getFileName(f os.FileInfo) string {
return f.Name()
}
func getDiff(prefix string, expected string, actual string) string {
mockT := &MockTestingT{}
assert.Equal(mockT, expected, actual, fmt.Sprintf("Unexpected %s. Expected:\n%s\nActual:\n%s\n", prefix, expected, actual))
return mockT.err
}
type MockTestingT struct {
err string
}
func (self *MockTestingT) Errorf(format string, args ...interface{}) {
self.err += fmt.Sprintf(format, args...)
}