gitea/modules/git/repo_index.go
Brecht Van Lommel b88e5fc72d
Fix slow patch checking with commits that add or remove many files (#31548)
Running git update-index for every individual file is slow, so add and
remove everything with a single git command.

When such a big commit lands in the default branch, it could cause PR
creation and patch checking for all open PRs to be slow, or time out
entirely. For example, a commit that removes 1383 files was measured to
take more than 60 seconds and timed out. With this change checking took
about a second.

This is related to #27967, though this will not help with commits that
change many lines in few files.
2024-07-04 18:57:11 +00:00

160 lines
4.6 KiB
Go

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
)
// ReadTreeToIndex reads a treeish to the index
func (repo *Repository) ReadTreeToIndex(treeish string, indexFilename ...string) error {
objectFormat, err := repo.GetObjectFormat()
if err != nil {
return err
}
if len(treeish) != objectFormat.FullLength() {
res, _, err := NewCommand(repo.Ctx, "rev-parse", "--verify").AddDynamicArguments(treeish).RunStdString(&RunOpts{Dir: repo.Path})
if err != nil {
return err
}
if len(res) > 0 {
treeish = res[:len(res)-1]
}
}
id, err := NewIDFromString(treeish)
if err != nil {
return err
}
return repo.readTreeToIndex(id, indexFilename...)
}
func (repo *Repository) readTreeToIndex(id ObjectID, indexFilename ...string) error {
var env []string
if len(indexFilename) > 0 {
env = append(os.Environ(), "GIT_INDEX_FILE="+indexFilename[0])
}
_, _, err := NewCommand(repo.Ctx, "read-tree").AddDynamicArguments(id.String()).RunStdString(&RunOpts{Dir: repo.Path, Env: env})
if err != nil {
return err
}
return nil
}
// ReadTreeToTemporaryIndex reads a treeish to a temporary index file
func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (filename, tmpDir string, cancel context.CancelFunc, err error) {
tmpDir, err = os.MkdirTemp("", "index")
if err != nil {
return filename, tmpDir, cancel, err
}
filename = filepath.Join(tmpDir, ".tmp-index")
cancel = func() {
err := util.RemoveAll(tmpDir)
if err != nil {
log.Error("failed to remove tmp index file: %v", err)
}
}
err = repo.ReadTreeToIndex(treeish, filename)
if err != nil {
defer cancel()
return "", "", func() {}, err
}
return filename, tmpDir, cancel, err
}
// EmptyIndex empties the index
func (repo *Repository) EmptyIndex() error {
_, _, err := NewCommand(repo.Ctx, "read-tree", "--empty").RunStdString(&RunOpts{Dir: repo.Path})
return err
}
// LsFiles checks if the given filenames are in the index
func (repo *Repository) LsFiles(filenames ...string) ([]string, error) {
cmd := NewCommand(repo.Ctx, "ls-files", "-z").AddDashesAndList(filenames...)
res, _, err := cmd.RunStdBytes(&RunOpts{Dir: repo.Path})
if err != nil {
return nil, err
}
filelist := make([]string, 0, len(filenames))
for _, line := range bytes.Split(res, []byte{'\000'}) {
filelist = append(filelist, string(line))
}
return filelist, err
}
// RemoveFilesFromIndex removes given filenames from the index - it does not check whether they are present.
func (repo *Repository) RemoveFilesFromIndex(filenames ...string) error {
objectFormat, err := repo.GetObjectFormat()
if err != nil {
return err
}
cmd := NewCommand(repo.Ctx, "update-index", "--remove", "-z", "--index-info")
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
buffer := new(bytes.Buffer)
for _, file := range filenames {
if file != "" {
// using format: mode SP type SP sha1 TAB path
buffer.WriteString("0 blob " + objectFormat.EmptyObjectID().String() + "\t" + file + "\000")
}
}
return cmd.Run(&RunOpts{
Dir: repo.Path,
Stdin: bytes.NewReader(buffer.Bytes()),
Stdout: stdout,
Stderr: stderr,
})
}
type IndexObjectInfo struct {
Mode string
Object ObjectID
Filename string
}
// AddObjectsToIndex adds the provided object hashes to the index at the provided filenames
func (repo *Repository) AddObjectsToIndex(objects ...IndexObjectInfo) error {
cmd := NewCommand(repo.Ctx, "update-index", "--add", "--replace", "-z", "--index-info")
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
buffer := new(bytes.Buffer)
for _, object := range objects {
// using format: mode SP type SP sha1 TAB path
buffer.WriteString(object.Mode + " blob " + object.Object.String() + "\t" + object.Filename + "\000")
}
return cmd.Run(&RunOpts{
Dir: repo.Path,
Stdin: bytes.NewReader(buffer.Bytes()),
Stdout: stdout,
Stderr: stderr,
})
}
// AddObjectToIndex adds the provided object hash to the index at the provided filename
func (repo *Repository) AddObjectToIndex(mode string, object ObjectID, filename string) error {
return repo.AddObjectsToIndex(IndexObjectInfo{Mode: mode, Object: object, Filename: filename})
}
// WriteTree writes the current index as a tree to the object db and returns its hash
func (repo *Repository) WriteTree() (*Tree, error) {
stdout, _, runErr := NewCommand(repo.Ctx, "write-tree").RunStdString(&RunOpts{Dir: repo.Path})
if runErr != nil {
return nil, runErr
}
id, err := NewIDFromString(strings.TrimSpace(stdout))
if err != nil {
return nil, err
}
return NewTree(repo, id), nil
}