add lfs_oid and lfs_size

Author: wxiaoguang

modules/git/blob.go

@@ -9,6 +9,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"io"
+	"strings"
 
 	"code.gitea.io/gitea/modules/typesniffer"
 	"code.gitea.io/gitea/modules/util"
@@ -63,33 +64,37 @@ func (b *Blob) GetBlobLineCount(w io.Writer) (int, error) {
 	}
 }
 
-// GetBlobContentBase64 Reads the content of the blob with a base64 encode and returns the encoded string
-func (b *Blob) GetBlobContentBase64() (string, error) {
+// GetBlobContentBase64 Reads the content of the blob with a base64 encoding and returns the encoded string
+func (b *Blob) GetBlobContentBase64(originContent *strings.Builder) (string, error) {
 	dataRc, err := b.DataAsync()
 	if err != nil {
 		return "", err
 	}
 	defer dataRc.Close()
 
-	pr, pw := io.Pipe()
-	encoder := base64.NewEncoder(base64.StdEncoding, pw)
-
-	go func() {
-		_, err := io.Copy(encoder, dataRc)
-		_ = encoder.Close()
-
-		if err != nil {
-			_ = pw.CloseWithError(err)
-		} else {
-			_ = pw.Close()
+	base64buf := &strings.Builder{}
+	encoder := base64.NewEncoder(base64.StdEncoding, base64buf)
+	buf := make([]byte, 32*1024)
+loop:
+	for {
+		n, err := dataRc.Read(buf)
+		if n > 0 {
+			if originContent != nil {
+				_, _ = originContent.Write(buf[:n])
+			}
+			if _, err := encoder.Write(buf[:n]); err != nil {
+				return "", err
+			}
+		}
+		switch {
+		case errors.Is(err, io.EOF):
+			break loop
+		case err != nil:
+			return "", err
 		}
-	}()
-
-	out, err := io.ReadAll(pr)
-	if err != nil {
-		return "", err
 	}
-	return string(out), nil
+	_ = encoder.Close()
+	return base64buf.String(), nil
 }
 
 // GuessContentType guesses the content type of the blob.

modules/lfs/pointer.go

@@ -15,15 +15,13 @@ import (
 	"strings"
 )
 
+// spec: https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
 const (
-	blobSizeCutoff = 1024
+	MetaFileMaxSize = 1024 // spec says the maximum size of a pointer file must be smaller than 1024
 
-	// MetaFileIdentifier is the string appearing at the first line of LFS pointer files.
-	// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
-	MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
+	MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" // the first line of a pointer file
 
-	// MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
-	MetaFileOidPrefix = "oid sha256:"
+	MetaFileOidPrefix = "oid sha256:" // spec says the only supported hash is sha256 at the moment
 )
 
 var (
@@ -39,7 +37,7 @@ var (
 
 // ReadPointer tries to read LFS pointer data from the reader
 func ReadPointer(reader io.Reader) (Pointer, error) {
-	buf := make([]byte, blobSizeCutoff)
+	buf := make([]byte, MetaFileMaxSize)
 	n, err := io.ReadFull(reader, buf)
 	if err != nil && err != io.ErrUnexpectedEOF {
 		return Pointer{}, err
@@ -65,6 +63,7 @@ func ReadPointerFromBuffer(buf []byte) (Pointer, error) {
 		return p, ErrInvalidStructure
 	}
 
+	// spec says "key/value pairs MUST be sorted alphabetically in ascending order (version is exception and must be the first)"
 	oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix)
 	if len(oid) != 64 || !oidPattern.MatchString(oid) {
 		return p, ErrInvalidOIDFormat

modules/lfs/pointer_scanner_gogit.go

@@ -31,7 +31,7 @@ func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan c
 			default:
 			}
 
-			if blob.Size > blobSizeCutoff {
+			if blob.Size > MetaFileMaxSize {
 				return nil
 			}
 

modules/structs/git_blob.go

@@ -10,4 +10,7 @@ type GitBlobResponse struct {
 	URL      string  `json:"url"`
 	SHA      string  `json:"sha"`
 	Size     int64   `json:"size"`
+
+	LfsOid  *string `json:"lfs_oid,omitempty"`
+	LfsSize *int64  `json:"lfs_size,omitempty"`
 }

modules/structs/repo_file.go

@@ -150,6 +150,9 @@ type ContentsResponse struct {
 	// `submodule_git_url` is populated when `type` is `submodule`, otherwise null
 	SubmoduleGitURL *string            `json:"submodule_git_url"`
 	Links           *FileLinksResponse `json:"_links"`
+
+	LfsOid  *string `json:"lfs_oid"`
+	LfsSize *int64  `json:"lfs_size"`
 }
 
 // FileCommitResponse contains information generated from a Git commit for a repo's file.

routers/api/v1/repo/blob.go

@@ -47,7 +47,7 @@ func GetBlob(ctx *context.APIContext) {
 		return
 	}
 
-	if blob, err := files_service.GetBlobBySHA(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, sha); err != nil {
+	if blob, err := files_service.GetBlobBySHA(ctx.Repo.Repository, ctx.Repo.GitRepo, sha); err != nil {
 		ctx.APIError(http.StatusBadRequest, err)
 	} else {
 		ctx.JSON(http.StatusOK, blob)

routers/api/v1/repo/file.go

@@ -908,8 +908,9 @@ func resolveRefCommit(ctx *context.APIContext, ref string, minCommitIDLen ...int
 func GetContentsExt(ctx *context.APIContext) {
 	// swagger:operation GET /repos/{owner}/{repo}/contents-ext/{filepath} repository repoGetContentsExt
 	// ---
-	// summary: The extended "contents" API, get file metadata and/or content, or list a directory.
+	// summary: The extended "contents" API, to get file metadata and/or content, or list a directory.
 	// description: It guarantees that only one of the response fields is set if the request succeeds.
+	//              Users can pass "includes=file_content" or "includes=lfs_metadata" to retrieve more fields.
 	// produces:
 	// - application/json
 	// parameters:
@@ -930,12 +931,13 @@ func GetContentsExt(ctx *context.APIContext) {
 	//   required: true
 	// - name: ref
 	//   in: query
-	//   description: "The name of the commit/branch/tag. Default to the repository’s default branch."
+	//   description: the name of the commit/branch/tag, default to the repository’s default branch.
 	//   type: string
 	//   required: false
 	// - name: includes
 	//   in: query
-	//   description: Set it to "file_content" to retrieve the file content when requesting a file, otherwise the response only contains the file's metadata.
+	//   description: By default this API's response only contains file's metadata. Use comma-separated "includes" options to retrieve more fields.
+	//                Option "file_content" will try to retrieve the file content, option "lfs_metadata" will try to retrieve LFS metadata.
 	//   type: string
 	//   required: false
 	// responses:
@@ -944,7 +946,6 @@ func GetContentsExt(ctx *context.APIContext) {
 	//   "404":
 	//     "$ref": "#/responses/notFound"
 
-	// TODO: add more includes options, like "lfs_content"
 	opts := files_service.GetContentsOrListOptions{TreePath: ctx.PathParam("*")}
 	for includeOpt := range strings.SplitSeq(ctx.FormString("includes"), ",") {
 		if includeOpt == "" {
@@ -953,6 +954,8 @@ func GetContentsExt(ctx *context.APIContext) {
 		switch includeOpt {
 		case "file_content":
 			opts.IncludeSingleFileContent = true
+		case "lfs_metadata":
+			opts.IncludeLfsMetadata = true
 		default:
 			ctx.APIError(http.StatusBadRequest, fmt.Sprintf("unknown include option %q", includeOpt))
 			return
@@ -965,7 +968,8 @@ func GetContentsExt(ctx *context.APIContext) {
 func GetContents(ctx *context.APIContext) {
 	// swagger:operation GET /repos/{owner}/{repo}/contents/{filepath} repository repoGetContents
 	// ---
-	// summary: Gets the metadata and contents (if a file) of an entry in a repository, or a list of entries if a dir
+	// summary: Gets the metadata and contents (if a file) of an entry in a repository, or a list of entries if a dir.
+	// description: This API follows GitHub's design, and it is not easy to use. Recommend to use our "contents-ext" API instead.
 	// produces:
 	// - application/json
 	// parameters:
@@ -1021,7 +1025,8 @@ func getRepoContents(ctx *context.APIContext, opts files_service.GetContentsOrLi
 func GetContentsList(ctx *context.APIContext) {
 	// swagger:operation GET /repos/{owner}/{repo}/contents repository repoGetContentsList
 	// ---
-	// summary: Gets the metadata of all the entries of the root dir
+	// summary: Gets the metadata of all the entries of the root dir.
+	// description: This API follows GitHub's design, and it is not easy to use. Recommend to use our "contents-ext" API instead.
 	// produces:
 	// - application/json
 	// parameters:

routers/api/v1/repo/wiki.go

@@ -499,7 +499,7 @@ func wikiContentsByEntry(ctx *context.APIContext, entry *git.TreeEntry) string {
 	if blob.Size() > setting.API.DefaultMaxBlobSize {
 		return ""
 	}
-	content, err := blob.GetBlobContentBase64()
+	content, err := blob.GetBlobContentBase64(nil)
 	if err != nil {
 		ctx.APIErrorInternal(err)
 		return ""

services/repository/files/content.go

@@ -5,11 +5,14 @@ package files
 
 import (
 	"context"
+	"io"
 	"net/url"
 	"path"
+	"strings"
 
 	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/setting"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
@@ -35,6 +38,7 @@ func (ct *ContentType) String() string {
 type GetContentsOrListOptions struct {
 	TreePath                 string
 	IncludeSingleFileContent bool // include the file's content when the tree path is a file
+	IncludeLfsMetadata       bool
 }
 
 // GetContentsOrList gets the metadata of a file's contents (*ContentsResponse) if treePath not a tree
@@ -65,9 +69,10 @@ func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, gitRepo
 	}
 	ret.DirContents = make([]*api.ContentsResponse, 0, len(entries))
 	for _, e := range entries {
-		// never include file content when listing a directory
-		subTreePath := path.Join(opts.TreePath, e.Name())
-		fileContentResponse, err := GetFileContents(ctx, repo, gitRepo, refCommit, GetContentsOrListOptions{TreePath: subTreePath, IncludeSingleFileContent: false})
+		subOpts := opts
+		subOpts.TreePath = path.Join(opts.TreePath, e.Name())
+		subOpts.IncludeSingleFileContent = false // never include file content when listing a directory
+		fileContentResponse, err := GetFileContents(ctx, repo, gitRepo, refCommit, subOpts)
 		if err != nil {
 			return ret, err
 		}
@@ -118,7 +123,7 @@ func GetFileContents(ctx context.Context, repo *repo_model.Repository, gitRepo *
 	return getFileContentsByEntryInternal(ctx, repo, gitRepo, refCommit, entry, opts)
 }
 
-func getFileContentsByEntryInternal(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, entry *git.TreeEntry, opts GetContentsOrListOptions) (*api.ContentsResponse, error) {
+func getFileContentsByEntryInternal(_ context.Context, repo *repo_model.Repository, gitRepo *git.Repository, refCommit *utils.RefCommit, entry *git.TreeEntry, opts GetContentsOrListOptions) (*api.ContentsResponse, error) {
 	refType := refCommit.RefName.RefType()
 	commit := refCommit.Commit
 	selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(opts.TreePath) + "?ref=" + url.QueryEscape(refCommit.InputRef))
@@ -164,12 +169,17 @@ func getFileContentsByEntryInternal(ctx context.Context, repo *repo_model.Reposi
 		contentsResponse.Type = string(ContentTypeRegular)
 		// if it is listing the repo root dir, don't waste system resources on reading content
 		if opts.IncludeSingleFileContent {
-			blobResponse, err := GetBlobBySHA(ctx, repo, gitRepo, entry.ID.String())
+			blobResponse, err := GetBlobBySHA(repo, gitRepo, entry.ID.String())
+			if err != nil {
+				return nil, err
+			}
+			contentsResponse.Encoding, contentsResponse.Content = blobResponse.Encoding, blobResponse.Content
+			contentsResponse.LfsOid, contentsResponse.LfsSize = blobResponse.LfsOid, blobResponse.LfsSize
+		} else if opts.IncludeLfsMetadata {
+			contentsResponse.LfsOid, contentsResponse.LfsSize, err = parsePossibleLfsPointerBlob(gitRepo, entry.ID.String())
 			if err != nil {
 				return nil, err
 			}
-			contentsResponse.Encoding = blobResponse.Encoding
-			contentsResponse.Content = blobResponse.Content
 		}
 	} else if entry.IsDir() {
 		contentsResponse.Type = string(ContentTypeDir)
@@ -221,8 +231,7 @@ func getFileContentsByEntryInternal(ctx context.Context, repo *repo_model.Reposi
 	return contentsResponse, nil
 }
 
-// GetBlobBySHA get the GitBlobResponse of a repository using a sha hash.
-func GetBlobBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) {
+func GetBlobBySHA(repo *repo_model.Repository, gitRepo *git.Repository, sha string) (*api.GitBlobResponse, error) {
 	gitBlob, err := gitRepo.GetBlob(sha)
 	if err != nil {
 		return nil, err
@@ -232,12 +241,49 @@ func GetBlobBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git
 		URL:  repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()),
 		Size: gitBlob.Size(),
 	}
-	if gitBlob.Size() <= setting.API.DefaultMaxBlobSize {
-		content, err := gitBlob.GetBlobContentBase64()
-		if err != nil {
-			return nil, err
-		}
-		ret.Encoding, ret.Content = util.ToPointer("base64"), &content
+
+	blobSize := gitBlob.Size()
+	if blobSize > setting.API.DefaultMaxBlobSize {
+		return ret, nil
+	}
+
+	var originContent *strings.Builder
+	if 0 < blobSize && blobSize < lfs.MetaFileMaxSize {
+		originContent = &strings.Builder{}
+	}
+
+	content, err := gitBlob.GetBlobContentBase64(originContent)
+	if err != nil {
+		return nil, err
+	}
+
+	ret.Encoding, ret.Content = util.ToPointer("base64"), &content
+	if originContent != nil {
+		ret.LfsOid, ret.LfsSize = parsePossibleLfsPointerBuffer(strings.NewReader(originContent.String()))
 	}
 	return ret, nil
 }
+
+func parsePossibleLfsPointerBuffer(r io.Reader) (*string, *int64) {
+	p, _ := lfs.ReadPointer(r)
+	if p.IsValid() {
+		return &p.Oid, &p.Size
+	}
+	return nil, nil
+}
+
+func parsePossibleLfsPointerBlob(gitRepo *git.Repository, sha string) (*string, *int64, error) {
+	gitBlob, err := gitRepo.GetBlob(sha)
+	if err != nil {
+		return nil, nil, err
+	}
+	if gitBlob.Size() > lfs.MetaFileMaxSize {
+		return nil, nil, nil // not a LFS pointer
+	}
+	buf, err := gitBlob.GetBlobContent(lfs.MetaFileMaxSize)
+	if err != nil {
+		return nil, nil, err
+	}
+	oid, size := parsePossibleLfsPointerBuffer(strings.NewReader(buf))
+	return oid, size, nil
+}

services/repository/files/content_test.go

@@ -107,7 +107,7 @@ func TestGetContents(t *testing.T) {
 		sha := "65f1bf27bc3bf70f64657658635e66094edbcb4d"
 		ctx.SetPathParam("id", "1")
 		ctx.SetPathParam("sha", sha)
-		gbr, err := GetBlobBySHA(ctx, ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("sha"))
+		gbr, err := GetBlobBySHA(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("sha"))
 		expectedGBR := &api.GitBlobResponse{
 			Content:  util.ToPointer("dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK"),
 			Encoding: util.ToPointer("base64"),