chore: Add tracing instrumentation to source code integration (#4607)

This will allow us to understand better what requests a fired and where
they might go wrong.

Author: simonswine

pkg/frontend/vcs/client/github.go

@@ -9,6 +9,7 @@ import (
 
 	"connectrpc.com/connect"
 	"github.com/google/go-github/v58/github"
+	"github.com/opentracing/opentracing-go"
 	"golang.org/x/oauth2"
 
 	vcsv1 "github.com/grafana/pyroscope/api/gen/proto/go/vcs/v1"
@@ -32,6 +33,12 @@ type githubClient struct {
 }
 
 func (gh *githubClient) GetCommit(ctx context.Context, owner, repo, ref string) (*vcsv1.CommitInfo, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "githubClient.GetCommit")
+	defer sp.Finish()
+	sp.SetTag("owner", owner)
+	sp.SetTag("repo", repo)
+	sp.SetTag("ref", ref)
+
 	commit, _, err := gh.repoService.GetCommit(ctx, owner, repo, ref, nil)
 	if err != nil {
 		var githubErr *github.ErrorResponse
@@ -67,6 +74,13 @@ func (gh *githubClient) GetCommit(ctx context.Context, owner, repo, ref string)
 }
 
 func (gh *githubClient) GetFile(ctx context.Context, req FileRequest) (File, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "githubClient.GetFile")
+	defer sp.Finish()
+	sp.SetTag("owner", req.Owner)
+	sp.SetTag("repo", req.Repo)
+	sp.SetTag("path", req.Path)
+	sp.SetTag("ref", req.Ref)
+
 	// We could abstract away git provider using git protocol
 	// git clone https://x-access-token:<token>@github.com/owner/repo.git
 	// For now we use the github client.

pkg/frontend/vcs/commit.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/opentracing/opentracing-go"
 	"golang.org/x/sync/errgroup"
 
 	vcsv1 "github.com/grafana/pyroscope/api/gen/proto/go/vcs/v1"
@@ -23,6 +24,12 @@ type gitHubCommitGetter interface {
 // This function provides partial success behavior, returning any commits
 // that were successfully fetched along with errors for those that failed.
 func getCommits(ctx context.Context, client gitHubCommitGetter, owner, repo string, refs []string) ([]*vcsv1.CommitInfo, []error, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "getCommits")
+	defer sp.Finish()
+	sp.SetTag("owner", owner)
+	sp.SetTag("repo", repo)
+	sp.SetTag("ref_count", len(refs))
+
 	type result struct {
 		commit *vcsv1.CommitInfo
 		err    error
@@ -71,6 +78,12 @@ func getCommits(ctx context.Context, client gitHubCommitGetter, owner, repo stri
 // tryGetCommit attempts to retrieve a commit using different ref formats (commit hash, branch, tag).
 // It tries each format in order and returns the first successful result.
 func tryGetCommit(ctx context.Context, client gitHubCommitGetter, owner, repo, ref string) (*vcsv1.CommitInfo, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "tryGetCommit")
+	defer sp.Finish()
+	sp.SetTag("owner", owner)
+	sp.SetTag("repo", repo)
+	sp.SetTag("ref", ref)
+
 	refFormats := []string{
 		ref,            // Try as a commit hash
 		"heads/" + ref, // Try as a branch

pkg/frontend/vcs/service.go

@@ -11,6 +11,7 @@ import (
 	"github.com/go-kit/log"
 	giturl "github.com/kubescape/go-git-url"
 	"github.com/kubescape/go-git-url/apis"
+	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/oauth2"
 
@@ -37,6 +38,9 @@ func New(logger log.Logger, reg prometheus.Registerer) *Service {
 }
 
 func (q *Service) GithubApp(ctx context.Context, req *connect.Request[vcsv1.GithubAppRequest]) (*connect.Response[vcsv1.GithubAppResponse], error) {
+	sp, _ := opentracing.StartSpanFromContext(ctx, "GithubApp")
+	defer sp.Finish()
+
 	err := isGitHubIntegrationConfigured()
 	if err != nil {
 		q.logger.Log("err", err, "msg", "GitHub integration is not configured")
@@ -50,6 +54,9 @@ func (q *Service) GithubApp(ctx context.Context, req *connect.Request[vcsv1.Gith
 }
 
 func (q *Service) GithubLogin(ctx context.Context, req *connect.Request[vcsv1.GithubLoginRequest]) (*connect.Response[vcsv1.GithubLoginResponse], error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "GithubLogin")
+	defer sp.Finish()
+
 	cfg, err := githubOAuthConfig()
 	if err != nil {
 		q.logger.Log("err", err, "msg", "failed to get GitHub OAuth config")
@@ -90,6 +97,9 @@ func (q *Service) GithubLogin(ctx context.Context, req *connect.Request[vcsv1.Gi
 }
 
 func (q *Service) GithubRefresh(ctx context.Context, req *connect.Request[vcsv1.GithubRefreshRequest]) (*connect.Response[vcsv1.GithubRefreshResponse], error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "GithubRefresh")
+	defer sp.Finish()
+
 	token, err := tokenFromRequest(ctx, req)
 	if err != nil {
 		q.logger.Log("err", err, "msg", "failed to extract token from request")
@@ -138,6 +148,13 @@ func (q *Service) GithubRefresh(ctx context.Context, req *connect.Request[vcsv1.
 }
 
 func (q *Service) GetFile(ctx context.Context, req *connect.Request[vcsv1.GetFileRequest]) (*connect.Response[vcsv1.GetFileResponse], error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "GetFile")
+	defer sp.Finish()
+	sp.SetTag("repository_url", req.Msg.RepositoryURL)
+	sp.SetTag("local_path", req.Msg.LocalPath)
+	sp.SetTag("root_path", req.Msg.RootPath)
+	sp.SetTag("ref", req.Msg.Ref)
+
 	token, err := tokenFromRequest(ctx, req)
 	if err != nil {
 		q.logger.Log("err", err, "msg", "failed to extract token from request")
@@ -184,6 +201,11 @@ func (q *Service) GetFile(ctx context.Context, req *connect.Request[vcsv1.GetFil
 }
 
 func (q *Service) GetCommit(ctx context.Context, req *connect.Request[vcsv1.GetCommitRequest]) (*connect.Response[vcsv1.GetCommitResponse], error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "GetCommit")
+	defer sp.Finish()
+	sp.SetTag("repository_url", req.Msg.RepositoryURL)
+	sp.SetTag("ref", req.Msg.Ref)
+
 	token, err := tokenFromRequest(ctx, req)
 	if err != nil {
 		q.logger.Log("err", err, "msg", "failed to extract token from request")
@@ -228,6 +250,9 @@ func (q *Service) GetCommit(ctx context.Context, req *connect.Request[vcsv1.GetC
 }
 
 func (q *Service) GetCommits(ctx context.Context, req *connect.Request[vcsv1.GetCommitsRequest]) (*connect.Response[vcsv1.GetCommitsResponse], error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "GetCommits")
+	defer sp.Finish()
+
 	token, err := tokenFromRequest(ctx, req)
 	if err != nil {
 		q.logger.Log("err", err, "msg", "failed to extract token from request")

pkg/frontend/vcs/source/find_go.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log/level"
+	"github.com/opentracing/opentracing-go"
 	"golang.org/x/mod/modfile"
 	"golang.org/x/mod/module"
 
@@ -23,6 +24,10 @@ const (
 
 // findGoFile finds a go file in a vcs repository.
 func (ff FileFinder) findGoFile(ctx context.Context) (*vcsv1.GetFileResponse, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "findGoFile")
+	defer sp.Finish()
+	sp.SetTag("path", ff.path)
+
 	if url, ok := golang.StandardLibraryURL(ff.path); ok {
 		return ff.fetchURL(ctx, url, false)
 	}
@@ -50,6 +55,12 @@ func (ff FileFinder) findGoFile(ctx context.Context) (*vcsv1.GetFileResponse, er
 }
 
 func (ff FileFinder) fetchGoMod(ctx context.Context) (*modfile.File, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "fetchGoMod")
+	defer sp.Finish()
+	sp.SetTag("owner", ff.repo.GetOwnerName())
+	sp.SetTag("repo", ff.repo.GetRepoName())
+	sp.SetTag("ref", ff.ref)
+
 	content, err := ff.client.GetFile(ctx, client.FileRequest{
 		Owner: ff.repo.GetOwnerName(),
 		Repo:  ff.repo.GetRepoName(),
@@ -63,6 +74,10 @@ func (ff FileFinder) fetchGoMod(ctx context.Context) (*modfile.File, error) {
 }
 
 func (ff FileFinder) fetchGoDependencyFile(ctx context.Context, module golang.Module) (*vcsv1.GetFileResponse, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "fetchGoDependencyFile")
+	defer sp.Finish()
+	sp.SetTag("module_path", module.Path)
+
 	switch {
 	case module.IsGitHub():
 		return ff.fetchGithubModuleFile(ctx, module)
@@ -73,12 +88,21 @@ func (ff FileFinder) fetchGoDependencyFile(ctx context.Context, module golang.Mo
 }
 
 func (ff FileFinder) fetchGithubModuleFile(ctx context.Context, mod golang.Module) (*vcsv1.GetFileResponse, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "fetchGithubModuleFile")
+	defer sp.Finish()
+	sp.SetTag("module_path", mod.Path)
+
 	// todo: what if this is not a github repo?
 	// 		VSClient should support querying multiple repo providers.
 	githubFile, err := mod.GithubFile()
 	if err != nil {
 		return nil, err
 	}
+	sp.SetTag("owner", githubFile.Owner)
+	sp.SetTag("repo", githubFile.Repo)
+	sp.SetTag("path", githubFile.Path)
+	sp.SetTag("ref", githubFile.Ref)
+
 	content, err := ff.client.GetFile(ctx, client.FileRequest{
 		Owner: githubFile.Owner,
 		Repo:  githubFile.Repo,
@@ -92,10 +116,15 @@ func (ff FileFinder) fetchGithubModuleFile(ctx context.Context, mod golang.Modul
 }
 
 func (ff FileFinder) fetchGoogleSourceDependencyFile(ctx context.Context, mod golang.Module) (*vcsv1.GetFileResponse, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "fetchGoogleSourceDependencyFile")
+	defer sp.Finish()
+	sp.SetTag("module_path", mod.Path)
+
 	url, err := mod.GoogleSourceURL()
 	if err != nil {
 		return nil, err
 	}
+	sp.SetTag("url", url)
 	return ff.fetchURL(ctx, url, true)
 }
 

pkg/frontend/vcs/source/find_go_test.go

@@ -89,14 +89,14 @@ func Test_tryFindGoFile(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			ctxMock := context.Context(nil)
+			ctx := context.Background()
 			sut := FileFinder{
 				path:     tt.searchedPath,
 				rootPath: tt.rootPath,
 				repo:     tt.repo,
 				client:   tt.clientMock,
 			}
-			_, err := sut.tryFindGoFile(ctxMock, tt.attempts)
+			_, err := sut.tryFindGoFile(ctx, tt.attempts)
 			assert.Equal(t, tt.expectedSearchedPaths, (*tt.clientMock).searchedSequence)
 			assert.Equal(t, tt.expectedError, err)
 		})