get github source id from onboarding api and generate gerrit source id (#84)

Signed-off-by: Ayman <[email protected]>
Co-authored-by: Ayman <[email protected]>

Author: khalifapro

Dockerfile

@@ -12,6 +12,7 @@ ENV GIT_SOURCE_ID='<GIT_SOURCE_ID>'
 ENV GIT_REPOSITORY_SOURCE='<GIT_REPOSITORY_SOURCE>'
 ENV LAST_SYNC='<LAST_SYNC>'
 ENV SPAN='<SPAN>'
+ENV INSIGHTS_SERVICE_URL_V2='<INSIGHTS_SERVICE_URL_V2>'
 RUN apk update && apk add git
 RUN apk add cloc
 RUN apk add --no-cache bash

cmd/git/git.go

@@ -10,7 +10,6 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/LF-Engineering/insights-datasource-shared/aws"
 	"io"
 	"math"
 	"net/url"
@@ -25,8 +24,11 @@ import (
 
 	"github.com/LF-Engineering/insights-datasource-git/build"
 	shared "github.com/LF-Engineering/insights-datasource-shared"
+	"github.com/LF-Engineering/insights-datasource-shared/auth0"
+	"github.com/LF-Engineering/insights-datasource-shared/aws"
 	"github.com/LF-Engineering/insights-datasource-shared/cache"
 	elastic "github.com/LF-Engineering/insights-datasource-shared/elastic"
+	"github.com/LF-Engineering/insights-datasource-shared/http"
 	logger "github.com/LF-Engineering/insights-datasource-shared/ingestjob"
 	"github.com/LF-Engineering/insights-datasource-shared/report"
 	"github.com/LF-Engineering/lfx-event-schema/service"
@@ -596,6 +598,7 @@ type DSGit struct {
 	cacheProvider    cache.Manager
 	endpoint         string
 	reportProvider   *report.Manager
+	auth0Client      *auth0.ClientProvider
 }
 
 // PublisherPushEvents - this is a fake function to test publisher locally
@@ -2526,6 +2529,25 @@ func (j *DSGit) Sync(ctx *shared.Ctx) (err error) {
 	if err != nil {
 		return
 	}
+
+	sourceID := ""
+	if j.RepositorySource == "github" {
+		sourceID, err = j.getGithubRepoSourceId()
+		if err != nil {
+			j.log.WithFields(logrus.Fields{"operation": "sync"}).Error(fmt.Errorf("getGithubRepoSourceId source id: %s, url: %s, source: %s.error:  %+v", j.SourceID, j.URL, j.RepositorySource, err))
+		}
+	}
+
+	if j.RepositorySource == "gerrit" {
+		sourceID, err = j.getGerritRepoSourceId()
+		if err != nil {
+			j.log.WithFields(logrus.Fields{"operation": "sync"}).Error(fmt.Errorf("getGerritRepoSourceId source id: %s, url: %s, source: %s.error:  %+v", j.SourceID, j.URL, j.RepositorySource, err))
+		}
+	}
+
+	if sourceID != "" {
+		j.SourceID = sourceID
+	}
 	// Continue with operations that need git ops
 	nThreads := 0
 	locFinished := false
@@ -2797,7 +2819,9 @@ func main() {
 		git.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("WriteLog Error : %+v", err)
 	}
 	shared.FatalOnError(err)
-
+	if err = git.addAuth0Client(); err != nil {
+		git.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("addAuth0Client Error : %+v", err)
+	}
 	err = git.Sync(&ctx)
 	if err != nil {
 		git.log.WithFields(logrus.Fields{"operation": "main"}).Errorf("Error: %+v", err)
@@ -3100,6 +3124,119 @@ func (j *DSGit) getHead(ctx *shared.Ctx) (string, error) {
 	return commitID, nil
 }
 
+func (j *DSGit) addAuth0Client() error {
+	esCacheClientProvider, err := elastic.NewClientProvider(&elastic.Params{
+		URL: os.Getenv("ES_CACHE_URL"),
+	})
+	if err != nil {
+		return err
+	}
+	httpClientProvider := http.NewClientProvider(time.Second*50, true)
+	authEnv := os.Getenv("STAGE")
+	if authEnv == "dev" {
+		authEnv = "test"
+	}
+	auth0Client, err := auth0.NewAuth0Client(
+		authEnv,
+		os.Getenv("AUTH_GRANT_TYPE"),
+		os.Getenv("AUTH_CLIENT_ID"),
+		os.Getenv("AUTH_CLIENT_SECRET"),
+		os.Getenv("AUTH_AUDIENCE"),
+		os.Getenv("AUTH0_URL"),
+		httpClientProvider,
+		esCacheClientProvider,
+		nil,
+		build.AppName)
+	if err != nil {
+		return err
+	}
+	j.auth0Client = auth0Client
+	return nil
+}
+
+func (j *DSGit) getGithubRepoSourceId() (string, error) {
+	sourceID := ""
+	if j.auth0Client == nil {
+		return "", nil
+	}
+
+	type response struct {
+		SourceID int64 `json:"sourceId"`
+	}
+
+	httpClientProvider := http.NewClientProvider(time.Second*50, true)
+	token, err := j.auth0Client.GetToken()
+	if err != nil {
+		return sourceID, err
+	}
+	headers := make(map[string]string)
+	headers["Authorization"] = fmt.Sprintf("Bearer %s", token)
+	orgRepoList := strings.Split(strings.TrimPrefix(j.URL, "https://github.com/"), "/")
+
+	URL := fmt.Sprintf("%sproject/source/github/organization/%s/repository/%s", os.Getenv("INSIGHTS_SERVICE_URL_V2"), orgRepoList[0], orgRepoList[1])
+	statusCode, res, err := httpClientProvider.Request(URL, "GET", headers, nil, nil)
+	if err != nil {
+		return sourceID, err
+	}
+	if statusCode != 200 {
+		return sourceID, fmt.Errorf("error getting source id status code: %d", statusCode)
+	}
+	var repo response
+	if err = jsoniter.Unmarshal(res, &repo); err != nil {
+		return sourceID, err
+	}
+	if repo.SourceID != 0 {
+		sourceID = strconv.FormatInt(repo.SourceID, 10)
+	}
+	return sourceID, nil
+}
+
+func (j *DSGit) getGerritRepoSourceId() (string, error) {
+	sourceID := ""
+	const (
+		repoSeparatorGerrit = "/gerrit/"
+		repoSeparatorR      = "/r/"
+	)
+	repoSeparatorList := []string{repoSeparatorR, repoSeparatorGerrit}
+	for _, repoSeparator := range repoSeparatorList {
+		repoSlice := strings.Split(j.URL, repoSeparator)
+		if len(repoSlice) > 1 {
+			if repoSeparator == repoSeparatorGerrit {
+				nRepoSlice := strings.Split(j.URL, repoSeparatorR)
+				if len(nRepoSlice) > 1 {
+					sourceID = strings.TrimSpace(repoSlice[1])
+					break
+				}
+			}
+			sourceID = strings.TrimSpace(repoSlice[1])
+			break
+		}
+	}
+
+	// check if url already contains a partial
+	hasPartial := false
+	u, err := url.Parse(j.URL)
+	if err != nil {
+		return sourceID, err
+	}
+
+	for _, p := range repoSeparatorList {
+		repoSlice := strings.Split(u.Path, p)
+		if len(repoSlice) > 1 {
+			hasPartial = true
+			break
+		}
+	}
+
+	if !hasPartial {
+		repoSlice := strings.Split(j.URL, strings.TrimSpace(fmt.Sprintf("%s://%s", u.Scheme, u.Host)))
+		if len(repoSlice) > 1 {
+			sourceID = strings.TrimPrefix(strings.TrimSuffix(strings.TrimSpace(repoSlice[1]), "/"), "/")
+		}
+	}
+	return sourceID, nil
+}
+
 // CommitCache single commit cache schema
 type CommitCache struct {
 	Timestamp      string `json:"timestamp"`

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/elastic/go-elasticsearch/v8 v8.0.0-20201229214741-2366c2514674 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect

go.sum

@@ -54,6 +54,7 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/elastic/go-elasticsearch/v8 v8.0.0-20201229214741-2366c2514674 h1:heH4w5l/KFP4Ry9Xp4+jbRx0Wn+TJD7+HlyoMJE4LvQ=
 github.com/elastic/go-elasticsearch/v8 v8.0.0-20201229214741-2366c2514674/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=