add report data functionality (#67)

Signed-off-by: Ibrahim Mbaziira <[email protected]>

Signed-off-by: Ibrahim Mbaziira <[email protected]>

Author: aultron

cmd/git/git.go

@@ -27,6 +27,7 @@ import (
 	"github.com/LF-Engineering/insights-datasource-shared/cache"
 	elastic "github.com/LF-Engineering/insights-datasource-shared/elastic"
 	logger "github.com/LF-Engineering/insights-datasource-shared/ingestjob"
+	"github.com/LF-Engineering/insights-datasource-shared/report"
 	"github.com/LF-Engineering/lfx-event-schema/service"
 	"github.com/LF-Engineering/lfx-event-schema/service/insights"
 	"github.com/LF-Engineering/lfx-event-schema/service/insights/git"
@@ -591,6 +592,7 @@ type DSGit struct {
 	log              *logrus.Entry
 	cacheProvider    cache.Manager
 	endpoint         string
+	reportProvider   *report.Manager
 }
 
 // PublisherPushEvents - this is a fake function to test publisher locally
@@ -1819,6 +1821,23 @@ func (j *DSGit) GitEnrichItems(ctx *shared.Ctx, thrN int, items []interface{}, d
 			if err != nil {
 				return
 			}
+
+			// write report data
+			rData := new(ReportData)
+			rData.NewCommits = int64(len(data))
+			rData.URL = j.URL
+			rData.Date = time.Now().UnixNano()
+
+			b, err := jsoniter.Marshal(rData)
+			if err != nil {
+				return
+			}
+
+			err = j.reportProvider.UpdateFileByKey(fmt.Sprintf("%+v-%+v.json", j.endpoint, time.Now().Unix()), b)
+			if err != nil {
+				return
+			}
+
 		}
 	}
 	if final {
@@ -2723,6 +2742,7 @@ func main() {
 	shared.SetLogLoggerError(false)
 	shared.AddLogger(&git.Logger, GitDataSource, logger.Internal, []map[string]string{{"REPO_URL": git.URL, "ProjectSlug": ctx.Project}})
 	git.AddCacheProvider()
+	git.AddReportProvider()
 	if os.Getenv("SPAN") != "" {
 		tracer.Start(tracer.WithGlobalTag("connector", "git"))
 		defer tracer.Stop()
@@ -2788,6 +2808,13 @@ func (j *DSGit) AddCacheProvider() {
 	j.endpoint = strings.ReplaceAll(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(j.URL, "https://"), "git://"), "http://"), "/", "-")
 }
 
+// AddReportProvider - adds report provider
+func (j *DSGit) AddReportProvider() {
+	reportProvider := report.NewManager(os.Getenv("STAGE"))
+	j.reportProvider = reportProvider
+	j.endpoint = strings.ReplaceAll(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(j.URL, "https://"), "git://"), "http://"), "/", "-")
+}
+
 func (j *DSGit) createCacheFile(cache []CommitCache, path string) error {
 	for _, comm := range cache {
 		comm.FileLocation = path
@@ -2901,3 +2928,15 @@ type CommitCache struct {
 	Orphaned       bool   `json:"orphaned"`
 	FromDL         bool   `json:"from_dl"`
 }
+
+// ReportData schema
+type ReportData struct {
+	ID              string `json:"id"`
+	SfdcID          string `json:"sfdc_id"`
+	ProjectName     string `json:"project_name"`
+	URL             string `json:"url"`
+	NewCommits      int64  `json:"new_commits"`
+	Date            int64  `json:"date"`
+	SyncStatus      string `json:"sync_status"`
+	OrphanedCommits int64  `json:"orphaned_commits"`
+}

go.mod

@@ -3,7 +3,7 @@ module github.com/LF-Engineering/insights-datasource-git
 go 1.17
 
 require (
-	github.com/LF-Engineering/insights-datasource-shared v1.5.18-0.20221031155159-425447b99760
+	github.com/LF-Engineering/insights-datasource-shared v1.5.21
 	github.com/LF-Engineering/lfx-event-schema v0.1.33
 	github.com/aws/aws-lambda-go v1.27.1
 	github.com/aws/aws-sdk-go v1.42.25

go.sum

@@ -9,9 +9,12 @@ github.com/DataDog/sketches-go v1.2.1 h1:qTBzWLnZ3kM2kw39ymh6rMcnN+5VULwFs++lEYU
 github.com/DataDog/sketches-go v1.2.1/go.mod h1:1xYmPLY1So10AwxV6MJV0J53XVH+WL9Ad1KetxVivVI=
 github.com/LF-Engineering/insights-datasource-shared v1.5.18-0.20221031155159-425447b99760 h1:mz4lyh+rx+HjDatpc8gc1viGaN2snN8OkGpu2igYTDg=
 github.com/LF-Engineering/insights-datasource-shared v1.5.18-0.20221031155159-425447b99760/go.mod h1:9DmFQbC8nnm1C7k+/tDo3Rmqzzx7AzmhPBlFouXaBZ8=
+github.com/LF-Engineering/insights-datasource-shared v1.5.21 h1:cZHytRoA5pZHsQlpeasV5K2b2jZ66ikylvNvzlBSj9s=
+github.com/LF-Engineering/insights-datasource-shared v1.5.21/go.mod h1:9DmFQbC8nnm1C7k+/tDo3Rmqzzx7AzmhPBlFouXaBZ8=
 github.com/LF-Engineering/lfx-event-schema v0.1.14/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
 github.com/LF-Engineering/lfx-event-schema v0.1.33 h1:5Yk20tPkPbyOKEckcEF87qEQtl1H8l6TyA9wjKi+cl0=
 github.com/LF-Engineering/lfx-event-schema v0.1.33/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
+github.com/LF-Engineering/lfx-event-schema v0.1.34/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=