Push LOC using cloc (#75)

* push LOC using cloc

Signed-off-by: Ayman <[email protected]>

* test changes

Signed-off-by: Ayman <[email protected]>

* code clean up

Signed-off-by: Ayman <[email protected]>

* include endpoint in input file name

Signed-off-by: Ayman <[email protected]>

---------

Signed-off-by: Ayman <[email protected]>
Co-authored-by: Ayman <[email protected]>

Author: khalifapro

cmd/git/git.go

@@ -524,7 +524,7 @@ var (
 
 // Publisher - for streaming data to Kinesis
 type Publisher interface {
-	PushEvents(action, source, eventType, subEventType, env string, data []interface{}) (string, error)
+	PushEvents(action, source, eventType, subEventType, env string, data []interface{}, endpoint string) (string, error)
 }
 
 // RawPLS - programming language summary (all fields as strings)
@@ -946,7 +946,8 @@ func (j *DSGit) EnrichItem(ctx *shared.Ctx, item map[string]interface{}) (rich m
 		err = fmt.Errorf("cannot parse author date from %v", iAuthorDate)
 		return
 	}
-
+	clocCount, _ := shared.Dig(commit, []string{"cloc_count"}, true, false)
+	rich["cloc_count"] = clocCount
 	authorLocation := time.FixedZone(fmt.Sprintf("UTC%v", authorTz), int(authorTz)*60*60)
 	authorLocalDate := time.Date(authorDate.Year(), authorDate.Month(), authorDate.Day(), authorDate.Hour(), authorDate.Minute(), authorDate.Second(), authorDate.Nanosecond(), authorLocation)
 	rich["orphaned"] = false
@@ -1342,6 +1343,10 @@ func (j *DSGit) GetModelData(ctx *shared.Ctx, docs []interface{}) []git.CommitCr
 			for _, value := range fileCache {
 				commit.Files = append(commit.Files, *value)
 			}
+			if len(commit.Files) > 0 {
+				clocCount, _ := doc["cloc_count"].(int)
+				commit.Files[len(commit.Files)-1].ActualLinesOfCode = clocCount
+			}
 		}
 		commit.MergeCommit = len(fileAry) == 0
 		// Event
@@ -1803,7 +1808,7 @@ func (j *DSGit) GitEnrichItems(ctx *shared.Ctx, thrN int, items []interface{}, d
 				}
 				path := ""
 				if len(formattedData) > 0 {
-					path, err = j.Publisher.PushEvents(CommitCreated, "insights", GitDataSource, "commits", os.Getenv("STAGE"), formattedData)
+					path, err = j.Publisher.PushEvents(CommitCreated, "insights", GitDataSource, "commits", os.Getenv("STAGE"), formattedData, j.endpoint)
 					if err != nil {
 						j.log.WithFields(logrus.Fields{"operation": "GitEnrichItems"}).Errorf("Error: %+v", err)
 						return
@@ -2561,6 +2566,20 @@ func (j *DSGit) Sync(ctx *shared.Ctx) (err error) {
 		return
 	}
 	processCommit := func(c chan error, commit map[string]interface{}) (wch chan error, e error) {
+		sha, _ := commit["commit"].(string)
+		cmdLine := []string{"cloc", "commit", sha, "--json"}
+		sout, serr, err := shared.ExecCommand(ctx, cmdLine, j.GitPath, GitDefaultEnv)
+		if err != nil {
+			j.log.WithFields(logrus.Fields{"operation": "Sync"}).Errorf("error executing command: %v, error: %v, output: %s, output error: %s", cmdLine, err, sout, serr)
+			return
+		}
+		r := make(map[string]clocResult)
+		err = jsoniter.Unmarshal([]byte(sout), &r)
+		if err != nil {
+			j.log.WithFields(logrus.Fields{"operation": "Sync"}).Errorf("error unmarshall: %v, error: %v", sout, err)
+			return
+		}
+		commit["cloc_count"] = r["SUM"].Code
 		defer func() {
 			if c != nil {
 				c <- e
@@ -2924,7 +2943,7 @@ func (j *DSGit) handleDataLakeOrphans() {
 	}
 
 	if len(formattedData) > 0 {
-		path, err := j.Publisher.PushEvents(CommitUpdated, "insights", GitDataSource, "commits", os.Getenv("STAGE"), formattedData)
+		path, err := j.Publisher.PushEvents(CommitUpdated, "insights", GitDataSource, "commits", os.Getenv("STAGE"), formattedData, j.endpoint)
 		if err != nil {
 			j.log.WithFields(logrus.Fields{"operation": "handleDataLakeOrphans"}).Errorf("error pushing data lake orphand commits: %+v", err)
 			return
@@ -2969,3 +2988,10 @@ type ReportData struct {
 	SyncStatus      string `json:"sync_status"`
 	OrphanedCommits int64  `json:"orphaned_commits"`
 }
+
+type clocResult struct {
+	Code          int `json:"code"`
+	Blank         int `json:"blank"`
+	Comment       int `json:"comment"`
+	NumberOfFiles int `json:"nFiles"`
+}

go.mod

@@ -4,7 +4,7 @@ go 1.17
 
 require (
 	github.com/LF-Engineering/insights-datasource-shared v1.5.21
-	github.com/LF-Engineering/lfx-event-schema v0.1.35
+	github.com/LF-Engineering/lfx-event-schema v0.1.37
 	github.com/aws/aws-lambda-go v1.27.1
 	github.com/aws/aws-sdk-go v1.42.25
 	github.com/json-iterator/go v1.1.11

go.sum

@@ -10,8 +10,8 @@ github.com/DataDog/sketches-go v1.2.1/go.mod h1:1xYmPLY1So10AwxV6MJV0J53XVH+WL9A
 github.com/LF-Engineering/insights-datasource-shared v1.5.21 h1:cZHytRoA5pZHsQlpeasV5K2b2jZ66ikylvNvzlBSj9s=
 github.com/LF-Engineering/insights-datasource-shared v1.5.21/go.mod h1:9DmFQbC8nnm1C7k+/tDo3Rmqzzx7AzmhPBlFouXaBZ8=
 github.com/LF-Engineering/lfx-event-schema v0.1.14/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
-github.com/LF-Engineering/lfx-event-schema v0.1.35 h1:kvhuZxXS6JRclVJJVYkkXbImaResQ+FeeM5k9sQooSE=
-github.com/LF-Engineering/lfx-event-schema v0.1.35/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
+github.com/LF-Engineering/lfx-event-schema v0.1.37 h1:ny46D2NdCXokvJZ01GJcw2RfQM64ousJjaYsrRj5zzg=
+github.com/LF-Engineering/lfx-event-schema v0.1.37/go.mod h1:CfFIZ4mwzo88umf5+KxDQEzqlVkPG7Vx8eLK2oDfWIs=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=