fix: add flag for data convert to dora

backend/core/models/domainlayer/devops/cicd_deployment.go

@@ -34,6 +34,7 @@ type CICDDeployment struct {
 	TaskDatesInfo
 	DurationSec       *float64
 	QueuedDurationSec *float64
+	SubtaskName       string `gorm:"type:varchar(255)"`
 }
 
 func (CICDDeployment) TableName() string {

backend/core/models/domainlayer/devops/cicd_deployment_commit.go

@@ -41,6 +41,7 @@ type CicdDeploymentCommit struct {
 	RepoId                        string `gorm:"type:varchar(255)"`
 	RepoUrl                       string `gorm:"index;not null"`
 	PrevSuccessDeploymentCommitId string `gorm:"type:varchar(255)"`
+	SubtaskName                   string `gorm:"type:varchar(255)"`
 }
 
 func (cicdDeploymentCommit CicdDeploymentCommit) TableName() string {

backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go

@@ -0,0 +1,63 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addSubtabknameToDeployment)(nil)
+
+type cicdDeployment20240308 struct {
+	SubtaskName string `gorm:"type:varchar(255)"`
+}
+
+func (cicdDeployment20240308) TableName() string {
+	return "cicd_deployments"
+}
+
+type cicdDeploymentCommits20240308 struct {
+	SubtaskName string `gorm:"type:varchar(255)"`
+}
+
+func (cicdDeploymentCommits20240308) TableName() string {
+	return "cicd_deployment_commits"
+}
+
+type addSubtabknameToDeployment struct{}
+
+func (*addSubtabknameToDeployment) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+	if err := db.AutoMigrate(&cicdDeployment20240308{}); err != nil {
+		return err
+	}
+	if err := db.AutoMigrate(&cicdDeploymentCommits20240308{}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (*addSubtabknameToDeployment) Version() uint64 {
+	return 20240308142101
+}
+
+func (*addSubtabknameToDeployment) Name() string {
+	return "add subtaskname to deployment"
+}

backend/core/models/migrationscripts/register.go

@@ -108,5 +108,6 @@ func All() []plugin.MigrationScript {
 		new(addUrgencyToIssues),
 		new(modifyRefsIdLength),
 		new(addOriginalEnvironmentToCicdDeploymentsAndCicdDeploymentCommits),
+		new(addSubtabknameToDeployment),
 	}
 }

backend/plugins/dora/tasks/change_lead_time_calculator.go

@@ -46,11 +46,11 @@ func CalculateChangeLeadTime(taskCtx plugin.SubTaskContext) errors.Error {
 	db := taskCtx.GetDal()
 	logger := taskCtx.GetLogger()
 	data := taskCtx.GetData().(*DoraTaskData)
-	// // Clear previous results from the project
-	// err := db.Exec("DELETE FROM project_pr_metrics WHERE project_name = ? ", data.Options.ProjectName)
-	// if err != nil {
-	// 	return errors.Default.Wrap(err, "error deleting previous project_pr_metrics")
-	// }
+	// Clear previous results from the project
+	err := db.Exec("DELETE FROM project_pr_metrics WHERE project_name = ? ", data.Options.ProjectName)
+	if err != nil {
+		return errors.Default.Wrap(err, "error deleting previous project_pr_metrics")
+	}
 
 	// Get pull requests by repo project_name
 	var clauses = []dal.Clause{

backend/plugins/dora/tasks/deployment_commits_generator.go

@@ -30,6 +30,8 @@ import (
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 )
 
+const DORAGenerateDeploymentCommits = "dora.generateDeploymentCommits"
+
 var DeploymentCommitsGeneratorMeta = plugin.SubTaskMeta{
 	Name:             "generateDeploymentCommits",
 	EntryPoint:       GenerateDeploymentCommits,
@@ -103,32 +105,32 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error {
 	}
 	if data.Options.ScopeId != nil {
 		clauses = append(clauses, dal.Where(`p.cicd_scope_id = ?`, data.Options.ScopeId))
-		// // Clear previous results from the project
-		// deleteSql := `DELETE FROM cicd_deployment_commits WHERE cicd_scope_id = ? ;`
-		// err := db.Exec(deleteSql, data.Options.ScopeId)
-		// if err != nil {
-		// 	return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
-		// }
+		// Clear previous results from the project
+		deleteSql := `DELETE FROM cicd_deployment_commits WHERE cicd_scope_id = ? and subtask_name = ?;`
+		err := db.Exec(deleteSql, data.Options.ScopeId, DORAGenerateDeploymentCommits)
+		if err != nil {
+			return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
+		}
 	} else {
 		clauses = append(clauses,
 			dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"),
 			dal.Where(`pm.project_name = ?`, data.Options.ProjectName),
 		)
-		// // Clear previous results from the project
-		// deleteSql := `DELETE FROM cicd_deployment_commits
-		// 	WHERE cicd_scope_id IN (
-		// 	SELECT cicd_scope_id
-		// 	FROM (
-		// 		SELECT cdc.cicd_scope_id
-		// 		FROM cicd_deployment_commits cdc
-		// 		LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id)
-		// 		WHERE pm.project_name = ?
-		// 	) AS subquery
-		// 	);`
-		// err := db.Exec(deleteSql, data.Options.ProjectName)
-		// if err != nil {
-		// 	return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
-		// }
+		// Clear previous results from the project
+		deleteSql := `DELETE FROM cicd_deployment_commits
+			WHERE cicd_scope_id IN (
+			SELECT cicd_scope_id
+			FROM (
+				SELECT cdc.cicd_scope_id
+				FROM cicd_deployment_commits cdc
+				LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id)
+				WHERE pm.project_name = ?
+			) AS subquery
+			) AND subtask_name = ?;`
+		err := db.Exec(deleteSql, data.Options.ProjectName, DORAGenerateDeploymentCommits)
+		if err != nil {
+			return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits")
+		}
 	}
 	cursor, err := db.Cursor(clauses...)
 	if err != nil {
@@ -198,6 +200,7 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error {
 					domainDeployCommit.Environment = devops.TESTING
 				}
 			}
+			domainDeployCommit.SubtaskName = DORAGenerateDeploymentCommits
 			return []interface{}{domainDeployCommit}, nil
 		},
 	})

backend/plugins/dora/tasks/deployment_generator.go

@@ -28,6 +28,8 @@ import (
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 )
 
+const DORAGenerateDeployment = "dora.generateDeployments"
+
 var DeploymentGeneratorMeta = plugin.SubTaskMeta{
 	Name:             "generateDeployments",
 	EntryPoint:       GenerateDeployment,
@@ -78,32 +80,32 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error {
 		clauses = append(clauses,
 			dal.Where("p.cicd_scope_id = ?", data.Options.ScopeId),
 		)
-		// // Clear previous results from the cicd_scope_id
-		// deleteSql := `DELETE FROM cicd_deployments WHERE cicd_scope_id = ?;`
-		// err := db.Exec(deleteSql, data.Options.ScopeId)
-		// if err != nil {
-		// 	return errors.Default.Wrap(err, "error deleting previous deployments")
-		// }
+		// Clear previous results from the cicd_scope_id
+		deleteSql := `DELETE FROM cicd_deployments WHERE cicd_scope_id = ? and subtask_name = ?;`
+		err := db.Exec(deleteSql, data.Options.ScopeId, DORAGenerateDeployment)
+		if err != nil {
+			return errors.Default.Wrap(err, "error deleting previous deployments")
+		}
 	} else {
 		clauses = append(clauses,
 			dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"),
 			dal.Where("pm.project_name = ?", data.Options.ProjectName),
 		)
-		// // Clear previous results from the project
-		// deleteSql := `DELETE FROM cicd_deployments
-		// 		WHERE cicd_scope_id IN (
-		// 		SELECT cicd_scope_id
-		// 		FROM (
-		// 			SELECT cd.cicd_scope_id
-		// 			FROM cicd_deployments cd
-		// 			LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id)
-		// 			WHERE pm.project_name = ?
-		// 		) AS subquery
-		// 		);`
-		// err := db.Exec(deleteSql, data.Options.ProjectName)
-		// if err != nil {
-		// 	return errors.Default.Wrap(err, "error deleting previous deployments")
-		// }
+		// Clear previous results from the project
+		deleteSql := `DELETE FROM cicd_deployments
+				WHERE cicd_scope_id IN (
+				SELECT cicd_scope_id
+				FROM (
+					SELECT cd.cicd_scope_id
+					FROM cicd_deployments cd
+					LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id)
+					WHERE pm.project_name = ?
+				) AS subquery
+				) AND subtask_name = ?;`
+		err := db.Exec(deleteSql, data.Options.ProjectName, DORAGenerateDeployment)
+		if err != nil {
+			return errors.Default.Wrap(err, "error deleting previous deployments")
+		}
 	}
 
 	cursor, err := db.Cursor(clauses...)
@@ -153,6 +155,7 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error {
 					domainDeployment.Environment = devops.TESTING
 				}
 			}
+			domainDeployment.SubtaskName = DORAGenerateDeployment
 			return []interface{}{domainDeployment}, nil
 		},
 	})

backend/plugins/dora/tasks/incident_deploy_connector.go

@@ -47,11 +47,11 @@ type simpleCicdDeploymentCommit struct {
 func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {
 	db := taskCtx.GetDal()
 	data := taskCtx.GetData().(*DoraTaskData)
-	// // Clear previous results from the project
-	// err := db.Exec("DELETE FROM project_issue_metrics WHERE project_name = ?", data.Options.ProjectName)
-	// if err != nil {
-	// 	return errors.Default.Wrap(err, "error deleting previous project_issue_metrics")
-	// }
+	// Clear previous results from the project
+	err := db.Exec("DELETE FROM project_issue_metrics WHERE project_name = ?", data.Options.ProjectName)
+	if err != nil {
+		return errors.Default.Wrap(err, "error deleting previous project_issue_metrics")
+	}
 	// select all issues belongs to the board
 	clauses := []dal.Clause{
 		dal.From(`issues i`),