From ca61c521c297d195411ae657f11ddf1c5be1fe90 Mon Sep 17 00:00:00 2001 From: abeizn Date: Fri, 8 Mar 2024 11:29:31 +0800 Subject: [PATCH 1/3] fix: add flag for data convert to dora --- .../domainlayer/devops/cicd_deployment.go | 1 + .../devops/cicd_deployment_commit.go | 1 + .../20240308_add_subtaskname_to_deployment.go | 63 +++++++++++++++++++ .../core/models/migrationscripts/register.go | 1 + .../dora/tasks/change_lead_time_calculator.go | 10 +-- .../tasks/deployment_commits_generator.go | 43 ++++++------- .../dora/tasks/deployment_generator.go | 43 ++++++------- .../dora/tasks/incident_deploy_connector.go | 10 +-- 8 files changed, 120 insertions(+), 52 deletions(-) create mode 100644 backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go diff --git a/backend/core/models/domainlayer/devops/cicd_deployment.go b/backend/core/models/domainlayer/devops/cicd_deployment.go index 33c22e2bb97..199e95fa24c 100644 --- a/backend/core/models/domainlayer/devops/cicd_deployment.go +++ b/backend/core/models/domainlayer/devops/cicd_deployment.go @@ -34,6 +34,7 @@ type CICDDeployment struct { TaskDatesInfo DurationSec *float64 QueuedDurationSec *float64 + SubtaskName string `gorm:"type:varchar(255)"` } func (CICDDeployment) TableName() string { diff --git a/backend/core/models/domainlayer/devops/cicd_deployment_commit.go b/backend/core/models/domainlayer/devops/cicd_deployment_commit.go index e04bf808552..aa8b9db8d94 100644 --- a/backend/core/models/domainlayer/devops/cicd_deployment_commit.go +++ b/backend/core/models/domainlayer/devops/cicd_deployment_commit.go @@ -41,6 +41,7 @@ type CicdDeploymentCommit struct { RepoId string `gorm:"type:varchar(255)"` RepoUrl string `gorm:"index;not null"` PrevSuccessDeploymentCommitId string `gorm:"type:varchar(255)"` + SubtaskName string `gorm:"type:varchar(255)"` } func (cicdDeploymentCommit CicdDeploymentCommit) TableName() string { diff --git a/backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go b/backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go new file mode 100644 index 00000000000..7bac0c34c45 --- /dev/null +++ b/backend/core/models/migrationscripts/20240308_add_subtaskname_to_deployment.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addSubtabknameToDeployment)(nil) + +type cicdDeployment20240308 struct { + SubtaskName string `gorm:"type:varchar(255)"` +} + +func (cicdDeployment20240308) TableName() string { + return "cicd_deployments" +} + +type cicdDeploymentCommits20240308 struct { + SubtaskName string `gorm:"type:varchar(255)"` +} + +func (cicdDeploymentCommits20240308) TableName() string { + return "cicd_deployment_commits" +} + +type addSubtabknameToDeployment struct{} + +func (*addSubtabknameToDeployment) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.AutoMigrate(&cicdDeployment20240308{}); err != nil { + return err + } + if err := db.AutoMigrate(&cicdDeploymentCommits20240308{}); err != nil { + return err + } + return nil +} + +func (*addSubtabknameToDeployment) Version() uint64 { + return 20240308142101 +} + +func (*addSubtabknameToDeployment) Name() string { + return "add subtaskname to deployment" +} diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go index 53350d9d0a0..03319bf9281 100644 --- a/backend/core/models/migrationscripts/register.go +++ b/backend/core/models/migrationscripts/register.go @@ -108,5 +108,6 @@ func All() []plugin.MigrationScript { new(addUrgencyToIssues), new(modifyRefsIdLength), new(addOriginalEnvironmentToCicdDeploymentsAndCicdDeploymentCommits), + new(addSubtabknameToDeployment), } } diff --git a/backend/plugins/dora/tasks/change_lead_time_calculator.go b/backend/plugins/dora/tasks/change_lead_time_calculator.go index c76b3d54b62..8e29f1678ee 100644 --- a/backend/plugins/dora/tasks/change_lead_time_calculator.go +++ b/backend/plugins/dora/tasks/change_lead_time_calculator.go @@ -46,11 +46,11 @@ func CalculateChangeLeadTime(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() logger := taskCtx.GetLogger() data := taskCtx.GetData().(*DoraTaskData) - // // Clear previous results from the project - // err := db.Exec("DELETE FROM project_pr_metrics WHERE project_name = ? ", data.Options.ProjectName) - // if err != nil { - // return errors.Default.Wrap(err, "error deleting previous project_pr_metrics") - // } + // Clear previous results from the project + err := db.Exec("DELETE FROM project_pr_metrics WHERE project_name = ? ", data.Options.ProjectName) + if err != nil { + return errors.Default.Wrap(err, "error deleting previous project_pr_metrics") + } // Get pull requests by repo project_name var clauses = []dal.Clause{ diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go b/backend/plugins/dora/tasks/deployment_commits_generator.go index e8a64e23927..20cab470e21 100644 --- a/backend/plugins/dora/tasks/deployment_commits_generator.go +++ b/backend/plugins/dora/tasks/deployment_commits_generator.go @@ -103,32 +103,32 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error { } if data.Options.ScopeId != nil { clauses = append(clauses, dal.Where(`p.cicd_scope_id = ?`, data.Options.ScopeId)) - // // Clear previous results from the project - // deleteSql := `DELETE FROM cicd_deployment_commits WHERE cicd_scope_id = ? ;` - // err := db.Exec(deleteSql, data.Options.ScopeId) - // if err != nil { - // return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits") - // } + // Clear previous results from the project + deleteSql := `DELETE FROM cicd_deployment_commits WHERE cicd_scope_id = ? and subtask_name = ?;` + err := db.Exec(deleteSql, data.Options.ScopeId, "dora.generateDeploymentCommits") + if err != nil { + return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits") + } } else { clauses = append(clauses, dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"), dal.Where(`pm.project_name = ?`, data.Options.ProjectName), ) - // // Clear previous results from the project - // deleteSql := `DELETE FROM cicd_deployment_commits - // WHERE cicd_scope_id IN ( - // SELECT cicd_scope_id - // FROM ( - // SELECT cdc.cicd_scope_id - // FROM cicd_deployment_commits cdc - // LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id) - // WHERE pm.project_name = ? - // ) AS subquery - // );` - // err := db.Exec(deleteSql, data.Options.ProjectName) - // if err != nil { - // return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits") - // } + // Clear previous results from the project + deleteSql := `DELETE FROM cicd_deployment_commits + WHERE cicd_scope_id IN ( + SELECT cicd_scope_id + FROM ( + SELECT cdc.cicd_scope_id + FROM cicd_deployment_commits cdc + LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id) + WHERE pm.project_name = ? and cdc.subtask_name = ? + ) AS subquery + );` + err := db.Exec(deleteSql, data.Options.ProjectName, "dora.generateDeploymentCommits") + if err != nil { + return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits") + } } cursor, err := db.Cursor(clauses...) if err != nil { @@ -198,6 +198,7 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error { domainDeployCommit.Environment = devops.TESTING } } + domainDeployCommit.SubtaskName = "dora.generateDeploymentCommits" return []interface{}{domainDeployCommit}, nil }, }) diff --git a/backend/plugins/dora/tasks/deployment_generator.go b/backend/plugins/dora/tasks/deployment_generator.go index cca452d6b1c..935a3d026b9 100644 --- a/backend/plugins/dora/tasks/deployment_generator.go +++ b/backend/plugins/dora/tasks/deployment_generator.go @@ -78,32 +78,32 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error { clauses = append(clauses, dal.Where("p.cicd_scope_id = ?", data.Options.ScopeId), ) - // // Clear previous results from the cicd_scope_id - // deleteSql := `DELETE FROM cicd_deployments WHERE cicd_scope_id = ?;` - // err := db.Exec(deleteSql, data.Options.ScopeId) - // if err != nil { - // return errors.Default.Wrap(err, "error deleting previous deployments") - // } + // Clear previous results from the cicd_scope_id + deleteSql := `DELETE FROM cicd_deployments WHERE cicd_scope_id = ? and subtask_name = ?;` + err := db.Exec(deleteSql, data.Options.ScopeId, "dora.generateDeployments") + if err != nil { + return errors.Default.Wrap(err, "error deleting previous deployments") + } } else { clauses = append(clauses, dal.Join("LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = p.cicd_scope_id)"), dal.Where("pm.project_name = ?", data.Options.ProjectName), ) - // // Clear previous results from the project - // deleteSql := `DELETE FROM cicd_deployments - // WHERE cicd_scope_id IN ( - // SELECT cicd_scope_id - // FROM ( - // SELECT cd.cicd_scope_id - // FROM cicd_deployments cd - // LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id) - // WHERE pm.project_name = ? - // ) AS subquery - // );` - // err := db.Exec(deleteSql, data.Options.ProjectName) - // if err != nil { - // return errors.Default.Wrap(err, "error deleting previous deployments") - // } + // Clear previous results from the project + deleteSql := `DELETE FROM cicd_deployments + WHERE cicd_scope_id IN ( + SELECT cicd_scope_id + FROM ( + SELECT cd.cicd_scope_id + FROM cicd_deployments cd + LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id) + WHERE pm.project_name = ? and cd.subtask_name = ? + ) AS subquery + );` + err := db.Exec(deleteSql, data.Options.ProjectName, "dora.generateDeployments") + if err != nil { + return errors.Default.Wrap(err, "error deleting previous deployments") + } } cursor, err := db.Cursor(clauses...) @@ -153,6 +153,7 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error { domainDeployment.Environment = devops.TESTING } } + domainDeployment.SubtaskName = "dora.generateDeployments" return []interface{}{domainDeployment}, nil }, }) diff --git a/backend/plugins/dora/tasks/incident_deploy_connector.go b/backend/plugins/dora/tasks/incident_deploy_connector.go index 2ec9f04cbe0..6d9a1365d63 100644 --- a/backend/plugins/dora/tasks/incident_deploy_connector.go +++ b/backend/plugins/dora/tasks/incident_deploy_connector.go @@ -47,11 +47,11 @@ type simpleCicdDeploymentCommit struct { func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() data := taskCtx.GetData().(*DoraTaskData) - // // Clear previous results from the project - // err := db.Exec("DELETE FROM project_issue_metrics WHERE project_name = ?", data.Options.ProjectName) - // if err != nil { - // return errors.Default.Wrap(err, "error deleting previous project_issue_metrics") - // } + // Clear previous results from the project + err := db.Exec("DELETE FROM project_issue_metrics WHERE project_name = ?", data.Options.ProjectName) + if err != nil { + return errors.Default.Wrap(err, "error deleting previous project_issue_metrics") + } // select all issues belongs to the board clauses := []dal.Clause{ dal.From(`issues i`), From f391a16d73b9e38530b4e73c92257ecc0f970d60 Mon Sep 17 00:00:00 2001 From: abeizn Date: Fri, 8 Mar 2024 11:41:11 +0800 Subject: [PATCH 2/3] fix: some minor optimizations --- .../plugins/dora/tasks/deployment_commits_generator.go | 8 +++++--- backend/plugins/dora/tasks/deployment_generator.go | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go b/backend/plugins/dora/tasks/deployment_commits_generator.go index 20cab470e21..70100731ffb 100644 --- a/backend/plugins/dora/tasks/deployment_commits_generator.go +++ b/backend/plugins/dora/tasks/deployment_commits_generator.go @@ -30,6 +30,8 @@ import ( "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) +const DORAGenerateDeploymentCommits = "dora.generateDeploymentCommits" + var DeploymentCommitsGeneratorMeta = plugin.SubTaskMeta{ Name: "generateDeploymentCommits", EntryPoint: GenerateDeploymentCommits, @@ -105,7 +107,7 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error { clauses = append(clauses, dal.Where(`p.cicd_scope_id = ?`, data.Options.ScopeId)) // Clear previous results from the project deleteSql := `DELETE FROM cicd_deployment_commits WHERE cicd_scope_id = ? and subtask_name = ?;` - err := db.Exec(deleteSql, data.Options.ScopeId, "dora.generateDeploymentCommits") + err := db.Exec(deleteSql, data.Options.ScopeId, DORAGenerateDeploymentCommits) if err != nil { return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits") } @@ -125,7 +127,7 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error { WHERE pm.project_name = ? and cdc.subtask_name = ? ) AS subquery );` - err := db.Exec(deleteSql, data.Options.ProjectName, "dora.generateDeploymentCommits") + err := db.Exec(deleteSql, data.Options.ProjectName, DORAGenerateDeploymentCommits) if err != nil { return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits") } @@ -198,7 +200,7 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error { domainDeployCommit.Environment = devops.TESTING } } - domainDeployCommit.SubtaskName = "dora.generateDeploymentCommits" + domainDeployCommit.SubtaskName = DORAGenerateDeploymentCommits return []interface{}{domainDeployCommit}, nil }, }) diff --git a/backend/plugins/dora/tasks/deployment_generator.go b/backend/plugins/dora/tasks/deployment_generator.go index 935a3d026b9..6cb6d770b66 100644 --- a/backend/plugins/dora/tasks/deployment_generator.go +++ b/backend/plugins/dora/tasks/deployment_generator.go @@ -28,6 +28,8 @@ import ( "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) +const DORAGenerateDeployment = "dora.generateDeployments" + var DeploymentGeneratorMeta = plugin.SubTaskMeta{ Name: "generateDeployments", EntryPoint: GenerateDeployment, @@ -80,7 +82,7 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error { ) // Clear previous results from the cicd_scope_id deleteSql := `DELETE FROM cicd_deployments WHERE cicd_scope_id = ? and subtask_name = ?;` - err := db.Exec(deleteSql, data.Options.ScopeId, "dora.generateDeployments") + err := db.Exec(deleteSql, data.Options.ScopeId, DORAGenerateDeployment) if err != nil { return errors.Default.Wrap(err, "error deleting previous deployments") } @@ -100,7 +102,7 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error { WHERE pm.project_name = ? and cd.subtask_name = ? ) AS subquery );` - err := db.Exec(deleteSql, data.Options.ProjectName, "dora.generateDeployments") + err := db.Exec(deleteSql, data.Options.ProjectName, DORAGenerateDeployment) if err != nil { return errors.Default.Wrap(err, "error deleting previous deployments") } @@ -153,7 +155,7 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error { domainDeployment.Environment = devops.TESTING } } - domainDeployment.SubtaskName = "dora.generateDeployments" + domainDeployment.SubtaskName = DORAGenerateDeployment return []interface{}{domainDeployment}, nil }, }) From 7921a453b039a913c6c2e8c087c1f688943bcc18 Mon Sep 17 00:00:00 2001 From: abeizn Date: Fri, 8 Mar 2024 14:30:47 +0800 Subject: [PATCH 3/3] fix: add subtask_name filter --- backend/plugins/dora/tasks/deployment_commits_generator.go | 4 ++-- backend/plugins/dora/tasks/deployment_generator.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go b/backend/plugins/dora/tasks/deployment_commits_generator.go index 70100731ffb..e298469971c 100644 --- a/backend/plugins/dora/tasks/deployment_commits_generator.go +++ b/backend/plugins/dora/tasks/deployment_commits_generator.go @@ -124,9 +124,9 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error { SELECT cdc.cicd_scope_id FROM cicd_deployment_commits cdc LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cdc.cicd_scope_id) - WHERE pm.project_name = ? and cdc.subtask_name = ? + WHERE pm.project_name = ? ) AS subquery - );` + ) AND subtask_name = ?;` err := db.Exec(deleteSql, data.Options.ProjectName, DORAGenerateDeploymentCommits) if err != nil { return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits") diff --git a/backend/plugins/dora/tasks/deployment_generator.go b/backend/plugins/dora/tasks/deployment_generator.go index 6cb6d770b66..08102fb4211 100644 --- a/backend/plugins/dora/tasks/deployment_generator.go +++ b/backend/plugins/dora/tasks/deployment_generator.go @@ -99,9 +99,9 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error { SELECT cd.cicd_scope_id FROM cicd_deployments cd LEFT JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = cd.cicd_scope_id) - WHERE pm.project_name = ? and cd.subtask_name = ? + WHERE pm.project_name = ? ) AS subquery - );` + ) AND subtask_name = ?;` err := db.Exec(deleteSql, data.Options.ProjectName, DORAGenerateDeployment) if err != nil { return errors.Default.Wrap(err, "error deleting previous deployments")