From 6de7b904257fd858f4cb735b86b0a60f823908a5 Mon Sep 17 00:00:00 2001 From: abeizn Date: Mon, 4 Mar 2024 18:01:30 +0800 Subject: [PATCH] fix: gitlab delate data by raw data table --- .../plugins/gitlab/e2e/pipelines_detail_test.go | 2 +- .../plugins/gitlab/tasks/mr_comment_convertor.go | 5 +++-- .../gitlab/tasks/pipeline_commit_convertor.go | 2 +- ...e_convertor.go => pipeline_detail_convertor.go} | 14 +++++++------- 4 files changed, 12 insertions(+), 11 deletions(-) rename backend/plugins/gitlab/tasks/{pipeline_convertor.go => pipeline_detail_convertor.go} (90%) diff --git a/backend/plugins/gitlab/e2e/pipelines_detail_test.go b/backend/plugins/gitlab/e2e/pipelines_detail_test.go index 1fa5b02d7e9..33c8b2862c9 100644 --- a/backend/plugins/gitlab/e2e/pipelines_detail_test.go +++ b/backend/plugins/gitlab/e2e/pipelines_detail_test.go @@ -81,7 +81,7 @@ func TestGitlabPipelineDetailDataFlow(t *testing.T) { dataflowTester.ImportCsvIntoTabler("./raw_tables/_tool_gitlab_projects.csv", &models.GitlabProject{}) dataflowTester.FlushTabler(&devops.CICDPipeline{}) dataflowTester.FlushTabler(&devops.CiCDPipelineCommit{}) - dataflowTester.Subtask(tasks.ConvertPipelineMeta, taskData) + dataflowTester.Subtask(tasks.ConvertDetailPipelineMeta, taskData) dataflowTester.Subtask(tasks.ConvertPipelineCommitMeta, taskData) dataflowTester.VerifyTableWithOptions(&devops.CICDPipeline{}, e2ehelper.TableOptions{ CSVRelPath: "./snapshot_tables/cicd_pipelines.csv", diff --git a/backend/plugins/gitlab/tasks/mr_comment_convertor.go b/backend/plugins/gitlab/tasks/mr_comment_convertor.go index 1ea38c9c68a..c309792cfaf 100644 --- a/backend/plugins/gitlab/tasks/mr_comment_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_comment_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,7 +28,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { @@ -43,7 +44,7 @@ var ConvertMrCommentMeta = plugin.SubTaskMeta{ } func ConvertMergeRequestComment(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_TABLE) + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_NOTES_TABLE) db := taskCtx.GetDal() clauses := []dal.Clause{ dal.From(&models.GitlabMrComment{}), diff --git a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go index 5e76cad0710..9950d39c499 100644 --- a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go @@ -39,7 +39,7 @@ var ConvertPipelineCommitMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Convert tool layer table gitlab_pipeline_project into domain layer table pipeline", DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, - Dependencies: []*plugin.SubTaskMeta{&ConvertPipelineMeta}, + Dependencies: []*plugin.SubTaskMeta{&ConvertDetailPipelineMeta}, } func ConvertPipelineCommits(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/gitlab/tasks/pipeline_convertor.go b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go similarity index 90% rename from backend/plugins/gitlab/tasks/pipeline_convertor.go rename to backend/plugins/gitlab/tasks/pipeline_detail_convertor.go index fc4a2d271f4..36f9bcea60c 100644 --- a/backend/plugins/gitlab/tasks/pipeline_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go @@ -33,19 +33,19 @@ import ( ) func init() { - RegisterSubtaskMeta(&ConvertPipelineMeta) + RegisterSubtaskMeta(&ConvertDetailPipelineMeta) } -var ConvertPipelineMeta = plugin.SubTaskMeta{ - Name: "convertPipelines", - EntryPoint: ConvertPipelines, +var ConvertDetailPipelineMeta = plugin.SubTaskMeta{ + Name: "convertDetailPipelines", + EntryPoint: ConvertDetailPipelines, EnabledByDefault: true, - Description: "Convert tool layer table gitlab_pipeline into domain layer table pipeline", + Description: "Convert tool layer table gitlab_detail_pipeline into domain layer table pipeline", DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, Dependencies: []*plugin.SubTaskMeta{&ConvertCommitsMeta}, } -func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error { +func ConvertDetailPipelines(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() data := taskCtx.GetData().(*GitlabTaskData) @@ -68,7 +68,7 @@ func ConvertPipelines(taskCtx plugin.SubTaskContext) errors.Error { ConnectionId: data.Options.ConnectionId, ProjectId: data.Options.ProjectId, }, - Table: RAW_PIPELINE_TABLE, + Table: RAW_PIPELINE_DETAILS_TABLE, }, Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { gitlabPipeline := inputRow.(*models.GitlabPipeline)