From 0eb3994f6a3ab9cefbd9c14df9c567c17e41e7cb Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Tue, 16 Jul 2024 10:29:18 +0800 Subject: [PATCH 1/4] fix(jira): don't use `tmpFromAccountId` and `tmpToAccountId` in issue changelogs --- .../jira/tasks/issue_changelog_convertor.go | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/backend/plugins/jira/tasks/issue_changelog_convertor.go b/backend/plugins/jira/tasks/issue_changelog_convertor.go index 11244d644bb..90dca2043fe 100644 --- a/backend/plugins/jira/tasks/issue_changelog_convertor.go +++ b/backend/plugins/jira/tasks/issue_changelog_convertor.go @@ -160,11 +160,11 @@ func ConvertIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { changelog.ToValue = getStdStatus(toStatus.StatusCategory) } default: - fromAccountId := tryToResolveAccountIdFromAccountLikeField(row.Field, row.TmpFromAccountId, row.FromValue, issueFieldMap) + fromAccountId := tryToResolveAccountIdFromAccountLikeField(row.Field, row.FromValue, issueFieldMap) if fromAccountId != "" { changelog.OriginalFromValue = accountIdGen.Generate(connectionId, fromAccountId) } - toAccountId := tryToResolveAccountIdFromAccountLikeField(row.Field, row.TmpToAccountId, row.ToValue, issueFieldMap) + toAccountId := tryToResolveAccountIdFromAccountLikeField(row.Field, row.ToValue, issueFieldMap) if toAccountId != "" { changelog.OriginalToValue = accountIdGen.Generate(connectionId, toAccountId) } @@ -181,21 +181,11 @@ func ConvertIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { return converter.Execute() } -func tryToResolveAccountIdFromAccountLikeField(fieldName string, tmpAccountId string, fromOrToValue string, issueFieldMap map[string]models.JiraIssueField) string { - if tmpAccountId != "" { - // process other account-like fields, it works on jira9 and jira cloud. - if fromOrToValue != "" { - return fromOrToValue - } else { - return tmpAccountId - } - } else { - // it works on jira8 - // notice: field name is not unique, but we cannot fetch field id here. - if v, ok := issueFieldMap[fieldName]; ok && v.SchemaType == "user" { - // field type is account - return fromOrToValue - } +func tryToResolveAccountIdFromAccountLikeField(fieldName string, fromOrToValue string, issueFieldMap map[string]models.JiraIssueField) string { + // notice: field name is not unique, but we cannot fetch field id here. + if v, ok := issueFieldMap[fieldName]; ok && v.SchemaType == "user" { + // field type is account + return fromOrToValue } return "" } From 4d010ccab307624eed94f8afb95322f22333b045 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Tue, 16 Jul 2024 11:22:57 +0800 Subject: [PATCH 2/4] fix(jira): remove all usage of `tmpFromAccountId` and `tmpToAccountId` --- .../jira/tasks/apiv2models/changelog.go | 14 ++----- .../plugins/jira/tasks/apiv2models/issue.go | 4 +- backend/plugins/jira/tasks/epic_extractor.go | 38 ++++++++++++++++++- .../jira/tasks/issue_changelog_convertor.go | 11 +----- .../jira/tasks/issue_changelog_extractor.go | 6 ++- backend/plugins/jira/tasks/issue_extractor.go | 10 +++-- 6 files changed, 56 insertions(+), 27 deletions(-) diff --git a/backend/plugins/jira/tasks/apiv2models/changelog.go b/backend/plugins/jira/tasks/apiv2models/changelog.go index 5c5c2c2a6c0..1ad6b98255c 100644 --- a/backend/plugins/jira/tasks/apiv2models/changelog.go +++ b/backend/plugins/jira/tasks/apiv2models/changelog.go @@ -76,23 +76,15 @@ func (c ChangelogItem) ToToolLayer(connectionId, changelogId uint64) *models.Jir return item } -func (c ChangelogItem) ExtractUser(connectionId uint64) []*models.JiraAccount { +func (c ChangelogItem) ExtractUser(connectionId uint64, userFieldMaps map[string]struct{}) []*models.JiraAccount { var result []*models.JiraAccount - // if `tmpFromAccountId` or `tmpToAccountId` is not empty, then this change log item stands for changes about accounts. - if c.TmpFromAccountId != "" { - // User `from` firstly + _, ok := userFieldMaps[c.Field] + if c.Field == "assignee" || c.Field == "reporter" || ok { if c.FromValue != "" { result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.FromValue}) - } else { - result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpFromAccountId}) } - } - if c.TmpToAccountId != "" { - // User `to` firstly if c.ToValue != "" { result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.ToValue}) - } else { - result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpToAccountId}) } } return result diff --git a/backend/plugins/jira/tasks/apiv2models/issue.go b/backend/plugins/jira/tasks/apiv2models/issue.go index 2ebace7c413..4a648d2e485 100644 --- a/backend/plugins/jira/tasks/apiv2models/issue.go +++ b/backend/plugins/jira/tasks/apiv2models/issue.go @@ -297,7 +297,7 @@ func (i *Issue) SetAllFields(raw json.RawMessage) errors.Error { return nil } -func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog, []*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems, []*models.JiraAccount) { +func (i Issue) ExtractEntities(connectionId uint64, userFieldMaps map[string]struct{}) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog, []*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems, []*models.JiraAccount) { issue := i.toToolLayer(connectionId) var comments []*models.JiraIssueComment var worklogs []*models.JiraWorklog @@ -338,7 +338,7 @@ func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue } for _, item := range changelog.Items { changelogItems = append(changelogItems, item.ToToolLayer(connectionId, changelog.ID)) - users = append(users, item.ExtractUser(connectionId)...) + users = append(users, item.ExtractUser(connectionId, userFieldMaps)...) } } } diff --git a/backend/plugins/jira/tasks/epic_extractor.go b/backend/plugins/jira/tasks/epic_extractor.go index 8c3c8eab3f2..c9a981f3d04 100644 --- a/backend/plugins/jira/tasks/epic_extractor.go +++ b/backend/plugins/jira/tasks/epic_extractor.go @@ -18,9 +18,12 @@ limitations under the License. package tasks import ( + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/jira/models" ) var _ plugin.SubTaskEntryPoint = ExtractEpics @@ -44,6 +47,10 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } + userFieldMap, err := getUserFieldMap(db, connectionId, logger) + if err != nil { + return err + } extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, @@ -54,7 +61,7 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { Table: RAW_EPIC_TABLE, }, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - return extractIssues(data, mappings, row) + return extractIssues(data, mappings, row, userFieldMap) }, }) if err != nil { @@ -62,3 +69,32 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { } return extractor.Execute() } + +func getIssueFieldMap(db dal.Dal, connectionId uint64, logger log.Logger) (map[string]models.JiraIssueField, errors.Error) { + var allIssueFields []models.JiraIssueField + if err := db.All(&allIssueFields, dal.Where("connection_id = ?", connectionId)); err != nil { + return nil, err + } + issueFieldMap := make(map[string]models.JiraIssueField) + for _, v := range allIssueFields { + if _, ok := issueFieldMap[v.Name]; ok { + logger.Warn(nil, "filed name %s is duplicated", v.Name) + } + issueFieldMap[v.Name] = v + } + return issueFieldMap, nil +} + +func getUserFieldMap(db dal.Dal, connectionId uint64, logger log.Logger) (map[string]struct{}, errors.Error) { + userFieldMap := make(map[string]struct{}) + issueFieldMap, err := getIssueFieldMap(db, connectionId, logger) + if err != nil { + return nil, err + } + for filedName, issueField := range issueFieldMap { + if issueField.SchemaType == "user" { + userFieldMap[filedName] = struct{}{} + } + } + return userFieldMap, nil +} diff --git a/backend/plugins/jira/tasks/issue_changelog_convertor.go b/backend/plugins/jira/tasks/issue_changelog_convertor.go index 90dca2043fe..6e433e6a6aa 100644 --- a/backend/plugins/jira/tasks/issue_changelog_convertor.go +++ b/backend/plugins/jira/tasks/issue_changelog_convertor.go @@ -89,17 +89,10 @@ func ConvertIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { } defer cursor.Close() - var allIssueFields []models.JiraIssueField - if err := db.All(&allIssueFields, dal.Where("connection_id = ?", connectionId)); err != nil { + issueFieldMap, err := getIssueFieldMap(db, connectionId, logger) + if err != nil { return err } - issueFieldMap := make(map[string]models.JiraIssueField) - for _, v := range allIssueFields { - if _, ok := issueFieldMap[v.Name]; ok { - logger.Warn(nil, "filed name %s is duplicated", v.Name) - } - issueFieldMap[v.Name] = v - } issueIdGenerator := didgen.NewDomainIdGenerator(&models.JiraIssue{}) sprintIdGenerator := didgen.NewDomainIdGenerator(&models.JiraSprint{}) diff --git a/backend/plugins/jira/tasks/issue_changelog_extractor.go b/backend/plugins/jira/tasks/issue_changelog_extractor.go index d9e8d659765..6afd1232930 100644 --- a/backend/plugins/jira/tasks/issue_changelog_extractor.go +++ b/backend/plugins/jira/tasks/issue_changelog_extractor.go @@ -42,6 +42,10 @@ func ExtractIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { return nil } connectionId := data.Options.ConnectionId + userFieldMap, err := getUserFieldMap(taskCtx.GetDal(), connectionId, taskCtx.GetLogger()) + if err != nil { + return err + } extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, @@ -76,7 +80,7 @@ func ExtractIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { // collect changelog_items for _, item := range changelog.Items { result = append(result, item.ToToolLayer(connectionId, changelog.ID)) - extractedUsersFromChangelogItem := item.ExtractUser(connectionId) + extractedUsersFromChangelogItem := item.ExtractUser(connectionId, userFieldMap) for _, u := range extractedUsersFromChangelogItem { if u != nil && u.AccountId != "" { result = append(result, u) diff --git a/backend/plugins/jira/tasks/issue_extractor.go b/backend/plugins/jira/tasks/issue_extractor.go index d2230bd747d..c8bfafc20ca 100644 --- a/backend/plugins/jira/tasks/issue_extractor.go +++ b/backend/plugins/jira/tasks/issue_extractor.go @@ -58,6 +58,10 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } + userFieldMap, err := getUserFieldMap(db, connectionId, logger) + if err != nil { + return err + } extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, @@ -75,7 +79,7 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { Table: RAW_ISSUE_TABLE, }, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - return extractIssues(data, mappings, row) + return extractIssues(data, mappings, row, userFieldMap) }, }) if err != nil { @@ -84,7 +88,7 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { return extractor.Execute() } -func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData) ([]interface{}, errors.Error) { +func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) { var apiIssue apiv2models.Issue err := errors.Convert(json.Unmarshal(row.Data, &apiIssue)) if err != nil { @@ -99,7 +103,7 @@ func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData) if apiIssue.Fields.Created == nil { return results, nil } - sprints, issue, comments, worklogs, changelogs, changelogItems, users := apiIssue.ExtractEntities(data.Options.ConnectionId) + sprints, issue, comments, worklogs, changelogs, changelogItems, users := apiIssue.ExtractEntities(data.Options.ConnectionId, userFieldMaps) for _, sprintId := range sprints { sprintIssue := &models.JiraSprintIssue{ ConnectionId: data.Options.ConnectionId, From 3017b7db3f4093e61558a49afc123cb59a729461 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Tue, 16 Jul 2024 11:30:45 +0800 Subject: [PATCH 3/4] refactor(jira): remove some codes --- backend/plugins/jira/tasks/epic_extractor.go | 6 ++++- .../jira/tasks/issue_changelog_convertor.go | 24 ++++++------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/backend/plugins/jira/tasks/epic_extractor.go b/backend/plugins/jira/tasks/epic_extractor.go index c9a981f3d04..cf1e7f28b89 100644 --- a/backend/plugins/jira/tasks/epic_extractor.go +++ b/backend/plugins/jira/tasks/epic_extractor.go @@ -79,8 +79,12 @@ func getIssueFieldMap(db dal.Dal, connectionId uint64, logger log.Logger) (map[s for _, v := range allIssueFields { if _, ok := issueFieldMap[v.Name]; ok { logger.Warn(nil, "filed name %s is duplicated", v.Name) + if v.SchemaType == "user" { + issueFieldMap[v.Name] = v + } + } else { + issueFieldMap[v.Name] = v } - issueFieldMap[v.Name] = v } return issueFieldMap, nil } diff --git a/backend/plugins/jira/tasks/issue_changelog_convertor.go b/backend/plugins/jira/tasks/issue_changelog_convertor.go index 6e433e6a6aa..c6cd6372813 100644 --- a/backend/plugins/jira/tasks/issue_changelog_convertor.go +++ b/backend/plugins/jira/tasks/issue_changelog_convertor.go @@ -153,16 +153,15 @@ func ConvertIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { changelog.ToValue = getStdStatus(toStatus.StatusCategory) } default: - fromAccountId := tryToResolveAccountIdFromAccountLikeField(row.Field, row.FromValue, issueFieldMap) - if fromAccountId != "" { - changelog.OriginalFromValue = accountIdGen.Generate(connectionId, fromAccountId) - } - toAccountId := tryToResolveAccountIdFromAccountLikeField(row.Field, row.ToValue, issueFieldMap) - if toAccountId != "" { - changelog.OriginalToValue = accountIdGen.Generate(connectionId, toAccountId) + if v, ok := issueFieldMap[row.FieldId]; ok && v.SchemaType == "user" { + if row.FromValue != "" { + changelog.OriginalFromValue = accountIdGen.Generate(connectionId, row.FromValue) + } + if row.ToValue != "" { + changelog.OriginalToValue = accountIdGen.Generate(connectionId, row.ToValue) + } } } - return []interface{}{changelog}, nil }, }) @@ -174,15 +173,6 @@ func ConvertIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { return converter.Execute() } -func tryToResolveAccountIdFromAccountLikeField(fieldName string, fromOrToValue string, issueFieldMap map[string]models.JiraIssueField) string { - // notice: field name is not unique, but we cannot fetch field id here. - if v, ok := issueFieldMap[fieldName]; ok && v.SchemaType == "user" { - // field type is account - return fromOrToValue - } - return "" -} - func convertIds(ids string, connectionId uint64, sprintIdGenerator *didgen.DomainIdGenerator) (string, errors.Error) { ss := strings.Split(ids, ",") var resultSlice []string From bb022b32978c71fb538e9e7341d4eccb82b74379 Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Tue, 16 Jul 2024 11:34:20 +0800 Subject: [PATCH 4/4] fix(jira): fix e2e test --- backend/plugins/jira/e2e/changelog_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/plugins/jira/e2e/changelog_test.go b/backend/plugins/jira/e2e/changelog_test.go index 670a7e12f35..8f6880f00cd 100644 --- a/backend/plugins/jira/e2e/changelog_test.go +++ b/backend/plugins/jira/e2e/changelog_test.go @@ -38,6 +38,7 @@ func TestIssueChangelogDataFlow(t *testing.T) { } // import raw data table dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issue_changelogs.csv", "_raw_jira_api_issue_changelogs") + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_fields.csv", &models.JiraIssueField{}) dataflowTester.FlushTabler(&models.JiraIssueChangelogs{}) dataflowTester.FlushTabler(&models.JiraIssueChangelogItems{}) dataflowTester.FlushTabler(&models.JiraAccount{})