diff --git a/backend/plugins/jira/e2e/changelog_test.go b/backend/plugins/jira/e2e/changelog_test.go index 670a7e12f35..8f6880f00cd 100644 --- a/backend/plugins/jira/e2e/changelog_test.go +++ b/backend/plugins/jira/e2e/changelog_test.go @@ -38,6 +38,7 @@ func TestIssueChangelogDataFlow(t *testing.T) { } // import raw data table dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issue_changelogs.csv", "_raw_jira_api_issue_changelogs") + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_fields.csv", &models.JiraIssueField{}) dataflowTester.FlushTabler(&models.JiraIssueChangelogs{}) dataflowTester.FlushTabler(&models.JiraIssueChangelogItems{}) dataflowTester.FlushTabler(&models.JiraAccount{}) diff --git a/backend/plugins/jira/tasks/apiv2models/changelog.go b/backend/plugins/jira/tasks/apiv2models/changelog.go index 5c5c2c2a6c0..1ad6b98255c 100644 --- a/backend/plugins/jira/tasks/apiv2models/changelog.go +++ b/backend/plugins/jira/tasks/apiv2models/changelog.go @@ -76,23 +76,15 @@ func (c ChangelogItem) ToToolLayer(connectionId, changelogId uint64) *models.Jir return item } -func (c ChangelogItem) ExtractUser(connectionId uint64) []*models.JiraAccount { +func (c ChangelogItem) ExtractUser(connectionId uint64, userFieldMaps map[string]struct{}) []*models.JiraAccount { var result []*models.JiraAccount - // if `tmpFromAccountId` or `tmpToAccountId` is not empty, then this change log item stands for changes about accounts. - if c.TmpFromAccountId != "" { - // User `from` firstly + _, ok := userFieldMaps[c.Field] + if c.Field == "assignee" || c.Field == "reporter" || ok { if c.FromValue != "" { result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.FromValue}) - } else { - result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpFromAccountId}) } - } - if c.TmpToAccountId != "" { - // User `to` firstly if c.ToValue != "" { result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.ToValue}) - } else { - result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpToAccountId}) } } return result diff --git a/backend/plugins/jira/tasks/apiv2models/issue.go b/backend/plugins/jira/tasks/apiv2models/issue.go index 2ebace7c413..4a648d2e485 100644 --- a/backend/plugins/jira/tasks/apiv2models/issue.go +++ b/backend/plugins/jira/tasks/apiv2models/issue.go @@ -297,7 +297,7 @@ func (i *Issue) SetAllFields(raw json.RawMessage) errors.Error { return nil } -func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog, []*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems, []*models.JiraAccount) { +func (i Issue) ExtractEntities(connectionId uint64, userFieldMaps map[string]struct{}) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog, []*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems, []*models.JiraAccount) { issue := i.toToolLayer(connectionId) var comments []*models.JiraIssueComment var worklogs []*models.JiraWorklog @@ -338,7 +338,7 @@ func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue } for _, item := range changelog.Items { changelogItems = append(changelogItems, item.ToToolLayer(connectionId, changelog.ID)) - users = append(users, item.ExtractUser(connectionId)...) + users = append(users, item.ExtractUser(connectionId, userFieldMaps)...) } } } diff --git a/backend/plugins/jira/tasks/epic_extractor.go b/backend/plugins/jira/tasks/epic_extractor.go index 8c3c8eab3f2..cf1e7f28b89 100644 --- a/backend/plugins/jira/tasks/epic_extractor.go +++ b/backend/plugins/jira/tasks/epic_extractor.go @@ -18,9 +18,12 @@ limitations under the License. package tasks import ( + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/jira/models" ) var _ plugin.SubTaskEntryPoint = ExtractEpics @@ -44,6 +47,10 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } + userFieldMap, err := getUserFieldMap(db, connectionId, logger) + if err != nil { + return err + } extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, @@ -54,7 +61,7 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { Table: RAW_EPIC_TABLE, }, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - return extractIssues(data, mappings, row) + return extractIssues(data, mappings, row, userFieldMap) }, }) if err != nil { @@ -62,3 +69,36 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { } return extractor.Execute() } + +func getIssueFieldMap(db dal.Dal, connectionId uint64, logger log.Logger) (map[string]models.JiraIssueField, errors.Error) { + var allIssueFields []models.JiraIssueField + if err := db.All(&allIssueFields, dal.Where("connection_id = ?", connectionId)); err != nil { + return nil, err + } + issueFieldMap := make(map[string]models.JiraIssueField) + for _, v := range allIssueFields { + if _, ok := issueFieldMap[v.Name]; ok { + logger.Warn(nil, "filed name %s is duplicated", v.Name) + if v.SchemaType == "user" { + issueFieldMap[v.Name] = v + } + } else { + issueFieldMap[v.Name] = v + } + } + return issueFieldMap, nil +} + +func getUserFieldMap(db dal.Dal, connectionId uint64, logger log.Logger) (map[string]struct{}, errors.Error) { + userFieldMap := make(map[string]struct{}) + issueFieldMap, err := getIssueFieldMap(db, connectionId, logger) + if err != nil { + return nil, err + } + for filedName, issueField := range issueFieldMap { + if issueField.SchemaType == "user" { + userFieldMap[filedName] = struct{}{} + } + } + return userFieldMap, nil +} diff --git a/backend/plugins/jira/tasks/issue_changelog_convertor.go b/backend/plugins/jira/tasks/issue_changelog_convertor.go index 11244d644bb..c6cd6372813 100644 --- a/backend/plugins/jira/tasks/issue_changelog_convertor.go +++ b/backend/plugins/jira/tasks/issue_changelog_convertor.go @@ -89,17 +89,10 @@ func ConvertIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { } defer cursor.Close() - var allIssueFields []models.JiraIssueField - if err := db.All(&allIssueFields, dal.Where("connection_id = ?", connectionId)); err != nil { + issueFieldMap, err := getIssueFieldMap(db, connectionId, logger) + if err != nil { return err } - issueFieldMap := make(map[string]models.JiraIssueField) - for _, v := range allIssueFields { - if _, ok := issueFieldMap[v.Name]; ok { - logger.Warn(nil, "filed name %s is duplicated", v.Name) - } - issueFieldMap[v.Name] = v - } issueIdGenerator := didgen.NewDomainIdGenerator(&models.JiraIssue{}) sprintIdGenerator := didgen.NewDomainIdGenerator(&models.JiraSprint{}) @@ -160,16 +153,15 @@ func ConvertIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { changelog.ToValue = getStdStatus(toStatus.StatusCategory) } default: - fromAccountId := tryToResolveAccountIdFromAccountLikeField(row.Field, row.TmpFromAccountId, row.FromValue, issueFieldMap) - if fromAccountId != "" { - changelog.OriginalFromValue = accountIdGen.Generate(connectionId, fromAccountId) - } - toAccountId := tryToResolveAccountIdFromAccountLikeField(row.Field, row.TmpToAccountId, row.ToValue, issueFieldMap) - if toAccountId != "" { - changelog.OriginalToValue = accountIdGen.Generate(connectionId, toAccountId) + if v, ok := issueFieldMap[row.FieldId]; ok && v.SchemaType == "user" { + if row.FromValue != "" { + changelog.OriginalFromValue = accountIdGen.Generate(connectionId, row.FromValue) + } + if row.ToValue != "" { + changelog.OriginalToValue = accountIdGen.Generate(connectionId, row.ToValue) + } } } - return []interface{}{changelog}, nil }, }) @@ -181,25 +173,6 @@ func ConvertIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { return converter.Execute() } -func tryToResolveAccountIdFromAccountLikeField(fieldName string, tmpAccountId string, fromOrToValue string, issueFieldMap map[string]models.JiraIssueField) string { - if tmpAccountId != "" { - // process other account-like fields, it works on jira9 and jira cloud. - if fromOrToValue != "" { - return fromOrToValue - } else { - return tmpAccountId - } - } else { - // it works on jira8 - // notice: field name is not unique, but we cannot fetch field id here. - if v, ok := issueFieldMap[fieldName]; ok && v.SchemaType == "user" { - // field type is account - return fromOrToValue - } - } - return "" -} - func convertIds(ids string, connectionId uint64, sprintIdGenerator *didgen.DomainIdGenerator) (string, errors.Error) { ss := strings.Split(ids, ",") var resultSlice []string diff --git a/backend/plugins/jira/tasks/issue_changelog_extractor.go b/backend/plugins/jira/tasks/issue_changelog_extractor.go index d9e8d659765..6afd1232930 100644 --- a/backend/plugins/jira/tasks/issue_changelog_extractor.go +++ b/backend/plugins/jira/tasks/issue_changelog_extractor.go @@ -42,6 +42,10 @@ func ExtractIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { return nil } connectionId := data.Options.ConnectionId + userFieldMap, err := getUserFieldMap(taskCtx.GetDal(), connectionId, taskCtx.GetLogger()) + if err != nil { + return err + } extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, @@ -76,7 +80,7 @@ func ExtractIssueChangelogs(taskCtx plugin.SubTaskContext) errors.Error { // collect changelog_items for _, item := range changelog.Items { result = append(result, item.ToToolLayer(connectionId, changelog.ID)) - extractedUsersFromChangelogItem := item.ExtractUser(connectionId) + extractedUsersFromChangelogItem := item.ExtractUser(connectionId, userFieldMap) for _, u := range extractedUsersFromChangelogItem { if u != nil && u.AccountId != "" { result = append(result, u) diff --git a/backend/plugins/jira/tasks/issue_extractor.go b/backend/plugins/jira/tasks/issue_extractor.go index d2230bd747d..c8bfafc20ca 100644 --- a/backend/plugins/jira/tasks/issue_extractor.go +++ b/backend/plugins/jira/tasks/issue_extractor.go @@ -58,6 +58,10 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } + userFieldMap, err := getUserFieldMap(db, connectionId, logger) + if err != nil { + return err + } extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, @@ -75,7 +79,7 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { Table: RAW_ISSUE_TABLE, }, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - return extractIssues(data, mappings, row) + return extractIssues(data, mappings, row, userFieldMap) }, }) if err != nil { @@ -84,7 +88,7 @@ func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { return extractor.Execute() } -func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData) ([]interface{}, errors.Error) { +func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) { var apiIssue apiv2models.Issue err := errors.Convert(json.Unmarshal(row.Data, &apiIssue)) if err != nil { @@ -99,7 +103,7 @@ func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData) if apiIssue.Fields.Created == nil { return results, nil } - sprints, issue, comments, worklogs, changelogs, changelogItems, users := apiIssue.ExtractEntities(data.Options.ConnectionId) + sprints, issue, comments, worklogs, changelogs, changelogItems, users := apiIssue.ExtractEntities(data.Options.ConnectionId, userFieldMaps) for _, sprintId := range sprints { sprintIssue := &models.JiraSprintIssue{ ConnectionId: data.Options.ConnectionId,