Remove the usage of TmpFromAccountId and TmpToAccountId in Jira issue related data.

backend/plugins/jira/e2e/changelog_test.go

@@ -38,6 +38,7 @@ func TestIssueChangelogDataFlow(t *testing.T) {
 	}
 	// import raw data table
 	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issue_changelogs.csv", "_raw_jira_api_issue_changelogs")
+	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_fields.csv", &models.JiraIssueField{})
 	dataflowTester.FlushTabler(&models.JiraIssueChangelogs{})
 	dataflowTester.FlushTabler(&models.JiraIssueChangelogItems{})
 	dataflowTester.FlushTabler(&models.JiraAccount{})

backend/plugins/jira/tasks/apiv2models/changelog.go

@@ -76,23 +76,15 @@ func (c ChangelogItem) ToToolLayer(connectionId, changelogId uint64) *models.Jir
 	return item
 }
 
-func (c ChangelogItem) ExtractUser(connectionId uint64) []*models.JiraAccount {
+func (c ChangelogItem) ExtractUser(connectionId uint64, userFieldMaps map[string]struct{}) []*models.JiraAccount {
 	var result []*models.JiraAccount
-	// if `tmpFromAccountId` or `tmpToAccountId` is not empty, then this change log item stands for changes about accounts.
-	if c.TmpFromAccountId != "" {
-		// User `from` firstly
+	_, ok := userFieldMaps[c.Field]
+	if c.Field == "assignee" || c.Field == "reporter" || ok {
 		if c.FromValue != "" {
 			result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.FromValue})
-		} else {
-			result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpFromAccountId})
 		}
-	}
-	if c.TmpToAccountId != "" {
-		// User `to` firstly
 		if c.ToValue != "" {
 			result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.ToValue})
-		} else {
-			result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpToAccountId})
 		}
 	}
 	return result

backend/plugins/jira/tasks/apiv2models/issue.go

@@ -300,7 +300,7 @@ func (i *Issue) SetAllFields(raw json.RawMessage) errors.Error {
 	return nil
 }
 
-func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog, []*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems, []*models.JiraAccount) {
+func (i Issue) ExtractEntities(connectionId uint64, userFieldMaps map[string]struct{}) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog, []*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems, []*models.JiraAccount) {
 	issue := i.toToolLayer(connectionId)
 	var comments []*models.JiraIssueComment
 	var worklogs []*models.JiraWorklog
@@ -341,7 +341,7 @@ func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue
 			}
 			for _, item := range changelog.Items {
 				changelogItems = append(changelogItems, item.ToToolLayer(connectionId, changelog.ID))
-				users = append(users, item.ExtractUser(connectionId)...)
+				users = append(users, item.ExtractUser(connectionId, userFieldMaps)...)
 			}
 		}
 	}

backend/plugins/jira/tasks/epic_extractor.go

@@ -18,9 +18,12 @@ limitations under the License.
 package tasks
 
 import (
+	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/log"
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/plugins/jira/models"
 )
 
 var _ plugin.SubTaskEntryPoint = ExtractEpics
@@ -44,6 +47,10 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error {
 	if err != nil {
 		return err
 	}
+	userFieldMap, err := getUserFieldMap(db, connectionId, logger)
+	if err != nil {
+		return err
+	}
 	extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
 		RawDataSubTaskArgs: api.RawDataSubTaskArgs{
 			Ctx: taskCtx,
@@ -54,11 +61,44 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error {
 			Table: RAW_EPIC_TABLE,
 		},
 		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			return extractIssues(data, mappings, row)
+			return extractIssues(data, mappings, row, userFieldMap)
 		},
 	})
 	if err != nil {
 		return err
 	}
 	return extractor.Execute()
 }
+
+func getIssueFieldMap(db dal.Dal, connectionId uint64, logger log.Logger) (map[string]models.JiraIssueField, errors.Error) {
+	var allIssueFields []models.JiraIssueField
+	if err := db.All(&allIssueFields, dal.Where("connection_id = ?", connectionId)); err != nil {
+		return nil, err
+	}
+	issueFieldMap := make(map[string]models.JiraIssueField)
+	for _, v := range allIssueFields {
+		if _, ok := issueFieldMap[v.Name]; ok {
+			logger.Warn(nil, "filed name %s is duplicated", v.Name)
+			if v.SchemaType == "user" {
+				issueFieldMap[v.Name] = v
+			}
+		} else {
+			issueFieldMap[v.Name] = v
+		}
+	}
+	return issueFieldMap, nil
+}
+
+func getUserFieldMap(db dal.Dal, connectionId uint64, logger log.Logger) (map[string]struct{}, errors.Error) {
+	userFieldMap := make(map[string]struct{})
+	issueFieldMap, err := getIssueFieldMap(db, connectionId, logger)
+	if err != nil {
+		return nil, err
+	}
+	for filedName, issueField := range issueFieldMap {
+		if issueField.SchemaType == "user" {
+			userFieldMap[filedName] = struct{}{}
+		}
+	}
+	return userFieldMap, nil
+}

backend/plugins/jira/tasks/issue_changelog_convertor.go

@@ -68,17 +68,10 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
 		statusMap[v.ID] = v
 	}
 
-	var allIssueFields []models.JiraIssueField
-	if err := db.All(&allIssueFields, dal.Where("connection_id = ?", connectionId)); err != nil {
+	issueFieldMap, err := getIssueFieldMap(db, connectionId, logger)
+	if err != nil {
 		return err
 	}
-	issueFieldMap := make(map[string]models.JiraIssueField)
-	for _, v := range allIssueFields {
-		if _, ok := issueFieldMap[v.Name]; ok {
-			logger.Warn(nil, "filed name %s is duplicated", v.Name)
-		}
-		issueFieldMap[v.Name] = v
-	}
 
 	issueIdGenerator := didgen.NewDomainIdGenerator(&models.JiraIssue{})
 	sprintIdGenerator := didgen.NewDomainIdGenerator(&models.JiraSprint{})
@@ -159,16 +152,15 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
 					changelog.ToValue = getStdStatus(toStatus.StatusCategory)
 				}
 			default:
-				fromAccountId := tryToResolveAccountIdFromAccountLikeField(row.Field, row.TmpFromAccountId, row.FromValue, issueFieldMap)
-				if fromAccountId != "" {
-					changelog.OriginalFromValue = accountIdGen.Generate(connectionId, fromAccountId)
-				}
-				toAccountId := tryToResolveAccountIdFromAccountLikeField(row.Field, row.TmpToAccountId, row.ToValue, issueFieldMap)
-				if toAccountId != "" {
-					changelog.OriginalToValue = accountIdGen.Generate(connectionId, toAccountId)
+				if v, ok := issueFieldMap[row.FieldId]; ok && v.SchemaType == "user" {
+					if row.FromValue != "" {
+						changelog.OriginalFromValue = accountIdGen.Generate(connectionId, row.FromValue)
+					}
+					if row.ToValue != "" {
+						changelog.OriginalToValue = accountIdGen.Generate(connectionId, row.ToValue)
+					}
 				}
 			}
-
 			return []interface{}{changelog}, nil
 
 		},
@@ -181,25 +173,6 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
 	return converter.Execute()
 }
 
-func tryToResolveAccountIdFromAccountLikeField(fieldName string, tmpAccountId string, fromOrToValue string, issueFieldMap map[string]models.JiraIssueField) string {
-	if tmpAccountId != "" {
-		// process other account-like fields, it works on jira9 and jira cloud.
-		if fromOrToValue != "" {
-			return fromOrToValue
-		} else {
-			return tmpAccountId
-		}
-	} else {
-		// it works on jira8
-		// notice: field name is not unique, but we cannot fetch field id here.
-		if v, ok := issueFieldMap[fieldName]; ok && v.SchemaType == "user" {
-			// field type is account
-			return fromOrToValue
-		}
-	}
-	return ""
-}
-
 func convertIds(ids string, connectionId uint64, sprintIdGenerator *didgen.DomainIdGenerator) (string, errors.Error) {
 	ss := strings.Split(ids, ",")
 	var resultSlice []string

backend/plugins/jira/tasks/issue_changelog_extractor.go

@@ -43,6 +43,10 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
 		return nil
 	}
 	connectionId := data.Options.ConnectionId
+	userFieldMap, err := getUserFieldMap(subtaskCtx.GetDal(), connectionId, subtaskCtx.GetLogger())
+	if err != nil {
+		return err
+	}
 	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{
 		SubtaskCommonArgs: &api.SubtaskCommonArgs{
 			SubTaskContext: subtaskCtx,
@@ -77,7 +81,7 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error {
 			// collect changelog_items
 			for _, item := range changelog.Items {
 				result = append(result, item.ToToolLayer(connectionId, changelog.ID))
-				extractedUsersFromChangelogItem := item.ExtractUser(connectionId)
+				extractedUsersFromChangelogItem := item.ExtractUser(connectionId, userFieldMap)
 				for _, u := range extractedUsersFromChangelogItem {
 					if u != nil && u.AccountId != "" {
 						result = append(result, u)

backend/plugins/jira/tasks/issue_extractor.go

@@ -58,6 +58,10 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
 	if err != nil {
 		return err
 	}
+	userFieldMap, err := getUserFieldMap(db, connectionId, logger)
+	if err != nil {
+		return err
+	}
 	extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{
 		SubtaskCommonArgs: &api.SubtaskCommonArgs{
 			SubTaskContext: subtaskCtx,
@@ -69,7 +73,7 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
 			SubtaskConfig: mappings,
 		},
 		Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
-			return extractIssues(data, mappings, row)
+			return extractIssues(data, mappings, row, userFieldMap)
 		},
 	})
 	if err != nil {
@@ -78,7 +82,7 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error {
 	return extractor.Execute()
 }
 
-func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData) ([]interface{}, errors.Error) {
+func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) {
 	var apiIssue apiv2models.Issue
 	err := errors.Convert(json.Unmarshal(row.Data, &apiIssue))
 	if err != nil {
@@ -93,7 +97,7 @@ func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData)
 	if apiIssue.Fields.Created == nil {
 		return results, nil
 	}
-	sprints, issue, comments, worklogs, changelogs, changelogItems, users := apiIssue.ExtractEntities(data.Options.ConnectionId)
+	sprints, issue, comments, worklogs, changelogs, changelogItems, users := apiIssue.ExtractEntities(data.Options.ConnectionId, userFieldMaps)
 	for _, sprintId := range sprints {
 		sprintIssue := &models.JiraSprintIssue{
 			ConnectionId:     data.Options.ConnectionId,