feat(jira): collector accounts from issue changelog items as much as possible

backend/helpers/e2ehelper/data_flow_tester.go

@@ -460,7 +460,7 @@ func formatDbValue(value interface{}, nullable bool) string {
 	return ``
 }
 
-// ColumnWithRawData create an Column string with _raw_data_* appending
+// ColumnWithRawData create a Column string with _raw_data_* appending
 func ColumnWithRawData(column ...string) []string {
 	return append(
 		column,

backend/plugins/jira/e2e/changelog_test.go

@@ -68,6 +68,8 @@ func TestIssueChangelogDataFlow(t *testing.T) {
 			"from_string",
 			"to_value",
 			"to_string",
+			"tmp_from_account_id",
+			"tmp_to_account_id",
 		),
 	)
 	dataflowTester.VerifyTable(
@@ -83,11 +85,36 @@ func TestIssueChangelogDataFlow(t *testing.T) {
 			"timezone",
 		),
 	)
+
+	// import raw data: _raw_jira_api_issue_fields
+	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_jira_api_issue_fields.csv", "_raw_jira_api_issue_fields")
+	dataflowTester.FlushTabler(&models.JiraIssueField{})
+	dataflowTester.Subtask(tasks.ExtractIssueFieldsMeta, taskData)
+	dataflowTester.VerifyTable(
+		models.JiraIssueField{},
+		"./snapshot_tables/_tool_jira_issue_fields.csv",
+		e2ehelper.ColumnWithRawData(
+			"connection_id",
+			"id",
+			"board_id",
+			"name",
+			"custom",
+			"orderable",
+			"navigable",
+			"searchable",
+			"schema_type",
+			"schema_items",
+			"schema_custom",
+			"schema_custom_id",
+			"sche_custom_system"),
+	)
+
 	// verify changelog conversion
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_changelogs.csv", &models.JiraIssueChangelogs{})
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_changelog_items.csv", &models.JiraIssueChangelogItems{})
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_statuses_for_changelog.csv", &models.JiraStatus{})
 	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_board_issues_for_changelog.csv", &models.JiraBoardIssue{})
+	dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_jira_issue_fields.csv", &models.JiraIssueField{})
 	dataflowTester.FlushTabler(&ticket.IssueChangelogs{})
 	dataflowTester.Subtask(tasks.ConvertIssueChangelogsMeta, taskData)
 	dataflowTester.VerifyTable(

backend/plugins/jira/e2e/raw_tables/_raw_jira_api_issue_fields.csv

@@ -0,0 +1,4 @@
+
+"id","params","data","url","input","created_at"
+70,"{""ConnectionId"":2,""BoardId"":8}","{""id"":""customfield_10315"",""name"":""cu用户单选"",""custom"":true,""orderable"":true,""navigable"":true,""searchable"":true,""clauseNames"":[""cf[10315]"",""user_single""],""schema"":{""type"":""user"",""custom"":""com.atlassian.jira.plugin.system.customfieldtypes:userpicker"",""customId"":10315}}","http://172.26.129.92:8092/rest/api/2/field","null","2024-07-11 06:49:14.517"
+66,"{""ConnectionId"":2,""BoardId"":8}","{""id"":""customfield_10400"",""name"":""gitBranch"",""custom"":true,""orderable"":true,""navigable"":true,""searchable"":true,""clauseNames"":[""cf[10400]"",""gitBranch""],""schema"":{""type"":""any"",""custom"":""com.xiplink.jira.git.jira_git_plugin:gitbranches"",""customId"":10400}}","http://172.26.129.92:8092/rest/api/2/field","null","2024-07-11 06:49:14.517"

backend/plugins/jira/e2e/snapshot_tables/_tool_jira_issue_fields.csv

@@ -0,0 +1,3 @@
+connection_id,board_id,id,name,custom,orderable,navigable,searchable,schema_type,schema_items,schema_custom,schema_custom_id,sche_custom_system,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+2,8,customfield_10315,cu用户单选,1,1,1,1,user,,com.atlassian.jira.plugin.system.customfieldtypes:userpicker,10315,,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issue_fields,70,
+2,8,customfield_10400,gitBranch,1,1,1,1,any,,com.xiplink.jira.git.jira_git_plugin:gitbranches,10400,,"{""ConnectionId"":2,""BoardId"":8}",_raw_jira_api_issue_fields,66,

backend/plugins/jira/impl/impl.go

@@ -90,6 +90,7 @@ func (p Jira) GetTablesInfo() []dal.Tabler {
 		&models.JiraIssueComment{},
 		&models.JiraIssueRelationship{},
 		&models.JiraScopeConfig{},
+		&models.JiraIssueField{},
 	}
 }
 
@@ -103,6 +104,9 @@ func (p Jira) Name() string {
 
 func (p Jira) SubTaskMetas() []plugin.SubTaskMeta {
 	return []plugin.SubTaskMeta{
+		tasks.CollectIssueFieldsMeta,
+		tasks.ExtractIssueFieldsMeta,
+
 		tasks.CollectBoardFilterBeginMeta,
 
 		tasks.CollectStatusMeta,

backend/plugins/jira/models/issue_changelog.go

@@ -40,15 +40,17 @@ type JiraIssueChangelogItems struct {
 	common.NoPKModel
 
 	// collected fields
-	ConnectionId uint64 `gorm:"primaryKey"`
-	ChangelogId  uint64 `gorm:"primaryKey"`
-	Field        string `gorm:"primaryKey"`
-	FieldType    string
-	FieldId      string
-	FromValue    string
-	FromString   string
-	ToValue      string
-	ToString     string
+	ConnectionId     uint64 `gorm:"primaryKey"`
+	ChangelogId      uint64 `gorm:"primaryKey"`
+	Field            string `gorm:"primaryKey"`
+	FieldType        string
+	FieldId          string
+	FromValue        string
+	FromString       string
+	ToValue          string
+	ToString         string
+	TmpFromAccountId string
+	TmpToAccountId   string
 }
 
 func (JiraIssueChangelogs) TableName() string {

backend/plugins/jira/models/issue_field.go

@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import "github.com/apache/incubator-devlake/core/models/common"
+
+type JiraIssueField struct {
+	common.NoPKModel
+	ConnectionId uint64 `gorm:"primaryKey"`
+	BoardId      uint64 `gorm:"primaryKey"`
+
+	ID         string `json:"id" gorm:"primaryKey"`
+	Name       string `json:"name"`
+	Custom     bool   `json:"custom"`
+	Orderable  bool   `json:"orderable"`
+	Navigable  bool   `json:"navigable"`
+	Searchable bool   `json:"searchable"`
+	//ClauseNames      []string `json:"clauseNames"`
+	SchemaType       string `json:"schema_type"`
+	SchemaItems      string `json:"schema_items"`
+	SchemaCustom     string `json:"schema_custom"`
+	SchemaCustomID   int    `json:"schema_custom_id"`
+	ScheCustomSystem string `json:"sche_custom_system"`
+}
+
+func (JiraIssueField) TableName() string {
+	return "_tool_jira_issue_fields"
+}

backend/plugins/jira/models/migrationscripts/20240709_add_tmp_account_id_to_issue_changelog_item.go

@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addTmpAccountIdToJiraIssueChangelogItem)(nil)
+
+type jiraIssueChangelogItems20240709 struct {
+	TmpFromAccountId string
+	TmpToAccountId   string
+}
+
+func (jiraIssueChangelogItems20240709) TableName() string {
+	return "_tool_jira_issue_changelog_items"
+}
+
+type addTmpAccountIdToJiraIssueChangelogItem struct{}
+
+func (script *addTmpAccountIdToJiraIssueChangelogItem) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().AutoMigrate(&jiraIssueChangelogItems20240709{})
+}
+
+func (*addTmpAccountIdToJiraIssueChangelogItem) Version() uint64 {
+	return 20240709134200
+}
+
+func (*addTmpAccountIdToJiraIssueChangelogItem) Name() string {
+	return "add TmpFromAccountId and TmpToAccountId to _tool_jira_issue_changelog_items"
+}

backend/plugins/jira/models/migrationscripts/20240710_add_issue_field_table.go

@@ -0,0 +1,41 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts/archived"
+)
+
+var _ plugin.MigrationScript = (*addIssueFieldTable)(nil)
+
+type addIssueFieldTable struct{}
+
+func (script *addIssueFieldTable) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().AutoMigrate(&archived.JiraIssueField{})
+}
+
+func (*addIssueFieldTable) Version() uint64 {
+	return 20240710100000
+}
+
+func (*addIssueFieldTable) Name() string {
+	return "init table _tool_jira_issue_fields"
+}

backend/plugins/jira/models/migrationscripts/archived/issue_field.go

@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import "github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+
+type JiraIssueField struct {
+	archived.NoPKModel
+	ConnectionId uint64 `gorm:"primaryKey"`
+	BoardId      uint64 `gorm:"primaryKey"`
+
+	ID               string `json:"id" gorm:"primaryKey"`
+	Name             string `json:"name"`
+	Custom           bool   `json:"custom"`
+	Orderable        bool   `json:"orderable"`
+	Navigable        bool   `json:"navigable"`
+	Searchable       bool   `json:"searchable"`
+	SchemaType       string `json:"schema_type"`
+	SchemaItems      string `json:"schema_items"`
+	SchemaCustom     string `json:"schema_custom"`
+	SchemaCustomID   int    `json:"schema_custom_id"`
+	ScheCustomSystem string `json:"sche_custom_system"`
+}
+
+func (JiraIssueField) TableName() string {
+	return "_tool_jira_issue_fields"
+}

backend/plugins/jira/models/migrationscripts/register.go

@@ -49,5 +49,7 @@ func All() []plugin.MigrationScript {
 		new(addFilterJQL),
 		new(addWorklogToIssue),
 		new(addSubtaskToIssue),
+		new(addTmpAccountIdToJiraIssueChangelogItem),
+		new(addIssueFieldTable),
 	}
 }

backend/plugins/jira/tasks/apiv2models/changelog.go

@@ -46,38 +46,54 @@ func (c Changelog) ToToolLayer(connectionId, issueId uint64, issueUpdated *time.
 }
 
 type ChangelogItem struct {
-	Field      string `json:"field"`
-	Fieldtype  string `json:"fieldtype"`
+	Field     string `json:"field"`
+	Fieldtype string `json:"fieldtype"`
+	FieldId   string `json:"fieldId"`
+
 	FromValue  string `json:"from"`
 	FromString string `json:"fromString"`
-	ToValue    string `json:"to"`
-	ToString   string `json:"toString"`
+
+	ToValue  string `json:"to"`
+	ToString string `json:"toString"`
+
+	TmpFromAccountId string `json:"tmpFromAccountId,omitempty"`
+	TmpToAccountId   string `json:"tmpToAccountId,omitempty"`
 }
 
 func (c ChangelogItem) ToToolLayer(connectionId, changelogId uint64) *models.JiraIssueChangelogItems {
 	item := &models.JiraIssueChangelogItems{
-		ConnectionId: connectionId,
-		ChangelogId:  changelogId,
-		Field:        c.Field,
-		FieldType:    c.Fieldtype,
-		FromValue:    c.FromValue,
-		FromString:   c.FromString,
-		ToValue:      c.ToValue,
-		ToString:     c.ToString,
+		ConnectionId:     connectionId,
+		ChangelogId:      changelogId,
+		Field:            c.Field,
+		FieldType:        c.Fieldtype,
+		FromValue:        c.FromValue,
+		FromString:       c.FromString,
+		ToValue:          c.ToValue,
+		ToString:         c.ToString,
+		TmpFromAccountId: c.TmpFromAccountId,
+		TmpToAccountId:   c.TmpToAccountId,
 	}
 	return item
 }
 
 func (c ChangelogItem) ExtractUser(connectionId uint64) []*models.JiraAccount {
-	if c.Field != "assignee" {
-		return nil
-	}
 	var result []*models.JiraAccount
-	if c.FromValue != "" {
-		result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.FromValue})
+	// if `tmpFromAccountId` or `tmpToAccountId` is not empty, then this change log item stands for changes about accounts.
+	if c.TmpFromAccountId != "" {
+		// User `from` firstly
+		if c.FromValue != "" {
+			result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.FromValue})
+		} else {
+			result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpFromAccountId})
+		}
 	}
-	if c.ToValue != "" {
-		result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.ToValue})
+	if c.TmpToAccountId != "" {
+		// User `to` firstly
+		if c.ToValue != "" {
+			result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.ToValue})
+		} else {
+			result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpToAccountId})
+		}
 	}
 	return result
 }