fix: github token expired may cause 500 error

backend/core/plugin/api_client.go

@@ -24,11 +24,24 @@ import (
 	"github.com/apache/incubator-devlake/core/errors"
 )
 
+// ApiAsyncCallback FIXME ...
+type ApiAsyncCallback func(*http.Response) errors.Error
+
+// ApiClientBeforeRequest FIXME ...
+type ApiClientBeforeRequest func(req *http.Request) errors.Error
+
+// ApiClientAfterResponse FIXME ...
+type ApiClientAfterResponse func(res *http.Response) errors.Error
+
 // ApiClientAbstract defines the functionalities needed by all plugins for Synchronized API Request
 type ApiClient interface {
 	SetData(name string, data interface{})
 	GetData(name string) interface{}
 	SetHeaders(headers map[string]string)
+	SetBeforeFunction(callback ApiClientBeforeRequest)
+	GetBeforeFunction() ApiClientBeforeRequest
+	SetAfterFunction(callback ApiClientAfterResponse)
+	GetAfterFunction() ApiClientAfterResponse
 	Get(path string, query url.Values, headers http.Header) (*http.Response, errors.Error)
 	Post(path string, query url.Values, body interface{}, headers http.Header) (*http.Response, errors.Error)
 }

backend/helpers/pluginhelper/api/api_async_client.go

@@ -30,7 +30,6 @@ import (
 	"github.com/apache/incubator-devlake/core/log"
 	plugin "github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/core/utils"
-	"github.com/apache/incubator-devlake/helpers/pluginhelper/common"
 )
 
 // HttpMinStatusRetryCode is which status will retry
@@ -152,7 +151,7 @@ func (apiClient *ApiAsyncClient) DoAsync(
 	query url.Values,
 	body interface{},
 	header http.Header,
-	handler common.ApiAsyncCallback,
+	handler plugin.ApiAsyncCallback,
 	retry int,
 ) {
 	var request func() errors.Error
@@ -224,7 +223,7 @@ func (apiClient *ApiAsyncClient) DoGetAsync(
 	path string,
 	query url.Values,
 	header http.Header,
-	handler common.ApiAsyncCallback,
+	handler plugin.ApiAsyncCallback,
 ) {
 	apiClient.DoAsync(http.MethodGet, path, query, nil, header, handler, 0)
 }
@@ -235,7 +234,7 @@ func (apiClient *ApiAsyncClient) DoPostAsync(
 	query url.Values,
 	body interface{},
 	header http.Header,
-	handler common.ApiAsyncCallback,
+	handler plugin.ApiAsyncCallback,
 ) {
 	apiClient.DoAsync(http.MethodPost, path, query, body, header, handler, 0)
 }
@@ -247,14 +246,14 @@ func (apiClient *ApiAsyncClient) GetNumOfWorkers() int {
 
 // RateLimitedApiClient FIXME ...
 type RateLimitedApiClient interface {
-	DoGetAsync(path string, query url.Values, header http.Header, handler common.ApiAsyncCallback)
-	DoPostAsync(path string, query url.Values, body interface{}, header http.Header, handler common.ApiAsyncCallback)
+	DoGetAsync(path string, query url.Values, header http.Header, handler plugin.ApiAsyncCallback)
+	DoPostAsync(path string, query url.Values, body interface{}, header http.Header, handler plugin.ApiAsyncCallback)
 	WaitAsync() errors.Error
 	HasError() bool
 	NextTick(task func() errors.Error)
 	GetNumOfWorkers() int
-	GetAfterFunction() common.ApiClientAfterResponse
-	SetAfterFunction(callback common.ApiClientAfterResponse)
+	GetAfterFunction() plugin.ApiClientAfterResponse
+	SetAfterFunction(callback plugin.ApiClientAfterResponse)
 	Reset(d time.Duration)
 	GetTickInterval() time.Duration
 	Release()

backend/helpers/pluginhelper/api/api_client.go

@@ -39,7 +39,6 @@ import (
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/log"
 	"github.com/apache/incubator-devlake/core/utils"
-	"github.com/apache/incubator-devlake/helpers/pluginhelper/common"
 )
 
 // ErrIgnoreAndContinue is a error which should be ignored
@@ -57,8 +56,8 @@ type ApiClient struct {
 	data       map[string]interface{}
 	data_mutex sync.Mutex
 
-	beforeRequest common.ApiClientBeforeRequest
-	afterResponse common.ApiClientAfterResponse
+	beforeRequest plugin.ApiClientBeforeRequest
+	afterResponse plugin.ApiClientAfterResponse
 	ctx           gocontext.Context
 	logger        log.Logger
 }
@@ -92,6 +91,17 @@ func NewApiClientFromConnection(
 		})
 	}
 
+	apiClient.SetAfterFunction(func(res *http.Response) errors.Error {
+		if res.StatusCode >= 400 {
+			bytes, err := io.ReadAll(res.Body)
+			if err != nil {
+				return errors.BadInput.Wrap(err, fmt.Sprintf("request failed with status code %d", res.StatusCode))
+			}
+			return errors.BadInput.New(fmt.Sprintf("request failed with status code %d, body: %s", res.StatusCode, string(bytes)))
+		}
+		return nil
+	})
+
 	return apiClient, nil
 }
 
@@ -224,23 +234,23 @@ func (apiClient *ApiClient) GetHeaders() map[string]string {
 }
 
 // GetBeforeFunction return beforeResponseFunction
-func (apiClient *ApiClient) GetBeforeFunction() common.ApiClientBeforeRequest {
+func (apiClient *ApiClient) GetBeforeFunction() plugin.ApiClientBeforeRequest {
 	return apiClient.beforeRequest
 }
 
 // SetBeforeFunction will set beforeResponseFunction
-func (apiClient *ApiClient) SetBeforeFunction(callback common.ApiClientBeforeRequest) {
+func (apiClient *ApiClient) SetBeforeFunction(callback plugin.ApiClientBeforeRequest) {
 	apiClient.beforeRequest = callback
 }
 
 // GetAfterFunction return afterResponseFunction
-func (apiClient *ApiClient) GetAfterFunction() common.ApiClientAfterResponse {
+func (apiClient *ApiClient) GetAfterFunction() plugin.ApiClientAfterResponse {
 	return apiClient.afterResponse
 }
 
 // SetAfterFunction will set afterResponseFunction
 // don't call this function directly in collector, use Collector.AfterResponse instead.
-func (apiClient *ApiClient) SetAfterFunction(callback common.ApiClientAfterResponse) {
+func (apiClient *ApiClient) SetAfterFunction(callback plugin.ApiClientAfterResponse) {
 	apiClient.afterResponse = callback
 }
 
@@ -327,14 +337,15 @@ func (apiClient *ApiClient) Do(
 	if apiClient.beforeRequest != nil {
 		err = apiClient.beforeRequest(req)
 		if err != nil {
-			return nil, errors.Default.Wrap(err, fmt.Sprintf("error running beforeRequest for %s", req.URL.String()))
+			apiClient.logError(err, "[api-client] beforeRequest returned error for %s", req.URL.String())
+			return nil, err
 		}
 	}
 	apiClient.logDebug("[api-client] %v %v", method, *uri)
 	res, err = errors.Convert01(apiClient.client.Do(req))
 	if err != nil {
 		apiClient.logError(err, "[api-client] failed to request %s with error", req.URL.String())
-		return nil, errors.Default.Wrap(err, fmt.Sprintf("error requesting %s", req.URL.String()))
+		return nil, err
 	}
 	// after receive
 	if apiClient.afterResponse != nil {
@@ -345,7 +356,8 @@ func (apiClient *ApiClient) Do(
 		}
 		if err != nil {
 			res.Body.Close()
-			return nil, errors.Default.Wrap(err, fmt.Sprintf("error running afterRequest for %s", req.URL.String()))
+			apiClient.logError(err, "[api-client] afterResponse returned error for %s", req.URL.String())
+			return nil, err
 		}
 	}
 	return res, nil

backend/helpers/pluginhelper/api/api_collector.go

@@ -32,7 +32,6 @@ import (
 
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
-	"github.com/apache/incubator-devlake/helpers/pluginhelper/common"
 )
 
 var _ plugin.SubTask = (*ApiCollector)(nil)
@@ -87,7 +86,7 @@ type ApiCollectorArgs struct {
 	// NORMALLY, DO NOT SPECIFY THIS PARAMETER, unless you know what it means
 	Concurrency    int
 	ResponseParser func(res *http.Response) ([]json.RawMessage, errors.Error)
-	AfterResponse  common.ApiClientAfterResponse
+	AfterResponse  plugin.ApiClientAfterResponse
 	RequestBody    func(reqData *RequestData) map[string]interface{}
 	Method         string
 }
@@ -389,12 +388,12 @@ func (collector *ApiCollector) generateUrl(pager *Pager, input interface{}) (str
 }
 
 // GetAfterResponse return apiClient's afterResponseFunction
-func (collector *ApiCollector) GetAfterResponse() common.ApiClientAfterResponse {
+func (collector *ApiCollector) GetAfterResponse() plugin.ApiClientAfterResponse {
 	return collector.args.ApiClient.GetAfterFunction()
 }
 
 // SetAfterResponse set apiClient's afterResponseFunction
-func (collector *ApiCollector) SetAfterResponse(f common.ApiClientAfterResponse) {
+func (collector *ApiCollector) SetAfterResponse(f plugin.ApiClientAfterResponse) {
 	collector.args.ApiClient.SetAfterFunction(f)
 }
 

backend/helpers/pluginhelper/api/api_collector_test.go

@@ -25,7 +25,7 @@ import (
 	"testing"
 
 	"github.com/apache/incubator-devlake/core/errors"
-	"github.com/apache/incubator-devlake/helpers/pluginhelper/common"
+	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/helpers/unithelper"
 	mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
 	mockapi "github.com/apache/incubator-devlake/mocks/helpers/pluginhelper/api"
@@ -74,7 +74,7 @@ func TestFetchPageUndetermined(t *testing.T) {
 			},
 			Body: io.NopCloser(bytes.NewBufferString(body)),
 		}
-		handler := args.Get(3).(common.ApiAsyncCallback)
+		handler := args.Get(3).(plugin.ApiAsyncCallback)
 		handler(res)
 	}).Twice()
 	mockApi.On("NextTick", mock.Anything).Run(func(args mock.Arguments) {

backend/helpers/pluginhelper/api/api_collector_with_state.go

@@ -27,7 +27,6 @@ import (
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models"
 	"github.com/apache/incubator-devlake/core/plugin"
-	"github.com/apache/incubator-devlake/helpers/pluginhelper/common"
 )
 
 // ApiCollectorStateManager save collector state in framework table
@@ -302,7 +301,7 @@ type FinalizableApiCollectorCommonArgs struct {
 	Header          func(reqData *RequestData, createdAfter *time.Time) (http.Header, errors.Error) // optional, build header for the request
 	RequestBody     func(reqData *RequestData) map[string]interface{}                               // optional, build request body for the request if the Method set to POST or PUT
 	MinTickInterval *time.Duration                                                                  // optional, minimum interval between two requests, some endpoints might have a more conservative rate limit than others within the same instance, you can mitigate this by setting a higher MinTickInterval to override the connection level rate limit.
-	AfterResponse   common.ApiClientAfterResponse                                                   // optional, hook to run after each response, would be called before the ResponseParser
+	AfterResponse   plugin.ApiClientAfterResponse                                                   // optional, hook to run after each response, would be called before the ResponseParser
 	ResponseParser  func(res *http.Response) ([]json.RawMessage, errors.Error)                      // required, parse the response body and return a list of entities
 }