diff --git a/.golangci.yml b/.golangci.yml index 79693ccdb..4f41991ee 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -70,7 +70,6 @@ linters: - golint - gomnd - goprintffuncname - - gosec - gosimple - govet - ineffassign diff --git a/cmd/main.go b/cmd/main.go index c8607f78f..7a3a7492d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "strings" "github.com/checkmarxDev/ast-cli/internal/wrappers" @@ -11,43 +12,63 @@ import ( ) const ( - astSchema = "AST_SCHEMA" - astHost = "AST_HOST" - astPort = "80" - scansPath = "SCANS_PATH" - projectsPath = "PROJECTS_PATH" - resultsPath = "RESULTS_PATH" - uploadsPath = "UPLOADS_PATH" - logLevel = "CLI_LOG_LEVEL" + astSchemaEnv = "AST_SCHEMA" + astHostEnv = "AST_HOST" + astPortEnv = "AST_PORT" + scansPathEnv = "SCANS_PATH" + projectsPathEnv = "PROJECTS_PATH" + resultsPathEnv = "RESULTS_PATH" + uploadsPathEnv = "UPLOADS_PATH" + successfulExitCode = 0 failureExitCode = 1 ) func main() { - viper.AutomaticEnv() - viper.AddConfigPath(".") - viper.SetConfigName("config") - viper.SetConfigType("env") - _ = viper.ReadInConfig() - - viper.SetDefault(astSchema, "http") - viper.SetDefault(astHost, "localhost") - viper.SetDefault(astPort, "80") - viper.SetDefault(scansPath, "scans") - viper.SetDefault(projectsPath, "projects") - viper.SetDefault(uploadsPath, "uploads") - viper.SetDefault(resultsPath, "results") - viper.SetDefault(logLevel, "DEBUG") - - schema := viper.GetString(astSchema) - host := viper.GetString(astHost) - port := viper.GetString(astPort) - ast := fmt.Sprintf("%s://%s:%s/api", schema, host, port) + // Key ast_schema will be bound to AST_SCHEMA + astSchemaKey := strings.ToLower(astSchemaEnv) + err := bindKeyToEnvAndDefault(astSchemaKey, astSchemaEnv, "http") + exitIfError(err) + schema := viper.GetString(astSchemaKey) + + astHostKey := strings.ToLower(astHostEnv) + err = bindKeyToEnvAndDefault(astHostKey, astHostEnv, "localhost") + exitIfError(err) + host := viper.GetString(astHostKey) + + astPortKey := strings.ToLower(astPortEnv) + err = bindKeyToEnvAndDefault(astPortKey, astPortEnv, "80") + exitIfError(err) + port := viper.GetString(astPortKey) + + scansPathKey := strings.ToLower(scansPathEnv) + err = bindKeyToEnvAndDefault(scansPathKey, scansPathEnv, "scans") + exitIfError(err) + scans := viper.GetString(scansPathKey) - scans := viper.GetString(scansPath) - uploads := viper.GetString(uploadsPath) - projects := viper.GetString(projectsPath) - results := viper.GetString(resultsPath) + projectsPathKey := strings.ToLower(projectsPathEnv) + err = bindKeyToEnvAndDefault(projectsPathKey, projectsPathEnv, "projects") + exitIfError(err) + projects := viper.GetString(projectsPathKey) + + resultsPathKey := strings.ToLower(resultsPathEnv) + err = bindKeyToEnvAndDefault(resultsPathKey, resultsPathEnv, "results") + exitIfError(err) + results := viper.GetString(resultsPathKey) + + uploadsPathKey := strings.ToLower(uploadsPathEnv) + err = bindKeyToEnvAndDefault(uploadsPathKey, uploadsPathEnv, "uploads") + exitIfError(err) + uploads := viper.GetString(uploadsPathKey) + + err = bindKeyToEnvAndDefault(commands.AccessKeyIDConfigKey, commands.AccessKeyIDEnv, "") + exitIfError(err) + err = bindKeyToEnvAndDefault(commands.AccessKeySecretConfigKey, commands.AccessKeySecretEnv, "") + exitIfError(err) + err = bindKeyToEnvAndDefault(commands.AstAuthenticationHostConfigKey, commands.AstAuthenticationHostEnv, "") + exitIfError(err) + + ast := fmt.Sprintf("%s://%s:%s/api", schema, host, port) scansURL := fmt.Sprintf("%s/%s", ast, scans) uploadsURL := fmt.Sprintf("%s/%s", ast, uploads) @@ -60,12 +81,23 @@ func main() { resultsWrapper := wrappers.NewHTTPResultsWrapper(resultsURL) astCli := commands.NewAstCLI(scansWrapper, uploadsWrapper, projectsWrapper, resultsWrapper) - err := astCli.Execute() + + err = astCli.Execute() + exitIfError(err) + os.Exit(successfulExitCode) +} + +func exitIfError(err error) { if err != nil { fmt.Println(err.Error()) os.Exit(failureExitCode) } - os.Exit(successfulExitCode) +} + +func bindKeyToEnvAndDefault(key, env, defaultVal string) error { + err := viper.BindEnv(key, env) + viper.SetDefault(key, defaultVal) + return err } // When building an executable for Windows and providing a name, diff --git a/config.env b/config.env deleted file mode 100644 index 64c375a5e..000000000 --- a/config.env +++ /dev/null @@ -1,8 +0,0 @@ -AST_SCHEMA=http -AST_HOST=localhost -AST_PORT=80 -SCANS_PATH=scans -PROJECTS_PATH=projects -UPLOADS_PATH=uploads -RESULTS_PATH=results -CLI_LOG_LEVEL=DEBUG \ No newline at end of file diff --git a/internal/commands/project_test.go b/internal/commands/project_test.go index e0b028bd1..8f3084e4f 100644 --- a/internal/commands/project_test.go +++ b/internal/commands/project_test.go @@ -44,7 +44,7 @@ func TestRunCreateProjectCommandWithInput(t *testing.T) { func TestRunCreateProjectCommandWithInputBadFormat(t *testing.T) { cmd := createASTTestCommand() - err := executeTestCommand(cmd, "-v", "scan", "create", "--input", "[]") + err := executeTestCommand(cmd, "-v", "project", "create", "--input", "[]") assert.Assert(t, err != nil) } diff --git a/internal/commands/root.go b/internal/commands/root.go index 0f66314a3..9b1261abc 100644 --- a/internal/commands/root.go +++ b/internal/commands/root.go @@ -2,26 +2,42 @@ package commands import ( "fmt" + "strings" "github.com/checkmarxDev/ast-cli/internal/wrappers" "github.com/spf13/cobra" + "github.com/spf13/viper" ) const ( - verboseFlag = "verbose" - verboseFlagSh = "v" - sourcesFlag = "sources" - sourcesFlagSh = "s" - inputFlag = "input" - inputFlagSh = "i" - inputFileFlag = "inputFile" - inputFileFlagSh = "f" - limitFlag = "limit" - limitFlagSh = "l" - limitUsage = "The number of items to return" - offsetFlag = "offset" - offsetFlagSh = "o" - offsetUsage = "The number of items to skip before collecting the results" + verboseFlag = "verbose" + verboseFlagSh = "v" + verboseUsage = "Verbose mode" + sourcesFlag = "sources" + sourcesFlagSh = "s" + inputFlag = "input" + inputFlagSh = "i" + inputFileFlag = "inputFile" + inputFileFlagSh = "f" + limitFlag = "limit" + limitFlagSh = "l" + limitUsage = "The number of items to return" + offsetFlag = "offset" + offsetFlagSh = "o" + offsetUsage = "The number of items to skip before collecting the results" + AccessKeyIDEnv = "AST_ACCESS_KEY_ID" + accessKeyIDFlag = "key" + accessKeyIDFlagUsage = "The access key ID for AST" + AccessKeySecretEnv = "AST_ACCESS_KEY_SECRET" + accessKeySecretFlag = "secret" + accessKeySecretFlagUsage = "The access key secret for AST" + AstAuthenticationHostEnv = "AST_AUTHENTICATION_HOST" +) + +var ( + AccessKeyIDConfigKey = strings.ToLower(AccessKeyIDEnv) + AccessKeySecretConfigKey = strings.ToLower(AccessKeySecretEnv) + AstAuthenticationHostConfigKey = strings.ToLower(AstAuthenticationHostEnv) ) // Return an AST CLI root command to execute @@ -33,17 +49,43 @@ func NewAstCLI(scansWrapper wrappers.ScansWrapper, Use: "ast", Short: "A CLI wrapping Checkmarx AST APIs", } - rootCmd.PersistentFlags().BoolP(verboseFlag, verboseFlagSh, false, "Verbose mode") + + rootCmd.PersistentFlags().BoolP(verboseFlag, verboseFlagSh, false, verboseUsage) + rootCmd.PersistentFlags().String(accessKeyIDFlag, "", accessKeyIDFlagUsage) + rootCmd.PersistentFlags().String(accessKeySecretFlag, "", accessKeySecretFlagUsage) + + // Bind the viper key ast_access_key_id to flag --key of the root command and + // to the environment variable AST_ACCESS_KEY_ID so that it will be taken from environment variables first + // and can be overridden by command flag --key + _ = viper.BindPFlag(AccessKeyIDConfigKey, rootCmd.PersistentFlags().Lookup(accessKeyIDFlag)) + // Bind the viper key ast_access_key_secret to flag --secret of the root command and + // to the environment variable AST_ACCESS_KEY_SECRET so that it will be taken from environment variables first + // and can be overridden by command flag --secret + _ = viper.BindPFlag(AccessKeySecretConfigKey, rootCmd.PersistentFlags().Lookup(accessKeySecretFlag)) scanCmd := NewScanCommand(scansWrapper, uploadsWrapper) projectCmd := NewProjectCommand(projectsWrapper) resultCmd := NewResultCommand(resultsWrapper) versionCmd := NewVersionCommand() + rootCmd.AddCommand(scanCmd, projectCmd, resultCmd, versionCmd) rootCmd.SilenceUsage = true return rootCmd } +/* +func login(cmd *cobra.Command, args []string) error { + accessKeyID := viper.GetString(AccessKeyIDConfigKey) + accessKeySecret := viper.GetString(AccessKeySecretConfigKey) + authHost := viper.GetString(AstAuthenticationHostConfigKey) + var err error + + fmt.Println("Authenticating with:", authHost) + fmt.Println("Key is:", accessKeyID) + fmt.Println("Secret IS:", accessKeySecret) + return err +} +*/ func PrintIfVerbose(verbose bool, msg string) { if verbose { fmt.Println(msg) diff --git a/internal/commands/scan.go b/internal/commands/scan.go index bd999bc78..c70418b36 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -117,7 +117,7 @@ func runCreateScanCommand(scansWrapper wrappers.ScansWrapper, if sourcesFile != "" { // Send a request to uploads service var preSignedURL *string - preSignedURL, err = uploadsWrapper.Create(sourcesFile) + preSignedURL, err = uploadsWrapper.UploadFile(sourcesFile) if err != nil { return errors.Wrapf(err, "%s: Failed to upload sources file\n", failedCreating) } diff --git a/internal/wrappers/credentials.go b/internal/wrappers/credentials.go new file mode 100644 index 000000000..8a05019d7 --- /dev/null +++ b/internal/wrappers/credentials.go @@ -0,0 +1,7 @@ +package wrappers + +type Credentials struct { + AuthenticationHost string + AccessKeyID string + AccessKeySecret string +} diff --git a/internal/wrappers/projects-http.go b/internal/wrappers/projects-http.go index 5325b6379..e405281e6 100644 --- a/internal/wrappers/projects-http.go +++ b/internal/wrappers/projects-http.go @@ -19,6 +19,7 @@ const ( type ProjectsHTTPWrapper struct { url string contentType string + credentials *Credentials } func NewHTTPProjectsWrapper(url string) ProjectsWrapper { diff --git a/internal/wrappers/results-http.go b/internal/wrappers/results-http.go index d27ea29cd..449aef36a 100644 --- a/internal/wrappers/results-http.go +++ b/internal/wrappers/results-http.go @@ -14,6 +14,7 @@ const ( type ResultsHTTPWrapper struct { url string contentType string + credentials *Credentials } func NewHTTPResultsWrapper(url string) ResultsWrapper { @@ -24,7 +25,7 @@ func NewHTTPResultsWrapper(url string) ResultsWrapper { } func (r *ResultsHTTPWrapper) GetByScanID(scanID string, limit, offset uint64) ([]ResultResponseModel, *ResultError, error) { - resp, err := getRequestWithLimitAndOffset(r.url+"/scan"+scanID+"/items", limit, offset) + resp, err := getRequestWithLimitAndOffset(r.url+"/"+scanID+"/items", limit, offset) if err != nil { return nil, nil, err } diff --git a/internal/wrappers/results.go b/internal/wrappers/results.go index c95a9a59d..0d5b4fcbd 100644 --- a/internal/wrappers/results.go +++ b/internal/wrappers/results.go @@ -1,9 +1,5 @@ package wrappers -import ( - "math/big" -) - type ResultsWrapper interface { GetByScanID(scanID string, limit, offset uint64) ([]ResultResponseModel, *ResultError, error) } @@ -39,7 +35,7 @@ type ResultNode struct { type ResultResponseModel struct { // Query ID - QueryID int32 `json:"queryID,omitempty"` + QueryID string `json:"queryID,omitempty"` // Query name QueryName string `json:"queryName,omitempty"` // Query group; sperate by ':' @@ -47,15 +43,15 @@ type ResultResponseModel struct { // Severity of result Severity string `json:"severity,omitempty"` // Common Weakness Enumeration ID - CweID int32 `json:"cweID,omitempty"` + CweID string `json:"cweID,omitempty"` // ID of the path. changes from scan to scan. PathID int32 `json:"pathID,omitempty"` // ID of the Similarity feature (Indicator to identify a result by its first and last nodes) - SimilarityID int32 `json:"similarityID,omitempty"` + SimilarityID string `json:"similarityID,omitempty"` // Same as similarityID but can change in the future (SAST feature) - UniqueID int32 `json:"uniqueID,omitempty"` + UniqueID string `json:"uniqueID,omitempty"` // Confidence Level of the exsitin of the result - ConfidenceLevel big.Float `json:"confidenceLevel,omitempty"` + ConfidenceLevel int32 `json:"confidenceLevel,omitempty"` Nodes []ResultNode `json:"nodes,omitempty"` // ID of the customer tenant @@ -65,7 +61,7 @@ type ResultResponseModel struct { // Creation date of the result CreatedAt string `json:"createdAt,omitempty"` - Classification string `json:"classification,omitempty"` + Classification int32 `json:"classification,omitempty"` // Groups arrays Groups []string `json:"groups,omitempty"` // ID of the customer tenant @@ -73,7 +69,7 @@ type ResultResponseModel struct { // ID created from queryMetaInfo + similarityID + files name PathSystemIDBySimiAndFilesPaths string `json:"pathSystemIDBySimiAndFilesPaths,omitempty"` // enum of the current state(new,old,fixed) - Status string `json:"status,omitempty"` + Status int32 `json:"status,omitempty"` // TBD MetadataJSON string `json:"metadataJSON,omitempty"` // TBD diff --git a/internal/wrappers/scans-http.go b/internal/wrappers/scans-http.go index c74715bc1..90bc9f287 100644 --- a/internal/wrappers/scans-http.go +++ b/internal/wrappers/scans-http.go @@ -17,6 +17,7 @@ const ( type ScansHTTPWrapper struct { url string contentType string + credentials *Credentials } func (s *ScansHTTPWrapper) Create(model *scansApi.Scan) (*scansApi.ScanResponseModel, *scansApi.ErrorModel, error) { diff --git a/internal/wrappers/uploads-http.go b/internal/wrappers/uploads-http.go index b597e5d35..e386b8513 100644 --- a/internal/wrappers/uploads-http.go +++ b/internal/wrappers/uploads-http.go @@ -4,11 +4,9 @@ import ( "bytes" "encoding/json" "fmt" - "io" - "mime/multipart" + "io/ioutil" "net/http" "os" - "path/filepath" "time" uploads "github.com/checkmarxDev/uploads/api/rest/v1" @@ -20,14 +18,15 @@ const ( ) type UploadsHTTPWrapper struct { - url string + url string + credentials *Credentials } -func (u UploadsHTTPWrapper) Create(sourcesFile string) (*string, error) { - var body bytes.Buffer - - // Create a multipart writer - multiPartWriter := multipart.NewWriter(&body) +func (u *UploadsHTTPWrapper) UploadFile(sourcesFile string) (*string, error) { + preSignedURL, err := u.getPresignedURLForUploading() + if err != nil { + return nil, errors.Errorf("Failed creating pre-signed URL - %s", err.Error()) + } file, err := os.Open(sourcesFile) if err != nil { @@ -36,40 +35,54 @@ func (u UploadsHTTPWrapper) Create(sourcesFile string) (*string, error) { // Close the file later defer file.Close() - // Initialize the file field - var fileWriter io.Writer - sourcesFileName := filepath.Base(sourcesFile) - fileWriter, err = multiPartWriter.CreateFormFile("sources", sourcesFileName) + // read all of the contents of our uploaded file into a + // byte array + fileBytes, err := ioutil.ReadAll(file) if err != nil { - return nil, errors.Errorf("Failed creating FormFile - %s", err.Error()) + return nil, errors.Errorf("Failed to read file %s: %s", sourcesFile, err.Error()) } - // Copy the actual file content to the field field's writer - _, err = io.Copy(fileWriter, file) - if err != nil { - return nil, errors.Errorf("Failed to copy file: %s", err.Error()) - } - // We completed adding the file and the fields, let's close the multipart writer - // So it writes the ending boundary - multiPartWriter.Close() - var req *http.Request - req, err = http.NewRequest("POST", u.url, &body) + req, err = http.NewRequest("PUT", *preSignedURL, bytes.NewReader(fileBytes)) if err != nil { return nil, errors.Errorf("Requesting error model failed - %s", err.Error()) } - // We need to set the content type from the writer, it includes necessary boundary as well - req.Header.Set("Content-Type", multiPartWriter.FormDataContentType()) var client = &http.Client{ Timeout: time.Second * time.Duration(httpClientTimeout), } var resp *http.Response - fmt.Printf("Uploading file to %s\n", u.url) + fmt.Printf("Uploading file to %s\n", *preSignedURL) resp, err = client.Do(req) if err != nil { return nil, errors.Errorf("Invoking HTTP request failed - %s", err.Error()) } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + return preSignedURL, nil + default: + return nil, errors.Errorf("Unknown response status code %d", resp.StatusCode) + } +} + +func (u *UploadsHTTPWrapper) getPresignedURLForUploading() (*string, error) { + req, err := http.NewRequest("POST", u.url, nil) + if err != nil { + return nil, errors.Errorf("Requesting pre-signed URL failed - %s", err.Error()) + } + + var client = &http.Client{ + Timeout: time.Second * time.Duration(httpClientTimeout), + } + var resp *http.Response + + resp, err = client.Do(req) + if err != nil { + return nil, errors.Errorf("Invoking HTTP request to get pre-signed URL failed - %s", err.Error()) + } + defer resp.Body.Close() decoder := json.NewDecoder(resp.Body) diff --git a/internal/wrappers/uploads-mock.go b/internal/wrappers/uploads-mock.go index ec68a2eeb..2d51b6098 100644 --- a/internal/wrappers/uploads-mock.go +++ b/internal/wrappers/uploads-mock.go @@ -5,7 +5,7 @@ import "fmt" type UploadsMockWrapper struct { } -func (u *UploadsMockWrapper) Create(sourcesFile string) (*string, error) { +func (u *UploadsMockWrapper) UploadFile(sourcesFile string) (*string, error) { fmt.Println("Called Create in UploadsMockWrapper") url := "/path/to/nowhere" return &url, nil diff --git a/internal/wrappers/uploads.go b/internal/wrappers/uploads.go index aa283781a..79687786d 100644 --- a/internal/wrappers/uploads.go +++ b/internal/wrappers/uploads.go @@ -1,5 +1,5 @@ package wrappers type UploadsWrapper interface { - Create(sourcesFile string) (*string, error) + UploadFile(sourcesFile string) (*string, error) } diff --git a/test/integration/result_test.go b/test/integration/result_test.go new file mode 100644 index 000000000..2f341045a --- /dev/null +++ b/test/integration/result_test.go @@ -0,0 +1,39 @@ +// +build integration + +package integration + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "strconv" + "testing" + + "github.com/checkmarxDev/ast-cli/internal/wrappers" + "gotest.tools/assert" +) + +const ( + numOfFullScanResults = 575 + numOfIncScanResults = 572 +) + +func getResultsNumberForScan(t *testing.T, scanID string) int { + b := bytes.NewBufferString("") + getResultsCmd := createASTIntegrationTestCommand() + getResultsCmd.SetOut(b) + var limit uint64 = 600 + var offset uint64 = 0 + l := strconv.FormatUint(limit, 10) + o := strconv.FormatUint(offset, 10) + err := execute(getResultsCmd, "-v", "result", "get", scanID, "--limit", l, "--offset", o) + assert.NilError(t, err, "Getting all results should pass") + // Read response from buffer + var getAllJSON []byte + getAllJSON, err = ioutil.ReadAll(b) + assert.NilError(t, err, "Reading all results response JSON should pass") + allResults := []wrappers.ResultResponseModel{} + err = json.Unmarshal(getAllJSON, &allResults) + assert.NilError(t, err, "Parsing all results response JSON should pass") + return len(allResults) +} diff --git a/test/integration/root_test.go b/test/integration/root_test.go index 20b2025f8..275af09c6 100644 --- a/test/integration/root_test.go +++ b/test/integration/root_test.go @@ -57,7 +57,7 @@ func createASTIntegrationTestCommand() *cobra.Command { viper.SetDefault(scansPath, "scans") viper.SetDefault(projectsPath, "projects") viper.SetDefault(uploadsPath, "uploads") - viper.SetDefault(resultsPath, "results") + viper.SetDefault(resultsPath, "scan") schema := viper.GetString(astSchema) host := viper.GetString(astHost) diff --git a/test/integration/scan_test.go b/test/integration/scan_test.go index 5a4fd50c8..b10f4394c 100644 --- a/test/integration/scan_test.go +++ b/test/integration/scan_test.go @@ -7,14 +7,15 @@ import ( "context" "encoding/json" "fmt" - scansRESTApi "github.com/checkmarxDev/scans/api/v1/rest/scans" - "gotest.tools/assert/cmp" "io/ioutil" "log" "strconv" "testing" "time" + scansRESTApi "github.com/checkmarxDev/scans/api/v1/rest/scans" + "gotest.tools/assert/cmp" + "github.com/spf13/viper" "gotest.tools/assert" ) @@ -33,6 +34,9 @@ func TestScansE2E(t *testing.T) { scanCompleted := <-scanCompletedCh assert.Assert(t, scanCompleted, "Full scan should be completed") + // Validate the results for full scan + scanResults := getResultsNumberForScan(t, scanID) + assert.Assert(t, scanResults == numOfFullScanResults, "Wrong number of full scan results") incScanID := createIncScan(t) log.Printf("Waiting %d seconds for the incremental scan to complete...\n", incScanWaitTime) // Wait for the inc scan to finish. See it's completed successfully @@ -41,6 +45,10 @@ func TestScansE2E(t *testing.T) { incScanCompleted := <-incScanCompletedCh assert.Assert(t, incScanCompleted, "Incremental scan should be completed") + // Validate the results for inc scan + incScanResults := getResultsNumberForScan(t, incScanID) + assert.Assert(t, incScanResults == numOfIncScanResults, "Wrong number of inc scan results") + getAllScans(t) getScansTags(t) } @@ -152,6 +160,9 @@ func pollScanUntilStatus(t *testing.T, scanID string, ch chan<- bool, requiredSt if string(scan.Status) == string(requiredStatus) { ch <- true return + } else if string(scan.Status) == scansRESTApi.ScanFailed || string(scan.Status) == scansRESTApi.ScanCanceled { + ch <- false + return } else { time.Sleep(time.Duration(sleep) * time.Second) }