From e8055788bd9eb0f324e07125cf2d3dfecf370ddc Mon Sep 17 00:00:00 2001 From: Kris Coleman Date: Mon, 14 Apr 2025 22:03:43 -0400 Subject: [PATCH 1/2] feat: introduce dagger for integration testing and ci Signed-off-by: Kris Coleman --- .github/workflows/pr-test.yml | 11 +++-- Makefile | 5 ++ go.mod | 7 +++ go.sum | 21 ++++++++ test/csharp-integration/README.md | 49 +++++++++++++++++++ test/csharp-integration/dagger.go | 80 +++++++++++++++++++++++++++++++ 6 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 test/csharp-integration/README.md create mode 100644 test/csharp-integration/dagger.go diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 050257d..cbebe3f 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -56,8 +56,11 @@ jobs: with: go-version-file: 'go.mod' - - name: Set up Docker - uses: docker/setup-buildx-action@v2 + - name: Install Dagger CLI + run: | + cd /usr/local + curl -L https://dl.dagger.io/dagger/install.sh | sh + dagger version - - name: 'Run C# integration test' - run: ./test/csharp-integration/test-compilation.sh \ No newline at end of file + - name: 'Run C# integration test with Dagger' + run: make test-csharp-dagger \ No newline at end of file diff --git a/Makefile b/Makefile index 2b68a76..e939a1b 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,11 @@ test-csharp: @echo "Running C# integration test..." @./test/csharp-integration/test-compilation.sh +.PHONY: test-csharp-dagger +test-csharp-dagger: + @echo "Running C# integration test with Dagger..." + @cd test/csharp-integration && go run dagger.go + generate-docs: @echo "Generating documentation..." @go run ./docs/generate-commands.go diff --git a/go.mod b/go.mod index 7178c5a..bb80b94 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,10 @@ require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect atomicgo.dev/schedule v0.1.0 // indirect + dagger.io/dagger v0.10.2 // indirect + github.com/99designs/gqlgen v0.17.31 // indirect + github.com/Khan/genqlient v0.6.0 // indirect + github.com/adrg/xdg v0.4.0 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/containerd/console v1.0.3 // indirect @@ -34,6 +38,7 @@ require ( github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -42,12 +47,14 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/vektah/gqlparser/v2 v2.5.6 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/term v0.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 993563b..eff2013 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,12 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8= atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ= atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs= atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= +dagger.io/dagger v0.10.2 h1:8q4AwKm48qAWZdY7O2NMZ3FveteDHJeM3WwmcNKZOM0= +dagger.io/dagger v0.10.2/go.mod h1:AonAYX6ZXNGsVvro4HhB/Uzsp9FU+aI41YfAEY9f5mI= +github.com/99designs/gqlgen v0.17.31 h1:VncSQ82VxieHkea8tz11p7h/zSbvHSxSDZfywqWt158= +github.com/99designs/gqlgen v0.17.31/go.mod h1:i4rEatMrzzu6RXaHydq1nmEPZkb3bKQsnxNRHS4DQB4= +github.com/Khan/genqlient v0.6.0 h1:Bwb1170ekuNIVIwTJEqvO8y7RxBxXu639VJOkKSrwAk= +github.com/Khan/genqlient v0.6.0/go.mod h1:rvChwWVTqXhiapdhLDV4bp9tz/Xvtewwkon4DpWWCRM= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII= @@ -15,6 +21,11 @@ github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/ github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE= github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4= github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= @@ -28,6 +39,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= @@ -66,6 +78,8 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -91,6 +105,7 @@ github.com/sagikazarmark/locafero v0.8.0 h1:mXaMVw7IqxNBxfv3LdWt9MDmcWDQ1fagDH91 github.com/sagikazarmark/locafero v0.8.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= @@ -113,6 +128,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/vektah/gqlparser/v2 v2.5.6 h1:Ou14T0N1s191eRMZ1gARVqohcbe1e8FrcONScsq8cRU= +github.com/vektah/gqlparser/v2 v2.5.6/go.mod h1:z8xXUff237NntSuH8mLFijZ+1tjV1swDbpDqjJmk6ME= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= @@ -140,12 +157,15 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -176,6 +196,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/test/csharp-integration/README.md b/test/csharp-integration/README.md new file mode 100644 index 0000000..798004a --- /dev/null +++ b/test/csharp-integration/README.md @@ -0,0 +1,49 @@ +# C# Integration Testing with Dagger + +This directory contains integration tests for the C# code generator. + +## Running the tests + +You can run the C# integration tests in two ways: + +### Using Dagger (Recommended) + +```bash +make test-csharp-dagger +``` + +This will: +1. Build the OpenFeature CLI +2. Generate C# client code using the sample manifest +3. Run the C# compilation test in an isolated environment +4. Report success or failure + +### Using the legacy shell script + +```bash +make test-csharp +``` + +or directly: + +```bash +./test/csharp-integration/test-compilation.sh +``` + +The shell script version uses Docker directly and will be deprecated in the future. + +## What the test does + +The integration test: +1. Builds the OpenFeature CLI +2. Generates C# client code using a sample manifest +3. Compiles the generated code with a sample program +4. Runs the compiled program to verify it works correctly + +## Implementation details + +The Dagger implementation (`dagger.go`) creates a pipeline that: +1. Builds the CLI in a Go container +2. Generates C# code using the CLI +3. Compiles and runs the C# code in a .NET SDK container +4. Reports success based on the exit code \ No newline at end of file diff --git a/test/csharp-integration/dagger.go b/test/csharp-integration/dagger.go new file mode 100644 index 0000000..8dfc089 --- /dev/null +++ b/test/csharp-integration/dagger.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "dagger.io/dagger" +) + +func main() { + ctx := context.Background() + + // Initialize Dagger client + client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stdout)) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to connect to Dagger engine: %v\n", err) + os.Exit(1) + } + defer client.Close() + + // Get project root directory + projectDir, err := filepath.Abs("../..") + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get project dir: %v\n", err) + os.Exit(1) + } + + // Get integration test directory + testDir, err := filepath.Abs(".") + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get test dir: %v\n", err) + os.Exit(1) + } + + // Source code container + source := client.Host().Directory(projectDir) + + // Build the CLI + cli := client.Container(). + From("golang:1.21-alpine"). + WithDirectory("/src", source). + WithWorkdir("/src"). + WithExec([]string{"go", "build", "-o", "cli"}) + + // Generate C# client + generated := cli.WithExec([]string{ + "./cli", "generate", "csharp", + "--manifest=/src/sample/sample_manifest.json", + "--output=/tmp/generated", + "--namespace=TestNamespace", + }) + + // Get generated files + generatedFiles := generated.Directory("/tmp/generated") + + // Test C# compilation with the generated files + dotnetContainer := client.Container(). + From("mcr.microsoft.com/dotnet/sdk:8.0"). + WithDirectory("/app/expected", generatedFiles). + WithDirectory("/app/test", client.Host().Directory(testDir, dagger.HostDirectoryOpts{ + Include: []string{"CompileTest.csproj", "Program.cs"}, + })). + WithWorkdir("/app"). + WithExec([]string{"cp", "/app/test/CompileTest.csproj", "."}). + WithExec([]string{"cp", "/app/test/Program.cs", "."}). + WithExec([]string{"dotnet", "restore"}). + WithExec([]string{"dotnet", "build"}). + WithExec([]string{"dotnet", "run"}) + + // Execute the pipeline + _, err = dotnetContainer.ExitCode(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "Pipeline failed: %v\n", err) + os.Exit(1) + } + + fmt.Println("=== Success: C# code compiles and executes correctly ===") +} \ No newline at end of file From 4d6e0617aa8cff0e54511c3082c9b0690906cbc1 Mon Sep 17 00:00:00 2001 From: Kris Coleman Date: Mon, 14 Apr 2025 23:21:18 -0400 Subject: [PATCH 2/2] feat: make integration testing extensible and modular my intention here is to make the pattern for integration testing more accessible. let's abstract the integration test pattern itself into a framework to harness the boiler plate. then reuse our csharp integration test code as our first integration-test. Signed-off-by: Kris Coleman --- .github/workflows/pr-test.yml | 25 ++- CONTRIBUTING.md | 28 +-- Makefile | 17 +- test/README.md | 65 +++++++ test/csharp-integration/CompileTest.csproj | 6 + test/csharp-integration/README.md | 42 ++--- test/csharp-integration/dagger.go | 80 --------- test/csharp-integration/test-compilation.sh | 34 ---- test/integration/cmd/csharp/run.go | 97 ++++++++++ test/integration/cmd/run.go | 26 +++ test/integration/integration.go | 44 +++++ test/new-generator.md | 190 ++++++++++++++++++++ 12 files changed, 485 insertions(+), 169 deletions(-) create mode 100644 test/README.md delete mode 100644 test/csharp-integration/dagger.go delete mode 100755 test/csharp-integration/test-compilation.sh create mode 100644 test/integration/cmd/csharp/run.go create mode 100644 test/integration/cmd/run.go create mode 100644 test/integration/integration.go create mode 100644 test/new-generator.md diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index cbebe3f..a8a51f3 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -44,23 +44,22 @@ jobs: git diff exit 1 fi - - csharp-test: - name: 'C# Generator Test' + + integration-tests: + name: 'Generator Integration Tests' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - + - name: Set up Go uses: actions/setup-go@v5 with: go-version-file: 'go.mod' - - - name: Install Dagger CLI - run: | - cd /usr/local - curl -L https://dl.dagger.io/dagger/install.sh | sh - dagger version - - - name: 'Run C# integration test with Dagger' - run: make test-csharp-dagger \ No newline at end of file + + - name: Run all integration tests with Dagger + uses: dagger/dagger-for-github@v5 + with: + workdir: . + verb: run + args: go run ./test/integration/cmd/run.go + version: 'latest' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0359bd3..0ed8726 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,23 +36,31 @@ We welcome contributions for new generators to extend the functionality of the O 11. **Address Feedback**: Be responsive to feedback from the maintainers. Make any necessary changes and update your pull request as needed. -### Integration Tests +### Testing -To verify that generated code compiles correctly, the project includes integration tests, for example, for c#: +The OpenFeature CLI includes both unit and integration tests to ensure quality and correctness. + +#### Unit Tests + +Run the unit tests with: ```bash -# Test the C# generator output -make test-csharp +go test ./... ``` -This will: +#### Integration Tests + +To verify that generated code compiles correctly, the project includes integration tests. The CLI uses a Dagger-based integration testing framework to test code generation for each supported language: -1. Build the CLI -2. Generate a C# client -3. Compile the C# code in a Docker container -4. Validate that the code compiles correctly +```bash +# Run all integration tests +make test-integration + +# Run tests for a specific language +make test-csharp-dagger +``` -Consider adding more integration tests for new generators. +For more information on the integration testing framework, see [Integration Testing](./docs/integration-testing.md). ## Setting Up Lefthook diff --git a/Makefile b/Makefile index e939a1b..78fb0ed 100644 --- a/Makefile +++ b/Makefile @@ -4,15 +4,16 @@ test: @go test -v ./... @echo "Tests passed successfully!" -.PHONY: test-csharp -test-csharp: - @echo "Running C# integration test..." - @./test/csharp-integration/test-compilation.sh - -.PHONY: test-csharp-dagger -test-csharp-dagger: +# Dagger-based integration tests +.PHONY: test-integration-csharp +test-integration-csharp: @echo "Running C# integration test with Dagger..." - @cd test/csharp-integration && go run dagger.go + @go run ./test/integration/cmd/csharp/run.go + +.PHONY: test-integration +test-integration: + @echo "Running all integration tests with Dagger..." + @go run ./test/integration/cmd/run.go generate-docs: @echo "Generating documentation..." diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..6d11ebb --- /dev/null +++ b/test/README.md @@ -0,0 +1,65 @@ +# OpenFeature CLI Integration Testing + +This directory contains integration tests for validating the OpenFeature CLI generators. + +## Integration Test Structure + +The integration tests use [Dagger](https://dagger.io/) to create reproducible test environments without needing to install dependencies locally. + +Each integration test: + +1. Builds the CLI from source +2. Generates code using a sample manifest file +3. Compiles and tests the generated code in a language-specific container +4. Reports success or failure + +## Running Tests + +### Run all integration tests + +```bash +make test-integration +``` + +### Run a specific integration test + +```bash +# For C# tests +make test-csharp-dagger +``` + +## Adding a New Integration Test + +To add an integration test for a new generator: + +1. Create a combined implementation and runner file in `test/integration/cmd//run.go` +2. Update the main runner in `test/integration/cmd/run.go` to execute your new test +3. Add a Makefile target for running your test individually + +See the step-by-step guide in [new-language.md](new-language.md) for detailed instructions. + +## How It Works + +The testing framework uses the following components: + +- `test/integration/integration.go`: Defines the `Test` interface and common utilities +- `test/integration/cmd/run.go`: Runner for all integration tests that executes each language-specific test +- `test/integration/cmd//run.go`: Combined implementation and runner for each language +- `test/-integration/`: Contains language-specific test files (code samples, project files) + +Each integration test uses Dagger to: + +1. Build the CLI in a clean environment +2. Generate code using a sample manifest +3. Compile and test the generated code in a language-specific container +4. Report success or failure + +## Benefits Over Shell Scripts + +Using Dagger for integration tests provides several advantages: + +1. **Reproducibility**: Tests run in containerized environments that are identical locally and in CI +2. **Language Support**: Easy to add new language tests with the same pattern +3. **Improved Debugging**: Clear separation of build, generate, and test steps +4. **Parallelization**: Tests can run in parallel when executed in different containers +5. **No Dependencies**: No need to install language-specific tooling locally diff --git a/test/csharp-integration/CompileTest.csproj b/test/csharp-integration/CompileTest.csproj index 98c3cbb..43c7d33 100644 --- a/test/csharp-integration/CompileTest.csproj +++ b/test/csharp-integration/CompileTest.csproj @@ -5,6 +5,7 @@ net8.0 enable enable + false @@ -12,4 +13,9 @@ + + + + + \ No newline at end of file diff --git a/test/csharp-integration/README.md b/test/csharp-integration/README.md index 798004a..cbdfe77 100644 --- a/test/csharp-integration/README.md +++ b/test/csharp-integration/README.md @@ -1,12 +1,10 @@ -# C# Integration Testing with Dagger +# C# Integration Testing This directory contains integration tests for the C# code generator. ## Running the tests -You can run the C# integration tests in two ways: - -### Using Dagger (Recommended) +Run the C# integration tests with Dagger: ```bash make test-csharp-dagger @@ -18,32 +16,28 @@ This will: 3. Run the C# compilation test in an isolated environment 4. Report success or failure -### Using the legacy shell script - -```bash -make test-csharp -``` - -or directly: - -```bash -./test/csharp-integration/test-compilation.sh -``` - -The shell script version uses Docker directly and will be deprecated in the future. - ## What the test does The integration test: -1. Builds the OpenFeature CLI +1. Builds the OpenFeature CLI inside a container 2. Generates C# client code using a sample manifest 3. Compiles the generated code with a sample program 4. Runs the compiled program to verify it works correctly -## Implementation details +## Test Files + +- `CompileTest.csproj`: .NET project file for compilation testing +- `Program.cs`: Test program that uses the generated code +- `expected/`: Directory containing expected output files (used for verification) + +## Implementation -The Dagger implementation (`dagger.go`) creates a pipeline that: -1. Builds the CLI in a Go container +The C# integration test uses Dagger to create a reproducible test environment: + +1. It builds the CLI in a Go container 2. Generates C# code using the CLI -3. Compiles and runs the C# code in a .NET SDK container -4. Reports success based on the exit code \ No newline at end of file +3. Tests the generated code in a .NET container + +The implementation is located in `test/integration/cmd/csharp/run.go`. + +For more implementation details, see the main [test/README.md](../README.md) file. \ No newline at end of file diff --git a/test/csharp-integration/dagger.go b/test/csharp-integration/dagger.go deleted file mode 100644 index 8dfc089..0000000 --- a/test/csharp-integration/dagger.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "path/filepath" - - "dagger.io/dagger" -) - -func main() { - ctx := context.Background() - - // Initialize Dagger client - client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stdout)) - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to connect to Dagger engine: %v\n", err) - os.Exit(1) - } - defer client.Close() - - // Get project root directory - projectDir, err := filepath.Abs("../..") - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to get project dir: %v\n", err) - os.Exit(1) - } - - // Get integration test directory - testDir, err := filepath.Abs(".") - if err != nil { - fmt.Fprintf(os.Stderr, "Failed to get test dir: %v\n", err) - os.Exit(1) - } - - // Source code container - source := client.Host().Directory(projectDir) - - // Build the CLI - cli := client.Container(). - From("golang:1.21-alpine"). - WithDirectory("/src", source). - WithWorkdir("/src"). - WithExec([]string{"go", "build", "-o", "cli"}) - - // Generate C# client - generated := cli.WithExec([]string{ - "./cli", "generate", "csharp", - "--manifest=/src/sample/sample_manifest.json", - "--output=/tmp/generated", - "--namespace=TestNamespace", - }) - - // Get generated files - generatedFiles := generated.Directory("/tmp/generated") - - // Test C# compilation with the generated files - dotnetContainer := client.Container(). - From("mcr.microsoft.com/dotnet/sdk:8.0"). - WithDirectory("/app/expected", generatedFiles). - WithDirectory("/app/test", client.Host().Directory(testDir, dagger.HostDirectoryOpts{ - Include: []string{"CompileTest.csproj", "Program.cs"}, - })). - WithWorkdir("/app"). - WithExec([]string{"cp", "/app/test/CompileTest.csproj", "."}). - WithExec([]string{"cp", "/app/test/Program.cs", "."}). - WithExec([]string{"dotnet", "restore"}). - WithExec([]string{"dotnet", "build"}). - WithExec([]string{"dotnet", "run"}) - - // Execute the pipeline - _, err = dotnetContainer.ExitCode(ctx) - if err != nil { - fmt.Fprintf(os.Stderr, "Pipeline failed: %v\n", err) - os.Exit(1) - } - - fmt.Println("=== Success: C# code compiles and executes correctly ===") -} \ No newline at end of file diff --git a/test/csharp-integration/test-compilation.sh b/test/csharp-integration/test-compilation.sh deleted file mode 100755 index c1fd4fe..0000000 --- a/test/csharp-integration/test-compilation.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -e - -# Script to test if the generated C# code compiles correctly -SCRIPT_DIR=$(dirname "$0") -CLI_ROOT=$(realpath "$SCRIPT_DIR/../..") -OUTPUT_DIR=$(realpath "$SCRIPT_DIR") - -echo "=== Building OpenFeature CLI ===" -cd "$CLI_ROOT" -go build - -echo "=== Generating C# client ===" -./cli generate csharp --manifest="$CLI_ROOT/sample/sample_manifest.json" --output="$OUTPUT_DIR/expected" --namespace="TestNamespace" - -if [ ! -f "$OUTPUT_DIR/expected/OpenFeature.cs" ]; then - echo "Error: OpenFeature.cs was not generated" - exit 1 -fi - -echo "=== Building Docker image to compile C# code ===" -cd "$OUTPUT_DIR" -docker build -t openfeature-csharp-test . - -echo "=== Testing C# compilation and execution ===" -docker run --rm openfeature-csharp-test - -if [ $? -eq 0 ]; then - echo "=== Success: C# code compiles and executes correctly ===" - exit 0 -else - echo "=== Error: C# code fails to compile or execute ===" - exit 1 -fi \ No newline at end of file diff --git a/test/integration/cmd/csharp/run.go b/test/integration/cmd/csharp/run.go new file mode 100644 index 0000000..3c245db --- /dev/null +++ b/test/integration/cmd/csharp/run.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "dagger.io/dagger" + "github.com/open-feature/cli/test/integration" +) + +// Test implements the integration test for the C# generator +type Test struct { + // ProjectDir is the absolute path to the root of the project + ProjectDir string + // TestDir is the absolute path to the test directory + TestDir string +} + +// New creates a new Test +func New(projectDir, testDir string) *Test { + return &Test{ + ProjectDir: projectDir, + TestDir: testDir, + } +} + +// Run executes the C# integration test using Dagger +func (t *Test) Run(ctx context.Context, client *dagger.Client) (*dagger.Container, error) { + // Source code container + source := client.Host().Directory(t.ProjectDir) + testFiles := client.Host().Directory(t.TestDir, dagger.HostDirectoryOpts{ + Include: []string{"CompileTest.csproj", "Program.cs"}, + }) + + // Build the CLI + cli := client.Container(). + From("golang:1.24-alpine"). + WithDirectory("/src", source). + WithWorkdir("/src"). + WithExec([]string{"go", "build", "-o", "cli"}) + + // Generate C# client + generated := cli.WithExec([]string{ + "./cli", "generate", "csharp", + "--manifest=/src/sample/sample_manifest.json", + "--output=/tmp/generated", + "--namespace=TestNamespace", + }) + + // Get generated files + generatedFiles := generated.Directory("/tmp/generated") + + // Test C# compilation with the generated files + dotnetContainer := client.Container(). + From("mcr.microsoft.com/dotnet/sdk:8.0"). + WithDirectory("/app/generated", generatedFiles). + WithDirectory("/app", testFiles). + WithWorkdir("/app"). + WithExec([]string{"dotnet", "restore"}). + WithExec([]string{"dotnet", "build"}). + WithExec([]string{"dotnet", "run"}) + + return dotnetContainer, nil +} + +// Name returns the name of the integration test +func (t *Test) Name() string { + return "csharp" +} + +func main() { + ctx := context.Background() + + // Get project root + projectDir, err := filepath.Abs(os.Getenv("PWD")) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get project dir: %v\n", err) + os.Exit(1) + } + + // Get test directory + testDir, err := filepath.Abs(filepath.Join(projectDir, "test/csharp-integration")) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get test dir: %v\n", err) + os.Exit(1) + } + + // Create and run the C# integration test + test := New(projectDir, testDir) + + if err := integration.RunTest(ctx, test); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} diff --git a/test/integration/cmd/run.go b/test/integration/cmd/run.go new file mode 100644 index 0000000..0306461 --- /dev/null +++ b/test/integration/cmd/run.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "os" + "os/exec" +) + +func main() { + + // Run the language-specific tests + fmt.Println("=== Running all integration tests ===") + + // Run the C# integration test + csharpCmd := exec.Command("go", "run", "github.com/open-feature/cli/test/integration/cmd/csharp") + csharpCmd.Stdout = os.Stdout + csharpCmd.Stderr = os.Stderr + if err := csharpCmd.Run(); err != nil { + fmt.Fprintf(os.Stderr, "Error running C# integration test: %v\n", err) + os.Exit(1) + } + + // Add more tests here as they are available + + fmt.Println("=== All integration tests passed successfully ===") +} diff --git a/test/integration/integration.go b/test/integration/integration.go new file mode 100644 index 0000000..f852e48 --- /dev/null +++ b/test/integration/integration.go @@ -0,0 +1,44 @@ +package integration + +import ( + "context" + "fmt" + "os" + + "dagger.io/dagger" +) + +// Test defines the interface for all integration tests +type Test interface { + // Run executes the integration test with the given Dagger client + Run(ctx context.Context, client *dagger.Client) (*dagger.Container, error) + // Name returns the name of the integration test + Name() string +} + +// RunTest runs a single integration test +func RunTest(ctx context.Context, test Test) error { + // Initialize Dagger client + client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stdout)) + if err != nil { + return fmt.Errorf("failed to connect to Dagger engine: %w", err) + } + defer client.Close() + + fmt.Printf("=== Running %s integration test ===\n", test.Name()) + + // Run the integration test + container, err := test.Run(ctx, client) + if err != nil { + return fmt.Errorf("failed to run %s integration test: %w", test.Name(), err) + } + + // Execute the pipeline and wait for it to complete + _, err = container.Stdout(ctx) + if err != nil { + return fmt.Errorf("%s integration test failed: %w", test.Name(), err) + } + + fmt.Printf("=== Success: %s integration test passed ===\n", test.Name()) + return nil +} diff --git a/test/new-generator.md b/test/new-generator.md new file mode 100644 index 0000000..6f68719 --- /dev/null +++ b/test/new-generator.md @@ -0,0 +1,190 @@ +# Adding a New Generator Integration Test + +This guide explains how to add integration tests for a new generator. + +## Directory Structure + +The integration testing framework has the following directory structure: + +``` +test/ + integration/ # Core integration test framework + integration.go # Test interface definition + cmd/ # Command-line runners and implementations + run.go # Runner for all tests + csharp/ # C# specific implementation and runner + run.go + python/ # Python specific implementation and runner (future) + run.go + csharp-integration/ # C# test files + python-integration/ # Python test files (future) +``` + +## Step 1: Create a generator-specific implementation and runner + +Create a file at `test/integration/cmd/python/run.go`: + +```go +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "dagger.io/dagger" + "github.com/open-feature/cli/test/integration" +) + +// Test implements the integration test for the Python generator +type Test struct { + ProjectDir string + TestDir string +} + +// New creates a new Test +func New(projectDir, testDir string) *Test { + return &Test{ + ProjectDir: projectDir, + TestDir: testDir, + } +} + +// Run executes the Python integration test +func (t *Test) Run(ctx context.Context, client *dagger.Client) (*dagger.Container, error) { + // Source code container + source := client.Host().Directory(t.ProjectDir) + testFiles := client.Host().Directory(t.TestDir, dagger.HostDirectoryOpts{ + Include: []string{"test_openfeature.py", "requirements.txt"}, + }) + + // Build the CLI + cli := client.Container(). + From("golang:1.24-alpine"). + WithDirectory("/src", source). + WithWorkdir("/src"). + WithExec([]string{"go", "build", "-o", "cli"}) + + // Generate Python client + generated := cli.WithExec([]string{ + "./cli", "generate", "python", + "--manifest=/src/sample/sample_manifest.json", + "--output=/tmp/generated", + "--package=openfeature_test", + }) + + // Get generated files + generatedFiles := generated.Directory("/tmp/generated") + + // Test Python with the generated files + pythonContainer := client.Container(). + From("python:3.11-slim"). + WithDirectory("/app/openfeature", generatedFiles). + WithDirectory("/app/test", testFiles). + WithWorkdir("/app"). + WithExec([]string{"pip", "install", "-r", "test/requirements.txt"}). + WithExec([]string{"python", "-m", "pytest", "test/test_openfeature.py", "-v"}) + + return pythonContainer, nil +} + +// Name returns the name of the integration test +func (t *Test) Name() string { + return "python" +} + +func main() { + ctx := context.Background() + + // Get project root + projectDir, err := filepath.Abs(os.Getenv("PWD")) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get project dir: %v\n", err) + os.Exit(1) + } + + // Get test directory + testDir, err := filepath.Abs(filepath.Join(projectDir, "test/python-integration")) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get test dir: %v\n", err) + os.Exit(1) + } + + // Create and run the Python integration test + test := New(projectDir, testDir) + + if err := integration.RunTest(ctx, test); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} +``` + +## Step 2: Add the test to the all-integration runner + +Update `test/integration/cmd/run.go` to include your test: + +```go +package main + +import ( + "fmt" + "os" + "os/exec" +) + +func main() { + // Run the generator-specific tests + fmt.Println("=== Running all integration tests ===") + + // Run the C# integration test + csharpCmd := exec.Command("go", "run", "github.com/open-feature/cli/test/integration/cmd/csharp") + csharpCmd.Stdout = os.Stdout + csharpCmd.Stderr = os.Stderr + if err := csharpCmd.Run(); err != nil { + fmt.Fprintf(os.Stderr, "Error running C# integration test: %v\n", err) + os.Exit(1) + } + + // Run the Python integration test + pythonCmd := exec.Command("go", "run", "github.com/open-feature/cli/test/integration/cmd/python") + pythonCmd.Stdout = os.Stdout + pythonCmd.Stderr = os.Stderr + if err := pythonCmd.Run(); err != nil { + fmt.Fprintf(os.Stderr, "Error running Python integration test: %v\n", err) + os.Exit(1) + } + + // Add more tests here as they are available + + fmt.Println("=== All integration tests passed successfully ===") +} +``` + +## Step 3: Create test files + +Create the following directory structure with your test files: + +``` +test/ + python-integration/ + requirements.txt + test_openfeature.py + README.md +``` + +## Step 4: Add a Makefile target + +Update the Makefile with a new target: + +```makefile +.PHONY: test-python-dagger +test-python-dagger: + @echo "Running Python integration test with Dagger..." + @go run ./test/integration/cmd/python/run.go +``` + +## Step 5: Update the documentation + +Update `test/README.md` to include your new test. \ No newline at end of file