[WIP] Node image builds for 3.10 #1224
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| { | ||
| "tag_specification": { | ||
| "cluster": "https://api.ci.openshift.org", | ||
| "namespace": "openshift", | ||
| "name": "origin-v3.10", | ||
| "tag": "", | ||
| "tag_overrides": {} | ||
| }, | ||
| "base_images": { | ||
| "base": { | ||
| "cluster": "https://api.ci.openshift.org", | ||
| "namespace": "azure", | ||
| "name": "plugin-base", | ||
| "tag": "latest" | ||
| } | ||
| }, | ||
| "test_base_image": { | ||
| "cluster": "https://api.ci.openshift.org", | ||
| "namespace": "azure", | ||
| "name": "test-base", | ||
| "tag": "latest" | ||
| }, | ||
|
|
||
| "canonical_go_repository": "github.com/openshift/openshift-azure", | ||
|
|
||
| "binary_build_commands": "make build sync", | ||
|
|
||
| "images": [ | ||
| { | ||
| "from": "base", | ||
| "to": "sync", | ||
| "dockerfile_path": "Dockerfile.sync", | ||
| "inputs": { | ||
| "bin": {"paths": [{"source_path": "/go/src/github.com/openshift/openshift-azure/sync", "destination_dir": "."}]}, | ||
| "root": {"as": ["sync"]} | ||
| } | ||
| } | ||
| ], | ||
|
|
||
| "tests": [ | ||
| { | ||
| "as": "verify", | ||
| "from": "src", | ||
| "commands": "make verify" | ||
| }, | ||
| { | ||
| "as": "unit", | ||
| "from": "src", | ||
| "commands": "make unit" | ||
| } | ||
| ], | ||
|
|
||
| "promotion": { | ||
| "cluster": "https://api.ci.openshift.org", | ||
| "namespace": "azure", | ||
| "name": "azure-plugins" | ||
| }, | ||
|
|
||
| "resources": { | ||
| "*": { | ||
| "requests": { "cpu": "100m", "memory": "200Mi" }, | ||
| "limits": { "cpu": "2", "memory": "4Gi" } | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,315 @@ | ||
| # This template is being used for testing OpenShift Ansible image builds for Azure. | ||
|
Contributor
There was a problem hiding this comment. This template needs to be moved in |
||
| # At the moment it cannot be reused to launch cluster for Origin tests | ||
| kind: Template | ||
| apiVersion: template.openshift.io/v1 | ||
|
|
||
| parameters: | ||
| - name: JOB_NAME_SAFE | ||
| required: true | ||
| - name: JOB_NAME_HASH | ||
| required: true | ||
| - name: LOCAL_IMAGE_BIN | ||
| required: true | ||
| - name: NAMESPACE | ||
| required: true | ||
| - name: CLUSTER_TYPE | ||
| value: "azure" | ||
| required: true | ||
| - name: AZURE_VM_SIZE | ||
| value: "Standard_D2s_v3" | ||
| required: true | ||
| - name: OPENSHIFT_RELEASE | ||
| value: "3.10" | ||
| required: true | ||
| - name: IMAGE_ANSIBLE | ||
| required: true | ||
| - name: DEPLOY_OS | ||
| value: "centos7" | ||
| required: true | ||
|
|
||
| objects: | ||
|
|
||
| # We want the cluster to be able to access these images | ||
| - kind: RoleBinding | ||
| apiVersion: authorization.openshift.io/v1 | ||
| metadata: | ||
| name: ${JOB_NAME_SAFE}-image-puller | ||
| namespace: ${NAMESPACE} | ||
| roleRef: | ||
| name: system:image-puller | ||
| subjects: | ||
| - kind: SystemGroup | ||
| name: system:unauthenticated | ||
|
|
||
| # The build pod will create an instance and install openshift | ||
| - kind: Pod | ||
| apiVersion: v1 | ||
| metadata: | ||
| name: ${JOB_NAME_SAFE} | ||
| namespace: ${NAMESPACE} | ||
| annotations: | ||
| # we want to gather the teardown logs no matter what | ||
| ci-operator.openshift.io/wait-for-container-artifacts: teardown | ||
| spec: | ||
| restartPolicy: Never | ||
| activeDeadlineSeconds: 10800 | ||
| terminationGracePeriodSeconds: 600 | ||
| volumes: | ||
| - name: shared-tmp | ||
| emptyDir: {} | ||
| - name: shared-data | ||
| emptyDir: {} | ||
| - name: cluster-secrets-azure | ||
| secret: | ||
| secretName: azure | ||
|
|
||
| containers: | ||
| - name: buildimage | ||
| image: ${IMAGE_ANSIBLE} | ||
| resources: | ||
| requests: | ||
| cpu: 1 | ||
| memory: 300Mi | ||
| limits: | ||
| cpu: 3 | ||
| memory: 4Gi | ||
| volumeMounts: | ||
| - name: shared-tmp | ||
| mountPath: /tmp/shared | ||
| - name: cluster-secrets-azure | ||
| mountPath: /usr/share/ansible/openshift-ansible/inventory/dynamic/injected | ||
| env: | ||
| - name: HOME | ||
| value: /usr/share/ansible/openshift-ansible | ||
| - name: AZURE_VM_SIZE | ||
| value: "${AZURE_VM_SIZE}" | ||
| - name: JOB_NAME_SAFE | ||
| value: "${JOB_NAME_SAFE}" | ||
| - name: JOB_NAME_HASH | ||
| value: "${JOB_NAME_HASH}" | ||
| - name: OPENSHIFT_RELEASE | ||
| value: "${OPENSHIFT_RELEASE}" | ||
| - name: TYPE | ||
| value: azure | ||
| - name: DEPLOY_OS | ||
| value: "${DEPLOY_OS}" | ||
| - name: INSTANCE_PREFIX | ||
| value: ${NAMESPACE}-${JOB_NAME_HASH} | ||
| command: | ||
| - /usr/local/bin/entrypoint-provider | ||
| args: | ||
| - /bin/bash | ||
| - -c | ||
| - | | ||
| #!/bin/bash | ||
| # error handling and sync code | ||
| set -euo pipefail | ||
|
|
||
| trap '[ $? == 0 ] || touch /tmp/shared/exit' EXIT | ||
| trap 'kill $(jobs -p); exit 0' TERM | ||
|
|
||
| # We need to persist this variable so that we can clean it up when we exit | ||
| echo "RESOURCE_GROUP=${INSTANCE_PREFIX}-$(TZ=Etc/UTC date +%Y%m%d%H%M)" >>/tmp/shared/environment | ||
| echo "IMAGE_NAME=${DEPLOY_OS}-$OPENSHIFT_RELEASE-$(TZ=Etc/UTC date +%Y%m%d%H%M)" >>/tmp/shared/environment | ||
|
|
||
| # Get resource group and image name | ||
| source /tmp/shared/environment | ||
| echo "RESOURCE_GROUP=${RESOURCE_GROUP}" | ||
| echo "IMAGE_NAME=${IMAGE_NAME}" | ||
| echo $(pwd) | ||
|
|
||
| # Source azure creds | ||
| source /usr/share/ansible/openshift-ansible/inventory/dynamic/injected/credentials | ||
|
|
||
| ansible-playbook \ | ||
| -e openshift_azure_resource_group_name=$RESOURCE_GROUP \ | ||
| -e openshift_azure_resource_location=eastus \ | ||
| -e openshift_azure_input_image_ns=images \ | ||
| -e openshift_azure_input_image_prefix=${DEPLOY_OS}-base \ | ||
| -e openshift_azure_output_image_ns=images \ | ||
| -e openshift_azure_output_image_name=$IMAGE_NAME \ | ||
| -e openshift_azure_storage_account=openshiftimages \ | ||
| -e openshift_azure_storage_account_ns=images \ | ||
| -e openshift_azure_container=images \ | ||
| -e openshift_azure_vm_size=$AZURE_VM_SIZE \ | ||
| -e openshift_azure_install_repo=$(curl -qs https://storage.googleapis.com/origin-ci-test/releases/openshift/origin/release-${OPENSHIFT_RELEASE}/.latest-rpms) \ | ||
| playbooks/azure/openshift-cluster/build_node_image.yml | ||
|
|
||
| az login --service-principal -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} --tenant ${AZURE_TENANT} &>/dev/null | ||
| az group delete -n ${RESOURCE_GROUP} -y | ||
|
|
||
| touch /tmp/shared/image_create | ||
|
|
||
| # Runs an install | ||
| - name: setup | ||
| image: ${LOCAL_IMAGE_BIN} | ||
|
Contributor
There was a problem hiding this comment. LOCAL_IMAGE_SRC should be enough I think |
||
| volumeMounts: | ||
| - name: shared-tmp | ||
| mountPath: /tmp/shared | ||
| - name: cluster-secrets-azure | ||
| mountPath: /etc/azure/credentials | ||
| env: | ||
| - name: TYPE | ||
| value: ${CLUSTER_TYPE} | ||
| - name: HOME | ||
| value: /tmp/shared/home | ||
| - name: OPENSHIFT_RELEASE | ||
| value: "${OPENSHIFT_RELEASE}" | ||
| - name: DEPLOY_OS | ||
| value: "${DEPLOY_OS}" | ||
| command: | ||
| - /bin/bash | ||
| - -c | ||
| - | | ||
| #!/bin/bash | ||
| # error handling and sync code | ||
| set -euo pipefail | ||
| echo "starting buildcluster" | ||
|
|
||
| trap '[ "$?" == 0 ] || touch /tmp/shared/exit' EXIT | ||
| trap 'kill $(jobs -p); exit 0' TERM | ||
|
|
||
| # Cluster creation specific configuration. | ||
| mkdir -p "${HOME}" | ||
|
|
||
| # wait until the setup job creates admin.kubeconfig | ||
| while true; do | ||
| if [[ ! -f /tmp/shared/image_create ]]; then | ||
| echo "sleeping waiting for image_create" | ||
| sleep 15 & wait | ||
| continue | ||
| fi | ||
| # if we got admin kubeconfig file with failure marker, ignore. Teardown is in progress. | ||
| if [[ -f /tmp/shared/exit ]]; then | ||
| echo "Exiting. Found /tmp/shared/exit." | ||
| exit 1 | ||
| fi | ||
| break | ||
| done | ||
| echo "Found image create marker file." | ||
|
|
||
| source /etc/azure/credentials/secret | ||
| az login --service-principal -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} --tenant ${AZURE_TENANT_ID} &>/dev/null | ||
| # TODO: Configure these from prow | ||
|
|
||
| export DNS_DOMAIN=osadev.cloud | ||
| export DNS_RESOURCEGROUP=dns | ||
| export DEPLOY_VERSION=v${OPENSHIFT_RELEASE} | ||
| export RUN_SYNC_LOCAL=true | ||
|
|
||
| export IMAGE_RESOURCEGROUP=images | ||
|
|
||
| source /tmp/shared/environment | ||
| export IMAGE_RESOURCENAME=${IMAGE_NAME} | ||
| cd /go/src/github.com/openshift/openshift-azure/ | ||
|
|
||
| # added timeout as this seems to loop infinitely awaiting cluster deployment | ||
| timeout 30m ./hack/create.sh ${RESOURCE_GROUP}-e2e | ||
|
|
||
| # if completed pass generated data to other stage | ||
| cp -r /go/src/github.com/openshift/openshift-azure/_data /tmp/shared/ | ||
|
|
||
| touch /tmp/shared/e2e_tests | ||
|
|
||
| - name: tagimage | ||
| image: ${IMAGE_ANSIBLE} | ||
| resources: | ||
| requests: | ||
| cpu: 1 | ||
| memory: 300Mi | ||
| limits: | ||
| cpu: 3 | ||
| memory: 4Gi | ||
| volumeMounts: | ||
| - name: shared-tmp | ||
| mountPath: /tmp/shared | ||
| - name: cluster-secrets-azure | ||
| mountPath: /usr/share/ansible/openshift-ansible/inventory/dynamic/injected | ||
| env: | ||
| - name: HOME | ||
| value: /usr/share/ansible/openshift-ansible | ||
| - name: TYPE | ||
| value: azure | ||
| command: | ||
| - /usr/local/bin/entrypoint-provider | ||
| args: | ||
| - /bin/bash | ||
| - -c | ||
| - | | ||
| #!/bin/bash | ||
| # error handling and sync code | ||
| set -euo pipefail | ||
|
|
||
| trap '[ "$?" == 0 ] || touch /tmp/shared/exit' EXIT | ||
| trap 'kill $(jobs -p); exit 0' TERM | ||
|
|
||
| # wait until testing our image has succeeded | ||
| while true; do | ||
| if [[ ! -f /tmp/shared/e2e_tests ]]; then | ||
| sleep 15 & wait | ||
| continue | ||
| fi | ||
| # if we got admin kubeconfig file with failure marker, ignore. Teardown is in progress. | ||
| if [[ -f /tmp/shared/exit ]]; then | ||
| exit 1 | ||
| fi | ||
| break | ||
| done | ||
| echo "Found image create marker file." | ||
|
|
||
| # Get resource group and image name | ||
| source /tmp/shared/environment | ||
|
|
||
| # Source azure creds | ||
| source /usr/share/ansible/openshift-ansible/inventory/dynamic/injected/credentials | ||
|
|
||
| ansible-playbook \ | ||
| -e openshift_azure_resource_group_name=$RESOURCE_GROUP \ | ||
| -e openshift_azure_input_image_ns=images \ | ||
| -e openshift_azure_input_image_name=$IMAGE_NAME \ | ||
| playbooks/azure/openshift-cluster/tag_image_as_valid.yml | ||
|
|
||
| touch /tmp/shared/exit | ||
|
|
||
| # Performs cleanup of all created resources | ||
| - name: teardown | ||
| image: ${LOCAL_IMAGE_BIN} | ||
|
Contributor
There was a problem hiding this comment. LOCAL_IMAGE_SRC should be enough I think
Contributor
There was a problem hiding this comment. BIN and SRC one difference is that BIN has generated code and binaries built already. We use binaries for sync image build in the local namespace. Didn't checked end-to-end but I think we need bin for this, as it is used for sync image build. Just to keep consistent.
Contributor
There was a problem hiding this comment. But we don't care about building the sync image and we can just pull it from quay since what is tested here is the node build.
Contributor
There was a problem hiding this comment. I guess a mismatch between the pushed image in quay and the plugin code that is pulled from git can fail the cluster install, assuming there is a breaking change in between.
Contributor
There was a problem hiding this comment. our experience shows we good at this...
Contributor
There was a problem hiding this comment. Probably a good one to have so we can stop breaking things :) |
||
| volumeMounts: | ||
| - name: shared-tmp | ||
| mountPath: /tmp/shared | ||
| - name: cluster-secrets-azure | ||
| mountPath: /etc/azure/credentials | ||
| env: | ||
| - name: TYPE | ||
| value: ${CLUSTER_TYPE} | ||
| - name: HOME | ||
| value: /tmp/shared/home | ||
| command: | ||
| - /bin/bash | ||
| - -c | ||
| - | | ||
| #!/bin/bash | ||
| function teardown() { | ||
| set +e | ||
| mkdir -p "${HOME}" | ||
| export HOME=/tmp/shared | ||
| export DNS_DOMAIN=osadev.cloud | ||
| export DNS_RESOURCEGROUP=dns | ||
|
|
||
| cp -r /tmp/shared/_data /go/src/github.com/openshift/openshift-azure/ | ||
| cd /go/src/github.com/openshift/openshift-azure/ | ||
| source /etc/azure/credentials/secret | ||
|
Contributor
There was a problem hiding this comment. You also need to source |
||
| az login --service-principal -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} --tenant ${AZURE_TENANT_ID} &>/dev/null | ||
| ./hack/delete.sh ${RESOURCE_GROUP} | ||
|
|
||
| trap 'teardown' EXIT | ||
| trap 'kill $(jobs -p); exit 0' TERM | ||
| } | ||
|
|
||
| trap 'teardown' EXIT | ||
| trap 'kill $(jobs -p); exit 0' TERM | ||
|
|
||
| for i in `seq 1 120`; do | ||
| if [[ -f /tmp/shared/exit ]]; then | ||
| exit 0 | ||
| fi | ||
| sleep 60 & wait | ||
| done | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
openshift-azure master is effectively origin 3.10 (we already use 3.10 images in master.yaml). I think this is the wrong place to branch node build jobs, and it's more appropriate to be using the openshift-ansible ci-operator specs.