From b80ec779051c82a23e962661bf9e47f408fcc1bd Mon Sep 17 00:00:00 2001 From: avdata99 Date: Mon, 20 Mar 2023 11:48:12 -0300 Subject: [PATCH 1/8] GH action to Google Artifact registry --- .github/workflows/publish-docker-to-gcr.yml | 125 ++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 .github/workflows/publish-docker-to-gcr.yml diff --git a/.github/workflows/publish-docker-to-gcr.yml b/.github/workflows/publish-docker-to-gcr.yml new file mode 100644 index 00000000..ab1916b1 --- /dev/null +++ b/.github/workflows/publish-docker-to-gcr.yml @@ -0,0 +1,125 @@ +name: Create and publish Docker images to Google Cloud Registry +# READ https://gist.github.com/palewire/12c4b2b974ef735d22da7493cf7f4d37 + +on: + push: + branches: + - 116_deploy_images_to-github_packages + +env: + GAR_ZONE: europe-southwest1 + GAR_REPO_NAME: docker-ckan-images + G_PROJECT_ID: oki-cloud + G_SERVICE_ACCOUT: sa-gh-action-registry + +jobs: + publish_to_gcr: + runs-on: ubuntu-latest + # Add "id-token" with the intended permissions. + permissions: + contents: 'read' + id-token: 'write' + + steps: + + - name: Checkout repository + uses: actions/checkout@v3 + + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + token_format: access_token + workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider + service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com + access_token_lifetime: 300s + + - name: Login to Artifact Registry + uses: docker/login-action@v1 + with: + registry: ${{ env.GAR_ZONE }}-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Set CKAN Versions + run: | + echo CKAN_v2_8=$(grep 'ENV GIT_BRANCH' ckan-base/2.8/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV + echo CKAN_v2_9=$(grep 'ENV GIT_BRANCH' ckan-base/2.9/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV + # TODO Update when there's an actual 2.10 released tag + echo CKAN_v2_10=2.10.0 >> $GITHUB_ENV + + - name: Build ckan-base 2.8 and ${{ env.CKAN_v2_8 }} + uses: docker/build-push-action@v4 + with: + context: ckan-base + file: ckan-base/2.8/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:2.8 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:${{ env.CKAN_v2_8 }} + + # - name: Build ckan-base 2.9 and ${{ env.CKAN_v2_9 }} + # uses: docker/build-push-action@v4 + # with: + # context: ckan-base + # file: ckan-base/2.9/Dockerfile + # push: true + # tags: | + # ${{ env.REGISTRY }}/openknowledge/ckan-base:2.9 + # ${{ env.REGISTRY }}/openknowledge/ckan-base:${{ env.CKAN_v2_9 }} + + # - name: Build ckan-base 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 + # uses: docker/build-push-action@v4 + # with: + # context: ckan-base + # file: ckan-base/2.9/Dockerfile.py2 + # push: true + # tags: | + # ${{ env.REGISTRY }}/openknowledge/ckan-base:2.9-py2 + # ${{ env.REGISTRY }}/openknowledge/ckan-base:${{ env.CKAN_v2_9 }}-py2 + + # - name: Build ckan-base 2.10 and ${{ env.CKAN_v2_10 }} + # uses: docker/build-push-action@v4 + # with: + # context: ckan-base + # file: ckan-base/2.10/Dockerfile + # push: true + # tags: | + # ${{ env.REGISTRY }}/openknowledge/ckan-base:2.10 + # ${{ env.REGISTRY }}/openknowledge/ckan-base:${{ env.CKAN_v2_10 }} + # - name: Build ckan-dev 2.8 and and ${{ env.CKAN_v2_8 }} + # uses: docker/build-push-action@v4 + # with: + # context: ckan-dev + # file: ckan-dev/2.8/Dockerfile + # push: true + # tags: | + # ${{ env.REGISTRY }}/openknowledge/ckan-dev:2.8 + # ${{ env.REGISTRY }}/openknowledge/ckan-dev:${{ env.CKAN_v2_8 }} + # - name: Build ckan-dev 2.9 and ${{ env.CKAN_v2_9 }} + # uses: docker/build-push-action@v4 + # with: + # context: ckan-dev + # file: ckan-dev/2.9/Dockerfile + # push: true + # tags: | + # ${{ env.REGISTRY }}/openknowledge/ckan-dev:2.9 + # ${{ env.REGISTRY }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }} + # - name: Build ckan-dev 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 + # uses: docker/build-push-action@v4 + # with: + # context: ckan-dev + # file: ckan-dev/2.9/Dockerfile.py2 + # push: true + # tags: | + # ${{ env.REGISTRY }}/openknowledge/ckan-dev:2.9-py2 + # ${{ env.REGISTRY }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }}-py2 + # - name: Build ckan-dev 2.10 and ${{ env.CKAN_v2_10 }} + # uses: docker/build-push-action@v4 + # with: + # context: ckan-dev + # file: ckan-dev/2.10/Dockerfile + # push: true + # tags: | + # ${{ env.REGISTRY }}/openknowledge/ckan-dev:2.10 + # ${{ env.REGISTRY }}/openknowledge/ckan-dev:${{ env.CKAN_v2_10 }} From 7a53936e7b42ed3262fcd6d35a0471fa4f92c955 Mon Sep 17 00:00:00 2001 From: avdata99 Date: Tue, 21 Mar 2023 16:13:59 -0300 Subject: [PATCH 2/8] Publish all images --- .github/workflows/publish-docker-to-gcr.yml | 126 ++++++++++---------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/.github/workflows/publish-docker-to-gcr.yml b/.github/workflows/publish-docker-to-gcr.yml index ab1916b1..0b46efc7 100644 --- a/.github/workflows/publish-docker-to-gcr.yml +++ b/.github/workflows/publish-docker-to-gcr.yml @@ -58,68 +58,68 @@ jobs: ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:2.8 ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:${{ env.CKAN_v2_8 }} - # - name: Build ckan-base 2.9 and ${{ env.CKAN_v2_9 }} - # uses: docker/build-push-action@v4 - # with: - # context: ckan-base - # file: ckan-base/2.9/Dockerfile - # push: true - # tags: | - # ${{ env.REGISTRY }}/openknowledge/ckan-base:2.9 - # ${{ env.REGISTRY }}/openknowledge/ckan-base:${{ env.CKAN_v2_9 }} + - name: Build ckan-base 2.9 and ${{ env.CKAN_v2_9 }} + uses: docker/build-push-action@v4 + with: + context: ckan-base + file: ckan-base/2.9/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:2.9 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:${{ env.CKAN_v2_9 }} - # - name: Build ckan-base 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 - # uses: docker/build-push-action@v4 - # with: - # context: ckan-base - # file: ckan-base/2.9/Dockerfile.py2 - # push: true - # tags: | - # ${{ env.REGISTRY }}/openknowledge/ckan-base:2.9-py2 - # ${{ env.REGISTRY }}/openknowledge/ckan-base:${{ env.CKAN_v2_9 }}-py2 + - name: Build ckan-base 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 + uses: docker/build-push-action@v4 + with: + context: ckan-base + file: ckan-base/2.9/Dockerfile.py2 + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:2.9-py2 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:${{ env.CKAN_v2_9 }}-py2 - # - name: Build ckan-base 2.10 and ${{ env.CKAN_v2_10 }} - # uses: docker/build-push-action@v4 - # with: - # context: ckan-base - # file: ckan-base/2.10/Dockerfile - # push: true - # tags: | - # ${{ env.REGISTRY }}/openknowledge/ckan-base:2.10 - # ${{ env.REGISTRY }}/openknowledge/ckan-base:${{ env.CKAN_v2_10 }} - # - name: Build ckan-dev 2.8 and and ${{ env.CKAN_v2_8 }} - # uses: docker/build-push-action@v4 - # with: - # context: ckan-dev - # file: ckan-dev/2.8/Dockerfile - # push: true - # tags: | - # ${{ env.REGISTRY }}/openknowledge/ckan-dev:2.8 - # ${{ env.REGISTRY }}/openknowledge/ckan-dev:${{ env.CKAN_v2_8 }} - # - name: Build ckan-dev 2.9 and ${{ env.CKAN_v2_9 }} - # uses: docker/build-push-action@v4 - # with: - # context: ckan-dev - # file: ckan-dev/2.9/Dockerfile - # push: true - # tags: | - # ${{ env.REGISTRY }}/openknowledge/ckan-dev:2.9 - # ${{ env.REGISTRY }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }} - # - name: Build ckan-dev 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 - # uses: docker/build-push-action@v4 - # with: - # context: ckan-dev - # file: ckan-dev/2.9/Dockerfile.py2 - # push: true - # tags: | - # ${{ env.REGISTRY }}/openknowledge/ckan-dev:2.9-py2 - # ${{ env.REGISTRY }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }}-py2 - # - name: Build ckan-dev 2.10 and ${{ env.CKAN_v2_10 }} - # uses: docker/build-push-action@v4 - # with: - # context: ckan-dev - # file: ckan-dev/2.10/Dockerfile - # push: true - # tags: | - # ${{ env.REGISTRY }}/openknowledge/ckan-dev:2.10 - # ${{ env.REGISTRY }}/openknowledge/ckan-dev:${{ env.CKAN_v2_10 }} + - name: Build ckan-base 2.10 and ${{ env.CKAN_v2_10 }} + uses: docker/build-push-action@v4 + with: + context: ckan-base + file: ckan-base/2.10/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:2.10 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:${{ env.CKAN_v2_10 }} + - name: Build ckan-dev 2.8 and and ${{ env.CKAN_v2_8 }} + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.8/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.8 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_8 }} + - name: Build ckan-dev 2.9 and ${{ env.CKAN_v2_9 }} + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.9/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.9 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }} + - name: Build ckan-dev 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.9/Dockerfile.py2 + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.9-py2 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }}-py2 + - name: Build ckan-dev 2.10 and ${{ env.CKAN_v2_10 }} + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.10/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.10 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_10 }} From 6dfe574115b6df27465cac9a3b3e37d2d675ddf6 Mon Sep 17 00:00:00 2001 From: avdata99 Date: Tue, 21 Mar 2023 16:34:32 -0300 Subject: [PATCH 3/8] Update deploy scripts --- .github/workflows/publish-docker-master.yml | 43 +++++++++++++++------ .github/workflows/publish-docker-to-gcr.yml | 3 +- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/.github/workflows/publish-docker-master.yml b/.github/workflows/publish-docker-master.yml index 5e5992e8..e25f0456 100644 --- a/.github/workflows/publish-docker-master.yml +++ b/.github/workflows/publish-docker-master.yml @@ -3,32 +3,53 @@ on: schedule: - cron: '15 5 * * *' +env: + GAR_ZONE: europe-southwest1 + GAR_REPO_NAME: docker-ckan-images + G_PROJECT_ID: oki-cloud + G_SERVICE_ACCOUT: sa-gh-action-registry + jobs: build: runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub + - name: Checkout repository + uses: actions/checkout@v3 + + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + token_format: access_token + workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider + service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com + access_token_lifetime: 600s + + - name: Login to Artifact Registry uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: ${{ env.GAR_ZONE }}-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + - name: Build ckan-base master - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: ckan-base file: ckan-base/master/Dockerfile push: true tags: | - openknowledge/ckan-base:master + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:master + - name: Build ckan-dev master - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: ckan-dev file: ckan-dev/master/Dockerfile push: true tags: | - openknowledge/ckan-dev:master + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:master diff --git a/.github/workflows/publish-docker-to-gcr.yml b/.github/workflows/publish-docker-to-gcr.yml index 0b46efc7..978a8dab 100644 --- a/.github/workflows/publish-docker-to-gcr.yml +++ b/.github/workflows/publish-docker-to-gcr.yml @@ -32,7 +32,8 @@ jobs: token_format: access_token workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com - access_token_lifetime: 300s + # Wait for all builds ... + access_token_lifetime: 3000s - name: Login to Artifact Registry uses: docker/login-action@v1 From 9b0558b91a2c1f77483b5faf3436694de836ee6f Mon Sep 17 00:00:00 2001 From: avdata99 Date: Tue, 21 Mar 2023 17:03:37 -0300 Subject: [PATCH 4/8] Notes, remove old action, add GAR reference --- .github/workflows/publish-docker-to-gcr.yml | 2 +- .github/workflows/publish-docker.yml | 97 --------------------- README.md | 13 ++- ckan-dev/2.10/Dockerfile | 3 +- ckan-dev/2.8/Dockerfile | 3 +- ckan-dev/2.9/Dockerfile | 3 +- ckan-dev/2.9/Dockerfile.py2 | 3 +- ckan-dev/master/Dockerfile | 3 +- ckan/Dockerfile | 3 +- ckan/Dockerfile.dev | 3 +- 10 files changed, 27 insertions(+), 106 deletions(-) delete mode 100644 .github/workflows/publish-docker.yml diff --git a/.github/workflows/publish-docker-to-gcr.yml b/.github/workflows/publish-docker-to-gcr.yml index 978a8dab..b48b8043 100644 --- a/.github/workflows/publish-docker-to-gcr.yml +++ b/.github/workflows/publish-docker-to-gcr.yml @@ -33,7 +33,7 @@ jobs: workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com # Wait for all builds ... - access_token_lifetime: 3000s + access_token_lifetime: 6000s - name: Login to Artifact Registry uses: docker/login-action@v1 diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml deleted file mode 100644 index 46c6a014..00000000 --- a/.github/workflows/publish-docker.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Build and publish all docker-ckan Images -on: - push: - branches: - - master - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set CKAN Versions - run: | - echo CKAN_v2_8=$(grep 'ENV GIT_BRANCH' ckan-base/2.8/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV - echo CKAN_v2_9=$(grep 'ENV GIT_BRANCH' ckan-base/2.9/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV - # TODO Update when there's an actual 2.10 released tag - echo CKAN_v2_10=2.10.0 >> $GITHUB_ENV - - name: Build ckan-base 2.8 and ${{ env.CKAN_v2_8 }} - uses: docker/build-push-action@v2 - with: - context: ckan-base - file: ckan-base/2.8/Dockerfile - push: true - tags: | - openknowledge/ckan-base:2.8 - openknowledge/ckan-base:${{ env.CKAN_v2_8 }} - - name: Build ckan-base 2.9 and ${{ env.CKAN_v2_9 }} - uses: docker/build-push-action@v2 - with: - context: ckan-base - file: ckan-base/2.9/Dockerfile - push: true - tags: | - openknowledge/ckan-base:2.9 - openknowledge/ckan-base:${{ env.CKAN_v2_9 }} - - name: Build ckan-base 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 - uses: docker/build-push-action@v2 - with: - context: ckan-base - file: ckan-base/2.9/Dockerfile.py2 - push: true - tags: | - openknowledge/ckan-base:2.9-py2 - openknowledge/ckan-base:${{ env.CKAN_v2_9 }}-py2 - - name: Build ckan-base 2.10 and ${{ env.CKAN_v2_10 }} - uses: docker/build-push-action@v2 - with: - context: ckan-base - file: ckan-base/2.10/Dockerfile - push: true - tags: | - openknowledge/ckan-base:2.10 - openknowledge/ckan-base:${{ env.CKAN_v2_10 }} - - name: Build ckan-dev 2.8 and and ${{ env.CKAN_v2_8 }} - uses: docker/build-push-action@v2 - with: - context: ckan-dev - file: ckan-dev/2.8/Dockerfile - push: true - tags: | - openknowledge/ckan-dev:2.8 - openknowledge/ckan-dev:${{ env.CKAN_v2_8 }} - - name: Build ckan-dev 2.9 and ${{ env.CKAN_v2_9 }} - uses: docker/build-push-action@v2 - with: - context: ckan-dev - file: ckan-dev/2.9/Dockerfile - push: true - tags: | - openknowledge/ckan-dev:2.9 - openknowledge/ckan-dev:${{ env.CKAN_v2_9 }} - - name: Build ckan-dev 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 - uses: docker/build-push-action@v2 - with: - context: ckan-dev - file: ckan-dev/2.9/Dockerfile.py2 - push: true - tags: | - openknowledge/ckan-dev:2.9-py2 - openknowledge/ckan-dev:${{ env.CKAN_v2_9 }}-py2 - - name: Build ckan-dev 2.10 and ${{ env.CKAN_v2_10 }} - uses: docker/build-push-action@v2 - with: - context: ckan-dev - file: ckan-dev/2.10/Dockerfile - push: true - tags: | - openknowledge/ckan-dev:2.10 - openknowledge/ckan-dev:${{ env.CKAN_v2_10 }} diff --git a/README.md b/README.md index f8ba3141..b7cb51f3 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,17 @@ The Docker images used to build your CKAN project are located in the `ckan/` fol From these two base images you can build your own customized image tailored to your project, installing any extensions and extra requirements needed. +### Pulling images + +Docker CKAN images lives at Google Artifact Registry. You can pull them using the following command: + +``` +docker pull europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images/openknowledge/ckan-base:2.8 +``` + +You can change the last part `ckan-base:2.8` for any of the available images. + + ### Extending the base images To perform extra initialization steps you can add scripts to your custom images and copy them to the `/docker-entrypoint.d` folder (The folder should be created for you when you build the image). Any `*.sh` and `*.py` file in that folder will be executed just after the main initialization script ([`prerun.py`](https://github.com/okfn/docker-ckan/blob/master/ckan-base/setup/prerun.py)) is executed and just before the web server and supervisor processes are started. @@ -147,7 +158,7 @@ paster --plugin=ckanext-validation validation init-db -c $CKAN_INI And then in our `Dockerfile` we install the extension and copy the initialization scripts: ```Dockerfile -FROM openknowledge/ckan-dev:2.9 +FROM europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images/openknowledge/ckan-dev:2.9 RUN pip install -e git+https://github.com/frictionlessdata/ckanext-validation.git#egg=ckanext-validation && \ pip install -r https://raw.githubusercontent.com/frictionlessdata/ckanext-validation/master/requirements.txt diff --git a/ckan-dev/2.10/Dockerfile b/ckan-dev/2.10/Dockerfile index 99318642..c8e8a881 100644 --- a/ckan-dev/2.10/Dockerfile +++ b/ckan-dev/2.10/Dockerfile @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:2.10 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:2.10 MAINTAINER Open Knowledge Foundation diff --git a/ckan-dev/2.8/Dockerfile b/ckan-dev/2.8/Dockerfile index cec5e837..a3a165c6 100644 --- a/ckan-dev/2.8/Dockerfile +++ b/ckan-dev/2.8/Dockerfile @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:2.8 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:2.8 MAINTAINER Open Knowledge Foundation diff --git a/ckan-dev/2.9/Dockerfile b/ckan-dev/2.9/Dockerfile index 2b21bca2..bf423ff7 100644 --- a/ckan-dev/2.9/Dockerfile +++ b/ckan-dev/2.9/Dockerfile @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:2.9 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:2.9 MAINTAINER Open Knowledge Foundation diff --git a/ckan-dev/2.9/Dockerfile.py2 b/ckan-dev/2.9/Dockerfile.py2 index 6c0c00bf..127b7d7a 100644 --- a/ckan-dev/2.9/Dockerfile.py2 +++ b/ckan-dev/2.9/Dockerfile.py2 @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:2.9-py2 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:2.9-py2 MAINTAINER Open Knowledge Foundation diff --git a/ckan-dev/master/Dockerfile b/ckan-dev/master/Dockerfile index f3e45341..b7cb2092 100644 --- a/ckan-dev/master/Dockerfile +++ b/ckan-dev/master/Dockerfile @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:master +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:master MAINTAINER Open Knowledge Foundation diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 0c2de0db..bb353649 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:2.10 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:2.10 MAINTAINER Your Name Here diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index cd3b296b..d78dfbbb 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-dev:2.10 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-dev:2.10 MAINTAINER Your Name Here From 7eddfa093299b84bd2d5503e73d3b660c592560d Mon Sep 17 00:00:00 2001 From: avdata99 Date: Tue, 21 Mar 2023 17:14:11 -0300 Subject: [PATCH 5/8] Split jobs --- ...gcr.yml => publish-docker-base-to-gcr.yml} | 40 +-------- .../publish-docker-dev-to-gcr copy.yml | 89 +++++++++++++++++++ 2 files changed, 91 insertions(+), 38 deletions(-) rename .github/workflows/{publish-docker-to-gcr.yml => publish-docker-base-to-gcr.yml} (63%) create mode 100644 .github/workflows/publish-docker-dev-to-gcr copy.yml diff --git a/.github/workflows/publish-docker-to-gcr.yml b/.github/workflows/publish-docker-base-to-gcr.yml similarity index 63% rename from .github/workflows/publish-docker-to-gcr.yml rename to .github/workflows/publish-docker-base-to-gcr.yml index b48b8043..ba01b4c2 100644 --- a/.github/workflows/publish-docker-to-gcr.yml +++ b/.github/workflows/publish-docker-base-to-gcr.yml @@ -1,4 +1,4 @@ -name: Create and publish Docker images to Google Cloud Registry +name: Create and publish BASE Docker images to Google Cloud Registry # READ https://gist.github.com/palewire/12c4b2b974ef735d22da7493cf7f4d37 on: @@ -33,7 +33,7 @@ jobs: workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com # Wait for all builds ... - access_token_lifetime: 6000s + access_token_lifetime: 3600s - name: Login to Artifact Registry uses: docker/login-action@v1 @@ -88,39 +88,3 @@ jobs: tags: | ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:2.10 ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:${{ env.CKAN_v2_10 }} - - name: Build ckan-dev 2.8 and and ${{ env.CKAN_v2_8 }} - uses: docker/build-push-action@v4 - with: - context: ckan-dev - file: ckan-dev/2.8/Dockerfile - push: true - tags: | - ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.8 - ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_8 }} - - name: Build ckan-dev 2.9 and ${{ env.CKAN_v2_9 }} - uses: docker/build-push-action@v4 - with: - context: ckan-dev - file: ckan-dev/2.9/Dockerfile - push: true - tags: | - ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.9 - ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }} - - name: Build ckan-dev 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 - uses: docker/build-push-action@v4 - with: - context: ckan-dev - file: ckan-dev/2.9/Dockerfile.py2 - push: true - tags: | - ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.9-py2 - ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }}-py2 - - name: Build ckan-dev 2.10 and ${{ env.CKAN_v2_10 }} - uses: docker/build-push-action@v4 - with: - context: ckan-dev - file: ckan-dev/2.10/Dockerfile - push: true - tags: | - ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.10 - ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_10 }} diff --git a/.github/workflows/publish-docker-dev-to-gcr copy.yml b/.github/workflows/publish-docker-dev-to-gcr copy.yml new file mode 100644 index 00000000..09a99283 --- /dev/null +++ b/.github/workflows/publish-docker-dev-to-gcr copy.yml @@ -0,0 +1,89 @@ +name: Create and publish DEV Docker images to Google Cloud Registry +# READ https://gist.github.com/palewire/12c4b2b974ef735d22da7493cf7f4d37 + +on: + workflow_run: + workflows: [Create and publish BASE Docker images to Google Cloud Registry] + types: + - completed + +env: + GAR_ZONE: europe-southwest1 + GAR_REPO_NAME: docker-ckan-images + G_PROJECT_ID: oki-cloud + G_SERVICE_ACCOUT: sa-gh-action-registry + +jobs: + publish_to_gcr: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + # Add "id-token" with the intended permissions. + permissions: + contents: 'read' + id-token: 'write' + + steps: + + - name: Checkout repository + uses: actions/checkout@v3 + + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + token_format: access_token + workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider + service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com + # Wait for all builds ... + access_token_lifetime: 3600s + + - name: Login to Artifact Registry + uses: docker/login-action@v1 + with: + registry: ${{ env.GAR_ZONE }}-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Set CKAN Versions + run: | + echo CKAN_v2_8=$(grep 'ENV GIT_BRANCH' ckan-base/2.8/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV + echo CKAN_v2_9=$(grep 'ENV GIT_BRANCH' ckan-base/2.9/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV + # TODO Update when there's an actual 2.10 released tag + echo CKAN_v2_10=2.10.0 >> $GITHUB_ENV + + - name: Build ckan-dev 2.8 and and ${{ env.CKAN_v2_8 }} + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.8/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.8 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_8 }} + - name: Build ckan-dev 2.9 and ${{ env.CKAN_v2_9 }} + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.9/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.9 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }} + - name: Build ckan-dev 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.9/Dockerfile.py2 + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.9-py2 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }}-py2 + - name: Build ckan-dev 2.10 and ${{ env.CKAN_v2_10 }} + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.10/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.10 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_10 }} From 98482ed8978f4600066301c135032f5c69105045 Mon Sep 17 00:00:00 2001 From: avdata99 Date: Tue, 21 Mar 2023 17:35:47 -0300 Subject: [PATCH 6/8] Fix workflows connection --- .github/workflows/publish-docker-base-to-gcr.yml | 2 +- .github/workflows/publish-docker-dev-to-gcr copy.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-docker-base-to-gcr.yml b/.github/workflows/publish-docker-base-to-gcr.yml index ba01b4c2..6abe1fd7 100644 --- a/.github/workflows/publish-docker-base-to-gcr.yml +++ b/.github/workflows/publish-docker-base-to-gcr.yml @@ -13,7 +13,7 @@ env: G_SERVICE_ACCOUT: sa-gh-action-registry jobs: - publish_to_gcr: + publish_base_to_gcr: runs-on: ubuntu-latest # Add "id-token" with the intended permissions. permissions: diff --git a/.github/workflows/publish-docker-dev-to-gcr copy.yml b/.github/workflows/publish-docker-dev-to-gcr copy.yml index 09a99283..d00196ce 100644 --- a/.github/workflows/publish-docker-dev-to-gcr copy.yml +++ b/.github/workflows/publish-docker-dev-to-gcr copy.yml @@ -6,6 +6,8 @@ on: workflows: [Create and publish BASE Docker images to Google Cloud Registry] types: - completed + branches: + - 116_deploy_images_to env: GAR_ZONE: europe-southwest1 @@ -14,7 +16,7 @@ env: G_SERVICE_ACCOUT: sa-gh-action-registry jobs: - publish_to_gcr: + publish_dev_to_gcr: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'success' }} # Add "id-token" with the intended permissions. From 81791f412b559744474665631fb15e29dc3788d2 Mon Sep 17 00:00:00 2001 From: avdata99 Date: Mon, 27 Mar 2023 12:04:00 -0300 Subject: [PATCH 7/8] Fix cascade workflow --- .github/workflows/publish-docker-base-to-gcr.yml | 2 +- ...docker-dev-to-gcr copy.yml => publish-docker-dev-to-gcr.yml} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{publish-docker-dev-to-gcr copy.yml => publish-docker-dev-to-gcr.yml} (97%) diff --git a/.github/workflows/publish-docker-base-to-gcr.yml b/.github/workflows/publish-docker-base-to-gcr.yml index 6abe1fd7..ed268363 100644 --- a/.github/workflows/publish-docker-base-to-gcr.yml +++ b/.github/workflows/publish-docker-base-to-gcr.yml @@ -1,4 +1,4 @@ -name: Create and publish BASE Docker images to Google Cloud Registry +name: Publish BASE Docker images # READ https://gist.github.com/palewire/12c4b2b974ef735d22da7493cf7f4d37 on: diff --git a/.github/workflows/publish-docker-dev-to-gcr copy.yml b/.github/workflows/publish-docker-dev-to-gcr.yml similarity index 97% rename from .github/workflows/publish-docker-dev-to-gcr copy.yml rename to .github/workflows/publish-docker-dev-to-gcr.yml index d00196ce..733c9c54 100644 --- a/.github/workflows/publish-docker-dev-to-gcr copy.yml +++ b/.github/workflows/publish-docker-dev-to-gcr.yml @@ -3,7 +3,7 @@ name: Create and publish DEV Docker images to Google Cloud Registry on: workflow_run: - workflows: [Create and publish BASE Docker images to Google Cloud Registry] + workflows: [Publish BASE Docker images] types: - completed branches: From 50f94a7289b422b9de1deda7bab7cfd8d81295bf Mon Sep 17 00:00:00 2001 From: avdata99 Date: Mon, 27 Mar 2023 12:54:38 -0300 Subject: [PATCH 8/8] Add a daily images cleaner --- .github/workflows/image-registry-cleaner.yml | 45 +++++++++++++++++++ .../workflows/publish-docker-base-to-gcr.yml | 2 +- .../workflows/publish-docker-dev-to-gcr.yml | 7 +-- .github/workflows/publish-docker-master.yml | 2 +- 4 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/image-registry-cleaner.yml diff --git a/.github/workflows/image-registry-cleaner.yml b/.github/workflows/image-registry-cleaner.yml new file mode 100644 index 00000000..35c0e52e --- /dev/null +++ b/.github/workflows/image-registry-cleaner.yml @@ -0,0 +1,45 @@ +name: 'Remove untagged images' + +on: + schedule: + - cron: '0 20 * * *' # runs daily + workflow_dispatch: # allows for manual invocation + +env: + GAR_ZONE: europe-southwest1 + GAR_REPO_NAME: docker-ckan-images + G_PROJECT_ID: oki-cloud + G_SERVICE_ACCOUT: sa-gh-action-registry + +jobs: + gcr-cleaner: + runs-on: 'ubuntu-latest' + steps: + + - name: Checkout repository + uses: actions/checkout@v3 + + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + token_format: access_token + workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider + service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com + # Wait for all builds ... + access_token_lifetime: 3600s + + - name: Login to Artifact Registry + uses: docker/login-action@v1 + with: + registry: ${{ env.GAR_ZONE }}-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + # customize based on the gcr-cleaner flags + - uses: 'docker://us-docker.pkg.dev/gcr-cleaner/gcr-cleaner/gcr-cleaner-cli' + with: + args: >- + -repo=${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base + -repo=${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev + -grace=48h diff --git a/.github/workflows/publish-docker-base-to-gcr.yml b/.github/workflows/publish-docker-base-to-gcr.yml index ed268363..e2b8eee5 100644 --- a/.github/workflows/publish-docker-base-to-gcr.yml +++ b/.github/workflows/publish-docker-base-to-gcr.yml @@ -4,7 +4,7 @@ name: Publish BASE Docker images on: push: branches: - - 116_deploy_images_to-github_packages + - master env: GAR_ZONE: europe-southwest1 diff --git a/.github/workflows/publish-docker-dev-to-gcr.yml b/.github/workflows/publish-docker-dev-to-gcr.yml index 733c9c54..2ac4f5d2 100644 --- a/.github/workflows/publish-docker-dev-to-gcr.yml +++ b/.github/workflows/publish-docker-dev-to-gcr.yml @@ -1,13 +1,10 @@ -name: Create and publish DEV Docker images to Google Cloud Registry +name: Publish DEV Docker images # READ https://gist.github.com/palewire/12c4b2b974ef735d22da7493cf7f4d37 on: workflow_run: workflows: [Publish BASE Docker images] - types: - - completed - branches: - - 116_deploy_images_to + types: [completed] env: GAR_ZONE: europe-southwest1 diff --git a/.github/workflows/publish-docker-master.yml b/.github/workflows/publish-docker-master.yml index e25f0456..ac32a323 100644 --- a/.github/workflows/publish-docker-master.yml +++ b/.github/workflows/publish-docker-master.yml @@ -1,4 +1,4 @@ -name: Build and publish the master docker-ckan image +name: Publish daily master docker-ckan image on: schedule: - cron: '15 5 * * *'