diff --git a/.github/workflows/image-registry-cleaner.yml b/.github/workflows/image-registry-cleaner.yml new file mode 100644 index 00000000..35c0e52e --- /dev/null +++ b/.github/workflows/image-registry-cleaner.yml @@ -0,0 +1,45 @@ +name: 'Remove untagged images' + +on: + schedule: + - cron: '0 20 * * *' # runs daily + workflow_dispatch: # allows for manual invocation + +env: + GAR_ZONE: europe-southwest1 + GAR_REPO_NAME: docker-ckan-images + G_PROJECT_ID: oki-cloud + G_SERVICE_ACCOUT: sa-gh-action-registry + +jobs: + gcr-cleaner: + runs-on: 'ubuntu-latest' + steps: + + - name: Checkout repository + uses: actions/checkout@v3 + + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + token_format: access_token + workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider + service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com + # Wait for all builds ... + access_token_lifetime: 3600s + + - name: Login to Artifact Registry + uses: docker/login-action@v1 + with: + registry: ${{ env.GAR_ZONE }}-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + # customize based on the gcr-cleaner flags + - uses: 'docker://us-docker.pkg.dev/gcr-cleaner/gcr-cleaner/gcr-cleaner-cli' + with: + args: >- + -repo=${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base + -repo=${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev + -grace=48h diff --git a/.github/workflows/publish-docker-base-to-gcr.yml b/.github/workflows/publish-docker-base-to-gcr.yml new file mode 100644 index 00000000..e2b8eee5 --- /dev/null +++ b/.github/workflows/publish-docker-base-to-gcr.yml @@ -0,0 +1,90 @@ +name: Publish BASE Docker images +# READ https://gist.github.com/palewire/12c4b2b974ef735d22da7493cf7f4d37 + +on: + push: + branches: + - master + +env: + GAR_ZONE: europe-southwest1 + GAR_REPO_NAME: docker-ckan-images + G_PROJECT_ID: oki-cloud + G_SERVICE_ACCOUT: sa-gh-action-registry + +jobs: + publish_base_to_gcr: + runs-on: ubuntu-latest + # Add "id-token" with the intended permissions. + permissions: + contents: 'read' + id-token: 'write' + + steps: + + - name: Checkout repository + uses: actions/checkout@v3 + + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + token_format: access_token + workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider + service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com + # Wait for all builds ... + access_token_lifetime: 3600s + + - name: Login to Artifact Registry + uses: docker/login-action@v1 + with: + registry: ${{ env.GAR_ZONE }}-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Set CKAN Versions + run: | + echo CKAN_v2_8=$(grep 'ENV GIT_BRANCH' ckan-base/2.8/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV + echo CKAN_v2_9=$(grep 'ENV GIT_BRANCH' ckan-base/2.9/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV + # TODO Update when there's an actual 2.10 released tag + echo CKAN_v2_10=2.10.0 >> $GITHUB_ENV + + - name: Build ckan-base 2.8 and ${{ env.CKAN_v2_8 }} + uses: docker/build-push-action@v4 + with: + context: ckan-base + file: ckan-base/2.8/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:2.8 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:${{ env.CKAN_v2_8 }} + + - name: Build ckan-base 2.9 and ${{ env.CKAN_v2_9 }} + uses: docker/build-push-action@v4 + with: + context: ckan-base + file: ckan-base/2.9/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:2.9 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:${{ env.CKAN_v2_9 }} + + - name: Build ckan-base 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 + uses: docker/build-push-action@v4 + with: + context: ckan-base + file: ckan-base/2.9/Dockerfile.py2 + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:2.9-py2 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:${{ env.CKAN_v2_9 }}-py2 + + - name: Build ckan-base 2.10 and ${{ env.CKAN_v2_10 }} + uses: docker/build-push-action@v4 + with: + context: ckan-base + file: ckan-base/2.10/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:2.10 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:${{ env.CKAN_v2_10 }} diff --git a/.github/workflows/publish-docker-dev-to-gcr.yml b/.github/workflows/publish-docker-dev-to-gcr.yml new file mode 100644 index 00000000..2ac4f5d2 --- /dev/null +++ b/.github/workflows/publish-docker-dev-to-gcr.yml @@ -0,0 +1,88 @@ +name: Publish DEV Docker images +# READ https://gist.github.com/palewire/12c4b2b974ef735d22da7493cf7f4d37 + +on: + workflow_run: + workflows: [Publish BASE Docker images] + types: [completed] + +env: + GAR_ZONE: europe-southwest1 + GAR_REPO_NAME: docker-ckan-images + G_PROJECT_ID: oki-cloud + G_SERVICE_ACCOUT: sa-gh-action-registry + +jobs: + publish_dev_to_gcr: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + # Add "id-token" with the intended permissions. + permissions: + contents: 'read' + id-token: 'write' + + steps: + + - name: Checkout repository + uses: actions/checkout@v3 + + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + token_format: access_token + workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider + service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com + # Wait for all builds ... + access_token_lifetime: 3600s + + - name: Login to Artifact Registry + uses: docker/login-action@v1 + with: + registry: ${{ env.GAR_ZONE }}-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Set CKAN Versions + run: | + echo CKAN_v2_8=$(grep 'ENV GIT_BRANCH' ckan-base/2.8/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV + echo CKAN_v2_9=$(grep 'ENV GIT_BRANCH' ckan-base/2.9/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV + # TODO Update when there's an actual 2.10 released tag + echo CKAN_v2_10=2.10.0 >> $GITHUB_ENV + + - name: Build ckan-dev 2.8 and and ${{ env.CKAN_v2_8 }} + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.8/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.8 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_8 }} + - name: Build ckan-dev 2.9 and ${{ env.CKAN_v2_9 }} + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.9/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.9 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }} + - name: Build ckan-dev 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.9/Dockerfile.py2 + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.9-py2 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_9 }}-py2 + - name: Build ckan-dev 2.10 and ${{ env.CKAN_v2_10 }} + uses: docker/build-push-action@v4 + with: + context: ckan-dev + file: ckan-dev/2.10/Dockerfile + push: true + tags: | + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:2.10 + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:${{ env.CKAN_v2_10 }} diff --git a/.github/workflows/publish-docker-master.yml b/.github/workflows/publish-docker-master.yml index 5e5992e8..ac32a323 100644 --- a/.github/workflows/publish-docker-master.yml +++ b/.github/workflows/publish-docker-master.yml @@ -1,34 +1,55 @@ -name: Build and publish the master docker-ckan image +name: Publish daily master docker-ckan image on: schedule: - cron: '15 5 * * *' +env: + GAR_ZONE: europe-southwest1 + GAR_REPO_NAME: docker-ckan-images + G_PROJECT_ID: oki-cloud + G_SERVICE_ACCOUT: sa-gh-action-registry + jobs: build: runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub + - name: Checkout repository + uses: actions/checkout@v3 + + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + token_format: access_token + workload_identity_provider: projects/3555635429/locations/global/workloadIdentityPools/gh-action-registry-pool/providers/gh-provider + service_account: ${{ env.G_SERVICE_ACCOUT }}@${{ env.G_PROJECT_ID }}.iam.gserviceaccount.com + access_token_lifetime: 600s + + - name: Login to Artifact Registry uses: docker/login-action@v1 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: ${{ env.GAR_ZONE }}-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + - name: Build ckan-base master - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: ckan-base file: ckan-base/master/Dockerfile push: true tags: | - openknowledge/ckan-base:master + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-base:master + - name: Build ckan-dev master - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: ckan-dev file: ckan-dev/master/Dockerfile push: true tags: | - openknowledge/ckan-dev:master + ${{ env.GAR_ZONE }}-docker.pkg.dev/${{ env.G_PROJECT_ID }}/${{ env.GAR_REPO_NAME }}/openknowledge/ckan-dev:master diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml deleted file mode 100644 index 46c6a014..00000000 --- a/.github/workflows/publish-docker.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Build and publish all docker-ckan Images -on: - push: - branches: - - master - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set CKAN Versions - run: | - echo CKAN_v2_8=$(grep 'ENV GIT_BRANCH' ckan-base/2.8/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV - echo CKAN_v2_9=$(grep 'ENV GIT_BRANCH' ckan-base/2.9/Dockerfile | cut -d '-' -f 2) >> $GITHUB_ENV - # TODO Update when there's an actual 2.10 released tag - echo CKAN_v2_10=2.10.0 >> $GITHUB_ENV - - name: Build ckan-base 2.8 and ${{ env.CKAN_v2_8 }} - uses: docker/build-push-action@v2 - with: - context: ckan-base - file: ckan-base/2.8/Dockerfile - push: true - tags: | - openknowledge/ckan-base:2.8 - openknowledge/ckan-base:${{ env.CKAN_v2_8 }} - - name: Build ckan-base 2.9 and ${{ env.CKAN_v2_9 }} - uses: docker/build-push-action@v2 - with: - context: ckan-base - file: ckan-base/2.9/Dockerfile - push: true - tags: | - openknowledge/ckan-base:2.9 - openknowledge/ckan-base:${{ env.CKAN_v2_9 }} - - name: Build ckan-base 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 - uses: docker/build-push-action@v2 - with: - context: ckan-base - file: ckan-base/2.9/Dockerfile.py2 - push: true - tags: | - openknowledge/ckan-base:2.9-py2 - openknowledge/ckan-base:${{ env.CKAN_v2_9 }}-py2 - - name: Build ckan-base 2.10 and ${{ env.CKAN_v2_10 }} - uses: docker/build-push-action@v2 - with: - context: ckan-base - file: ckan-base/2.10/Dockerfile - push: true - tags: | - openknowledge/ckan-base:2.10 - openknowledge/ckan-base:${{ env.CKAN_v2_10 }} - - name: Build ckan-dev 2.8 and and ${{ env.CKAN_v2_8 }} - uses: docker/build-push-action@v2 - with: - context: ckan-dev - file: ckan-dev/2.8/Dockerfile - push: true - tags: | - openknowledge/ckan-dev:2.8 - openknowledge/ckan-dev:${{ env.CKAN_v2_8 }} - - name: Build ckan-dev 2.9 and ${{ env.CKAN_v2_9 }} - uses: docker/build-push-action@v2 - with: - context: ckan-dev - file: ckan-dev/2.9/Dockerfile - push: true - tags: | - openknowledge/ckan-dev:2.9 - openknowledge/ckan-dev:${{ env.CKAN_v2_9 }} - - name: Build ckan-dev 2.9-py2 and ${{ env.CKAN_v2_9 }}-py2 - uses: docker/build-push-action@v2 - with: - context: ckan-dev - file: ckan-dev/2.9/Dockerfile.py2 - push: true - tags: | - openknowledge/ckan-dev:2.9-py2 - openknowledge/ckan-dev:${{ env.CKAN_v2_9 }}-py2 - - name: Build ckan-dev 2.10 and ${{ env.CKAN_v2_10 }} - uses: docker/build-push-action@v2 - with: - context: ckan-dev - file: ckan-dev/2.10/Dockerfile - push: true - tags: | - openknowledge/ckan-dev:2.10 - openknowledge/ckan-dev:${{ env.CKAN_v2_10 }} diff --git a/README.md b/README.md index f8ba3141..b7cb51f3 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,17 @@ The Docker images used to build your CKAN project are located in the `ckan/` fol From these two base images you can build your own customized image tailored to your project, installing any extensions and extra requirements needed. +### Pulling images + +Docker CKAN images lives at Google Artifact Registry. You can pull them using the following command: + +``` +docker pull europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images/openknowledge/ckan-base:2.8 +``` + +You can change the last part `ckan-base:2.8` for any of the available images. + + ### Extending the base images To perform extra initialization steps you can add scripts to your custom images and copy them to the `/docker-entrypoint.d` folder (The folder should be created for you when you build the image). Any `*.sh` and `*.py` file in that folder will be executed just after the main initialization script ([`prerun.py`](https://github.com/okfn/docker-ckan/blob/master/ckan-base/setup/prerun.py)) is executed and just before the web server and supervisor processes are started. @@ -147,7 +158,7 @@ paster --plugin=ckanext-validation validation init-db -c $CKAN_INI And then in our `Dockerfile` we install the extension and copy the initialization scripts: ```Dockerfile -FROM openknowledge/ckan-dev:2.9 +FROM europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images/openknowledge/ckan-dev:2.9 RUN pip install -e git+https://github.com/frictionlessdata/ckanext-validation.git#egg=ckanext-validation && \ pip install -r https://raw.githubusercontent.com/frictionlessdata/ckanext-validation/master/requirements.txt diff --git a/ckan-dev/2.10/Dockerfile b/ckan-dev/2.10/Dockerfile index 99318642..c8e8a881 100644 --- a/ckan-dev/2.10/Dockerfile +++ b/ckan-dev/2.10/Dockerfile @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:2.10 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:2.10 MAINTAINER Open Knowledge Foundation diff --git a/ckan-dev/2.8/Dockerfile b/ckan-dev/2.8/Dockerfile index cec5e837..a3a165c6 100644 --- a/ckan-dev/2.8/Dockerfile +++ b/ckan-dev/2.8/Dockerfile @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:2.8 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:2.8 MAINTAINER Open Knowledge Foundation diff --git a/ckan-dev/2.9/Dockerfile b/ckan-dev/2.9/Dockerfile index 2b21bca2..bf423ff7 100644 --- a/ckan-dev/2.9/Dockerfile +++ b/ckan-dev/2.9/Dockerfile @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:2.9 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:2.9 MAINTAINER Open Knowledge Foundation diff --git a/ckan-dev/2.9/Dockerfile.py2 b/ckan-dev/2.9/Dockerfile.py2 index 6c0c00bf..127b7d7a 100644 --- a/ckan-dev/2.9/Dockerfile.py2 +++ b/ckan-dev/2.9/Dockerfile.py2 @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:2.9-py2 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:2.9-py2 MAINTAINER Open Knowledge Foundation diff --git a/ckan-dev/master/Dockerfile b/ckan-dev/master/Dockerfile index f3e45341..b7cb2092 100644 --- a/ckan-dev/master/Dockerfile +++ b/ckan-dev/master/Dockerfile @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:master +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:master MAINTAINER Open Knowledge Foundation diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 0c2de0db..bb353649 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-base:2.10 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-base:2.10 MAINTAINER Your Name Here diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev index cd3b296b..d78dfbbb 100644 --- a/ckan/Dockerfile.dev +++ b/ckan/Dockerfile.dev @@ -1,4 +1,5 @@ -FROM openknowledge/ckan-dev:2.10 +ARG BASE_IMG=europe-southwest1-docker.pkg.dev/oki-cloud/docker-ckan-images +FROM ${BASE_IMG}/openknowledge/ckan-dev:2.10 MAINTAINER Your Name Here