Postgres implementation

.github/workflows/oci-dist-spec-content-discovery.yml

@@ -1,18 +1,41 @@
-name: OCI Spec - Content Discovery
+name: OCI Distribution Spec
 
 on:
   pull_request:
 
 jobs:
   content-discovery:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          PGUSER: postgres
+          POSTGRES_DB: open_registry
+          POSTGRES_PASSWORD: Qwerty@123
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
-      - name: set up docker
-        uses: docker-practice/actions-setup-docker@0.0.1
-        with:
-          docker_version: 18.09
-          docker_channel: stable
       - uses: actions/checkout@v2
+      - name: Install Migrate CLI
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+      - name: Setup PostgreSQL Database
+        run: |
+          IP=`hostname -I | awk '{print $1}'`
+          POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable
+          migrate -database ${POSTGRESQL_URL} -path db/migrations up
+        env:
+           PGDATABASE: open_registry
+           PGPASSWORD: Qwerty@123
+           PGUSER: postgres
       - name: start distribution server
         run: |
           IP=`hostname -I | awk '{print $1}'`
@@ -21,8 +44,10 @@ jobs:
           DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)"
           docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" .
           sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example
+          sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example
           echo CI_SYS_ADDR=$IP:5000 >> env-vars.example
-          docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}"
+          echo REGISTRY_STORAGE_DELETE_ENABLED=true 
+          docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}"
           sleep 5
           curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup"
       - name: Run OCI Distribution Spec conformance tests

.github/workflows/oci-dist-spec-content-management.yml

@@ -1,18 +1,41 @@
-name: OCI Spec - Content Management
+name: OCI Distribution Spec
 
 on:
   pull_request:
 
 jobs:
   content-management:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          PGUSER: postgres
+          POSTGRES_DB: open_registry
+          POSTGRES_PASSWORD: Qwerty@123
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
-      - name: set up docker
-        uses: docker-practice/actions-setup-docker@0.0.1
-        with:
-          docker_version: 18.09
-          docker_channel: stable
       - uses: actions/checkout@v2
+      - name: Install Migrate CLI
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+      - name: Setup PostgreSQL Database
+        run: |
+          IP=`hostname -I | awk '{print $1}'`
+          POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable
+          migrate -database ${POSTGRESQL_URL} -path db/migrations up
+        env:
+           PGDATABASE: open_registry
+           PGPASSWORD: Qwerty@123
+           PGUSER: postgres
       - name: start distribution server
         run: |
           IP=`hostname -I | awk '{print $1}'`
@@ -21,8 +44,10 @@ jobs:
           DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)"
           docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" .
           sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example
+          sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example
           echo CI_SYS_ADDR=$IP:5000 >> env-vars.example
-          docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}"
+          echo REGISTRY_STORAGE_DELETE_ENABLED=true 
+          docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}"
           sleep 5
           curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup"
       - name: Run OCI Distribution Spec conformance tests

.github/workflows/oci-dist-spec-pull.yml

@@ -1,18 +1,41 @@
-name: OCI Spec - Pull
+name: OCI Distribution Spec
 
 on:
   pull_request:
 
 jobs:
   pull:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          PGUSER: postgres
+          POSTGRES_DB: open_registry
+          POSTGRES_PASSWORD: Qwerty@123
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
-      - name: set up docker
-        uses: docker-practice/actions-setup-docker@0.0.1
-        with:
-          docker_version: 18.09
-          docker_channel: stable
       - uses: actions/checkout@v2
+      - name: Install Migrate CLI
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+      - name: Setup PostgreSQL Database
+        run: |
+          IP=`hostname -I | awk '{print $1}'`
+          POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable
+          migrate -database ${POSTGRESQL_URL} -path db/migrations up
+        env:
+           PGDATABASE: open_registry
+           PGPASSWORD: Qwerty@123
+           PGUSER: postgres
       - name: start distribution server
         run: |
           IP=`hostname -I | awk '{print $1}'`
@@ -21,8 +44,10 @@ jobs:
           DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)"
           docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" .
           sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example
+          sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example
           echo CI_SYS_ADDR=$IP:5000 >> env-vars.example
-          docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}"
+          echo REGISTRY_STORAGE_DELETE_ENABLED=true 
+          docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}"
           sleep 5
           curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup"
       - name: Run OCI Distribution Spec conformance tests

.github/workflows/oci-dist-spec-push.yml

@@ -1,18 +1,41 @@
-name: OCI Spec - Push
+name: OCI Distribution Spec
 
 on:
   pull_request:
 
 jobs:
   push:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres
+        env:
+          PGUSER: postgres
+          POSTGRES_DB: open_registry
+          POSTGRES_PASSWORD: Qwerty@123
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
-      - name: set up docker
-        uses: docker-practice/actions-setup-docker@0.0.1
-        with:
-          docker_version: 18.09
-          docker_channel: stable
       - uses: actions/checkout@v2
+      - name: Install Migrate CLI
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+      - name: Setup PostgreSQL Database
+        run: |
+          IP=`hostname -I | awk '{print $1}'`
+          POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable
+          migrate -database ${POSTGRESQL_URL} -path db/migrations up
+        env:
+           PGDATABASE: open_registry
+           PGPASSWORD: Qwerty@123
+           PGUSER: postgres
       - name: start distribution server
         run: |
           IP=`hostname -I | awk '{print $1}'`
@@ -21,8 +44,10 @@ jobs:
           DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)"
           docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" .
           sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example
+          sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example
           echo CI_SYS_ADDR=$IP:5000 >> env-vars.example
-          docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}"
+          echo REGISTRY_STORAGE_DELETE_ENABLED=true 
+          docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}"
           sleep 5
           curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup"
       - name: Run OCI Distribution Spec conformance tests

auth/auth.go

@@ -3,6 +3,7 @@ package auth
 import (
 	"github.com/containerish/OpenRegistry/cache"
 	"github.com/containerish/OpenRegistry/config"
+	"github.com/containerish/OpenRegistry/store/postgres"
 	"github.com/containerish/OpenRegistry/telemetry"
 	"github.com/labstack/echo/v4"
 )
@@ -18,13 +19,19 @@ type Authentication interface {
 }
 
 type auth struct {
-	store  cache.Store
-	c      *config.RegistryConfig
-	logger telemetry.Logger
+	pgStore postgres.PersistentStore
+	store   cache.Store
+	c       *config.RegistryConfig
+	logger  telemetry.Logger
 }
 
 // New is the constructor function returns an Authentication implementation
-func New(s cache.Store, c *config.RegistryConfig, logger telemetry.Logger) Authentication {
-	a := &auth{store: s, c: c, logger: logger}
+func New(
+	s cache.Store,
+	c *config.RegistryConfig,
+	pgStore postgres.PersistentStore,
+	logger telemetry.Logger,
+) Authentication {
+	a := &auth{store: s, c: c, pgStore: pgStore, logger: logger}
 	return a
 }

auth/signin.go

@@ -2,7 +2,6 @@ package auth
 
 import (
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"time"
 
@@ -47,35 +46,33 @@ func (a *auth) SignIn(ctx echo.Context) error {
 				"error": err.Error(),
 			})
 		}
-		key = fmt.Sprintf("%s/%s", UserNameSpace, user.Email)
+		key = user.Email
 	} else {
-		key = fmt.Sprintf("%s/%s", UserNameSpace, user.Username)
+		key = user.Username
 	}
 
-	bz, err := a.store.Get([]byte(key))
+	//bz, err := a.store.Get([]byte(key))
+	userFromDb, err := a.pgStore.GetUser(ctx.Request().Context(), key)
 	if err != nil {
 		ctx.Set(types.HttpEndpointErrorKey, err.Error())
 		return ctx.JSON(http.StatusBadRequest, echo.Map{
 			"error": err.Error(),
 		})
 	}
 
-	var userFromDb User
-	if err := json.Unmarshal(bz, &userFromDb); err != nil {
-		ctx.Set(types.HttpEndpointErrorKey, err.Error())
-		return ctx.JSON(http.StatusInternalServerError, echo.Map{
-			"error": err.Error(),
-		})
-	}
-
 	if !a.verifyPassword(userFromDb.Password, user.Password) {
 		errMsg := "invalid password"
 		ctx.Set(types.HttpEndpointErrorKey, errMsg)
 		return ctx.JSON(http.StatusUnauthorized, errMsg)
 	}
 
 	tokenLife := time.Now().Add(time.Hour * 24 * 14).Unix()
-	token, err := a.newToken(userFromDb, tokenLife)
+	uu := User{
+		Email:    userFromDb.Email,
+		Username: userFromDb.Username,
+	}
+
+	token, err := a.newToken(uu, tokenLife)
 	if err != nil {
 		ctx.Set(types.HttpEndpointErrorKey, err.Error())
 		return ctx.JSON(http.StatusInternalServerError, echo.Map{

auth/signup.go

@@ -178,24 +178,15 @@ func (a *auth) SignUp(ctx echo.Context) error {
 		})
 	}
 	u.Password = hpwd
-	bz, err = json.Marshal(u)
-	if err != nil {
-		ctx.Set(types.HttpEndpointErrorKey, err.Error())
-		return ctx.JSON(http.StatusInternalServerError, echo.Map{
-			"error": err.Error(),
-		})
-	}
 
-	key := fmt.Sprintf("%s/%s", UserNameSpace, u.Username)
-	if err := a.store.Set([]byte(key), bz); err != nil {
-		ctx.Set(types.HttpEndpointErrorKey, err.Error())
-		return ctx.JSON(http.StatusInternalServerError, echo.Map{
-			"error": err.Error(),
-		})
+	newUser := &types.User{
+		Email:    u.Email,
+		Username: u.Username,
+		Password: u.Password,
 	}
 
-	key = fmt.Sprintf("%s/%s", UserNameSpace, u.Email)
-	if err := a.store.Set([]byte(key), bz); err != nil {
+	err = a.pgStore.AddUser(ctx.Request().Context(), newUser)
+	if err != nil {
 		ctx.Set(types.HttpEndpointErrorKey, err.Error())
 		return ctx.JSON(http.StatusInternalServerError, echo.Map{
 			"error": err.Error(),

auth/validate_user.go

@@ -1,7 +1,7 @@
 package auth
 
 import (
-	"encoding/json"
+	"context"
 	"fmt"
 	"time"
 
@@ -13,17 +13,11 @@ func (a *auth) validateUser(username, password string) (map[string]interface{},
 		return nil, fmt.Errorf("Email/Password cannot be empty")
 	}
 
-	key := fmt.Sprintf("%s/%s", UserNameSpace, username)
-	bz, err := a.store.Get([]byte(key))
+	userFromDb, err := a.pgStore.GetUser(context.Background(), username)
 	if err != nil {
 		return nil, err
 	}
 
-	var userFromDb User
-	if err := json.Unmarshal(bz, &userFromDb); err != nil {
-		return nil, err
-	}
-
 	if !a.verifyPassword(userFromDb.Password, password) {
 		return nil, fmt.Errorf("invalid password")
 	}