diff --git a/.github/workflows/oci-dist-spec-content-discovery.yml b/.github/workflows/oci-dist-spec-content-discovery.yml index e156ab0f..a0a7e4f1 100644 --- a/.github/workflows/oci-dist-spec-content-discovery.yml +++ b/.github/workflows/oci-dist-spec-content-discovery.yml @@ -1,4 +1,4 @@ -name: OCI Spec - Content Discovery +name: OCI Distribution Spec on: pull_request: @@ -6,13 +6,36 @@ on: jobs: content-discovery: runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + PGUSER: postgres + POSTGRES_DB: open_registry + POSTGRES_PASSWORD: Qwerty@123 + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: - - name: set up docker - uses: docker-practice/actions-setup-docker@0.0.1 - with: - docker_version: 18.09 - docker_channel: stable - uses: actions/checkout@v2 + - name: Install Migrate CLI + run: | + curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz + sudo mv migrate /usr/bin/migrate + - name: Setup PostgreSQL Database + run: | + IP=`hostname -I | awk '{print $1}'` + POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable + migrate -database ${POSTGRESQL_URL} -path db/migrations up + env: + PGDATABASE: open_registry + PGPASSWORD: Qwerty@123 + PGUSER: postgres - name: start distribution server run: | IP=`hostname -I | awk '{print $1}'` @@ -21,8 +44,10 @@ jobs: DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)" docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" . sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example + sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example echo CI_SYS_ADDR=$IP:5000 >> env-vars.example - docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}" + echo REGISTRY_STORAGE_DELETE_ENABLED=true + docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}" sleep 5 curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup" - name: Run OCI Distribution Spec conformance tests diff --git a/.github/workflows/oci-dist-spec-content-management.yml b/.github/workflows/oci-dist-spec-content-management.yml index 538c67b2..8a7f7101 100644 --- a/.github/workflows/oci-dist-spec-content-management.yml +++ b/.github/workflows/oci-dist-spec-content-management.yml @@ -1,4 +1,4 @@ -name: OCI Spec - Content Management +name: OCI Distribution Spec on: pull_request: @@ -6,13 +6,36 @@ on: jobs: content-management: runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + PGUSER: postgres + POSTGRES_DB: open_registry + POSTGRES_PASSWORD: Qwerty@123 + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: - - name: set up docker - uses: docker-practice/actions-setup-docker@0.0.1 - with: - docker_version: 18.09 - docker_channel: stable - uses: actions/checkout@v2 + - name: Install Migrate CLI + run: | + curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz + sudo mv migrate /usr/bin/migrate + - name: Setup PostgreSQL Database + run: | + IP=`hostname -I | awk '{print $1}'` + POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable + migrate -database ${POSTGRESQL_URL} -path db/migrations up + env: + PGDATABASE: open_registry + PGPASSWORD: Qwerty@123 + PGUSER: postgres - name: start distribution server run: | IP=`hostname -I | awk '{print $1}'` @@ -21,8 +44,10 @@ jobs: DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)" docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" . sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example + sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example echo CI_SYS_ADDR=$IP:5000 >> env-vars.example - docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}" + echo REGISTRY_STORAGE_DELETE_ENABLED=true + docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}" sleep 5 curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup" - name: Run OCI Distribution Spec conformance tests diff --git a/.github/workflows/oci-dist-spec-pull.yml b/.github/workflows/oci-dist-spec-pull.yml index efec79c6..ed3e33df 100644 --- a/.github/workflows/oci-dist-spec-pull.yml +++ b/.github/workflows/oci-dist-spec-pull.yml @@ -1,4 +1,4 @@ -name: OCI Spec - Pull +name: OCI Distribution Spec on: pull_request: @@ -6,13 +6,36 @@ on: jobs: pull: runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + PGUSER: postgres + POSTGRES_DB: open_registry + POSTGRES_PASSWORD: Qwerty@123 + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: - - name: set up docker - uses: docker-practice/actions-setup-docker@0.0.1 - with: - docker_version: 18.09 - docker_channel: stable - uses: actions/checkout@v2 + - name: Install Migrate CLI + run: | + curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz + sudo mv migrate /usr/bin/migrate + - name: Setup PostgreSQL Database + run: | + IP=`hostname -I | awk '{print $1}'` + POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable + migrate -database ${POSTGRESQL_URL} -path db/migrations up + env: + PGDATABASE: open_registry + PGPASSWORD: Qwerty@123 + PGUSER: postgres - name: start distribution server run: | IP=`hostname -I | awk '{print $1}'` @@ -21,8 +44,10 @@ jobs: DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)" docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" . sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example + sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example echo CI_SYS_ADDR=$IP:5000 >> env-vars.example - docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}" + echo REGISTRY_STORAGE_DELETE_ENABLED=true + docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}" sleep 5 curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup" - name: Run OCI Distribution Spec conformance tests diff --git a/.github/workflows/oci-dist-spec-push.yml b/.github/workflows/oci-dist-spec-push.yml index 885863a3..1b4d39a6 100644 --- a/.github/workflows/oci-dist-spec-push.yml +++ b/.github/workflows/oci-dist-spec-push.yml @@ -1,4 +1,4 @@ -name: OCI Spec - Push +name: OCI Distribution Spec on: pull_request: @@ -6,13 +6,36 @@ on: jobs: push: runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + PGUSER: postgres + POSTGRES_DB: open_registry + POSTGRES_PASSWORD: Qwerty@123 + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: - - name: set up docker - uses: docker-practice/actions-setup-docker@0.0.1 - with: - docker_version: 18.09 - docker_channel: stable - uses: actions/checkout@v2 + - name: Install Migrate CLI + run: | + curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz + sudo mv migrate /usr/bin/migrate + - name: Setup PostgreSQL Database + run: | + IP=`hostname -I | awk '{print $1}'` + POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable + migrate -database ${POSTGRESQL_URL} -path db/migrations up + env: + PGDATABASE: open_registry + PGPASSWORD: Qwerty@123 + PGUSER: postgres - name: start distribution server run: | IP=`hostname -I | awk '{print $1}'` @@ -21,8 +44,10 @@ jobs: DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)" docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" . sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example + sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example echo CI_SYS_ADDR=$IP:5000 >> env-vars.example - docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}" + echo REGISTRY_STORAGE_DELETE_ENABLED=true + docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}" sleep 5 curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup" - name: Run OCI Distribution Spec conformance tests diff --git a/auth/auth.go b/auth/auth.go index 9ad703e4..76fbd64a 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -3,6 +3,7 @@ package auth import ( "github.com/containerish/OpenRegistry/cache" "github.com/containerish/OpenRegistry/config" + "github.com/containerish/OpenRegistry/store/postgres" "github.com/containerish/OpenRegistry/telemetry" "github.com/labstack/echo/v4" ) @@ -18,13 +19,19 @@ type Authentication interface { } type auth struct { - store cache.Store - c *config.RegistryConfig - logger telemetry.Logger + pgStore postgres.PersistentStore + store cache.Store + c *config.RegistryConfig + logger telemetry.Logger } // New is the constructor function returns an Authentication implementation -func New(s cache.Store, c *config.RegistryConfig, logger telemetry.Logger) Authentication { - a := &auth{store: s, c: c, logger: logger} +func New( + s cache.Store, + c *config.RegistryConfig, + pgStore postgres.PersistentStore, + logger telemetry.Logger, +) Authentication { + a := &auth{store: s, c: c, pgStore: pgStore, logger: logger} return a } diff --git a/auth/signin.go b/auth/signin.go index 7b9bde9b..1d738619 100644 --- a/auth/signin.go +++ b/auth/signin.go @@ -2,7 +2,6 @@ package auth import ( "encoding/json" - "fmt" "net/http" "time" @@ -47,12 +46,13 @@ func (a *auth) SignIn(ctx echo.Context) error { "error": err.Error(), }) } - key = fmt.Sprintf("%s/%s", UserNameSpace, user.Email) + key = user.Email } else { - key = fmt.Sprintf("%s/%s", UserNameSpace, user.Username) + key = user.Username } - bz, err := a.store.Get([]byte(key)) + //bz, err := a.store.Get([]byte(key)) + userFromDb, err := a.pgStore.GetUser(ctx.Request().Context(), key) if err != nil { ctx.Set(types.HttpEndpointErrorKey, err.Error()) return ctx.JSON(http.StatusBadRequest, echo.Map{ @@ -60,14 +60,6 @@ func (a *auth) SignIn(ctx echo.Context) error { }) } - var userFromDb User - if err := json.Unmarshal(bz, &userFromDb); err != nil { - ctx.Set(types.HttpEndpointErrorKey, err.Error()) - return ctx.JSON(http.StatusInternalServerError, echo.Map{ - "error": err.Error(), - }) - } - if !a.verifyPassword(userFromDb.Password, user.Password) { errMsg := "invalid password" ctx.Set(types.HttpEndpointErrorKey, errMsg) @@ -75,7 +67,12 @@ func (a *auth) SignIn(ctx echo.Context) error { } tokenLife := time.Now().Add(time.Hour * 24 * 14).Unix() - token, err := a.newToken(userFromDb, tokenLife) + uu := User{ + Email: userFromDb.Email, + Username: userFromDb.Username, + } + + token, err := a.newToken(uu, tokenLife) if err != nil { ctx.Set(types.HttpEndpointErrorKey, err.Error()) return ctx.JSON(http.StatusInternalServerError, echo.Map{ diff --git a/auth/signup.go b/auth/signup.go index ef69b9e1..c2bb80e6 100644 --- a/auth/signup.go +++ b/auth/signup.go @@ -178,24 +178,15 @@ func (a *auth) SignUp(ctx echo.Context) error { }) } u.Password = hpwd - bz, err = json.Marshal(u) - if err != nil { - ctx.Set(types.HttpEndpointErrorKey, err.Error()) - return ctx.JSON(http.StatusInternalServerError, echo.Map{ - "error": err.Error(), - }) - } - key := fmt.Sprintf("%s/%s", UserNameSpace, u.Username) - if err := a.store.Set([]byte(key), bz); err != nil { - ctx.Set(types.HttpEndpointErrorKey, err.Error()) - return ctx.JSON(http.StatusInternalServerError, echo.Map{ - "error": err.Error(), - }) + newUser := &types.User{ + Email: u.Email, + Username: u.Username, + Password: u.Password, } - key = fmt.Sprintf("%s/%s", UserNameSpace, u.Email) - if err := a.store.Set([]byte(key), bz); err != nil { + err = a.pgStore.AddUser(ctx.Request().Context(), newUser) + if err != nil { ctx.Set(types.HttpEndpointErrorKey, err.Error()) return ctx.JSON(http.StatusInternalServerError, echo.Map{ "error": err.Error(), diff --git a/auth/validate_user.go b/auth/validate_user.go index 9c3488c4..32f303ea 100644 --- a/auth/validate_user.go +++ b/auth/validate_user.go @@ -1,7 +1,7 @@ package auth import ( - "encoding/json" + "context" "fmt" "time" @@ -13,17 +13,11 @@ func (a *auth) validateUser(username, password string) (map[string]interface{}, return nil, fmt.Errorf("Email/Password cannot be empty") } - key := fmt.Sprintf("%s/%s", UserNameSpace, username) - bz, err := a.store.Get([]byte(key)) + userFromDb, err := a.pgStore.GetUser(context.Background(), username) if err != nil { return nil, err } - var userFromDb User - if err := json.Unmarshal(bz, &userFromDb); err != nil { - return nil, err - } - if !a.verifyPassword(userFromDb.Password, password) { return nil, fmt.Errorf("invalid password") } diff --git a/cache/store.go b/cache/store.go index 6917287f..c6f5dd41 100644 --- a/cache/store.go +++ b/cache/store.go @@ -19,11 +19,14 @@ type dataStore struct { type Store interface { Get(key []byte) ([]byte, error) Set(key, value []byte) error + GetDigest(digest string) (*types.LayerRef, error) SetDigest(digest, skylink string) error DeleteDigest(digest string) error + DeleteLayer(namespace, digest string) error Update(key, value []byte) error + ListAll() ([]byte, error) ListWithPrefix(prefix []byte) ([]byte, error) Delete(key []byte) error @@ -50,6 +53,159 @@ func New(storeLocation string) (Store, error) { return &dataStore{db: db}, nil } +func (ds *dataStore) Get(key []byte) ([]byte, error) { + var res []byte + + err := ds.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(key) + if err != nil { + return err + } + err = item.Value(func(v []byte) error { + res = make([]byte, len(v)) + copy(res, v) + return nil + }) + return err + }) + if err != nil { + return nil, err + } + + return res, nil +} + +func (ds *dataStore) GetDigest(digest string) (*types.LayerRef, error) { + key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) + bz, err := ds.Get([]byte(key)) + if err != nil { + return nil, err + } + + var layerRef types.LayerRef + err = json.Unmarshal(bz, &layerRef) + return &layerRef, err +} + +func (ds *dataStore) GetSkynetURL(key, ref string) (string, error) { + var res []byte + err := ds.db.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) + if err != nil { + return err + } + + return item.Value(func(v []byte) error { + res = make([]byte, len(v)) + copy(res, v) + return nil + }) + }) + + if err != nil { + return "", err + } + + var md types.Metadata + err = json.Unmarshal(res, &md) + if err != nil { + return "", err + } + + return md.FindLinkForDigest(ref) +} + +func (ds *dataStore) ListAll() ([]byte, error) { + var buf []types.Metadata + + err := ds.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchSize = 10 + it := txn.NewIterator(opts) + defer it.Close() + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + err := item.Value(func(v []byte) error { + var md types.Metadata + if err := json.Unmarshal(v, &md); err != nil { + return err + } + buf = append(buf, md) + return nil + }) + + if err != nil { + return err + } + } + return nil + }) + if err != nil { + return nil, err + } + + return json.Marshal(buf) +} + +func (ds *dataStore) ListWithPrefix(prefix []byte) ([]byte, error) { + + var buf []byte + + err := ds.db.View(func(txn *badger.Txn) error { + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + item := it.Item() + err := item.Value(func(v []byte) error { + buf = make([]byte, len(v)) + copy(buf, v) + return nil + }) + if err != nil { + return err + } + } + return nil + }) + return buf, err +} + +func (ds *dataStore) ResolveManifestRef(namespace, ref string) (string, error) { + var res []byte + fn := func(txn *badger.Txn) error { + item, err := txn.Get([]byte(namespace)) + if err != nil { + return err + } + + return item.Value(func(v []byte) error { + res = make([]byte, len(v)) + copy(res, v) + return nil + }) + } + + if err := ds.db.View(fn); err != nil { + return "", err + } + + var md types.Metadata + err := json.Unmarshal(res, &md) + if err != nil { + return "", err + } + + for _, c := range md.Manifest.Config { + mdRef := c.Reference + mdDigest := c.Digest + if ref == mdRef || ref == mdDigest { + return c.SkynetLink, nil + } + } + + return "", fmt.Errorf("ref not found") +} + func (ds *dataStore) LayerDigests(ctx echo.Context) error { bz, err := ds.ListWithPrefix([]byte(layerDigestNamespace)) if err != nil { @@ -59,6 +215,39 @@ func (ds *dataStore) LayerDigests(ctx echo.Context) error { return ctx.JSONBlob(http.StatusOK, bz) } +// QueryMetaData is similar to ListWithPrefix with only a few changes +// making these changes in ListWithPrefix was causing the oci tests to fail +func (ds *dataStore) QueryMetaData(prefix []byte) ([]byte, error) { + + var mds []types.Metadata + + err := ds.db.View(func(txn *badger.Txn) error { + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + var md types.Metadata + item := it.Item() + err := item.Value(func(v []byte) error { + err := json.Unmarshal(v, &md) + if err != nil { + return err + } + mds = append(mds, md) + return nil + }) + if err != nil { + return err + } + } + return nil + }) + + if err != nil { + return nil, err + } + return json.Marshal(mds) +} + func (ds *dataStore) Metadata(ctx echo.Context) error { key := ctx.QueryParam("namespace") dig := ctx.QueryParam("digest") @@ -115,32 +304,30 @@ func (ds *dataStore) Metadata(ctx echo.Context) error { return ctx.JSON(http.StatusOK, reg) } -func (ds *dataStore) DeleteLayer(namespace, digest string) error { - bz, err := ds.Get([]byte(namespace)) - if err != nil { - return err - } +const layerDigestNamespace = "layers/digests" - var md types.Metadata - if err = json.Unmarshal(bz, &md); err != nil { +func (ds *dataStore) Set(key, value []byte) error { + txn := ds.db.NewTransaction(true) + + if err := txn.Set(key, value); err != nil { return err } - var match bool - for i, v := range md.Manifest.Layers { - if v.Digest == digest { - l := len(md.Manifest.Layers) - md.Manifest.Layers[i] = md.Manifest.Layers[l-1] - md.Manifest.Layers = md.Manifest.Layers[:l-1] - match = true - break - } + return txn.Commit() +} + +func (ds *dataStore) SetDigest(digest, skylink string) error { + key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) + value := types.LayerRef{ + Digest: digest, + Skylink: skylink, } - if !match { - return fmt.Errorf("layer/blob not found for digest %s", digest) + + if err := ds.Set([]byte(key), value.Bytes()); err != nil { + return err } - return ds.Set([]byte(namespace), md.Bytes()) + return nil } func (ds *dataStore) Update(key, value []byte) error { @@ -200,30 +387,6 @@ EndLoop: return ds.Set(key, bz) } -func (ds *dataStore) removeDuplicateLayers(src, dst []*types.Layer) ([]*types.Layer, error) { - size := uint(len(src) + len(dst)) - - if size > math.MaxInt16 { - return nil, fmt.Errorf("ERROR_TOO_MANY_LAYERS") - } - - list := make([]*types.Layer, size) - list = append(list, src...) - list = append(list, dst...) - - seenMap := make(map[string]bool) - var layers []*types.Layer - - for _, l := range list { - if l != nil && !seenMap[l.Digest] { - seenMap[l.Digest] = true - layers = append(layers, l) - } - } - - return layers, nil -} - func (ds *dataStore) UpdateManifestRef(namespace, ref string) error { var res []byte fn := func(txn *badger.Txn) error { @@ -270,222 +433,62 @@ func (ds *dataStore) UpdateManifestRef(namespace, ref string) error { return ds.Set([]byte(namespace), md.Bytes()) } -func (ds *dataStore) ResolveManifestRef(namespace, ref string) (string, error) { - var res []byte - fn := func(txn *badger.Txn) error { - item, err := txn.Get([]byte(namespace)) - if err != nil { - return err - } +func (ds *dataStore) removeDuplicateLayers(src, dst []*types.Layer) ([]*types.Layer, error) { + size := uint(len(src) + len(dst)) - return item.Value(func(v []byte) error { - res = make([]byte, len(v)) - copy(res, v) - return nil - }) + if size > math.MaxInt16 { + return nil, fmt.Errorf("ERROR_TOO_MANY_LAYERS") } - if err := ds.db.View(fn); err != nil { - return "", err - } + list := make([]*types.Layer, size) + list = append(list, src...) + list = append(list, dst...) - var md types.Metadata - err := json.Unmarshal(res, &md) - if err != nil { - return "", err - } + seenMap := make(map[string]bool) + var layers []*types.Layer - for _, c := range md.Manifest.Config { - mdRef := c.Reference - mdDigest := c.Digest - if ref == mdRef || ref == mdDigest { - return c.SkynetLink, nil + for _, l := range list { + if l != nil && !seenMap[l.Digest] { + seenMap[l.Digest] = true + layers = append(layers, l) } } - return "", fmt.Errorf("ref not found") -} - -const layerDigestNamespace = "layers/digests" - -func (ds *dataStore) SetDigest(digest, skylink string) error { - key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) - value := types.LayerRef{ - Digest: digest, - Skylink: skylink, - } - - if err := ds.Set([]byte(key), value.Bytes()); err != nil { - return err - } - - return nil -} - -func (ds *dataStore) DeleteDigest(digest string) error { - key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) - - return ds.Delete([]byte(key)) -} - -func (ds *dataStore) GetDigest(digest string) (*types.LayerRef, error) { - key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) - bz, err := ds.Get([]byte(key)) - if err != nil { - return nil, err - } - - var layerRef types.LayerRef - err = json.Unmarshal(bz, &layerRef) - return &layerRef, err + return layers, nil } -func (ds *dataStore) GetSkynetURL(key, ref string) (string, error) { - var res []byte - err := ds.db.View(func(txn *badger.Txn) error { - item, err := txn.Get([]byte(key)) - if err != nil { - return err - } - - return item.Value(func(v []byte) error { - res = make([]byte, len(v)) - copy(res, v) - return nil - }) - }) - +func (ds *dataStore) DeleteLayer(namespace, digest string) error { + bz, err := ds.Get([]byte(namespace)) if err != nil { - return "", err + return err } var md types.Metadata - err = json.Unmarshal(res, &md) - if err != nil { - return "", err - } - - return md.FindLinkForDigest(ref) -} - -func (ds *dataStore) Set(key, value []byte) error { - txn := ds.db.NewTransaction(true) - - if err := txn.Set(key, value); err != nil { + if err = json.Unmarshal(bz, &md); err != nil { return err } - return txn.Commit() -} - -func (ds *dataStore) Get(key []byte) ([]byte, error) { - var res []byte - - err := ds.db.View(func(txn *badger.Txn) error { - item, err := txn.Get(key) - if err != nil { - return err + var match bool + for i, v := range md.Manifest.Layers { + if v.Digest == digest { + l := len(md.Manifest.Layers) + md.Manifest.Layers[i] = md.Manifest.Layers[l-1] + md.Manifest.Layers = md.Manifest.Layers[:l-1] + match = true + break } - err = item.Value(func(v []byte) error { - res = make([]byte, len(v)) - copy(res, v) - return nil - }) - return err - }) - if err != nil { - return nil, err } - - return res, nil -} - -func (ds *dataStore) ListAll() ([]byte, error) { - var buf []types.Metadata - - err := ds.db.View(func(txn *badger.Txn) error { - opts := badger.DefaultIteratorOptions - opts.PrefetchSize = 10 - it := txn.NewIterator(opts) - defer it.Close() - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - err := item.Value(func(v []byte) error { - var md types.Metadata - if err := json.Unmarshal(v, &md); err != nil { - return err - } - buf = append(buf, md) - return nil - }) - - if err != nil { - return err - } - } - return nil - }) - if err != nil { - return nil, err + if !match { + return fmt.Errorf("layer/blob not found for digest %s", digest) } - return json.Marshal(buf) -} - -func (ds *dataStore) ListWithPrefix(prefix []byte) ([]byte, error) { - - var buf []byte - - err := ds.db.View(func(txn *badger.Txn) error { - it := txn.NewIterator(badger.DefaultIteratorOptions) - defer it.Close() - for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { - item := it.Item() - err := item.Value(func(v []byte) error { - buf = make([]byte, len(v)) - copy(buf, v) - return nil - }) - if err != nil { - return err - } - } - return nil - }) - return buf, err + return ds.Set([]byte(namespace), md.Bytes()) } -// QueryMetaData is similar to ListWithPrefix with only a few changes -// making these changes in ListWithPrefix was causing the oci tests to fail -func (ds *dataStore) QueryMetaData(prefix []byte) ([]byte, error) { - - var mds []types.Metadata - - err := ds.db.View(func(txn *badger.Txn) error { - it := txn.NewIterator(badger.DefaultIteratorOptions) - defer it.Close() - for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { - var md types.Metadata - item := it.Item() - err := item.Value(func(v []byte) error { - err := json.Unmarshal(v, &md) - if err != nil { - return err - } - mds = append(mds, md) - return nil - }) - if err != nil { - return err - } - } - return nil - }) +func (ds *dataStore) DeleteDigest(digest string) error { + key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) - if err != nil { - return nil, err - } - return json.Marshal(mds) + return ds.Delete([]byte(key)) } func (ds *dataStore) Delete(key []byte) error { diff --git a/config/config.go b/config/config.go index 565101aa..225a447b 100644 --- a/config/config.go +++ b/config/config.go @@ -12,6 +12,7 @@ import ( type ( RegistryConfig struct { + StoreConfig *StoreConfig `mapstructure:"store_config"` AuthConfig AuthConfig `mapstructure:"auth_config"` LogConfig LogConfig `mapstructure:"log_config"` SkynetConfig SkynetConfig `mapstructure:"skynet_config"` @@ -42,12 +43,40 @@ type ( Username string `mapstructure:"username"` Password string `mapstructure:"password"` } + + StoreConfig struct { + User string `mapstructure:"user"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + } ) func (r *RegistryConfig) Address() string { return fmt.Sprintf("%s:%d", r.Host, r.Port) } +func NewStoreConfig() (*StoreConfig, error) { + viper.SetEnvPrefix("OPEN_REGISTRY") + viper.AutomaticEnv() + + storeConfig := &StoreConfig{ + User: viper.GetString("DB_USER"), + Password: viper.GetString("DB_PASSWORD"), + Database: viper.GetString("DB_NAME"), + Host: viper.GetString("DB_HOST"), + Port: viper.GetInt("DB_PORT"), + } + + return storeConfig, nil +} + +func (sc *StoreConfig) Endpoint() string { + return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?pool_max_conns=1000&sslmode=disable", + sc.User, sc.Password, sc.Host, sc.Port, sc.Database) +} + func LoadFromENV() (*RegistryConfig, error) { // TODO - Add Support for loading from config // loading config from path is not possible right now, @@ -82,6 +111,13 @@ func LoadFromENV() (*RegistryConfig, error) { Username: viper.GetString("LOG_SERVICE_USER"), Password: viper.GetString("LOG_SERVICE_PASSWORD"), }, + StoreConfig: &StoreConfig{ + User: viper.GetString("DB_USER"), + Host: viper.GetString("DB_HOST"), + Port: viper.GetInt("DB_PORT"), + Password: viper.GetString("DB_PASSWORD"), + Database: viper.GetString("DB_NAME"), + }, } for _, service := range strings.Split(viper.GetString("SUPPORTED_SERVICES"), ",") { diff --git a/db/migrations/000001_create_users_table.down.sql b/db/migrations/000001_create_users_table.down.sql new file mode 100644 index 00000000..c99ddcdc --- /dev/null +++ b/db/migrations/000001_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS users; diff --git a/db/migrations/000001_create_users_table.up.sql b/db/migrations/000001_create_users_table.up.sql new file mode 100644 index 00000000..4b7ed546 --- /dev/null +++ b/db/migrations/000001_create_users_table.up.sql @@ -0,0 +1,10 @@ +CREATE TABLE "users" ( + "id" uuid PRIMARY KEY, + "is_active" boolean, + "username" varchar NOT NULL UNIQUE, + "email" varchar NOT NULL UNIQUE, + "password" varchar, + "created_at" timestamp, + "updated_at" timestamp, + "country_code" int +); diff --git a/db/migrations/000002_create_blob_table.down.sql b/db/migrations/000002_create_blob_table.down.sql new file mode 100644 index 00000000..a662c0ab --- /dev/null +++ b/db/migrations/000002_create_blob_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS blob; diff --git a/db/migrations/000002_create_blob_table.up.sql b/db/migrations/000002_create_blob_table.up.sql new file mode 100644 index 00000000..72e690a5 --- /dev/null +++ b/db/migrations/000002_create_blob_table.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE "blob" ( + "uuid" uuid, + "digest" text PRIMARY KEY, + "sky_link" text, + "start_range" int, + "end_range" int +); + diff --git a/db/migrations/000003_create_layer_table.down.sql b/db/migrations/000003_create_layer_table.down.sql new file mode 100644 index 00000000..8051de1e --- /dev/null +++ b/db/migrations/000003_create_layer_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS layer; diff --git a/db/migrations/000003_create_layer_table.up.sql b/db/migrations/000003_create_layer_table.up.sql new file mode 100644 index 00000000..8a3bd4f6 --- /dev/null +++ b/db/migrations/000003_create_layer_table.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE "layer" ( + "uuid" uuid PRIMARY KEY, + "digest" text UNIQUE NOT NULL, + "blob_ids" text[], + "media_type" text, + "sky_link" text, + "size" int +); diff --git a/db/migrations/000004_create_config_table.down.sql b/db/migrations/000004_create_config_table.down.sql new file mode 100644 index 00000000..2efbbee0 --- /dev/null +++ b/db/migrations/000004_create_config_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS config; diff --git a/db/migrations/000004_create_config_table.up.sql b/db/migrations/000004_create_config_table.up.sql new file mode 100644 index 00000000..32d64c4b --- /dev/null +++ b/db/migrations/000004_create_config_table.up.sql @@ -0,0 +1,12 @@ +CREATE TABLE "config" ( + "uuid" uuid UNIQUE NOT NULL, + "namespace" text NOT NULL, + "reference" text NOT NULL, + "digest" text NOT NULL, + "sky_link" text, + "media_type" text, + "layers" text[], + "size" int, + PRIMARY KEY(namespace, reference) +); + diff --git a/db/migrations/000005_create_image_manifest_table.down.sql b/db/migrations/000005_create_image_manifest_table.down.sql new file mode 100644 index 00000000..17e896ba --- /dev/null +++ b/db/migrations/000005_create_image_manifest_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS image_manifest; diff --git a/db/migrations/000005_create_image_manifest_table.up.sql b/db/migrations/000005_create_image_manifest_table.up.sql new file mode 100644 index 00000000..02f75671 --- /dev/null +++ b/db/migrations/000005_create_image_manifest_table.up.sql @@ -0,0 +1,7 @@ +CREATE TABLE "image_manifest" ( + "uuid" uuid PRIMARY KEY, + "namespace" text UNIQUE NOT NULL, + "media_type" text, + "schema_version" int +); + diff --git a/env-vars.example b/env-vars.example index 8ccbf80c..bf958746 100644 --- a/env-vars.example +++ b/env-vars.example @@ -11,3 +11,8 @@ OPEN_REGISTRY_LOG_SERVICE_NAME=grafana-loki OPEN_REGISTRY_LOG_SERVICE_HOST=http://0.0.0.0:9880/app.log OPEN_REGISTRY_LOG_SERVICE_AUTH_KIND=basic OPEN_REGISTRY_SKYNET_API_KEY="" +OPEN_REGISTRY_DB_HOST=0.0.0.0 +OPEN_REGISTRY_DB_PORT=5432 +OPEN_REGISTRY_DB_USER=postgres +OPEN_REGISTRY_DB_PASSWORD=Qwerty@123 +OPEN_REGISTRY_DB_NAME=open_registry diff --git a/go.mod b/go.mod index f6ce7855..a1eb0ad4 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,21 @@ require ( golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 ) +require ( + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.9.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.10.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.1.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.8.1 // indirect + github.com/jackc/puddle v1.1.3 // indirect + github.com/leodido/go-urn v1.2.1 // indirect +) + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect @@ -32,6 +47,7 @@ require ( github.com/golang/snappy v0.0.3 // indirect github.com/google/flatbuffers v1.12.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jackc/pgx/v4 v4.13.0 github.com/klauspost/compress v1.12.3 // indirect github.com/labstack/gommon v0.3.0 // indirect github.com/magiconair/properties v1.8.5 // indirect @@ -54,7 +70,7 @@ require ( gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect - golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect + golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect google.golang.org/protobuf v1.26.0 // indirect diff --git a/go.sum b/go.sum index 7c2f6256..cb21f951 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -88,6 +89,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerish/go-skynet/v2 v2.0.2-0.20211206044851-95b8ecc10d44 h1:0oS0IvCItos8SeUg6hFGI6duDGiIQtx6lcLXKtvBggU= @@ -97,6 +99,8 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -146,12 +150,21 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.9.0 h1:NgTtmN58D0m8+UuxtYmGztBJB7VnPgjj221I1QHci2A= +github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -278,6 +291,53 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= +github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= +github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= +github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -299,11 +359,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -314,6 +378,12 @@ github.com/labstack/echo/v4 v4.5.0 h1:JXk6H5PAw9I3GwizqUHhYyS4f45iyGebR/c1xNCeOC github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -321,11 +391,15 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -386,6 +460,7 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -427,16 +502,25 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.24.0 h1:76ivFxmVSRs1u2wUwJVg5VZDYQgeH1JpoS6ndgr9Wy8= github.com/rs/zerolog v1.24.0/go.mod h1:7KHcEGe0QZPOm2IE4Kpb5rTh6n1h2hIgS5OOnu1rUaI= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -471,6 +555,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -500,6 +585,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8 h1:gZfMjx7Jr6N8b7iJO4eUjDsn6xJqoyXg8D+ogdoAfKY= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -518,12 +604,16 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= @@ -531,13 +621,17 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -657,6 +751,7 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -668,6 +763,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -705,6 +801,9 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 h1:siQdpVirKtzPhKl3lZWozZraCFObP8S1v6PRp0bLrtU= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -733,12 +832,14 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -780,6 +881,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -906,11 +1009,14 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/main.go b/main.go index c3860e6e..87207ebf 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "github.com/containerish/OpenRegistry/registry/v2" "github.com/containerish/OpenRegistry/router" "github.com/containerish/OpenRegistry/skynet" + "github.com/containerish/OpenRegistry/store/postgres" "github.com/containerish/OpenRegistry/telemetry" fluentbit "github.com/containerish/OpenRegistry/telemetry/fluent-bit" "github.com/fatih/color" @@ -31,6 +32,12 @@ func main() { } defer localCache.Close() + pgStore, err := postgres.New(cfg.StoreConfig) + if err != nil { + color.Red("error here: %s", err.Error()) + return + } + fluentBitCollector, err := fluentbit.New(cfg) if err != nil { color.Red("error initializing fluentbit collector: %s\n", err) @@ -38,15 +45,15 @@ func main() { } logger := telemetry.ZLogger(telemetry.SetupLogger(), fluentBitCollector) - authSvc := auth.New(localCache, cfg, logger) + authSvc := auth.New(localCache, cfg, pgStore, logger) skynetClient := skynet.NewClient(cfg) - reg, err := registry.NewRegistry(skynetClient, localCache, logger) + reg, err := registry.NewRegistry(skynetClient, localCache, logger, pgStore) if err != nil { e.Logger.Errorf("error creating new container registry: %s", err) return } - router.Register(cfg, e, reg, authSvc, localCache) + router.Register(cfg, e, reg, authSvc, localCache, pgStore) logger.Errorf("error initialising OpenRegistry Server: %s", e.Start(cfg.Address())) } diff --git a/registry/v2/blobs.go b/registry/v2/blobs.go index 3ec78e0f..5c2be8f5 100644 --- a/registry/v2/blobs.go +++ b/registry/v2/blobs.go @@ -38,10 +38,11 @@ func (b *blobs) HEAD(ctx echo.Context) error { }() digest := ctx.Param("digest") - layerRef, err := b.registry.localCache.GetDigest(digest) + + layerRef, err := b.registry.store.GetLayer(ctx.Request().Context(), digest) if err != nil { details := echo.Map{ - "skynet": "skynet link not found", + "skynet": "layer not found", } errMsg := b.errorResponse(RegistryErrorCodeManifestBlobUnknown, err.Error(), details) @@ -49,9 +50,12 @@ func (b *blobs) HEAD(ctx echo.Context) error { return ctx.JSONBlob(http.StatusNotFound, errMsg) } - size, ok := b.registry.skynet.Metadata(layerRef.Skylink) + size, ok := b.registry.skynet.Metadata(layerRef.SkynetLink) if !ok { - errMsg := b.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", nil) + details := echo.Map{ + "skynet": "skynet link not found", + } + errMsg := b.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", details) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) } @@ -61,6 +65,12 @@ func (b *blobs) HEAD(ctx echo.Context) error { return ctx.String(http.StatusOK, "OK") } +/* +UploadBlob +for postgres +insert into blob table one blob at a time +these will be part of the txn in StartUpload +*/ func (b *blobs) UploadBlob(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { @@ -88,6 +98,16 @@ func (b *blobs) UploadBlob(ctx echo.Context) error { b.uploads[uuid] = bz + if err := b.blobTransaction(ctx, bz, uuid); err != nil { + errMsg := b.errorResponse( + RegistryErrorCodeBlobUploadInvalid, + err.Error(), + nil, + ) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + locationHeader := fmt.Sprintf("/v2/%s/blobs/uploads/%s", namespace, uuid) ctx.Response().Header().Set("Location", locationHeader) ctx.Response().Header().Set("Range", fmt.Sprintf("0-%d", len(bz)-1)) @@ -126,8 +146,41 @@ func (b *blobs) UploadBlob(ctx echo.Context) error { ctx.Request().Body.Close() b.uploads[uuid] = buf.Bytes() + if err := b.blobTransaction(ctx, buf.Bytes(), uuid); err != nil { + errMsg := b.errorResponse( + RegistryErrorCodeBlobUploadInvalid, + err.Error(), + nil, + ) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } locationHeader := fmt.Sprintf("/v2/%s/blobs/uploads/%s", namespace, uuid) ctx.Response().Header().Set("Location", locationHeader) ctx.Response().Header().Set("Range", fmt.Sprintf("0-%d", buf.Len()-1)) return ctx.NoContent(http.StatusAccepted) } + +func (b *blobs) blobTransaction(ctx echo.Context, bz []byte, uuid string) error { + blob := &types.Blob{ + Digest: digest(bz), + Skylink: "", + UUID: uuid, + RangeStart: 0, + RangeEnd: uint32(len(bz) - 1), + } + + txnOp, ok := b.registry.txnMap[uuid] + if !ok { + return fmt.Errorf("txn has not been initialised for uuid - " + uuid) + } + + if err := b.registry.store.SetBlob(ctx.Request().Context(), txnOp.txn, blob); err != nil { + color.Red("aborting txn: %s\n", err.Error()) + return b.registry.store.Abort(ctx.Request().Context(), txnOp.txn) + } + + txnOp.blobDigests = append(txnOp.blobDigests, blob.Digest) + b.registry.txnMap[uuid] = txnOp + return nil +} diff --git a/registry/v2/registry.go b/registry/v2/registry.go index 2eabb60e..14d30de9 100644 --- a/registry/v2/registry.go +++ b/registry/v2/registry.go @@ -2,12 +2,12 @@ package registry import ( "bytes" + "context" "encoding/json" "fmt" "io" "net/http" "path" - "sort" "strconv" "strings" "sync" @@ -15,6 +15,7 @@ import ( "github.com/containerish/OpenRegistry/cache" "github.com/containerish/OpenRegistry/skynet" + "github.com/containerish/OpenRegistry/store/postgres" "github.com/containerish/OpenRegistry/telemetry" "github.com/containerish/OpenRegistry/types" "github.com/docker/distribution/uuid" @@ -25,6 +26,7 @@ func NewRegistry( skynetClient *skynet.Client, c cache.Store, logger telemetry.Logger, + pgStore postgres.PersistentStore, ) (Registry, error) { r := ®istry{ debug: true, @@ -38,6 +40,8 @@ func NewRegistry( localCache: c, logger: logger, mu: &sync.RWMutex{}, + store: pgStore, + txnMap: map[string]TxnStore{}, } r.b.registry = r @@ -45,267 +49,19 @@ func NewRegistry( return r, nil } -// Catalog - The list of available repositories is made available through the catalog. -//GET /v2/_catalog -func (r *registry) Catalog(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - bz, err := r.localCache.ListAll() - if err != nil { - logMsg := echo.Map{ - "error": err.Error(), - } - - ctx.Set(types.HttpEndpointErrorKey, logMsg) - return ctx.JSON(http.StatusBadRequest, echo.Map{ - "error": err.Error(), - }) - } - - var md []types.Metadata - err = json.Unmarshal(bz, &md) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - } - - var result []string - for _, el := range md { - var repo []string - ns := el.Namespace - - for _, c := range el.Manifest.Config { - repo = append(repo, fmt.Sprintf("%s:%s", ns, c.Reference)) - } - - result = append(result, repo...) - } - - return ctx.JSON(http.StatusOK, echo.Map{ - "repositories": result, - }) -} - -func (r *registry) DeleteTagOrManifest(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - ref := ctx.Param("reference") - - if ref == "" { - reqURI := strings.Split(ctx.Request().RequestURI, "/") - if len(reqURI) == 6 { - ref = reqURI[5] - } - } - - if err := r.localCache.UpdateManifestRef(namespace, ref); err != nil { - details := map[string]interface{}{ - "namespace": namespace, - "digest": ref, - } - errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), details) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - return ctx.NoContent(http.StatusAccepted) -} - -func (r *registry) DeleteLayer(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - dig := ctx.Param("digest") - - var m types.Metadata - _, err := r.localCache.GetDigest(dig) - if err != nil { - bz, err := r.localCache.Get([]byte(namespace)) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - if err = json.Unmarshal(bz, &m); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - } - } - - err = r.localCache.DeleteLayer(namespace, dig) - if err != nil { - logMsg := echo.Map{ - "error": err.Error(), - "caller": "DeleteLayer", - } - - bz, err := json.Marshal(logMsg) - if err == nil { - ctx.Set(types.HttpEndpointErrorKey, logMsg) - } - return ctx.JSONBlob(http.StatusInternalServerError, bz) - } - - if err = r.localCache.DeleteDigest(dig); err != nil { - logMsg := echo.Map{ - "error": err.Error(), - "caller": "DeleteLayer", - } - - ctx.Set(types.HttpEndpointErrorKey, logMsg) - bz, err := json.Marshal(logMsg) - if err != nil { - ctx.Set(types.HttpEndpointErrorKey, err.Error()) - r.logger.Log(ctx) - } - return ctx.JSONBlob(http.StatusInternalServerError, bz) - } - return ctx.NoContent(http.StatusAccepted) -} - -// PUT /v2//blobs/uploads/?digest= -func (r *registry) MonolithicUpload(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - uuid := ctx.Param("uuid") - digest := ctx.QueryParam("digest") - - bz, err := io.ReadAll(ctx.Request().Body) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - ctx.Request().Body.Close() - - link, err := r.skynet.Upload(namespace, digest, bz, true) - if err != nil { - detail := echo.Map{ - "error": err.Error(), - "caller": "MonolithicUpload", - } - errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), detail) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusInternalServerError, bz) - } - - metadata := types.Metadata{ - Namespace: namespace, - Manifest: types.ImageManifest{ - SchemaVersion: 2, - MediaType: "", - Layers: []*types.Layer{{MediaType: "", Size: len(bz), Digest: digest, SkynetLink: link, UUID: uuid}}, - }, - } - - err = r.localCache.Update([]byte(namespace), metadata.Bytes()) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - - locationHeader := link - ctx.Response().Header().Set("Location", locationHeader) - return ctx.NoContent(http.StatusCreated) -} - -// PUT /v2//blobs/uploads/?digest= -func (r *registry) CompleteUpload(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - dig := ctx.QueryParam("digest") - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - uuid := ctx.Param("uuid") - - bz, err := io.ReadAll(ctx.Request().Body) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeDigestInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - _ = ctx.Request().Body.Close() - - buf := bytes.NewBuffer(r.b.uploads[uuid]) - buf.Write(bz) - ourHash := digest(buf.Bytes()) - delete(r.b.uploads, uuid) - - if ourHash != dig { - details := map[string]interface{}{ - "headerDigest": dig, "serverSideDigest": ourHash, "bodyDigest": digest(bz), - } - errMsg := r.errorResponse(RegistryErrorCodeDigestInvalid, "digest mismatch", details) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - - blobNamespace := fmt.Sprintf("%s/blobs", namespace) - skylink, err := r.skynet.Upload(blobNamespace, dig, buf.Bytes(), true) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg) - } - if err := r.localCache.SetDigest(ourHash, skylink); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - } - - val := types.Metadata{ - Namespace: namespace, - Manifest: types.ImageManifest{ - SchemaVersion: 2, - MediaType: "", - Layers: []*types.Layer{ - { - MediaType: "", Size: len(bz), Digest: dig, SkynetLink: skylink, UUID: uuid, - }, - }, - }, - } - - if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - } - - locationHeader := fmt.Sprintf("/v2/%s/blobs/%s", namespace, ourHash) - ctx.Response().Header().Set("Content-Length", "0") - ctx.Response().Header().Set("Docker-Content-Digest", ourHash) - ctx.Response().Header().Set("Location", locationHeader) - return ctx.NoContent(http.StatusCreated) -} - +// LayerExists // HEAD /v2//blobs/ // 200 OK // Content-Length: // Docker-Content-Digest: +// OK func (r *registry) LayerExists(ctx echo.Context) error { return r.b.HEAD(ctx) } +// ManifestExists // HEAD /v2//manifests/ +// OK func (r *registry) ManifestExists(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { @@ -315,10 +71,11 @@ func (r *registry) ManifestExists(ctx echo.Context) error { namespace := ctx.Param("username") + "/" + ctx.Param("imagename") ref := ctx.Param("reference") // ref can be either tag or digest - skylink, err := r.localCache.ResolveManifestRef(namespace, ref) + manifest, err := r.store.GetManifestByReference(ctx.Request().Context(), namespace, ref) + if err != nil { details := echo.Map{ - "skynet": "skynet link not found", + "skynet": "manifest not found", "error": err.Error(), } @@ -328,42 +85,14 @@ func (r *registry) ManifestExists(ctx echo.Context) error { return ctx.JSONBlob(http.StatusNotFound, errMsg) } - size, ok := r.skynet.Metadata(skylink) + size, ok := r.skynet.Metadata(manifest.Skylink) if !ok { - lm := logMsg{ + detail := map[string]interface{}{ "error": "metadata not found for skylink", - "skylink": skylink, - } - r.logger.Error(lm) - errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - - bz, err := r.localCache.Get([]byte(namespace)) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - - var md types.Metadata - if err = json.Unmarshal(bz, &md); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - lm := logMsg{ - "errorUnmarshal": fmt.Sprintf("%s\n", errMsg), + "skylink": manifest.Skylink, } - r.logger.Error(lm) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - - manifest, err := md.GetManifestByRef(ref) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), nil) + errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", detail) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) @@ -388,55 +117,95 @@ func (r *registry) ManifestExists(ctx echo.Context) error { return ctx.NoContent(http.StatusOK) } -// PATCH /v2//blobs/uploads/ -func (r *registry) ChunkedUpload(ctx echo.Context) error { - return r.b.UploadBlob(ctx) -} +// Catalog - The list of available repositories is made available through the catalog. +// GET /v2/_catalog +// OK +func (r *registry) Catalog(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + defer func() { + r.logger.Log(ctx).Send() + }() + + catalogList, err := r.store.GetCatalog(ctx.Request().Context()) + if err != nil { + ctx.Set(types.HttpEndpointErrorKey, err.Error()) + return ctx.JSON(http.StatusInternalServerError, echo.Map{ + "error": err.Error(), + }) + } + return ctx.JSON(http.StatusOK, echo.Map{ + "repositories": catalogList, + }) -func (r *registry) CancelUpload(ctx echo.Context) error { - return nil } -// PullManifest GET /v2//manifests/ -func (r *registry) PullManifest(ctx echo.Context) error { +// ListTags Content discovery +// GET /v2//tags/list +// OK +func (r *registry) ListTags(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { r.logger.Log(ctx).Send() }() namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - ref := ctx.Param("reference") + limit := ctx.QueryParam("n") - bz, err := r.localCache.Get([]byte(namespace)) + tags, err := r.store.GetImageTags(ctx.Request().Context(), namespace) if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), nil) + errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) - err = ctx.JSONBlob(http.StatusNotFound, errMsg) - return err - } - var md types.Metadata - if err = json.Unmarshal(bz, &md); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) } - skynetLink, err := r.localCache.ResolveManifestRef(namespace, ref) + if limit != "" { + n, err := strconv.ParseInt(limit, 10, 32) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + + return ctx.JSONBlob(http.StatusNotFound, errMsg) + } + if n > 0 { + tags = tags[0:n] + } + if n == 0 { + tags = nil + } + } + + return ctx.JSON(http.StatusOK, echo.Map{ + "name": namespace, + "tags": tags, + }) +} +func (r *registry) List(ctx echo.Context) error { + return fmt.Errorf("error") +} + +// PullManifest +// GET /v2//manifests/ +// OK +func (r *registry) PullManifest(ctx echo.Context) error { + namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + ref := ctx.Param("reference") + + manifest, err := r.store.GetManifestByReference(ctx.Request().Context(), namespace, ref) if err != nil { errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) } - resp, err := r.skynet.Download(skynetLink) + resp, err := r.skynet.Download(manifest.Skylink) if err != nil { errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) } - bz, err = io.ReadAll(resp) + bz, err := io.ReadAll(resp) if err != nil { errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) @@ -444,200 +213,141 @@ func (r *registry) PullManifest(ctx echo.Context) error { } _ = resp.Close() - manifest, err := md.GetManifestByRef(ref) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - ctx.Response().Header().Set("Docker-Content-Digest", manifest.Digest) - ctx.Response().Header().Set("X-Docker-Content-ID", skynetLink) + ctx.Response().Header().Set("X-Docker-Content-ID", manifest.Skylink) ctx.Response().Header().Set("Content-Type", manifest.MediaType) - ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", manifest.Size)) + ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", len(bz))) return ctx.JSONBlob(http.StatusOK, bz) } -func (r *registry) PushManifest(ctx echo.Context) error { +// PullLayer +// GET /v2//blobs/ +// OK, error: binary output can mess your system ... +func (r *registry) PullLayer(ctx echo.Context) error { + //namespace := ctx.Param("username") + "/" + ctx.Param("imagename") ctx.Set(types.HandlerStartTime, time.Now()) defer func() { r.logger.Log(ctx).Send() }() - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - ref := ctx.Param("reference") - contentType := ctx.Request().Header.Get("Content-Type") + clientDigest := ctx.Param("digest") - bz, err := io.ReadAll(ctx.Request().Body) + layer, err := r.store.GetLayer(ctx.Request().Context(), clientDigest) if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - ctx.Request().Body.Close() - - dig := digest(bz) - - var manifest ManifestList - if err = json.Unmarshal(bz, &manifest); err != nil { errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - - mfNamespace := fmt.Sprintf("%s/manifests", namespace) - skylink, err := r.skynet.Upload(mfNamespace, dig, bz, true) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - manifestConfig := &types.Config{ - MediaType: contentType, - Size: len(bz), - Digest: dig, - SkynetLink: skylink, - Reference: ref, } - metadata := types.Metadata{ - Namespace: namespace, - Manifest: types.ImageManifest{ - SchemaVersion: 2, - Config: []*types.Config{manifestConfig}, - }, - } - - if err = r.localCache.Update([]byte(namespace), metadata.Bytes()); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) + if layer.SkynetLink == "" { + detail := map[string]interface{}{ + "error": "skylink is empty", + } + e := fmt.Errorf("skylink is empty").Error() + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, e, detail) ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusBadRequest, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) } - locationHeader := r.getHttpUrlFromSkylink(skylink) - ctx.Response().Header().Set("Location", locationHeader) - ctx.Response().Header().Set("Docker-Content-Digest", dig) - ctx.Response().Header().Set("X-Docker-Content-ID", skylink) - - return ctx.String(http.StatusCreated, "Created") -} - -// Content discovery GET /v2//tags/list - -func (r *registry) ListTags(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - limit := ctx.QueryParam("n") - - l, err := r.localCache.ListWithPrefix([]byte(namespace)) + resp, err := r.skynet.Download(layer.SkynetLink) if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) + detail := map[string]interface{}{ + "error": err.Error(), + "skylink": layer.SkynetLink, + } + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), detail) ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) } - var md types.Metadata - err = json.Unmarshal(l, &md) + + bz, err := io.ReadAll(resp) if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - var tags []string - for _, v := range md.Manifest.Config { - tags = append(tags, v.Reference) + return ctx.JSONBlob(http.StatusInternalServerError, errMsg) } - if limit != "" { - n, err := strconv.ParseInt(limit, 10, 32) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) + _ = resp.Close() - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - if n > 0 { - tags = tags[0:n] - } - if n == 0 { - tags = []string{} + dig := digest(bz) + if dig != clientDigest { + details := map[string]interface{}{ + "clientDigest": clientDigest, + "computedDigest": dig, } + errMsg := r.errorResponse( + RegistryErrorCodeBlobUploadUnknown, + "client digest is different than computed digest", + details, + ) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) } - sort.Strings(tags) - return ctx.JSON(http.StatusOK, echo.Map{ - "name": namespace, - "tags": tags, - }) -} -func (r *registry) List(ctx echo.Context) error { - return fmt.Errorf("error") -} - -// GET /v2//blobs/ - -func (r *registry) PullLayer(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() + ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", len(bz))) + ctx.Response().Header().Set("Docker-Content-Digest", dig) + return ctx.Blob(http.StatusOK, "application/octet-stream", bz) +} +// MonolithicUpload +// PUT /v2//blobs/uploads/?digest= +func (r *registry) MonolithicUpload(ctx echo.Context) error { namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - clientDigest := ctx.Param("digest") + uuid := ctx.Param("uuid") + digest := ctx.QueryParam("digest") - layerRef, err := r.localCache.GetDigest(clientDigest) + bz, err := io.ReadAll(ctx.Request().Body) if err != nil { - skynetLink, err := r.localCache.GetSkynetURL(namespace, clientDigest) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + ctx.Request().Body.Close() - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - layerRef = &types.LayerRef{ - Digest: clientDigest, - Skylink: skynetLink, + link, err := r.skynet.Upload(namespace, digest, bz, true) + if err != nil { + detail := echo.Map{ + "error": err.Error(), + "caller": "MonolithicUpload", } + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), detail) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusInternalServerError, bz) } - size, ok := r.skynet.Metadata(layerRef.Skylink) - if ok { - url := fmt.Sprintf("https://siasky.net/%s", - strings.Replace(layerRef.Skylink, "sia://", "", 1)) - ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", size)) - http.Redirect(ctx.Response(), ctx.Request(), url, http.StatusTemporaryRedirect) - return nil + metadata := types.Metadata{ + Namespace: namespace, + Manifest: types.ImageManifest{ + SchemaVersion: 2, + MediaType: "", + Layers: []*types.Layer{{MediaType: "", Size: len(bz), Digest: digest, SkynetLink: link, UUID: uuid}}, + }, } - detail := map[string]interface{}{ - "error": "skylink is empty", + err = r.localCache.Update([]byte(namespace), metadata.Bytes()) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) } - e := fmt.Errorf("skylink is empty").Error() - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, e, detail) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusNotFound, errMsg) -} -//BlobMount to be implemented by guacamole at a later stage -func (r *registry) BlobMount(ctx echo.Context) error { - return nil + locationHeader := link + ctx.Response().Header().Set("Location", locationHeader) + return ctx.NoContent(http.StatusCreated) } -//PushImage is already implemented through StartUpload and ChunkedUpload -func (r *registry) PushImage(ctx echo.Context) error { - return nil +// ChunkedUpload +// PATCH /v2//blobs/uploads/ +func (r *registry) ChunkedUpload(ctx echo.Context) error { + return r.b.UploadBlob(ctx) } +/*StartUpload +for postgres: +start a tnx +registry.tnxMap[uuid] = {txn,blobs[],timeout} +*/ +// POST /v2//blobs/uploads/ func (r *registry) StartUpload(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { @@ -661,9 +371,7 @@ func (r *registry) StartUpload(ctx echo.Context) error { ) ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } ctx.Request().Body.Close() // why defer? body is already read :) dig := digest(bz) @@ -691,33 +399,56 @@ func (r *registry) StartUpload(ctx echo.Context) error { return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg) } - layer := &types.Layer{ - MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", - Size: len(bz), - Digest: dig, - SkynetLink: skylink, - UUID: "", + layerV2 := &types.LayerV2{ + MediaType: ctx.Request().Header.Get("content-type"), + Digest: dig, + SkynetLink: skylink, + UUID: uuid.Generate().String(), + BlobDigests: nil, + Size: len(bz), } - var val types.Metadata - val.Namespace = namespace - val.Manifest.Layers = append(val.Manifest.Layers, layer) - - link := r.getHttpUrlFromSkylink(skylink) - if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) + txnOp, err := r.store.NewTxn(ctx.Request().Context()) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusInternalServerError, errMsg) } - ctx.Response().Header().Set("Location", link) + if err := r.store.SetLayer(ctx.Request().Context(), txnOp, layerV2); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + if err := r.store.Commit(ctx.Request().Context(), txnOp); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + link := r.getHttpUrlFromSkylink(skylink) + ctx.Response().Header().Set("Location", link) return ctx.NoContent(http.StatusCreated) } id := uuid.Generate() locationHeader := fmt.Sprintf("/v2/%s/blobs/uploads/%s", namespace, id.String()) - + txn, err := r.store.NewTxn(ctx.Request().Context()) + if err != nil { + errMsg := r.errorResponse( + RegistryErrorCodeUnknown, + err.Error(), + nil, + ) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusInternalServerError, errMsg) + } + r.txnMap[id.String()] = TxnStore{ + txn: txn, + blobDigests: []string{}, + timeout: time.Minute * 30, + } ctx.Response().Header().Set("Location", locationHeader) ctx.Response().Header().Set("Content-Length", "0") ctx.Response().Header().Set("Docker-Upload-UUID", id.String()) @@ -763,6 +494,179 @@ func (r *registry) UploadProgress(ctx echo.Context) error { return ctx.NoContent(http.StatusNoContent) } +// CompleteUpload +/*PUT /v2//blobs/uploads/?digest= +for postgres: +this is where we insert into the layer after all the blobs have been accumulated +and inserted in the blob table +thus committing the txn +*/ +func (r *registry) CompleteUpload(ctx echo.Context) error { + dig := ctx.QueryParam("digest") + namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + id := ctx.Param("uuid") + + bz, err := io.ReadAll(ctx.Request().Body) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeDigestInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + _ = ctx.Request().Body.Close() + // insert if bz is not nil + buf := bytes.NewBuffer(r.b.uploads[id]) + buf.Write(bz) + ourHash := digest(buf.Bytes()) + delete(r.b.uploads, id) + + if ourHash != dig { + details := map[string]interface{}{ + "headerDigest": dig, "serverSideDigest": ourHash, "bodyDigest": digest(bz), + } + errMsg := r.errorResponse(RegistryErrorCodeDigestInvalid, "digest mismatch", details) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + blobNamespace := fmt.Sprintf("%s/blobs", namespace) + skylink, err := r.skynet.Upload(blobNamespace, dig, buf.Bytes(), true) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg) + } + + txnOp, ok := r.txnMap[id] + layer := &types.LayerV2{ + MediaType: "", + Digest: dig, + SkynetLink: skylink, + UUID: id, + BlobDigests: txnOp.blobDigests, + Size: len(bz), + } + if !ok { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, "transaction does not exist for uuid -"+id, nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + if err := r.store.SetLayer(ctx.Request().Context(), txnOp.txn, layer); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), echo.Map{ + "error_detail": "set layer issues", + }) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + if err := r.store.Commit(ctx.Request().Context(), txnOp.txn); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), echo.Map{ + "error_detail": "commitment issue", + }) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + delete(r.txnMap, id) + + locationHeader := fmt.Sprintf("/v2/%s/blobs/%s", namespace, ourHash) + ctx.Response().Header().Set("Content-Length", "0") + ctx.Response().Header().Set("Docker-Content-Digest", ourHash) + ctx.Response().Header().Set("Location", locationHeader) + return ctx.NoContent(http.StatusCreated) +} + +//BlobMount to be implemented by guacamole at a later stage +func (r *registry) BlobMount(ctx echo.Context) error { + return nil +} + +//PushImage is already implemented through StartUpload and ChunkedUpload +func (r *registry) PushImage(ctx echo.Context) error { + return nil +} + +func (r *registry) PushManifest(ctx echo.Context) error { + namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + ref := ctx.Param("reference") + contentType := ctx.Request().Header.Get("Content-Type") + + bz, err := io.ReadAll(ctx.Request().Body) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + ctx.Request().Body.Close() + + dig := digest(bz) + + var manifest ImageManifest + if err = json.Unmarshal(bz, &manifest); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + mfNamespace := fmt.Sprintf("%s/manifests", namespace) + skylink, err := r.skynet.Upload(mfNamespace, dig, bz, true) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) + } + + var layerIDs []string + for _, layer := range manifest.Layers { + layerIDs = append(layerIDs, layer.Digest) + } + + id := uuid.Generate() + mfc := types.ConfigV2{ + UUID: id.String(), + Namespace: namespace, + Reference: ref, + Digest: dig, + Skylink: skylink, + MediaType: contentType, + Layers: layerIDs, + Size: 0, + } + + val := &types.ImageManifestV2{ + Uuid: uuid.Generate().String(), + Namespace: namespace, + MediaType: "", + SchemaVersion: 2, + } + + txnOp, _ := r.store.NewTxn(context.Background()) + + if err := r.store.SetManifest(ctx.Request().Context(), txnOp, val); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + _ = r.store.Abort(ctx.Request().Context(), txnOp) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + if err := r.store.SetConfig(ctx.Request().Context(), txnOp, mfc); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + _ = r.store.Abort(ctx.Request().Context(), txnOp) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + if err = r.store.Commit(ctx.Request().Context(), txnOp); err != nil { + return err + } + + locationHeader := r.getHttpUrlFromSkylink(skylink) + ctx.Response().Header().Set("Location", locationHeader) + ctx.Response().Header().Set("Docker-Content-Digest", dig) + ctx.Response().Header().Set("X-Docker-Content-ID", skylink) + return ctx.String(http.StatusCreated, "Created") +} + +// PushLayer // POST /v2//blobs/uploads/ func (r *registry) PushLayer(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) @@ -793,6 +697,92 @@ func (r *registry) PushLayer(ctx echo.Context) error { return ctx.NoContent(http.StatusAccepted) } +func (r *registry) CancelUpload(ctx echo.Context) error { + return nil +} + +// DeleteTagOrManifest +// DELETE /v2//manifest/ or +func (r *registry) DeleteTagOrManifest(ctx echo.Context) error { + namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + ref := ctx.Param("reference") + + if ref == "" { + reqURI := strings.Split(ctx.Request().RequestURI, "/") + if len(reqURI) == 6 { + ref = reqURI[5] + } + } + txnOp, _ := r.store.NewTxn(context.Background()) + if err := r.store.DeleteManifestOrTag(ctx.Request().Context(), txnOp, ref); err != nil { + //if err := r.localCache.UpdateManifestRef(namespace, ref); err != nil { + details := map[string]interface{}{ + "namespace": namespace, + "digest": ref, + } + errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), details) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) + } + + _ = r.store.Commit(ctx.Request().Context(), txnOp) + return ctx.NoContent(http.StatusAccepted) +} + +func (r *registry) DeleteLayer(ctx echo.Context) error { + //namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + dig := ctx.Param("digest") + + //var m types.Metadata + + layer, err := r.store.GetLayer(ctx.Request().Context(), dig) + //_, err := r.localCache.GetDigest(dig) + if err != nil { + + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) + } + blobs := layer.BlobDigests + + //err = r.localCache.DeleteLayer(namespace, dig) + txnOp, _ := r.store.NewTxn(context.Background()) + err = r.store.DeleteLayerV2(ctx.Request().Context(), txnOp, dig) + if err != nil { + logMsg := echo.Map{ + "error": err.Error(), + "caller": "DeleteLayer", + } + + bz, err := json.Marshal(logMsg) + if err == nil { + ctx.Set(types.HttpEndpointErrorKey, logMsg) + } + + return ctx.JSONBlob(http.StatusInternalServerError, bz) + } + + for i := range blobs { + //if err = r.localCache.DeleteDigest(dig); err != nil { + if err = r.store.DeleteBlobV2(ctx.Request().Context(), txnOp, blobs[i]); err != nil { + logMsg := echo.Map{ + "error": err.Error(), + "caller": "DeleteLayer", + } + + ctx.Set(types.HttpEndpointErrorKey, logMsg) + bz, err := json.Marshal(logMsg) + if err != nil { + r.log.Err(err).Send() + } + + return ctx.JSONBlob(http.StatusInternalServerError, bz) + } + } + _ = r.store.Commit(ctx.Request().Context(), txnOp) + return ctx.NoContent(http.StatusAccepted) +} + // Should also look into 401 Code // https://docs.docker.com/registry/spec/api/ func (r *registry) ApiVersion(ctx echo.Context) error { diff --git a/registry/v2/types.go b/registry/v2/types.go index 0ce33d76..60fd2a02 100644 --- a/registry/v2/types.go +++ b/registry/v2/types.go @@ -2,12 +2,15 @@ package registry import ( "sync" - - "github.com/containerish/OpenRegistry/telemetry" + "time" "github.com/containerish/OpenRegistry/cache" "github.com/containerish/OpenRegistry/skynet" + "github.com/containerish/OpenRegistry/store/postgres" + "github.com/containerish/OpenRegistry/telemetry" + "github.com/jackc/pgx/v4" "github.com/labstack/echo/v4" + "github.com/rs/zerolog" ) /* @@ -52,9 +55,9 @@ type RegistryErrors struct { } type RegistryError struct { - Detail map[string]interface{} `json:"detail,omitempty"` Code string `json:"code"` Message string `json:"message"` + Detail map[string]interface{} `json:"detail,omitempty"` } // OCI - Distribution Spec compliant Headers @@ -85,12 +88,21 @@ const ( type ( registry struct { + log zerolog.Logger b blobs logger telemetry.Logger localCache cache.Store skynet *skynet.Client mu *sync.RWMutex debug bool + txnMap map[string]TxnStore + store postgres.PersistentStore + } + + TxnStore struct { + txn pgx.Tx + blobDigests []string + timeout time.Duration } blobs struct { @@ -101,8 +113,6 @@ type ( registry *registry } - logMsg map[string]interface{} - ManifestList struct { SchemaVersion int `json:"schemaVersion"` MediaType string `json:"mediaType"` diff --git a/router/router.go b/router/router.go index 260bc594..6692b1c2 100644 --- a/router/router.go +++ b/router/router.go @@ -7,6 +7,7 @@ import ( "github.com/containerish/OpenRegistry/cache" "github.com/containerish/OpenRegistry/config" "github.com/containerish/OpenRegistry/registry/v2" + "github.com/containerish/OpenRegistry/store/postgres" "github.com/google/uuid" "github.com/labstack/echo-contrib/prometheus" "github.com/labstack/echo/v4" @@ -15,7 +16,14 @@ import ( // Register is the entry point that registers all the endpoints // nolint -func Register(cfg *config.RegistryConfig, e *echo.Echo, reg registry.Registry, authSvc auth.Authentication, localCache cache.Store) { +func Register( + cfg *config.RegistryConfig, + e *echo.Echo, + reg registry.Registry, + authSvc auth.Authentication, + localCache cache.Store, + pStore postgres.PersistentStore, +) { e.Use(middleware.Recover()) e.Use(middleware.CORS()) diff --git a/scripts/postgres/OpenRegistry.sql b/scripts/postgres/OpenRegistry.sql new file mode 100644 index 00000000..5b9027d0 --- /dev/null +++ b/scripts/postgres/OpenRegistry.sql @@ -0,0 +1,48 @@ + +CREATE TABLE "users" ( + "id" uuid PRIMARY KEY, + "is_active" boolean, + "username" varchar NOT NULL UNIQUE, + "email" varchar NOT NULL UNIQUE, + "password" varchar, + "created_at" timestamp, + "updated_at" timestamp, + "country_code" int +); + +CREATE TABLE "config" ( + "uuid" uuid UNIQUE NOT NULL, + "namespace" text NOT NULL, + "reference" text NOT NULL, + "digest" text NOT NULL, + "sky_link" text, + "media_type" text, + "layers" text[], + "size" int, + PRIMARY KEY(namespace, reference) +); + +CREATE TABLE "blob" ( + "uuid" uuid, + "digest" text PRIMARY KEY, + "sky_link" text, + "start_range" int, + "end_range" int +); + +CREATE TABLE "layer" ( + "uuid" uuid PRIMARY KEY, + "digest" text UNIQUE NOT NULL, + "blob_ids" text[], + "media_type" text, + "sky_link" text, + "size" int +); + +CREATE TABLE "image_manifest" ( + "uuid" uuid PRIMARY KEY, + "namespace" text UNIQUE NOT NULL, + "media_type" text, + "schema_version" int +); + diff --git a/store/postgres/container_image.go b/store/postgres/container_image.go new file mode 100644 index 00000000..fb227418 --- /dev/null +++ b/store/postgres/container_image.go @@ -0,0 +1,283 @@ +package postgres + +import ( + "context" + "strings" + "time" + + "github.com/containerish/OpenRegistry/store/postgres/queries" + "github.com/containerish/OpenRegistry/types" + "github.com/jackc/pgx/v4" +) + +func (p *pg) GetLayer(ctx context.Context, digest string) (*types.LayerV2, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + row := p.conn.QueryRow(childCtx, queries.GetLayer, digest) + var layer types.LayerV2 + if err := row.Scan( + &layer.UUID, + &layer.Digest, + &layer.BlobDigests, + &layer.MediaType, + &layer.SkynetLink, + &layer.Size, + ); err != nil { + return nil, err + } + + return &layer, nil + +} + +func (p *pg) SetLayer(ctx context.Context, txn pgx.Tx, l *types.LayerV2) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + _, err := txn.Exec(childCtx, queries.SetLayer, l.MediaType, l.Digest, l.SkynetLink, l.UUID, l.BlobDigests, l.Size) + return err +} + +func (p *pg) GetManifest(ctx context.Context, namespace string) (*types.ImageManifestV2, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + row := p.conn.QueryRow(childCtx, queries.GetManifest, namespace) + var im *types.ImageManifestV2 + if err := row.Scan( + &im.Uuid, + &im.Namespace, + &im.MediaType, + &im.SchemaVersion, + ); err != nil { + return nil, err + } + return im, nil +} +func (p *pg) GetManifestByReference(ctx context.Context, namespace string, ref string) (*types.ConfigV2, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + query := queries.GetManifestByRef + if strings.HasPrefix(ref, "sha256") { + query = queries.GetManifestByDig + } + + row := p.conn.QueryRow(childCtx, query, namespace, ref) + var im types.ConfigV2 + if err := row.Scan( + &im.UUID, + &im.Namespace, + &im.Reference, + &im.Digest, + &im.Skylink, + &im.MediaType, + &im.Layers, + &im.Size, + ); err != nil { + return nil, err + } + return &im, nil +} + +func (p *pg) SetManifest(ctx context.Context, txn pgx.Tx, im *types.ImageManifestV2) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + _, err := txn.Exec(childCtx, queries.SetImageManifest, im.Uuid, im.Namespace, im.MediaType, im.SchemaVersion) + return err +} + +func (p *pg) GetBlob(ctx context.Context, digest string) ([]*types.Blob, error) { + + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + rows, err := p.conn.Query(childCtx, queries.GetBlob, digest) + if err != nil { + return nil, err + } + defer rows.Close() + + blobList := make([]*types.Blob, 0) + for i := 0; rows.Next(); i++ { + var blob types.Blob + if err := rows.Scan( + &blob.UUID, + &blob.Digest, + &blob.Skylink, + &blob.RangeStart, + &blob.RangeEnd, + ); err != nil { + return nil, err + } + + blobList = append(blobList, &blob) + } + + return blobList, nil +} + +func (p *pg) SetBlob(ctx context.Context, txn pgx.Tx, b *types.Blob) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + _, err := txn.Exec(childCtx, queries.SetBlob, b.UUID, b.Digest, b.Skylink, b.RangeStart, b.RangeEnd) + return err + +} + +func (p *pg) GetConfig(ctx context.Context, namespace string) ([]*types.ConfigV2, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + rows, err := p.conn.Query(childCtx, queries.GetConfig, namespace) + if err != nil { + return nil, err + } + defer rows.Close() + cfgList := make([]*types.ConfigV2, 0) + + for i := 0; rows.Next(); i++ { + var cfg types.ConfigV2 + if err := rows.Scan( + &cfg.UUID, + &cfg.Namespace, + &cfg.Reference, + &cfg.Digest, + &cfg.Skylink, + &cfg.MediaType, + &cfg.Layers, + &cfg.Size, + ); err != nil { + return nil, err + } + + cfgList = append(cfgList, &cfg) + } + + return cfgList, nil +} +func (p *pg) GetImageTags(ctx context.Context, namespace string) ([]string, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + rows, err := p.conn.Query(childCtx, queries.GetImageTags, namespace) + if err != nil { + return nil, err + } + defer rows.Close() + var tags []string + + for i := 0; rows.Next(); i++ { + var tag string + if err := rows.Scan(&tag); err != nil { + return nil, err + } + + tags = append(tags, tag) + } + + return tags, nil +} + +func (p *pg) SetConfig(ctx context.Context, txn pgx.Tx, cfg types.ConfigV2) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + if _, err := txn.Exec(childCtx, + queries.SetConfig, + cfg.UUID, + cfg.Namespace, + cfg.Reference, + cfg.Digest, + cfg.Skylink, + cfg.MediaType, + cfg.Layers, + cfg.Size, + ); err != nil { + return err + } + return nil +} + +func (p *pg) GetCatalog(ctx context.Context) ([]*types.ConfigV2, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + rows, err := p.conn.Query(childCtx, queries.GetCatalog) + if err != nil { + return nil, err + } + + cfgList := make([]*types.ConfigV2, 0) + for i := 0; rows.Next(); i++ { + var cfg types.ConfigV2 + if err := rows.Scan( + &cfg.Namespace, + &cfg.Reference, + &cfg.Digest, + ); err != nil { + return nil, err + } + + cfgList = append(cfgList, &cfg) + } + + return cfgList, nil +} + +func (p *pg) DeleteLayerV2(ctx context.Context, txn pgx.Tx, digest string) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + if _, err := txn.Exec(childCtx, queries.DeleteLayer, digest); err != nil { + return err + } + return nil +} + +func (p *pg) DeleteBlobV2(ctx context.Context, txn pgx.Tx, digest string) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + if _, err := txn.Exec(childCtx, queries.DeleteBlob, digest); err != nil { + return err + } + return nil +} + +func (p *pg) DeleteManifestOrTag(ctx context.Context, txn pgx.Tx, reference string) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + query := queries.DeleteManifestByRef + if strings.HasPrefix(reference, "sha256") { + query = queries.DeleteManifestByDig + } + if _, err := txn.Exec(childCtx, query, reference); err != nil { + return err + } + return nil +} + +func (p *pg) NewTxn(ctx context.Context) (pgx.Tx, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute*30) + defer cancel() + + return p.conn.Begin(childCtx) +} + +func (p *pg) Abort(ctx context.Context, txn pgx.Tx) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute*30) + defer cancel() + + return txn.Rollback(childCtx) +} + +func (p *pg) Commit(ctx context.Context, txn pgx.Tx) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute*30) + defer cancel() + + return txn.Commit(childCtx) +} diff --git a/store/postgres/postgres.go b/store/postgres/postgres.go new file mode 100644 index 00000000..bf26cd2d --- /dev/null +++ b/store/postgres/postgres.go @@ -0,0 +1,107 @@ +package postgres + +import ( + "context" + "time" + + "github.com/containerish/OpenRegistry/config" + "github.com/containerish/OpenRegistry/types" + "github.com/fatih/color" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/labstack/echo/v4" +) + +type PersistentStore interface { + UserStore + RegistryStore + Get(key []byte) ([]byte, error) + Set(key, value []byte) error + GetDigest(digest string) (*types.LayerRef, error) + SetDigest(digest, skylink string) error + DeleteDigest(digest string) error + DeleteLayer(namespace, digest string) error + Update(key, value []byte) error + ListAll() ([]byte, error) + ListWithPrefix(prefix []byte) ([]byte, error) + Delete(key []byte) error + GetSkynetURL(key string, ref string) (string, error) + UpdateManifestRef(namespace, ref string) error + ResolveManifestRef(namespace, ref string) (string, error) + Metadata(ctx echo.Context) error + LayerDigests(ctx echo.Context) error + RegisterForBeta(ctx echo.Context) error + GetAllEmail(ctx echo.Context) error + Close() error +} + +type UserStore interface { + AddUser(ctx context.Context, u *types.User) error + GetUser(ctx context.Context, identifier string) (*types.User, error) + UpdateUser(ctx context.Context, identifier string, u *types.User) error + DeleteUser(ctx context.Context, identifier string) error + IsActive(ctx context.Context, identifier string) bool +} + +type RegistryStore interface { + NewTxn(ctx context.Context) (pgx.Tx, error) + Abort(ctx context.Context, txn pgx.Tx) error + Commit(ctx context.Context, txn pgx.Tx) error + SetLayer(ctx context.Context, txn pgx.Tx, l *types.LayerV2) error + SetManifest(ctx context.Context, txn pgx.Tx, im *types.ImageManifestV2) error + SetBlob(ctx context.Context, txn pgx.Tx, b *types.Blob) error + SetConfig(ctx context.Context, txn pgx.Tx, cfg types.ConfigV2) error + GetManifest(ctx context.Context, ref string) (*types.ImageManifestV2, error) + GetManifestByReference(ctx context.Context, namespace string, ref string) (*types.ConfigV2, error) + GetLayer(ctx context.Context, digest string) (*types.LayerV2, error) + GetBlob(ctx context.Context, digest string) ([]*types.Blob, error) + GetConfig(ctx context.Context, namespace string) ([]*types.ConfigV2, error) + GetImageTags(ctx context.Context, namespace string) ([]string, error) + GetCatalog(ctx context.Context) ([]*types.ConfigV2, error) + DeleteLayerV2(ctx context.Context, txn pgx.Tx, digest string) error + DeleteBlobV2(ctx context.Context, txn pgx.Tx, digest string) error + DeleteManifestOrTag(ctx context.Context, txn pgx.Tx, reference string) error +} + +type pg struct { + conn *pgxpool.Pool +} + +func New(cfg *config.StoreConfig) (PersistentStore, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + pgxCofig, err := pgxpool.ParseConfig(cfg.Endpoint()) + if err != nil { + return nil, err + } + + conn, err := pgxpool.ConnectConfig(ctx, pgxCofig) + if err != nil { + return nil, err + } + + color.Green("connection to database successful") + return &pg{conn: conn}, nil +} + +func (p *pg) RegisterForBeta(ctx echo.Context) error { return nil } +func (p *pg) Get(key []byte) ([]byte, error) { return nil, nil } +func (p *pg) Set(key, value []byte) error { return nil } +func (p *pg) Update(key, value []byte) error { return nil } +func (p *pg) Delete(key []byte) error { return nil } +func (p *pg) ListAll() ([]byte, error) { return nil, nil } +func (p *pg) ListWithPrefix(prefix []byte) ([]byte, error) { return nil, nil } +func (p *pg) Metadata(ctx echo.Context) error { return nil } +func (p *pg) GetAllEmail(ctx echo.Context) error { return nil } + +func (p *pg) GetSkynetURL(key string, ref string) (string, error) { return "", nil } +func (p *pg) DeleteLayer(namespace, digest string) error { return nil } +func (p *pg) LayerDigests(ctx echo.Context) error { return nil } +func (p *pg) UpdateManifestRef(namespace, ref string) error { return nil } +func (p *pg) ResolveManifestRef(namespace, ref string) (string, error) { return "", nil } + +func (p *pg) GetDigest(digest string) (*types.LayerRef, error) { return nil, nil } +func (p *pg) SetDigest(digest, skylink string) error { return nil } +func (p *pg) DeleteDigest(digest string) error { return nil } +func (p *pg) Close() error { return nil } diff --git a/store/postgres/queries/registry.go b/store/postgres/queries/registry.go new file mode 100644 index 00000000..4787f312 --- /dev/null +++ b/store/postgres/queries/registry.go @@ -0,0 +1,41 @@ +//nolint +package queries + +// insert queries +var ( + // + SetImageManifest = `insert into image_manifest (uuid, namespace, media_type, schema_version) + values ($1, $2, $3, $4) on conflict (namespace) do update set schema_version=3` + SetLayer = `insert into layer (media_type, digest, sky_link, uuid, blob_ids, size) + values ($1, $2, $3, $4, $5, $6) on conflict (digest) do nothing;` + + // SetBlob TODO - (guacamole/jay-dee7) find a better way to handle duplicates in blob + SetBlob = `insert into blob (uuid, digest, sky_link, start_range, end_range) + values ($1, $2, $3, $4, $5) on conflict (digest) do nothing;` + + SetConfig = `insert into config (uuid, namespace, reference, digest, sky_link, media_type, layers, size) + values ($1, $2, $3, $4, $5, $6,$7, $8) on conflict (namespace,reference) + do update set digest=$4, sky_link=$5,layers=$7;` +) + +// select queries +var ( + GetDigest = `select digest from layers where digest=$1;` + ReadMetadata = `select * from metadata where namespace=$1;` + GetLayer = `select * from layer where digest=$1;` + GetManifest = `select * from image_manifest where namespace=$1;` + GetBlob = `select * from blob where digest=$1;` + GetConfig = `select * from config where namespace=$1;` + GetImageTags = `select reference from config where namespace=$1;` + GetManifestByRef = `select * from config where namespace=$1 and reference=$2;` + GetManifestByDig = `select * from config where namespace=$1 and digest=$2;` + GetCatalog = `select namespace,reference,digest from config;` +) + +// delete queries +var ( + DeleteLayer = `delete from layer where digest=$1;` + DeleteBlob = `delete from blob where digest=$1;` + DeleteManifestByRef = `delete from config where reference=$1;` + DeleteManifestByDig = `delete from config where digest=$1;` +) diff --git a/store/postgres/queries/users.go b/store/postgres/queries/users.go new file mode 100644 index 00000000..f3c3d40e --- /dev/null +++ b/store/postgres/queries/users.go @@ -0,0 +1,12 @@ +//nolint +package queries + +var ( + AddUser = `insert into users (id, is_active, username, email, password, created_at, updated_at) +values ($1, $2, $3, $4, $5, $6, $7);` + GetUser = `select is_active, username, email, password, created_at, updated_at from users where email=$1 + or username=$1;` + UpdateUser = `update user set username = $1, email = $2, password = $3, updated_at = $4 where username = $5;` + DeleteUser = `delete from user where username = $1;` + GetAllEmails = `select email from users;` +) diff --git a/store/postgres/users.go b/store/postgres/users.go new file mode 100644 index 00000000..5c8e5bb5 --- /dev/null +++ b/store/postgres/users.go @@ -0,0 +1,82 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "github.com/containerish/OpenRegistry/store/postgres/queries" + "github.com/containerish/OpenRegistry/types" + "github.com/google/uuid" +) + +func (p *pg) AddUser(ctx context.Context, u *types.User) error { + if err := u.Validate(); err != nil { + return err + } + + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*100) + defer cancel() + + t := time.Now() + id := uuid.New() + _, err := p.conn.Exec(childCtx, queries.AddUser, id.String(), true, u.Username, u.Email, u.Password, t, t) + if err != nil { + return fmt.Errorf("error adding user to database: %w", err) + } + + return nil +} + +func (p *pg) GetUser(ctx context.Context, identifier string) (*types.User, error) { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*100) + defer cancel() + + row := p.conn.QueryRow(childCtx, queries.GetUser, identifier) + + var user types.User + err := row.Scan(&user.IsActive, &user.Username, &user.Email, &user.Password, &user.CreatedAt, &user.UpdatedAt) + if err != nil { + return nil, fmt.Errorf("user not found") + } + + return &user, nil +} + +// UpdateUser +//update user set username = $1, email = $2, password = $3, updated_at = $4 where username = $5 +func (p *pg) UpdateUser(ctx context.Context, identifier string, u *types.User) error { + if err := u.Validate(); err != nil { + return err + } + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*100) + defer cancel() + + t := time.Now() + _, err := p.conn.Exec(childCtx, queries.UpdateUser, u.Username, u.Email, u.Password, t, identifier) + if err != nil { + return fmt.Errorf("error updating user: %s", identifier) + } + return nil +} + +// DeleteUser - delete from user where username = $1; +func (p *pg) DeleteUser(ctx context.Context, identifier string) error { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*100) + defer cancel() + + _, err := p.conn.Exec(childCtx, queries.DeleteUser, identifier) + if err != nil { + return fmt.Errorf("error deleting user: %s", identifier) + } + return nil +} + +//IsActive - if the user has logged in, isActive returns true +// this method is also useful for limiting access of malicious actors +func (p *pg) IsActive(ctx context.Context, identifier string) bool { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*100) + defer cancel() + row := p.conn.QueryRow(childCtx, queries.GetUser, identifier) + return row != nil +} diff --git a/types/types.go b/types/types.go index e36ad394..ce13faf4 100644 --- a/types/types.go +++ b/types/types.go @@ -11,12 +11,6 @@ type ( Manifest ImageManifest } - Manifest struct { - SkynetLink string - Reference string - Digest string - } - ImageManifest struct { MediaType string `json:"mediaType"` Layers []*Layer `json:"layers"` @@ -24,6 +18,13 @@ type ( SchemaVersion int `json:"schemaVersion"` } + ImageManifestV2 struct { + Uuid string `json:"uuid"` + Namespace string `json:"namespace"` + MediaType string `json:"mediaType"` + SchemaVersion int `json:"schemaVersion"` + } + Blob struct { Digest string Skylink string @@ -41,6 +42,15 @@ type ( Size int `json:"size"` } + LayerV2 struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + SkynetLink string `json:"skynetLink"` + UUID string `json:"uuid"` + BlobDigests []string `json:"blobs"` + Size int `json:"size"` + } + LayerRef struct { Digest string Skylink string @@ -53,6 +63,17 @@ type ( Reference string `json:"reference"` Size int `json:"size"` } + + ConfigV2 struct { + UUID string `json:"uuid"` + Namespace string `json:"namespace"` + Reference string `json:"reference"` + Digest string `json:"digest"` + Skylink string `json:"sky_link"` + MediaType string `json:"media_type"` + Layers []string `json:"layers"` + Size int `json:"size"` + } ) func (md Metadata) GetManifestByRef(ref string) (*Config, error) { diff --git a/types/users.go b/types/users.go new file mode 100644 index 00000000..feb129e4 --- /dev/null +++ b/types/users.go @@ -0,0 +1,25 @@ +package types + +import ( + "time" + + "github.com/go-playground/validator/v10" +) + +type ( + User struct { + Id string `json:"id" validate:"-"` + Email string `json:"email" validate:"email"` + Username string `json:"username" validate:"gte=4"` + Password string `json:"password" validate:"required,gte=8"` + CreatedAt time.Time `json:"created_at" validate:"-"` + UpdatedAt time.Time `json:"updated_at" validate:"-"` + IsActive bool `json:"is_active" validate:"-"` + } +) + +func (u *User) Validate() error { + v := validator.New() + + return v.Struct(u) +}