From 316c9d562ed2dd5ea3574ed2a31eaf4fb0a3d481 Mon Sep 17 00:00:00 2001 From: jay-dee7 Date: Tue, 23 Nov 2021 10:54:20 +0530 Subject: [PATCH 01/16] Feat: Pluggable Storage Backends Signed-off-by: jay-dee7 --- auth/auth.go | 6 +- auth/signup.go | 37 ++++++----- config/config.go | 22 +++++++ go.mod | 17 +++++- go.sum | 105 ++++++++++++++++++++++++++++++++ main.go | 11 +++- store/postgres/postgres.go | 87 ++++++++++++++++++++++++++ store/postgres/queries/users.go | 7 +++ store/postgres/users.go | 56 +++++++++++++++++ types/users.go | 25 ++++++++ 10 files changed, 354 insertions(+), 19 deletions(-) create mode 100644 store/postgres/postgres.go create mode 100644 store/postgres/queries/users.go create mode 100644 store/postgres/users.go create mode 100644 types/users.go diff --git a/auth/auth.go b/auth/auth.go index 9ad703e4..21706b06 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -3,6 +3,7 @@ package auth import ( "github.com/containerish/OpenRegistry/cache" "github.com/containerish/OpenRegistry/config" + "github.com/containerish/OpenRegistry/store/postgres" "github.com/containerish/OpenRegistry/telemetry" "github.com/labstack/echo/v4" ) @@ -18,13 +19,14 @@ type Authentication interface { } type auth struct { + pgStore postgres.PersistentStore store cache.Store c *config.RegistryConfig logger telemetry.Logger } // New is the constructor function returns an Authentication implementation -func New(s cache.Store, c *config.RegistryConfig, logger telemetry.Logger) Authentication { - a := &auth{store: s, c: c, logger: logger} +func New(s cache.Store, c *config.RegistryConfig, pgStore postgres.PersistentStore, logger telemetry.Logger) Authentication { + a := &auth{store: s, c: c, pgStore: pgStore, logger: logger} return a } diff --git a/auth/signup.go b/auth/signup.go index ef69b9e1..3404381a 100644 --- a/auth/signup.go +++ b/auth/signup.go @@ -178,30 +178,37 @@ func (a *auth) SignUp(ctx echo.Context) error { }) } u.Password = hpwd - bz, err = json.Marshal(u) - if err != nil { - ctx.Set(types.HttpEndpointErrorKey, err.Error()) - return ctx.JSON(http.StatusInternalServerError, echo.Map{ - "error": err.Error(), - }) - } - key := fmt.Sprintf("%s/%s", UserNameSpace, u.Username) - if err := a.store.Set([]byte(key), bz); err != nil { - ctx.Set(types.HttpEndpointErrorKey, err.Error()) - return ctx.JSON(http.StatusInternalServerError, echo.Map{ - "error": err.Error(), - }) + newUser := &types.User{ + Email: u.Email, + Username: u.Username, + Password: u.Password, } - key = fmt.Sprintf("%s/%s", UserNameSpace, u.Email) - if err := a.store.Set([]byte(key), bz); err != nil { + err = a.pgStore.AddUser(ctx.Request().Context(), newUser) + + // bz, err = json.Marshal(u) + if err != nil { ctx.Set(types.HttpEndpointErrorKey, err.Error()) return ctx.JSON(http.StatusInternalServerError, echo.Map{ "error": err.Error(), }) } + // key := fmt.Sprintf("%s/%s", UserNameSpace, u.Username) + // if err := a.store.Set([]byte(key), bz); err != nil { + // return ctx.JSON(http.StatusInternalServerError, echo.Map{ + // "error": err.Error(), + // }) + // } + + // key = fmt.Sprintf("%s/%s", UserNameSpace, u.Email) + // if err := a.store.Set([]byte(key), bz); err != nil { + // return ctx.JSON(http.StatusInternalServerError, echo.Map{ + // "error": err.Error(), + // }) + // } + return ctx.JSON(http.StatusCreated, echo.Map{ "message": "user successfully created", }) diff --git a/config/config.go b/config/config.go index 565101aa..42ea39a7 100644 --- a/config/config.go +++ b/config/config.go @@ -42,12 +42,34 @@ type ( Username string `mapstructure:"username"` Password string `mapstructure:"password"` } + + StoreConfig struct { + User string `mapstructure:"user"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + } ) func (r *RegistryConfig) Address() string { return fmt.Sprintf("%s:%d", r.Host, r.Port) } +func NewStoreConfig() (*StoreConfig, error) { + return &StoreConfig{ + User: "postgres", + Password: "Qwerty@123", + Database: "postgres", + Host: "0.0.0.0", + Port: 5432, + }, nil +} + +func (sc *StoreConfig) Endpoint() string { + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", sc.User, sc.Password, sc.Host, sc.Port, sc.Database) +} + func LoadFromENV() (*RegistryConfig, error) { // TODO - Add Support for loading from config // loading config from path is not possible right now, diff --git a/go.mod b/go.mod index f6ce7855..f47034b1 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,20 @@ require ( golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 ) +require ( + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.9.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.10.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.1.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.8.1 // indirect + github.com/leodido/go-urn v1.2.1 // indirect +) + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect @@ -32,6 +46,7 @@ require ( github.com/golang/snappy v0.0.3 // indirect github.com/google/flatbuffers v1.12.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jackc/pgx/v4 v4.13.0 github.com/klauspost/compress v1.12.3 // indirect github.com/labstack/gommon v0.3.0 // indirect github.com/magiconair/properties v1.8.5 // indirect @@ -54,7 +69,7 @@ require ( gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect - golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect + golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect google.golang.org/protobuf v1.26.0 // indirect diff --git a/go.sum b/go.sum index 7c2f6256..fa6c53c9 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -88,6 +89,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerish/go-skynet/v2 v2.0.2-0.20211206044851-95b8ecc10d44 h1:0oS0IvCItos8SeUg6hFGI6duDGiIQtx6lcLXKtvBggU= @@ -97,6 +99,8 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -146,12 +150,21 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.9.0 h1:NgTtmN58D0m8+UuxtYmGztBJB7VnPgjj221I1QHci2A= +github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -278,6 +291,52 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= +github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= +github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= +github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -299,11 +358,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -314,6 +377,12 @@ github.com/labstack/echo/v4 v4.5.0 h1:JXk6H5PAw9I3GwizqUHhYyS4f45iyGebR/c1xNCeOC github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -321,11 +390,15 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -386,6 +459,7 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -427,16 +501,25 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.24.0 h1:76ivFxmVSRs1u2wUwJVg5VZDYQgeH1JpoS6ndgr9Wy8= github.com/rs/zerolog v1.24.0/go.mod h1:7KHcEGe0QZPOm2IE4Kpb5rTh6n1h2hIgS5OOnu1rUaI= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -471,6 +554,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -500,6 +584,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8 h1:gZfMjx7Jr6N8b7iJO4eUjDsn6xJqoyXg8D+ogdoAfKY= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -518,12 +603,16 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= @@ -531,13 +620,17 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -657,6 +750,7 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -668,6 +762,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -705,6 +800,9 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 h1:siQdpVirKtzPhKl3lZWozZraCFObP8S1v6PRp0bLrtU= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -733,12 +831,14 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -780,6 +880,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -906,11 +1008,14 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= diff --git a/main.go b/main.go index c3860e6e..9fe1473a 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "github.com/containerish/OpenRegistry/registry/v2" "github.com/containerish/OpenRegistry/router" "github.com/containerish/OpenRegistry/skynet" + "github.com/containerish/OpenRegistry/store/postgres" "github.com/containerish/OpenRegistry/telemetry" fluentbit "github.com/containerish/OpenRegistry/telemetry/fluent-bit" "github.com/fatih/color" @@ -31,6 +32,14 @@ func main() { } defer localCache.Close() + pgConfig, _ := config.NewStoreConfig() + pgStore, err := postgres.New(pgConfig) + if err != nil { + color.Red("error here: %s", err.Error()) + return + } + + log := telemetry.SetupLogger() fluentBitCollector, err := fluentbit.New(cfg) if err != nil { color.Red("error initializing fluentbit collector: %s\n", err) @@ -38,7 +47,7 @@ func main() { } logger := telemetry.ZLogger(telemetry.SetupLogger(), fluentBitCollector) - authSvc := auth.New(localCache, cfg, logger) + authSvc := auth.New(localCache, cfg, pgStore, logger) skynetClient := skynet.NewClient(cfg) reg, err := registry.NewRegistry(skynetClient, localCache, logger) diff --git a/store/postgres/postgres.go b/store/postgres/postgres.go new file mode 100644 index 00000000..0c2b22ff --- /dev/null +++ b/store/postgres/postgres.go @@ -0,0 +1,87 @@ +package postgres + +import ( + "context" + "time" + + "github.com/containerish/OpenRegistry/config" + "github.com/containerish/OpenRegistry/types" + "github.com/fatih/color" + "github.com/jackc/pgx/v4" + "github.com/labstack/echo/v4" +) + +type PersistentStore interface { + UserStore + Get(key []byte) ([]byte, error) + Set(key, value []byte) error + GetDigest(digest string) (*types.LayerRef, error) + SetDigest(digest, skylink string) error + DeleteDigest(digest string) error + DeleteLayer(namespace, digest string) error + Update(key, value []byte) error + ListAll() ([]byte, error) + ListWithPrefix(prefix []byte) ([]byte, error) + Delete(key []byte) error + GetSkynetURL(key string, ref string) (string, error) + UpdateManifestRef(namespace, ref string) error + ResolveManifestRef(namespace, ref string) (string, error) + Metadata(ctx echo.Context) error + LayerDigests(ctx echo.Context) error + RegisterForBeta(ctx echo.Context) error + GetAllEmail(ctx echo.Context) error + Close() error +} + +type UserStore interface { + AddUser(ctx context.Context, u *types.User) error + GetUser(ctx context.Context, identifier string) (*types.User, error) + UpdateUser(identifier string, u *types.User) error + DeleteUser(identifier string) error + IsActive(identifier string) bool +} + +type pg struct { + conn *pgx.Conn +} + +func New(cfg *config.StoreConfig) (PersistentStore, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + conn, err := pgx.Connect(ctx, cfg.Endpoint()) + if err != nil { + return nil, err + } + + color.Green("connection to database successful") + + return &pg{conn: conn}, nil +} + +func (p *pg) Get(key []byte) ([]byte, error) { + return nil, nil +} + +func (p *pg) Set(key, value []byte) error { + return nil +} +func (p *pg) GetDigest(digest string) (*types.LayerRef, error) { + return nil, nil +} + +func (p *pg) SetDigest(digest, skylink string) error { return nil } +func (p *pg) DeleteDigest(digest string) error { return nil } +func (p *pg) DeleteLayer(namespace, digest string) error { return nil } +func (p *pg) Update(key, value []byte) error { return nil } +func (p *pg) ListAll() ([]byte, error) { return nil, nil } +func (p *pg) ListWithPrefix(prefix []byte) ([]byte, error) { return nil, nil } +func (p *pg) Delete(key []byte) error { return nil } +func (p *pg) GetSkynetURL(key string, ref string) (string, error) { return "", nil } +func (p *pg) UpdateManifestRef(namespace, ref string) error { return nil } +func (p *pg) ResolveManifestRef(namespace, ref string) (string, error) { return "", nil } +func (p *pg) Metadata(ctx echo.Context) error { return nil } +func (p *pg) LayerDigests(ctx echo.Context) error { return nil } +func (p *pg) RegisterForBeta(ctx echo.Context) error { return nil } +func (p *pg) GetAllEmail(ctx echo.Context) error { return nil } +func (p *pg) Close() error { return nil } diff --git a/store/postgres/queries/users.go b/store/postgres/queries/users.go new file mode 100644 index 00000000..25255f43 --- /dev/null +++ b/store/postgres/queries/users.go @@ -0,0 +1,7 @@ +package queries + +var ( + AddUser = `insert into users (id, is_active, username, email, password, created_at, updated_at) +values ($1, $2, $3, $4, $5, $6, $7);` + GetUser = `select username, is_active, email, created_at, updated_at where email = $1` +) diff --git a/store/postgres/users.go b/store/postgres/users.go new file mode 100644 index 00000000..e34af0df --- /dev/null +++ b/store/postgres/users.go @@ -0,0 +1,56 @@ +package postgres + +import ( + "context" + "fmt" + "time" + + "github.com/containerish/OpenRegistry/store/postgres/queries" + "github.com/containerish/OpenRegistry/types" + "github.com/google/uuid" +) + +func (p *pg) AddUser(ctx context.Context, u *types.User) error { + if err := u.Validate(); err != nil { + return err + } + + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*100) + defer cancel() + + t := time.Now() + id := uuid.New() + _, err := p.conn.Exec(childCtx, queries.AddUser, id.String(), true, u.Username, u.Email, u.Password, t, t) + if err != nil { + return fmt.Errorf("error adding user to database: %w", err) + } + + return nil +} + +func (p *pg) GetUser(ctx context.Context, identifier string) (*types.User, error) { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*100) + defer cancel() + + row := p.conn.QueryRow(childCtx, queries.GetUser, identifier) + + var user types.User + err := row.Scan(&user.Username, &user.IsActive, &user.Email, &user.CreatedAt, &user.UpdatedAt) + if err != nil { + return nil, err + } + + return &user, nil +} + +func (p *pg) UpdateUser(identifier string, u *types.User) error { + return nil +} + +func (p *pg) DeleteUser(identifier string) error { + return nil +} + +func (p *pg) IsActive(identifier string) bool { + return false +} diff --git a/types/users.go b/types/users.go new file mode 100644 index 00000000..feb129e4 --- /dev/null +++ b/types/users.go @@ -0,0 +1,25 @@ +package types + +import ( + "time" + + "github.com/go-playground/validator/v10" +) + +type ( + User struct { + Id string `json:"id" validate:"-"` + Email string `json:"email" validate:"email"` + Username string `json:"username" validate:"gte=4"` + Password string `json:"password" validate:"required,gte=8"` + CreatedAt time.Time `json:"created_at" validate:"-"` + UpdatedAt time.Time `json:"updated_at" validate:"-"` + IsActive bool `json:"is_active" validate:"-"` + } +) + +func (u *User) Validate() error { + v := validator.New() + + return v.Struct(u) +} From 5579e522907bdf086b3800b8badcf2e511335b0c Mon Sep 17 00:00:00 2001 From: guacamole Date: Fri, 26 Nov 2021 11:59:38 +0530 Subject: [PATCH 02/16] Feat: second commit in series of database migration to postgres - added queries and apis for users table Signed-off-by: guacamole --- cache/store.go | 470 ++++++++++++++--------------- router/router.go | 7 +- store/postgres/postgres.go | 44 ++- store/postgres/queries/registry.go | 6 + store/postgres/queries/users.go | 4 + store/postgres/users.go | 36 ++- 6 files changed, 301 insertions(+), 266 deletions(-) create mode 100644 store/postgres/queries/registry.go diff --git a/cache/store.go b/cache/store.go index 6917287f..7cb89fde 100644 --- a/cache/store.go +++ b/cache/store.go @@ -50,6 +50,159 @@ func New(storeLocation string) (Store, error) { return &dataStore{db: db}, nil } +func (ds *dataStore) Get(key []byte) ([]byte, error) { + var res []byte + + err := ds.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(key) + if err != nil { + return err + } + err = item.Value(func(v []byte) error { + res = make([]byte, len(v)) + copy(res, v) + return nil + }) + return err + }) + if err != nil { + return nil, err + } + + return res, nil +} + +func (ds *dataStore) GetDigest(digest string) (*types.LayerRef, error) { + key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) + bz, err := ds.Get([]byte(key)) + if err != nil { + return nil, err + } + + var layerRef types.LayerRef + err = json.Unmarshal(bz, &layerRef) + return &layerRef, err +} + +func (ds *dataStore) GetSkynetURL(key, ref string) (string, error) { + var res []byte + err := ds.db.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) + if err != nil { + return err + } + + return item.Value(func(v []byte) error { + res = make([]byte, len(v)) + copy(res, v) + return nil + }) + }) + + if err != nil { + return "", err + } + + var md types.Metadata + err = json.Unmarshal(res, &md) + if err != nil { + return "", err + } + + return md.FindLinkForDigest(ref) +} + +func (ds *dataStore) ListAll() ([]byte, error) { + var buf []types.Metadata + + err := ds.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchSize = 10 + it := txn.NewIterator(opts) + defer it.Close() + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + err := item.Value(func(v []byte) error { + var md types.Metadata + if err := json.Unmarshal(v, &md); err != nil { + return err + } + buf = append(buf, md) + return nil + }) + + if err != nil { + return err + } + } + return nil + }) + if err != nil { + return nil, err + } + + return json.Marshal(buf) +} + +func (ds *dataStore) ListWithPrefix(prefix []byte) ([]byte, error) { + + var buf []byte + + err := ds.db.View(func(txn *badger.Txn) error { + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + item := it.Item() + err := item.Value(func(v []byte) error { + buf = make([]byte, len(v)) + copy(buf, v) + return nil + }) + if err != nil { + return err + } + } + return nil + }) + return buf, err +} + +func (ds *dataStore) ResolveManifestRef(namespace, ref string) (string, error) { + var res []byte + fn := func(txn *badger.Txn) error { + item, err := txn.Get([]byte(namespace)) + if err != nil { + return err + } + + return item.Value(func(v []byte) error { + res = make([]byte, len(v)) + copy(res, v) + return nil + }) + } + + if err := ds.db.View(fn); err != nil { + return "", err + } + + var md types.Metadata + err := json.Unmarshal(res, &md) + if err != nil { + return "", err + } + + for _, c := range md.Manifest.Config { + mdRef := c.Reference + mdDigest := c.Digest + if ref == mdRef || ref == mdDigest { + return c.SkynetLink, nil + } + } + + return "", fmt.Errorf("ref not found") +} + func (ds *dataStore) LayerDigests(ctx echo.Context) error { bz, err := ds.ListWithPrefix([]byte(layerDigestNamespace)) if err != nil { @@ -59,6 +212,39 @@ func (ds *dataStore) LayerDigests(ctx echo.Context) error { return ctx.JSONBlob(http.StatusOK, bz) } +// QueryMetaData is similar to ListWithPrefix with only a few changes +// making these changes in ListWithPrefix was causing the oci tests to fail +func (ds *dataStore) QueryMetaData(prefix []byte) ([]byte, error) { + + var mds []types.Metadata + + err := ds.db.View(func(txn *badger.Txn) error { + it := txn.NewIterator(badger.DefaultIteratorOptions) + defer it.Close() + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + var md types.Metadata + item := it.Item() + err := item.Value(func(v []byte) error { + err := json.Unmarshal(v, &md) + if err != nil { + return err + } + mds = append(mds, md) + return nil + }) + if err != nil { + return err + } + } + return nil + }) + + if err != nil { + return nil, err + } + return json.Marshal(mds) +} + func (ds *dataStore) Metadata(ctx echo.Context) error { key := ctx.QueryParam("namespace") dig := ctx.QueryParam("digest") @@ -115,32 +301,30 @@ func (ds *dataStore) Metadata(ctx echo.Context) error { return ctx.JSON(http.StatusOK, reg) } -func (ds *dataStore) DeleteLayer(namespace, digest string) error { - bz, err := ds.Get([]byte(namespace)) - if err != nil { - return err - } +const layerDigestNamespace = "layers/digests" - var md types.Metadata - if err = json.Unmarshal(bz, &md); err != nil { +func (ds *dataStore) Set(key, value []byte) error { + txn := ds.db.NewTransaction(true) + + if err := txn.Set(key, value); err != nil { return err } - var match bool - for i, v := range md.Manifest.Layers { - if v.Digest == digest { - l := len(md.Manifest.Layers) - md.Manifest.Layers[i] = md.Manifest.Layers[l-1] - md.Manifest.Layers = md.Manifest.Layers[:l-1] - match = true - break - } + return txn.Commit() +} + +func (ds *dataStore) SetDigest(digest, skylink string) error { + key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) + value := types.LayerRef{ + Digest: digest, + Skylink: skylink, } - if !match { - return fmt.Errorf("layer/blob not found for digest %s", digest) + + if err := ds.Set([]byte(key), value.Bytes()); err != nil { + return err } - return ds.Set([]byte(namespace), md.Bytes()) + return nil } func (ds *dataStore) Update(key, value []byte) error { @@ -200,30 +384,6 @@ EndLoop: return ds.Set(key, bz) } -func (ds *dataStore) removeDuplicateLayers(src, dst []*types.Layer) ([]*types.Layer, error) { - size := uint(len(src) + len(dst)) - - if size > math.MaxInt16 { - return nil, fmt.Errorf("ERROR_TOO_MANY_LAYERS") - } - - list := make([]*types.Layer, size) - list = append(list, src...) - list = append(list, dst...) - - seenMap := make(map[string]bool) - var layers []*types.Layer - - for _, l := range list { - if l != nil && !seenMap[l.Digest] { - seenMap[l.Digest] = true - layers = append(layers, l) - } - } - - return layers, nil -} - func (ds *dataStore) UpdateManifestRef(namespace, ref string) error { var res []byte fn := func(txn *badger.Txn) error { @@ -270,222 +430,62 @@ func (ds *dataStore) UpdateManifestRef(namespace, ref string) error { return ds.Set([]byte(namespace), md.Bytes()) } -func (ds *dataStore) ResolveManifestRef(namespace, ref string) (string, error) { - var res []byte - fn := func(txn *badger.Txn) error { - item, err := txn.Get([]byte(namespace)) - if err != nil { - return err - } +func (ds *dataStore) removeDuplicateLayers(src, dst []*types.Layer) ([]*types.Layer, error) { + size := uint(len(src) + len(dst)) - return item.Value(func(v []byte) error { - res = make([]byte, len(v)) - copy(res, v) - return nil - }) + if size > math.MaxInt16 { + return nil, fmt.Errorf("ERROR_TOO_MANY_LAYERS") } - if err := ds.db.View(fn); err != nil { - return "", err - } + list := make([]*types.Layer, size) + list = append(list, src...) + list = append(list, dst...) - var md types.Metadata - err := json.Unmarshal(res, &md) - if err != nil { - return "", err - } + seenMap := make(map[string]bool) + var layers []*types.Layer - for _, c := range md.Manifest.Config { - mdRef := c.Reference - mdDigest := c.Digest - if ref == mdRef || ref == mdDigest { - return c.SkynetLink, nil + for _, l := range list { + if l != nil && !seenMap[l.Digest] { + seenMap[l.Digest] = true + layers = append(layers, l) } } - return "", fmt.Errorf("ref not found") -} - -const layerDigestNamespace = "layers/digests" - -func (ds *dataStore) SetDigest(digest, skylink string) error { - key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) - value := types.LayerRef{ - Digest: digest, - Skylink: skylink, - } - - if err := ds.Set([]byte(key), value.Bytes()); err != nil { - return err - } - - return nil -} - -func (ds *dataStore) DeleteDigest(digest string) error { - key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) - - return ds.Delete([]byte(key)) -} - -func (ds *dataStore) GetDigest(digest string) (*types.LayerRef, error) { - key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) - bz, err := ds.Get([]byte(key)) - if err != nil { - return nil, err - } - - var layerRef types.LayerRef - err = json.Unmarshal(bz, &layerRef) - return &layerRef, err + return layers, nil } -func (ds *dataStore) GetSkynetURL(key, ref string) (string, error) { - var res []byte - err := ds.db.View(func(txn *badger.Txn) error { - item, err := txn.Get([]byte(key)) - if err != nil { - return err - } - - return item.Value(func(v []byte) error { - res = make([]byte, len(v)) - copy(res, v) - return nil - }) - }) - +func (ds *dataStore) DeleteLayer(namespace, digest string) error { + bz, err := ds.Get([]byte(namespace)) if err != nil { - return "", err + return err } var md types.Metadata - err = json.Unmarshal(res, &md) - if err != nil { - return "", err - } - - return md.FindLinkForDigest(ref) -} - -func (ds *dataStore) Set(key, value []byte) error { - txn := ds.db.NewTransaction(true) - - if err := txn.Set(key, value); err != nil { + if err = json.Unmarshal(bz, &md); err != nil { return err } - return txn.Commit() -} - -func (ds *dataStore) Get(key []byte) ([]byte, error) { - var res []byte - - err := ds.db.View(func(txn *badger.Txn) error { - item, err := txn.Get(key) - if err != nil { - return err + var match bool + for i, v := range md.Manifest.Layers { + if v.Digest == digest { + l := len(md.Manifest.Layers) + md.Manifest.Layers[i] = md.Manifest.Layers[l-1] + md.Manifest.Layers = md.Manifest.Layers[:l-1] + match = true + break } - err = item.Value(func(v []byte) error { - res = make([]byte, len(v)) - copy(res, v) - return nil - }) - return err - }) - if err != nil { - return nil, err } - - return res, nil -} - -func (ds *dataStore) ListAll() ([]byte, error) { - var buf []types.Metadata - - err := ds.db.View(func(txn *badger.Txn) error { - opts := badger.DefaultIteratorOptions - opts.PrefetchSize = 10 - it := txn.NewIterator(opts) - defer it.Close() - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - err := item.Value(func(v []byte) error { - var md types.Metadata - if err := json.Unmarshal(v, &md); err != nil { - return err - } - buf = append(buf, md) - return nil - }) - - if err != nil { - return err - } - } - return nil - }) - if err != nil { - return nil, err + if !match { + return fmt.Errorf("layer/blob not found for digest %s", digest) } - return json.Marshal(buf) -} - -func (ds *dataStore) ListWithPrefix(prefix []byte) ([]byte, error) { - - var buf []byte - - err := ds.db.View(func(txn *badger.Txn) error { - it := txn.NewIterator(badger.DefaultIteratorOptions) - defer it.Close() - for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { - item := it.Item() - err := item.Value(func(v []byte) error { - buf = make([]byte, len(v)) - copy(buf, v) - return nil - }) - if err != nil { - return err - } - } - return nil - }) - return buf, err + return ds.Set([]byte(namespace), md.Bytes()) } -// QueryMetaData is similar to ListWithPrefix with only a few changes -// making these changes in ListWithPrefix was causing the oci tests to fail -func (ds *dataStore) QueryMetaData(prefix []byte) ([]byte, error) { - - var mds []types.Metadata - - err := ds.db.View(func(txn *badger.Txn) error { - it := txn.NewIterator(badger.DefaultIteratorOptions) - defer it.Close() - for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { - var md types.Metadata - item := it.Item() - err := item.Value(func(v []byte) error { - err := json.Unmarshal(v, &md) - if err != nil { - return err - } - mds = append(mds, md) - return nil - }) - if err != nil { - return err - } - } - return nil - }) +func (ds *dataStore) DeleteDigest(digest string) error { + key := fmt.Sprintf("%s/%s", layerDigestNamespace, digest) - if err != nil { - return nil, err - } - return json.Marshal(mds) + return ds.Delete([]byte(key)) } func (ds *dataStore) Delete(key []byte) error { diff --git a/router/router.go b/router/router.go index 260bc594..a8af5ae6 100644 --- a/router/router.go +++ b/router/router.go @@ -1,6 +1,8 @@ package router import ( + "github.com/containerish/OpenRegistry/store/postgres" + "log" "net/http" "github.com/containerish/OpenRegistry/auth" @@ -15,7 +17,10 @@ import ( // Register is the entry point that registers all the endpoints // nolint -func Register(cfg *config.RegistryConfig, e *echo.Echo, reg registry.Registry, authSvc auth.Authentication, localCache cache.Store) { +func Register( + cfg *config.RegistryConfig, e *echo.Echo, reg registry.Registry, authSvc auth.Authentication, + localCache cache.Store, pStore postgres.PersistentStore, + ) { e.Use(middleware.Recover()) e.Use(middleware.CORS()) diff --git a/store/postgres/postgres.go b/store/postgres/postgres.go index 0c2b22ff..03ca8bec 100644 --- a/store/postgres/postgres.go +++ b/store/postgres/postgres.go @@ -36,9 +36,9 @@ type PersistentStore interface { type UserStore interface { AddUser(ctx context.Context, u *types.User) error GetUser(ctx context.Context, identifier string) (*types.User, error) - UpdateUser(identifier string, u *types.User) error - DeleteUser(identifier string) error - IsActive(identifier string) bool + UpdateUser(ctx context.Context, identifier string, u *types.User) error + DeleteUser(ctx context.Context, identifier string) error + IsActive(ctx context.Context, identifier string) bool } type pg struct { @@ -59,29 +59,23 @@ func New(cfg *config.StoreConfig) (PersistentStore, error) { return &pg{conn: conn}, nil } -func (p *pg) Get(key []byte) ([]byte, error) { - return nil, nil -} +func (p *pg) RegisterForBeta(ctx echo.Context) error { return nil } +func (p *pg) Get(key []byte) ([]byte, error) { return nil, nil } +func (p *pg) Set(key, value []byte) error { return nil } +func (p *pg) Update(key, value []byte) error { return nil } +func (p *pg) Delete(key []byte) error { return nil } +func (p *pg) ListAll() ([]byte, error) { return nil, nil } +func (p *pg) ListWithPrefix(prefix []byte) ([]byte, error) { return nil, nil } +func (p *pg) Metadata(ctx echo.Context) error { return nil } +func (p *pg) GetAllEmail(ctx echo.Context) error { return nil } -func (p *pg) Set(key, value []byte) error { - return nil -} -func (p *pg) GetDigest(digest string) (*types.LayerRef, error) { - return nil, nil -} - -func (p *pg) SetDigest(digest, skylink string) error { return nil } -func (p *pg) DeleteDigest(digest string) error { return nil } -func (p *pg) DeleteLayer(namespace, digest string) error { return nil } -func (p *pg) Update(key, value []byte) error { return nil } -func (p *pg) ListAll() ([]byte, error) { return nil, nil } -func (p *pg) ListWithPrefix(prefix []byte) ([]byte, error) { return nil, nil } -func (p *pg) Delete(key []byte) error { return nil } func (p *pg) GetSkynetURL(key string, ref string) (string, error) { return "", nil } +func (p *pg) DeleteLayer(namespace, digest string) error { return nil } +func (p *pg) LayerDigests(ctx echo.Context) error { return nil } func (p *pg) UpdateManifestRef(namespace, ref string) error { return nil } func (p *pg) ResolveManifestRef(namespace, ref string) (string, error) { return "", nil } -func (p *pg) Metadata(ctx echo.Context) error { return nil } -func (p *pg) LayerDigests(ctx echo.Context) error { return nil } -func (p *pg) RegisterForBeta(ctx echo.Context) error { return nil } -func (p *pg) GetAllEmail(ctx echo.Context) error { return nil } -func (p *pg) Close() error { return nil } + +func (p *pg) GetDigest(digest string) (*types.LayerRef, error) { return nil, nil } +func (p *pg) SetDigest(digest, skylink string) error { return nil } +func (p *pg) DeleteDigest(digest string) error { return nil } +func (p *pg) Close() error { return nil } diff --git a/store/postgres/queries/registry.go b/store/postgres/queries/registry.go new file mode 100644 index 00000000..03c67335 --- /dev/null +++ b/store/postgres/queries/registry.go @@ -0,0 +1,6 @@ +package queries + +var ( + GetAllEmails = `select email from users;` + +) diff --git a/store/postgres/queries/users.go b/store/postgres/queries/users.go index 25255f43..050973ee 100644 --- a/store/postgres/queries/users.go +++ b/store/postgres/queries/users.go @@ -4,4 +4,8 @@ var ( AddUser = `insert into users (id, is_active, username, email, password, created_at, updated_at) values ($1, $2, $3, $4, $5, $6, $7);` GetUser = `select username, is_active, email, created_at, updated_at where email = $1` + UpdateUser = `update user set username = $1, email = $2, password = $3, updated_at = $4 where username = $5;` + DeleteUser = `delete from user where username = $1;` + ) + diff --git a/store/postgres/users.go b/store/postgres/users.go index e34af0df..a9f03580 100644 --- a/store/postgres/users.go +++ b/store/postgres/users.go @@ -37,20 +37,46 @@ func (p *pg) GetUser(ctx context.Context, identifier string) (*types.User, error var user types.User err := row.Scan(&user.Username, &user.IsActive, &user.Email, &user.CreatedAt, &user.UpdatedAt) if err != nil { - return nil, err + return nil, fmt.Errorf("user not found") } return &user, nil } -func (p *pg) UpdateUser(identifier string, u *types.User) error { +// UpdateUser +//update user set username = $1, email = $2, password = $3, updated_at = $4 where username = $5 +func (p *pg) UpdateUser(ctx context.Context, identifier string, u *types.User) error { + if err := u.Validate(); err!= nil { + return err + } + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond* 100) + defer cancel() + + t := time.Now() + _, err := p.conn.Exec(childCtx, queries.UpdateUser, u.Username, u.Email, u.Password, t, identifier) + if err != nil { + return fmt.Errorf("error updating user: %s",identifier) + } return nil } -func (p *pg) DeleteUser(identifier string) error { +// DeleteUser - delete from user where username = $1; +func (p *pg) DeleteUser(ctx context.Context, identifier string) error { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond * 100) + defer cancel() + + _, err := p.conn.Exec(childCtx, queries.DeleteUser, identifier) + if err != nil { + return fmt.Errorf("error deleting user: %s", identifier) + } return nil } -func (p *pg) IsActive(identifier string) bool { - return false +//IsActive - if the user has logged in, isActive returns true +// this method is also useful for limiting access of malicious actors +func (p *pg) IsActive(ctx context.Context, identifier string) bool { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond * 100) + defer cancel() + row := p.conn.QueryRow(childCtx, queries.GetUser, identifier) + return row != nil } From 4c420f8fa7f9feb0d92f8087df89d069879ea28b Mon Sep 17 00:00:00 2001 From: guacamole Date: Tue, 30 Nov 2021 11:46:55 +0530 Subject: [PATCH 03/16] WIP: for postgres implementation Signed-off-by: guacamole --- cache/store.go | 3 +++ router/router.go | 8 ++++++-- store/postgres/container_image.go | 1 + store/postgres/queries/registry.go | 17 ++++++++++++++++- 4 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 store/postgres/container_image.go diff --git a/cache/store.go b/cache/store.go index 7cb89fde..c6f5dd41 100644 --- a/cache/store.go +++ b/cache/store.go @@ -19,11 +19,14 @@ type dataStore struct { type Store interface { Get(key []byte) ([]byte, error) Set(key, value []byte) error + GetDigest(digest string) (*types.LayerRef, error) SetDigest(digest, skylink string) error DeleteDigest(digest string) error + DeleteLayer(namespace, digest string) error Update(key, value []byte) error + ListAll() ([]byte, error) ListWithPrefix(prefix []byte) ([]byte, error) Delete(key []byte) error diff --git a/router/router.go b/router/router.go index a8af5ae6..617a03ad 100644 --- a/router/router.go +++ b/router/router.go @@ -18,8 +18,12 @@ import ( // Register is the entry point that registers all the endpoints // nolint func Register( - cfg *config.RegistryConfig, e *echo.Echo, reg registry.Registry, authSvc auth.Authentication, - localCache cache.Store, pStore postgres.PersistentStore, + cfg *config.RegistryConfig, + e *echo.Echo, + reg registry.Registry, + authSvc auth.Authentication, + localCache cache.Store, + pStore postgres.PersistentStore, ) { e.Use(middleware.Recover()) e.Use(middleware.CORS()) diff --git a/store/postgres/container_image.go b/store/postgres/container_image.go new file mode 100644 index 00000000..bf560bea --- /dev/null +++ b/store/postgres/container_image.go @@ -0,0 +1 @@ +package postgres diff --git a/store/postgres/queries/registry.go b/store/postgres/queries/registry.go index 03c67335..afa819ac 100644 --- a/store/postgres/queries/registry.go +++ b/store/postgres/queries/registry.go @@ -1,6 +1,21 @@ package queries +// insert queries var ( - GetAllEmails = `select email from users;` + SetMetadata = `insert into metadata (uuid, namespace, manifest) values($1, $2, $3);` + SetImageManifest = `insert into image_manifest (uuid, media_type, layers, config, + schema_version) values ($1, $2, $3, $4, $5);` + SetLayer = `insert into layer (media_type, digest, sky_link, uuid, blobs, size) + values ($1, $2, $3, $4, $5, $6);` + SetBlob = `insert into blob (uuid, digest, sky_link, start_range, end_range) + values ($1, $2, $3, $4, $5);` + SetConfig = `insert into config (media_type, digest, sky_link, reference, size) + values ($1, $2, $3, $4, $5);` +) +// select queries +var ( + GetAllEmails = `select email from users;` + ReadMetadata = `select * from metadata where namespace=$1;` ) + From e9722a5d6f616f377ebc3ecfe62b63205733a6bb Mon Sep 17 00:00:00 2001 From: guacamole Date: Sat, 4 Dec 2021 22:21:18 +0530 Subject: [PATCH 04/16] Add: Added Registry Database Operations Signed-off-by: guacamole --- auth/signin.go | 23 +++--- auth/validate_user.go | 21 ++++-- config/config.go | 2 +- registry/v2/blobs.go | 45 ++++++++++- registry/v2/registry.go | 103 +++++++++++++++++++++----- registry/v2/types.go | 15 +++- store/postgres/container_image.go | 115 +++++++++++++++++++++++++++++ store/postgres/postgres.go | 13 ++++ store/postgres/queries/registry.go | 12 ++- store/postgres/queries/users.go | 4 +- types/types.go | 37 ++++++++-- 11 files changed, 338 insertions(+), 52 deletions(-) diff --git a/auth/signin.go b/auth/signin.go index 7b9bde9b..1d738619 100644 --- a/auth/signin.go +++ b/auth/signin.go @@ -2,7 +2,6 @@ package auth import ( "encoding/json" - "fmt" "net/http" "time" @@ -47,12 +46,13 @@ func (a *auth) SignIn(ctx echo.Context) error { "error": err.Error(), }) } - key = fmt.Sprintf("%s/%s", UserNameSpace, user.Email) + key = user.Email } else { - key = fmt.Sprintf("%s/%s", UserNameSpace, user.Username) + key = user.Username } - bz, err := a.store.Get([]byte(key)) + //bz, err := a.store.Get([]byte(key)) + userFromDb, err := a.pgStore.GetUser(ctx.Request().Context(), key) if err != nil { ctx.Set(types.HttpEndpointErrorKey, err.Error()) return ctx.JSON(http.StatusBadRequest, echo.Map{ @@ -60,14 +60,6 @@ func (a *auth) SignIn(ctx echo.Context) error { }) } - var userFromDb User - if err := json.Unmarshal(bz, &userFromDb); err != nil { - ctx.Set(types.HttpEndpointErrorKey, err.Error()) - return ctx.JSON(http.StatusInternalServerError, echo.Map{ - "error": err.Error(), - }) - } - if !a.verifyPassword(userFromDb.Password, user.Password) { errMsg := "invalid password" ctx.Set(types.HttpEndpointErrorKey, errMsg) @@ -75,7 +67,12 @@ func (a *auth) SignIn(ctx echo.Context) error { } tokenLife := time.Now().Add(time.Hour * 24 * 14).Unix() - token, err := a.newToken(userFromDb, tokenLife) + uu := User{ + Email: userFromDb.Email, + Username: userFromDb.Username, + } + + token, err := a.newToken(uu, tokenLife) if err != nil { ctx.Set(types.HttpEndpointErrorKey, err.Error()) return ctx.JSON(http.StatusInternalServerError, echo.Map{ diff --git a/auth/validate_user.go b/auth/validate_user.go index 9c3488c4..e28878b4 100644 --- a/auth/validate_user.go +++ b/auth/validate_user.go @@ -1,7 +1,7 @@ package auth import ( - "encoding/json" + "context" "fmt" "time" @@ -13,17 +13,22 @@ func (a *auth) validateUser(username, password string) (map[string]interface{}, return nil, fmt.Errorf("Email/Password cannot be empty") } - key := fmt.Sprintf("%s/%s", UserNameSpace, username) - bz, err := a.store.Get([]byte(key)) + //key := fmt.Sprintf("%s/%s", UserNameSpace, username) + //bz, err := a.store.Get([]byte(key)) + //if err != nil { + // return nil, err + //} + // + //var userFromDb User + //if err := json.Unmarshal(bz, &userFromDb); err != nil { + // return nil, err + //} + + userFromDb, err := a.pgStore.GetUser(context.Background(), username) if err != nil { return nil, err } - var userFromDb User - if err := json.Unmarshal(bz, &userFromDb); err != nil { - return nil, err - } - if !a.verifyPassword(userFromDb.Password, password) { return nil, fmt.Errorf("invalid password") } diff --git a/config/config.go b/config/config.go index 42ea39a7..49645e9f 100644 --- a/config/config.go +++ b/config/config.go @@ -60,7 +60,7 @@ func NewStoreConfig() (*StoreConfig, error) { return &StoreConfig{ User: "postgres", Password: "Qwerty@123", - Database: "postgres", + Database: "open_registry", Host: "0.0.0.0", Port: 5432, }, nil diff --git a/registry/v2/blobs.go b/registry/v2/blobs.go index 3ec78e0f..a061c331 100644 --- a/registry/v2/blobs.go +++ b/registry/v2/blobs.go @@ -60,7 +60,12 @@ func (b *blobs) HEAD(ctx echo.Context) error { ctx.Response().Header().Set("Docker-Content-Digest", digest) return ctx.String(http.StatusOK, "OK") } - +/* +UploadBlob +for postgres +insert into blob table one blob at a time +these will be part of the txn in StartUpload + */ func (b *blobs) UploadBlob(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { @@ -88,6 +93,15 @@ func (b *blobs) UploadBlob(ctx echo.Context) error { b.uploads[uuid] = bz + if err := b.blobTransaction(ctx, bz, uuid); err!= nil { + errMsg := b.errorResponse( + RegistryErrorCodeBlobUploadInvalid, + err.Error(), + nil, + ) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } locationHeader := fmt.Sprintf("/v2/%s/blobs/uploads/%s", namespace, uuid) ctx.Response().Header().Set("Location", locationHeader) ctx.Response().Header().Set("Range", fmt.Sprintf("0-%d", len(bz)-1)) @@ -126,8 +140,37 @@ func (b *blobs) UploadBlob(ctx echo.Context) error { ctx.Request().Body.Close() b.uploads[uuid] = buf.Bytes() + if err := b.blobTransaction(ctx, buf.Bytes(), uuid); err!= nil { + errMsg := b.errorResponse( + RegistryErrorCodeBlobUploadInvalid, + err.Error(), + nil, + ) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } locationHeader := fmt.Sprintf("/v2/%s/blobs/uploads/%s", namespace, uuid) ctx.Response().Header().Set("Location", locationHeader) ctx.Response().Header().Set("Range", fmt.Sprintf("0-%d", buf.Len()-1)) return ctx.NoContent(http.StatusAccepted) } + +func (b *blobs) blobTransaction(ctx echo.Context, bz []byte, uuid string) error { + blob := &types.Blob{ + Digest: digest(bz), + Skylink: "", + UUID: uuid, + RangeStart: 0, + RangeEnd: uint32(len(bz)-1), + } + + txnOp, ok := b.registry.txnMap[uuid] + if !ok { + return fmt.Errorf("txn has not been initialised for uuid - " + uuid) + } + + if err := b.registry.store.SetBlob(ctx.Request().Context(), txnOp.txn, blob); err != nil { + return b.registry.store.Abort(ctx.Request().Context(),txnOp.txn) + } + return nil +} diff --git a/registry/v2/registry.go b/registry/v2/registry.go index 2eabb60e..8649c537 100644 --- a/registry/v2/registry.go +++ b/registry/v2/registry.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/fatih/color" "io" "net/http" "path" @@ -225,7 +226,14 @@ func (r *registry) MonolithicUpload(ctx echo.Context) error { return ctx.NoContent(http.StatusCreated) } -// PUT /v2//blobs/uploads/?digest= +// CompleteUpload +//PUT /v2//blobs/uploads/?digest= +/* +for postgres: +this is where we insert into the layer after all the blobs have been accumulated +and inserted in the blob table +thus committing the txn +*/ func (r *registry) CompleteUpload(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { @@ -243,7 +251,7 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { return ctx.JSONBlob(http.StatusBadRequest, errMsg) } _ = ctx.Request().Body.Close() - + // insert if bz is not nil buf := bytes.NewBuffer(r.b.uploads[uuid]) buf.Write(bz) ourHash := digest(buf.Bytes()) @@ -270,24 +278,51 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusInternalServerError, errMsg) } + txnOp , ok := r.txnMap[uuid] + layer := &types.LayerV2{ + MediaType: "", + Digest: dig, + SkynetLink: skylink, + UUID: uuid, + BlobDigests: txnOp.blobDigests, + Size: len(bz), + } + if !ok { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, "transaction does not exist for uuid -"+ uuid, nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + if err := r.store.SetLayer(ctx.Request().Context(), txnOp.txn, layer); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } - val := types.Metadata{ - Namespace: namespace, - Manifest: types.ImageManifest{ - SchemaVersion: 2, - MediaType: "", - Layers: []*types.Layer{ - { - MediaType: "", Size: len(bz), Digest: dig, SkynetLink: skylink, UUID: uuid, - }, - }, - }, + + + val := &types.ImageManifestV2{ + Uuid: uuid, + Namespace: namespace, + MediaType: "", + SchemaVersion: 2, } - if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) + //if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { + // errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) + // ctx.Set(types.HttpEndpointErrorKey, errMsg) + // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) + //} + + if err := r.store.SetManifest(ctx.Request().Context(), txnOp.txn, val);err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusInternalServerError, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + if err := r.store.Commit(ctx.Request().Context(),txnOp.txn); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) } locationHeader := fmt.Sprintf("/v2/%s/blobs/%s", namespace, ourHash) @@ -388,6 +423,7 @@ func (r *registry) ManifestExists(ctx echo.Context) error { return ctx.NoContent(http.StatusOK) } +// ChunkedUpload // PATCH /v2//blobs/uploads/ func (r *registry) ChunkedUpload(ctx echo.Context) error { return r.b.UploadBlob(ctx) @@ -487,6 +523,8 @@ func (r *registry) PushManifest(ctx echo.Context) error { return ctx.JSONBlob(http.StatusBadRequest, errMsg) } + color.Red("manifest list: %v\n", manifest) + mfNamespace := fmt.Sprintf("%s/manifests", namespace) skylink, err := r.skynet.Upload(mfNamespace, dig, bz, true) if err != nil { @@ -496,6 +534,18 @@ func (r *registry) PushManifest(ctx echo.Context) error { return ctx.JSONBlob(http.StatusNotFound, errMsg) } + //id := uuid.Generate() + //mfc := &types.ConfigV2{ + // UUID: id.String(), + // Namespace: namespace, + // Reference: ref, + // Digest: dig, + // Skylink: skylink, + // MediaType: contentType, + // Layers: nil, + // Size: 0, + //} + manifestConfig := &types.Config{ MediaType: contentType, Size: len(bz), @@ -638,6 +688,11 @@ func (r *registry) PushImage(ctx echo.Context) error { return nil } +/*StartUpload +for postgres: +start a tnx +registry.tnxMap[uuid] = {txn,blobs[],timeout} +*/ func (r *registry) StartUpload(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { @@ -717,7 +772,21 @@ func (r *registry) StartUpload(ctx echo.Context) error { id := uuid.Generate() locationHeader := fmt.Sprintf("/v2/%s/blobs/uploads/%s", namespace, id.String()) - + txn, err := r.store.NewTxn(ctx.Request().Context()) + if err != nil { + errMsg := r.errorResponse( + RegistryErrorCodeUnknown, + "error creating new database transaction", + nil, + ) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusInternalServerError, errMsg) + } + r.txnMap[id.String()] = TxnStore{ + txn: txn, + blobDigests: []string{}, + timeout: time.Minute * 30, + } ctx.Response().Header().Set("Location", locationHeader) ctx.Response().Header().Set("Content-Length", "0") ctx.Response().Header().Set("Docker-Upload-UUID", id.String()) diff --git a/registry/v2/types.go b/registry/v2/types.go index 0ce33d76..a093a35e 100644 --- a/registry/v2/types.go +++ b/registry/v2/types.go @@ -1,7 +1,11 @@ package registry import ( + "github.com/containerish/OpenRegistry/store/postgres" + "github.com/jackc/pgx/v4" + "github.com/rs/zerolog" "sync" + "time" "github.com/containerish/OpenRegistry/telemetry" @@ -52,9 +56,9 @@ type RegistryErrors struct { } type RegistryError struct { - Detail map[string]interface{} `json:"detail,omitempty"` Code string `json:"code"` Message string `json:"message"` + Detail map[string]interface{} `json:"detail,omitempty"` } // OCI - Distribution Spec compliant Headers @@ -85,12 +89,21 @@ const ( type ( registry struct { + log zerolog.Logger b blobs logger telemetry.Logger localCache cache.Store skynet *skynet.Client mu *sync.RWMutex debug bool + txnMap map[string]TxnStore + store postgres.PersistentStore + } + + TxnStore struct { + txn pgx.Tx + blobDigests []string + timeout time.Duration } blobs struct { diff --git a/store/postgres/container_image.go b/store/postgres/container_image.go index bf560bea..81322341 100644 --- a/store/postgres/container_image.go +++ b/store/postgres/container_image.go @@ -1 +1,116 @@ package postgres + +import ( + "context" + "fmt" + "github.com/containerish/OpenRegistry/store/postgres/queries" + "github.com/containerish/OpenRegistry/types" + "github.com/jackc/pgx/v4" + "time" +) + +func (p *pg) GetLayer(ctx context.Context, txn pgx.Tx, digest string) (*types.Layer, error) { + childCtx, cancel := context.WithTimeout(ctx,time.Millisecond*200) + defer cancel() + + row := txn.QueryRow(childCtx,queries.GetLayer,digest) + var layer types.Layer + if err := row.Scan( + &layer.MediaType, + &layer.Digest, + &layer.SkynetLink, + &layer.UUID, + &layer.Blobs, + &layer.Size, + ); err!= nil { + return nil, err + } + + return &layer, nil + +} + +func (p *pg) SetLayer(ctx context.Context, txn pgx.Tx, l *types.LayerV2) error { + childCtx, cancel := context.WithTimeout(ctx,time.Millisecond*200) + defer cancel() + + _, err := txn.Exec(childCtx, queries.SetLayer, l.MediaType, l.Digest, l.SkynetLink, l.UUID, l.BlobDigests, l.Size) + return err +} + +func (p *pg) GetManifest(ctx context.Context, txn pgx.Tx, ref string) (*types.ImageManifestV2, error) { + childCtx, cancel := context.WithTimeout(ctx,time.Millisecond*200) + defer cancel() + + row := txn.QueryRow(childCtx, queries.GetManifest, ref) + var im *types.ImageManifestV2 + if err := row.Scan( + &im.Uuid, + &im.Namespace, + &im.MediaType, + &im.SchemaVersion, + ); err != nil { + return nil, err + } + return im, nil +} + +func (p *pg) SetManifest(ctx context.Context, txn pgx.Tx, im *types.ImageManifestV2) error { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*200) + defer cancel() + + _, err := txn.Exec(childCtx, queries.SetImageManifest, im.Uuid, im.Namespace, im.MediaType, im.SchemaVersion) + return err +} + +func (p *pg) SetBlob(ctx context.Context, txn pgx.Tx, b *types.Blob) error { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond* 200) + defer cancel() + + _, err := txn.Exec(childCtx,queries.SetBlob,b.UUID, b.Digest, b.Skylink, b.RangeStart, b.RangeEnd) + return err + +} + +func (p *pg) GetBlob(ctx context.Context, txn pgx.Tx, digest string) (*types.Blob, error) { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond* 200) + defer cancel() + + row := txn.QueryRow(childCtx, queries.GetBlob, digest) + if row == nil { + return nil, fmt.Errorf("error blob not found") + } + var blob *types.Blob + if err := row.Scan( + &blob.UUID, + &blob.Digest, + &blob.Skylink, + &blob.RangeStart, + &blob.RangeEnd, + ); err != nil { + return nil, err + } + + return blob, nil +} + +func (p *pg) NewTxn(ctx context.Context) (pgx.Tx,error) { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*200) + defer cancel() + + return p.conn.Begin(childCtx) +} + +func (p *pg) Abort(ctx context.Context, txn pgx.Tx) error { + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*200) + defer cancel() + + return txn.Rollback(childCtx) +} + +func (p *pg) Commit(ctx context.Context, txn pgx.Tx) error { + childCtx, cancel := context.WithTimeout(ctx, time.Second * 30) + defer cancel() + + return txn.Commit(childCtx) +} diff --git a/store/postgres/postgres.go b/store/postgres/postgres.go index 03ca8bec..202a0df8 100644 --- a/store/postgres/postgres.go +++ b/store/postgres/postgres.go @@ -13,6 +13,7 @@ import ( type PersistentStore interface { UserStore + RegistryStore Get(key []byte) ([]byte, error) Set(key, value []byte) error GetDigest(digest string) (*types.LayerRef, error) @@ -41,6 +42,18 @@ type UserStore interface { IsActive(ctx context.Context, identifier string) bool } +type RegistryStore interface { + NewTxn(ctx context.Context) (pgx.Tx, error) + Abort(ctx context.Context, txn pgx.Tx) error + Commit(ctx context.Context, txn pgx.Tx) error + GetLayer(ctx context.Context, txn pgx.Tx, digest string) (*types.Layer, error) + SetLayer(ctx context.Context, txn pgx.Tx, l *types.LayerV2) error + GetManifest(ctx context.Context, txn pgx.Tx, ref string) (*types.ImageManifestV2, error) + SetManifest(ctx context.Context, txn pgx.Tx, im *types.ImageManifestV2) error + SetBlob(ctx context.Context, txn pgx.Tx, b *types.Blob) error + GetBlob(ctx context.Context, txn pgx.Tx, digest string) (*types.Blob, error) +} + type pg struct { conn *pgx.Conn } diff --git a/store/postgres/queries/registry.go b/store/postgres/queries/registry.go index afa819ac..ce208be7 100644 --- a/store/postgres/queries/registry.go +++ b/store/postgres/queries/registry.go @@ -2,20 +2,24 @@ package queries // insert queries var ( - SetMetadata = `insert into metadata (uuid, namespace, manifest) values($1, $2, $3);` - SetImageManifest = `insert into image_manifest (uuid, media_type, layers, config, - schema_version) values ($1, $2, $3, $4, $5);` + SetDigest = `insert into ` + SetImageManifest = `insert into image_manifest (uuid, namespace, media_type, schema_version) values ($1, $2, $3, $4);` SetLayer = `insert into layer (media_type, digest, sky_link, uuid, blobs, size) values ($1, $2, $3, $4, $5, $6);` + SetBlob = `insert into blob (uuid, digest, sky_link, start_range, end_range) values ($1, $2, $3, $4, $5);` + SetConfig = `insert into config (media_type, digest, sky_link, reference, size) values ($1, $2, $3, $4, $5);` ) // select queries var ( - GetAllEmails = `select email from users;` + GetDigest = `select digest from layers where digest=$1;` ReadMetadata = `select * from metadata where namespace=$1;` + GetLayer =`select * from layer where digest=$1;` + GetManifest =`select * from image_manifest where namespace=$1;` + GetBlob = `select * from blob where digest=$1;` ) diff --git a/store/postgres/queries/users.go b/store/postgres/queries/users.go index 050973ee..16277ec5 100644 --- a/store/postgres/queries/users.go +++ b/store/postgres/queries/users.go @@ -3,9 +3,9 @@ package queries var ( AddUser = `insert into users (id, is_active, username, email, password, created_at, updated_at) values ($1, $2, $3, $4, $5, $6, $7);` - GetUser = `select username, is_active, email, created_at, updated_at where email = $1` + GetUser = `select username, is_active, email, created_at, updated_at where email=$1 or username=$1;` UpdateUser = `update user set username = $1, email = $2, password = $3, updated_at = $4 where username = $5;` DeleteUser = `delete from user where username = $1;` - + GetAllEmails = `select email from users;` ) diff --git a/types/types.go b/types/types.go index e36ad394..1a74c188 100644 --- a/types/types.go +++ b/types/types.go @@ -11,11 +11,11 @@ type ( Manifest ImageManifest } - Manifest struct { - SkynetLink string - Reference string - Digest string - } + //Manifest struct { + // SkynetLink string + // Reference string + // Digest string + //} ImageManifest struct { MediaType string `json:"mediaType"` @@ -24,6 +24,13 @@ type ( SchemaVersion int `json:"schemaVersion"` } + ImageManifestV2 struct { + Uuid string `json:"uuid"` + Namespace string `json:"namespace"` + MediaType string `json:"mediaType"` + SchemaVersion int `json:"schemaVersion"` + } + Blob struct { Digest string Skylink string @@ -41,6 +48,15 @@ type ( Size int `json:"size"` } + LayerV2 struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + SkynetLink string `json:"skynetLink"` + UUID string `json:"uuid"` + BlobDigests []string `json:"blobs"` + Size int `json:"size"` + } + LayerRef struct { Digest string Skylink string @@ -53,6 +69,17 @@ type ( Reference string `json:"reference"` Size int `json:"size"` } + + ConfigV2 struct { + UUID string `json:"uuid"` + Namespace string `json:"namespace"` + Reference string `json:"reference"` + Digest string `json:"digest"` + Skylink string `json:"sky_link"` + MediaType string `json:"media_type"` + Layers []string `json:"layers"` + Size int `json:"size"` + } ) func (md Metadata) GetManifestByRef(ref string) (*Config, error) { From bac008311e6478db45a7d006509b50aff44da1cc Mon Sep 17 00:00:00 2001 From: jay-dee7 Date: Sat, 4 Dec 2021 23:39:11 +0530 Subject: [PATCH 05/16] Fix: Add Support for PersistentStore Backend Signed-off-by: jay-dee7 --- config/config.go | 2 +- go.mod | 1 + go.sum | 1 + main.go | 5 +-- registry/v2/blobs.go | 21 +++++++---- registry/v2/registry.go | 59 +++++++++++++++++++++--------- store/postgres/container_image.go | 39 ++++++++++---------- store/postgres/postgres.go | 11 ++++-- store/postgres/queries/registry.go | 16 ++++---- store/postgres/queries/users.go | 7 ++-- store/postgres/users.go | 12 +++--- 11 files changed, 104 insertions(+), 70 deletions(-) diff --git a/config/config.go b/config/config.go index 49645e9f..972250ba 100644 --- a/config/config.go +++ b/config/config.go @@ -67,7 +67,7 @@ func NewStoreConfig() (*StoreConfig, error) { } func (sc *StoreConfig) Endpoint() string { - return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", sc.User, sc.Password, sc.Host, sc.Port, sc.Database) + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?pool_max_conns=1000", sc.User, sc.Password, sc.Host, sc.Port, sc.Database) } func LoadFromENV() (*RegistryConfig, error) { diff --git a/go.mod b/go.mod index f47034b1..a1eb0ad4 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/jackc/pgproto3/v2 v2.1.1 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/pgtype v1.8.1 // indirect + github.com/jackc/puddle v1.1.3 // indirect github.com/leodido/go-urn v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index fa6c53c9..cb21f951 100644 --- a/go.sum +++ b/go.sum @@ -336,6 +336,7 @@ github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/main.go b/main.go index 9fe1473a..a97a7bf9 100644 --- a/main.go +++ b/main.go @@ -39,7 +39,6 @@ func main() { return } - log := telemetry.SetupLogger() fluentBitCollector, err := fluentbit.New(cfg) if err != nil { color.Red("error initializing fluentbit collector: %s\n", err) @@ -50,12 +49,12 @@ func main() { authSvc := auth.New(localCache, cfg, pgStore, logger) skynetClient := skynet.NewClient(cfg) - reg, err := registry.NewRegistry(skynetClient, localCache, logger) + reg, err := registry.NewRegistry(skynetClient, localCache, logger, pgStore) if err != nil { e.Logger.Errorf("error creating new container registry: %s", err) return } - router.Register(cfg, e, reg, authSvc, localCache) + router.Register(cfg, e, reg, authSvc, localCache, pgStore) logger.Errorf("error initialising OpenRegistry Server: %s", e.Start(cfg.Address())) } diff --git a/registry/v2/blobs.go b/registry/v2/blobs.go index a061c331..8e6773dd 100644 --- a/registry/v2/blobs.go +++ b/registry/v2/blobs.go @@ -60,12 +60,13 @@ func (b *blobs) HEAD(ctx echo.Context) error { ctx.Response().Header().Set("Docker-Content-Digest", digest) return ctx.String(http.StatusOK, "OK") } + /* UploadBlob for postgres insert into blob table one blob at a time these will be part of the txn in StartUpload - */ +*/ func (b *blobs) UploadBlob(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { @@ -93,7 +94,7 @@ func (b *blobs) UploadBlob(ctx echo.Context) error { b.uploads[uuid] = bz - if err := b.blobTransaction(ctx, bz, uuid); err!= nil { + if err := b.blobTransaction(ctx, bz, uuid); err != nil { errMsg := b.errorResponse( RegistryErrorCodeBlobUploadInvalid, err.Error(), @@ -102,6 +103,7 @@ func (b *blobs) UploadBlob(ctx echo.Context) error { ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusBadRequest, errMsg) } + locationHeader := fmt.Sprintf("/v2/%s/blobs/uploads/%s", namespace, uuid) ctx.Response().Header().Set("Location", locationHeader) ctx.Response().Header().Set("Range", fmt.Sprintf("0-%d", len(bz)-1)) @@ -140,7 +142,7 @@ func (b *blobs) UploadBlob(ctx echo.Context) error { ctx.Request().Body.Close() b.uploads[uuid] = buf.Bytes() - if err := b.blobTransaction(ctx, buf.Bytes(), uuid); err!= nil { + if err := b.blobTransaction(ctx, buf.Bytes(), uuid); err != nil { errMsg := b.errorResponse( RegistryErrorCodeBlobUploadInvalid, err.Error(), @@ -161,7 +163,7 @@ func (b *blobs) blobTransaction(ctx echo.Context, bz []byte, uuid string) error Skylink: "", UUID: uuid, RangeStart: 0, - RangeEnd: uint32(len(bz)-1), + RangeEnd: uint32(len(bz) - 1), } txnOp, ok := b.registry.txnMap[uuid] @@ -169,8 +171,11 @@ func (b *blobs) blobTransaction(ctx echo.Context, bz []byte, uuid string) error return fmt.Errorf("txn has not been initialised for uuid - " + uuid) } - if err := b.registry.store.SetBlob(ctx.Request().Context(), txnOp.txn, blob); err != nil { - return b.registry.store.Abort(ctx.Request().Context(),txnOp.txn) - } - return nil + if err := b.registry.store.SetBlob(ctx.Request().Context(), txnOp.txn, blob); err != nil { + return b.registry.store.Abort(ctx.Request().Context(), txnOp.txn) + } + + txnOp.blobDigests = append(txnOp.blobDigests, blob.Digest) + b.registry.txnMap[uuid] = txnOp + return nil } diff --git a/registry/v2/registry.go b/registry/v2/registry.go index 8649c537..1a91dda4 100644 --- a/registry/v2/registry.go +++ b/registry/v2/registry.go @@ -2,6 +2,7 @@ package registry import ( "bytes" + "context" "encoding/json" "fmt" "github.com/fatih/color" @@ -16,6 +17,7 @@ import ( "github.com/containerish/OpenRegistry/cache" "github.com/containerish/OpenRegistry/skynet" + "github.com/containerish/OpenRegistry/store/postgres" "github.com/containerish/OpenRegistry/telemetry" "github.com/containerish/OpenRegistry/types" "github.com/docker/distribution/uuid" @@ -26,6 +28,7 @@ func NewRegistry( skynetClient *skynet.Client, c cache.Store, logger telemetry.Logger, + pgStore postgres.PersistentStore, ) (Registry, error) { r := ®istry{ debug: true, @@ -39,6 +42,8 @@ func NewRegistry( localCache: c, logger: logger, mu: &sync.RWMutex{}, + store: pgStore, + txnMap: map[string]TxnStore{}, } r.b.registry = r @@ -278,7 +283,7 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusInternalServerError, errMsg) } - txnOp , ok := r.txnMap[uuid] + txnOp, ok := r.txnMap[uuid] layer := &types.LayerV2{ MediaType: "", Digest: dig, @@ -288,7 +293,7 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { Size: len(bz), } if !ok { - errMsg := r.errorResponse(RegistryErrorCodeUnknown, "transaction does not exist for uuid -"+ uuid, nil) + errMsg := r.errorResponse(RegistryErrorCodeUnknown, "transaction does not exist for uuid -"+uuid, nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusBadRequest, errMsg) } @@ -298,14 +303,12 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { return ctx.JSONBlob(http.StatusBadRequest, errMsg) } - - - val := &types.ImageManifestV2{ - Uuid: uuid, - Namespace: namespace, - MediaType: "", - SchemaVersion: 2, - } + // val := &types.ImageManifestV2{ + // Uuid: uuid, + // Namespace: namespace, + // MediaType: "", + // SchemaVersion: 2, + // } //if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { // errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) @@ -313,17 +316,18 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) //} - if err := r.store.SetManifest(ctx.Request().Context(), txnOp.txn, val);err != nil { - errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } + // if err := r.store.SetManifest(ctx.Request().Context(), txnOp.txn, val); err != nil { + // errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) + // ctx.Set(types.HttpEndpointErrorKey, errMsg) + // return ctx.JSONBlob(http.StatusBadRequest, errMsg) + // } - if err := r.store.Commit(ctx.Request().Context(),txnOp.txn); err != nil { + if err := r.store.Commit(ctx.Request().Context(), txnOp.txn); err != nil { errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusBadRequest, errMsg) } + delete(r.txnMap, uuid) locationHeader := fmt.Sprintf("/v2/%s/blobs/%s", namespace, ourHash) ctx.Response().Header().Set("Content-Length", "0") @@ -523,7 +527,7 @@ func (r *registry) PushManifest(ctx echo.Context) error { return ctx.JSONBlob(http.StatusBadRequest, errMsg) } - color.Red("manifest list: %v\n", manifest) + color.Red("manifest list: %s\n", bz) mfNamespace := fmt.Sprintf("%s/manifests", namespace) skylink, err := r.skynet.Upload(mfNamespace, dig, bz, true) @@ -569,6 +573,25 @@ func (r *registry) PushManifest(ctx echo.Context) error { return ctx.JSONBlob(http.StatusBadRequest, errMsg) } + val := &types.ImageManifestV2{ + Uuid: uuid.Generate().String(), + Namespace: namespace, + MediaType: "", + SchemaVersion: 2, + } + + //if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { + // errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) + // ctx.Set(types.HttpEndpointErrorKey, errMsg) + // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) + //} + txnOp, _ := r.store.NewTxn(context.Background()) + + if err := r.store.SetManifest(ctx.Request().Context(), txnOp, val); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } locationHeader := r.getHttpUrlFromSkylink(skylink) ctx.Response().Header().Set("Location", locationHeader) ctx.Response().Header().Set("Docker-Content-Digest", dig) @@ -776,7 +799,7 @@ func (r *registry) StartUpload(ctx echo.Context) error { if err != nil { errMsg := r.errorResponse( RegistryErrorCodeUnknown, - "error creating new database transaction", + err.Error(), nil, ) ctx.Set(types.HttpEndpointErrorKey, errMsg) diff --git a/store/postgres/container_image.go b/store/postgres/container_image.go index 81322341..6011e59e 100644 --- a/store/postgres/container_image.go +++ b/store/postgres/container_image.go @@ -3,17 +3,18 @@ package postgres import ( "context" "fmt" + "time" + "github.com/containerish/OpenRegistry/store/postgres/queries" "github.com/containerish/OpenRegistry/types" "github.com/jackc/pgx/v4" - "time" ) -func (p *pg) GetLayer(ctx context.Context, txn pgx.Tx, digest string) (*types.Layer, error) { - childCtx, cancel := context.WithTimeout(ctx,time.Millisecond*200) +func (p *pg) GetLayer(ctx context.Context, txn pgx.Tx, digest string) (*types.Layer, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - row := txn.QueryRow(childCtx,queries.GetLayer,digest) + row := txn.QueryRow(childCtx, queries.GetLayer, digest) var layer types.Layer if err := row.Scan( &layer.MediaType, @@ -22,7 +23,7 @@ func (p *pg) GetLayer(ctx context.Context, txn pgx.Tx, digest string) (*types.L &layer.UUID, &layer.Blobs, &layer.Size, - ); err!= nil { + ); err != nil { return nil, err } @@ -31,7 +32,7 @@ func (p *pg) GetLayer(ctx context.Context, txn pgx.Tx, digest string) (*types.L } func (p *pg) SetLayer(ctx context.Context, txn pgx.Tx, l *types.LayerV2) error { - childCtx, cancel := context.WithTimeout(ctx,time.Millisecond*200) + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() _, err := txn.Exec(childCtx, queries.SetLayer, l.MediaType, l.Digest, l.SkynetLink, l.UUID, l.BlobDigests, l.Size) @@ -39,7 +40,7 @@ func (p *pg) SetLayer(ctx context.Context, txn pgx.Tx, l *types.LayerV2) error { } func (p *pg) GetManifest(ctx context.Context, txn pgx.Tx, ref string) (*types.ImageManifestV2, error) { - childCtx, cancel := context.WithTimeout(ctx,time.Millisecond*200) + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() row := txn.QueryRow(childCtx, queries.GetManifest, ref) @@ -49,14 +50,14 @@ func (p *pg) GetManifest(ctx context.Context, txn pgx.Tx, ref string) (*types.Im &im.Namespace, &im.MediaType, &im.SchemaVersion, - ); err != nil { + ); err != nil { return nil, err } return im, nil } func (p *pg) SetManifest(ctx context.Context, txn pgx.Tx, im *types.ImageManifestV2) error { - childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*200) + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() _, err := txn.Exec(childCtx, queries.SetImageManifest, im.Uuid, im.Namespace, im.MediaType, im.SchemaVersion) @@ -64,52 +65,52 @@ func (p *pg) SetManifest(ctx context.Context, txn pgx.Tx, im *types.ImageManifes } func (p *pg) SetBlob(ctx context.Context, txn pgx.Tx, b *types.Blob) error { - childCtx, cancel := context.WithTimeout(ctx, time.Millisecond* 200) + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - _, err := txn.Exec(childCtx,queries.SetBlob,b.UUID, b.Digest, b.Skylink, b.RangeStart, b.RangeEnd) + _, err := txn.Exec(childCtx, queries.SetBlob, b.UUID, b.Digest, b.Skylink, b.RangeStart, b.RangeEnd) return err } func (p *pg) GetBlob(ctx context.Context, txn pgx.Tx, digest string) (*types.Blob, error) { - childCtx, cancel := context.WithTimeout(ctx, time.Millisecond* 200) + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() row := txn.QueryRow(childCtx, queries.GetBlob, digest) if row == nil { return nil, fmt.Errorf("error blob not found") } - var blob *types.Blob + var blob types.Blob if err := row.Scan( &blob.UUID, &blob.Digest, &blob.Skylink, &blob.RangeStart, &blob.RangeEnd, - ); err != nil { + ); err != nil { return nil, err } - return blob, nil + return &blob, nil } -func (p *pg) NewTxn(ctx context.Context) (pgx.Tx,error) { - childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*200) +func (p *pg) NewTxn(ctx context.Context) (pgx.Tx, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() return p.conn.Begin(childCtx) } func (p *pg) Abort(ctx context.Context, txn pgx.Tx) error { - childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*200) + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() return txn.Rollback(childCtx) } func (p *pg) Commit(ctx context.Context, txn pgx.Tx) error { - childCtx, cancel := context.WithTimeout(ctx, time.Second * 30) + childCtx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() return txn.Commit(childCtx) diff --git a/store/postgres/postgres.go b/store/postgres/postgres.go index 202a0df8..c085577e 100644 --- a/store/postgres/postgres.go +++ b/store/postgres/postgres.go @@ -8,6 +8,7 @@ import ( "github.com/containerish/OpenRegistry/types" "github.com/fatih/color" "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" "github.com/labstack/echo/v4" ) @@ -55,20 +56,24 @@ type RegistryStore interface { } type pg struct { - conn *pgx.Conn + conn *pgxpool.Pool } func New(cfg *config.StoreConfig) (PersistentStore, error) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - conn, err := pgx.Connect(ctx, cfg.Endpoint()) + pgxCofig, err := pgxpool.ParseConfig(cfg.Endpoint()) if err != nil { return nil, err } - color.Green("connection to database successful") + conn, err := pgxpool.ConnectConfig(ctx, pgxCofig) + if err != nil { + return nil, err + } + color.Green("connection to database successful") return &pg{conn: conn}, nil } diff --git a/store/postgres/queries/registry.go b/store/postgres/queries/registry.go index ce208be7..b3d01aeb 100644 --- a/store/postgres/queries/registry.go +++ b/store/postgres/queries/registry.go @@ -2,9 +2,10 @@ package queries // insert queries var ( - SetDigest = `insert into ` - SetImageManifest = `insert into image_manifest (uuid, namespace, media_type, schema_version) values ($1, $2, $3, $4);` - SetLayer = `insert into layer (media_type, digest, sky_link, uuid, blobs, size) + SetDigest = `insert into ` + SetImageManifest = `insert into image_manifest (uuid, namespace, media_type, schema_version) values ($1, $2, $3, $4) + on conflict (namespace) do update set schema_version=3` + SetLayer = `insert into layer (media_type, digest, sky_link, uuid, blob_ids, size) values ($1, $2, $3, $4, $5, $6);` SetBlob = `insert into blob (uuid, digest, sky_link, start_range, end_range) @@ -16,10 +17,9 @@ var ( // select queries var ( - GetDigest = `select digest from layers where digest=$1;` + GetDigest = `select digest from layers where digest=$1;` ReadMetadata = `select * from metadata where namespace=$1;` - GetLayer =`select * from layer where digest=$1;` - GetManifest =`select * from image_manifest where namespace=$1;` - GetBlob = `select * from blob where digest=$1;` + GetLayer = `select * from layer where digest=$1;` + GetManifest = `select * from image_manifest where namespace=$1;` + GetBlob = `select * from blob where digest=$1;` ) - diff --git a/store/postgres/queries/users.go b/store/postgres/queries/users.go index 16277ec5..5e5b5598 100644 --- a/store/postgres/queries/users.go +++ b/store/postgres/queries/users.go @@ -3,9 +3,8 @@ package queries var ( AddUser = `insert into users (id, is_active, username, email, password, created_at, updated_at) values ($1, $2, $3, $4, $5, $6, $7);` - GetUser = `select username, is_active, email, created_at, updated_at where email=$1 or username=$1;` - UpdateUser = `update user set username = $1, email = $2, password = $3, updated_at = $4 where username = $5;` - DeleteUser = `delete from user where username = $1;` + GetUser = `select is_active, username, email, password, created_at, updated_at from users where email=$1 or username=$1;` + UpdateUser = `update user set username = $1, email = $2, password = $3, updated_at = $4 where username = $5;` + DeleteUser = `delete from user where username = $1;` GetAllEmails = `select email from users;` ) - diff --git a/store/postgres/users.go b/store/postgres/users.go index a9f03580..5c8e5bb5 100644 --- a/store/postgres/users.go +++ b/store/postgres/users.go @@ -35,7 +35,7 @@ func (p *pg) GetUser(ctx context.Context, identifier string) (*types.User, error row := p.conn.QueryRow(childCtx, queries.GetUser, identifier) var user types.User - err := row.Scan(&user.Username, &user.IsActive, &user.Email, &user.CreatedAt, &user.UpdatedAt) + err := row.Scan(&user.IsActive, &user.Username, &user.Email, &user.Password, &user.CreatedAt, &user.UpdatedAt) if err != nil { return nil, fmt.Errorf("user not found") } @@ -46,23 +46,23 @@ func (p *pg) GetUser(ctx context.Context, identifier string) (*types.User, error // UpdateUser //update user set username = $1, email = $2, password = $3, updated_at = $4 where username = $5 func (p *pg) UpdateUser(ctx context.Context, identifier string, u *types.User) error { - if err := u.Validate(); err!= nil { + if err := u.Validate(); err != nil { return err } - childCtx, cancel := context.WithTimeout(ctx, time.Millisecond* 100) + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*100) defer cancel() t := time.Now() _, err := p.conn.Exec(childCtx, queries.UpdateUser, u.Username, u.Email, u.Password, t, identifier) if err != nil { - return fmt.Errorf("error updating user: %s",identifier) + return fmt.Errorf("error updating user: %s", identifier) } return nil } // DeleteUser - delete from user where username = $1; func (p *pg) DeleteUser(ctx context.Context, identifier string) error { - childCtx, cancel := context.WithTimeout(ctx, time.Millisecond * 100) + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*100) defer cancel() _, err := p.conn.Exec(childCtx, queries.DeleteUser, identifier) @@ -75,7 +75,7 @@ func (p *pg) DeleteUser(ctx context.Context, identifier string) error { //IsActive - if the user has logged in, isActive returns true // this method is also useful for limiting access of malicious actors func (p *pg) IsActive(ctx context.Context, identifier string) bool { - childCtx, cancel := context.WithTimeout(ctx, time.Millisecond * 100) + childCtx, cancel := context.WithTimeout(ctx, time.Millisecond*100) defer cancel() row := p.conn.QueryRow(childCtx, queries.GetUser, identifier) return row != nil From 5eee398f289ff50af70dc7772819223ec436a15d Mon Sep 17 00:00:00 2001 From: guacamole Date: Mon, 6 Dec 2021 14:31:14 +0530 Subject: [PATCH 06/16] Feat: third commit in series "implementing postgres" - Added support for catalog api, content discovery apis Signed-off-by: guacamole --- registry/v2/blobs.go | 13 +- registry/v2/registry.go | 984 ++++++++++++++--------------- store/postgres/container_image.go | 210 +++++- store/postgres/postgres.go | 14 +- store/postgres/queries/registry.go | 32 +- 5 files changed, 690 insertions(+), 563 deletions(-) diff --git a/registry/v2/blobs.go b/registry/v2/blobs.go index 8e6773dd..5c2be8f5 100644 --- a/registry/v2/blobs.go +++ b/registry/v2/blobs.go @@ -38,10 +38,11 @@ func (b *blobs) HEAD(ctx echo.Context) error { }() digest := ctx.Param("digest") - layerRef, err := b.registry.localCache.GetDigest(digest) + + layerRef, err := b.registry.store.GetLayer(ctx.Request().Context(), digest) if err != nil { details := echo.Map{ - "skynet": "skynet link not found", + "skynet": "layer not found", } errMsg := b.errorResponse(RegistryErrorCodeManifestBlobUnknown, err.Error(), details) @@ -49,9 +50,12 @@ func (b *blobs) HEAD(ctx echo.Context) error { return ctx.JSONBlob(http.StatusNotFound, errMsg) } - size, ok := b.registry.skynet.Metadata(layerRef.Skylink) + size, ok := b.registry.skynet.Metadata(layerRef.SkynetLink) if !ok { - errMsg := b.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", nil) + details := echo.Map{ + "skynet": "skynet link not found", + } + errMsg := b.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", details) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) } @@ -172,6 +176,7 @@ func (b *blobs) blobTransaction(ctx echo.Context, bz []byte, uuid string) error } if err := b.registry.store.SetBlob(ctx.Request().Context(), txnOp.txn, blob); err != nil { + color.Red("aborting txn: %s\n", err.Error()) return b.registry.store.Abort(ctx.Request().Context(), txnOp.txn) } diff --git a/registry/v2/registry.go b/registry/v2/registry.go index 1a91dda4..b32737ad 100644 --- a/registry/v2/registry.go +++ b/registry/v2/registry.go @@ -5,11 +5,9 @@ import ( "context" "encoding/json" "fmt" - "github.com/fatih/color" "io" "net/http" "path" - "sort" "strconv" "strings" "sync" @@ -51,300 +49,19 @@ func NewRegistry( return r, nil } -// Catalog - The list of available repositories is made available through the catalog. -//GET /v2/_catalog -func (r *registry) Catalog(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - bz, err := r.localCache.ListAll() - if err != nil { - logMsg := echo.Map{ - "error": err.Error(), - } - - ctx.Set(types.HttpEndpointErrorKey, logMsg) - return ctx.JSON(http.StatusBadRequest, echo.Map{ - "error": err.Error(), - }) - } - - var md []types.Metadata - err = json.Unmarshal(bz, &md) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - } - - var result []string - for _, el := range md { - var repo []string - ns := el.Namespace - - for _, c := range el.Manifest.Config { - repo = append(repo, fmt.Sprintf("%s:%s", ns, c.Reference)) - } - - result = append(result, repo...) - } - - return ctx.JSON(http.StatusOK, echo.Map{ - "repositories": result, - }) -} - -func (r *registry) DeleteTagOrManifest(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - ref := ctx.Param("reference") - - if ref == "" { - reqURI := strings.Split(ctx.Request().RequestURI, "/") - if len(reqURI) == 6 { - ref = reqURI[5] - } - } - - if err := r.localCache.UpdateManifestRef(namespace, ref); err != nil { - details := map[string]interface{}{ - "namespace": namespace, - "digest": ref, - } - errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), details) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - return ctx.NoContent(http.StatusAccepted) -} - -func (r *registry) DeleteLayer(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - dig := ctx.Param("digest") - - var m types.Metadata - _, err := r.localCache.GetDigest(dig) - if err != nil { - bz, err := r.localCache.Get([]byte(namespace)) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - if err = json.Unmarshal(bz, &m); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - } - } - - err = r.localCache.DeleteLayer(namespace, dig) - if err != nil { - logMsg := echo.Map{ - "error": err.Error(), - "caller": "DeleteLayer", - } - - bz, err := json.Marshal(logMsg) - if err == nil { - ctx.Set(types.HttpEndpointErrorKey, logMsg) - } - return ctx.JSONBlob(http.StatusInternalServerError, bz) - } - - if err = r.localCache.DeleteDigest(dig); err != nil { - logMsg := echo.Map{ - "error": err.Error(), - "caller": "DeleteLayer", - } - - ctx.Set(types.HttpEndpointErrorKey, logMsg) - bz, err := json.Marshal(logMsg) - if err != nil { - ctx.Set(types.HttpEndpointErrorKey, err.Error()) - r.logger.Log(ctx) - } - return ctx.JSONBlob(http.StatusInternalServerError, bz) - } - return ctx.NoContent(http.StatusAccepted) -} - -// PUT /v2//blobs/uploads/?digest= -func (r *registry) MonolithicUpload(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - uuid := ctx.Param("uuid") - digest := ctx.QueryParam("digest") - - bz, err := io.ReadAll(ctx.Request().Body) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - ctx.Request().Body.Close() - - link, err := r.skynet.Upload(namespace, digest, bz, true) - if err != nil { - detail := echo.Map{ - "error": err.Error(), - "caller": "MonolithicUpload", - } - errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), detail) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusInternalServerError, bz) - } - - metadata := types.Metadata{ - Namespace: namespace, - Manifest: types.ImageManifest{ - SchemaVersion: 2, - MediaType: "", - Layers: []*types.Layer{{MediaType: "", Size: len(bz), Digest: digest, SkynetLink: link, UUID: uuid}}, - }, - } - - err = r.localCache.Update([]byte(namespace), metadata.Bytes()) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - - locationHeader := link - ctx.Response().Header().Set("Location", locationHeader) - return ctx.NoContent(http.StatusCreated) -} - -// CompleteUpload -//PUT /v2//blobs/uploads/?digest= -/* -for postgres: -this is where we insert into the layer after all the blobs have been accumulated -and inserted in the blob table -thus committing the txn -*/ -func (r *registry) CompleteUpload(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - dig := ctx.QueryParam("digest") - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - uuid := ctx.Param("uuid") - - bz, err := io.ReadAll(ctx.Request().Body) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeDigestInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - _ = ctx.Request().Body.Close() - // insert if bz is not nil - buf := bytes.NewBuffer(r.b.uploads[uuid]) - buf.Write(bz) - ourHash := digest(buf.Bytes()) - delete(r.b.uploads, uuid) - - if ourHash != dig { - details := map[string]interface{}{ - "headerDigest": dig, "serverSideDigest": ourHash, "bodyDigest": digest(bz), - } - errMsg := r.errorResponse(RegistryErrorCodeDigestInvalid, "digest mismatch", details) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - - blobNamespace := fmt.Sprintf("%s/blobs", namespace) - skylink, err := r.skynet.Upload(blobNamespace, dig, buf.Bytes(), true) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg) - } - if err := r.localCache.SetDigest(ourHash, skylink); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - } - txnOp, ok := r.txnMap[uuid] - layer := &types.LayerV2{ - MediaType: "", - Digest: dig, - SkynetLink: skylink, - UUID: uuid, - BlobDigests: txnOp.blobDigests, - Size: len(bz), - } - if !ok { - errMsg := r.errorResponse(RegistryErrorCodeUnknown, "transaction does not exist for uuid -"+uuid, nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - if err := r.store.SetLayer(ctx.Request().Context(), txnOp.txn, layer); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - - // val := &types.ImageManifestV2{ - // Uuid: uuid, - // Namespace: namespace, - // MediaType: "", - // SchemaVersion: 2, - // } - - //if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { - // errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) - // ctx.Set(types.HttpEndpointErrorKey, errMsg) - // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - //} - - // if err := r.store.SetManifest(ctx.Request().Context(), txnOp.txn, val); err != nil { - // errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) - // ctx.Set(types.HttpEndpointErrorKey, errMsg) - // return ctx.JSONBlob(http.StatusBadRequest, errMsg) - // } - - if err := r.store.Commit(ctx.Request().Context(), txnOp.txn); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - delete(r.txnMap, uuid) - - locationHeader := fmt.Sprintf("/v2/%s/blobs/%s", namespace, ourHash) - ctx.Response().Header().Set("Content-Length", "0") - ctx.Response().Header().Set("Docker-Content-Digest", ourHash) - ctx.Response().Header().Set("Location", locationHeader) - return ctx.NoContent(http.StatusCreated) -} - +// LayerExists // HEAD /v2//blobs/ // 200 OK // Content-Length: // Docker-Content-Digest: +// OK func (r *registry) LayerExists(ctx echo.Context) error { return r.b.HEAD(ctx) } +// ManifestExists // HEAD /v2//manifests/ +// OK func (r *registry) ManifestExists(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { @@ -354,10 +71,11 @@ func (r *registry) ManifestExists(ctx echo.Context) error { namespace := ctx.Param("username") + "/" + ctx.Param("imagename") ref := ctx.Param("reference") // ref can be either tag or digest - skylink, err := r.localCache.ResolveManifestRef(namespace, ref) + manifest, err := r.store.GetManifestByReference(ctx.Request().Context(), namespace, ref) + if err != nil { details := echo.Map{ - "skynet": "skynet link not found", + "skynet": "manifest not found", "error": err.Error(), } @@ -367,42 +85,14 @@ func (r *registry) ManifestExists(ctx echo.Context) error { return ctx.JSONBlob(http.StatusNotFound, errMsg) } - size, ok := r.skynet.Metadata(skylink) + size, ok := r.skynet.Metadata(manifest.Skylink) if !ok { - lm := logMsg{ + detail := map[string]interface{}{ "error": "metadata not found for skylink", - "skylink": skylink, + "skylink": manifest.Skylink, } - r.logger.Error(lm) - errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - - bz, err := r.localCache.Get([]byte(namespace)) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - - var md types.Metadata - if err = json.Unmarshal(bz, &md); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - lm := logMsg{ - "errorUnmarshal": fmt.Sprintf("%s\n", errMsg), - } - r.logger.Error(lm) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - - manifest, err := md.GetManifestByRef(ref) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), nil) + errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, "Manifest does not exist", detail) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) @@ -427,56 +117,95 @@ func (r *registry) ManifestExists(ctx echo.Context) error { return ctx.NoContent(http.StatusOK) } -// ChunkedUpload -// PATCH /v2//blobs/uploads/ -func (r *registry) ChunkedUpload(ctx echo.Context) error { - return r.b.UploadBlob(ctx) -} +// Catalog - The list of available repositories is made available through the catalog. +// GET /v2/_catalog +// OK +func (r *registry) Catalog(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + defer func() { + r.logger.Log(ctx).Send() + }() + + catalogList, err := r.store.GetCatalog(ctx.Request().Context()) + if err != nil { + ctx.Set(types.HttpEndpointErrorKey,err.Error()) + return ctx.JSON(http.StatusInternalServerError, echo.Map{ + "error": err.Error(), + }) + } + return ctx.JSON(http.StatusOK, echo.Map{ + "repositories": catalogList, + }) -func (r *registry) CancelUpload(ctx echo.Context) error { - return nil } -// PullManifest GET /v2//manifests/ -func (r *registry) PullManifest(ctx echo.Context) error { +// ListTags Content discovery +// GET /v2//tags/list +// OK +func (r *registry) ListTags(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { r.logger.Log(ctx).Send() }() namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - ref := ctx.Param("reference") + limit := ctx.QueryParam("n") - bz, err := r.localCache.Get([]byte(namespace)) + tags, err := r.store.GetImageTags(ctx.Request().Context(), namespace) if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), nil) + errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) - err = ctx.JSONBlob(http.StatusNotFound, errMsg) - return err - } - var md types.Metadata - if err = json.Unmarshal(bz, &md); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) } - skynetLink, err := r.localCache.ResolveManifestRef(namespace, ref) + if limit != "" { + n, err := strconv.ParseInt(limit, 10, 32) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + + return ctx.JSONBlob(http.StatusNotFound, errMsg) + } + if n > 0 { + tags = tags[0:n] + } + if n == 0 { + tags = nil + } + } + + return ctx.JSON(http.StatusOK, echo.Map{ + "name": namespace, + "tags": tags, + }) +} +func (r *registry) List(ctx echo.Context) error { + return fmt.Errorf("error") +} + +// PullManifest +// GET /v2//manifests/ +// OK +func (r *registry) PullManifest(ctx echo.Context) error { + namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + ref := ctx.Param("reference") + + manifest, err := r.store.GetManifestByReference(ctx.Request().Context(), namespace, ref) if err != nil { errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) } - resp, err := r.skynet.Download(skynetLink) + resp, err := r.skynet.Download(manifest.Skylink) if err != nil { errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) } - bz, err = io.ReadAll(resp) + bz, err := io.ReadAll(resp) if err != nil { errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) @@ -484,231 +213,134 @@ func (r *registry) PullManifest(ctx echo.Context) error { } _ = resp.Close() - manifest, err := md.GetManifestByRef(ref) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - ctx.Response().Header().Set("Docker-Content-Digest", manifest.Digest) - ctx.Response().Header().Set("X-Docker-Content-ID", skynetLink) + ctx.Response().Header().Set("X-Docker-Content-ID", manifest.Skylink) ctx.Response().Header().Set("Content-Type", manifest.MediaType) - ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", manifest.Size)) + ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", len(bz))) return ctx.JSONBlob(http.StatusOK, bz) } -func (r *registry) PushManifest(ctx echo.Context) error { +// PullLayer +// GET /v2//blobs/ +// OK, error: binary output can mess your system ... +func (r *registry) PullLayer(ctx echo.Context) error { + //namespace := ctx.Param("username") + "/" + ctx.Param("imagename") ctx.Set(types.HandlerStartTime, time.Now()) defer func() { r.logger.Log(ctx).Send() }() - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - ref := ctx.Param("reference") - contentType := ctx.Request().Header.Get("Content-Type") + clientDigest := ctx.Param("digest") - bz, err := io.ReadAll(ctx.Request().Body) + layer, err := r.store.GetLayer(ctx.Request().Context(), clientDigest) + //layerRef, err := r.localCache.GetDigest(clientDigest) if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) + //color.Red("error pull layer") + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) } - ctx.Request().Body.Close() - dig := digest(bz) - - var manifest ManifestList - if err = json.Unmarshal(bz, &manifest); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) + if layer.SkynetLink == "" { + detail := map[string]interface{}{ + "error": "skylink is empty", + } + e := fmt.Errorf("skylink is empty").Error() + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, e, detail) ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusBadRequest, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) } - color.Red("manifest list: %s\n", bz) - - mfNamespace := fmt.Sprintf("%s/manifests", namespace) - skylink, err := r.skynet.Upload(mfNamespace, dig, bz, true) + resp, err := r.skynet.Download(layer.SkynetLink) if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, err.Error(), nil) + detail := map[string]interface{}{ + "error": err.Error(), + } + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), detail) ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) } - //id := uuid.Generate() - //mfc := &types.ConfigV2{ - // UUID: id.String(), - // Namespace: namespace, - // Reference: ref, - // Digest: dig, - // Skylink: skylink, - // MediaType: contentType, - // Layers: nil, - // Size: 0, - //} - - manifestConfig := &types.Config{ - MediaType: contentType, - Size: len(bz), - Digest: dig, - SkynetLink: skylink, - Reference: ref, - } - - metadata := types.Metadata{ - Namespace: namespace, - Manifest: types.ImageManifest{ - SchemaVersion: 2, - Config: []*types.Config{manifestConfig}, - }, - } - - if err = r.localCache.Update([]byte(namespace), metadata.Bytes()); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) + bz, err := io.ReadAll(resp) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusBadRequest, errMsg) - } - - val := &types.ImageManifestV2{ - Uuid: uuid.Generate().String(), - Namespace: namespace, - MediaType: "", - SchemaVersion: 2, + return ctx.JSONBlob(http.StatusInternalServerError, errMsg) } + _ = resp.Close() - //if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { - // errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) - // ctx.Set(types.HttpEndpointErrorKey, errMsg) - // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - //} - txnOp, _ := r.store.NewTxn(context.Background()) - - if err := r.store.SetManifest(ctx.Request().Context(), txnOp, val); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) + dig := digest(bz) + if dig != clientDigest { + details := map[string]interface{}{ + "clientDigest": clientDigest, + "computedDigest": dig, + } + errMsg := r.errorResponse( + RegistryErrorCodeBlobUploadUnknown, + "client digest is different than computed digest", + details, + ) ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusBadRequest, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) } - locationHeader := r.getHttpUrlFromSkylink(skylink) - ctx.Response().Header().Set("Location", locationHeader) - ctx.Response().Header().Set("Docker-Content-Digest", dig) - ctx.Response().Header().Set("X-Docker-Content-ID", skylink) - return ctx.String(http.StatusCreated, "Created") + ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", len(bz))) + ctx.Response().Header().Set("Docker-Content-Digest", dig) + return ctx.Blob(http.StatusOK, "application/octet-stream", bz) } -// Content discovery GET /v2//tags/list - -func (r *registry) ListTags(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - +// MonolithicUpload +// PUT /v2//blobs/uploads/?digest= +func (r *registry) MonolithicUpload(ctx echo.Context) error { namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - limit := ctx.QueryParam("n") - - l, err := r.localCache.ListWithPrefix([]byte(namespace)) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) + uuid := ctx.Param("uuid") + digest := ctx.QueryParam("digest") - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - var md types.Metadata - err = json.Unmarshal(l, &md) + bz, err := io.ReadAll(ctx.Request().Body) if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - var tags []string - for _, v := range md.Manifest.Config { - tags = append(tags, v.Reference) - } - if limit != "" { - n, err := strconv.ParseInt(limit, 10, 32) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeTagInvalid, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - if n > 0 { - tags = tags[0:n] - } - if n == 0 { - tags = []string{} - } + return ctx.JSONBlob(http.StatusBadRequest, errMsg) } + ctx.Request().Body.Close() - sort.Strings(tags) - return ctx.JSON(http.StatusOK, echo.Map{ - "name": namespace, - "tags": tags, - }) -} -func (r *registry) List(ctx echo.Context) error { - return fmt.Errorf("error") -} - -// GET /v2//blobs/ - -func (r *registry) PullLayer(ctx echo.Context) error { - ctx.Set(types.HandlerStartTime, time.Now()) - defer func() { - r.logger.Log(ctx).Send() - }() - - namespace := ctx.Param("username") + "/" + ctx.Param("imagename") - clientDigest := ctx.Param("digest") - - layerRef, err := r.localCache.GetDigest(clientDigest) + link, err := r.skynet.Upload(namespace, digest, bz) if err != nil { - skynetLink, err := r.localCache.GetSkynetURL(namespace, clientDigest) - if err != nil { - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } - layerRef = &types.LayerRef{ - Digest: clientDigest, - Skylink: skynetLink, + detail := echo.Map{ + "error": err.Error(), + "caller": "MonolithicUpload", } + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), detail) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusInternalServerError, bz) } - size, ok := r.skynet.Metadata(layerRef.Skylink) - if ok { - url := fmt.Sprintf("https://siasky.net/%s", - strings.Replace(layerRef.Skylink, "sia://", "", 1)) - ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", size)) - http.Redirect(ctx.Response(), ctx.Request(), url, http.StatusTemporaryRedirect) - return nil + metadata := types.Metadata{ + Namespace: namespace, + Manifest: types.ImageManifest{ + SchemaVersion: 2, + MediaType: "", + Layers: []*types.Layer{{MediaType: "", Size: len(bz), Digest: digest, SkynetLink: link, UUID: uuid}}, + }, } - detail := map[string]interface{}{ - "error": "skylink is empty", + err = r.localCache.Update([]byte(namespace), metadata.Bytes()) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) } - e := fmt.Errorf("skylink is empty").Error() - errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, e, detail) - ctx.Set(types.HttpEndpointErrorKey, errMsg) - - return ctx.JSONBlob(http.StatusNotFound, errMsg) -} -//BlobMount to be implemented by guacamole at a later stage -func (r *registry) BlobMount(ctx echo.Context) error { - return nil + locationHeader := link + ctx.Response().Header().Set("Location", locationHeader) + return ctx.NoContent(http.StatusCreated) } -//PushImage is already implemented through StartUpload and ChunkedUpload -func (r *registry) PushImage(ctx echo.Context) error { - return nil +// ChunkedUpload +// PATCH /v2//blobs/uploads/ +func (r *registry) ChunkedUpload(ctx echo.Context) error { + return r.b.UploadBlob(ctx) } /*StartUpload @@ -716,6 +348,7 @@ for postgres: start a tnx registry.tnxMap[uuid] = {txn,blobs[],timeout} */ +// POST /v2//blobs/uploads/ func (r *registry) StartUpload(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) defer func() { @@ -855,6 +488,224 @@ func (r *registry) UploadProgress(ctx echo.Context) error { return ctx.NoContent(http.StatusNoContent) } +// CompleteUpload +/*PUT /v2//blobs/uploads/?digest= +for postgres: +this is where we insert into the layer after all the blobs have been accumulated +and inserted in the blob table +thus committing the txn +*/ +func (r *registry) CompleteUpload(ctx echo.Context) error { + dig := ctx.QueryParam("digest") + namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + id := ctx.Param("uuid") + + bz, err := io.ReadAll(ctx.Request().Body) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeDigestInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + _ = ctx.Request().Body.Close() + // insert if bz is not nil + buf := bytes.NewBuffer(r.b.uploads[id]) + buf.Write(bz) + ourHash := digest(buf.Bytes()) + delete(r.b.uploads, id) + + if ourHash != dig { + details := map[string]interface{}{ + "headerDigest": dig, "serverSideDigest": ourHash, "bodyDigest": digest(bz), + } + errMsg := r.errorResponse(RegistryErrorCodeDigestInvalid, "digest mismatch", details) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + blobNamespace := fmt.Sprintf("%s/blobs", namespace) + skylink, err := r.skynet.Upload(blobNamespace, dig, buf.Bytes()) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg) + } + //if err := r.localCache.SetDigest(ourHash, skylink); err != nil { + // errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) + // ctx.Set(types.HttpEndpointErrorKey, errMsg) + // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) + //} + txnOp, ok := r.txnMap[id] + layer := &types.LayerV2{ + MediaType: "", + Digest: dig, + SkynetLink: skylink, + UUID: id, + BlobDigests: txnOp.blobDigests, + Size: len(bz), + } + if !ok { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, "transaction does not exist for uuid -"+id, nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + if err := r.store.SetLayer(ctx.Request().Context(), txnOp.txn, layer); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), echo.Map{ + "error_detail": "set layer issues", + }) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + // val := &types.ImageManifestV2{ + // Uuid: uuid, + // Namespace: namespace, + // MediaType: "", + // SchemaVersion: 2, + // } + + //if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { + // errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) + // ctx.Set(types.HttpEndpointErrorKey, errMsg) + // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) + //} + + // if err := r.store.SetManifest(ctx.Request().Context(), txnOp.txn, val); err != nil { + // errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) + // ctx.Set(types.HttpEndpointErrorKey, errMsg) + // return ctx.JSONBlob(http.StatusBadRequest, errMsg) + // } + + if err := r.store.Commit(ctx.Request().Context(), txnOp.txn); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), echo.Map{ + "error_detail": "commitment issue", + }) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + delete(r.txnMap, id) + + locationHeader := fmt.Sprintf("/v2/%s/blobs/%s", namespace, ourHash) + ctx.Response().Header().Set("Content-Length", "0") + ctx.Response().Header().Set("Docker-Content-Digest", ourHash) + ctx.Response().Header().Set("Location", locationHeader) + return ctx.NoContent(http.StatusCreated) +} + +//BlobMount to be implemented by guacamole at a later stage +func (r *registry) BlobMount(ctx echo.Context) error { + return nil +} + +//PushImage is already implemented through StartUpload and ChunkedUpload +func (r *registry) PushImage(ctx echo.Context) error { + return nil +} + +func (r *registry) PushManifest(ctx echo.Context) error { + namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + ref := ctx.Param("reference") + contentType := ctx.Request().Header.Get("Content-Type") + + bz, err := io.ReadAll(ctx.Request().Body) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + ctx.Request().Body.Close() + + dig := digest(bz) + + var manifest ImageManifest + if err = json.Unmarshal(bz, &manifest); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + mfNamespace := fmt.Sprintf("%s/manifests", namespace) + skylink, err := r.skynet.Upload(mfNamespace, dig, bz) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) + } + + id := uuid.Generate() + mfc := types.ConfigV2{ + UUID: id.String(), + Namespace: namespace, + Reference: ref, + Digest: dig, + Skylink: skylink, + MediaType: contentType, + Layers: nil, + Size: 0, + } + + //manifestConfig := &types.Config{ + // MediaType: contentType, + // Size: len(bz), + // Digest: dig, + // SkynetLink: skylink, + // Reference: ref, + //} + + //metadata := types.Metadata{ + // Namespace: namespace, + // Manifest: types.ImageManifest{ + // SchemaVersion: 2, + // Config: []*types.Config{manifestConfig}, + // }, + //} + + //if err = r.localCache.Update([]byte(namespace), metadata.Bytes()); err != nil { + // errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) + // ctx.Set(types.HttpEndpointErrorKey, errMsg) + // return ctx.JSONBlob(http.StatusBadRequest, errMsg) + //} + + val := &types.ImageManifestV2{ + Uuid: uuid.Generate().String(), + Namespace: namespace, + MediaType: "", + SchemaVersion: 2, + } + + //if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { + // errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) + // ctx.Set(types.HttpEndpointErrorKey, errMsg) + // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) + //} + txnOp, _ := r.store.NewTxn(context.Background()) + + if err := r.store.SetManifest(ctx.Request().Context(), txnOp, val); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + _ = r.store.Abort(ctx.Request().Context(), txnOp) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + if err := r.store.SetConfig(ctx.Request().Context(), txnOp, mfc); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + _ = r.store.Abort(ctx.Request().Context(), txnOp) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + if err = r.store.Commit(ctx.Request().Context(), txnOp); err != nil { + return err + } + + locationHeader := r.getHttpUrlFromSkylink(skylink) + ctx.Response().Header().Set("Location", locationHeader) + ctx.Response().Header().Set("Docker-Content-Digest", dig) + ctx.Response().Header().Set("X-Docker-Content-ID", skylink) + return ctx.String(http.StatusCreated, "Created") +} + +// PushLayer // POST /v2//blobs/uploads/ func (r *registry) PushLayer(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) @@ -885,6 +736,99 @@ func (r *registry) PushLayer(ctx echo.Context) error { return ctx.NoContent(http.StatusAccepted) } +func (r *registry) CancelUpload(ctx echo.Context) error { + return nil +} + +// DeleteTagOrManifest +// DELETE /v2//manifest/ or +func (r *registry) DeleteTagOrManifest(ctx echo.Context) error { + namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + ref := ctx.Param("reference") + + if ref == "" { + reqURI := strings.Split(ctx.Request().RequestURI, "/") + if len(reqURI) == 6 { + ref = reqURI[5] + } + } + txnOp, _ := r.store.NewTxn(context.Background()) + if err := r.store.DeleteManifestOrTag(ctx.Request().Context(), txnOp, ref); err != nil { + //if err := r.localCache.UpdateManifestRef(namespace, ref); err != nil { + details := map[string]interface{}{ + "namespace": namespace, + "digest": ref, + } + errMsg := r.errorResponse(RegistryErrorCodeManifestUnknown, err.Error(), details) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) + } + + _ = r.store.Commit(ctx.Request().Context(), txnOp) + return ctx.NoContent(http.StatusAccepted) +} + +func (r *registry) DeleteLayer(ctx echo.Context) error { + //namespace := ctx.Param("username") + "/" + ctx.Param("imagename") + dig := ctx.Param("digest") + + //var m types.Metadata + + layer, err := r.store.GetLayer(ctx.Request().Context(), dig) + //_, err := r.localCache.GetDigest(dig) + if err != nil { + //bz, err := r.localCache.Get([]byte(namespace)) + //if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusNotFound, errMsg) + //} + //if err = json.Unmarshal(bz, &m); err != nil { + // errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) + // ctx.Set(types.HttpEndpointErrorKey, errMsg) + // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) + //} + } + blobs := layer.BlobDigests + + //err = r.localCache.DeleteLayer(namespace, dig) + txnOp, _ := r.store.NewTxn(context.Background()) + err = r.store.DeleteLayerV2(ctx.Request().Context(), txnOp, dig) + if err != nil { + logMsg := echo.Map{ + "error": err.Error(), + "caller": "DeleteLayer", + } + + bz, err := json.Marshal(logMsg) + if err == nil { + ctx.Set(types.HttpEndpointErrorKey, logMsg) + } + + return ctx.JSONBlob(http.StatusInternalServerError, bz) + } + + for i, _ := range blobs { + //if err = r.localCache.DeleteDigest(dig); err != nil { + if err = r.store.DeleteBlobV2(ctx.Request().Context(), txnOp, blobs[i]); err != nil { + logMsg := echo.Map{ + "error": err.Error(), + "caller": "DeleteLayer", + } + + ctx.Set(types.HttpEndpointErrorKey, logMsg) + bz, err := json.Marshal(logMsg) + if err != nil { + r.log.Err(err).Send() + } + + return ctx.JSONBlob(http.StatusInternalServerError, bz) + } + } + _ = r.store.Commit(ctx.Request().Context(), txnOp) + return ctx.NoContent(http.StatusAccepted) +} + // Should also look into 401 Code // https://docs.docker.com/registry/spec/api/ func (r *registry) ApiVersion(ctx echo.Context) error { diff --git a/store/postgres/container_image.go b/store/postgres/container_image.go index 6011e59e..f91c53ef 100644 --- a/store/postgres/container_image.go +++ b/store/postgres/container_image.go @@ -2,7 +2,7 @@ package postgres import ( "context" - "fmt" + "strings" "time" "github.com/containerish/OpenRegistry/store/postgres/queries" @@ -10,18 +10,18 @@ import ( "github.com/jackc/pgx/v4" ) -func (p *pg) GetLayer(ctx context.Context, txn pgx.Tx, digest string) (*types.Layer, error) { +func (p *pg) GetLayer(ctx context.Context, digest string) (*types.LayerV2, error) { childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - row := txn.QueryRow(childCtx, queries.GetLayer, digest) - var layer types.Layer + row := p.conn.QueryRow(childCtx, queries.GetLayer, digest) + var layer types.LayerV2 if err := row.Scan( - &layer.MediaType, + &layer.UUID, &layer.Digest, + &layer.BlobDigests, + &layer.MediaType, &layer.SkynetLink, - &layer.UUID, - &layer.Blobs, &layer.Size, ); err != nil { return nil, err @@ -39,11 +39,11 @@ func (p *pg) SetLayer(ctx context.Context, txn pgx.Tx, l *types.LayerV2) error { return err } -func (p *pg) GetManifest(ctx context.Context, txn pgx.Tx, ref string) (*types.ImageManifestV2, error) { +func (p *pg) GetManifest(ctx context.Context, namespace string) (*types.ImageManifestV2, error) { childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - row := txn.QueryRow(childCtx, queries.GetManifest, ref) + row := p.conn.QueryRow(childCtx, queries.GetManifest, namespace) var im *types.ImageManifestV2 if err := row.Scan( &im.Uuid, @@ -55,6 +55,31 @@ func (p *pg) GetManifest(ctx context.Context, txn pgx.Tx, ref string) (*types.Im } return im, nil } +func (p *pg) GetManifestByReference(ctx context.Context, namespace string, ref string) (*types.ConfigV2, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + query := queries.GetManifestByRef + if strings.HasPrefix(ref, "sha256") { + query = queries.GetManifestByDig + } + + row := p.conn.QueryRow(childCtx, query, namespace, ref) + var im types.ConfigV2 + if err := row.Scan( + &im.UUID, + &im.Namespace, + &im.Reference, + &im.Digest, + &im.Skylink, + &im.MediaType, + &im.Layers, + &im.Size, + ); err != nil { + return nil, err + } + return &im, nil +} func (p *pg) SetManifest(ctx context.Context, txn pgx.Tx, im *types.ImageManifestV2) error { childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) @@ -64,6 +89,36 @@ func (p *pg) SetManifest(ctx context.Context, txn pgx.Tx, im *types.ImageManifes return err } +func (p *pg) GetBlob(ctx context.Context, digest string) ([]*types.Blob, error) { + + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + rows, err := p.conn.Query(childCtx, queries.GetBlob, digest) + if err != nil { + return nil, err + } + defer rows.Close() + + blobList := make([]*types.Blob, 0) + for i := 0; rows.Next(); i++ { + var blob types.Blob + if err := rows.Scan( + &blob.UUID, + &blob.Digest, + &blob.Skylink, + &blob.RangeStart, + &blob.RangeEnd, + ); err != nil { + return nil, err + } + + blobList = append(blobList, &blob) + } + + return blobList, nil +} + func (p *pg) SetBlob(ctx context.Context, txn pgx.Tx, b *types.Blob) error { childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -73,45 +128,146 @@ func (p *pg) SetBlob(ctx context.Context, txn pgx.Tx, b *types.Blob) error { } -func (p *pg) GetBlob(ctx context.Context, txn pgx.Tx, digest string) (*types.Blob, error) { +func (p *pg) GetConfig(ctx context.Context, namespace string) ([]*types.ConfigV2, error) { childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - row := txn.QueryRow(childCtx, queries.GetBlob, digest) - if row == nil { - return nil, fmt.Errorf("error blob not found") + rows, err := p.conn.Query(childCtx, queries.GetConfig, namespace) + if err != nil { + return nil, err } - var blob types.Blob - if err := row.Scan( - &blob.UUID, - &blob.Digest, - &blob.Skylink, - &blob.RangeStart, - &blob.RangeEnd, - ); err != nil { + defer rows.Close() + cfgList := make([]*types.ConfigV2, 0) + + for i := 0; rows.Next(); i++ { + var cfg types.ConfigV2 + if err := rows.Scan( + &cfg.UUID, + &cfg.Namespace, + &cfg.Reference, + &cfg.Digest, + &cfg.Skylink, + &cfg.MediaType, + &cfg.Layers, + &cfg.Size, + ); err != nil { + return nil, err + } + + cfgList = append(cfgList, &cfg) + } + + return cfgList, nil +} +func (p *pg) GetImageTags(ctx context.Context, namespace string) ([]string, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + rows, err := p.conn.Query(childCtx, queries.GetImageTags, namespace) + if err != nil { + return nil, err + } + defer rows.Close() + var tags []string + + for i := 0; rows.Next(); i++ { + var tag string + if err := rows.Scan(&tag); err != nil { + return nil, err + } + + tags = append(tags, tag) + } + + return tags, nil +} + +func (p *pg) SetConfig(ctx context.Context, txn pgx.Tx, cfg types.ConfigV2) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + if _, err := txn.Exec(childCtx, queries.SetConfig, cfg.UUID, cfg.Namespace, cfg.Reference, cfg.Digest, cfg.Skylink, cfg.MediaType, cfg.Layers, cfg.Size); err != nil { + return err + } + return nil +} + +func (p *pg) GetCatalog(ctx context.Context) ([]*types.ConfigV2, error) { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + rows, err := p.conn.Query(childCtx, queries.GetCatalog) + if err != nil { return nil, err } - return &blob, nil + cfgList := make([]*types.ConfigV2, 0) + for i := 0; rows.Next(); i++ { + var cfg types.ConfigV2 + if err := rows.Scan( + &cfg.Namespace, + &cfg.Reference, + &cfg.Digest, + ); err != nil { + return nil, err + } + + cfgList = append(cfgList, &cfg) + } + + return cfgList, nil } -func (p *pg) NewTxn(ctx context.Context) (pgx.Tx, error) { +func (p *pg) DeleteLayerV2(ctx context.Context, txn pgx.Tx, digest string) error { childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - return p.conn.Begin(childCtx) + if _, err := txn.Exec(childCtx, queries.DeleteLayer, digest); err != nil { + return err + } + return nil } -func (p *pg) Abort(ctx context.Context, txn pgx.Tx) error { +func (p *pg) DeleteBlobV2(ctx context.Context, txn pgx.Tx, digest string) error { + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + if _, err := txn.Exec(childCtx, queries.DeleteBlob, digest); err != nil { + return err + } + return nil +} + +func (p *pg) DeleteManifestOrTag(ctx context.Context, txn pgx.Tx, reference string) error { childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() + query := queries.DeleteManifestByRef + if strings.HasPrefix(reference, "sha256") { + query = queries.DeleteManifestByDig + } + if _, err := txn.Exec(childCtx, query, reference); err != nil { + return err + } + return nil +} + +func (p *pg) NewTxn(ctx context.Context) (pgx.Tx, error) { + childCtx, _ := context.WithTimeout(context.Background(), time.Minute*30) + //defer cancel() + + return p.conn.Begin(childCtx) +} + +func (p *pg) Abort(ctx context.Context, txn pgx.Tx) error { + childCtx, _ := context.WithTimeout(context.Background(), time.Minute*30) + //defer cancel() return txn.Rollback(childCtx) } func (p *pg) Commit(ctx context.Context, txn pgx.Tx) error { - childCtx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() + childCtx, _ := context.WithTimeout(context.Background(), time.Minute*30) + //defer cancel() return txn.Commit(childCtx) } diff --git a/store/postgres/postgres.go b/store/postgres/postgres.go index c085577e..bf26cd2d 100644 --- a/store/postgres/postgres.go +++ b/store/postgres/postgres.go @@ -47,12 +47,20 @@ type RegistryStore interface { NewTxn(ctx context.Context) (pgx.Tx, error) Abort(ctx context.Context, txn pgx.Tx) error Commit(ctx context.Context, txn pgx.Tx) error - GetLayer(ctx context.Context, txn pgx.Tx, digest string) (*types.Layer, error) SetLayer(ctx context.Context, txn pgx.Tx, l *types.LayerV2) error - GetManifest(ctx context.Context, txn pgx.Tx, ref string) (*types.ImageManifestV2, error) SetManifest(ctx context.Context, txn pgx.Tx, im *types.ImageManifestV2) error SetBlob(ctx context.Context, txn pgx.Tx, b *types.Blob) error - GetBlob(ctx context.Context, txn pgx.Tx, digest string) (*types.Blob, error) + SetConfig(ctx context.Context, txn pgx.Tx, cfg types.ConfigV2) error + GetManifest(ctx context.Context, ref string) (*types.ImageManifestV2, error) + GetManifestByReference(ctx context.Context, namespace string, ref string) (*types.ConfigV2, error) + GetLayer(ctx context.Context, digest string) (*types.LayerV2, error) + GetBlob(ctx context.Context, digest string) ([]*types.Blob, error) + GetConfig(ctx context.Context, namespace string) ([]*types.ConfigV2, error) + GetImageTags(ctx context.Context, namespace string) ([]string, error) + GetCatalog(ctx context.Context) ([]*types.ConfigV2, error) + DeleteLayerV2(ctx context.Context, txn pgx.Tx, digest string) error + DeleteBlobV2(ctx context.Context, txn pgx.Tx, digest string) error + DeleteManifestOrTag(ctx context.Context, txn pgx.Tx, reference string) error } type pg struct { diff --git a/store/postgres/queries/registry.go b/store/postgres/queries/registry.go index b3d01aeb..c1867c4d 100644 --- a/store/postgres/queries/registry.go +++ b/store/postgres/queries/registry.go @@ -6,20 +6,34 @@ var ( SetImageManifest = `insert into image_manifest (uuid, namespace, media_type, schema_version) values ($1, $2, $3, $4) on conflict (namespace) do update set schema_version=3` SetLayer = `insert into layer (media_type, digest, sky_link, uuid, blob_ids, size) - values ($1, $2, $3, $4, $5, $6);` + values ($1, $2, $3, $4, $5, $6) on conflict (digest) do nothing;` + // SetBlob TODO - (guacamole/jay-dee7) find a better way to handle duplicates in blob SetBlob = `insert into blob (uuid, digest, sky_link, start_range, end_range) - values ($1, $2, $3, $4, $5);` + values ($1, $2, $3, $4, $5) on conflict (digest) do nothing;` - SetConfig = `insert into config (media_type, digest, sky_link, reference, size) - values ($1, $2, $3, $4, $5);` + SetConfig = `insert into config (uuid, namespace, reference, digest, sky_link, media_type, layers, size) + values ($1, $2, $3, $4, $5, $6,$7, $8) on conflict (digest) do set digest=$4;` ) // select queries var ( - GetDigest = `select digest from layers where digest=$1;` - ReadMetadata = `select * from metadata where namespace=$1;` - GetLayer = `select * from layer where digest=$1;` - GetManifest = `select * from image_manifest where namespace=$1;` - GetBlob = `select * from blob where digest=$1;` + GetDigest = `select digest from layers where digest=$1;` + ReadMetadata = `select * from metadata where namespace=$1;` + GetLayer = `select * from layer where digest=$1;` + GetManifest = `select * from image_manifest where namespace=$1;` + GetBlob = `select * from blob where digest=$1;` + GetConfig = `select * from config where namespace=$1;` + GetImageTags = `select reference from config where namespace=$1;` + GetManifestByRef = `select * from config where namespace=$1 and reference=$2;` + GetManifestByDig = `select * from config where namespace=$1 and digest=$2;` + GetCatalog = `select namespace,reference,digest from config;` +) + +// delete queries +var ( + DeleteLayer = `delete from layer where digest=$1;` + DeleteBlob = `delete from blob where digest=$1;` + DeleteManifestByRef = `delete from config where reference=$1;` + DeleteManifestByDig = `delete from config where digest=$1;` ) From c81c5c6d29a0e285f19fe06340121c0d36016332 Mon Sep 17 00:00:00 2001 From: guacamole Date: Sun, 12 Dec 2021 21:05:37 +0530 Subject: [PATCH 07/16] WIP: 58 out of 59 conformance cases are passing Signed-off-by: guacamole --- registry/v2/registry.go | 54 +++--------------------------- router/router.go | 3 +- store/postgres/queries/registry.go | 2 +- 3 files changed, 6 insertions(+), 53 deletions(-) diff --git a/registry/v2/registry.go b/registry/v2/registry.go index b32737ad..8a12f5f3 100644 --- a/registry/v2/registry.go +++ b/registry/v2/registry.go @@ -128,7 +128,7 @@ func (r *registry) Catalog(ctx echo.Context) error { catalogList, err := r.store.GetCatalog(ctx.Request().Context()) if err != nil { - ctx.Set(types.HttpEndpointErrorKey,err.Error()) + ctx.Set(types.HttpEndpointErrorKey, err.Error()) return ctx.JSON(http.StatusInternalServerError, echo.Map{ "error": err.Error(), }) @@ -305,7 +305,7 @@ func (r *registry) MonolithicUpload(ctx echo.Context) error { } ctx.Request().Body.Close() - link, err := r.skynet.Upload(namespace, digest, bz) + link, err := r.skynet.Upload(namespace, digest, bz, true) if err != nil { detail := echo.Map{ "error": err.Error(), @@ -523,7 +523,7 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { } blobNamespace := fmt.Sprintf("%s/blobs", namespace) - skylink, err := r.skynet.Upload(blobNamespace, dig, buf.Bytes()) + skylink, err := r.skynet.Upload(blobNamespace, dig, buf.Bytes(), true) if err != nil { errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) @@ -557,25 +557,6 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { return ctx.JSONBlob(http.StatusBadRequest, errMsg) } - // val := &types.ImageManifestV2{ - // Uuid: uuid, - // Namespace: namespace, - // MediaType: "", - // SchemaVersion: 2, - // } - - //if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { - // errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) - // ctx.Set(types.HttpEndpointErrorKey, errMsg) - // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - //} - - // if err := r.store.SetManifest(ctx.Request().Context(), txnOp.txn, val); err != nil { - // errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) - // ctx.Set(types.HttpEndpointErrorKey, errMsg) - // return ctx.JSONBlob(http.StatusBadRequest, errMsg) - // } - if err := r.store.Commit(ctx.Request().Context(), txnOp.txn); err != nil { errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), echo.Map{ "error_detail": "commitment issue", @@ -625,7 +606,7 @@ func (r *registry) PushManifest(ctx echo.Context) error { } mfNamespace := fmt.Sprintf("%s/manifests", namespace) - skylink, err := r.skynet.Upload(mfNamespace, dig, bz) + skylink, err := r.skynet.Upload(mfNamespace, dig, bz, true) if err != nil { errMsg := r.errorResponse(RegistryErrorCodeManifestBlobUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) @@ -644,28 +625,6 @@ func (r *registry) PushManifest(ctx echo.Context) error { Size: 0, } - //manifestConfig := &types.Config{ - // MediaType: contentType, - // Size: len(bz), - // Digest: dig, - // SkynetLink: skylink, - // Reference: ref, - //} - - //metadata := types.Metadata{ - // Namespace: namespace, - // Manifest: types.ImageManifest{ - // SchemaVersion: 2, - // Config: []*types.Config{manifestConfig}, - // }, - //} - - //if err = r.localCache.Update([]byte(namespace), metadata.Bytes()); err != nil { - // errMsg := r.errorResponse(RegistryErrorCodeManifestInvalid, err.Error(), nil) - // ctx.Set(types.HttpEndpointErrorKey, errMsg) - // return ctx.JSONBlob(http.StatusBadRequest, errMsg) - //} - val := &types.ImageManifestV2{ Uuid: uuid.Generate().String(), Namespace: namespace, @@ -673,11 +632,6 @@ func (r *registry) PushManifest(ctx echo.Context) error { SchemaVersion: 2, } - //if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { - // errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) - // ctx.Set(types.HttpEndpointErrorKey, errMsg) - // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - //} txnOp, _ := r.store.NewTxn(context.Background()) if err := r.store.SetManifest(ctx.Request().Context(), txnOp, val); err != nil { diff --git a/router/router.go b/router/router.go index 617a03ad..ac0ce7d0 100644 --- a/router/router.go +++ b/router/router.go @@ -2,7 +2,6 @@ package router import ( "github.com/containerish/OpenRegistry/store/postgres" - "log" "net/http" "github.com/containerish/OpenRegistry/auth" @@ -24,7 +23,7 @@ func Register( authSvc auth.Authentication, localCache cache.Store, pStore postgres.PersistentStore, - ) { +) { e.Use(middleware.Recover()) e.Use(middleware.CORS()) diff --git a/store/postgres/queries/registry.go b/store/postgres/queries/registry.go index c1867c4d..afd9b5e2 100644 --- a/store/postgres/queries/registry.go +++ b/store/postgres/queries/registry.go @@ -13,7 +13,7 @@ var ( values ($1, $2, $3, $4, $5) on conflict (digest) do nothing;` SetConfig = `insert into config (uuid, namespace, reference, digest, sky_link, media_type, layers, size) - values ($1, $2, $3, $4, $5, $6,$7, $8) on conflict (digest) do set digest=$4;` + values ($1, $2, $3, $4, $5, $6,$7, $8) on conflict (digest) do nothing;` ) // select queries From a86401673c54ca0e6bff8e36433c44cba485dad4 Mon Sep 17 00:00:00 2001 From: guacamole Date: Mon, 13 Dec 2021 16:08:34 +0530 Subject: [PATCH 08/16] Fix: monolithic upload function to support single POST layer upload Signed-off-by: guacamole --- .../oci-dist-spec-content-discovery.yml | 22 ++++++- registry/v2/registry.go | 66 +++++++++---------- store/postgres/queries/registry.go | 2 +- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/.github/workflows/oci-dist-spec-content-discovery.yml b/.github/workflows/oci-dist-spec-content-discovery.yml index e156ab0f..593619f9 100644 --- a/.github/workflows/oci-dist-spec-content-discovery.yml +++ b/.github/workflows/oci-dist-spec-content-discovery.yml @@ -6,7 +6,27 @@ on: jobs: content-discovery: runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + POSTGRES_DB: open_registry + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: + - name: set up postgres database + run: | + psql --version + psql -d open_registry -c '\dt' --host=localhost --port=5432 -U postgres + env: + PGPASSWORD: postgres - name: set up docker uses: docker-practice/actions-setup-docker@0.0.1 with: @@ -22,7 +42,7 @@ jobs: docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" . sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example echo CI_SYS_ADDR=$IP:5000 >> env-vars.example - docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}" + docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -i "${DISTRIBUTION_REF}" sleep 5 curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup" - name: Run OCI Distribution Spec conformance tests diff --git a/registry/v2/registry.go b/registry/v2/registry.go index 8a12f5f3..0183577f 100644 --- a/registry/v2/registry.go +++ b/registry/v2/registry.go @@ -233,9 +233,7 @@ func (r *registry) PullLayer(ctx echo.Context) error { clientDigest := ctx.Param("digest") layer, err := r.store.GetLayer(ctx.Request().Context(), clientDigest) - //layerRef, err := r.localCache.GetDigest(clientDigest) if err != nil { - //color.Red("error pull layer") errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) @@ -255,7 +253,8 @@ func (r *registry) PullLayer(ctx echo.Context) error { resp, err := r.skynet.Download(layer.SkynetLink) if err != nil { detail := map[string]interface{}{ - "error": err.Error(), + "error": err.Error(), + "skylink": layer.SkynetLink, } errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), detail) ctx.Set(types.HttpEndpointErrorKey, errMsg) @@ -372,9 +371,7 @@ func (r *registry) StartUpload(ctx echo.Context) error { ) ctx.Set(types.HttpEndpointErrorKey, errMsg) - return ctx.JSONBlob(http.StatusNotFound, errMsg) - } ctx.Request().Body.Close() // why defer? body is already read :) dig := digest(bz) @@ -402,27 +399,36 @@ func (r *registry) StartUpload(ctx echo.Context) error { return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg) } - layer := &types.Layer{ - MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", - Size: len(bz), - Digest: dig, - SkynetLink: skylink, - UUID: "", + layerV2 := &types.LayerV2{ + MediaType: ctx.Request().Header.Get("content-type"), + Digest: dig, + SkynetLink: skylink, + UUID: uuid.Generate().String(), + BlobDigests: nil, + Size: len(bz), } - var val types.Metadata - val.Namespace = namespace - val.Manifest.Layers = append(val.Manifest.Layers, layer) - - link := r.getHttpUrlFromSkylink(skylink) - if err = r.localCache.Update([]byte(namespace), val.Bytes()); err != nil { - errMsg := r.errorResponse(RegistryErrorCodeUnsupported, err.Error(), nil) + txnOp, err := r.store.NewTxn(ctx.Request().Context()) + if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusInternalServerError, errMsg) } - ctx.Response().Header().Set("Location", link) + if err := r.store.SetLayer(ctx.Request().Context(), txnOp, layerV2); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + if err := r.store.Commit(ctx.Request().Context(), txnOp); err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUploadInvalid, err.Error(), nil) + ctx.Set(types.HttpEndpointErrorKey, errMsg) + return ctx.JSONBlob(http.StatusBadRequest, errMsg) + } + + link := r.getHttpUrlFromSkylink(skylink) + ctx.Response().Header().Set("Location", link) return ctx.NoContent(http.StatusCreated) } @@ -529,11 +535,7 @@ func (r *registry) CompleteUpload(ctx echo.Context) error { ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusRequestedRangeNotSatisfiable, errMsg) } - //if err := r.localCache.SetDigest(ourHash, skylink); err != nil { - // errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) - // ctx.Set(types.HttpEndpointErrorKey, errMsg) - // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - //} + txnOp, ok := r.txnMap[id] layer := &types.LayerV2{ MediaType: "", @@ -613,6 +615,11 @@ func (r *registry) PushManifest(ctx echo.Context) error { return ctx.JSONBlob(http.StatusNotFound, errMsg) } + var layerIDs []string + for _, layer := range manifest.Layers { + layerIDs = append(layerIDs, layer.Digest) + } + id := uuid.Generate() mfc := types.ConfigV2{ UUID: id.String(), @@ -621,7 +628,7 @@ func (r *registry) PushManifest(ctx echo.Context) error { Digest: dig, Skylink: skylink, MediaType: contentType, - Layers: nil, + Layers: layerIDs, Size: 0, } @@ -731,17 +738,10 @@ func (r *registry) DeleteLayer(ctx echo.Context) error { layer, err := r.store.GetLayer(ctx.Request().Context(), dig) //_, err := r.localCache.GetDigest(dig) if err != nil { - //bz, err := r.localCache.Get([]byte(namespace)) - //if err != nil { + errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) ctx.Set(types.HttpEndpointErrorKey, errMsg) return ctx.JSONBlob(http.StatusNotFound, errMsg) - //} - //if err = json.Unmarshal(bz, &m); err != nil { - // errMsg := r.errorResponse(RegistryErrorCodeBlobUnknown, err.Error(), nil) - // ctx.Set(types.HttpEndpointErrorKey, errMsg) - // return ctx.JSONBlob(http.StatusInternalServerError, errMsg) - //} } blobs := layer.BlobDigests diff --git a/store/postgres/queries/registry.go b/store/postgres/queries/registry.go index afd9b5e2..49044e9f 100644 --- a/store/postgres/queries/registry.go +++ b/store/postgres/queries/registry.go @@ -13,7 +13,7 @@ var ( values ($1, $2, $3, $4, $5) on conflict (digest) do nothing;` SetConfig = `insert into config (uuid, namespace, reference, digest, sky_link, media_type, layers, size) - values ($1, $2, $3, $4, $5, $6,$7, $8) on conflict (digest) do nothing;` + values ($1, $2, $3, $4, $5, $6,$7, $8) on conflict (namespace,reference) do update set digest=$4, sky_link=$5, layers=$7;` ) // select queries From 84f64b38dbd847d2aa31178558d52897ed445bf9 Mon Sep 17 00:00:00 2001 From: jay-dee7 Date: Mon, 13 Dec 2021 20:10:53 +0530 Subject: [PATCH 09/16] Fix: PostgreSQL in Github Actions Signed-off-by: jay-dee7 --- .../oci-dist-spec-content-discovery.yml | 12 ++--- scripts/postgres/OpenRegistry.sql | 46 +++++++++++++++++++ scripts/postgres/setup.sh | 9 ++++ 3 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 scripts/postgres/OpenRegistry.sql create mode 100755 scripts/postgres/setup.sh diff --git a/.github/workflows/oci-dist-spec-content-discovery.yml b/.github/workflows/oci-dist-spec-content-discovery.yml index 593619f9..9d3af290 100644 --- a/.github/workflows/oci-dist-spec-content-discovery.yml +++ b/.github/workflows/oci-dist-spec-content-discovery.yml @@ -21,18 +21,18 @@ jobs: ports: - 5432:5432 steps: - - name: set up postgres database - run: | - psql --version - psql -d open_registry -c '\dt' --host=localhost --port=5432 -U postgres - env: - PGPASSWORD: postgres - name: set up docker uses: docker-practice/actions-setup-docker@0.0.1 with: docker_version: 18.09 docker_channel: stable - uses: actions/checkout@v2 + - name: Setup PostgreSQL Database + run: sh scripts/postgres/setup.sh + env: + PGDATABASE: open_registry + PGUSER: postgres + PGPASSWORD: postgres - name: start distribution server run: | IP=`hostname -I | awk '{print $1}'` diff --git a/scripts/postgres/OpenRegistry.sql b/scripts/postgres/OpenRegistry.sql new file mode 100644 index 00000000..6a9b63dc --- /dev/null +++ b/scripts/postgres/OpenRegistry.sql @@ -0,0 +1,46 @@ +CREATE TABLE "users" ( + "id" uuid PRIMARY KEY, + "is_active" boolean, + "username" varchar NOT NULL UNIQUE, + "email" varchar NOT NULL UNIQUE, + "password" varchar, + "created_at" timestamp, + "updated_at" timestamp, + "country_code" int +); + +CREATE TABLE "config" ( + "uuid" uuid UNIQUE NOT NULL, + "namespace" text, + "reference" text, + "digest" text UNIQUE NOT NULL, + "sky_link" text, + "media_type" text, + "layers" text[], + "size" int, + PRIMARY KEY (namespace,reference) +); + +CREATE TABLE "blob" ( + "uuid" uuid, + "digest" text PRIMARY KEY, + "sky_link" text, + "start_range" int, + "end_range" int +); + +CREATE TABLE "layer" ( + "uuid" uuid PRIMARY KEY, + "digest" text UNIQUE NOT NULL, + "blob_ids" text[], + "media_type" text, + "sky_link" text, + "size" int +); + +CREATE TABLE "image_manifest" ( + "uuid" uuid PRIMARY KEY, + "namespace" text UNIQUE NOT NULL, + "media_type" text, + "schema_version" int +); diff --git a/scripts/postgres/setup.sh b/scripts/postgres/setup.sh new file mode 100755 index 00000000..e1d37abb --- /dev/null +++ b/scripts/postgres/setup.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e +/Applications/Postgres.app/Contents/Versions/14/bin/psql -v ON_ERROR_STOP=1 --username "$PGUSER" --dbname "$PGDATABASE" <<-EOSQL + CREATE USER jane_doe; + GRANT ALL PRIVILEGES ON DATABASE open_registry TO jane_doe; + \c open_registry; + \i scripts/postgres/OpenRegistry.sql +EOSQL From 9da5942ca706977dd0818416558b1c9673c4449f Mon Sep 17 00:00:00 2001 From: guacamole Date: Tue, 14 Dec 2021 12:02:17 +0530 Subject: [PATCH 10/16] Fix: Partially fixes linter errors Signed-off-by: guacamole --- config/config.go | 3 ++- registry/v2/types.go | 5 +---- router/router.go | 2 +- store/postgres/container_image.go | 12 +++++++++++- store/postgres/queries/registry.go | 7 ++++--- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/config/config.go b/config/config.go index 972250ba..9cc33f3b 100644 --- a/config/config.go +++ b/config/config.go @@ -67,7 +67,8 @@ func NewStoreConfig() (*StoreConfig, error) { } func (sc *StoreConfig) Endpoint() string { - return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?pool_max_conns=1000", sc.User, sc.Password, sc.Host, sc.Port, sc.Database) + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?pool_max_conns=1000", + sc.User, sc.Password, sc.Host, sc.Port, sc.Database) } func LoadFromENV() (*RegistryConfig, error) { diff --git a/registry/v2/types.go b/registry/v2/types.go index a093a35e..647d903e 100644 --- a/registry/v2/types.go +++ b/registry/v2/types.go @@ -7,10 +7,9 @@ import ( "sync" "time" - "github.com/containerish/OpenRegistry/telemetry" - "github.com/containerish/OpenRegistry/cache" "github.com/containerish/OpenRegistry/skynet" + "github.com/containerish/OpenRegistry/telemetry" "github.com/labstack/echo/v4" ) @@ -114,8 +113,6 @@ type ( registry *registry } - logMsg map[string]interface{} - ManifestList struct { SchemaVersion int `json:"schemaVersion"` MediaType string `json:"mediaType"` diff --git a/router/router.go b/router/router.go index ac0ce7d0..6692b1c2 100644 --- a/router/router.go +++ b/router/router.go @@ -1,13 +1,13 @@ package router import ( - "github.com/containerish/OpenRegistry/store/postgres" "net/http" "github.com/containerish/OpenRegistry/auth" "github.com/containerish/OpenRegistry/cache" "github.com/containerish/OpenRegistry/config" "github.com/containerish/OpenRegistry/registry/v2" + "github.com/containerish/OpenRegistry/store/postgres" "github.com/google/uuid" "github.com/labstack/echo-contrib/prometheus" "github.com/labstack/echo/v4" diff --git a/store/postgres/container_image.go b/store/postgres/container_image.go index f91c53ef..bbfe2b7b 100644 --- a/store/postgres/container_image.go +++ b/store/postgres/container_image.go @@ -186,7 +186,17 @@ func (p *pg) SetConfig(ctx context.Context, txn pgx.Tx, cfg types.ConfigV2) erro childCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - if _, err := txn.Exec(childCtx, queries.SetConfig, cfg.UUID, cfg.Namespace, cfg.Reference, cfg.Digest, cfg.Skylink, cfg.MediaType, cfg.Layers, cfg.Size); err != nil { + if _, err := txn.Exec(childCtx, + queries.SetConfig, + cfg.UUID, + cfg.Namespace, + cfg.Reference, + cfg.Digest, + cfg.Skylink, + cfg.MediaType, + cfg.Layers, + cfg.Size, + ); err != nil { return err } return nil diff --git a/store/postgres/queries/registry.go b/store/postgres/queries/registry.go index 49044e9f..d94d5b6b 100644 --- a/store/postgres/queries/registry.go +++ b/store/postgres/queries/registry.go @@ -3,8 +3,8 @@ package queries // insert queries var ( SetDigest = `insert into ` - SetImageManifest = `insert into image_manifest (uuid, namespace, media_type, schema_version) values ($1, $2, $3, $4) - on conflict (namespace) do update set schema_version=3` + SetImageManifest = `insert into image_manifest (uuid, namespace, media_type, schema_version) + values ($1, $2, $3, $4) on conflict (namespace) do update set schema_version=3` SetLayer = `insert into layer (media_type, digest, sky_link, uuid, blob_ids, size) values ($1, $2, $3, $4, $5, $6) on conflict (digest) do nothing;` @@ -13,7 +13,8 @@ var ( values ($1, $2, $3, $4, $5) on conflict (digest) do nothing;` SetConfig = `insert into config (uuid, namespace, reference, digest, sky_link, media_type, layers, size) - values ($1, $2, $3, $4, $5, $6,$7, $8) on conflict (namespace,reference) do update set digest=$4, sky_link=$5, layers=$7;` + values ($1, $2, $3, $4, $5, $6,$7, $8) on conflict (namespace,reference) + do update set digest=$4, sky_link=$5,layers=$7;` ) // select queries From b3fb04a986f1fc90ceeb376d346d92a742d579e2 Mon Sep 17 00:00:00 2001 From: guacamole Date: Tue, 14 Dec 2021 12:10:04 +0530 Subject: [PATCH 11/16] Fix: attempt to fix github actions to include postgres config Signed-off-by: guacamole --- scripts/postgres/OpenRegistry.sql | 66 ++++++++++++++++--------------- scripts/postgres/setup.sh | 4 +- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/scripts/postgres/OpenRegistry.sql b/scripts/postgres/OpenRegistry.sql index 6a9b63dc..5b9027d0 100644 --- a/scripts/postgres/OpenRegistry.sql +++ b/scripts/postgres/OpenRegistry.sql @@ -1,46 +1,48 @@ + CREATE TABLE "users" ( - "id" uuid PRIMARY KEY, - "is_active" boolean, - "username" varchar NOT NULL UNIQUE, - "email" varchar NOT NULL UNIQUE, - "password" varchar, - "created_at" timestamp, - "updated_at" timestamp, - "country_code" int + "id" uuid PRIMARY KEY, + "is_active" boolean, + "username" varchar NOT NULL UNIQUE, + "email" varchar NOT NULL UNIQUE, + "password" varchar, + "created_at" timestamp, + "updated_at" timestamp, + "country_code" int ); CREATE TABLE "config" ( - "uuid" uuid UNIQUE NOT NULL, - "namespace" text, - "reference" text, - "digest" text UNIQUE NOT NULL, - "sky_link" text, - "media_type" text, - "layers" text[], - "size" int, - PRIMARY KEY (namespace,reference) + "uuid" uuid UNIQUE NOT NULL, + "namespace" text NOT NULL, + "reference" text NOT NULL, + "digest" text NOT NULL, + "sky_link" text, + "media_type" text, + "layers" text[], + "size" int, + PRIMARY KEY(namespace, reference) ); CREATE TABLE "blob" ( - "uuid" uuid, - "digest" text PRIMARY KEY, - "sky_link" text, - "start_range" int, - "end_range" int + "uuid" uuid, + "digest" text PRIMARY KEY, + "sky_link" text, + "start_range" int, + "end_range" int ); CREATE TABLE "layer" ( - "uuid" uuid PRIMARY KEY, - "digest" text UNIQUE NOT NULL, - "blob_ids" text[], - "media_type" text, - "sky_link" text, - "size" int + "uuid" uuid PRIMARY KEY, + "digest" text UNIQUE NOT NULL, + "blob_ids" text[], + "media_type" text, + "sky_link" text, + "size" int ); CREATE TABLE "image_manifest" ( - "uuid" uuid PRIMARY KEY, - "namespace" text UNIQUE NOT NULL, - "media_type" text, - "schema_version" int + "uuid" uuid PRIMARY KEY, + "namespace" text UNIQUE NOT NULL, + "media_type" text, + "schema_version" int ); + diff --git a/scripts/postgres/setup.sh b/scripts/postgres/setup.sh index e1d37abb..f15d42ee 100755 --- a/scripts/postgres/setup.sh +++ b/scripts/postgres/setup.sh @@ -1,9 +1,7 @@ #!/bin/bash set -e -/Applications/Postgres.app/Contents/Versions/14/bin/psql -v ON_ERROR_STOP=1 --username "$PGUSER" --dbname "$PGDATABASE" <<-EOSQL - CREATE USER jane_doe; - GRANT ALL PRIVILEGES ON DATABASE open_registry TO jane_doe; +psql -v ON_ERROR_STOP=1 --host="0.0.0.0" --port="5432" --username "$PGUSER" --dbname "$PGDATABASE" <<-EOSQL \c open_registry; \i scripts/postgres/OpenRegistry.sql EOSQL From e1b81b431668b02ce40edbcfccc25e6f11526003 Mon Sep 17 00:00:00 2001 From: guacamole Date: Tue, 14 Dec 2021 12:10:04 +0530 Subject: [PATCH 12/16] Fix: attempt to fix github actions to include postgres config Signed-off-by: guacamole Signed-off-by: jay-dee7 --- .../oci-dist-spec-content-discovery.yml | 8 ++- scripts/postgres/OpenRegistry.sql | 66 ++++++++++--------- scripts/postgres/setup.sh | 4 +- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/.github/workflows/oci-dist-spec-content-discovery.yml b/.github/workflows/oci-dist-spec-content-discovery.yml index 9d3af290..05baee86 100644 --- a/.github/workflows/oci-dist-spec-content-discovery.yml +++ b/.github/workflows/oci-dist-spec-content-discovery.yml @@ -28,7 +28,13 @@ jobs: docker_channel: stable - uses: actions/checkout@v2 - name: Setup PostgreSQL Database - run: sh scripts/postgres/setup.sh + run: | + psql --version + IP=`hostname -I | awk '{print $1}'` + psql -v ON_ERROR_STOP=1 --host=$IP --port=5432 --username $PGUSER --dbname $PGDATABASE <<-EOSQL + \c open_registry; + \i scripts/postgres/OpenRegistry.sql + EOSQL env: PGDATABASE: open_registry PGUSER: postgres diff --git a/scripts/postgres/OpenRegistry.sql b/scripts/postgres/OpenRegistry.sql index 6a9b63dc..5b9027d0 100644 --- a/scripts/postgres/OpenRegistry.sql +++ b/scripts/postgres/OpenRegistry.sql @@ -1,46 +1,48 @@ + CREATE TABLE "users" ( - "id" uuid PRIMARY KEY, - "is_active" boolean, - "username" varchar NOT NULL UNIQUE, - "email" varchar NOT NULL UNIQUE, - "password" varchar, - "created_at" timestamp, - "updated_at" timestamp, - "country_code" int + "id" uuid PRIMARY KEY, + "is_active" boolean, + "username" varchar NOT NULL UNIQUE, + "email" varchar NOT NULL UNIQUE, + "password" varchar, + "created_at" timestamp, + "updated_at" timestamp, + "country_code" int ); CREATE TABLE "config" ( - "uuid" uuid UNIQUE NOT NULL, - "namespace" text, - "reference" text, - "digest" text UNIQUE NOT NULL, - "sky_link" text, - "media_type" text, - "layers" text[], - "size" int, - PRIMARY KEY (namespace,reference) + "uuid" uuid UNIQUE NOT NULL, + "namespace" text NOT NULL, + "reference" text NOT NULL, + "digest" text NOT NULL, + "sky_link" text, + "media_type" text, + "layers" text[], + "size" int, + PRIMARY KEY(namespace, reference) ); CREATE TABLE "blob" ( - "uuid" uuid, - "digest" text PRIMARY KEY, - "sky_link" text, - "start_range" int, - "end_range" int + "uuid" uuid, + "digest" text PRIMARY KEY, + "sky_link" text, + "start_range" int, + "end_range" int ); CREATE TABLE "layer" ( - "uuid" uuid PRIMARY KEY, - "digest" text UNIQUE NOT NULL, - "blob_ids" text[], - "media_type" text, - "sky_link" text, - "size" int + "uuid" uuid PRIMARY KEY, + "digest" text UNIQUE NOT NULL, + "blob_ids" text[], + "media_type" text, + "sky_link" text, + "size" int ); CREATE TABLE "image_manifest" ( - "uuid" uuid PRIMARY KEY, - "namespace" text UNIQUE NOT NULL, - "media_type" text, - "schema_version" int + "uuid" uuid PRIMARY KEY, + "namespace" text UNIQUE NOT NULL, + "media_type" text, + "schema_version" int ); + diff --git a/scripts/postgres/setup.sh b/scripts/postgres/setup.sh index e1d37abb..f15d42ee 100755 --- a/scripts/postgres/setup.sh +++ b/scripts/postgres/setup.sh @@ -1,9 +1,7 @@ #!/bin/bash set -e -/Applications/Postgres.app/Contents/Versions/14/bin/psql -v ON_ERROR_STOP=1 --username "$PGUSER" --dbname "$PGDATABASE" <<-EOSQL - CREATE USER jane_doe; - GRANT ALL PRIVILEGES ON DATABASE open_registry TO jane_doe; +psql -v ON_ERROR_STOP=1 --host="0.0.0.0" --port="5432" --username "$PGUSER" --dbname "$PGDATABASE" <<-EOSQL \c open_registry; \i scripts/postgres/OpenRegistry.sql EOSQL From 1d11ce1c130bd9f4ba4d1dad3fd3fd36f91b71cb Mon Sep 17 00:00:00 2001 From: jay-dee7 Date: Tue, 14 Dec 2021 13:21:23 +0530 Subject: [PATCH 13/16] Chore: added migrate cli and changed schema load config Signed-off-by: jay-dee7 --- .../oci-dist-spec-content-discovery.yml | 28 +++++++-------- .../oci-dist-spec-content-management.yml | 36 +++++++++++++++---- .github/workflows/oci-dist-spec-pull.yml | 34 +++++++++++++++--- .github/workflows/oci-dist-spec-push.yml | 36 +++++++++++++++---- config/config.go | 29 ++++++++++----- .../000001_create_users_table.down.sql | 1 + .../000001_create_users_table.up.sql | 10 ++++++ .../000002_create_blob_table.down.sql | 1 + db/migrations/000002_create_blob_table.up.sql | 8 +++++ .../000003_create_layer_table.down.sql | 1 + .../000003_create_layer_table.up.sql | 8 +++++ .../000004_create_config_table.down.sql | 1 + .../000004_create_config_table.up.sql | 12 +++++++ ...00005_create_image_manifest_table.down.sql | 1 + .../000005_create_image_manifest_table.up.sql | 7 ++++ env-vars.example | 5 +++ main.go | 3 +- 17 files changed, 180 insertions(+), 41 deletions(-) create mode 100644 db/migrations/000001_create_users_table.down.sql create mode 100644 db/migrations/000001_create_users_table.up.sql create mode 100644 db/migrations/000002_create_blob_table.down.sql create mode 100644 db/migrations/000002_create_blob_table.up.sql create mode 100644 db/migrations/000003_create_layer_table.down.sql create mode 100644 db/migrations/000003_create_layer_table.up.sql create mode 100644 db/migrations/000004_create_config_table.down.sql create mode 100644 db/migrations/000004_create_config_table.up.sql create mode 100644 db/migrations/000005_create_image_manifest_table.down.sql create mode 100644 db/migrations/000005_create_image_manifest_table.up.sql diff --git a/.github/workflows/oci-dist-spec-content-discovery.yml b/.github/workflows/oci-dist-spec-content-discovery.yml index 05baee86..24556be8 100644 --- a/.github/workflows/oci-dist-spec-content-discovery.yml +++ b/.github/workflows/oci-dist-spec-content-discovery.yml @@ -1,4 +1,4 @@ -name: OCI Spec - Content Discovery +name: OCI Distribution Spec on: pull_request: @@ -10,8 +10,9 @@ jobs: postgres: image: postgres env: + PGUSER: postgres POSTGRES_DB: open_registry - POSTGRES_PASSWORD: postgres + POSTGRES_PASSWORD: Qwerty@123 POSTGRES_USER: postgres options: >- --health-cmd pg_isready @@ -21,24 +22,21 @@ jobs: ports: - 5432:5432 steps: - - name: set up docker - uses: docker-practice/actions-setup-docker@0.0.1 - with: - docker_version: 18.09 - docker_channel: stable - uses: actions/checkout@v2 + - name: Install Migrate CLI + run: | + curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz + sudo mv migrate /usr/bin/migrate + sudo netstat -tulnp - name: Setup PostgreSQL Database run: | - psql --version IP=`hostname -I | awk '{print $1}'` - psql -v ON_ERROR_STOP=1 --host=$IP --port=5432 --username $PGUSER --dbname $PGDATABASE <<-EOSQL - \c open_registry; - \i scripts/postgres/OpenRegistry.sql - EOSQL + POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable + migrate -database ${POSTGRESQL_URL} -path db/migrations up env: PGDATABASE: open_registry + PGPASSWORD: Qwerty@123 PGUSER: postgres - PGPASSWORD: postgres - name: start distribution server run: | IP=`hostname -I | awk '{print $1}'` @@ -47,8 +45,10 @@ jobs: DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)" docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" . sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example + sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example echo CI_SYS_ADDR=$IP:5000 >> env-vars.example - docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -i "${DISTRIBUTION_REF}" + echo REGISTRY_STORAGE_DELETE_ENABLED=true + docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}" sleep 5 curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup" - name: Run OCI Distribution Spec conformance tests diff --git a/.github/workflows/oci-dist-spec-content-management.yml b/.github/workflows/oci-dist-spec-content-management.yml index 538c67b2..b5937b62 100644 --- a/.github/workflows/oci-dist-spec-content-management.yml +++ b/.github/workflows/oci-dist-spec-content-management.yml @@ -1,4 +1,4 @@ -name: OCI Spec - Content Management +name: OCI Distribution Spec on: pull_request: @@ -6,13 +6,37 @@ on: jobs: content-management: runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + PGUSER: postgres + POSTGRES_DB: open_registry + POSTGRES_PASSWORD: Qwerty@123 + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: - - name: set up docker - uses: docker-practice/actions-setup-docker@0.0.1 - with: - docker_version: 18.09 - docker_channel: stable - uses: actions/checkout@v2 + - name: Install Migrate CLI + run: | + curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz + sudo mv migrate /usr/bin/migrate + sudo netstat -tulnp + - name: Setup PostgreSQL Database + run: | + IP=`hostname -I | awk '{print $1}'` + POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable + migrate -database ${POSTGRESQL_URL} -path db/migrations up + env: + PGDATABASE: open_registry + PGPASSWORD: Qwerty@123 + PGUSER: postgres - name: start distribution server run: | IP=`hostname -I | awk '{print $1}'` diff --git a/.github/workflows/oci-dist-spec-pull.yml b/.github/workflows/oci-dist-spec-pull.yml index efec79c6..c1636df5 100644 --- a/.github/workflows/oci-dist-spec-pull.yml +++ b/.github/workflows/oci-dist-spec-pull.yml @@ -6,13 +6,37 @@ on: jobs: pull: runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + PGUSER: postgres + POSTGRES_DB: open_registry + POSTGRES_PASSWORD: Qwerty@123 + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: - - name: set up docker - uses: docker-practice/actions-setup-docker@0.0.1 - with: - docker_version: 18.09 - docker_channel: stable - uses: actions/checkout@v2 + - name: Install Migrate CLI + run: | + curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz + sudo mv migrate /usr/bin/migrate + sudo netstat -tulnp + - name: Setup PostgreSQL Database + run: | + IP=`hostname -I | awk '{print $1}'` + POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable + migrate -database ${POSTGRESQL_URL} -path db/migrations up + env: + PGDATABASE: open_registry + PGPASSWORD: Qwerty@123 + PGUSER: postgres - name: start distribution server run: | IP=`hostname -I | awk '{print $1}'` diff --git a/.github/workflows/oci-dist-spec-push.yml b/.github/workflows/oci-dist-spec-push.yml index 885863a3..06887e4d 100644 --- a/.github/workflows/oci-dist-spec-push.yml +++ b/.github/workflows/oci-dist-spec-push.yml @@ -1,4 +1,4 @@ -name: OCI Spec - Push +name: OCI Distribution Spec on: pull_request: @@ -6,13 +6,37 @@ on: jobs: push: runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + PGUSER: postgres + POSTGRES_DB: open_registry + POSTGRES_PASSWORD: Qwerty@123 + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: - - name: set up docker - uses: docker-practice/actions-setup-docker@0.0.1 - with: - docker_version: 18.09 - docker_channel: stable - uses: actions/checkout@v2 + - name: Install Migrate CLI + run: | + curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz + sudo mv migrate /usr/bin/migrate + sudo netstat -tulnp + - name: Setup PostgreSQL Database + run: | + IP=`hostname -I | awk '{print $1}'` + POSTGRESQL_URL=postgres://$PGUSER:$PGPASSWORD@$IP:5432/$PGDATABASE?sslmode=disable + migrate -database ${POSTGRESQL_URL} -path db/migrations up + env: + PGDATABASE: open_registry + PGPASSWORD: Qwerty@123 + PGUSER: postgres - name: start distribution server run: | IP=`hostname -I | awk '{print $1}'` diff --git a/config/config.go b/config/config.go index 9cc33f3b..225a447b 100644 --- a/config/config.go +++ b/config/config.go @@ -12,6 +12,7 @@ import ( type ( RegistryConfig struct { + StoreConfig *StoreConfig `mapstructure:"store_config"` AuthConfig AuthConfig `mapstructure:"auth_config"` LogConfig LogConfig `mapstructure:"log_config"` SkynetConfig SkynetConfig `mapstructure:"skynet_config"` @@ -57,17 +58,22 @@ func (r *RegistryConfig) Address() string { } func NewStoreConfig() (*StoreConfig, error) { - return &StoreConfig{ - User: "postgres", - Password: "Qwerty@123", - Database: "open_registry", - Host: "0.0.0.0", - Port: 5432, - }, nil + viper.SetEnvPrefix("OPEN_REGISTRY") + viper.AutomaticEnv() + + storeConfig := &StoreConfig{ + User: viper.GetString("DB_USER"), + Password: viper.GetString("DB_PASSWORD"), + Database: viper.GetString("DB_NAME"), + Host: viper.GetString("DB_HOST"), + Port: viper.GetInt("DB_PORT"), + } + + return storeConfig, nil } func (sc *StoreConfig) Endpoint() string { - return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?pool_max_conns=1000", + return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?pool_max_conns=1000&sslmode=disable", sc.User, sc.Password, sc.Host, sc.Port, sc.Database) } @@ -105,6 +111,13 @@ func LoadFromENV() (*RegistryConfig, error) { Username: viper.GetString("LOG_SERVICE_USER"), Password: viper.GetString("LOG_SERVICE_PASSWORD"), }, + StoreConfig: &StoreConfig{ + User: viper.GetString("DB_USER"), + Host: viper.GetString("DB_HOST"), + Port: viper.GetInt("DB_PORT"), + Password: viper.GetString("DB_PASSWORD"), + Database: viper.GetString("DB_NAME"), + }, } for _, service := range strings.Split(viper.GetString("SUPPORTED_SERVICES"), ",") { diff --git a/db/migrations/000001_create_users_table.down.sql b/db/migrations/000001_create_users_table.down.sql new file mode 100644 index 00000000..c99ddcdc --- /dev/null +++ b/db/migrations/000001_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS users; diff --git a/db/migrations/000001_create_users_table.up.sql b/db/migrations/000001_create_users_table.up.sql new file mode 100644 index 00000000..4b7ed546 --- /dev/null +++ b/db/migrations/000001_create_users_table.up.sql @@ -0,0 +1,10 @@ +CREATE TABLE "users" ( + "id" uuid PRIMARY KEY, + "is_active" boolean, + "username" varchar NOT NULL UNIQUE, + "email" varchar NOT NULL UNIQUE, + "password" varchar, + "created_at" timestamp, + "updated_at" timestamp, + "country_code" int +); diff --git a/db/migrations/000002_create_blob_table.down.sql b/db/migrations/000002_create_blob_table.down.sql new file mode 100644 index 00000000..a662c0ab --- /dev/null +++ b/db/migrations/000002_create_blob_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS blob; diff --git a/db/migrations/000002_create_blob_table.up.sql b/db/migrations/000002_create_blob_table.up.sql new file mode 100644 index 00000000..72e690a5 --- /dev/null +++ b/db/migrations/000002_create_blob_table.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE "blob" ( + "uuid" uuid, + "digest" text PRIMARY KEY, + "sky_link" text, + "start_range" int, + "end_range" int +); + diff --git a/db/migrations/000003_create_layer_table.down.sql b/db/migrations/000003_create_layer_table.down.sql new file mode 100644 index 00000000..8051de1e --- /dev/null +++ b/db/migrations/000003_create_layer_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS layer; diff --git a/db/migrations/000003_create_layer_table.up.sql b/db/migrations/000003_create_layer_table.up.sql new file mode 100644 index 00000000..8a3bd4f6 --- /dev/null +++ b/db/migrations/000003_create_layer_table.up.sql @@ -0,0 +1,8 @@ +CREATE TABLE "layer" ( + "uuid" uuid PRIMARY KEY, + "digest" text UNIQUE NOT NULL, + "blob_ids" text[], + "media_type" text, + "sky_link" text, + "size" int +); diff --git a/db/migrations/000004_create_config_table.down.sql b/db/migrations/000004_create_config_table.down.sql new file mode 100644 index 00000000..2efbbee0 --- /dev/null +++ b/db/migrations/000004_create_config_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS config; diff --git a/db/migrations/000004_create_config_table.up.sql b/db/migrations/000004_create_config_table.up.sql new file mode 100644 index 00000000..32d64c4b --- /dev/null +++ b/db/migrations/000004_create_config_table.up.sql @@ -0,0 +1,12 @@ +CREATE TABLE "config" ( + "uuid" uuid UNIQUE NOT NULL, + "namespace" text NOT NULL, + "reference" text NOT NULL, + "digest" text NOT NULL, + "sky_link" text, + "media_type" text, + "layers" text[], + "size" int, + PRIMARY KEY(namespace, reference) +); + diff --git a/db/migrations/000005_create_image_manifest_table.down.sql b/db/migrations/000005_create_image_manifest_table.down.sql new file mode 100644 index 00000000..17e896ba --- /dev/null +++ b/db/migrations/000005_create_image_manifest_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS image_manifest; diff --git a/db/migrations/000005_create_image_manifest_table.up.sql b/db/migrations/000005_create_image_manifest_table.up.sql new file mode 100644 index 00000000..02f75671 --- /dev/null +++ b/db/migrations/000005_create_image_manifest_table.up.sql @@ -0,0 +1,7 @@ +CREATE TABLE "image_manifest" ( + "uuid" uuid PRIMARY KEY, + "namespace" text UNIQUE NOT NULL, + "media_type" text, + "schema_version" int +); + diff --git a/env-vars.example b/env-vars.example index 8ccbf80c..bf958746 100644 --- a/env-vars.example +++ b/env-vars.example @@ -11,3 +11,8 @@ OPEN_REGISTRY_LOG_SERVICE_NAME=grafana-loki OPEN_REGISTRY_LOG_SERVICE_HOST=http://0.0.0.0:9880/app.log OPEN_REGISTRY_LOG_SERVICE_AUTH_KIND=basic OPEN_REGISTRY_SKYNET_API_KEY="" +OPEN_REGISTRY_DB_HOST=0.0.0.0 +OPEN_REGISTRY_DB_PORT=5432 +OPEN_REGISTRY_DB_USER=postgres +OPEN_REGISTRY_DB_PASSWORD=Qwerty@123 +OPEN_REGISTRY_DB_NAME=open_registry diff --git a/main.go b/main.go index a97a7bf9..87207ebf 100644 --- a/main.go +++ b/main.go @@ -32,8 +32,7 @@ func main() { } defer localCache.Close() - pgConfig, _ := config.NewStoreConfig() - pgStore, err := postgres.New(pgConfig) + pgStore, err := postgres.New(cfg.StoreConfig) if err != nil { color.Red("error here: %s", err.Error()) return From bcb17d3d0dc24746ae8d4ad84b39eaef37dd3d1b Mon Sep 17 00:00:00 2001 From: jay-dee7 Date: Tue, 14 Dec 2021 20:10:08 +0530 Subject: [PATCH 14/16] Chore: Fix Github Actions for postgres-implementation Signed-off-by: jay-dee7 --- .github/workflows/oci-dist-spec-content-management.yml | 4 +++- .github/workflows/oci-dist-spec-pull.yml | 6 ++++-- .github/workflows/oci-dist-spec-push.yml | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/oci-dist-spec-content-management.yml b/.github/workflows/oci-dist-spec-content-management.yml index b5937b62..40dbe25b 100644 --- a/.github/workflows/oci-dist-spec-content-management.yml +++ b/.github/workflows/oci-dist-spec-content-management.yml @@ -45,8 +45,10 @@ jobs: DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)" docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" . sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example + sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example echo CI_SYS_ADDR=$IP:5000 >> env-vars.example - docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}" + echo REGISTRY_STORAGE_DELETE_ENABLED=true + docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}" sleep 5 curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup" - name: Run OCI Distribution Spec conformance tests diff --git a/.github/workflows/oci-dist-spec-pull.yml b/.github/workflows/oci-dist-spec-pull.yml index c1636df5..f066a06e 100644 --- a/.github/workflows/oci-dist-spec-pull.yml +++ b/.github/workflows/oci-dist-spec-pull.yml @@ -1,4 +1,4 @@ -name: OCI Spec - Pull +name: OCI Distribution Spec on: pull_request: @@ -45,8 +45,10 @@ jobs: DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)" docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" . sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example + sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example echo CI_SYS_ADDR=$IP:5000 >> env-vars.example - docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}" + echo REGISTRY_STORAGE_DELETE_ENABLED=true + docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}" sleep 5 curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup" - name: Run OCI Distribution Spec conformance tests diff --git a/.github/workflows/oci-dist-spec-push.yml b/.github/workflows/oci-dist-spec-push.yml index 06887e4d..44fd4312 100644 --- a/.github/workflows/oci-dist-spec-push.yml +++ b/.github/workflows/oci-dist-spec-push.yml @@ -45,8 +45,10 @@ jobs: DISTRIBUTION_REF="local-distribution:v$(date +%Y%m%d%H%M%S)" docker build -f ./Dockerfile -t "${DISTRIBUTION_REF}" . sed -in "s/OPEN_REGISTRY_ENVIRONMENT=local/OPEN_REGISTRY_ENVIRONMENT=ci/g" env-vars.example + sed -in "s/OPEN_REGISTRY_DB_HOST=0.0.0.0/OPEN_REGISTRY_DB_HOST=${IP}/g" env-vars.example echo CI_SYS_ADDR=$IP:5000 >> env-vars.example - docker run --rm -p 5000:5000 --env-file ./env-vars.example -e REGISTRY_STORAGE_DELETE_ENABLED=true -d "${DISTRIBUTION_REF}" + echo REGISTRY_STORAGE_DELETE_ENABLED=true + docker run --rm -p 5000:5000 --env-file ./env-vars.example -d "${DISTRIBUTION_REF}" sleep 5 curl -XPOST -d ${{ secrets.OPENREGISTRY_SIGNUP_PAYLOAD }} "http://${IP}:5000/auth/signup" - name: Run OCI Distribution Spec conformance tests From 13998245682ffebb77732873eda3e42530f16d41 Mon Sep 17 00:00:00 2001 From: guacamole Date: Wed, 15 Dec 2021 11:09:08 +0530 Subject: [PATCH 15/16] Fix: linting errors caused by databse updates Signed-off-by: guacamole --- auth/auth.go | 13 +++++++++---- registry/v2/registry.go | 2 +- registry/v2/types.go | 6 +++--- store/postgres/container_image.go | 12 ++++++------ store/postgres/queries/registry.go | 3 ++- store/postgres/queries/users.go | 4 +++- 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/auth/auth.go b/auth/auth.go index 21706b06..76fbd64a 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -20,13 +20,18 @@ type Authentication interface { type auth struct { pgStore postgres.PersistentStore - store cache.Store - c *config.RegistryConfig - logger telemetry.Logger + store cache.Store + c *config.RegistryConfig + logger telemetry.Logger } // New is the constructor function returns an Authentication implementation -func New(s cache.Store, c *config.RegistryConfig, pgStore postgres.PersistentStore, logger telemetry.Logger) Authentication { +func New( + s cache.Store, + c *config.RegistryConfig, + pgStore postgres.PersistentStore, + logger telemetry.Logger, +) Authentication { a := &auth{store: s, c: c, pgStore: pgStore, logger: logger} return a } diff --git a/registry/v2/registry.go b/registry/v2/registry.go index 0183577f..14d30de9 100644 --- a/registry/v2/registry.go +++ b/registry/v2/registry.go @@ -762,7 +762,7 @@ func (r *registry) DeleteLayer(ctx echo.Context) error { return ctx.JSONBlob(http.StatusInternalServerError, bz) } - for i, _ := range blobs { + for i := range blobs { //if err = r.localCache.DeleteDigest(dig); err != nil { if err = r.store.DeleteBlobV2(ctx.Request().Context(), txnOp, blobs[i]); err != nil { logMsg := echo.Map{ diff --git a/registry/v2/types.go b/registry/v2/types.go index 647d903e..60fd2a02 100644 --- a/registry/v2/types.go +++ b/registry/v2/types.go @@ -1,16 +1,16 @@ package registry import ( - "github.com/containerish/OpenRegistry/store/postgres" - "github.com/jackc/pgx/v4" - "github.com/rs/zerolog" "sync" "time" "github.com/containerish/OpenRegistry/cache" "github.com/containerish/OpenRegistry/skynet" + "github.com/containerish/OpenRegistry/store/postgres" "github.com/containerish/OpenRegistry/telemetry" + "github.com/jackc/pgx/v4" "github.com/labstack/echo/v4" + "github.com/rs/zerolog" ) /* diff --git a/store/postgres/container_image.go b/store/postgres/container_image.go index bbfe2b7b..fb227418 100644 --- a/store/postgres/container_image.go +++ b/store/postgres/container_image.go @@ -262,22 +262,22 @@ func (p *pg) DeleteManifestOrTag(ctx context.Context, txn pgx.Tx, reference stri } func (p *pg) NewTxn(ctx context.Context) (pgx.Tx, error) { - childCtx, _ := context.WithTimeout(context.Background(), time.Minute*30) - //defer cancel() + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute*30) + defer cancel() return p.conn.Begin(childCtx) } func (p *pg) Abort(ctx context.Context, txn pgx.Tx) error { - childCtx, _ := context.WithTimeout(context.Background(), time.Minute*30) - //defer cancel() + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute*30) + defer cancel() return txn.Rollback(childCtx) } func (p *pg) Commit(ctx context.Context, txn pgx.Tx) error { - childCtx, _ := context.WithTimeout(context.Background(), time.Minute*30) - //defer cancel() + childCtx, cancel := context.WithTimeout(context.Background(), time.Minute*30) + defer cancel() return txn.Commit(childCtx) } diff --git a/store/postgres/queries/registry.go b/store/postgres/queries/registry.go index d94d5b6b..4787f312 100644 --- a/store/postgres/queries/registry.go +++ b/store/postgres/queries/registry.go @@ -1,8 +1,9 @@ +//nolint package queries // insert queries var ( - SetDigest = `insert into ` + // SetImageManifest = `insert into image_manifest (uuid, namespace, media_type, schema_version) values ($1, $2, $3, $4) on conflict (namespace) do update set schema_version=3` SetLayer = `insert into layer (media_type, digest, sky_link, uuid, blob_ids, size) diff --git a/store/postgres/queries/users.go b/store/postgres/queries/users.go index 5e5b5598..f3c3d40e 100644 --- a/store/postgres/queries/users.go +++ b/store/postgres/queries/users.go @@ -1,9 +1,11 @@ +//nolint package queries var ( AddUser = `insert into users (id, is_active, username, email, password, created_at, updated_at) values ($1, $2, $3, $4, $5, $6, $7);` - GetUser = `select is_active, username, email, password, created_at, updated_at from users where email=$1 or username=$1;` + GetUser = `select is_active, username, email, password, created_at, updated_at from users where email=$1 + or username=$1;` UpdateUser = `update user set username = $1, email = $2, password = $3, updated_at = $4 where username = $5;` DeleteUser = `delete from user where username = $1;` GetAllEmails = `select email from users;` From ab4024411cd704456877f6a950b35f42e7352a1b Mon Sep 17 00:00:00 2001 From: guacamole Date: Sun, 19 Dec 2021 16:10:13 +0530 Subject: [PATCH 16/16] Chore: removed comments and unused code Signed-off-by: guacamole --- .../oci-dist-spec-content-discovery.yml | 1 - .../oci-dist-spec-content-management.yml | 1 - .github/workflows/oci-dist-spec-pull.yml | 1 - .github/workflows/oci-dist-spec-push.yml | 1 - auth/signup.go | 16 ---------------- auth/validate_user.go | 11 ----------- scripts/postgres/setup.sh | 7 ------- types/types.go | 6 ------ 8 files changed, 44 deletions(-) delete mode 100755 scripts/postgres/setup.sh diff --git a/.github/workflows/oci-dist-spec-content-discovery.yml b/.github/workflows/oci-dist-spec-content-discovery.yml index 24556be8..a0a7e4f1 100644 --- a/.github/workflows/oci-dist-spec-content-discovery.yml +++ b/.github/workflows/oci-dist-spec-content-discovery.yml @@ -27,7 +27,6 @@ jobs: run: | curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz sudo mv migrate /usr/bin/migrate - sudo netstat -tulnp - name: Setup PostgreSQL Database run: | IP=`hostname -I | awk '{print $1}'` diff --git a/.github/workflows/oci-dist-spec-content-management.yml b/.github/workflows/oci-dist-spec-content-management.yml index 40dbe25b..8a7f7101 100644 --- a/.github/workflows/oci-dist-spec-content-management.yml +++ b/.github/workflows/oci-dist-spec-content-management.yml @@ -27,7 +27,6 @@ jobs: run: | curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz sudo mv migrate /usr/bin/migrate - sudo netstat -tulnp - name: Setup PostgreSQL Database run: | IP=`hostname -I | awk '{print $1}'` diff --git a/.github/workflows/oci-dist-spec-pull.yml b/.github/workflows/oci-dist-spec-pull.yml index f066a06e..ed3e33df 100644 --- a/.github/workflows/oci-dist-spec-pull.yml +++ b/.github/workflows/oci-dist-spec-pull.yml @@ -27,7 +27,6 @@ jobs: run: | curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz sudo mv migrate /usr/bin/migrate - sudo netstat -tulnp - name: Setup PostgreSQL Database run: | IP=`hostname -I | awk '{print $1}'` diff --git a/.github/workflows/oci-dist-spec-push.yml b/.github/workflows/oci-dist-spec-push.yml index 44fd4312..1b4d39a6 100644 --- a/.github/workflows/oci-dist-spec-push.yml +++ b/.github/workflows/oci-dist-spec-push.yml @@ -27,7 +27,6 @@ jobs: run: | curl -L https://github.com/golang-migrate/migrate/releases/download/v4.15.1/migrate.linux-amd64.tar.gz | tar xvz sudo mv migrate /usr/bin/migrate - sudo netstat -tulnp - name: Setup PostgreSQL Database run: | IP=`hostname -I | awk '{print $1}'` diff --git a/auth/signup.go b/auth/signup.go index 3404381a..c2bb80e6 100644 --- a/auth/signup.go +++ b/auth/signup.go @@ -186,8 +186,6 @@ func (a *auth) SignUp(ctx echo.Context) error { } err = a.pgStore.AddUser(ctx.Request().Context(), newUser) - - // bz, err = json.Marshal(u) if err != nil { ctx.Set(types.HttpEndpointErrorKey, err.Error()) return ctx.JSON(http.StatusInternalServerError, echo.Map{ @@ -195,20 +193,6 @@ func (a *auth) SignUp(ctx echo.Context) error { }) } - // key := fmt.Sprintf("%s/%s", UserNameSpace, u.Username) - // if err := a.store.Set([]byte(key), bz); err != nil { - // return ctx.JSON(http.StatusInternalServerError, echo.Map{ - // "error": err.Error(), - // }) - // } - - // key = fmt.Sprintf("%s/%s", UserNameSpace, u.Email) - // if err := a.store.Set([]byte(key), bz); err != nil { - // return ctx.JSON(http.StatusInternalServerError, echo.Map{ - // "error": err.Error(), - // }) - // } - return ctx.JSON(http.StatusCreated, echo.Map{ "message": "user successfully created", }) diff --git a/auth/validate_user.go b/auth/validate_user.go index e28878b4..32f303ea 100644 --- a/auth/validate_user.go +++ b/auth/validate_user.go @@ -13,17 +13,6 @@ func (a *auth) validateUser(username, password string) (map[string]interface{}, return nil, fmt.Errorf("Email/Password cannot be empty") } - //key := fmt.Sprintf("%s/%s", UserNameSpace, username) - //bz, err := a.store.Get([]byte(key)) - //if err != nil { - // return nil, err - //} - // - //var userFromDb User - //if err := json.Unmarshal(bz, &userFromDb); err != nil { - // return nil, err - //} - userFromDb, err := a.pgStore.GetUser(context.Background(), username) if err != nil { return nil, err diff --git a/scripts/postgres/setup.sh b/scripts/postgres/setup.sh deleted file mode 100755 index f15d42ee..00000000 --- a/scripts/postgres/setup.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e -psql -v ON_ERROR_STOP=1 --host="0.0.0.0" --port="5432" --username "$PGUSER" --dbname "$PGDATABASE" <<-EOSQL - \c open_registry; - \i scripts/postgres/OpenRegistry.sql -EOSQL diff --git a/types/types.go b/types/types.go index 1a74c188..ce13faf4 100644 --- a/types/types.go +++ b/types/types.go @@ -11,12 +11,6 @@ type ( Manifest ImageManifest } - //Manifest struct { - // SkynetLink string - // Reference string - // Digest string - //} - ImageManifest struct { MediaType string `json:"mediaType"` Layers []*Layer `json:"layers"`