Skip to content

backport-6.5: migrate to thanos objstore backend #537

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
btaani wants to merge 4 commits into
base: release-6.5
Choose a base branch
from
Open

backport-6.5: migrate to thanos objstore backend #537

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
acaa0a0
feat(operator): migrate to thanos objstore backend (#20475)
btaani Apr 10, 2026
263548a
fix build_test.go
btaani Aug 18, 2026
cd4c758
fix(operator): Ruler doesn't evaluate rules due to use_thanos_objstor…
JoaoBraveCoding May 5, 2026
1fc79da
fix(operator): Use thanos objstore CA flags for S3 and Swift storage …
shirzady1934 Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 43 additions & 16 deletions operator/internal/handlers/internal/storage/secrets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,9 @@ var (
errSecretUnknownCredentialMode = errors.New("unknown credential mode")

errAzureManagedIdentityNoOverride = errors.New("when in managed mode, storage secret can not contain credentials")
errAzureInvalidEnvironment = errors.New("azure environment invalid (valid values: AzureGlobal, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment)")
errAzureInvalidEnvironment = errors.New("azure environment invalid (valid values: AzureGlobal, AzurePublicCloud, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment)")
errAzureInvalidAccountKey = errors.New("azure account key is not valid base64")
errAzureInvalidEndpointSuffix = errors.New("azure endpoint suffix invalid")

errS3EndpointUnparseable = errors.New("can not parse S3 endpoint as URL")
errS3EndpointNoURL = errors.New("endpoint for S3 must be an HTTP or HTTPS URL")
Expand All @@ -50,11 +51,12 @@ var (
errGCPWrongCredentialSourceFile = errors.New("credential source in secret needs to point to token file")
errGCPInvalidCredentialsFile = errors.New("gcp credentials file contains invalid fields")

azureValidEnvironments = map[string]bool{
"AzureGlobal": true,
"AzureChinaCloud": true,
"AzureGermanCloud": true,
"AzureUSGovernment": true,
azureEnvironmentEndpointSuffix = map[string]string{
"AzureGlobal": "blob.core.windows.net",
"AzurePublicCloud": "blob.core.windows.net",
"AzureChinaCloud": "blob.core.chinacloudapi.cn",
"AzureGermanCloud": "blob.core.cloudapi.de",
"AzureUSGovernment": "blob.core.usgovcloudapi.net",
}
)

Expand Down Expand Up @@ -239,14 +241,24 @@ func hashSecretData(s *corev1.Secret) (string, error) {
func extractAzureConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMode) (*storage.AzureStorageConfig, error) {
// Extract and validate mandatory fields
env := string(s.Data[storage.KeyAzureEnvironmentName])
if env == "" {
return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureEnvironmentName)
endpointSuffix := string(s.Data[storage.KeyAzureStorageEndpointSuffix])
if env == "" && endpointSuffix == "" {
return nil, fmt.Errorf("%w: either %s or %s should be set", errSecretMissingField, storage.KeyAzureEnvironmentName, storage.KeyAzureStorageEndpointSuffix)
}

if !azureValidEnvironments[env] {
envEndpointSuffix, ok := azureEnvironmentEndpointSuffix[env]
if env != "" && !ok {
return nil, fmt.Errorf("%w: %s", errAzureInvalidEnvironment, env)
}

if endpointSuffix == "" {
endpointSuffix = envEndpointSuffix
}

if !endpointSuffixExists(azureEnvironmentEndpointSuffix, endpointSuffix) {
return nil, fmt.Errorf("%w: %s", errAzureInvalidEndpointSuffix, endpointSuffix)
}

accountName := s.Data[storage.KeyAzureStorageAccountName]
if len(accountName) == 0 {
return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageAccountName)
Expand All @@ -263,15 +275,13 @@ func extractAzureConfigSecret(s *corev1.Secret, credentialMode lokiv1.Credential
}

// Extract and validate optional fields
endpointSuffix := s.Data[storage.KeyAzureStorageEndpointSuffix]
audience := s.Data[storage.KeyAzureAudience]

if !workloadIdentity && len(audience) > 0 {
return nil, fmt.Errorf("%w: %s", errSecretFieldNotAllowed, storage.KeyAzureAudience)
}

return &storage.AzureStorageConfig{
Env: env,
Container: string(container),
EndpointSuffix: string(endpointSuffix),
Audience: string(audience),
Expand Down Expand Up @@ -405,7 +415,7 @@ func extractS3ConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMod

var (
// Fields related with static authentication
endpoint = s.Data[storage.KeyAWSEndpoint]
endpoint = string(s.Data[storage.KeyAWSEndpoint])
id = s.Data[storage.KeyAWSAccessKeyID]
secret = s.Data[storage.KeyAWSAccessKeySecret]
// Fields related with STS authentication
Expand All @@ -417,7 +427,7 @@ func extractS3ConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMod

// Determine if we should use path style URLs for S3
// default to false for non-AWS endpoints
forcePathStyle := !strings.HasSuffix(string(endpoint), awsEndpointSuffix)
forcePathStyle := !strings.HasSuffix(endpoint, awsEndpointSuffix)
// Check if the user has specified forcepathstyle
if configForcePathStyle, ok := s.Data[storage.KeyAWSForcePathStyle]; ok {
strForcePathStyle := string(configForcePathStyle)
Expand Down Expand Up @@ -454,13 +464,20 @@ func extractS3ConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMod
if len(region) == 0 {
return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAWSRegion)
}

return cfg, nil
case lokiv1.CredentialModeStatic:
cfg.Endpoint = string(endpoint)

if err := validateS3Endpoint(string(endpoint), string(region)); err != nil {
if err := validateS3Endpoint(endpoint, string(region)); err != nil {
return nil, err
}
parsedURL, err := url.Parse(endpoint)
if err != nil {
return nil, fmt.Errorf("%w:%s", errS3EndpointUnparseable, storage.KeyAWSEndpoint)
}

cfg.Endpoint = parsedURL.Host
cfg.Insecure = strings.HasPrefix(endpoint, "http://")

if len(id) == 0 {
return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAWSAccessKeyID)
}
Expand All @@ -477,6 +494,7 @@ func extractS3ConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMod
if len(region) == 0 {
return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAWSRegion)
}

return cfg, nil
default:
return nil, fmt.Errorf("%w: %s", errSecretUnknownCredentialMode, credentialMode)
Expand Down Expand Up @@ -634,3 +652,12 @@ func extractAlibabaCloudConfigSecret(s *corev1.Secret) (*storage.AlibabaCloudSto
Bucket: string(bucket),
}, nil
}

func endpointSuffixExists(m map[string]string, value string) bool {
for _, v := range m {
if v == value {
return true
}
}
return false
}
52 changes: 44 additions & 8 deletions operator/internal/handlers/internal/storage/secrets_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,44 @@ func TestAzureExtract(t *testing.T) {
}
table := []test{
{
name: "missing environment",
name: "missing environment and endpoint_suffix",
secret: &corev1.Secret{},
wantError: "missing secret field: environment",
wantError: "missing secret field: either environment or endpoint_suffix should be set",
},
{
name: "missing only endpoint_suffix",
secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "test"},
Data: map[string][]byte{
"environment": []byte("AzureGlobal"),
"container": []byte("this,that"),
"account_name": []byte("test-account-name"),
"account_key": []byte("dGVzdC1hY2NvdW50LWtleQ=="),
},
},
wantCredentialMode: lokiv1.CredentialModeStatic,
},
{
name: "missing only environment",
secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "test"},
Data: map[string][]byte{
"endpoint_suffix": []byte("blob.core.windows.net"),
"container": []byte("this,that"),
"account_name": []byte("test-account-name"),
"account_key": []byte("dGVzdC1hY2NvdW50LWtleQ=="),
},
},
wantCredentialMode: lokiv1.CredentialModeStatic,
},
{
name: "invalid endpoint_suffix",
secret: &corev1.Secret{
Data: map[string][]byte{
"endpoint_suffix": []byte("invalid-endpoint-suffix"),
},
},
wantError: "azure endpoint suffix invalid: invalid-endpoint-suffix",
},
{
name: "invalid environment",
Expand All @@ -93,7 +128,7 @@ func TestAzureExtract(t *testing.T) {
"environment": []byte("invalid-environment"),
},
},
wantError: "azure environment invalid (valid values: AzureGlobal, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment): invalid-environment",
wantError: "azure environment invalid (valid values: AzureGlobal, AzurePublicCloud, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment): invalid-environment",
},
{
name: "missing account_name",
Expand Down Expand Up @@ -245,7 +280,7 @@ func TestAzureExtract(t *testing.T) {
"container": []byte("this,that"),
"account_name": []byte("id"),
"account_key": []byte("dGVzdC1hY2NvdW50LWtleQ=="), // test-account-key
"endpoint_suffix": []byte("suffix"),
"endpoint_suffix": []byte("blob.core.windows.net"),
},
},
wantCredentialMode: lokiv1.CredentialModeStatic,
Expand Down Expand Up @@ -702,7 +737,7 @@ func TestS3Extract_ForcePathStyle(t *testing.T) {
},
},
wantOptions: &storage.S3StorageConfig{
Endpoint: "https://s3.region.amazonaws.com",
Endpoint: "s3.region.amazonaws.com",
Region: "region",
Buckets: "this,that",
ForcePathStyle: false, // defaults to virtual style for AWS endpoints
Expand All @@ -721,10 +756,11 @@ func TestS3Extract_ForcePathStyle(t *testing.T) {
},
},
wantOptions: &storage.S3StorageConfig{
Endpoint: "http://minio:9000",
Endpoint: "minio:9000",
Region: "",
Buckets: "this,that",
ForcePathStyle: true, // defaults to path style for non-AWS endpoints
Insecure: true,
},
},
{
Expand All @@ -741,7 +777,7 @@ func TestS3Extract_ForcePathStyle(t *testing.T) {
},
},
wantOptions: &storage.S3StorageConfig{
Endpoint: "https://s3.region.amazonaws.com",
Endpoint: "s3.region.amazonaws.com",
Region: "region",
Buckets: "this,that",
ForcePathStyle: true,
Expand All @@ -752,7 +788,7 @@ func TestS3Extract_ForcePathStyle(t *testing.T) {
secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "test"},
Data: map[string][]byte{
"endpoint": []byte("https://s3.region.amazonaws.com"),
"endpoint": []byte("s3.region.amazonaws.com"),
"region": []byte("region"),
"bucketnames": []byte("this,that"),
"access_key_id": []byte("id"),
Expand Down
Loading