diff --git a/operator/internal/handlers/internal/storage/secrets.go b/operator/internal/handlers/internal/storage/secrets.go index cb89619eb3a..c1ed4c111db 100644 --- a/operator/internal/handlers/internal/storage/secrets.go +++ b/operator/internal/handlers/internal/storage/secrets.go @@ -36,8 +36,9 @@ var ( errSecretUnknownCredentialMode = errors.New("unknown credential mode") errAzureManagedIdentityNoOverride = errors.New("when in managed mode, storage secret can not contain credentials") - errAzureInvalidEnvironment = errors.New("azure environment invalid (valid values: AzureGlobal, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment)") + errAzureInvalidEnvironment = errors.New("azure environment invalid (valid values: AzureGlobal, AzurePublicCloud, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment)") errAzureInvalidAccountKey = errors.New("azure account key is not valid base64") + errAzureInvalidEndpointSuffix = errors.New("azure endpoint suffix invalid") errS3EndpointUnparseable = errors.New("can not parse S3 endpoint as URL") errS3EndpointNoURL = errors.New("endpoint for S3 must be an HTTP or HTTPS URL") @@ -50,11 +51,12 @@ var ( errGCPWrongCredentialSourceFile = errors.New("credential source in secret needs to point to token file") errGCPInvalidCredentialsFile = errors.New("gcp credentials file contains invalid fields") - azureValidEnvironments = map[string]bool{ - "AzureGlobal": true, - "AzureChinaCloud": true, - "AzureGermanCloud": true, - "AzureUSGovernment": true, + azureEnvironmentEndpointSuffix = map[string]string{ + "AzureGlobal": "blob.core.windows.net", + "AzurePublicCloud": "blob.core.windows.net", + "AzureChinaCloud": "blob.core.chinacloudapi.cn", + "AzureGermanCloud": "blob.core.cloudapi.de", + "AzureUSGovernment": "blob.core.usgovcloudapi.net", } ) @@ -239,14 +241,24 @@ func hashSecretData(s *corev1.Secret) (string, error) { func extractAzureConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMode) (*storage.AzureStorageConfig, error) { // Extract and validate mandatory fields env := string(s.Data[storage.KeyAzureEnvironmentName]) - if env == "" { - return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureEnvironmentName) + endpointSuffix := string(s.Data[storage.KeyAzureStorageEndpointSuffix]) + if env == "" && endpointSuffix == "" { + return nil, fmt.Errorf("%w: either %s or %s should be set", errSecretMissingField, storage.KeyAzureEnvironmentName, storage.KeyAzureStorageEndpointSuffix) } - if !azureValidEnvironments[env] { + envEndpointSuffix, ok := azureEnvironmentEndpointSuffix[env] + if env != "" && !ok { return nil, fmt.Errorf("%w: %s", errAzureInvalidEnvironment, env) } + if endpointSuffix == "" { + endpointSuffix = envEndpointSuffix + } + + if !endpointSuffixExists(azureEnvironmentEndpointSuffix, endpointSuffix) { + return nil, fmt.Errorf("%w: %s", errAzureInvalidEndpointSuffix, endpointSuffix) + } + accountName := s.Data[storage.KeyAzureStorageAccountName] if len(accountName) == 0 { return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAzureStorageAccountName) @@ -263,7 +275,6 @@ func extractAzureConfigSecret(s *corev1.Secret, credentialMode lokiv1.Credential } // Extract and validate optional fields - endpointSuffix := s.Data[storage.KeyAzureStorageEndpointSuffix] audience := s.Data[storage.KeyAzureAudience] if !workloadIdentity && len(audience) > 0 { @@ -271,7 +282,6 @@ func extractAzureConfigSecret(s *corev1.Secret, credentialMode lokiv1.Credential } return &storage.AzureStorageConfig{ - Env: env, Container: string(container), EndpointSuffix: string(endpointSuffix), Audience: string(audience), @@ -405,7 +415,7 @@ func extractS3ConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMod var ( // Fields related with static authentication - endpoint = s.Data[storage.KeyAWSEndpoint] + endpoint = string(s.Data[storage.KeyAWSEndpoint]) id = s.Data[storage.KeyAWSAccessKeyID] secret = s.Data[storage.KeyAWSAccessKeySecret] // Fields related with STS authentication @@ -417,7 +427,7 @@ func extractS3ConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMod // Determine if we should use path style URLs for S3 // default to false for non-AWS endpoints - forcePathStyle := !strings.HasSuffix(string(endpoint), awsEndpointSuffix) + forcePathStyle := !strings.HasSuffix(endpoint, awsEndpointSuffix) // Check if the user has specified forcepathstyle if configForcePathStyle, ok := s.Data[storage.KeyAWSForcePathStyle]; ok { strForcePathStyle := string(configForcePathStyle) @@ -454,13 +464,20 @@ func extractS3ConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMod if len(region) == 0 { return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAWSRegion) } + return cfg, nil case lokiv1.CredentialModeStatic: - cfg.Endpoint = string(endpoint) - - if err := validateS3Endpoint(string(endpoint), string(region)); err != nil { + if err := validateS3Endpoint(endpoint, string(region)); err != nil { return nil, err } + parsedURL, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("%w:%s", errS3EndpointUnparseable, storage.KeyAWSEndpoint) + } + + cfg.Endpoint = parsedURL.Host + cfg.Insecure = strings.HasPrefix(endpoint, "http://") + if len(id) == 0 { return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAWSAccessKeyID) } @@ -477,6 +494,7 @@ func extractS3ConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMod if len(region) == 0 { return nil, fmt.Errorf("%w: %s", errSecretMissingField, storage.KeyAWSRegion) } + return cfg, nil default: return nil, fmt.Errorf("%w: %s", errSecretUnknownCredentialMode, credentialMode) @@ -634,3 +652,12 @@ func extractAlibabaCloudConfigSecret(s *corev1.Secret) (*storage.AlibabaCloudSto Bucket: string(bucket), }, nil } + +func endpointSuffixExists(m map[string]string, value string) bool { + for _, v := range m { + if v == value { + return true + } + } + return false +} diff --git a/operator/internal/handlers/internal/storage/secrets_test.go b/operator/internal/handlers/internal/storage/secrets_test.go index aa7197e2bef..4516d24a46a 100644 --- a/operator/internal/handlers/internal/storage/secrets_test.go +++ b/operator/internal/handlers/internal/storage/secrets_test.go @@ -82,9 +82,44 @@ func TestAzureExtract(t *testing.T) { } table := []test{ { - name: "missing environment", + name: "missing environment and endpoint_suffix", secret: &corev1.Secret{}, - wantError: "missing secret field: environment", + wantError: "missing secret field: either environment or endpoint_suffix should be set", + }, + { + name: "missing only endpoint_suffix", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "environment": []byte("AzureGlobal"), + "container": []byte("this,that"), + "account_name": []byte("test-account-name"), + "account_key": []byte("dGVzdC1hY2NvdW50LWtleQ=="), + }, + }, + wantCredentialMode: lokiv1.CredentialModeStatic, + }, + { + name: "missing only environment", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "endpoint_suffix": []byte("blob.core.windows.net"), + "container": []byte("this,that"), + "account_name": []byte("test-account-name"), + "account_key": []byte("dGVzdC1hY2NvdW50LWtleQ=="), + }, + }, + wantCredentialMode: lokiv1.CredentialModeStatic, + }, + { + name: "invalid endpoint_suffix", + secret: &corev1.Secret{ + Data: map[string][]byte{ + "endpoint_suffix": []byte("invalid-endpoint-suffix"), + }, + }, + wantError: "azure endpoint suffix invalid: invalid-endpoint-suffix", }, { name: "invalid environment", @@ -93,7 +128,7 @@ func TestAzureExtract(t *testing.T) { "environment": []byte("invalid-environment"), }, }, - wantError: "azure environment invalid (valid values: AzureGlobal, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment): invalid-environment", + wantError: "azure environment invalid (valid values: AzureGlobal, AzurePublicCloud, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment): invalid-environment", }, { name: "missing account_name", @@ -245,7 +280,7 @@ func TestAzureExtract(t *testing.T) { "container": []byte("this,that"), "account_name": []byte("id"), "account_key": []byte("dGVzdC1hY2NvdW50LWtleQ=="), // test-account-key - "endpoint_suffix": []byte("suffix"), + "endpoint_suffix": []byte("blob.core.windows.net"), }, }, wantCredentialMode: lokiv1.CredentialModeStatic, @@ -702,7 +737,7 @@ func TestS3Extract_ForcePathStyle(t *testing.T) { }, }, wantOptions: &storage.S3StorageConfig{ - Endpoint: "https://s3.region.amazonaws.com", + Endpoint: "s3.region.amazonaws.com", Region: "region", Buckets: "this,that", ForcePathStyle: false, // defaults to virtual style for AWS endpoints @@ -721,10 +756,11 @@ func TestS3Extract_ForcePathStyle(t *testing.T) { }, }, wantOptions: &storage.S3StorageConfig{ - Endpoint: "http://minio:9000", + Endpoint: "minio:9000", Region: "", Buckets: "this,that", ForcePathStyle: true, // defaults to path style for non-AWS endpoints + Insecure: true, }, }, { @@ -741,7 +777,7 @@ func TestS3Extract_ForcePathStyle(t *testing.T) { }, }, wantOptions: &storage.S3StorageConfig{ - Endpoint: "https://s3.region.amazonaws.com", + Endpoint: "s3.region.amazonaws.com", Region: "region", Buckets: "this,that", ForcePathStyle: true, @@ -752,7 +788,7 @@ func TestS3Extract_ForcePathStyle(t *testing.T) { secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Data: map[string][]byte{ - "endpoint": []byte("https://s3.region.amazonaws.com"), + "endpoint": []byte("s3.region.amazonaws.com"), "region": []byte("region"), "bucketnames": []byte("this,that"), "access_key_id": []byte("id"), diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index bd1df70629d..807aed9fe50 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -1,6 +1,7 @@ package config import ( + "os" "strings" "testing" "time" @@ -14,172 +15,9 @@ import ( ) func TestBuild_ConfigAndRuntimeConfig_NoRuntimeConfigGenerated(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 536870912 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: true - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/no-runtime-config/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -277,172 +115,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_BothGenerated(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: false -` + expCfgBytes, err := os.ReadFile("testdata/both-generated/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -708,226 +383,9 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) { } func TestBuild_ConfigAndRuntimeConfig_RulerConfigGenerated_WithHeaderAuthorization(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -ruler: - enable_api: true - enable_sharding: true - evaluation_interval: 1m - poll_interval: 1m - external_url: http://alert.me/now - external_labels: - key1: val1 - key2: val2 - alertmanager_url: http://alerthost1,http://alerthost2 - enable_alertmanager_v2: true - enable_alertmanager_discovery: true - alertmanager_refresh_interval: 1m - notification_queue_capacity: 1000 - notification_timeout: 1m - for_outage_tolerance: 10m - for_grace_period: 5m - resend_delay: 2m - remote_write: - enabled: true - config_refresh_period: 1m - client: - name: remote-write-me - url: http://remote.write.me - remote_timeout: 10s - proxy_url: http://proxy.through.me - follow_redirects: true - headers: - more: foryou - less: forme - authorization: - type: bearer - credentials: supersecret - queue_config: - capacity: 1000 - max_shards: 100 - min_shards: 50 - max_samples_per_send: 1000 - batch_send_deadline: 10s - min_backoff: 30ms - max_backoff: 100ms - wal: - dir: /tmp/wal - truncate_frequency: 60m - min_age: 5m - max_age: 4h - rule_path: /tmp/loki - storage: - type: local - local: - directory: /tmp/rules - ring: - kvstore: - store: memberlist -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/ruler-with-auth-header/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -1071,226 +529,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_RulerConfigGenerated_WithBasicAuthorization(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -ruler: - enable_api: true - enable_sharding: true - evaluation_interval: 1m - poll_interval: 1m - external_url: http://alert.me/now - external_labels: - key1: val1 - key2: val2 - alertmanager_url: http://alerthost1,http://alerthost2 - enable_alertmanager_v2: true - enable_alertmanager_discovery: true - alertmanager_refresh_interval: 1m - notification_queue_capacity: 1000 - notification_timeout: 1m - for_outage_tolerance: 10m - for_grace_period: 5m - resend_delay: 2m - remote_write: - enabled: true - config_refresh_period: 1m - client: - name: remote-write-me - url: http://remote.write.me - remote_timeout: 10s - proxy_url: http://proxy.through.me - follow_redirects: true - headers: - more: foryou - less: forme - basic_auth: - username: user - password: passwd - queue_config: - capacity: 1000 - max_shards: 100 - min_shards: 50 - max_samples_per_send: 1000 - batch_send_deadline: 10s - min_backoff: 30ms - max_backoff: 100ms - wal: - dir: /tmp/wal - truncate_frequency: 60m - min_age: 5m - max_age: 4h - rule_path: /tmp/loki - storage: - type: local - local: - directory: /tmp/rules - ring: - kvstore: - store: memberlist -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/ruler-with-auth-basic/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -1435,239 +676,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_RulerConfigGenerated_WithRelabelConfigs(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -ruler: - enable_api: true - enable_sharding: true - evaluation_interval: 1m - poll_interval: 1m - external_url: http://alert.me/now - external_labels: - key1: val1 - key2: val2 - alertmanager_url: http://alerthost1,http://alerthost2 - enable_alertmanager_v2: true - enable_alertmanager_discovery: true - alertmanager_refresh_interval: 1m - notification_queue_capacity: 1000 - notification_timeout: 1m - for_outage_tolerance: 10m - for_grace_period: 5m - resend_delay: 2m - remote_write: - enabled: true - config_refresh_period: 1m - client: - name: remote-write-me - url: http://remote.write.me - remote_timeout: 10s - proxy_url: http://proxy.through.me - follow_redirects: true - headers: - more: foryou - less: forme - write_relabel_configs: - - source_labels: ["labela","labelb"] - regex: "ALERTS.*" - action: "drop" - separator: "\\" - replacement: "$1" - - source_labels: ["labelc","labeld"] - regex: "ALERTS.*" - action: "drop" - separator: "" - replacement: "$1" - target_label: "labeld" - modulus: 123 - basic_auth: - username: user - password: passwd - queue_config: - capacity: 1000 - max_shards: 100 - min_shards: 50 - max_samples_per_send: 1000 - batch_send_deadline: 10s - min_backoff: 30ms - max_backoff: 100ms - wal: - dir: /tmp/wal - truncate_frequency: 60m - min_age: 5m - max_age: 4h - rule_path: /tmp/loki - storage: - type: local - local: - directory: /tmp/rules - ring: - kvstore: - store: memberlist -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/ruler-with-relabel-configs/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -1829,181 +840,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_WithRetention(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor - retention_enabled: true - retention_delete_delay: 4h - retention_delete_worker_count: 50 - delete_request_store: s3 -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - retention_period: 15d - retention_stream: - - selector: '{environment="development"}' - priority: 1 - period: 3d - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: false -` + expCfgBytes, err := os.ReadFile("testdata/with-retention/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -2173,252 +1012,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_RulerConfigGenerated_WithAlertRelabelConfigs(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 2m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -ruler: - enable_api: true - enable_sharding: true - evaluation_interval: 1m - poll_interval: 1m - external_url: http://alert.me/now - external_labels: - key1: val1 - key2: val2 - alertmanager_url: http://alerthost1,http://alerthost2 - enable_alertmanager_v2: true - enable_alertmanager_discovery: true - alertmanager_refresh_interval: 1m - notification_queue_capacity: 1000 - notification_timeout: 1m - alert_relabel_configs: - - source_labels: ["source1", "source2"] - regex: "ALERTS.*" - action: "drop" - separator: "\\" - replacement: "$1" - - source_labels: ["source3", "source4"] - regex: "ALERTS.*" - action: "keep" - separator: "" - replacement: "$1" - target_label: "target" - modulus: 42 - for_outage_tolerance: 10m - for_grace_period: 5m - resend_delay: 2m - remote_write: - enabled: true - config_refresh_period: 1m - client: - name: remote-write-me - url: http://remote.write.me - remote_timeout: 10s - proxy_url: http://proxy.through.me - follow_redirects: true - headers: - more: foryou - less: forme - write_relabel_configs: - - source_labels: ["labela","labelb"] - regex: "ALERTS.*" - action: "drop" - separator: "\\" - replacement: "$1" - - source_labels: ["labelc","labeld"] - regex: "ALERTS.*" - action: "drop" - separator: "" - replacement: "$1" - target_label: "labeld" - modulus: 123 - basic_auth: - username: user - password: passwd - queue_config: - capacity: 1000 - max_shards: 100 - min_shards: 50 - max_samples_per_send: 1000 - batch_send_deadline: 10s - min_backoff: 30ms - max_backoff: 100ms - wal: - dir: /tmp/wal - truncate_frequency: 60m - min_age: 5m - max_age: 4h - rule_path: /tmp/loki - storage: - type: local - local: - directory: /tmp/rules - ring: - kvstore: - store: memberlist -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/ruler-with-alert-relabel-configs/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -2597,229 +1193,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_WithTLS(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://s3.us-east.amazonaws.com - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - tail_tls_config: - tls_cert_path: /var/run/tls/http/tls.crt - tls_key_path: /var/run/tls/http/tls.key - tls_ca_path: /var/run/tls/ca.pem - tls_server_name: querier-http.svc - tls_cipher_suites: cipher1,cipher2 - tls_min_version: VersionTLS12 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 - tls_enabled: true - tls_cert_path: /var/run/tls/grpc/tls.crt - tls_key_path: /var/run/tls/grpc/tls.key - tls_ca_path: /var/run/tls/ca.pem - tls_server_name: query-frontend-grpc.svc - tls_cipher_suites: cipher1,cipher2 - tls_min_version: VersionTLS12 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - tls_enabled: true - tls_cert_path: /var/run/tls/grpc/tls.crt - tls_key_path: /var/run/tls/grpc/tls.key - tls_ca_path: /var/run/tls/ca.pem - tls_server_name: ingester-grpc.svc - tls_cipher_suites: cipher1,cipher2 - tls_min_version: VersionTLS12 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -compactor_grpc_client: - tls_enabled: true - tls_cert_path: /var/run/tls/grpc/tls.crt - tls_key_path: /var/run/tls/grpc/tls.key - tls_ca_path: /var/run/tls/ca.pem - tls_server_name: compactor-grpc.svc - tls_cipher_suites: cipher1,cipher2 - tls_min_version: VersionTLS12 -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper - -internal_server: - enable: true - http_listen_address: "" - tls_min_version: VersionTLS12 - tls_cipher_suites: cipher1,cipher2 - http_tls_config: - cert_file: /var/run/tls/http/tls.crt - key_file: /var/run/tls/http/tls.key -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - tls_min_version: VersionTLS12 - tls_cipher_suites: cipher1,cipher2 - http_tls_config: - cert_file: /var/run/tls/http/tls.crt - key_file: /var/run/tls/http/tls.key - client_auth_type: RequireAndVerifyClientCert - client_ca_file: /var/run/tls/ca.pem - grpc_tls_config: - cert_file: /var/run/tls/grpc/tls.crt - key_file: /var/run/tls/grpc/tls.key - client_auth_type: RequireAndVerifyClientCert - client_ca_file: /var/run/tls/ca.pem - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - tls_enabled: true - tls_cert_path: /var/run/tls/grpc/tls.crt - tls_key_path: /var/run/tls/grpc/tls.key - tls_ca_path: /var/run/tls/ca.pem - tls_server_name: index-gateway-grpc.svc - tls_cipher_suites: cipher1,cipher2 - tls_min_version: VersionTLS12 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/with-tls/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -2946,252 +1322,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_RulerConfigGenerated_WithAlertmanagerOverrides(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 2m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -ruler: - enable_api: true - enable_sharding: true - evaluation_interval: 1m - poll_interval: 1m - external_url: http://alert.me/now - external_labels: - key1: val1 - key2: val2 - alertmanager_url: http://alerthost1,http://alerthost2 - enable_alertmanager_v2: true - enable_alertmanager_discovery: true - alertmanager_refresh_interval: 1m - notification_queue_capacity: 1000 - notification_timeout: 1m - alert_relabel_configs: - - source_labels: ["source1", "source2"] - regex: "ALERTS.*" - action: "drop" - separator: "\\" - replacement: "$1" - - source_labels: ["source3", "source4"] - regex: "ALERTS.*" - action: "keep" - separator: "" - replacement: "$1" - target_label: "target" - modulus: 42 - for_outage_tolerance: 10m - for_grace_period: 5m - resend_delay: 2m - remote_write: - enabled: true - config_refresh_period: 1m - client: - name: remote-write-me - url: http://remote.write.me - remote_timeout: 10s - proxy_url: http://proxy.through.me - follow_redirects: true - headers: - more: foryou - less: forme - write_relabel_configs: - - source_labels: ["labela","labelb"] - regex: "ALERTS.*" - action: "drop" - separator: "\\" - replacement: "$1" - - source_labels: ["labelc","labeld"] - regex: "ALERTS.*" - action: "drop" - separator: "" - replacement: "$1" - target_label: "labeld" - modulus: 123 - basic_auth: - username: user - password: passwd - queue_config: - capacity: 1000 - max_shards: 100 - min_shards: 50 - max_samples_per_send: 1000 - batch_send_deadline: 10s - min_backoff: 30ms - max_backoff: 100ms - wal: - dir: /tmp/wal - truncate_frequency: 60m - min_age: 5m - max_age: 4h - rule_path: /tmp/loki - storage: - type: local - local: - directory: /tmp/rules - ring: - kvstore: - store: memberlist -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/ruler-with-alertmanager-overrides/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -3447,174 +1580,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_WithHashRingSpec(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_addr: ${HASH_RING_INSTANCE_ADDR} - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_addr: ${HASH_RING_INSTANCE_ADDR} - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/with-hashring-spec/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -3712,175 +1680,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_WithHashRingSpec_EnableIPv6(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_addr: ${HASH_RING_INSTANCE_ADDR} - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - enable_inet6: true - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_addr: ${HASH_RING_INSTANCE_ADDR} - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/with-hashring-spec-ipv6/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -3979,174 +1781,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_WithReplicationSpec(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 - zone_awareness_enabled: true - instance_availability_zone: ${INSTANCE_AVAILABILITY_ZONE} -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/with-replication-spec/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -4244,177 +1881,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_WithS3SSEKMS(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - sse: - type: SSE-KMS - kms_key_id: test - kms_encryption_context: | - ${AWS_SSE_KMS_ENCRYPTION_CONTEXT} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: false -` + expCfgBytes, err := os.ReadFile("testdata/with-s3-sse-kms/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -4551,174 +2020,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_WithS3SSES3(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - sse: - type: SSE-S3 - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: false -` + expCfgBytes, err := os.ReadFile("testdata/with-s3-sse-s3/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -4855,168 +2159,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_WithManualPerStreamRateLimits(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - per_stream_rate_limit: 3MB - per_stream_rate_limit_burst: 15MB - split_queries_by_interval: 30m - tsdb_max_query_parallelism: 512 - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/with-manual-stream-ratelimits/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -5357,155 +2502,9 @@ func TestBuild_ConfigAndRuntimeConfig_Schemas(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - per_stream_rate_limit: 3MB - per_stream_rate_limit_burst: 15MB - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - ${STORAGE_STRUCTURED_METADATA} -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: -${SCHEMA_CONFIG} -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: -${STORAGE_CONFIG} -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/schemas-template.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expCfg = strings.Replace(expCfg, "${SCHEMA_CONFIG}", tc.expSchemaConfig, 1) expCfg = strings.Replace(expCfg, "${STORAGE_CONFIG}", tc.expStorageConfig, 1) expCfg = strings.Replace(expCfg, "${STORAGE_STRUCTURED_METADATA}", tc.expStructuredMetadata, 1) @@ -5537,169 +2536,9 @@ func TestBuild_ConfigAndRuntimeConfig_STS(t *testing.T) { }, }, } - expStorageConfig := ` - s3: - bucketnames: my-bucket - region: my-region - s3forcepathstyle: false` - - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: -${STORAGE_CONFIG} - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - per_stream_rate_limit: 3MB - per_stream_rate_limit_burst: 15MB - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` - expCfg = strings.Replace(expCfg, "${STORAGE_CONFIG}", expStorageConfig, 1) + expCfgBytes, err := os.ReadFile("testdata/sts/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) opts := defaultOptions() opts.ObjectStorage = objStorageConfig @@ -5710,237 +2549,9 @@ analytics: } func TestBuild_ConfigAndRuntimeConfig_RulerConfigGenerated_WithAlertmanagerClient(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: false -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2020-10-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v11 - store: boltdb-shipper -ruler: - enable_api: true - enable_sharding: true - evaluation_interval: 1m - poll_interval: 1m - external_url: http://alert.me/now - external_labels: - key1: val1 - key2: val2 - alertmanager_url: http://alerthost1,http://alerthost2 - enable_alertmanager_v2: true - enable_alertmanager_discovery: true - alertmanager_refresh_interval: 1m - notification_queue_capacity: 1000 - notification_timeout: 1m - alertmanager_client: - tls_cert_path: "custom/path" - tls_key_path: "custom/key" - tls_ca_path: "custom/CA" - tls_server_name: "custom-servername" - tls_insecure_skip_verify: false - basic_auth_password: "pass" - basic_auth_username: "user" - credentials: "creds" - credentials_file: "cred/file" - type: "auth" - for_outage_tolerance: 10m - for_grace_period: 5m - resend_delay: 2m - remote_write: - enabled: true - config_refresh_period: 1m - client: - name: remote-write-me - url: http://remote.write.me - remote_timeout: 10s - proxy_url: http://proxy.through.me - follow_redirects: true - headers: - more: foryou - less: forme - authorization: - type: bearer - credentials: supersecret - queue_config: - capacity: 1000 - max_shards: 100 - min_shards: 50 - max_samples_per_send: 1000 - batch_send_deadline: 10s - min_backoff: 30ms - max_backoff: 100ms - wal: - dir: /tmp/wal - truncate_frequency: 60m - min_age: 5m - max_age: 4h - rule_path: /tmp/loki - storage: - type: local - local: - directory: /tmp/rules - ring: - kvstore: - store: memberlist -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - boltdb_shipper: - active_index_directory: /tmp/loki/index - cache_location: /tmp/loki/index_cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/ruler-with-alertmanager-client/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: @@ -6102,260 +2713,9 @@ overrides: } func TestBuild_ConfigAndRuntimeConfig_OTLPConfigGenerated(t *testing.T) { - expCfg := ` ---- -auth_enabled: true -chunk_store_config: - chunk_cache_config: - embedded_cache: - enabled: true - max_size_mb: 500 -common: - storage: - s3: - endpoint: http://test.default.svc.cluster.local.:9000 - bucketnames: loki - region: us-east - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - s3forcepathstyle: true - compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 - ring: - kvstore: - store: memberlist - heartbeat_period: 5s - heartbeat_timeout: 1m - instance_port: 9095 -compactor: - compaction_interval: 2h - working_directory: /tmp/loki/compactor -distributor: - otlp_config: - default_resource_attributes_as_index_labels: [] -frontend: - tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 - compress_responses: true - max_outstanding_per_tenant: 4096 - log_queries_longer_than: 5s -frontend_worker: - frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 - grpc_client_config: - max_send_msg_size: 104857600 -ingester: - chunk_block_size: 262144 - chunk_encoding: snappy - chunk_idle_period: 1h - chunk_retain_period: 5m - chunk_target_size: 2097152 - flush_op_timeout: 10m - lifecycler: - final_sleep: 0s - join_after: 30s - num_tokens: 512 - ring: - replication_factor: 1 - max_chunk_age: 2h - autoforget_unhealthy: true - wal: - enabled: true - dir: /tmp/wal - replay_memory_ceiling: 2147483648 -ingester_client: - grpc_client_config: - max_recv_msg_size: 67108864 - remote_timeout: 5s -# NOTE: Keep the order of keys as in Loki docs -# to enable easy diffs when vendoring newer -# Loki releases. -# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) -# -# Values for not exposed fields are taken from the grafana/loki production -# configuration manifests. -# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) -limits_config: - ingestion_rate_strategy: global - ingestion_rate_mb: 4 - ingestion_burst_size_mb: 6 - max_label_name_length: 1024 - max_label_value_length: 2048 - max_label_names_per_series: 30 - reject_old_samples: true - reject_old_samples_max_age: 168h - creation_grace_period: 10m - # Keep max_streams_per_user always to 0 to default - # using max_global_streams_per_user always. - # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) - max_streams_per_user: 0 - max_line_size: 256000 - max_entries_limit_per_query: 5000 - discover_service_name: [] - discover_log_levels: false - max_global_streams_per_user: 0 - max_chunks_per_query: 2000000 - max_query_length: 721h - max_query_parallelism: 32 - tsdb_max_query_parallelism: 512 - max_query_series: 500 - cardinality_limit: 100000 - max_streams_matchers_per_query: 1000 - max_cache_freshness_per_query: 10m - split_queries_by_interval: 30m - query_timeout: 1m - volume_enabled: true - volume_max_series: 1000 - per_stream_rate_limit: 5MB - per_stream_rate_limit_burst: 15MB - shard_streams: - enabled: true - desired_rate: 3MB - time_sharding_enabled: true - allow_structured_metadata: true - otlp_config: - resource_attributes: - attributes_config: - - action: index_label - attributes: - - res.foo.bar - - res.bar.baz - - action: drop - attributes: - - res.service.env - scope_attributes: - - action: drop - attributes: - - scope.foo.bar - - scope.bar.baz - log_attributes: - - action: drop - attributes: - - log.foo.bar - - log.bar.baz -memberlist: - abort_if_cluster_join_fails: true - advertise_port: 7946 - bind_port: 7946 - join_members: - - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - rejoin_interval: 90s -querier: - engine: - max_look_back_period: 30s - extra_query_delay: 0s - max_concurrent: 2 - query_ingesters_within: 3h - tail_max_duration: 1h -query_range: - align_queries_with_step: true - cache_results: true - max_retries: 5 - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 500 - parallelise_shardable_queries: true -schema_config: - configs: - - from: "2024-01-01" - index: - period: 24h - prefix: index_ - object_store: s3 - schema: v13 - store: tsdb -ruler: - enable_api: true - enable_sharding: true - evaluation_interval: 1m - poll_interval: 1m - external_url: http://alert.me/now - external_labels: - key1: val1 - key2: val2 - alertmanager_url: http://alerthost1,http://alerthost2 - enable_alertmanager_v2: true - enable_alertmanager_discovery: true - alertmanager_refresh_interval: 1m - notification_queue_capacity: 1000 - notification_timeout: 1m - alertmanager_client: - tls_cert_path: "custom/path" - tls_key_path: "custom/key" - tls_ca_path: "custom/CA" - tls_server_name: "custom-servername" - tls_insecure_skip_verify: false - basic_auth_password: "pass" - basic_auth_username: "user" - credentials: "creds" - credentials_file: "cred/file" - type: "auth" - for_outage_tolerance: 10m - for_grace_period: 5m - resend_delay: 2m - remote_write: - enabled: true - config_refresh_period: 1m - client: - name: remote-write-me - url: http://remote.write.me - remote_timeout: 10s - proxy_url: http://proxy.through.me - follow_redirects: true - headers: - more: foryou - less: forme - authorization: - type: bearer - credentials: supersecret - queue_config: - capacity: 1000 - max_shards: 100 - min_shards: 50 - max_samples_per_send: 1000 - batch_send_deadline: 10s - min_backoff: 30ms - max_backoff: 100ms - wal: - dir: /tmp/wal - truncate_frequency: 60m - min_age: 5m - max_age: 4h - rule_path: /tmp/loki - storage: - type: local - local: - directory: /tmp/rules - ring: - kvstore: - store: memberlist -server: - graceful_shutdown_timeout: 5s - grpc_server_min_time_between_pings: '10s' - grpc_server_ping_without_stream_allowed: true - grpc_server_max_concurrent_streams: 1000 - grpc_server_max_recv_msg_size: 104857600 - grpc_server_max_send_msg_size: 104857600 - http_listen_port: 3100 - http_server_idle_timeout: 30s - http_server_read_timeout: 30s - http_server_write_timeout: 10m0s - log_level: info -storage_config: - tsdb_shipper: - active_index_directory: /tmp/loki/tsdb-index - cache_location: /tmp/loki/tsdb-cache - cache_ttl: 24h - resync_interval: 5m - index_gateway_client: - server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 -tracing: - enabled: false -analytics: - reporting_enabled: true -` + expCfgBytes, err := os.ReadFile("testdata/with-otlp-config/config.yaml") + require.NoError(t, err) + expCfg := string(expCfgBytes) expRCfg := ` --- overrides: diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index d469701e64f..bdf7f24a460 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -8,78 +8,78 @@ chunk_store_config: max_size_mb: 500 common: storage: - {{- with .ObjectStorage.Azure }} - azure: - environment: {{ .Env }} - container_name: {{ .Container }} - account_name: ${AZURE_STORAGE_ACCOUNT_NAME} - {{- if .WorkloadIdentity }} - use_federated_token: true - {{- else }} - account_key: ${AZURE_STORAGE_ACCOUNT_KEY} + object_store: + {{- with .ObjectStorage.Azure }} + azure: + container_name: {{ .Container }} + account_name: ${AZURE_STORAGE_ACCOUNT_NAME} + endpoint_suffix: {{ .EndpointSuffix }} + {{- if not .WorkloadIdentity }} + account_key: ${AZURE_STORAGE_ACCOUNT_KEY} + {{- end }} {{- end }} - {{- with .EndpointSuffix }} - endpoint_suffix: {{ . }} + {{- with .ObjectStorage.GCS }} + gcs: + bucket_name: {{ .Bucket }} {{- end }} - {{- end }} - {{- with .ObjectStorage.GCS }} - gcs: - bucket_name: {{ .Bucket }} - {{- end }} - {{- with .ObjectStorage.S3 }} - s3: - {{- if .STS }} - bucketnames: {{.Buckets}} - region: {{.Region}} - s3forcepathstyle: false - {{- else }} - endpoint: {{ .Endpoint }} - bucketnames: {{ .Buckets }} - region: {{ .Region }} - access_key_id: ${AWS_ACCESS_KEY_ID} - secret_access_key: ${AWS_ACCESS_KEY_SECRET} - {{- if .ForcePathStyle }} - s3forcepathstyle: true - {{- end}} - {{- end }} - {{- with .SSE }} - {{- if .Type }} - sse: - type: {{ .Type }} - {{- if eq .Type "SSE-KMS" }} - kms_key_id: {{ .KMSKeyID }} - {{- with .KMSEncryptionContext }} - kms_encryption_context: | - ${AWS_SSE_KMS_ENCRYPTION_CONTEXT} - {{- end }} + {{- with .ObjectStorage.S3 }} + s3: + {{- if .STS }} + bucket_name: {{ .Buckets }} + region: {{ .Region }} + endpoint: s3.{{.Region}}.amazonaws.com + native_aws_auth_enabled: true + {{- else }} + endpoint: {{ .Endpoint }} + bucket_name: {{ .Buckets }} + region: {{ .Region }} + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + {{- if .Insecure }} + insecure: true + {{- end}} + {{- if .ForcePathStyle }} + bucket_lookup_type: path {{- end}} + {{- end }} + {{- with .SSE }} + {{- if .Type }} + sse: + type: {{ .Type }} + {{- if eq .Type "SSE-KMS" }} + kms_key_id: {{ .KMSKeyID }} + {{- with .KMSEncryptionContext }} + kms_encryption_context: | + ${AWS_SSE_KMS_ENCRYPTION_CONTEXT} + {{- end }} + {{- end}} + {{- end }} + {{- end }} {{- end }} + {{- with .ObjectStorage.Swift }} + swift: + auth_url: {{ .AuthURL }} + username: ${SWIFT_USERNAME} + user_domain_name: {{ .UserDomainName }} + user_domain_id: {{ .UserDomainID }} + user_id: {{ .UserID }} + password: ${SWIFT_PASSWORD} + domain_id: {{ .DomainID }} + domain_name: {{ .DomainName }} + project_id: {{ .ProjectID }} + project_name: {{ .ProjectName }} + project_domain_id: {{ .ProjectDomainID }} + project_domain_name: {{ .ProjectDomainName }} + region_name: {{ .Region }} + container_name: {{ .Container }} + {{- end }} + {{- with .ObjectStorage.AlibabaCloud}} + alibaba: + bucket: {{ .Bucket }} + endpoint: {{ .Endpoint }} + access_key_id: ${ALIBABA_CLOUD_ACCESS_KEY_ID} + access_key_secret: ${ALIBABA_CLOUD_ACCESS_KEY_SECRET} {{- end }} - {{- end }} - {{- with .ObjectStorage.Swift }} - swift: - auth_url: {{ .AuthURL }} - username: ${SWIFT_USERNAME} - user_domain_name: {{ .UserDomainName }} - user_domain_id: {{ .UserDomainID }} - user_id: {{ .UserID }} - password: ${SWIFT_PASSWORD} - domain_id: {{ .DomainID }} - domain_name: {{ .DomainName }} - project_id: {{ .ProjectID }} - project_name: {{ .ProjectName }} - project_domain_id: {{ .ProjectDomainID }} - project_domain_name: {{ .ProjectDomainName }} - region_name: {{ .Region }} - container_name: {{ .Container }} - {{- end }} - {{- with .ObjectStorage.AlibabaCloud}} - alibabacloud: - bucket: {{ .Bucket }} - endpoint: {{ .Endpoint }} - access_key_id: ${ALIBABA_CLOUD_ACCESS_KEY_ID} - secret_access_key: ${ALIBABA_CLOUD_ACCESS_KEY_SECRET} - {{- end }} compactor_grpc_address: {{ .Compactor.FQDN }}:{{ .Compactor.Port }} {{- with .GossipRing }} ring: @@ -343,6 +343,10 @@ schema_config: {{- end}} {{- end }} {{ if .Ruler.Enabled }} +ruler_storage: + backend: local + local: + directory: {{ .Ruler.RulesStorageDirectory }} ruler: enable_api: true enable_sharding: true @@ -536,10 +540,6 @@ ruler: min_age: 5m max_age: 4h rule_path: {{ .StorageDirectory }} - storage: - type: local - local: - directory: {{ .Ruler.RulesStorageDirectory }} ring: kvstore: store: memberlist @@ -595,6 +595,7 @@ server: {{- end }} log_level: info storage_config: + use_thanos_objstore: true {{- range $_, $ship := .Shippers }} {{- if eq $ship "boltdb" }} boltdb_shipper: diff --git a/operator/internal/manifests/internal/config/testdata/both-generated/config.yaml b/operator/internal/manifests/internal/config/testdata/both-generated/config.yaml new file mode 100644 index 00000000000..252ac994f52 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/both-generated/config.yaml @@ -0,0 +1,166 @@ +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: false diff --git a/operator/internal/manifests/internal/config/testdata/no-runtime-config/config.yaml b/operator/internal/manifests/internal/config/testdata/no-runtime-config/config.yaml new file mode 100644 index 00000000000..cd1fa83196e --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/no-runtime-config/config.yaml @@ -0,0 +1,167 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 536870912 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: true + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/ruler-with-alert-relabel-configs/config.yaml b/operator/internal/manifests/internal/config/testdata/ruler-with-alert-relabel-configs/config.yaml new file mode 100644 index 00000000000..078636df6e7 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/ruler-with-alert-relabel-configs/config.yaml @@ -0,0 +1,247 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 2m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +ruler: + enable_api: true + enable_sharding: true + evaluation_interval: 1m + poll_interval: 1m + external_url: http://alert.me/now + external_labels: + key1: val1 + key2: val2 + alertmanager_url: http://alerthost1,http://alerthost2 + enable_alertmanager_v2: true + enable_alertmanager_discovery: true + alertmanager_refresh_interval: 1m + notification_queue_capacity: 1000 + notification_timeout: 1m + alert_relabel_configs: + - source_labels: ["source1", "source2"] + regex: "ALERTS.*" + action: "drop" + separator: "\\" + replacement: "$1" + - source_labels: ["source3", "source4"] + regex: "ALERTS.*" + action: "keep" + separator: "" + replacement: "$1" + target_label: "target" + modulus: 42 + for_outage_tolerance: 10m + for_grace_period: 5m + resend_delay: 2m + remote_write: + enabled: true + config_refresh_period: 1m + client: + name: remote-write-me + url: http://remote.write.me + remote_timeout: 10s + proxy_url: http://proxy.through.me + follow_redirects: true + headers: + more: foryou + less: forme + write_relabel_configs: + - source_labels: ["labela","labelb"] + regex: "ALERTS.*" + action: "drop" + separator: "\\" + replacement: "$1" + - source_labels: ["labelc","labeld"] + regex: "ALERTS.*" + action: "drop" + separator: "" + replacement: "$1" + target_label: "labeld" + modulus: 123 + basic_auth: + username: user + password: passwd + queue_config: + capacity: 1000 + max_shards: 100 + min_shards: 50 + max_samples_per_send: 1000 + batch_send_deadline: 10s + min_backoff: 30ms + max_backoff: 100ms + wal: + dir: /tmp/wal + truncate_frequency: 60m + min_age: 5m + max_age: 4h + rule_path: /tmp/loki + ring: + kvstore: + store: memberlist +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +ruler_storage: + backend: local + local: + directory: /tmp/rules +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/ruler-with-alertmanager-client/config.yaml b/operator/internal/manifests/internal/config/testdata/ruler-with-alertmanager-client/config.yaml new file mode 100644 index 00000000000..035cff16ed0 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/ruler-with-alertmanager-client/config.yaml @@ -0,0 +1,232 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +ruler: + enable_api: true + enable_sharding: true + evaluation_interval: 1m + poll_interval: 1m + external_url: http://alert.me/now + external_labels: + key1: val1 + key2: val2 + alertmanager_url: http://alerthost1,http://alerthost2 + enable_alertmanager_v2: true + enable_alertmanager_discovery: true + alertmanager_refresh_interval: 1m + notification_queue_capacity: 1000 + notification_timeout: 1m + alertmanager_client: + tls_cert_path: "custom/path" + tls_key_path: "custom/key" + tls_ca_path: "custom/CA" + tls_server_name: "custom-servername" + tls_insecure_skip_verify: false + basic_auth_password: "pass" + basic_auth_username: "user" + credentials: "creds" + credentials_file: "cred/file" + type: "auth" + for_outage_tolerance: 10m + for_grace_period: 5m + resend_delay: 2m + remote_write: + enabled: true + config_refresh_period: 1m + client: + name: remote-write-me + url: http://remote.write.me + remote_timeout: 10s + proxy_url: http://proxy.through.me + follow_redirects: true + headers: + more: foryou + less: forme + authorization: + type: bearer + credentials: supersecret + queue_config: + capacity: 1000 + max_shards: 100 + min_shards: 50 + max_samples_per_send: 1000 + batch_send_deadline: 10s + min_backoff: 30ms + max_backoff: 100ms + wal: + dir: /tmp/wal + truncate_frequency: 60m + min_age: 5m + max_age: 4h + rule_path: /tmp/loki + ring: + kvstore: + store: memberlist +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +ruler_storage: + backend: local + local: + directory: /tmp/rules +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/ruler-with-alertmanager-overrides/config.yaml b/operator/internal/manifests/internal/config/testdata/ruler-with-alertmanager-overrides/config.yaml new file mode 100644 index 00000000000..078636df6e7 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/ruler-with-alertmanager-overrides/config.yaml @@ -0,0 +1,247 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 2m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +ruler: + enable_api: true + enable_sharding: true + evaluation_interval: 1m + poll_interval: 1m + external_url: http://alert.me/now + external_labels: + key1: val1 + key2: val2 + alertmanager_url: http://alerthost1,http://alerthost2 + enable_alertmanager_v2: true + enable_alertmanager_discovery: true + alertmanager_refresh_interval: 1m + notification_queue_capacity: 1000 + notification_timeout: 1m + alert_relabel_configs: + - source_labels: ["source1", "source2"] + regex: "ALERTS.*" + action: "drop" + separator: "\\" + replacement: "$1" + - source_labels: ["source3", "source4"] + regex: "ALERTS.*" + action: "keep" + separator: "" + replacement: "$1" + target_label: "target" + modulus: 42 + for_outage_tolerance: 10m + for_grace_period: 5m + resend_delay: 2m + remote_write: + enabled: true + config_refresh_period: 1m + client: + name: remote-write-me + url: http://remote.write.me + remote_timeout: 10s + proxy_url: http://proxy.through.me + follow_redirects: true + headers: + more: foryou + less: forme + write_relabel_configs: + - source_labels: ["labela","labelb"] + regex: "ALERTS.*" + action: "drop" + separator: "\\" + replacement: "$1" + - source_labels: ["labelc","labeld"] + regex: "ALERTS.*" + action: "drop" + separator: "" + replacement: "$1" + target_label: "labeld" + modulus: 123 + basic_auth: + username: user + password: passwd + queue_config: + capacity: 1000 + max_shards: 100 + min_shards: 50 + max_samples_per_send: 1000 + batch_send_deadline: 10s + min_backoff: 30ms + max_backoff: 100ms + wal: + dir: /tmp/wal + truncate_frequency: 60m + min_age: 5m + max_age: 4h + rule_path: /tmp/loki + ring: + kvstore: + store: memberlist +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +ruler_storage: + backend: local + local: + directory: /tmp/rules +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/ruler-with-auth-basic/config.yaml b/operator/internal/manifests/internal/config/testdata/ruler-with-auth-basic/config.yaml new file mode 100644 index 00000000000..7d9068ec4b7 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/ruler-with-auth-basic/config.yaml @@ -0,0 +1,221 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +ruler: + enable_api: true + enable_sharding: true + evaluation_interval: 1m + poll_interval: 1m + external_url: http://alert.me/now + external_labels: + key1: val1 + key2: val2 + alertmanager_url: http://alerthost1,http://alerthost2 + enable_alertmanager_v2: true + enable_alertmanager_discovery: true + alertmanager_refresh_interval: 1m + notification_queue_capacity: 1000 + notification_timeout: 1m + for_outage_tolerance: 10m + for_grace_period: 5m + resend_delay: 2m + remote_write: + enabled: true + config_refresh_period: 1m + client: + name: remote-write-me + url: http://remote.write.me + remote_timeout: 10s + proxy_url: http://proxy.through.me + follow_redirects: true + headers: + more: foryou + less: forme + basic_auth: + username: user + password: passwd + queue_config: + capacity: 1000 + max_shards: 100 + min_shards: 50 + max_samples_per_send: 1000 + batch_send_deadline: 10s + min_backoff: 30ms + max_backoff: 100ms + wal: + dir: /tmp/wal + truncate_frequency: 60m + min_age: 5m + max_age: 4h + rule_path: /tmp/loki + ring: + kvstore: + store: memberlist +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +ruler_storage: + backend: local + local: + directory: /tmp/rules +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/ruler-with-auth-header/config.yaml b/operator/internal/manifests/internal/config/testdata/ruler-with-auth-header/config.yaml new file mode 100644 index 00000000000..e2835d44a51 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/ruler-with-auth-header/config.yaml @@ -0,0 +1,220 @@ +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +ruler: + enable_api: true + enable_sharding: true + evaluation_interval: 1m + poll_interval: 1m + external_url: http://alert.me/now + external_labels: + key1: val1 + key2: val2 + alertmanager_url: http://alerthost1,http://alerthost2 + enable_alertmanager_v2: true + enable_alertmanager_discovery: true + alertmanager_refresh_interval: 1m + notification_queue_capacity: 1000 + notification_timeout: 1m + for_outage_tolerance: 10m + for_grace_period: 5m + resend_delay: 2m + remote_write: + enabled: true + config_refresh_period: 1m + client: + name: remote-write-me + url: http://remote.write.me + remote_timeout: 10s + proxy_url: http://proxy.through.me + follow_redirects: true + headers: + more: foryou + less: forme + authorization: + type: bearer + credentials: supersecret + queue_config: + capacity: 1000 + max_shards: 100 + min_shards: 50 + max_samples_per_send: 1000 + batch_send_deadline: 10s + min_backoff: 30ms + max_backoff: 100ms + wal: + dir: /tmp/wal + truncate_frequency: 60m + min_age: 5m + max_age: 4h + rule_path: /tmp/loki + ring: + kvstore: + store: memberlist +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +ruler_storage: + backend: local + local: + directory: /tmp/rules +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/ruler-with-relabel-configs/config.yaml b/operator/internal/manifests/internal/config/testdata/ruler-with-relabel-configs/config.yaml new file mode 100644 index 00000000000..82d26e33275 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/ruler-with-relabel-configs/config.yaml @@ -0,0 +1,234 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +ruler: + enable_api: true + enable_sharding: true + evaluation_interval: 1m + poll_interval: 1m + external_url: http://alert.me/now + external_labels: + key1: val1 + key2: val2 + alertmanager_url: http://alerthost1,http://alerthost2 + enable_alertmanager_v2: true + enable_alertmanager_discovery: true + alertmanager_refresh_interval: 1m + notification_queue_capacity: 1000 + notification_timeout: 1m + for_outage_tolerance: 10m + for_grace_period: 5m + resend_delay: 2m + remote_write: + enabled: true + config_refresh_period: 1m + client: + name: remote-write-me + url: http://remote.write.me + remote_timeout: 10s + proxy_url: http://proxy.through.me + follow_redirects: true + headers: + more: foryou + less: forme + write_relabel_configs: + - source_labels: ["labela","labelb"] + regex: "ALERTS.*" + action: "drop" + separator: "\\" + replacement: "$1" + - source_labels: ["labelc","labeld"] + regex: "ALERTS.*" + action: "drop" + separator: "" + replacement: "$1" + target_label: "labeld" + modulus: 123 + basic_auth: + username: user + password: passwd + queue_config: + capacity: 1000 + max_shards: 100 + min_shards: 50 + max_samples_per_send: 1000 + batch_send_deadline: 10s + min_backoff: 30ms + max_backoff: 100ms + wal: + dir: /tmp/wal + truncate_frequency: 60m + min_age: 5m + max_age: 4h + rule_path: /tmp/loki + ring: + kvstore: + store: memberlist +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +ruler_storage: + backend: local + local: + directory: /tmp/rules +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/schemas-template.yaml b/operator/internal/manifests/internal/config/testdata/schemas-template.yaml new file mode 100644 index 00000000000..243c9c1c0d0 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/schemas-template.yaml @@ -0,0 +1,149 @@ +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + per_stream_rate_limit: 3MB + per_stream_rate_limit_burst: 15MB + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + ${STORAGE_STRUCTURED_METADATA} +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + ${SCHEMA_CONFIG} +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + ${STORAGE_CONFIG} +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/sts/config.yaml b/operator/internal/manifests/internal/config/testdata/sts/config.yaml new file mode 100644 index 00000000000..dc3087c5b79 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/sts/config.yaml @@ -0,0 +1,160 @@ +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + bucket_name: my-bucket + endpoint: s3.my-region.amazonaws.com + native_aws_auth_enabled: true + region: my-region + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + per_stream_rate_limit: 3MB + per_stream_rate_limit_burst: 15MB + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/with-hashring-spec-ipv6/config.yaml b/operator/internal/manifests/internal/config/testdata/with-hashring-spec-ipv6/config.yaml new file mode 100644 index 00000000000..b520bf8fd60 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/with-hashring-spec-ipv6/config.yaml @@ -0,0 +1,170 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_addr: ${HASH_RING_INSTANCE_ADDR} + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + enable_inet6: true + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_addr: ${HASH_RING_INSTANCE_ADDR} + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/with-hashring-spec/config.yaml b/operator/internal/manifests/internal/config/testdata/with-hashring-spec/config.yaml new file mode 100644 index 00000000000..ef4df20bfe6 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/with-hashring-spec/config.yaml @@ -0,0 +1,169 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_addr: ${HASH_RING_INSTANCE_ADDR} + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_addr: ${HASH_RING_INSTANCE_ADDR} + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/with-manual-stream-ratelimits/config.yaml b/operator/internal/manifests/internal/config/testdata/with-manual-stream-ratelimits/config.yaml new file mode 100644 index 00000000000..6cfcc965353 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/with-manual-stream-ratelimits/config.yaml @@ -0,0 +1,163 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + per_stream_rate_limit: 3MB + per_stream_rate_limit_burst: 15MB + split_queries_by_interval: 30m + tsdb_max_query_parallelism: 512 + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/with-otlp-config/config.yaml b/operator/internal/manifests/internal/config/testdata/with-otlp-config/config.yaml new file mode 100644 index 00000000000..f3819819da4 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/with-otlp-config/config.yaml @@ -0,0 +1,255 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +distributor: + otlp_config: + default_resource_attributes_as_index_labels: [] +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: true + otlp_config: + resource_attributes: + attributes_config: + - action: index_label + attributes: + - res.foo.bar + - res.bar.baz + - action: drop + attributes: + - res.service.env + scope_attributes: + - action: drop + attributes: + - scope.foo.bar + - scope.bar.baz + log_attributes: + - action: drop + attributes: + - log.foo.bar + - log.bar.baz +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2024-01-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v13 + store: tsdb +ruler: + enable_api: true + enable_sharding: true + evaluation_interval: 1m + poll_interval: 1m + external_url: http://alert.me/now + external_labels: + key1: val1 + key2: val2 + alertmanager_url: http://alerthost1,http://alerthost2 + enable_alertmanager_v2: true + enable_alertmanager_discovery: true + alertmanager_refresh_interval: 1m + notification_queue_capacity: 1000 + notification_timeout: 1m + alertmanager_client: + tls_cert_path: "custom/path" + tls_key_path: "custom/key" + tls_ca_path: "custom/CA" + tls_server_name: "custom-servername" + tls_insecure_skip_verify: false + basic_auth_password: "pass" + basic_auth_username: "user" + credentials: "creds" + credentials_file: "cred/file" + type: "auth" + for_outage_tolerance: 10m + for_grace_period: 5m + resend_delay: 2m + remote_write: + enabled: true + config_refresh_period: 1m + client: + name: remote-write-me + url: http://remote.write.me + remote_timeout: 10s + proxy_url: http://proxy.through.me + follow_redirects: true + headers: + more: foryou + less: forme + authorization: + type: bearer + credentials: supersecret + queue_config: + capacity: 1000 + max_shards: 100 + min_shards: 50 + max_samples_per_send: 1000 + batch_send_deadline: 10s + min_backoff: 30ms + max_backoff: 100ms + wal: + dir: /tmp/wal + truncate_frequency: 60m + min_age: 5m + max_age: 4h + rule_path: /tmp/loki + ring: + kvstore: + store: memberlist +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +ruler_storage: + backend: local + local: + directory: /tmp/rules +storage_config: + use_thanos_objstore: true + tsdb_shipper: + active_index_directory: /tmp/loki/tsdb-index + cache_location: /tmp/loki/tsdb-cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/with-replication-spec/config.yaml b/operator/internal/manifests/internal/config/testdata/with-replication-spec/config.yaml new file mode 100644 index 00000000000..3bfad595ad7 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/with-replication-spec/config.yaml @@ -0,0 +1,169 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 + zone_awareness_enabled: true + instance_availability_zone: ${INSTANCE_AVAILABILITY_ZONE} +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/internal/config/testdata/with-retention/config.yaml b/operator/internal/manifests/internal/config/testdata/with-retention/config.yaml new file mode 100644 index 00000000000..0a986787f2e --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/with-retention/config.yaml @@ -0,0 +1,176 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor + retention_enabled: true + retention_delete_delay: 4h + retention_delete_worker_count: 50 + delete_request_store: s3 +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + retention_period: 15d + retention_stream: + - selector: '{environment="development"}' + priority: 1 + period: 3d + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: false diff --git a/operator/internal/manifests/internal/config/testdata/with-s3-sse-kms/config.yaml b/operator/internal/manifests/internal/config/testdata/with-s3-sse-kms/config.yaml new file mode 100644 index 00000000000..0afd03fb956 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/with-s3-sse-kms/config.yaml @@ -0,0 +1,172 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + sse: + type: SSE-KMS + kms_key_id: test + kms_encryption_context: | + ${AWS_SSE_KMS_ENCRYPTION_CONTEXT} + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: false diff --git a/operator/internal/manifests/internal/config/testdata/with-s3-sse-s3/config.yaml b/operator/internal/manifests/internal/config/testdata/with-s3-sse-s3/config.yaml new file mode 100644 index 00000000000..f7732b09377 --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/with-s3-sse-s3/config.yaml @@ -0,0 +1,169 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + bucket_lookup_type: path + sse: + type: SSE-S3 + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: false diff --git a/operator/internal/manifests/internal/config/testdata/with-tls/config.yaml b/operator/internal/manifests/internal/config/testdata/with-tls/config.yaml new file mode 100644 index 00000000000..cf512e0200b --- /dev/null +++ b/operator/internal/manifests/internal/config/testdata/with-tls/config.yaml @@ -0,0 +1,224 @@ + +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + object_store: + s3: + endpoint: http://s3.us-east.amazonaws.com + bucket_name: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + tail_tls_config: + tls_cert_path: /var/run/tls/http/tls.crt + tls_key_path: /var/run/tls/http/tls.key + tls_ca_path: /var/run/tls/ca.pem + tls_server_name: querier-http.svc + tls_cipher_suites: cipher1,cipher2 + tls_min_version: VersionTLS12 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 + tls_enabled: true + tls_cert_path: /var/run/tls/grpc/tls.crt + tls_key_path: /var/run/tls/grpc/tls.key + tls_ca_path: /var/run/tls/ca.pem + tls_server_name: query-frontend-grpc.svc + tls_cipher_suites: cipher1,cipher2 + tls_min_version: VersionTLS12 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + autoforget_unhealthy: true + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + tls_enabled: true + tls_cert_path: /var/run/tls/grpc/tls.crt + tls_key_path: /var/run/tls/grpc/tls.key + tls_ca_path: /var/run/tls/ca.pem + tls_server_name: ingester-grpc.svc + tls_cipher_suites: cipher1,cipher2 + tls_min_version: VersionTLS12 + remote_timeout: 5s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + discover_service_name: [] + discover_log_levels: false + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + time_sharding_enabled: true + allow_structured_metadata: false +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + rejoin_interval: 90s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +compactor_grpc_client: + tls_enabled: true + tls_cert_path: /var/run/tls/grpc/tls.crt + tls_key_path: /var/run/tls/grpc/tls.key + tls_ca_path: /var/run/tls/ca.pem + tls_server_name: compactor-grpc.svc + tls_cipher_suites: cipher1,cipher2 + tls_min_version: VersionTLS12 +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2020-10-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v11 + store: boltdb-shipper + +internal_server: + enable: true + http_listen_address: "" + tls_min_version: VersionTLS12 + tls_cipher_suites: cipher1,cipher2 + http_tls_config: + cert_file: /var/run/tls/http/tls.crt + key_file: /var/run/tls/http/tls.key +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + tls_min_version: VersionTLS12 + tls_cipher_suites: cipher1,cipher2 + http_tls_config: + cert_file: /var/run/tls/http/tls.crt + key_file: /var/run/tls/http/tls.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /var/run/tls/ca.pem + grpc_tls_config: + cert_file: /var/run/tls/grpc/tls.crt + key_file: /var/run/tls/grpc/tls.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /var/run/tls/ca.pem + log_level: info +storage_config: + use_thanos_objstore: true + boltdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + tls_enabled: true + tls_cert_path: /var/run/tls/grpc/tls.crt + tls_key_path: /var/run/tls/grpc/tls.key + tls_ca_path: /var/run/tls/ca.pem + tls_server_name: index-gateway-grpc.svc + tls_cipher_suites: cipher1,cipher2 + tls_min_version: VersionTLS12 +tracing: + enabled: false +analytics: + reporting_enabled: true diff --git a/operator/internal/manifests/storage/configure.go b/operator/internal/manifests/storage/configure.go index a33944dbcc5..295ea47d4c6 100644 --- a/operator/internal/manifests/storage/configure.go +++ b/operator/internal/manifests/storage/configure.go @@ -303,11 +303,11 @@ func ensureCAForObjectStorage(p *corev1.PodSpec, tls *TLSConfig, secretType loki switch secretType { case lokiv1.ObjectStorageSecretS3: container.Args = append(container.Args, - fmt.Sprintf("-s3.http.ca-file=%s", path.Join(caDirectory, tls.Key)), + fmt.Sprintf("-common.storage.object-store.s3.http.tls-ca-path=%s", path.Join(caDirectory, tls.Key)), ) case lokiv1.ObjectStorageSecretSwift: container.Args = append(container.Args, - fmt.Sprintf("-swift.http.tls-ca-path=%s", path.Join(caDirectory, tls.Key)), + fmt.Sprintf("-common.storage.object-store.swift.http.tls-ca-path=%s", path.Join(caDirectory, tls.Key)), ) } diff --git a/operator/internal/manifests/storage/configure_test.go b/operator/internal/manifests/storage/configure_test.go index 7b52593ffe2..cfe8300c089 100644 --- a/operator/internal/manifests/storage/configure_test.go +++ b/operator/internal/manifests/storage/configure_test.go @@ -2562,7 +2562,7 @@ func TestConfigureDeploymentForStorageCA(t *testing.T) { }, }, Args: []string{ - "-s3.http.ca-file=/etc/storage/ca/service-ca.crt", + "-common.storage.object-store.s3.http.tls-ca-path=/etc/storage/ca/service-ca.crt", }, Env: []corev1.EnvVar{ { @@ -2658,7 +2658,7 @@ func TestConfigureDeploymentForStorageCA(t *testing.T) { }, }, Args: []string{ - "-swift.http.tls-ca-path=/etc/storage/ca/service-ca.crt", + "-common.storage.object-store.swift.http.tls-ca-path=/etc/storage/ca/service-ca.crt", }, Env: []corev1.EnvVar{ { @@ -2852,7 +2852,7 @@ func TestConfigureStatefulSetForStorageCA(t *testing.T) { }, }, Args: []string{ - "-s3.http.ca-file=/etc/storage/ca/service-ca.crt", + "-common.storage.object-store.s3.http.tls-ca-path=/etc/storage/ca/service-ca.crt", }, Env: []corev1.EnvVar{ { @@ -2948,7 +2948,7 @@ func TestConfigureStatefulSetForStorageCA(t *testing.T) { }, }, Args: []string{ - "-swift.http.tls-ca-path=/etc/storage/ca/service-ca.crt", + "-common.storage.object-store.swift.http.tls-ca-path=/etc/storage/ca/service-ca.crt", }, Env: []corev1.EnvVar{ { diff --git a/operator/internal/manifests/storage/options.go b/operator/internal/manifests/storage/options.go index 59618953d1f..0a80db845a8 100644 --- a/operator/internal/manifests/storage/options.go +++ b/operator/internal/manifests/storage/options.go @@ -27,7 +27,6 @@ type Options struct { // AzureStorageConfig for Azure storage config type AzureStorageConfig struct { - Env string Container string EndpointSuffix string Audience string @@ -50,6 +49,7 @@ type S3StorageConfig struct { STS bool SSE S3SSEConfig ForcePathStyle bool + Insecure bool } type S3SSEType string