数据存储

blocks_storage:
  tsdb:
    dir: ./data/cortex/tsdb
  bucket_store:
    sync_dir: ./data/cortex/tsdb-sync
  backend: filesystem

storage:
  engine: blocks

blocks_storage:
  backend: filesystem
  s3:
    endpoint: ""
    region: ""
    bucket_name: ""
    secret_access_key: ""
    access_key_id: ""
    insecure: false
    signature_version: v4
    bucket_lookup_type: auto
    sse:
      type: ""
      kms_key_id: ""
      kms_encryption_context: ""
    http:
      idle_conn_timeout: 1m30s
      response_header_timeout: 2m0s
      insecure_skip_verify: false
      tls_handshake_timeout: 10s
      expect_continue_timeout: 1s
      max_idle_connections: 100
      max_idle_connections_per_host: 100
      max_connections_per_host: 0
  gcs:
    ...
 
  azure:
    ...

  swift:
    ...

  filesystem:
    dir: ./data/tsdb

  bucket_store:
    sync_dir: ./data/cortex/tsdb-sync
    sync_interval: 15m0s
    max_concurrent: 100
    tenant_sync_concurrency: 10
    block_sync_concurrency: 20
    meta_sync_concurrency: 20
    consistency_delay: 0s
    index_cache:
      backend: inmemory
      inmemory:
        max_size_bytes: 1073741824
      memcached:
        addresses: ""
        timeout: 100ms
        max_idle_connections: 16
        max_async_concurrency: 50
        max_async_buffer_size: 10000
        max_get_multi_concurrency: 100
        max_get_multi_batch_size: 0
        max_item_size: 1048576
        auto_discovery: false

    chunks_cache:
      backend: ""
      memcached:
        addresses: ""
        timeout: 100ms
        max_idle_connections: 16
        max_async_concurrency: 50
        max_async_buffer_size: 10000
        max_get_multi_concurrency: 100
        max_get_multi_batch_size: 0
        max_item_size: 1048576
        auto_discovery: false
      subrange_size: 16000
      max_get_range_requests: 3
      attributes_ttl: 168h0m0s
      subrange_ttl: 24h0m0s

    metadata_cache:
      backend: ""
      memcached:
        addresses: ""
        timeout: 100ms
        max_idle_connections: 16
        max_async_concurrency: 50
        max_async_buffer_size: 10000
        max_get_multi_concurrency: 100
        max_get_multi_batch_size: 0
        max_item_size: 1048576
        auto_discovery: false
      tenants_list_ttl: 15m0s
      tenant_blocks_list_ttl: 5m0s
      chunks_list_ttl: 24h0m0s
      metafile_exists_ttl: 2h0m0s
      metafile_doesnt_exist_ttl: 5m0s
      metafile_content_ttl: 24h0m0s
      metafile_max_size_bytes: 1048576
      metafile_attributes_ttl: 168h0m0s
      block_index_attributes_ttl: 168h0m0s
      bucket_index_content_ttl: 5m0s
      bucket_index_max_size_bytes: 1048576
    ignore_deletion_mark_delay: 6h0m0s
    bucket_index:
      enabled: false
      update_on_error_interval: 1m0s
      idle_timeout: 1h0m0s
      max_stale_period: 1h0m0s
    max_chunk_pool_bytes: 2147483648
    chunk_pool_min_bucket_size_bytes: 16000
    chunk_pool_max_bucket_size_bytes: 50000000
    index_header_lazy_loading_enabled: false
    index_header_lazy_loading_idle_timeout: 20m0s
    partitioner_max_gap_bytes: 524288
    postings_offsets_in_mem_sampling: 32

  tsdb:
    dir: ./data/cortex/tsdb
    block_ranges_period:
    - 2h0m0s
    retention_period: 6h0m0s
    ship_interval: 1m0s
    ship_concurrency: 10
    head_compaction_interval: 1m0s
    head_compaction_concurrency: 5
    head_compaction_idle_timeout: 1h0m0s
    head_chunks_write_buffer_size_bytes: 4194304
    stripe_size: 16384
    wal_compression_enabled: false
    wal_segment_size_bytes: 134217728
    flush_blocks_on_shutdown: false
    close_idle_tsdb_timeout: 0s
    max_tsdb_opening_concurrency_on_startup: 10
    max_exemplars: 0

// BlocksStorageConfig holds the config information for the blocks storage.
//
//nolint:revive
type BlocksStorageConfig struct {
    Bucket      bucket.Config     `yaml:",inline"`
    BucketStore BucketStoreConfig `yaml:"bucket_store" doc:"description=This configures how the querier and store-gateway discover and synchronize blocks stored in the bucket."`
    TSDB        TSDBConfig        `yaml:"tsdb"`
}

// BucketStoreConfig holds the config information for Bucket Stores used by the querier and store-gateway.
type BucketStoreConfig struct {
    SyncDir                  string              `yaml:"sync_dir"`
    SyncInterval             time.Duration       `yaml:"sync_interval"`
    MaxConcurrent            int                 `yaml:"max_concurrent"`
    TenantSyncConcurrency    int                 `yaml:"tenant_sync_concurrency"`
    BlockSyncConcurrency     int                 `yaml:"block_sync_concurrency"`
    MetaSyncConcurrency      int                 `yaml:"meta_sync_concurrency"`
    ConsistencyDelay         time.Duration       `yaml:"consistency_delay"`
    IndexCache               IndexCacheConfig    `yaml:"index_cache"`
    ChunksCache              ChunksCacheConfig   `yaml:"chunks_cache"`
    MetadataCache            MetadataCacheConfig `yaml:"metadata_cache"`
    IgnoreDeletionMarksDelay time.Duration       `yaml:"ignore_deletion_mark_delay"`
    IgnoreBlocksWithin       time.Duration       `yaml:"ignore_blocks_within"`
    BucketIndex              BucketIndexConfig   `yaml:"bucket_index"`

    // Chunk pool.
    MaxChunkPoolBytes           uint64 `yaml:"max_chunk_pool_bytes"`
    ChunkPoolMinBucketSizeBytes int    `yaml:"chunk_pool_min_bucket_size_bytes" doc:"hidden"`
    ChunkPoolMaxBucketSizeBytes int    `yaml:"chunk_pool_max_bucket_size_bytes" doc:"hidden"`

    // Controls whether index-header lazy loading is enabled.
    IndexHeaderLazyLoadingEnabled     bool          `yaml:"index_header_lazy_loading_enabled"`
    IndexHeaderLazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout"`

    // Controls the partitioner, used to aggregate multiple GET object API requests.
    // The config option is hidden until experimental.
    PartitionerMaxGapBytes uint64 `yaml:"partitioner_max_gap_bytes" doc:"hidden"`

    // Controls what is the ratio of postings offsets store will hold in memory.
    // Larger value will keep less offsets, which will increase CPU cycles needed for query touching those postings.
    // It's meant for setups that want low baseline memory pressure and where less traffic is expected.
    // On the contrary, smaller value will increase baseline memory usage, but improve latency slightly.
    // 1 will keep all in memory. Default value is the same as in Prometheus which gives a good balance.
    PostingOffsetsInMemSampling int `yaml:"postings_offsets_in_mem_sampling" doc:"hidden"`
}

// TSDBConfig holds the config for TSDB opened in the ingesters.
//
//nolint:revive
type TSDBConfig struct {
    Dir                       string        `yaml:"dir"`
    BlockRanges               DurationList  `yaml:"block_ranges_period"`
    Retention                 time.Duration `yaml:"retention_period"`
    ShipInterval              time.Duration `yaml:"ship_interval"`
    ShipConcurrency           int           `yaml:"ship_concurrency"`
    HeadCompactionInterval    time.Duration `yaml:"head_compaction_interval"`
    HeadCompactionConcurrency int           `yaml:"head_compaction_concurrency"`
    HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout"`
    HeadChunksWriteBufferSize int           `yaml:"head_chunks_write_buffer_size_bytes"`
    StripeSize                int           `yaml:"stripe_size"`
    WALCompressionEnabled     bool          `yaml:"wal_compression_enabled"`
    WALSegmentSizeBytes       int           `yaml:"wal_segment_size_bytes"`
    FlushBlocksOnShutdown     bool          `yaml:"flush_blocks_on_shutdown"`
    CloseIdleTSDBTimeout      time.Duration `yaml:"close_idle_tsdb_timeout"`
    // The size of the in-memory queue used before flushing chunks to the disk.
    HeadChunksWriteQueueSize int `yaml:"head_chunks_write_queue_size"`

    // MaxTSDBOpeningConcurrencyOnStartup limits the number of concurrently opening TSDB's during startup.
    MaxTSDBOpeningConcurrencyOnStartup int `yaml:"max_tsdb_opening_concurrency_on_startup"`

    // If true, user TSDBs are not closed on shutdown. Only for testing.
    // If false (default), user TSDBs are closed to make sure all resources are released and closed properly.
    KeepUserTSDBOpenOnShutdown bool `yaml:"-"`

    // How often to check for idle TSDBs for closing. DefaultCloseIdleTSDBInterval is not suitable for testing, so tests can override.
    CloseIdleTSDBInterval time.Duration `yaml:"-"`

    // Positive value enables experimental support for exemplars. 0 or less to disable.
    MaxExemplars int `yaml:"max_exemplars"`

    // Enable snapshotting of in-memory TSDB data on disk when shutting down.
    MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown"`

    // OutOfOrderCapMax is maximum capacity for OOO chunks (in samples).
    OutOfOrderCapMax int64 `yaml:"out_of_order_cap_max"`
}

// Config holds configuration for accessing long-term storage.
type Config struct {
    Backend string `yaml:"backend"`
    // Backends
    S3         s3.Config         `yaml:"s3"`
    GCS        gcs.Config        `yaml:"gcs"`
    Azure      azure.Config      `yaml:"azure"`
    Swift      swift.Config      `yaml:"swift"`
    Filesystem filesystem.Config `yaml:"filesystem"`

    // Not used internally, meant to allow callers to wrap Buckets
    // created using this config
    Middlewares []func(objstore.Bucket) (objstore.Bucket, error) `yaml:"-"`

    // Used to inject additional backends into the config. Allows for this config to
    // be embedded in multiple contexts and support non-object storage based backends.
    ExtraBackends []string `yaml:"-"`
}

// Config holds the config options for an S3 backend
type Config struct {
    Endpoint         string         `yaml:"endpoint"`
    Region           string         `yaml:"region"`
    BucketName       string         `yaml:"bucket_name"`
    SecretAccessKey  flagext.Secret `yaml:"secret_access_key"`
    AccessKeyID      string         `yaml:"access_key_id"`
    Insecure         bool           `yaml:"insecure"`
    SignatureVersion string         `yaml:"signature_version"`
    BucketLookupType string         `yaml:"bucket_lookup_type"`

    SSE  SSEConfig  `yaml:"sse"`
    HTTP HTTPConfig `yaml:"http"`
}

// Config stores the configuration for storing and accessing objects in the local filesystem.
type Config struct {
    Directory string `yaml:"dir"`
}

// SSEConfig configures S3 server side encryption
// struct that is going to receive user input (through config file or CLI)
type SSEConfig struct {
    Type                 string `yaml:"type"`
    KMSKeyID             string `yaml:"kms_key_id"`
    KMSEncryptionContext string `yaml:"kms_encryption_context"`
}

const (
    SignatureVersionV4 = "v4"
    SignatureVersionV2 = "v2"

    // SSEKMS config type constant to configure S3 server side encryption using KMS
    // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
    SSEKMS = "SSE-KMS"

    // SSES3 config type constant to configure S3 server side encryption with AES-256
    // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
    SSES3 = "SSE-S3"

    BucketAutoLookup        = "auto"
    BucketVirtualHostLookup = "virtual-hosted"
    BucketPathLookup        = "path"
)