diff --git a/.run/api.run.xml b/.run/api.run.xml index 82b7baff2..44f95bf39 100644 --- a/.run/api.run.xml +++ b/.run/api.run.xml @@ -2,7 +2,7 @@ - + diff --git a/internal/agent/config.go b/internal/agent/config.go index 10359fb73..045882326 100644 --- a/internal/agent/config.go +++ b/internal/agent/config.go @@ -69,9 +69,9 @@ func DefaultConfig() Config { HistoricWindow: 24 * 3600, MaxHistoricDiskSize: 20 << 30, // enough for default SampleBudget per MaxHistoricWindow SampleBudgets: true, - SampleNamespaces: false, - SampleGroups: false, - SampleKeys: false, + SampleNamespaces: true, + SampleGroups: true, + SampleKeys: true, StringTopCapacity: 100, StringTopCountSend: 20, LivenessResponsesWindowLength: 5, diff --git a/internal/aggregator/aggregator_insert.go b/internal/aggregator/aggregator_insert.go index 2dc0087c4..666d4665b 100644 --- a/internal/aggregator/aggregator_insert.go +++ b/internal/aggregator/aggregator_insert.go @@ -349,7 +349,6 @@ func (a *Aggregator) rowDataMarshalAppendPositions(buckets []*aggregatorBucket, } usedTimestamps := map[uint32]struct{}{} - usedBufferTimestamps := map[uint32]struct{}{} appendCtx := appendContext{ metricCache: makeMetricCache(a.metricStorage), unknownTags: map[string]createMappingExtra{}, @@ -359,15 +358,7 @@ func (a *Aggregator) rowDataMarshalAppendPositions(buckets []*aggregatorBucket, insertItem := func(item *data_model.MultiItem, sf float64, bucketTs uint32) { // lambda is convenient here is := insertSize{} - bufferedInsert := false - if configR.BufferedInsertAgeSec > 0 && item.Key.Timestamp+uint32(configR.BufferedInsertAgeSec) < recentTs { - bufferedInsert = true - } - if bufferedInsert { - usedBufferTimestamps[item.Key.Timestamp] = struct{}{} - } else { - usedTimestamps[item.Key.Timestamp] = struct{}{} - } + usedTimestamps[item.Key.Timestamp] = struct{}{} resPos := len(res) if !item.Tail.Empty() { // only tail @@ -514,12 +505,6 @@ func (a *Aggregator) rowDataMarshalAppendPositions(buckets []*aggregatorBucket, key = data_model.Key{Timestamp: t, Metric: format.BuiltinMetricIDContributorsLogRev, Tags: [format.MaxTags]int32{0, int32(insertTimeUnix)}} res = appendSimpleValueStat(rnd, res, &key, float64(insertTimeUnix)-float64(t), 1, a.aggregatorHostTag.I, appendCtx) } - for t := range usedBufferTimestamps { - key := data_model.Key{Timestamp: insertTimeUnix, Metric: format.BuiltinMetricIDContributorsLog, Tags: [format.MaxTags]int32{0, int32(t)}} - res = appendSimpleValueStat(rnd, res, &key, float64(insertTimeUnix)-float64(t), 1, a.aggregatorHostTag.I, appendCtx) - key = data_model.Key{Timestamp: t, Metric: format.BuiltinMetricIDContributorsLogRev, Tags: [format.MaxTags]int32{0, int32(insertTimeUnix)}} - res = appendSimpleValueStat(rnd, res, &key, float64(insertTimeUnix)-float64(t), 1, a.aggregatorHostTag.I, appendCtx) - } a.addUnknownTags(appendCtx.unknownTags, recentTs, rnd, a.aggregatorHostTag, data_model.AgentEnvRouteArch{}) // no agent info here return res, sampler.SamplerBuffers, stats, time.Since(startTime) } diff --git a/internal/aggregator/config.go b/internal/aggregator/config.go index 87bce8474..799ed41ab 100644 --- a/internal/aggregator/config.go +++ b/internal/aggregator/config.go @@ -41,7 +41,6 @@ type ConfigAggregatorRemote struct { V3InsertSettings string MappingCacheSize int64 MappingCacheTTL int - BufferedInsertAgeSec int // age in seconds of data that should be sent to buffer table MigrationTimeRange string // format: "{begin timestamp}-{end timestamp}" MigrationV3DisabledShards map[int32]struct{} MigrationDelaySec int // delay in seconds between migration steps @@ -196,7 +195,6 @@ func (c *ConfigAggregatorRemote) Bind(f *flag.FlagSet, d ConfigAggregatorRemote, f.StringVar(&c.V3InsertSettings, "v3-insert-settings", d.V3InsertSettings, "Settings when inserting into v3 table") f.Int64Var(&c.MappingCacheSize, "mappings-cache-size-agg", d.MappingCacheSize, "Mappings cache size both in memory and on disk for aggregator.") f.IntVar(&c.MappingCacheTTL, "mappings-cache-ttl-agg", d.MappingCacheTTL, "Mappings cache item TTL since last used for aggregator.") - f.IntVar(&c.BufferedInsertAgeSec, "buffered-insert-age-sec", d.BufferedInsertAgeSec, "Age in seconds of data that should be inserted via buffer table") f.StringVar(&c.MigrationTimeRange, "migration", d.MigrationTimeRange, "Migration time range: \"{start timestamp}-{end timestamp}\" (start > end because of backwards migration)") f.Func("migration-v3-disabled-shards", "List of disabled shards for migration v3", c.setMigrationV3DisabledShards) f.IntVar(&c.MigrationDelaySec, "migration-delay-sec", d.MigrationDelaySec, "Delay in seconds between migration steps") diff --git a/internal/api/httputil.go b/internal/api/httputil.go index 7c9bf2c8d..bb9a75002 100644 --- a/internal/api/httputil.go +++ b/internal/api/httputil.go @@ -8,6 +8,7 @@ package api import ( "context" + "encoding/base64" "encoding/csv" "errors" "fmt" @@ -167,6 +168,7 @@ func respondJSON(h *httpRequestHandler, resp interface{}, cache time.Duration, c } if h.announcement != "" { w.Header().Set("X-Statshouse-Announcment", h.announcement) + w.Header().Set("X-Statshouse-Announcement", base64.StdEncoding.EncodeToString([]byte(h.announcement))) } start := time.Now() var jw jwriter.Writer diff --git a/internal/loadgen/loadgen.go b/internal/loadgen/loadgen.go index e5597f167..3355d7aa0 100644 --- a/internal/loadgen/loadgen.go +++ b/internal/loadgen/loadgen.go @@ -118,12 +118,8 @@ func RunEnableNewPipeline() { ctx := makeInterruptibleContext() c := api.NewClient("http://127.0.0.1:10888", "loadgen") ensureMetricWithDescription(ctx, c, format.StatshouseAPIRemoteConfig, ``) - ensureMetricWithDescription(ctx, c, format.StatshouseAgentRemoteConfigMetric, ` --conveyor-v3-staging=production,staging1,staging2,development -`) - ensureMetricWithDescription(ctx, c, format.StatshouseAggregatorRemoteConfigMetric, ` --map-string-top=true -`) + ensureMetricWithDescription(ctx, c, format.StatshouseAgentRemoteConfigMetric, ``) + ensureMetricWithDescription(ctx, c, format.StatshouseAggregatorRemoteConfigMetric, ``) ensureMetricWithDescription(ctx, c, format.StatshouseJournalDump, ` `) } diff --git a/localdebug/run-api.sh b/localdebug/run-api.sh index b46ceb173..e7e007c0d 100755 --- a/localdebug/run-api.sh +++ b/localdebug/run-api.sh @@ -7,7 +7,7 @@ mkdir -p cache/api --clickhouse-v2-addrs=localhost:9000,localhost:9000,localhost:9000 \ --listen-rpc-addr=localhost:10889 \ --verbose --listen-addr localhost:10888 --static-dir ../statshouse-ui/build/ \ - --metadata-addr "127.0.0.1:2442" --available-shards "1" --cache-dir=cache/api --announcement=Hello \ + --metadata-addr "127.0.0.1:2442" --available-shards "1" --cache-dir=cache/api --announcement=Крокозяблик \ "$@" # --rqlite-addrs "127.0.0.1:4001"