Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions apis/fluentd/v1alpha1/plugins/output/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ type Output struct {
Elasticsearch *Elasticsearch `json:"elasticsearch,omitempty"`
// out_es datastreams plugin
ElasticsearchDataStream *ElasticsearchDataStream `json:"elasticsearchDataStream,omitempty"`
// out_es dynamic plugin (elasticsearch_dynamic). It takes the same parameters as elasticsearch,
// but host, port, index and logstash values can use placeholders like ${tag_parts[1]} or ${record['key']}.
ElasticsearchDynamic *Elasticsearch `json:"elasticsearchDynamic,omitempty"`
// out_opensearch plugin
Opensearch *Opensearch `json:"opensearch,omitempty"`
// out_kafka plugin
Expand Down Expand Up @@ -135,14 +138,19 @@ func (o *Output) Params(loader plugins.SecretLoader) (*params.PluginStore, error

if o.Elasticsearch != nil {
ps.InsertType(string(params.ElasticsearchOutputType))
return o.elasticsearchPlugin(ps, loader)
return o.elasticsearchPlugin(o.Elasticsearch, ps, loader)
}

if o.ElasticsearchDataStream != nil {
ps.InsertType(string(params.ElasticsearchDataStreamOutputType))
return o.elasticsearchDataStreamPlugin(ps, loader)
}

if o.ElasticsearchDynamic != nil {
ps.InsertType(string(params.ElasticsearchDynamicOutputType))
return o.elasticsearchPlugin(o.ElasticsearchDynamic, ps, loader)
}

if o.Opensearch != nil {
ps.InsertType(string(params.OpensearchOutputType))
return o.opensearchPlugin(ps, loader)
Expand Down Expand Up @@ -401,22 +409,22 @@ func (o *Output) elasticsearchPluginCommon(cmn *ElasticsearchCommon, parent *par
return parent, nil
}

func (o *Output) elasticsearchPlugin(parent *params.PluginStore, loader plugins.SecretLoader) (*params.PluginStore, error) {
parent, err := o.elasticsearchPluginCommon(&o.Elasticsearch.ElasticsearchCommon, parent, loader)
func (o *Output) elasticsearchPlugin(es *Elasticsearch, parent *params.PluginStore, loader plugins.SecretLoader) (*params.PluginStore, error) {
parent, err := o.elasticsearchPluginCommon(&es.ElasticsearchCommon, parent, loader)
if err != nil {
return nil, err
}

if o.Elasticsearch.IndexName != nil {
parent.InsertPairs("index_name", fmt.Sprint(*o.Elasticsearch.IndexName))
if es.IndexName != nil {
parent.InsertPairs("index_name", fmt.Sprint(*es.IndexName))
}

if o.Elasticsearch.LogstashFormat != nil {
parent.InsertPairs("logstash_format", fmt.Sprint(*o.Elasticsearch.LogstashFormat))
if es.LogstashFormat != nil {
parent.InsertPairs("logstash_format", fmt.Sprint(*es.LogstashFormat))
}

if o.Elasticsearch.LogstashPrefix != nil {
parent.InsertPairs("logstash_prefix", fmt.Sprint(*o.Elasticsearch.LogstashPrefix))
if es.LogstashPrefix != nil {
parent.InsertPairs("logstash_prefix", fmt.Sprint(*es.LogstashPrefix))
}

return parent, nil
Expand Down
1 change: 1 addition & 0 deletions apis/fluentd/v1alpha1/plugins/params/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ const (
KafkaOutputType OutputType = "kafka2"
ElasticsearchOutputType OutputType = "elasticsearch"
ElasticsearchDataStreamOutputType OutputType = "elasticsearch_data_stream"
ElasticsearchDynamicOutputType OutputType = "elasticsearch_dynamic"
OpensearchOutputType OutputType = "opensearch"
S3OutputType OutputType = "s3"
LokiOutputType OutputType = "loki"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
<source>
@type forward
bind 0.0.0.0
port 24224
</source>
<match **>
@id main
@type label_router
<route>
@label @a2170d34e9940ec56d328100e375c43e
<match>
namespaces default,kube-system
</match>
</route>
</match>
<label @a2170d34e9940ec56d328100e375c43e>
<match **>
@id ClusterFluentdConfig-cluster-fluentd-config::cluster::clusteroutput::fluentd-output-es-dynamic-0
@type elasticsearch_dynamic
host elasticsearch-logging-data.kubesphere-logging-system.svc
logstash_format true
logstash_prefix ks-${record['kubernetes']['namespace_name']}
port 9200
</match>
</label>
5 changes: 5 additions & 0 deletions apis/fluentd/v1alpha1/tests/helper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ func Test_ClusterCfgOutput2ESBulkThreshold(t *testing.T) {
testClusterConfigWithFiltersAndOutputs(t, sl, Fluentd, &FluentdClusterFluentdConfig1, []fluentdv1alpha1.ClusterFilter{}, []fluentdv1alpha1.ClusterOutput{FluentdclusterOutput2ESBulkThreshold}, "./expected/fluentd-cluster-cfg-output-es-bulk-threshold.cfg", false)
}

func Test_ClusterCfgOutput2ESDynamic(t *testing.T) {
sl := plugins.NewSecretLoader(nil, Fluentd.Namespace, logr.Logger{})
testClusterConfigWithFiltersAndOutputs(t, sl, Fluentd, &FluentdClusterFluentdConfig1, []fluentdv1alpha1.ClusterFilter{}, []fluentdv1alpha1.ClusterOutput{FluentdclusterOutput2ESDynamic}, "./expected/fluentd-cluster-cfg-output-es-dynamic.cfg", false)
}

func Test_ClusterCfgOutput2ESDataStream(t *testing.T) {
sl := plugins.NewSecretLoader(nil, Fluentd.Namespace, logr.Logger{})
testClusterConfigWithFiltersAndOutputs(t, sl, Fluentd, &FluentdClusterFluentdConfig1, []fluentdv1alpha1.ClusterFilter{}, []fluentdv1alpha1.ClusterOutput{FluentdclusterOutput2ESDataStream}, "./expected/fluentd-cluster-cfg-output-es-data-stream.cfg", false)
Expand Down
18 changes: 18 additions & 0 deletions apis/fluentd/v1alpha1/tests/tools.go
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,23 @@ spec:
bulkMessageRequestThreshold: 20000000
`

FluentdclusterOutput2ESDynamic fluentdv1alpha1.ClusterOutput
FluentdclusterOutput2ESDynamicRaw = `
apiVersion: fluentd.fluent.io/v1alpha1
kind: ClusterOutput
metadata:
name: fluentd-output-es-dynamic
labels:
output.fluentd.fluent.io/enabled: "true"
spec:
outputs:
- elasticsearchDynamic:
host: elasticsearch-logging-data.kubesphere-logging-system.svc
port: 9200
logstashFormat: true
logstashPrefix: ks-${record['kubernetes']['namespace_name']}
`

FluentdclusterOutput2ESDataStream fluentdv1alpha1.ClusterOutput
FluentdclusterOutput2ESDataStreamRaw = `
apiVersion: fluentd.fluent.io/v1alpha1
Expand Down Expand Up @@ -1279,6 +1296,7 @@ func init() {
MustParseIntoObject(FluentdClusterOutputMemoryBufferRaw, &FluentdClusterOutputMemoryBuffer)
MustParseIntoObject(FluentdclusterOutput2ESRaw, &FluentdclusterOutput2ES)
MustParseIntoObject(FluentdclusterOutput2ESBulkThresholdRaw, &FluentdclusterOutput2ESBulkThreshold)
MustParseIntoObject(FluentdclusterOutput2ESDynamicRaw, &FluentdclusterOutput2ESDynamic)
MustParseIntoObject(FluentdclusterOutput2ESDataStreamRaw, &FluentdclusterOutput2ESDataStream)
MustParseIntoObject(FluentdclusterOutput2CopyESDataStreamRaw, &FluentdclusterOutput2CopyESDataStream)
MustParseIntoObject(FluentdOutput2ES1Raw, &FluentdOutput2ES1)
Expand Down
Loading
Loading