diff --git a/apis/fluentbit/v1alpha2/clusterfilter_types.go b/apis/fluentbit/v1alpha2/clusterfilter_types.go index c657dec50..eef7d38da 100644 --- a/apis/fluentbit/v1alpha2/clusterfilter_types.go +++ b/apis/fluentbit/v1alpha2/clusterfilter_types.go @@ -77,6 +77,8 @@ type FilterItem struct { Wasm *filter.Wasm `json:"wasm,omitempty"` // Sysinfo defines a Sysinfo Filter configuration. Sysinfo *filter.Sysinfo `json:"sysinfo,omitempty"` + // GeoIP2 defines a GeoIP2 Filter configuration. + GeoIP2 *filter.GeoIP2 `json:"geoip2,omitempty"` // CustomPlugin defines a Custom plugin configuration. CustomPlugin *custom.CustomPlugin `json:"customPlugin,omitempty"` } diff --git a/apis/fluentbit/v1alpha2/clusterfilter_types_test.go b/apis/fluentbit/v1alpha2/clusterfilter_types_test.go index c51103f7f..447272fec 100644 --- a/apis/fluentbit/v1alpha2/clusterfilter_types_test.go +++ b/apis/fluentbit/v1alpha2/clusterfilter_types_test.go @@ -797,3 +797,60 @@ func TestClusterFilterList_Load_Sysinfo(t *testing.T) { hostname_key: hostname `)) } + +func TestClusterFilterList_Load_GeoIP2(t *testing.T) { + g := NewGomegaWithT(t) + sl := plugins.NewSecretLoader(nil, "testnamespace") + + filters := ClusterFilterList{ + Items: []ClusterFilter{ + { + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterFilter", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "filter-geoip2", + }, + Spec: FilterSpec{ + Match: "nginx.*", + FilterItems: []FilterItem{ + { + GeoIP2: &filter.GeoIP2{ + Database: "/fluent-bit/geoip/GeoLite2-City.mmdb", + LookupKeys: []string{"remote_addr"}, + Records: []filter.GeoIP2Record{ + {Key: "country", LookupKey: "remote_addr", Value: "%{country.names.en}"}, + {Key: "city", LookupKey: "remote_addr", Value: "%{city.names.en}"}, + }, + }, + }, + }, + }, + }, + }, + } + + clusterFilters, err := filters.Load(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(clusterFilters).To(Equal(`[Filter] + Name geoip2 + Match nginx.* + Database /fluent-bit/geoip/GeoLite2-City.mmdb + Lookup_Key remote_addr + Record country remote_addr %{country.names.en} + Record city remote_addr %{city.names.en} +`)) + + clusterFilters, err = filters.LoadAsYaml(sl, 0) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(clusterFilters).To(Equal(`filters: + - name: geoip2 + match: "nginx.*" + database: /fluent-bit/geoip/GeoLite2-City.mmdb + lookup_key: remote_addr + record: + - country remote_addr %{country.names.en} + - city remote_addr %{city.names.en} +`)) +} diff --git a/apis/fluentbit/v1alpha2/plugins/filter/geoip2_types.go b/apis/fluentbit/v1alpha2/plugins/filter/geoip2_types.go new file mode 100644 index 000000000..d3ce52e01 --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/filter/geoip2_types.go @@ -0,0 +1,67 @@ +package filter + +import ( + "fmt" + + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/params" +) + +// +kubebuilder:object:generate:=true + +// The GeoIP2 Filter plugin looks up IP addresses of a record in a MaxMind GeoIP2 or GeoLite2 database
+// and appends the location information, such as the country or the city, to the record.
+// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/filters/geoip2-filter** +type GeoIP2 struct { + plugins.CommonParams `json:",inline"` + // Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb. + // The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Database string `json:"database"` + // Keys of the record whose value is an IP address to look up. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:items:Pattern:="^\\S+$" + LookupKeys []string `json:"lookupKeys"` + // Keys to append to the record, with the value looked up in the database. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinItems:=1 + Records []GeoIP2Record `json:"records"` +} + +// +kubebuilder:object:generate:=true + +// GeoIP2Record appends Key to the record, with the value found at Value in the database for the IP address in LookupKey. +type GeoIP2Record struct { + // Key appended to the record. + // +kubebuilder:validation:Pattern:="^\\S+$" + Key string `json:"key"` + // Lookup key holding the IP address to look up. It must be one of lookupKeys, otherwise the value is null. + // +kubebuilder:validation:Pattern:="^\\S+$" + LookupKey string `json:"lookupKey"` + // Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}. + // The path is at most 63 characters long. + // +kubebuilder:validation:Pattern:="^%\\{[^\\s}]{1,63}\\}$" + Value string `json:"value"` +} + +func (*GeoIP2) Name() string { + return "geoip2" +} + +func (g *GeoIP2) Params(_ plugins.SecretLoader) (*params.KVs, error) { + kvs := params.NewKVs() + err := g.AddCommonParams(kvs) + if err != nil { + return kvs, err + } + plugins.InsertKVString(kvs, "Database", g.Database) + for _, key := range g.LookupKeys { + kvs.Insert("Lookup_Key", key) + } + for _, r := range g.Records { + kvs.Insert("Record", fmt.Sprintf("%s %s %s", r.Key, r.LookupKey, r.Value)) + } + return kvs, nil +} diff --git a/apis/fluentbit/v1alpha2/plugins/filter/geoip2_types_test.go b/apis/fluentbit/v1alpha2/plugins/filter/geoip2_types_test.go new file mode 100644 index 000000000..a4ac246ca --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/filter/geoip2_types_test.go @@ -0,0 +1,62 @@ +package filter + +import ( + "testing" + + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/params" + . "github.com/onsi/gomega" +) + +func TestFilter_GeoIP2_Params(t *testing.T) { + g := NewGomegaWithT(t) + + sl := plugins.NewSecretLoader(nil, "test namespace") + + geoip2 := GeoIP2{ + Database: "/fluent-bit/geoip/GeoLite2-City.mmdb", + LookupKeys: []string{"remote_addr", "x_forwarded_for"}, + Records: []GeoIP2Record{ + {Key: "country", LookupKey: "remote_addr", Value: "%{country.names.en}"}, + {Key: "latitude", LookupKey: "remote_addr", Value: "%{location.latitude}"}, + {Key: "client_country", LookupKey: "x_forwarded_for", Value: "%{country.iso_code}"}, + }, + } + + expected := params.NewKVs() + expected.Insert("Database", "/fluent-bit/geoip/GeoLite2-City.mmdb") + expected.Insert("Lookup_Key", "remote_addr") + expected.Insert("Lookup_Key", "x_forwarded_for") + expected.Insert("Record", "country remote_addr %{country.names.en}") + expected.Insert("Record", "latitude remote_addr %{location.latitude}") + expected.Insert("Record", "client_country x_forwarded_for %{country.iso_code}") + + kvs, err := geoip2.Params(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(kvs).To(Equal(expected)) +} + +func TestFilter_GeoIP2_ParamsCommonParams(t *testing.T) { + g := NewGomegaWithT(t) + + sl := plugins.NewSecretLoader(nil, "test namespace") + + geoip2 := GeoIP2{ + CommonParams: plugins.CommonParams{Alias: "geoip"}, + Database: "/geoip/GeoLite2-Country.mmdb", + LookupKeys: []string{"ip"}, + Records: []GeoIP2Record{ + {Key: "country", LookupKey: "ip", Value: "%{country.iso_code}"}, + }, + } + + expected := params.NewKVs() + expected.Insert("Alias", "geoip") + expected.Insert("Database", "/geoip/GeoLite2-Country.mmdb") + expected.Insert("Lookup_Key", "ip") + expected.Insert("Record", "country ip %{country.iso_code}") + + kvs, err := geoip2.Params(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(kvs).To(Equal(expected)) +} diff --git a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go index 209509999..dd6453b65 100644 --- a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go @@ -142,6 +142,47 @@ func (in *Condition) DeepCopy() *Condition { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GeoIP2) DeepCopyInto(out *GeoIP2) { + *out = *in + out.CommonParams = in.CommonParams + if in.LookupKeys != nil { + in, out := &in.LookupKeys, &out.LookupKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Records != nil { + in, out := &in.Records, &out.Records + *out = make([]GeoIP2Record, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GeoIP2. +func (in *GeoIP2) DeepCopy() *GeoIP2 { + if in == nil { + return nil + } + out := new(GeoIP2) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GeoIP2Record) DeepCopyInto(out *GeoIP2Record) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GeoIP2Record. +func (in *GeoIP2Record) DeepCopy() *GeoIP2Record { + if in == nil { + return nil + } + out := new(GeoIP2Record) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Grep) DeepCopyInto(out *Grep) { *out = *in diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go index c6b5a86fa..114e80fb9 100644 --- a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go @@ -708,6 +708,11 @@ func (in *FilterItem) DeepCopyInto(out *FilterItem) { *out = new(filter.Sysinfo) **out = **in } + if in.GeoIP2 != nil { + in, out := &in.GeoIP2, &out.GeoIP2 + *out = new(filter.GeoIP2) + (*in).DeepCopyInto(*out) + } if in.CustomPlugin != nil { in, out := &in.CustomPlugin, &out.CustomPlugin *out = new(custom.CustomPlugin) diff --git a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterfilters.yaml index e946c26c6..0c939c422 100644 --- a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_clusterfilters.yaml @@ -113,6 +113,72 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + geoip2: + description: GeoIP2 defines a GeoIP2 Filter configuration. + properties: + alias: + description: Alias for the plugin + type: string + database: + description: |- + Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb. + The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource. + minLength: 1 + type: string + lookupKeys: + description: Keys of the record whose value is an IP address + to look up. + items: + pattern: ^\S+$ + type: string + minItems: 1 + type: array + records: + description: Keys to append to the record, with the value + looked up in the database. + items: + description: GeoIP2Record appends Key to the record, with + the value found at Value in the database for the IP + address in LookupKey. + properties: + key: + description: Key appended to the record. + pattern: ^\S+$ + type: string + lookupKey: + description: Lookup key holding the IP address to + look up. It must be one of lookupKeys, otherwise + the value is null. + pattern: ^\S+$ + type: string + value: + description: |- + Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}. + The path is at most 63 characters long. + pattern: ^%\{[^\s}]{1,63}\}$ + type: string + required: + - key + - lookupKey + - value + type: object + minItems: 1 + type: array + retryLimit: + description: 'RetryLimit describes how many times fluent-bit + should retry to send data to a specific output. If set + to false fluent-bit will try indefinitely. If set to any + integer N>0 it will try at most N+1 times. Leading zeros + are not allowed (values such as 007, 0150, 01 do not work). + If this property is not defined fluent-bit will use the + default value: 1.' + pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$ + type: string + required: + - database + - lookupKeys + - records + type: object grep: description: Grep defines Grep Filter configuration. properties: diff --git a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_filters.yaml index 9656643e8..4c9c50705 100644 --- a/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator-fluent-bit-crds/templates/fluentbit.fluent.io_filters.yaml @@ -113,6 +113,72 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + geoip2: + description: GeoIP2 defines a GeoIP2 Filter configuration. + properties: + alias: + description: Alias for the plugin + type: string + database: + description: |- + Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb. + The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource. + minLength: 1 + type: string + lookupKeys: + description: Keys of the record whose value is an IP address + to look up. + items: + pattern: ^\S+$ + type: string + minItems: 1 + type: array + records: + description: Keys to append to the record, with the value + looked up in the database. + items: + description: GeoIP2Record appends Key to the record, with + the value found at Value in the database for the IP + address in LookupKey. + properties: + key: + description: Key appended to the record. + pattern: ^\S+$ + type: string + lookupKey: + description: Lookup key holding the IP address to + look up. It must be one of lookupKeys, otherwise + the value is null. + pattern: ^\S+$ + type: string + value: + description: |- + Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}. + The path is at most 63 characters long. + pattern: ^%\{[^\s}]{1,63}\}$ + type: string + required: + - key + - lookupKey + - value + type: object + minItems: 1 + type: array + retryLimit: + description: 'RetryLimit describes how many times fluent-bit + should retry to send data to a specific output. If set + to false fluent-bit will try indefinitely. If set to any + integer N>0 it will try at most N+1 times. Leading zeros + are not allowed (values such as 007, 0150, 01 do not work). + If this property is not defined fluent-bit will use the + default value: 1.' + pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$ + type: string + required: + - database + - lookupKeys + - records + type: object grep: description: Grep defines Grep Filter configuration. properties: diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_clusterfilters.yaml index 974042bf5..253114616 100644 --- a/charts/fluent-operator/crds/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator/crds/fluentbit.fluent.io_clusterfilters.yaml @@ -111,6 +111,72 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + geoip2: + description: GeoIP2 defines a GeoIP2 Filter configuration. + properties: + alias: + description: Alias for the plugin + type: string + database: + description: |- + Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb. + The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource. + minLength: 1 + type: string + lookupKeys: + description: Keys of the record whose value is an IP address + to look up. + items: + pattern: ^\S+$ + type: string + minItems: 1 + type: array + records: + description: Keys to append to the record, with the value + looked up in the database. + items: + description: GeoIP2Record appends Key to the record, with + the value found at Value in the database for the IP + address in LookupKey. + properties: + key: + description: Key appended to the record. + pattern: ^\S+$ + type: string + lookupKey: + description: Lookup key holding the IP address to + look up. It must be one of lookupKeys, otherwise + the value is null. + pattern: ^\S+$ + type: string + value: + description: |- + Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}. + The path is at most 63 characters long. + pattern: ^%\{[^\s}]{1,63}\}$ + type: string + required: + - key + - lookupKey + - value + type: object + minItems: 1 + type: array + retryLimit: + description: 'RetryLimit describes how many times fluent-bit + should retry to send data to a specific output. If set + to false fluent-bit will try indefinitely. If set to any + integer N>0 it will try at most N+1 times. Leading zeros + are not allowed (values such as 007, 0150, 01 do not work). + If this property is not defined fluent-bit will use the + default value: 1.' + pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$ + type: string + required: + - database + - lookupKeys + - records + type: object grep: description: Grep defines Grep Filter configuration. properties: diff --git a/charts/fluent-operator/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/crds/fluentbit.fluent.io_filters.yaml index 792eedb12..16ab1dec3 100644 --- a/charts/fluent-operator/crds/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator/crds/fluentbit.fluent.io_filters.yaml @@ -111,6 +111,72 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + geoip2: + description: GeoIP2 defines a GeoIP2 Filter configuration. + properties: + alias: + description: Alias for the plugin + type: string + database: + description: |- + Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb. + The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource. + minLength: 1 + type: string + lookupKeys: + description: Keys of the record whose value is an IP address + to look up. + items: + pattern: ^\S+$ + type: string + minItems: 1 + type: array + records: + description: Keys to append to the record, with the value + looked up in the database. + items: + description: GeoIP2Record appends Key to the record, with + the value found at Value in the database for the IP + address in LookupKey. + properties: + key: + description: Key appended to the record. + pattern: ^\S+$ + type: string + lookupKey: + description: Lookup key holding the IP address to + look up. It must be one of lookupKeys, otherwise + the value is null. + pattern: ^\S+$ + type: string + value: + description: |- + Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}. + The path is at most 63 characters long. + pattern: ^%\{[^\s}]{1,63}\}$ + type: string + required: + - key + - lookupKey + - value + type: object + minItems: 1 + type: array + retryLimit: + description: 'RetryLimit describes how many times fluent-bit + should retry to send data to a specific output. If set + to false fluent-bit will try indefinitely. If set to any + integer N>0 it will try at most N+1 times. Leading zeros + are not allowed (values such as 007, 0150, 01 do not work). + If this property is not defined fluent-bit will use the + default value: 1.' + pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$ + type: string + required: + - database + - lookupKeys + - records + type: object grep: description: Grep defines Grep Filter configuration. properties: diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml index 93ca5af07..26052b923 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml @@ -112,6 +112,72 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + geoip2: + description: GeoIP2 defines a GeoIP2 Filter configuration. + properties: + alias: + description: Alias for the plugin + type: string + database: + description: |- + Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb. + The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource. + minLength: 1 + type: string + lookupKeys: + description: Keys of the record whose value is an IP address + to look up. + items: + pattern: ^\S+$ + type: string + minItems: 1 + type: array + records: + description: Keys to append to the record, with the value + looked up in the database. + items: + description: GeoIP2Record appends Key to the record, with + the value found at Value in the database for the IP + address in LookupKey. + properties: + key: + description: Key appended to the record. + pattern: ^\S+$ + type: string + lookupKey: + description: Lookup key holding the IP address to + look up. It must be one of lookupKeys, otherwise + the value is null. + pattern: ^\S+$ + type: string + value: + description: |- + Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}. + The path is at most 63 characters long. + pattern: ^%\{[^\s}]{1,63}\}$ + type: string + required: + - key + - lookupKey + - value + type: object + minItems: 1 + type: array + retryLimit: + description: 'RetryLimit describes how many times fluent-bit + should retry to send data to a specific output. If set + to false fluent-bit will try indefinitely. If set to any + integer N>0 it will try at most N+1 times. Leading zeros + are not allowed (values such as 007, 0150, 01 do not work). + If this property is not defined fluent-bit will use the + default value: 1.' + pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$ + type: string + required: + - database + - lookupKeys + - records + type: object grep: description: Grep defines Grep Filter configuration. properties: diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml index 6ea074e4a..63b37631d 100644 --- a/config/crd/bases/fluentbit.fluent.io_filters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_filters.yaml @@ -112,6 +112,72 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + geoip2: + description: GeoIP2 defines a GeoIP2 Filter configuration. + properties: + alias: + description: Alias for the plugin + type: string + database: + description: |- + Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb. + The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource. + minLength: 1 + type: string + lookupKeys: + description: Keys of the record whose value is an IP address + to look up. + items: + pattern: ^\S+$ + type: string + minItems: 1 + type: array + records: + description: Keys to append to the record, with the value + looked up in the database. + items: + description: GeoIP2Record appends Key to the record, with + the value found at Value in the database for the IP + address in LookupKey. + properties: + key: + description: Key appended to the record. + pattern: ^\S+$ + type: string + lookupKey: + description: Lookup key holding the IP address to + look up. It must be one of lookupKeys, otherwise + the value is null. + pattern: ^\S+$ + type: string + value: + description: |- + Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}. + The path is at most 63 characters long. + pattern: ^%\{[^\s}]{1,63}\}$ + type: string + required: + - key + - lookupKey + - value + type: object + minItems: 1 + type: array + retryLimit: + description: 'RetryLimit describes how many times fluent-bit + should retry to send data to a specific output. If set + to false fluent-bit will try indefinitely. If set to any + integer N>0 it will try at most N+1 times. Leading zeros + are not allowed (values such as 007, 0150, 01 do not work). + If this property is not defined fluent-bit will use the + default value: 1.' + pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$ + type: string + required: + - database + - lookupKeys + - records + type: object grep: description: Grep defines Grep Filter configuration. properties: diff --git a/docs/fluentbit.md b/docs/fluentbit.md index d8f6a9b18..8a9d5b8dd 100644 --- a/docs/fluentbit.md +++ b/docs/fluentbit.md @@ -289,6 +289,7 @@ Filter is the Schema for namespace level filter API | logToMetrics | LogToMetrics defines a Log to Metrics Filter configuration. | *[filter.LogToMetrics](plugins/fluentbit/filter/log_to_metrics.md) | | wasm | Wasm defines a Wasm configuration. | *[filter.Wasm](plugins/fluentbit/filter/wasm.md) | | sysinfo | Sysinfo defines a Sysinfo Filter configuration. | *[filter.Sysinfo](plugins/fluentbit/filter/sysinfo.md) | +| geoip2 | GeoIP2 defines a GeoIP2 Filter configuration. | *[filter.GeoIP2](plugins/fluentbit/filter/geoip2.md) | | customPlugin | CustomPlugin defines a Custom plugin configuration. | *[custom.CustomPlugin](plugins/fluentbit/custom/custom_plugin.md) | [Back to TOC](#table-of-contents) diff --git a/docs/plugins/fluentbit/filter/geoip2.md b/docs/plugins/fluentbit/filter/geoip2.md new file mode 100644 index 000000000..a0d9f4d76 --- /dev/null +++ b/docs/plugins/fluentbit/filter/geoip2.md @@ -0,0 +1,20 @@ +# GeoIP2 + +The GeoIP2 Filter plugin looks up IP addresses of a record in a MaxMind GeoIP2 or GeoLite2 database
and appends the location information, such as the country or the city, to the record.
**For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/filters/geoip2-filter** + + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| database | Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb. The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource. | string | +| lookupKeys | Keys of the record whose value is an IP address to look up. | []string | +| records | Keys to append to the record, with the value looked up in the database. | [][GeoIP2Record](#geoip2record) | +# GeoIP2Record + +GeoIP2Record appends Key to the record, with the value found at Value in the database for the IP address in LookupKey. + + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| key | Key appended to the record. | string | +| lookupKey | Lookup key holding the IP address to look up. It must be one of lookupKeys, otherwise the value is null. | string | +| value | Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}. The path is at most 63 characters long. | string | diff --git a/docs/plugins/fluentbit/index.md b/docs/plugins/fluentbit/index.md index b90fe4f6e..1cd1fd4c1 100644 --- a/docs/plugins/fluentbit/index.md +++ b/docs/plugins/fluentbit/index.md @@ -12,6 +12,7 @@ - [regex](parser/regex.md) - [ClusterFilter](../../fluentbit.md#ClusterFilter) - [aws](filter/aws.md) + - [geoip2](filter/geoip2.md) - [grep](filter/grep.md) - [kubernetes](filter/kubernetes.md) - [lua](filter/lua.md) diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 199b6dca7..0e1f65f46 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -111,6 +111,72 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + geoip2: + description: GeoIP2 defines a GeoIP2 Filter configuration. + properties: + alias: + description: Alias for the plugin + type: string + database: + description: |- + Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb. + The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource. + minLength: 1 + type: string + lookupKeys: + description: Keys of the record whose value is an IP address + to look up. + items: + pattern: ^\S+$ + type: string + minItems: 1 + type: array + records: + description: Keys to append to the record, with the value + looked up in the database. + items: + description: GeoIP2Record appends Key to the record, with + the value found at Value in the database for the IP + address in LookupKey. + properties: + key: + description: Key appended to the record. + pattern: ^\S+$ + type: string + lookupKey: + description: Lookup key holding the IP address to + look up. It must be one of lookupKeys, otherwise + the value is null. + pattern: ^\S+$ + type: string + value: + description: |- + Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}. + The path is at most 63 characters long. + pattern: ^%\{[^\s}]{1,63}\}$ + type: string + required: + - key + - lookupKey + - value + type: object + minItems: 1 + type: array + retryLimit: + description: 'RetryLimit describes how many times fluent-bit + should retry to send data to a specific output. If set + to false fluent-bit will try indefinitely. If set to any + integer N>0 it will try at most N+1 times. Leading zeros + are not allowed (values such as 007, 0150, 01 do not work). + If this property is not defined fluent-bit will use the + default value: 1.' + pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$ + type: string + required: + - database + - lookupKeys + - records + type: object grep: description: Grep defines Grep Filter configuration. properties: @@ -16389,6 +16455,72 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + geoip2: + description: GeoIP2 defines a GeoIP2 Filter configuration. + properties: + alias: + description: Alias for the plugin + type: string + database: + description: |- + Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb. + The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource. + minLength: 1 + type: string + lookupKeys: + description: Keys of the record whose value is an IP address + to look up. + items: + pattern: ^\S+$ + type: string + minItems: 1 + type: array + records: + description: Keys to append to the record, with the value + looked up in the database. + items: + description: GeoIP2Record appends Key to the record, with + the value found at Value in the database for the IP + address in LookupKey. + properties: + key: + description: Key appended to the record. + pattern: ^\S+$ + type: string + lookupKey: + description: Lookup key holding the IP address to + look up. It must be one of lookupKeys, otherwise + the value is null. + pattern: ^\S+$ + type: string + value: + description: |- + Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}. + The path is at most 63 characters long. + pattern: ^%\{[^\s}]{1,63}\}$ + type: string + required: + - key + - lookupKey + - value + type: object + minItems: 1 + type: array + retryLimit: + description: 'RetryLimit describes how many times fluent-bit + should retry to send data to a specific output. If set + to false fluent-bit will try indefinitely. If set to any + integer N>0 it will try at most N+1 times. Leading zeros + are not allowed (values such as 007, 0150, 01 do not work). + If this property is not defined fluent-bit will use the + default value: 1.' + pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$ + type: string + required: + - database + - lookupKeys + - records + type: object grep: description: Grep defines Grep Filter configuration. properties: