Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apis/fluentbit/v1alpha2/clusterfilter_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ type FilterItem struct {
Wasm *filter.Wasm `json:"wasm,omitempty"`
// Sysinfo defines a Sysinfo Filter configuration.
Sysinfo *filter.Sysinfo `json:"sysinfo,omitempty"`
// GeoIP2 defines a GeoIP2 Filter configuration.
GeoIP2 *filter.GeoIP2 `json:"geoip2,omitempty"`
// CustomPlugin defines a Custom plugin configuration.
CustomPlugin *custom.CustomPlugin `json:"customPlugin,omitempty"`
}
Expand Down
57 changes: 57 additions & 0 deletions apis/fluentbit/v1alpha2/clusterfilter_types_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -797,3 +797,60 @@ func TestClusterFilterList_Load_Sysinfo(t *testing.T) {
hostname_key: hostname
`))
}

func TestClusterFilterList_Load_GeoIP2(t *testing.T) {
g := NewGomegaWithT(t)
sl := plugins.NewSecretLoader(nil, "testnamespace")

filters := ClusterFilterList{
Items: []ClusterFilter{
{
TypeMeta: metav1.TypeMeta{
APIVersion: "fluentbit.fluent.io/v1alpha2",
Kind: "ClusterFilter",
},
ObjectMeta: metav1.ObjectMeta{
Name: "filter-geoip2",
},
Spec: FilterSpec{
Match: "nginx.*",
FilterItems: []FilterItem{
{
GeoIP2: &filter.GeoIP2{
Database: "/fluent-bit/geoip/GeoLite2-City.mmdb",
LookupKeys: []string{"remote_addr"},
Records: []filter.GeoIP2Record{
{Key: "country", LookupKey: "remote_addr", Value: "%{country.names.en}"},
{Key: "city", LookupKey: "remote_addr", Value: "%{city.names.en}"},
},
},
},
},
},
},
},
}

clusterFilters, err := filters.Load(sl)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(clusterFilters).To(Equal(`[Filter]
Name geoip2
Match nginx.*
Database /fluent-bit/geoip/GeoLite2-City.mmdb
Lookup_Key remote_addr
Record country remote_addr %{country.names.en}
Record city remote_addr %{city.names.en}
`))

clusterFilters, err = filters.LoadAsYaml(sl, 0)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(clusterFilters).To(Equal(`filters:
- name: geoip2
match: "nginx.*"
database: /fluent-bit/geoip/GeoLite2-City.mmdb
lookup_key: remote_addr
record:
- country remote_addr %{country.names.en}
- city remote_addr %{city.names.en}
`))
}
67 changes: 67 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/filter/geoip2_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package filter

import (
"fmt"

"github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins"
"github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/params"
)

// +kubebuilder:object:generate:=true

// The GeoIP2 Filter plugin looks up IP addresses of a record in a MaxMind GeoIP2 or GeoLite2 database <br />
// and appends the location information, such as the country or the city, to the record. <br />
// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/filters/geoip2-filter**
type GeoIP2 struct {
plugins.CommonParams `json:",inline"`
// Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb.
// The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource.
// +kubebuilder:validation:Required
// +kubebuilder:validation:MinLength=1
Database string `json:"database"`
// Keys of the record whose value is an IP address to look up.
// +kubebuilder:validation:Required
// +kubebuilder:validation:MinItems:=1
// +kubebuilder:validation:items:Pattern:="^\\S+$"
LookupKeys []string `json:"lookupKeys"`
// Keys to append to the record, with the value looked up in the database.
// +kubebuilder:validation:Required
// +kubebuilder:validation:MinItems:=1
Records []GeoIP2Record `json:"records"`
}

// +kubebuilder:object:generate:=true

// GeoIP2Record appends Key to the record, with the value found at Value in the database for the IP address in LookupKey.
type GeoIP2Record struct {
// Key appended to the record.
// +kubebuilder:validation:Pattern:="^\\S+$"
Key string `json:"key"`
// Lookup key holding the IP address to look up. It must be one of lookupKeys, otherwise the value is null.
// +kubebuilder:validation:Pattern:="^\\S+$"
LookupKey string `json:"lookupKey"`
// Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}.
// The path is at most 63 characters long.
// +kubebuilder:validation:Pattern:="^%\\{[^\\s}]{1,63}\\}$"
Value string `json:"value"`
}

func (*GeoIP2) Name() string {
return "geoip2"
}

func (g *GeoIP2) Params(_ plugins.SecretLoader) (*params.KVs, error) {
kvs := params.NewKVs()
err := g.AddCommonParams(kvs)
if err != nil {
return kvs, err
}
plugins.InsertKVString(kvs, "Database", g.Database)
for _, key := range g.LookupKeys {
kvs.Insert("Lookup_Key", key)
}
for _, r := range g.Records {
kvs.Insert("Record", fmt.Sprintf("%s %s %s", r.Key, r.LookupKey, r.Value))
}
return kvs, nil
}
62 changes: 62 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/filter/geoip2_types_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package filter

import (
"testing"

"github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins"
"github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/params"
. "github.com/onsi/gomega"
)

func TestFilter_GeoIP2_Params(t *testing.T) {
g := NewGomegaWithT(t)

sl := plugins.NewSecretLoader(nil, "test namespace")

geoip2 := GeoIP2{
Database: "/fluent-bit/geoip/GeoLite2-City.mmdb",
LookupKeys: []string{"remote_addr", "x_forwarded_for"},
Records: []GeoIP2Record{
{Key: "country", LookupKey: "remote_addr", Value: "%{country.names.en}"},
{Key: "latitude", LookupKey: "remote_addr", Value: "%{location.latitude}"},
{Key: "client_country", LookupKey: "x_forwarded_for", Value: "%{country.iso_code}"},
},
}

expected := params.NewKVs()
expected.Insert("Database", "/fluent-bit/geoip/GeoLite2-City.mmdb")
expected.Insert("Lookup_Key", "remote_addr")
expected.Insert("Lookup_Key", "x_forwarded_for")
expected.Insert("Record", "country remote_addr %{country.names.en}")
expected.Insert("Record", "latitude remote_addr %{location.latitude}")
expected.Insert("Record", "client_country x_forwarded_for %{country.iso_code}")

kvs, err := geoip2.Params(sl)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(kvs).To(Equal(expected))
}

func TestFilter_GeoIP2_ParamsCommonParams(t *testing.T) {
g := NewGomegaWithT(t)

sl := plugins.NewSecretLoader(nil, "test namespace")

geoip2 := GeoIP2{
CommonParams: plugins.CommonParams{Alias: "geoip"},
Database: "/geoip/GeoLite2-Country.mmdb",
LookupKeys: []string{"ip"},
Records: []GeoIP2Record{
{Key: "country", LookupKey: "ip", Value: "%{country.iso_code}"},
},
}

expected := params.NewKVs()
expected.Insert("Alias", "geoip")
expected.Insert("Database", "/geoip/GeoLite2-Country.mmdb")
expected.Insert("Lookup_Key", "ip")
expected.Insert("Record", "country ip %{country.iso_code}")

kvs, err := geoip2.Params(sl)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(kvs).To(Equal(expected))
}
41 changes: 41 additions & 0 deletions apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions apis/fluentbit/v1alpha2/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,72 @@ spec:
type: object
x-kubernetes-preserve-unknown-fields: true
type: object
geoip2:
description: GeoIP2 defines a GeoIP2 Filter configuration.
properties:
alias:
description: Alias for the plugin
type: string
database:
description: |-
Path to the MaxMind database file (.mmdb), e.g. /fluent-bit/geoip/GeoLite2-City.mmdb.
The file can be mounted into the Fluent Bit pod with the volumes and volumeMounts of the FluentBit resource.
minLength: 1
type: string
lookupKeys:
description: Keys of the record whose value is an IP address
to look up.
items:
pattern: ^\S+$
type: string
minItems: 1
type: array
records:
description: Keys to append to the record, with the value
looked up in the database.
items:
description: GeoIP2Record appends Key to the record, with
the value found at Value in the database for the IP
address in LookupKey.
properties:
key:
description: Key appended to the record.
pattern: ^\S+$
type: string
lookupKey:
description: Lookup key holding the IP address to
look up. It must be one of lookupKeys, otherwise
the value is null.
pattern: ^\S+$
type: string
value:
description: |-
Path of the value in the database, written as %{path}, e.g. %{country.names.en} or %{location.latitude}.
The path is at most 63 characters long.
pattern: ^%\{[^\s}]{1,63}\}$
type: string
required:
- key
- lookupKey
- value
type: object
minItems: 1
type: array
retryLimit:
description: 'RetryLimit describes how many times fluent-bit
should retry to send data to a specific output. If set
to false fluent-bit will try indefinitely. If set to any
integer N>0 it will try at most N+1 times. Leading zeros
are not allowed (values such as 007, 0150, 01 do not work).
If this property is not defined fluent-bit will use the
default value: 1.'
pattern: ^(((f|F)alse)|(no_limits)|(no_retries)|([1-9]+[0-9]*))$
type: string
required:
- database
- lookupKeys
- records
type: object
grep:
description: Grep defines Grep Filter configuration.
properties:
Expand Down
Loading
Loading