Skip to content

Introduce masterPoolerServiceAnnotations & replicaPoolerServiceAnnotations #3050

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
privatecoder wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Introduce masterPoolerServiceAnnotations & replicaPoolerServiceAnnotations #3050

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions charts/postgres-operator/crds/postgresqls.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,10 @@ spec:
items:
type: string
pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$'
masterPoolerServiceAnnotations:
type: object
additionalProperties:
type: string
masterServiceAnnotations:
type: object
additionalProperties:
Expand Down Expand Up @@ -408,6 +412,10 @@ spec:
replicaLoadBalancer:
type: boolean
description: deprecated
replicaPoolerServiceAnnotations:
type: object
additionalProperties:
type: string
replicaServiceAnnotations:
type: object
additionalProperties:
Expand Down
1 change: 1 addition & 0 deletions docs/administrator.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -905,6 +905,7 @@ precedence):
2. Globally configured `custom_service_annotations`
3. `serviceAnnotations` specified in the cluster manifest
4. `masterServiceAnnotations` and `replicaServiceAnnotations` specified in the cluster manifest
5. `masterPoolerServiceAnnotations` and `replicaPoolerServiceAnnotations` specified in the cluster manifest (only for connection pooler services)

To limit the range of IP addresses that can reach a load balancer, specify the
desired ranges in the `allowedSourceRanges` field (applies to both master and
Expand Down
12 changes: 12 additions & 0 deletions docs/reference/cluster_manifest.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,18 @@ These parameters are grouped directly under the `spec` key in the manifest.
This field overrides `serviceAnnotations` with the same key for the replica
service if not empty.

* **masterPoolerServiceAnnotations**
A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)
to the master connection pooler service created for the database cluster.
This field overrides `serviceAnnotations` and `masterServiceAnnotations`
with the same key for the master pooler service if not empty.

* **replicaPoolerServiceAnnotations**
A map of key value pairs that gets attached as [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)
to the replica connection pooler service created for the database cluster.
This field overrides `serviceAnnotations` and `replicaServiceAnnotations`
with the same key for the replica pooler service if not empty.

* **enableShmVolume**
Start a database pod without limitations on shm memory. By default Docker
limit `/dev/shm` to `64M` (see e.g. the [docker
Expand Down
12 changes: 12 additions & 0 deletions manifests/postgresql.crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3256,6 +3256,12 @@ spec:
pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$'
type: string
type: array
masterPoolerServiceAnnotations:
additionalProperties:
type: string
description: MasterPoolerServiceAnnotations takes precedence over other
annotations for master pooler service if not empty
type: object
masterServiceAnnotations:
additionalProperties:
type: string
Expand Down Expand Up @@ -3560,6 +3566,12 @@ spec:
replicaLoadBalancer:
description: deprecated
type: boolean
replicaPoolerServiceAnnotations:
additionalProperties:
type: string
description: ReplicaPoolerServiceAnnotations takes precedence over other
annotations for replica pooler service if not empty
type: object
replicaServiceAnnotations:
additionalProperties:
type: string
Expand Down
12 changes: 12 additions & 0 deletions pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3256,6 +3256,12 @@ spec:
pattern: '^\ *((Mon|Tue|Wed|Thu|Fri|Sat|Sun):(2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))-((2[0-3]|[01]?\d):([0-5]?\d)|(2[0-3]|[01]?\d):([0-5]?\d))\ *$'
type: string
type: array
masterPoolerServiceAnnotations:
additionalProperties:
type: string
description: MasterPoolerServiceAnnotations takes precedence over other
annotations for master pooler service if not empty
type: object
masterServiceAnnotations:
additionalProperties:
type: string
Expand Down Expand Up @@ -3560,6 +3566,12 @@ spec:
replicaLoadBalancer:
description: deprecated
type: boolean
replicaPoolerServiceAnnotations:
additionalProperties:
type: string
description: ReplicaPoolerServiceAnnotations takes precedence over other
annotations for replica pooler service if not empty
type: object
replicaServiceAnnotations:
additionalProperties:
type: string
Expand Down
8 changes: 6 additions & 2 deletions pkg/apis/acid.zalan.do/v1/postgresql_type.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,12 @@ type PostgresSpec struct {
// MasterServiceAnnotations takes precedence over ServiceAnnotations for master role if not empty
MasterServiceAnnotations map[string]string `json:"masterServiceAnnotations,omitempty"`
// ReplicaServiceAnnotations takes precedence over ServiceAnnotations for replica role if not empty
ReplicaServiceAnnotations map[string]string `json:"replicaServiceAnnotations,omitempty"`
TLS *TLSDescription `json:"tls,omitempty"`
ReplicaServiceAnnotations map[string]string `json:"replicaServiceAnnotations,omitempty"`
// MasterPoolerServiceAnnotations takes precedence over other annotations for master pooler service if not empty
MasterPoolerServiceAnnotations map[string]string `json:"masterPoolerServiceAnnotations,omitempty"`
// ReplicaPoolerServiceAnnotations takes precedence over other annotations for replica pooler service if not empty
ReplicaPoolerServiceAnnotations map[string]string `json:"replicaPoolerServiceAnnotations,omitempty"`
TLS *TLSDescription `json:"tls,omitempty"`
AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"`
Streams []Stream `json:"streams,omitempty"`
Env []v1.EnvVar `json:"env,omitempty"`
Expand Down
14 changes: 14 additions & 0 deletions pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

130 changes: 130 additions & 0 deletions pkg/cluster/cluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1026,6 +1026,136 @@ func TestServiceAnnotations(t *testing.T) {
}
}

func TestPoolerServiceAnnotations(t *testing.T) {
enabled := true
tests := []struct {
about string
role PostgresRole
enableMasterPoolerLoadBalancer *bool
enableReplicaPoolerLoadBalancer *bool
operatorAnnotations map[string]string
serviceAnnotations map[string]string
masterServiceAnnotations map[string]string
replicaServiceAnnotations map[string]string
masterPoolerServiceAnnotations map[string]string
replicaPoolerServiceAnnotations map[string]string
expect map[string]string
}{
{
about: "Master pooler annotations override service annotations",
role: "master",
operatorAnnotations: make(map[string]string),
serviceAnnotations: map[string]string{
"foo": "bar",
},
masterServiceAnnotations: map[string]string{
"baz": "qux",
},
masterPoolerServiceAnnotations: map[string]string{
"foo": "pooler-bar",
"pooler": "master",
},
expect: map[string]string{
"foo": "pooler-bar",
"baz": "qux",
"pooler": "master",
},
},
{
about: "Replica pooler annotations override service annotations",
role: "replica",
operatorAnnotations: make(map[string]string),
serviceAnnotations: map[string]string{
"foo": "bar",
},
replicaServiceAnnotations: map[string]string{
"baz": "qux",
},
replicaPoolerServiceAnnotations: map[string]string{
"foo": "pooler-bar",
"pooler": "replica",
},
expect: map[string]string{
"foo": "pooler-bar",
"baz": "qux",
"pooler": "replica",
},
},
{
about: "Master pooler with load balancer and pooler annotations",
role: "master",
enableMasterPoolerLoadBalancer: &enabled,
operatorAnnotations: make(map[string]string),
serviceAnnotations: make(map[string]string),
masterPoolerServiceAnnotations: map[string]string{
"pooler-key": "pooler-value",
},
expect: map[string]string{
"external-dns.alpha.kubernetes.io/hostname": "acid-test-pooler-stg.test.db.example.com",
"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "3600",
"pooler-key": "pooler-value",
},
},
{
about: "Master pooler annotations not applied to replica pooler",
role: "replica",
operatorAnnotations: make(map[string]string),
serviceAnnotations: make(map[string]string),
masterPoolerServiceAnnotations: map[string]string{
"master-only": "value",
},
expect: make(map[string]string),
},
{
about: "Replica pooler annotations not applied to master pooler",
role: "master",
operatorAnnotations: make(map[string]string),
serviceAnnotations: make(map[string]string),
replicaPoolerServiceAnnotations: map[string]string{
"replica-only": "value",
},
expect: make(map[string]string),
},
}

for _, tt := range tests {
t.Run(tt.about, func(t *testing.T) {
cl.OpConfig.CustomServiceAnnotations = tt.operatorAnnotations
cl.OpConfig.EnableMasterPoolerLoadBalancer = false
cl.OpConfig.EnableReplicaPoolerLoadBalancer = false
cl.OpConfig.MasterDNSNameFormat = "{cluster}-stg.{namespace}.{hostedzone}"
cl.OpConfig.MasterLegacyDNSNameFormat = "{cluster}-stg.{team}.{hostedzone}"
cl.OpConfig.ReplicaDNSNameFormat = "{cluster}-stg-repl.{namespace}.{hostedzone}"
cl.OpConfig.ReplicaLegacyDNSNameFormat = "{cluster}-stg-repl.{team}.{hostedzone}"
cl.OpConfig.DbHostedZone = "db.example.com"

cl.Postgresql.Spec.ClusterName = ""
cl.Postgresql.Spec.TeamID = "acid"
cl.Postgresql.Spec.ServiceAnnotations = tt.serviceAnnotations
cl.Postgresql.Spec.MasterServiceAnnotations = tt.masterServiceAnnotations
cl.Postgresql.Spec.ReplicaServiceAnnotations = tt.replicaServiceAnnotations
cl.Postgresql.Spec.MasterPoolerServiceAnnotations = tt.masterPoolerServiceAnnotations
cl.Postgresql.Spec.ReplicaPoolerServiceAnnotations = tt.replicaPoolerServiceAnnotations
cl.Postgresql.Spec.EnableMasterPoolerLoadBalancer = tt.enableMasterPoolerLoadBalancer
cl.Postgresql.Spec.EnableReplicaPoolerLoadBalancer = tt.enableReplicaPoolerLoadBalancer

got := cl.generatePoolerServiceAnnotations(tt.role, &cl.Postgresql.Spec)
if got == nil {
got = make(map[string]string)
}
if len(tt.expect) != len(got) {
t.Errorf("expected %d annotation(s), got %d: %v", len(tt.expect), len(got), got)
return
}
for k, v := range got {
if tt.expect[k] != v {
t.Errorf("expected annotation '%v' with value '%v', got value '%v'", k, tt.expect[k], v)
}
}
})
}
}

func TestInitSystemUsers(t *testing.T) {
// reset system users, pooler and stream section
cl.systemUsers = make(map[string]spec.PgUser)
Expand Down
10 changes: 10 additions & 0 deletions pkg/cluster/connection_pooler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/zalando/postgres-operator/pkg/util/constants"
"github.com/zalando/postgres-operator/pkg/util/k8sutil"
"github.com/zalando/postgres-operator/pkg/util/retryutil"
"maps"
)

var poolerRunAsUser = int64(100)
Expand Down Expand Up @@ -532,6 +533,15 @@ func (c *Cluster) generatePoolerServiceAnnotations(role PostgresRole, spec *acid
var dnsString string
annotations := c.getCustomServiceAnnotations(role, spec)

if spec != nil {
switch role {
case Master:
maps.Copy(annotations, spec.MasterPoolerServiceAnnotations)
case Replica:
maps.Copy(annotations, spec.ReplicaPoolerServiceAnnotations)
}
}

if c.shouldCreateLoadBalancerForPoolerService(role, spec) {
// set ELB Timeout annotation with default value
if _, ok := annotations[constants.ElbTimeoutAnnotationName]; !ok {
Expand Down
Loading