Skip to content

Commit e0e8a64

Browse files
alok87alok87
committed
Beta release changes
1 parent c0c132c commit e0e8a64

3 files changed

Lines changed: 232 additions & 1 deletion

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ The pipeline is a combination of services deployed independently. This repo hold
2020
kubectl get kafka
2121
```
2222

23-
- **Producer** Install Producer using [Strimzi](http://strimzi.io/) CRDs and [Debezium](https://debezium.io/). Creating the kafkaconect and kafkaconnector creates a kafkaconnect pod in the cluster which start streaming the data from the source(MYSQL, RDS, etc..) to Kafka.
23+
- **Producer** Install Producer using [Strimzi](http://strimzi.io/) CRDs and [Debezium](https://debezium.io/). Creating the kafkaconnect and kafkaconnector creates a kafkaconnect pod in the cluster which start streaming the data from the source(MYSQL, RDS, etc..) to Kafka.
2424
```
2525
kubectl get kafkaconnect
2626
kubectl get kafkaconnector

‎cmd/redshiftsink/main.go‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
/*
2+
Licensed under the Apache License, Version 2.0 (the "License");
3+
you may not use this file except in compliance with the License.
4+
You may obtain a copy of the License at
5+
6+
http://www.apache.org/licenses/LICENSE-2.0
7+
8+
Unless required by applicable law or agreed to in writing, software
9+
distributed under the License is distributed on an "AS IS" BASIS,
10+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
See the License for the specific language governing permissions and
12+
limitations under the License.
13+
*/
14+
15+
package main
16+
17+
import (
18+
"context"
19+
"flag"
20+
"math/rand"
21+
"os"
22+
"strings"
23+
"sync"
24+
"time"
25+
26+
"github.com/practo/klog/v2"
27+
prometheus "github.com/practo/tipoca-stream/pkg/prometheus"
28+
redshift "github.com/practo/tipoca-stream/pkg/redshift"
29+
pflag "github.com/spf13/pflag"
30+
"k8s.io/apimachinery/pkg/runtime"
31+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
32+
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
33+
"k8s.io/klog/klogr"
34+
ctrl "sigs.k8s.io/controller-runtime"
35+
client "sigs.k8s.io/controller-runtime/pkg/client"
36+
metrics "sigs.k8s.io/controller-runtime/pkg/metrics"
37+
38+
tipocav1 "github.com/practo/tipoca-stream/api/v1"
39+
"github.com/practo/tipoca-stream/controllers"
40+
// +kubebuilder:scaffold:imports
41+
)
42+
43+
var (
44+
scheme = runtime.NewScheme()
45+
setupLog = ctrl.Log.WithName("setup")
46+
)
47+
48+
func init() {
49+
klog.InitFlags(nil)
50+
pflag.CommandLine.AddGoFlag(flag.CommandLine.Lookup("v"))
51+
_ = clientgoscheme.AddToScheme(scheme)
52+
53+
_ = tipocav1.AddToScheme(scheme)
54+
// +kubebuilder:scaffold:scheme
55+
}
56+
57+
func main() {
58+
rand.Seed(time.Now().UnixNano())
59+
60+
var enableLeaderElection, collectRedshiftMetrics bool
61+
var batcherImage, loaderImage, secretRefName, secretRefNamespace, kafkaVersion, metricsAddr, allowedRsks, prometheusURL string
62+
var redshiftMaxOpenConns, redshiftMaxIdleConns int
63+
flag.StringVar(&batcherImage, "default-batcher-image", "practodev/redshiftbatcher:v1.0.0-beta.1", "image to use for the redshiftbatcher")
64+
flag.StringVar(&loaderImage, "default-loader-image", "practodev/redshiftloader:v1.0.0-beta.1", "image to use for the redshiftloader")
65+
flag.StringVar(&secretRefName, "default-secret-ref-name", "redshiftsink-secret", "default secret name for all redshiftsink secret")
66+
flag.StringVar(&secretRefNamespace, "default-secret-ref-namespace", "ts-redshiftsink-latest", "default namespace where redshiftsink secret is there")
67+
flag.BoolVar(&collectRedshiftMetrics, "collect-redshift-metrics", false, "collectRedshiftMetrics when enabled collects redshift metrics for better calculations, used for calculating throttling seconds value at present for each table")
68+
flag.StringVar(&kafkaVersion, "default-kafka-version", "2.6.0", "default kafka version")
69+
flag.StringVar(&metricsAddr, "metrics-addr", ":8443", "The address the metric endpoint binds to.")
70+
flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
71+
flag.IntVar(&redshiftMaxOpenConns, "default-redshift-max-open-conns", 10, "the maximum number of open connections allowed to redshift per redshiftsink resource")
72+
flag.IntVar(&redshiftMaxIdleConns, "default-redshift-max-idle-conns", 2, "the maximum number of idle connections allowed to redshift per redshiftsink resource")
73+
flag.StringVar(&allowedRsks, "allowed-rsks", "", "comma separated list of names of rsk resources to allow, if empty all rsk resources are allowed")
74+
flag.StringVar(&prometheusURL, "prometheus-url", "", "optional, giving prometheus makes the operator enable new features using time series data. Features: loader throttling, resetting offsets of 0 throughput topics.")
75+
flag.Parse()
76+
77+
ctrl.SetLogger(klogr.New())
78+
79+
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
80+
Scheme: scheme,
81+
MetricsBindAddress: metricsAddr,
82+
Port: 9443,
83+
LeaderElection: enableLeaderElection,
84+
LeaderElectionID: "854ae6e3.",
85+
})
86+
if err != nil {
87+
setupLog.Error(err, "unable to start manager")
88+
os.Exit(1)
89+
}
90+
91+
uncachedClient, err := client.New(
92+
mgr.GetConfig(),
93+
client.Options{Scheme: mgr.GetScheme()},
94+
)
95+
if err != nil {
96+
setupLog.Error(err, "unable to make uncached client")
97+
os.Exit(1)
98+
}
99+
var prometheusClient prometheus.Client
100+
if prometheusURL != "" {
101+
prometheusClient, err = prometheus.NewClient(prometheusURL)
102+
if err != nil {
103+
setupLog.Error(err, "unable to init prometheus")
104+
os.Exit(1)
105+
}
106+
}
107+
var allowedResources []string
108+
if allowedRsks != "" {
109+
allowedResources = strings.Split(allowedRsks, ",")
110+
}
111+
112+
if err = (&controllers.RedshiftSinkReconciler{
113+
Client: uncachedClient,
114+
Log: ctrl.Log.WithName("controllers").WithName("RedshiftSink"),
115+
Scheme: mgr.GetScheme(),
116+
Recorder: mgr.GetEventRecorderFor("redshiftsink-reconciler"),
117+
KafkaClients: new(sync.Map),
118+
KafkaTopicRegexes: new(sync.Map),
119+
KafkaTopicsCache: new(sync.Map),
120+
KafkaRealtimeCache: new(sync.Map),
121+
ReleaseCache: new(sync.Map),
122+
GitCache: new(sync.Map),
123+
IncludeTablesCache: new(sync.Map),
124+
DefaultBatcherImage: batcherImage,
125+
DefaultLoaderImage: loaderImage,
126+
DefaultSecretRefName: secretRefName,
127+
DefaultSecretRefNamespace: secretRefNamespace,
128+
DefaultKafkaVersion: kafkaVersion,
129+
DefaultRedshiftMaxOpenConns: redshiftMaxOpenConns,
130+
DefaultRedshiftMaxIdleConns: redshiftMaxIdleConns,
131+
AllowedResources: allowedResources,
132+
PrometheusClient: prometheusClient,
133+
RedshiftMetrics: collectRedshiftMetrics,
134+
}).SetupWithManager(mgr); err != nil {
135+
setupLog.Error(err, "unable to create controller", "controller", "RedshiftSink")
136+
os.Exit(1)
137+
}
138+
// +kubebuilder:scaffold:builder
139+
140+
if !collectRedshiftMetrics {
141+
setupLog.Info("Starting Operator... (redshift metrics feature is disabled)")
142+
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
143+
setupLog.Error(err, "problem running manager")
144+
os.Exit(1)
145+
}
146+
return
147+
}
148+
149+
ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
150+
defer cancel()
151+
152+
setupLog.Info("Configuring Redshift exporter...")
153+
redshiftClient, err := controllers.NewRedshiftConn(uncachedClient, secretRefName, secretRefNamespace)
154+
if err != nil {
155+
setupLog.Error(err, "problem initializing redshift connection")
156+
os.Exit(1)
157+
}
158+
redshiftCollector := redshift.NewRedshiftCollector(redshiftClient)
159+
wg := &sync.WaitGroup{}
160+
wg.Add(1)
161+
go redshiftCollector.Fetch(ctx, wg)
162+
metrics.Registry.MustRegister(redshiftCollector)
163+
164+
setupLog.Info("Starting Operator...")
165+
if err := mgr.Start(ctx); err != nil {
166+
setupLog.Error(err, "problem running manager")
167+
os.Exit(1)
168+
}
169+
170+
wg.Wait()
171+
}

‎config/operator/redshiftsink_operator.yaml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: redshiftsink-operator
5+
namespace: kube-system
6+
---
7+
apiVersion: apps/v1
8+
kind: Deployment
9+
metadata:
10+
name: redshiftsink-operator
11+
namespace: kube-system
12+
labels:
13+
app: redshiftsink-operator
14+
spec:
15+
strategy:
16+
type: Recreate
17+
replicas: 1
18+
selector:
19+
matchLabels:
20+
app: redshiftsink-operator
21+
template:
22+
metadata:
23+
labels:
24+
app: redshiftsink-operator
25+
spec:
26+
serviceAccountName: redshiftsink-operator
27+
tolerations:
28+
- effect: NoExecute
29+
operator: Exists
30+
- effect: NoSchedule
31+
operator: Exists
32+
volumes:
33+
- name: redshiftsink-secret-volume
34+
secret:
35+
secretName: redshiftsink-secret
36+
containers:
37+
- name: rsk-operator
38+
image: practodev/redshiftsink:v1.0.0-beta.1
39+
imagePullPolicy: Always
40+
volumeMounts:
41+
- name: redshiftsink-secret-volume
42+
mountPath: /secret.yaml
43+
command:
44+
- /redshiftsink
45+
args:
46+
- -v=2
47+
- --default-batcher-image=practodev/redshiftbatcher:v1.0.0-beta.1
48+
- --default-loader-image=practodev/redshiftloader:v1.0.0-beta.1
49+
- --default-redshift-max-open-conns=10
50+
- --default-redshift-max-idle-conns=2
51+
- --allowed-rsks=
52+
- --promethus-url=
53+
- --collect-redshift-metrics=false
54+
resources:
55+
limits:
56+
cpu: 300m
57+
memory: 300Mi
58+
requests:
59+
cpu: 100m
60+
memory: 200Mi

0 commit comments

Comments
 (0)