Skip to content

refactor service caching #855

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sknat merged 1 commit into from
Jan 14, 2026
Merged

refactor service caching #855

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 75 additions & 47 deletions calico-vpp-agent/services/service_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func getCnatVipDstPort(servicePort *v1.ServicePort, isNodePort bool) uint16 {
return uint16(servicePort.Port)
}

func (s *Server) buildCnatEntryForServicePort(servicePort *v1.ServicePort, service *v1.Service, epslices []*discoveryv1.EndpointSlice, serviceIP net.IP, isNodePort bool, svcInfo serviceInfo, isLocalOnly bool) *types.CnatTranslateEntry {
func (s *Server) buildCnatEntryForServicePort(servicePort *v1.ServicePort, epslices []*discoveryv1.EndpointSlice, serviceIP net.IP, isNodePort bool, svcInfo serviceInfo, isLocalOnly bool) *types.CnatTranslateEntry {
backends := make([]types.CnatEndpointTuple, 0)
// Find the endpoint subset port that exposes the port we're interested in
for _, epslice := range epslices {
Expand Down Expand Up @@ -167,15 +167,15 @@ func (s *Server) GetLocalService(service *v1.Service, epSlicesMap map[string]*di
for _, servicePort := range service.Spec.Ports {
for _, cip := range clusterIPs {
if !cip.IsUnspecified() && len(cip) > 0 {
entry := s.buildCnatEntryForServicePort(&servicePort, service, epSlices, cip, false /* isNodePort */, *serviceSpec, InternalIsLocalOnly(service))
entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, cip, false /* isNodePort */, *serviceSpec, InternalIsLocalOnly(service))
localService.Entries = append(localService.Entries, *entry)
}
}

for _, eip := range service.Spec.ExternalIPs {
extIP := net.ParseIP(eip)
if !extIP.IsUnspecified() && len(extIP) > 0 {
entry := s.buildCnatEntryForServicePort(&servicePort, service, epSlices, extIP, false /* isNodePort */, *serviceSpec, ExternalIsLocalOnly(service))
entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, extIP, false /* isNodePort */, *serviceSpec, ExternalIsLocalOnly(service))
localService.Entries = append(localService.Entries, *entry)
if ExternalIsLocalOnly(service) && len(entry.Backends) > 0 {
localService.SpecificRoutes = append(localService.SpecificRoutes, extIP)
Expand All @@ -186,7 +186,7 @@ func (s *Server) GetLocalService(service *v1.Service, epSlicesMap map[string]*di
for _, ingress := range service.Status.LoadBalancer.Ingress {
ingressIP := net.ParseIP(ingress.IP)
if !ingressIP.IsUnspecified() && len(ingressIP) > 0 {
entry := s.buildCnatEntryForServicePort(&servicePort, service, epSlices, ingressIP, false /* isNodePort */, *serviceSpec, ExternalIsLocalOnly(service))
entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, ingressIP, false /* isNodePort */, *serviceSpec, ExternalIsLocalOnly(service))
localService.Entries = append(localService.Entries, *entry)
if ExternalIsLocalOnly(service) && len(entry.Backends) > 0 {
localService.SpecificRoutes = append(localService.SpecificRoutes, ingressIP)
Expand All @@ -197,7 +197,7 @@ func (s *Server) GetLocalService(service *v1.Service, epSlicesMap map[string]*di
if service.Spec.Type == v1.ServiceTypeNodePort {
for _, nip := range nodeIPs {
if !nip.IsUnspecified() && len(nip) > 0 {
entry := s.buildCnatEntryForServicePort(&servicePort, service, epSlices, nip, true /* isNodePort */, *serviceSpec, false)
entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, nip, true /* isNodePort */, *serviceSpec, false)
localService.Entries = append(localService.Entries, *entry)
}
}
Expand All @@ -209,7 +209,7 @@ func (s *Server) GetLocalService(service *v1.Service, epSlicesMap map[string]*di
if service.Spec.Type == v1.ServiceTypeLoadBalancer && *service.Spec.AllocateLoadBalancerNodePorts {
for _, nip := range nodeIPs {
if !nip.IsUnspecified() && len(nip) > 0 {
entry := s.buildCnatEntryForServicePort(&servicePort, service, epSlices, nip, true /* isNodePort */, *serviceSpec, false)
entry := s.buildCnatEntryForServicePort(&servicePort, epSlices, nip, true /* isNodePort */, *serviceSpec, false)
localService.Entries = append(localService.Entries, *entry)
}
}
Expand Down Expand Up @@ -249,53 +249,60 @@ func (s *Server) advertiseSpecificRoute(added []net.IP, deleted []net.IP) {
}
}

func (s *Server) deleteServiceEntries(entries []types.CnatTranslateEntry, oldService *LocalService) {
for _, entry := range entries {
oldServiceState, found := s.serviceStateMap[entry.Key()]
if !found {
s.log.Infof("svc(del) key=%s Cnat entry not found", entry.Key())
continue
}
s.log.Infof("svc(del) key=%s %s vpp-id=%d", entry.Key(), entry.String(), oldServiceState.VppID)
if oldServiceState.OwnerServiceID != oldService.ServiceID {
s.log.Infof("Cnat entry found but changed owner since")
continue
}

err := s.vpp.CnatTranslateDel(oldServiceState.VppID)
func (s *Server) deleteServiceEntry(key, serviceID string) {
if _, found := s.serviceIDByKey[key]; !found {
s.log.Warnf("svc(del) entry %s not found", key)
return
} else if s.serviceIDByKey[key] != serviceID {
// do nothing in vpp, this service is not activated
s.log.Debugf("svc(del) entry %s not created in vpp for service %s", key, serviceID)
} else if len(s.cnatEntryByKeyAndSid[key]) == 1 {
err := s.vpp.CnatTranslateDel(s.cnatEntryByKeyAndSid[key][serviceID].vppID)
if err != nil {
s.log.Errorf("Cnat entry delete errored %s", err)
continue
}
delete(s.serviceStateMap, entry.Key())
}
}

func (s *Server) deleteServiceByName(serviceID string) {
s.lock.Lock()
defer s.lock.Unlock()

for key, oldServiceState := range s.serviceStateMap {
if oldServiceState.OwnerServiceID != serviceID {
continue
delete(s.serviceIDByKey, key)
} else if len(s.cnatEntryByKeyAndSid[key]) > 1 {
// the entry is referenced by another service, recreate the lexicographically smallest service entry
s.log.Warnf("svc(del) entry %s was referenced by multiple services", key)
var chosenService string
for svc := range s.cnatEntryByKeyAndSid[key] {
if (chosenService == "" || svc < chosenService) && chosenService != serviceID {
chosenService = svc
}
}
err := s.vpp.CnatTranslateDel(oldServiceState.VppID)
s.log.Infof("svc(re-add) adding service %s for entry %s", chosenService, key)
entryID, err := s.vpp.CnatTranslateAdd(&s.cnatEntryByKeyAndSid[key][chosenService].entry)
if err != nil {
s.log.Errorf("Cnat entry delete errored %s", err)
continue
s.log.Errorf("svc(add) Error adding translation %s %s", s.cnatEntryByKeyAndSid[key][chosenService].entry.String(), err)
}
delete(s.serviceStateMap, key)
s.cnatEntryByKeyAndSid[key][chosenService].vppID = entryID
s.serviceIDByKey[key] = chosenService
} else {
panic("this should not happen")
}
delete(s.cnatEntryByKeyAndSid[key], serviceID)
delete(s.cnatEntryBySidAndKey[serviceID], key)
// cleanup maps if empty
if len(s.cnatEntryByKeyAndSid[key]) == 0 {
delete(s.cnatEntryByKeyAndSid, key)
}
if len(s.cnatEntryBySidAndKey[serviceID]) == 0 {
delete(s.cnatEntryBySidAndKey, serviceID)
}
}

func (s *Server) sameServiceEntries(entries []types.CnatTranslateEntry, service *LocalService) {
func (s *Server) deleteServiceEntries(entries []types.CnatTranslateEntry, oldService *LocalService) {
for _, entry := range entries {
if serviceState, found := s.serviceStateMap[entry.Key()]; found {
serviceState.OwnerServiceID = service.ServiceID
s.serviceStateMap[entry.Key()] = serviceState
} else {
s.log.Warnf("Cnat entry not found key=%s", entry.Key())
}
s.deleteServiceEntry(entry.Key(), oldService.ServiceID)
}
}

func (s *Server) deleteServiceByName(serviceID string) {
s.lock.Lock()
defer s.lock.Unlock()
for key := range s.cnatEntryBySidAndKey[serviceID] {
s.deleteServiceEntry(key, serviceID)
}
}

Expand All @@ -306,10 +313,31 @@ func (s *Server) addServiceEntries(entries []types.CnatTranslateEntry, service *
s.log.Errorf("svc(add) Error adding translation %s %s", entry.String(), err)
continue
}
s.log.Infof("svc(add) key=%s %s vpp-id=%d", entry.Key(), entry.String(), entryID)
s.serviceStateMap[entry.Key()] = ServiceState{
OwnerServiceID: service.ServiceID,
VppID: entryID,
if _, found := s.cnatEntryBySidAndKey[service.ServiceID]; !found {
s.log.Infof("svc(add) adding service id %s to cache", service.ServiceID)
s.cnatEntryBySidAndKey[service.ServiceID] = make(map[string]*cnatEntry)
}
if _, found := s.cnatEntryByKeyAndSid[entry.Key()]; !found {
s.log.Infof("svc(add) adding entry key=%s to cache", entry.Key())
s.cnatEntryByKeyAndSid[entry.Key()] = make(map[string]*cnatEntry)
}
s.log.Infof("svc(add) adding service %s to entry key=%s cache", service.ServiceID, entry.Key())
s.cnatEntryByKeyAndSid[entry.Key()][service.ServiceID] = &cnatEntry{
entry: entry,
vppID: entryID,
}
s.cnatEntryBySidAndKey[service.ServiceID][entry.Key()] = &cnatEntry{
entry: entry,
vppID: entryID,
}
s.serviceIDByKey[entry.Key()] = service.ServiceID
if len(s.cnatEntryByKeyAndSid[entry.Key()]) > 1 {
s.log.Warnf("svc(add) entry %s is referenced by multiple services; overriding previous value and using the latest", entry.Key())
Comment thread
sknat marked this conversation as resolved.
for svc := range s.cnatEntryByKeyAndSid[entry.Key()] {
if svc != service.ServiceID {
s.cnatEntryByKeyAndSid[entry.Key()][svc].vppID = ^uint32(0)
}
}
}
}
}
26 changes: 15 additions & 11 deletions calico-vpp-agent/services/service_server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ type LocalService struct {
}

/**
* Store VPP's state in a map [CnatTranslateEntry.Key()]->ServiceState
* Store VPP's state in a map [CnatTranslateEntry.Key()]->map[serviceId]->cnatEntry
*/
type ServiceState struct {
OwnerServiceID string /* serviceID(service.ObjectMeta) of the service that created this entry */
VppID uint32 /* cnat translation ID in VPP */
type cnatEntry struct {
entry types.CnatTranslateEntry
vppID uint32
}

type Server struct {
Expand All @@ -74,7 +74,12 @@ type Server struct {
BGPConf *calicov3.BGPConfigurationSpec
nodeBGPSpec *common.LocalNodeSpec

serviceStateMap map[string]ServiceState
// map entry key to a map of service id to entry in vpp
cnatEntryByKeyAndSid map[string]map[string]*cnatEntry
// map service id to a map of entry key to entry in vpp
cnatEntryBySidAndKey map[string]map[string]*cnatEntry
// map entry key to the active k8s service id
serviceIDByKey map[string]string
// cache of all endpoint slices, by service name
endpointSlicesByService map[string]map[string]*discoveryv1.EndpointSlice
endpointSlices map[string]*discoveryv1.EndpointSlice
Expand Down Expand Up @@ -156,7 +161,9 @@ func NewServiceServer(vpp *vpplink.VppLink, k8sclient *kubernetes.Clientset, log
server := Server{
vpp: vpp,
log: log,
serviceStateMap: make(map[string]ServiceState),
cnatEntryByKeyAndSid: make(map[string]map[string]*cnatEntry),
cnatEntryBySidAndKey: make(map[string]map[string]*cnatEntry),
serviceIDByKey: make(map[string]string),
endpointSlicesByService: make(map[string]map[string]*discoveryv1.EndpointSlice),
endpointSlices: make(map[string]*discoveryv1.EndpointSlice),
}
Expand Down Expand Up @@ -404,7 +411,7 @@ func (s *Server) getServiceFromStore(key string) *v1.Service {
* who should be deleted (first) and then re-added. It supports update
* when the entries can be updated with the add call
*/
func compareEntryLists(service *LocalService, oldService *LocalService) (added, same, deleted []types.CnatTranslateEntry, changed bool) {
func compareEntryLists(service *LocalService, oldService *LocalService) (added, deleted []types.CnatTranslateEntry, changed bool) {
if service == nil && oldService == nil {
} else if service == nil {
deleted = oldService.Entries
Expand All @@ -426,8 +433,6 @@ func compareEntryLists(service *LocalService, oldService *LocalService) (added,
deleted = append(deleted, oldService)
} else if newService.Equal(&oldService) == types.ShouldRecreateObj {
deleted = append(deleted, oldService)
} else {
same = append(same, oldService)
}
}
for _, newService := range service.Entries {
Expand Down Expand Up @@ -464,9 +469,8 @@ func compareSpecificRoutes(service *LocalService, oldService *LocalService) (add
}

func (s *Server) handleServiceEndpointEvent(service *LocalService, oldService *LocalService) {
if added, same, deleted, changed := compareEntryLists(service, oldService); changed {
if added, deleted, changed := compareEntryLists(service, oldService); changed {
s.deleteServiceEntries(deleted, oldService)
s.sameServiceEntries(same, service)
s.addServiceEntries(added, service)
}
if added, deleted, changed := compareSpecificRoutes(service, oldService); changed {
Expand Down
Loading