Ignore "not our ref" errors from gitlab-sshd error metrics

If a client requests a ref that cannot be found in the repository,
previously gitlab-sshd would record it as part of its service level
indicator metric. This is really an application error between the
client and the Git repository, so we exclude it from our metrics.

Relates to
https://gitlab.com/gitlab-com/gl-infra/reliability/-/issues/15848

Changelog: fixed

Author: stanhu

internal/sshd/connection.go

@@ -21,7 +21,10 @@ import (
 	"gitlab.com/gitlab-org/labkit/log"
 )
 
-const KeepAliveMsg = "keepalive@openssh.com"
+const (
+	KeepAliveMsg   = "keepalive@openssh.com"
+	NotOurRefError = `exit status 128, stderr: "fatal: git upload-pack: not our ref `
+)
 
 var EOFTimeout = 10 * time.Second
 
@@ -173,6 +176,8 @@ func (c *connection) trackError(ctxlog *logrus.Entry, err error) {
 	grpcCode := grpcstatus.Code(err)
 	if grpcCode == grpccodes.Canceled || grpcCode == grpccodes.Unavailable {
 		return
+	} else if grpcCode == grpccodes.Internal && strings.Contains(err.Error(), NotOurRefError) {
+		return
 	}
 
 	metrics.SliSshdSessionsErrorsTotal.Inc()

internal/sshd/connection_test.go

@@ -221,6 +221,7 @@ func TestSessionsMetrics(t *testing.T) {
 		{"unavailable Gitaly", grpcstatus.Error(grpccodes.Unavailable, "unavailable")},
 		{"api error", &client.ApiError{"api error"}},
 		{"disallowed command", disallowedcommand.Error},
+		{"not our ref", grpcstatus.Error(grpccodes.Internal, `rpc error: code = Internal desc = cmd wait: exit status 128, stderr: "fatal: git upload-pack: not our ref 9106d18f6a1b8022f6517f479696f3e3ea5e68c1"`)},
 	} {
 		t.Run(ignoredError.desc, func(t *testing.T) {
 			conn, chans = setup(1, newChannel)