Skip to content

Commit ac5c79e

Browse files
cthulhu-ridercthulhu-rider
committed
internal/object: Add header and parent header protobuf parsers
Allows to work with object binary w/o full unmarshalling. Going to be used in #3783. Signed-off-by: Leonard Lyubich <leonard@morphbits.io>
1 parent fd1e659 commit ac5c79e

6 files changed

Lines changed: 451 additions & 11 deletions

File tree

‎internal/object/wire.go‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ import (
44
"errors"
55
"fmt"
66
"io"
7+
"strconv"
78

9+
iprotobuf "github.com/nspcc-dev/neofs-node/internal/protobuf"
810
"github.com/nspcc-dev/neofs-sdk-go/object"
911
protoobject "github.com/nspcc-dev/neofs-sdk-go/proto/object"
1012
"github.com/nspcc-dev/neofs-sdk-go/proto/refs"
@@ -19,6 +21,27 @@ const (
1921
fieldObjectSignature
2022
fieldObjectHeader
2123
fieldObjectPayload
24+
25+
FieldHeaderVersion = 1
26+
FieldHeaderContainerID = 2
27+
FieldHeaderOwnerID = 3
28+
FieldHeaderCreationEpoch = 4
29+
FieldHeaderPayloadLength = 5
30+
FieldHeaderPayloadHash = 6
31+
FieldHeaderType = 7
32+
FieldHeaderHomoHash = 8
33+
FieldHeaderSessionToken = 9
34+
FieldHeaderAttributes = 10
35+
FieldHeaderSplit = 11
36+
FieldHeaderSessionTokenV2 = 12
37+
38+
FieldHeaderSplitParent = 1
39+
FieldHeaderSplitPrevious = 2
40+
FieldHeaderSplitParentSignature = 3
41+
FieldHeaderSplitParentHeader = 4
42+
FieldHeaderSplitChildren = 5
43+
FieldHeaderSplitSplitID = 6
44+
FieldHeaderSplitFirst = 7
2245
)
2346

2447
// WriteWithoutPayload writes the object header to the given writer without the payload.
@@ -108,3 +131,148 @@ func ReadHeaderPrefix(r io.Reader) (*object.Object, []byte, error) {
108131
}
109132
return ExtractHeaderAndPayload(buf[:n])
110133
}
134+
135+
// GetNonPayloadFieldBounds seeks ID, signature and header in object message and
136+
// parses their boundaries.
137+
//
138+
// If any field is missing, no error is returned.
139+
//
140+
// Message should have ascending field order, otherwise error returns.
141+
func GetNonPayloadFieldBounds(buf []byte) (iprotobuf.FieldBounds, iprotobuf.FieldBounds, iprotobuf.FieldBounds, error) {
142+
var idf, sigf, hdrf iprotobuf.FieldBounds
143+
var off int
144+
var prevNum protowire.Number
145+
loop:
146+
for {
147+
num, typ, n, err := iprotobuf.ParseTag(buf[off:])
148+
if err != nil {
149+
return idf, sigf, hdrf, err
150+
}
151+
152+
if num > fieldObjectHeader {
153+
break
154+
}
155+
if num < prevNum {
156+
return idf, sigf, hdrf, iprotobuf.NewUnorderedFieldsError(prevNum, num)
157+
}
158+
if num == prevNum {
159+
return idf, sigf, hdrf, iprotobuf.NewRepeatedFieldError(num)
160+
}
161+
prevNum = num
162+
163+
switch num {
164+
case fieldObjectID:
165+
idf, err = iprotobuf.ParseLENFieldBounds(buf, off, n, num, typ)
166+
if err != nil {
167+
return idf, sigf, hdrf, err
168+
}
169+
off = idf.To
170+
case fieldObjectSignature:
171+
sigf, err = iprotobuf.ParseLENFieldBounds(buf, off, n, num, typ)
172+
if err != nil {
173+
return idf, sigf, hdrf, err
174+
}
175+
off = sigf.To
176+
case fieldObjectHeader:
177+
hdrf, err = iprotobuf.ParseLENFieldBounds(buf, off, n, num, typ)
178+
if err != nil {
179+
return idf, sigf, hdrf, err
180+
}
181+
break loop
182+
default:
183+
panic("unreachable with num " + strconv.Itoa(int(num)))
184+
}
185+
186+
if off == len(buf) {
187+
break
188+
}
189+
}
190+
191+
return idf, sigf, hdrf, nil
192+
}
193+
194+
// GetParentNonPayloadFieldBounds seeks parent's ID, signature and header in child
195+
// object message and parses their boundaries.
196+
//
197+
// If any field is missing, no error is returned.
198+
//
199+
// Message should have ascending field order, otherwise error returns.
200+
func GetParentNonPayloadFieldBounds(buf []byte) (iprotobuf.FieldBounds, iprotobuf.FieldBounds, iprotobuf.FieldBounds, error) {
201+
var idf, sigf, hdrf iprotobuf.FieldBounds
202+
203+
rootHdrf, err := iprotobuf.GetLENFieldBounds(buf, fieldObjectHeader)
204+
if err != nil {
205+
return idf, sigf, hdrf, err
206+
}
207+
208+
if rootHdrf.IsMissing() {
209+
return idf, sigf, hdrf, nil
210+
}
211+
212+
splitf, err := iprotobuf.GetLENFieldBounds(buf[rootHdrf.ValueFrom:rootHdrf.To], FieldHeaderSplit)
213+
if err != nil {
214+
return idf, sigf, hdrf, err
215+
}
216+
217+
if splitf.IsMissing() {
218+
return idf, sigf, hdrf, nil
219+
}
220+
221+
buf = buf[:rootHdrf.ValueFrom+splitf.To]
222+
off := rootHdrf.ValueFrom + splitf.ValueFrom
223+
var prevNum protowire.Number
224+
loop:
225+
for {
226+
num, typ, n, err := iprotobuf.ParseTag(buf[off:])
227+
if err != nil {
228+
return idf, sigf, hdrf, err
229+
}
230+
231+
if num > FieldHeaderSplitParentHeader {
232+
break
233+
}
234+
if num < prevNum {
235+
return idf, sigf, hdrf, iprotobuf.NewUnorderedFieldsError(prevNum, num)
236+
}
237+
if num == prevNum {
238+
return idf, sigf, hdrf, iprotobuf.NewRepeatedFieldError(num)
239+
}
240+
prevNum = num
241+
242+
switch num {
243+
case FieldHeaderSplitParent:
244+
idf, err = iprotobuf.ParseLENFieldBounds(buf, off, n, num, typ)
245+
if err != nil {
246+
return idf, sigf, hdrf, err
247+
}
248+
off = idf.To
249+
case FieldHeaderSplitPrevious:
250+
off += n
251+
ln, n, err := iprotobuf.ParseLENField(buf[off:], num, typ)
252+
if err != nil {
253+
return idf, sigf, hdrf, err
254+
}
255+
off += n + ln
256+
case FieldHeaderSplitParentSignature:
257+
sigf, err = iprotobuf.ParseLENFieldBounds(buf, off, n, num, typ)
258+
if err != nil {
259+
return idf, sigf, hdrf, err
260+
}
261+
off = sigf.To
262+
case FieldHeaderSplitParentHeader:
263+
hdrf, err = iprotobuf.ParseLENFieldBounds(buf, off, n, num, typ)
264+
if err != nil {
265+
return idf, sigf, hdrf, err
266+
}
267+
break loop
268+
default:
269+
panic("unreachable with num " + strconv.Itoa(int(num)))
270+
}
271+
272+
if off == len(buf) {
273+
break
274+
}
275+
}
276+
277+
return idf, sigf, hdrf, nil
278+
}

‎internal/object/wire_test.go‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,52 @@ package object_test
33
import (
44
"bytes"
55
"crypto/rand"
6+
"crypto/sha256"
67
"io"
8+
"math"
9+
"slices"
710
"testing"
811

912
iobject "github.com/nspcc-dev/neofs-node/internal/object"
13+
iprotobuf "github.com/nspcc-dev/neofs-node/internal/protobuf"
14+
"github.com/nspcc-dev/neofs-node/internal/testutil"
15+
"github.com/nspcc-dev/neofs-sdk-go/checksum"
16+
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
17+
neofscrypto "github.com/nspcc-dev/neofs-sdk-go/crypto"
18+
neofscryptotest "github.com/nspcc-dev/neofs-sdk-go/crypto/test"
1019
"github.com/nspcc-dev/neofs-sdk-go/object"
20+
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
21+
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
1122
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
23+
"github.com/nspcc-dev/neofs-sdk-go/version"
24+
"github.com/nspcc-dev/tzhash/tz"
1225
"github.com/stretchr/testify/require"
26+
"google.golang.org/protobuf/encoding/protowire"
1327
)
1428

29+
func TestFields(t *testing.T) {
30+
require.EqualValues(t, 1, iobject.FieldHeaderVersion)
31+
require.EqualValues(t, 2, iobject.FieldHeaderContainerID)
32+
require.EqualValues(t, 3, iobject.FieldHeaderOwnerID)
33+
require.EqualValues(t, 4, iobject.FieldHeaderCreationEpoch)
34+
require.EqualValues(t, 5, iobject.FieldHeaderPayloadLength)
35+
require.EqualValues(t, 6, iobject.FieldHeaderPayloadHash)
36+
require.EqualValues(t, 7, iobject.FieldHeaderType)
37+
require.EqualValues(t, 8, iobject.FieldHeaderHomoHash)
38+
require.EqualValues(t, 9, iobject.FieldHeaderSessionToken)
39+
require.EqualValues(t, 10, iobject.FieldHeaderAttributes)
40+
require.EqualValues(t, 11, iobject.FieldHeaderSplit)
41+
require.EqualValues(t, 12, iobject.FieldHeaderSessionTokenV2)
42+
43+
require.EqualValues(t, 1, iobject.FieldHeaderSplitParent)
44+
require.EqualValues(t, 2, iobject.FieldHeaderSplitPrevious)
45+
require.EqualValues(t, 3, iobject.FieldHeaderSplitParentSignature)
46+
require.EqualValues(t, 4, iobject.FieldHeaderSplitParentHeader)
47+
require.EqualValues(t, 5, iobject.FieldHeaderSplitChildren)
48+
require.EqualValues(t, 6, iobject.FieldHeaderSplitSplitID)
49+
require.EqualValues(t, 7, iobject.FieldHeaderSplitFirst)
50+
}
51+
1552
func TestWriteWithoutPayload(t *testing.T) {
1653
t.Run("write empty object", func(t *testing.T) {
1754
var buf bytes.Buffer
@@ -97,3 +134,158 @@ func TestReadHeaderPrefix(t *testing.T) {
97134
require.Equal(t, expectedSize, len(payloadPrefix))
98135
require.Equal(t, payload[:expectedSize], payloadPrefix)
99136
}
137+
138+
func TestGetNonPayloadFieldBounds(t *testing.T) {
139+
id := oidtest.ID()
140+
sig := neofscryptotest.Signature()
141+
142+
obj := objecttest.Object()
143+
obj.SetID(id)
144+
obj.SetSignature(&sig)
145+
146+
buf := obj.Marshal()
147+
148+
idf, sigf, hdrf, err := iobject.GetNonPayloadFieldBounds(buf)
149+
require.NoError(t, err)
150+
151+
assertFound := func(t *testing.T, f iprotobuf.FieldBounds, tag byte, exp []byte) {
152+
require.False(t, f.IsMissing())
153+
require.EqualValues(t, tag, buf[f.From])
154+
ln, n, err := iprotobuf.ParseLENField(buf[f.From+1:], 42, protowire.BytesType)
155+
require.NoError(t, err)
156+
require.EqualValues(t, 1+n, f.ValueFrom-f.From)
157+
require.EqualValues(t, ln, f.To-f.ValueFrom)
158+
require.True(t, bytes.Equal(exp, buf[f.ValueFrom:f.To]))
159+
}
160+
161+
assertFound(t, idf, iprotobuf.TagBytes1, id.Marshal())
162+
assertFound(t, sigf, iprotobuf.TagBytes2, sig.Marshal())
163+
164+
hdr := obj.ProtoMessage().Header
165+
hdrBuf := make([]byte, hdr.MarshaledSize())
166+
hdr.MarshalStable(hdrBuf)
167+
assertFound(t, hdrf, iprotobuf.TagBytes3, hdrBuf)
168+
}
169+
170+
func BenchmarkGetNonPayloadFieldBounds(b *testing.B) {
171+
id := oidtest.ID()
172+
const sigLen = 100
173+
const hdrLen = 16 << 10
174+
175+
buf := slices.Concat(
176+
[]byte{iprotobuf.TagBytes1, oid.Size + 2, iprotobuf.TagBytes1, oid.Size}, id[:],
177+
[]byte{iprotobuf.TagBytes2, sigLen}, testutil.RandByteSlice(sigLen),
178+
[]byte{iprotobuf.TagBytes3, 128, 128, 1}, testutil.RandByteSlice(hdrLen),
179+
)
180+
181+
idf, sigf, hdrf, err := iobject.GetNonPayloadFieldBounds(buf)
182+
require.NoError(b, err)
183+
require.EqualValues(b, 0, idf.From)
184+
require.EqualValues(b, idf.From+2, idf.ValueFrom)
185+
require.EqualValues(b, idf.ValueFrom+oid.Size, idf.To)
186+
require.EqualValues(b, idf.To, sigf.From)
187+
require.EqualValues(b, sigf.From+2, sigf.ValueFrom)
188+
require.EqualValues(b, sigf.ValueFrom+sigLen, sigf.To)
189+
require.EqualValues(b, sigf.To, hdrf.From)
190+
require.EqualValues(b, hdrf.From+4, hdrf.ValueFrom)
191+
require.EqualValues(b, hdrf.ValueFrom+hdrLen, hdrf.To)
192+
require.EqualValues(b, len(buf), hdrf.To)
193+
194+
b.ReportAllocs()
195+
for b.Loop() {
196+
_, _, _, err = iobject.GetNonPayloadFieldBounds(buf)
197+
require.NoError(b, err)
198+
}
199+
}
200+
201+
func TestGetParentNonPayloadFieldBounds(t *testing.T) {
202+
parID := oidtest.ID()
203+
parSig := neofscryptotest.Signature()
204+
205+
par := objecttest.Object()
206+
par.SetID(parID)
207+
par.SetSignature(&parSig)
208+
par.ResetRelations()
209+
210+
obj := objecttest.Object()
211+
obj.SetParent(&par)
212+
213+
buf := obj.Marshal()
214+
215+
idf, sigf, hdrf, err := iobject.GetParentNonPayloadFieldBounds(buf)
216+
require.NoError(t, err)
217+
218+
assertFound := func(t *testing.T, f iprotobuf.FieldBounds, tag byte, exp []byte) {
219+
require.False(t, f.IsMissing())
220+
require.EqualValues(t, tag, buf[f.From])
221+
ln, n, err := iprotobuf.ParseLENField(buf[f.From+1:], 42, protowire.BytesType)
222+
require.NoError(t, err)
223+
require.EqualValues(t, 1+n, f.ValueFrom-f.From)
224+
require.EqualValues(t, ln, f.To-f.ValueFrom)
225+
require.True(t, bytes.Equal(exp, buf[f.ValueFrom:f.To]))
226+
}
227+
228+
assertFound(t, idf, iprotobuf.TagBytes1, parID.Marshal())
229+
assertFound(t, sigf, iprotobuf.TagBytes3, parSig.Marshal())
230+
231+
parHdr := par.ProtoMessage().Header
232+
parHdrBuf := make([]byte, parHdr.MarshaledSize())
233+
parHdr.MarshalStable(parHdrBuf)
234+
assertFound(t, hdrf, iprotobuf.TagBytes4, parHdrBuf)
235+
}
236+
237+
func BenchmarkGetParentNonPayloadFieldBounds(b *testing.B) {
238+
parID := oidtest.ID()
239+
240+
sig := neofscrypto.NewSignatureFromRawKey(neofscrypto.ECDSA_DETERMINISTIC_SHA256, testutil.RandByteSlice(33), testutil.RandByteSlice(64))
241+
ver := version.New(123, 456)
242+
pldHash := checksum.New(checksum.SHA256, testutil.RandByteSlice(sha256.Size))
243+
pldHomoHash := checksum.New(checksum.TillichZemor, testutil.RandByteSlice(tz.Size))
244+
245+
fillHeader := func(obj *object.Object) {
246+
obj.SetID(oidtest.ID())
247+
obj.SetSignature(&sig)
248+
obj.SetVersion(&ver)
249+
obj.SetContainerID(cidtest.ID())
250+
obj.SetCreationEpoch(math.MaxUint64)
251+
obj.SetPayloadSize(math.MaxUint64)
252+
obj.SetPayloadChecksum(pldHash)
253+
obj.SetType(math.MaxInt32)
254+
obj.SetPayloadHomomorphicHash(pldHomoHash)
255+
obj.SetAttributes(
256+
object.NewAttribute("key1", "val1"),
257+
object.NewAttribute("key2", "val2"),
258+
object.NewAttribute("key3", "val3"),
259+
)
260+
}
261+
262+
var par object.Object
263+
fillHeader(&par)
264+
par.SetID(parID)
265+
266+
var obj object.Object
267+
fillHeader(&obj)
268+
obj.SetPreviousID(oidtest.ID())
269+
obj.SetParent(&par)
270+
271+
buf := obj.Marshal()
272+
273+
idf, sigf, hdrf, err := iobject.GetParentNonPayloadFieldBounds(buf)
274+
require.NoError(b, err)
275+
require.Positive(b, idf.From)
276+
require.EqualValues(b, idf.From+2, idf.ValueFrom)
277+
require.EqualValues(b, idf.ValueFrom+2+oid.Size, idf.To)
278+
require.EqualValues(b, idf.To+2+2+oid.Size, sigf.From)
279+
require.EqualValues(b, sigf.From+2, sigf.ValueFrom)
280+
require.EqualValues(b, sigf.ValueFrom+len(sig.Marshal()), sigf.To)
281+
require.EqualValues(b, sigf.To, hdrf.From)
282+
require.EqualValues(b, hdrf.From+3, hdrf.ValueFrom)
283+
require.EqualValues(b, hdrf.ValueFrom+par.HeaderLen(), hdrf.To)
284+
require.EqualValues(b, len(buf), hdrf.To)
285+
286+
b.ReportAllocs()
287+
for b.Loop() {
288+
_, _, _, err = iobject.GetNonPayloadFieldBounds(buf)
289+
require.NoError(b, err)
290+
}
291+
}

0 commit comments

Comments
 (0)