Skip to content

Commit 4ce0c4b

Browse files
LexLuthrrvagg
andauthored
chore: bump boost-graphsync to v0.13.7 (#1647)
* bump boost-graphsync to v0.13.7 * chore: add selector retrieval tests (#1652) --------- Co-authored-by: Rod Vagg <[email protected]>
1 parent 75e40e3 commit 4ce0c4b

File tree

9 files changed

+289
-30
lines changed

9 files changed

+289
-30
lines changed

.circleci/config.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,12 @@ workflows:
325325
suite: itest-markets_v1_deal
326326
target: "./itests/markets_v1_deal_test.go"
327327

328+
329+
- test:
330+
name: test-itest-markets_v1_retrieval
331+
suite: itest-markets_v1_retrieval
332+
target: "./itests/markets_v1_retrieval_test.go"
333+
328334
- test:
329335
name: test-itest-markets_v1_offline_deal
330336
suite: itest-markets_v1_offline_deal

go.mod

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ require (
5151
github.com/ipfs/go-cid v0.4.1
5252
github.com/ipfs/go-cidutil v0.1.0
5353
github.com/ipfs/go-datastore v0.6.0
54-
github.com/ipfs/go-graphsync v0.14.6
54+
github.com/ipfs/go-graphsync v0.14.8
5555
github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
5656
github.com/ipfs/go-ipfs-blocksutil v0.0.1
5757
github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
@@ -67,8 +67,8 @@ require (
6767
github.com/ipfs/go-metrics-interface v0.0.1
6868
github.com/ipfs/go-unixfs v0.4.5 // indirect
6969
github.com/ipld/go-car v0.6.1
70-
github.com/ipld/go-car/v2 v2.10.1
71-
github.com/ipld/go-ipld-prime v0.20.0
70+
github.com/ipld/go-car/v2 v2.11.0
71+
github.com/ipld/go-ipld-prime v0.21.0
7272
github.com/ipld/go-ipld-selector-text-lite v0.0.1
7373
github.com/ipni/index-provider v0.13.4
7474
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
@@ -223,7 +223,7 @@ require (
223223
github.com/ipfs/go-ipld-cbor v0.0.6
224224
github.com/ipfs/go-log v1.0.5 // indirect
225225
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
226-
github.com/ipfs/go-unixfsnode v1.7.1
226+
github.com/ipfs/go-unixfsnode v1.8.0
227227
github.com/ipfs/go-verifcid v0.0.2 // indirect
228228
github.com/ipld/go-codec-dagpb v1.6.0
229229
github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect
@@ -334,7 +334,7 @@ require (
334334

335335
require (
336336
github.com/filecoin-project/boost-gfm v1.26.7
337-
github.com/filecoin-project/boost-graphsync v0.13.6
337+
github.com/filecoin-project/boost-graphsync v0.13.7
338338
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7
339339
github.com/filecoin-project/go-fil-markets v1.28.3
340340
github.com/filecoin-project/lotus v1.23.2-0.20230622154405-168d022018ce

go.sum

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -333,8 +333,8 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF
333333
github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
334334
github.com/filecoin-project/boost-gfm v1.26.7 h1:ENJEqx1OzY072QnUP37YrGVmUiCewRwHAjbtTxyW74Y=
335335
github.com/filecoin-project/boost-gfm v1.26.7/go.mod h1:OhG2y7WeDx3KU9DPjgWllS+3/ospPjm8/XDrvN6uOfk=
336-
github.com/filecoin-project/boost-graphsync v0.13.6 h1:UZZHwUt9epr3r1KUAy7A01vbmkgYRgpm9O0ZCVa/fzE=
337-
github.com/filecoin-project/boost-graphsync v0.13.6/go.mod h1:WL0AQccJeNASH4XG5CaXJ0zbkqoTaqOZCekyCFly75s=
336+
github.com/filecoin-project/boost-graphsync v0.13.7 h1:c/CQOSd6Bpw7qnv9KUL/hbWOfN0l5xwyX8UNxQk4ElM=
337+
github.com/filecoin-project/boost-graphsync v0.13.7/go.mod h1:bc2M5ZLZJtXHl8kjnqtn4L1MsdEqpJErDaIeY0bJ9wk=
338338
github.com/filecoin-project/dagstore v0.7.0 h1:IS0R+69za8dguYWeqz/MI+nb7ONpk03tAkxPCBXEKm0=
339339
github.com/filecoin-project/dagstore v0.7.0/go.mod h1:YKn4qXih+/2xQWpfJsaKGOi4POw5vH5grDmfPCCnx8g=
340340
github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8=
@@ -442,7 +442,7 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM
442442
github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
443443
github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
444444
github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
445-
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
445+
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
446446
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
447447
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
448448
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
@@ -804,8 +804,8 @@ github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9
804804
github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM=
805805
github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
806806
github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
807-
github.com/ipfs/go-graphsync v0.14.6 h1:NPxvuUy4Z08Mg8dwpBzwgbv/PGLIufSJ1sle6iAX8yo=
808-
github.com/ipfs/go-graphsync v0.14.6/go.mod h1:yT0AfjFgicOoWdAlUJ96tQ5AkuGI4r1taIQX/aHbBQo=
807+
github.com/ipfs/go-graphsync v0.14.8 h1:NFFHquTNnwPi05tJhdpPj4CJMnqRBLxpZd+IfPRauf4=
808+
github.com/ipfs/go-graphsync v0.14.8/go.mod h1:qyHjUvHey6EfKUDMQPwCuVkMOurRG3hcjRm+FaVP6bE=
809809
github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
810810
github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
811811
github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE=
@@ -915,8 +915,8 @@ github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/
915915
github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
916916
github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg=
917917
github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo=
918-
github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s=
919-
github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
918+
github.com/ipfs/go-unixfsnode v1.8.0 h1:yCkakzuE365glu+YkgzZt6p38CSVEBPgngL9ZkfnyQU=
919+
github.com/ipfs/go-unixfsnode v1.8.0/go.mod h1:HxRu9HYHOjK6HUqFBAi++7DVoWAHn0o4v/nZ/VA+0g8=
920920
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
921921
github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
922922
github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
@@ -927,8 +927,8 @@ github.com/ipld/go-car v0.6.1 h1:blWbEHf1j62JMWFIqWE//YR0m7k5ZMw0AuUOU5hjrH8=
927927
github.com/ipld/go-car v0.6.1/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8=
928928
github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI=
929929
github.com/ipld/go-car/v2 v2.4.1/go.mod h1:zjpRf0Jew9gHqSvjsKVyoq9OY9SWoEKdYCQUKVaaPT0=
930-
github.com/ipld/go-car/v2 v2.10.1 h1:MRDqkONNW9WRhB79u+Z3U5b+NoN7lYA5B8n8qI3+BoI=
931-
github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg=
930+
github.com/ipld/go-car/v2 v2.11.0 h1:lkAPwbbTFqbdfawgm+bfmFc8PjGC7D12VcaLXPCLNfM=
931+
github.com/ipld/go-car/v2 v2.11.0/go.mod h1:aDszqev0zjtU8l96g4lwXHaU9bzArj56Y7eEN0q/xqA=
932932
github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s=
933933
github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA=
934934
github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY=
@@ -944,8 +944,8 @@ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHt
944944
github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM=
945945
github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA=
946946
github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4=
947-
github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
948-
github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
947+
github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
948+
github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
949949
github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU=
950950
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY=
951951
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
@@ -1871,7 +1871,7 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp
18711871
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
18721872
github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
18731873
github.com/warpfork/go-testmark v0.10.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
1874-
github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U=
1874+
github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
18751875
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
18761876
github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
18771877
github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=

itests/dummydeal_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,6 @@ func TestDummydealOnline(t *testing.T) {
9494
require.NoError(t, err)
9595

9696
// rootCid is an identity CID
97-
outFile := f.RetrieveDirect(ctx, t, rootCid, &res.DealParams.ClientDealProposal.Proposal.PieceCID, true)
97+
outFile := f.RetrieveDirect(ctx, t, rootCid, &res.DealParams.ClientDealProposal.Proposal.PieceCID, true, true, nil)
9898
kit.AssertFilesEqual(t, randomFilepath, outFile)
9999
}

itests/framework/framework.go

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,13 @@ import (
5959
"github.com/ipfs/go-cid"
6060
"github.com/ipfs/go-datastore"
6161
ipldcbor "github.com/ipfs/go-ipld-cbor"
62-
ipld "github.com/ipfs/go-ipld-format"
62+
ipldformat "github.com/ipfs/go-ipld-format"
6363
logging "github.com/ipfs/go-log/v2"
6464
"github.com/ipld/go-car"
65+
"github.com/ipld/go-ipld-prime"
66+
"github.com/ipld/go-ipld-prime/codec/dagjson"
67+
"github.com/ipld/go-ipld-prime/datamodel"
68+
"github.com/ipld/go-ipld-prime/traversal/selector"
6569
"github.com/libp2p/go-libp2p"
6670
"github.com/libp2p/go-libp2p/core/host"
6771
"github.com/multiformats/go-multihash"
@@ -711,7 +715,7 @@ func (f *TestFramework) WaitDealSealed(ctx context.Context, deal *cid.Cid) error
711715
}
712716
}
713717

714-
func (f *TestFramework) Retrieve(ctx context.Context, t *testing.T, deal *cid.Cid, root cid.Cid, carExport bool) string {
718+
func (f *TestFramework) Retrieve(ctx context.Context, t *testing.T, deal *cid.Cid, root cid.Cid, extractCar bool, selectorNode datamodel.Node) string {
715719
// perform retrieval.
716720
info, err := f.FullNode.ClientGetDealInfo(ctx, *deal)
717721
require.NoError(t, err)
@@ -720,7 +724,7 @@ func (f *TestFramework) Retrieve(ctx context.Context, t *testing.T, deal *cid.Ci
720724
require.NoError(t, err)
721725
require.NotEmpty(t, offers, "no offers")
722726

723-
return f.retrieve(ctx, t, offers[0], carExport)
727+
return f.retrieve(ctx, t, offers[0], extractCar, selectorNode)
724728
}
725729

726730
func (f *TestFramework) ExtractFileFromCAR(ctx context.Context, t *testing.T, file *os.File) string {
@@ -739,7 +743,7 @@ func (f *TestFramework) ExtractFileFromCAR(ctx context.Context, t *testing.T, fi
739743
require.NoError(t, err)
740744
}
741745

742-
reg := ipld.Registry{}
746+
reg := ipldformat.Registry{}
743747
reg.Register(cid.DagProtobuf, dag.DecodeProtobufBlock)
744748
reg.Register(cid.DagCBOR, ipldcbor.DecodeBlock)
745749
reg.Register(cid.Raw, dag.DecodeRawBlock)
@@ -758,15 +762,17 @@ func (f *TestFramework) ExtractFileFromCAR(ctx context.Context, t *testing.T, fi
758762
return tmpFile
759763
}
760764

761-
func (f *TestFramework) RetrieveDirect(ctx context.Context, t *testing.T, root cid.Cid, pieceCid *cid.Cid, carExport bool) string {
765+
func (f *TestFramework) RetrieveDirect(ctx context.Context, t *testing.T, root cid.Cid, pieceCid *cid.Cid, carExport bool, extractCar bool, selectorNode datamodel.Node) string {
762766
offer, err := f.FullNode.ClientMinerQueryOffer(ctx, f.MinerAddr, root, pieceCid)
763767
require.NoError(t, err)
764768

765-
return f.retrieve(ctx, t, offer, carExport)
769+
return f.retrieve(ctx, t, offer, extractCar, selectorNode)
766770
}
767771

768-
func (f *TestFramework) retrieve(ctx context.Context, t *testing.T, offer lapi.QueryOffer, carExport bool) string {
772+
func (f *TestFramework) retrieve(ctx context.Context, t *testing.T, offer lapi.QueryOffer, extractCar bool, selectorNode datamodel.Node) string {
769773
p := path.Join(t.TempDir(), "ret-car-"+t.Name())
774+
err := os.MkdirAll(path.Dir(p), 0755)
775+
require.NoError(t, err)
770776
carFile, err := os.Create(p)
771777
require.NoError(t, err)
772778

@@ -779,7 +785,17 @@ func (f *TestFramework) retrieve(ctx context.Context, t *testing.T, offer lapi.Q
779785
updates, err := f.FullNode.ClientGetRetrievalUpdates(updatesCtx)
780786
require.NoError(t, err)
781787

782-
retrievalRes, err := f.FullNode.ClientRetrieve(ctx, offer.Order(caddr))
788+
order := offer.Order(caddr)
789+
if selectorNode != nil {
790+
_, err := selector.CompileSelector(selectorNode)
791+
require.NoError(t, err)
792+
jsonSelector, err := ipld.Encode(selectorNode, dagjson.Encode)
793+
require.NoError(t, err)
794+
sel := lapi.Selector(jsonSelector)
795+
order.DataSelector = &sel
796+
}
797+
798+
retrievalRes, err := f.FullNode.ClientRetrieve(ctx, order)
783799
require.NoError(t, err)
784800
consumeEvents:
785801
for {
@@ -812,11 +828,11 @@ consumeEvents:
812828
},
813829
lapi.FileRef{
814830
Path: carFile.Name(),
815-
IsCAR: carExport,
831+
IsCAR: true,
816832
}))
817833

818834
ret := carFile.Name()
819-
if carExport {
835+
if extractCar {
820836
ret = f.ExtractFileFromCAR(ctx, t, carFile)
821837
}
822838

itests/markets_v1_deal_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func TestMarketsV1Deal(t *testing.T) {
4848
require.NoError(t, err)
4949

5050
log.Debugw("deal is sealed, starting retrieval", "cid", dealProposalCid, "root", res.Root)
51-
outPath := f.Retrieve(ctx, t, dealProposalCid, res.Root, true)
51+
outPath := f.Retrieve(ctx, t, dealProposalCid, res.Root, true, nil)
5252

5353
log.Debugw("retrieval is done, compare in- and out- files", "in", inPath, "out", outPath)
5454
kit.AssertFilesEqual(t, inPath, outPath)

itests/markets_v1_offline_deal_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func TestMarketsV1OfflineDeal(t *testing.T) {
8383
require.NoError(t, err)
8484

8585
log.Debugw("offline deal is sealed, starting retrieval", "cid", dealProposalCid, "root", res.Root)
86-
outPath := f.Retrieve(ctx, t, dealProposalCid, res.Root, true)
86+
outPath := f.Retrieve(ctx, t, dealProposalCid, res.Root, true, nil)
8787

8888
log.Debugw("retrieval of offline deal is done, compare in- and out- files", "in", inPath, "out", outPath)
8989
kit.AssertFilesEqual(t, inPath, outPath)

0 commit comments

Comments
 (0)