From 33b6f1b8a6691b22ed3799e7fe2eafb8b759d27c Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 10 Aug 2018 16:09:09 -0400 Subject: [PATCH 1/3] gx updates This commit was moved from ipfs/go-unixfs@f84f499c6ef52e9c5c788626747c1e8e08ed5c1d --- unixfs/ipld-merkledag/importer/importer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unixfs/ipld-merkledag/importer/importer_test.go b/unixfs/ipld-merkledag/importer/importer_test.go index 823abaa4b..b39aff57d 100644 --- a/unixfs/ipld-merkledag/importer/importer_test.go +++ b/unixfs/ipld-merkledag/importer/importer_test.go @@ -38,7 +38,7 @@ func getTrickleDag(t testing.TB, size int64, blksize int64) (ipld.Node, ipld.DAG func TestStableCid(t *testing.T) { ds := mdtest.Mock() - buf := make([]byte, 10 * 1024 * 1024) + buf := make([]byte, 10*1024*1024) u.NewSeededRand(0xdeadbeef).Read(buf) r := bytes.NewReader(buf) From a0c04d6278110437be0575c1a897cb400f8d4922 Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 10 Aug 2018 23:43:14 -0400 Subject: [PATCH 2/3] Update to use new Builder interface for creating CIDs. This commit was moved from ipfs/go-unixfs@dc95f6c3f0ac4ab13ebbbae213d3334ce9c23c13 --- unixfs/ipld-merkledag/hamt/hamt.go | 10 +++++----- .../importer/helpers/dagbuilder.go | 12 ++++++------ .../ipld-merkledag/importer/helpers/helpers.go | 4 ++-- unixfs/ipld-merkledag/io/directory.go | 18 +++++++++--------- unixfs/ipld-merkledag/mod/dagmodifier.go | 2 +- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/unixfs/ipld-merkledag/hamt/hamt.go b/unixfs/ipld-merkledag/hamt/hamt.go index 6887a148d..a28bf0725 100644 --- a/unixfs/ipld-merkledag/hamt/hamt.go +++ b/unixfs/ipld-merkledag/hamt/hamt.go @@ -52,7 +52,7 @@ type Shard struct { tableSize int tableSizeLg2 int - prefix *cid.Prefix + prefix cid.Builder hashFunc uint64 prefixPadStr string @@ -124,25 +124,25 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) { ds.children = make([]child, len(pbnd.Links())) ds.bitfield.SetBytes(pbd.GetData()) ds.hashFunc = pbd.GetHashType() - ds.prefix = &ds.nd.Prefix + ds.prefix = ds.nd.CidBuilder() return ds, nil } // SetPrefix sets the CID Prefix -func (ds *Shard) SetPrefix(prefix *cid.Prefix) { +func (ds *Shard) SetPrefix(prefix cid.Builder) { ds.prefix = prefix } // Prefix gets the CID Prefix, may be nil if unset -func (ds *Shard) Prefix() *cid.Prefix { +func (ds *Shard) Prefix() cid.Builder { return ds.prefix } // Node serializes the HAMT structure into a merkledag node with unixfs formatting func (ds *Shard) Node() (ipld.Node, error) { out := new(dag.ProtoNode) - out.SetPrefix(ds.prefix) + out.SetCidBuilder(ds.prefix) cindex := 0 // TODO: optimized 'for each set bit' diff --git a/unixfs/ipld-merkledag/importer/helpers/dagbuilder.go b/unixfs/ipld-merkledag/importer/helpers/dagbuilder.go index 387b26304..1331ddbfa 100644 --- a/unixfs/ipld-merkledag/importer/helpers/dagbuilder.go +++ b/unixfs/ipld-merkledag/importer/helpers/dagbuilder.go @@ -26,7 +26,7 @@ type DagBuilderHelper struct { nextData []byte // the next item to return. maxlinks int batch *ipld.Batch - prefix *cid.Prefix + prefix cid.Builder // Filestore support variables. // ---------------------------- @@ -54,7 +54,7 @@ type DagBuilderParams struct { RawLeaves bool // CID Prefix to use if set - Prefix *cid.Prefix + Prefix cid.Builder // DAGService to write blocks to (required) Dagserv ipld.DAGService @@ -146,7 +146,7 @@ func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode { } // GetPrefix returns the internal `cid.Prefix` set in the builder. -func (db *DagBuilderHelper) GetPrefix() *cid.Prefix { +func (db *DagBuilderHelper) GetPrefix() cid.Builder { return db.prefix } @@ -166,7 +166,7 @@ func (db *DagBuilderHelper) NewLeaf(data []byte) (*UnixfsNode, error) { raw: true, }, nil } - rawnode, err := dag.NewRawNodeWPrefix(data, *db.prefix) + rawnode, err := dag.NewRawNodeWPrefix(data, db.prefix) if err != nil { return nil, err } @@ -197,7 +197,7 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte) (ipld.Node, error) { if db.prefix == nil { return dag.NewRawNode(data), nil } - rawnode, err := dag.NewRawNodeWPrefix(data, *db.prefix) + rawnode, err := dag.NewRawNodeWPrefix(data, db.prefix) if err != nil { return nil, err } @@ -401,7 +401,7 @@ type FSNodeOverDag struct { func (db *DagBuilderHelper) NewFSNodeOverDag(fsNodeType pb.Data_DataType) *FSNodeOverDag { node := new(FSNodeOverDag) node.dag = new(dag.ProtoNode) - node.dag.SetPrefix(db.GetPrefix()) + node.dag.SetCidBuilder(db.GetPrefix()) node.file = ft.NewFSNode(fsNodeType) diff --git a/unixfs/ipld-merkledag/importer/helpers/helpers.go b/unixfs/ipld-merkledag/importer/helpers/helpers.go index 6fb0f83c8..c8aca63d1 100644 --- a/unixfs/ipld-merkledag/importer/helpers/helpers.go +++ b/unixfs/ipld-merkledag/importer/helpers/helpers.go @@ -61,8 +61,8 @@ func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) { } // SetPrefix sets the CID Prefix -func (n *UnixfsNode) SetPrefix(prefix *cid.Prefix) { - n.node.SetPrefix(prefix) +func (n *UnixfsNode) SetPrefix(prefix cid.Builder) { + n.node.SetCidBuilder(prefix) } // NumChildren returns the number of children referenced by this UnixfsNode. diff --git a/unixfs/ipld-merkledag/io/directory.go b/unixfs/ipld-merkledag/io/directory.go index 89864566e..64960531f 100644 --- a/unixfs/ipld-merkledag/io/directory.go +++ b/unixfs/ipld-merkledag/io/directory.go @@ -30,7 +30,7 @@ var DefaultShardWidth = 256 type Directory interface { // SetPrefix sets the CID prefix of the root node. - SetPrefix(*cid.Prefix) + SetPrefix(cid.Builder) // AddChild adds a (name, key) pair to the root node. AddChild(context.Context, string, ipld.Node) error @@ -52,7 +52,7 @@ type Directory interface { GetNode() (ipld.Node, error) // GetPrefix returns the CID Prefix used. - GetPrefix() *cid.Prefix + GetPrefix() cid.Builder } // TODO: Evaluate removing `dserv` from this layer and providing it in MFS. @@ -128,8 +128,8 @@ func NewDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (Directory, err } // SetPrefix implements the `Directory` interface. -func (d *BasicDirectory) SetPrefix(prefix *cid.Prefix) { - d.node.SetPrefix(prefix) +func (d *BasicDirectory) SetPrefix(prefix cid.Builder) { + d.node.SetCidBuilder(prefix) } // AddChild implements the `Directory` interface. It adds (or replaces) @@ -180,8 +180,8 @@ func (d *BasicDirectory) GetNode() (ipld.Node, error) { } // GetPrefix implements the `Directory` interface. -func (d *BasicDirectory) GetPrefix() *cid.Prefix { - return &d.node.Prefix +func (d *BasicDirectory) GetPrefix() cid.Builder { + return d.node.CidBuilder() } // SwitchToSharding returns a HAMT implementation of this directory. @@ -193,7 +193,7 @@ func (d *BasicDirectory) SwitchToSharding(ctx context.Context) (Directory, error if err != nil { return nil, err } - shard.SetPrefix(&d.node.Prefix) + shard.SetPrefix(d.node.CidBuilder()) hamtDir.shard = shard for _, lnk := range d.node.Links() { @@ -212,7 +212,7 @@ func (d *BasicDirectory) SwitchToSharding(ctx context.Context) (Directory, error } // SetPrefix implements the `Directory` interface. -func (d *HAMTDirectory) SetPrefix(prefix *cid.Prefix) { +func (d *HAMTDirectory) SetPrefix(prefix cid.Builder) { d.shard.SetPrefix(prefix) } @@ -252,6 +252,6 @@ func (d *HAMTDirectory) GetNode() (ipld.Node, error) { } // GetPrefix implements the `Directory` interface. -func (d *HAMTDirectory) GetPrefix() *cid.Prefix { +func (d *HAMTDirectory) GetPrefix() cid.Builder { return d.shard.Prefix() } diff --git a/unixfs/ipld-merkledag/mod/dagmodifier.go b/unixfs/ipld-merkledag/mod/dagmodifier.go index 09665b80c..509b3cc81 100644 --- a/unixfs/ipld-merkledag/mod/dagmodifier.go +++ b/unixfs/ipld-merkledag/mod/dagmodifier.go @@ -256,7 +256,7 @@ func (dm *DagModifier) modifyDag(n ipld.Node, offset uint64) (*cid.Cid, error) { nd := new(mdag.ProtoNode) nd.SetData(b) - nd.SetPrefix(&nd0.Prefix) + nd.SetCidBuilder(nd0.CidBuilder()) err = dm.dagserv.Add(dm.ctx, nd) if err != nil { return nil, err From 86f106ac10ad00cb1b9eeee804ecc4a7a938b48b Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Sat, 11 Aug 2018 00:13:57 -0400 Subject: [PATCH 3/3] Rename Prefix in names to CidBuilder when a builder is used. This commit was moved from ipfs/go-unixfs@b12f2d836ee0ad9bc8b4b80fa15e1a8fc561a0ca --- unixfs/ipld-merkledag/hamt/hamt.go | 20 +++---- .../importer/helpers/dagbuilder.go | 52 +++++++++---------- .../importer/helpers/helpers.go | 6 +-- unixfs/ipld-merkledag/io/directory.go | 32 ++++++------ unixfs/ipld-merkledag/mod/dagmodifier.go | 8 +-- unixfs/ipld-merkledag/test/utils.go | 8 +-- 6 files changed, 63 insertions(+), 63 deletions(-) diff --git a/unixfs/ipld-merkledag/hamt/hamt.go b/unixfs/ipld-merkledag/hamt/hamt.go index a28bf0725..4d3bd3b8e 100644 --- a/unixfs/ipld-merkledag/hamt/hamt.go +++ b/unixfs/ipld-merkledag/hamt/hamt.go @@ -52,7 +52,7 @@ type Shard struct { tableSize int tableSizeLg2 int - prefix cid.Builder + builder cid.Builder hashFunc uint64 prefixPadStr string @@ -124,25 +124,25 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) { ds.children = make([]child, len(pbnd.Links())) ds.bitfield.SetBytes(pbd.GetData()) ds.hashFunc = pbd.GetHashType() - ds.prefix = ds.nd.CidBuilder() + ds.builder = ds.nd.CidBuilder() return ds, nil } -// SetPrefix sets the CID Prefix -func (ds *Shard) SetPrefix(prefix cid.Builder) { - ds.prefix = prefix +// SetCidBuilder sets the CID Builder +func (ds *Shard) SetCidBuilder(builder cid.Builder) { + ds.builder = builder } -// Prefix gets the CID Prefix, may be nil if unset -func (ds *Shard) Prefix() cid.Builder { - return ds.prefix +// CidBuilder gets the CID Builder, may be nil if unset +func (ds *Shard) CidBuilder() cid.Builder { + return ds.builder } // Node serializes the HAMT structure into a merkledag node with unixfs formatting func (ds *Shard) Node() (ipld.Node, error) { out := new(dag.ProtoNode) - out.SetCidBuilder(ds.prefix) + out.SetCidBuilder(ds.builder) cindex := 0 // TODO: optimized 'for each set bit' @@ -494,7 +494,7 @@ func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val if err != nil { return err } - ns.prefix = ds.prefix + ns.builder = ds.builder chhv := &hashBits{ b: hash([]byte(child.key)), consumed: hv.consumed, diff --git a/unixfs/ipld-merkledag/importer/helpers/dagbuilder.go b/unixfs/ipld-merkledag/importer/helpers/dagbuilder.go index 1331ddbfa..4c897fd48 100644 --- a/unixfs/ipld-merkledag/importer/helpers/dagbuilder.go +++ b/unixfs/ipld-merkledag/importer/helpers/dagbuilder.go @@ -19,14 +19,14 @@ import ( // DagBuilderHelper wraps together a bunch of objects needed to // efficiently create unixfs dag trees type DagBuilderHelper struct { - dserv ipld.DAGService - spl chunker.Splitter - recvdErr error - rawLeaves bool - nextData []byte // the next item to return. - maxlinks int - batch *ipld.Batch - prefix cid.Builder + dserv ipld.DAGService + spl chunker.Splitter + recvdErr error + rawLeaves bool + nextData []byte // the next item to return. + maxlinks int + batch *ipld.Batch + cidBuilder cid.Builder // Filestore support variables. // ---------------------------- @@ -53,8 +53,8 @@ type DagBuilderParams struct { // instead of using the unixfs TRaw type RawLeaves bool - // CID Prefix to use if set - Prefix cid.Builder + // CID Builder to use if set + CidBuilder cid.Builder // DAGService to write blocks to (required) Dagserv ipld.DAGService @@ -73,12 +73,12 @@ type DagBuilderParams struct { // chunker.Splitter as data source. func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper { db := &DagBuilderHelper{ - dserv: dbp.Dagserv, - spl: spl, - rawLeaves: dbp.RawLeaves, - prefix: dbp.Prefix, - maxlinks: dbp.Maxlinks, - batch: ipld.NewBatch(context.TODO(), dbp.Dagserv), + dserv: dbp.Dagserv, + spl: spl, + rawLeaves: dbp.RawLeaves, + cidBuilder: dbp.CidBuilder, + maxlinks: dbp.Maxlinks, + batch: ipld.NewBatch(context.TODO(), dbp.Dagserv), } if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok { db.fullPath = fi.AbsPath() @@ -141,13 +141,13 @@ func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode { node: new(dag.ProtoNode), ufmt: ft.NewFSNode(ft.TFile), } - n.SetPrefix(db.prefix) + n.SetCidBuilder(db.cidBuilder) return n } -// GetPrefix returns the internal `cid.Prefix` set in the builder. -func (db *DagBuilderHelper) GetPrefix() cid.Builder { - return db.prefix +// GetCidBuilder returns the internal `cid.CidBuilder` set in the builder. +func (db *DagBuilderHelper) GetCidBuilder() cid.Builder { + return db.cidBuilder } // NewLeaf creates a leaf node filled with data. If rawLeaves is @@ -160,13 +160,13 @@ func (db *DagBuilderHelper) NewLeaf(data []byte) (*UnixfsNode, error) { } if db.rawLeaves { - if db.prefix == nil { + if db.cidBuilder == nil { return &UnixfsNode{ rawnode: dag.NewRawNode(data), raw: true, }, nil } - rawnode, err := dag.NewRawNodeWPrefix(data, db.prefix) + rawnode, err := dag.NewRawNodeWPrefix(data, db.cidBuilder) if err != nil { return nil, err } @@ -194,10 +194,10 @@ func (db *DagBuilderHelper) NewLeafNode(data []byte) (ipld.Node, error) { if db.rawLeaves { // Encapsulate the data in a raw node. - if db.prefix == nil { + if db.cidBuilder == nil { return dag.NewRawNode(data), nil } - rawnode, err := dag.NewRawNodeWPrefix(data, db.prefix) + rawnode, err := dag.NewRawNodeWPrefix(data, db.cidBuilder) if err != nil { return nil, err } @@ -229,7 +229,7 @@ func (db *DagBuilderHelper) newUnixfsBlock() *UnixfsNode { node: new(dag.ProtoNode), ufmt: ft.NewFSNode(ft.TRaw), } - n.SetPrefix(db.prefix) + n.SetCidBuilder(db.cidBuilder) return n } @@ -401,7 +401,7 @@ type FSNodeOverDag struct { func (db *DagBuilderHelper) NewFSNodeOverDag(fsNodeType pb.Data_DataType) *FSNodeOverDag { node := new(FSNodeOverDag) node.dag = new(dag.ProtoNode) - node.dag.SetCidBuilder(db.GetPrefix()) + node.dag.SetCidBuilder(db.GetCidBuilder()) node.file = ft.NewFSNode(fsNodeType) diff --git a/unixfs/ipld-merkledag/importer/helpers/helpers.go b/unixfs/ipld-merkledag/importer/helpers/helpers.go index c8aca63d1..a2e443ea3 100644 --- a/unixfs/ipld-merkledag/importer/helpers/helpers.go +++ b/unixfs/ipld-merkledag/importer/helpers/helpers.go @@ -60,9 +60,9 @@ func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) { }, nil } -// SetPrefix sets the CID Prefix -func (n *UnixfsNode) SetPrefix(prefix cid.Builder) { - n.node.SetCidBuilder(prefix) +// SetCidBuilder sets the CID Builder +func (n *UnixfsNode) SetCidBuilder(builder cid.Builder) { + n.node.SetCidBuilder(builder) } // NumChildren returns the number of children referenced by this UnixfsNode. diff --git a/unixfs/ipld-merkledag/io/directory.go b/unixfs/ipld-merkledag/io/directory.go index 64960531f..aa1ec8de7 100644 --- a/unixfs/ipld-merkledag/io/directory.go +++ b/unixfs/ipld-merkledag/io/directory.go @@ -29,8 +29,8 @@ var DefaultShardWidth = 256 // (which is the main consumer of this interface). type Directory interface { - // SetPrefix sets the CID prefix of the root node. - SetPrefix(cid.Builder) + // SetCidBuilder sets the CID Builder of the root node. + SetCidBuilder(cid.Builder) // AddChild adds a (name, key) pair to the root node. AddChild(context.Context, string, ipld.Node) error @@ -51,8 +51,8 @@ type Directory interface { // GetNode returns the root of this directory. GetNode() (ipld.Node, error) - // GetPrefix returns the CID Prefix used. - GetPrefix() cid.Builder + // GetCidBuilder returns the CID Builder used. + GetCidBuilder() cid.Builder } // TODO: Evaluate removing `dserv` from this layer and providing it in MFS. @@ -127,9 +127,9 @@ func NewDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (Directory, err return nil, ErrNotADir } -// SetPrefix implements the `Directory` interface. -func (d *BasicDirectory) SetPrefix(prefix cid.Builder) { - d.node.SetCidBuilder(prefix) +// SetCidBuilder implements the `Directory` interface. +func (d *BasicDirectory) SetCidBuilder(builder cid.Builder) { + d.node.SetCidBuilder(builder) } // AddChild implements the `Directory` interface. It adds (or replaces) @@ -179,8 +179,8 @@ func (d *BasicDirectory) GetNode() (ipld.Node, error) { return d.node, nil } -// GetPrefix implements the `Directory` interface. -func (d *BasicDirectory) GetPrefix() cid.Builder { +// GetCidBuilder implements the `Directory` interface. +func (d *BasicDirectory) GetCidBuilder() cid.Builder { return d.node.CidBuilder() } @@ -193,7 +193,7 @@ func (d *BasicDirectory) SwitchToSharding(ctx context.Context) (Directory, error if err != nil { return nil, err } - shard.SetPrefix(d.node.CidBuilder()) + shard.SetCidBuilder(d.node.CidBuilder()) hamtDir.shard = shard for _, lnk := range d.node.Links() { @@ -211,9 +211,9 @@ func (d *BasicDirectory) SwitchToSharding(ctx context.Context) (Directory, error return hamtDir, nil } -// SetPrefix implements the `Directory` interface. -func (d *HAMTDirectory) SetPrefix(prefix cid.Builder) { - d.shard.SetPrefix(prefix) +// SetCidBuilder implements the `Directory` interface. +func (d *HAMTDirectory) SetCidBuilder(builder cid.Builder) { + d.shard.SetCidBuilder(builder) } // AddChild implements the `Directory` interface. @@ -251,7 +251,7 @@ func (d *HAMTDirectory) GetNode() (ipld.Node, error) { return d.shard.Node() } -// GetPrefix implements the `Directory` interface. -func (d *HAMTDirectory) GetPrefix() cid.Builder { - return d.shard.Prefix() +// GetCidBuilder implements the `Directory` interface. +func (d *HAMTDirectory) GetCidBuilder() cid.Builder { + return d.shard.CidBuilder() } diff --git a/unixfs/ipld-merkledag/mod/dagmodifier.go b/unixfs/ipld-merkledag/mod/dagmodifier.go index 509b3cc81..f6e5f4820 100644 --- a/unixfs/ipld-merkledag/mod/dagmodifier.go +++ b/unixfs/ipld-merkledag/mod/dagmodifier.go @@ -345,10 +345,10 @@ func (dm *DagModifier) appendData(nd ipld.Node, spl chunker.Splitter) (ipld.Node switch nd := nd.(type) { case *mdag.ProtoNode, *mdag.RawNode: dbp := &help.DagBuilderParams{ - Dagserv: dm.dagserv, - Maxlinks: help.DefaultLinksPerBlock, - Prefix: &dm.Prefix, - RawLeaves: dm.RawLeaves, + Dagserv: dm.dagserv, + Maxlinks: help.DefaultLinksPerBlock, + CidBuilder: dm.Prefix, + RawLeaves: dm.RawLeaves, } return trickle.Append(dm.ctx, nd, dbp.New(spl)) default: diff --git a/unixfs/ipld-merkledag/test/utils.go b/unixfs/ipld-merkledag/test/utils.go index cc8fe4dd0..fdd307c56 100644 --- a/unixfs/ipld-merkledag/test/utils.go +++ b/unixfs/ipld-merkledag/test/utils.go @@ -61,10 +61,10 @@ func GetNode(t testing.TB, dserv ipld.DAGService, data []byte, opts NodeOpts) ip in := bytes.NewReader(data) dbp := h.DagBuilderParams{ - Dagserv: dserv, - Maxlinks: h.DefaultLinksPerBlock, - Prefix: &opts.Prefix, - RawLeaves: opts.RawLeavesUsed, + Dagserv: dserv, + Maxlinks: h.DefaultLinksPerBlock, + CidBuilder: opts.Prefix, + RawLeaves: opts.RawLeavesUsed, } node, err := trickle.Layout(dbp.New(SizeSplitterGen(500)(in)))