Add function to write hash device (microsoft#1235)

Split hash-device computation and writing into a separate function.
This allows to store hash device in a separate file, which (e.g.) can
be converted to VHDs and exposed inside VMs as separate block devices.

Update `dmverity-vhd` command-line utility to support writing hash device
as a separate VHD

Signed-off-by: Maksim An <[email protected]>

cmd/dmverity-vhd/main.go

@@ -20,12 +20,13 @@ import (
 const usage = `dmverity-vhd is a command line tool for creating LCOW layer VHDs with dm-verity hashes.`
 
 const (
- usernameFlag = "username"
- passwordFlag = "password"
- imageFlag = "image"
- verboseFlag = "verbose"
- outputDirFlag = "out-dir"
- maxVHDSize = dmverity.RecommendedVHDSizeGB
+ usernameFlag = "username"
+ passwordFlag = "password"
+ imageFlag = "image"
+ verboseFlag = "verbose"
+ outputDirFlag = "out-dir"
+ hashDeviceVhdFlag = "hash-dev-vhd"
+ maxVHDSize = dmverity.RecommendedVHDSizeGB
 )
 
 func init() {
@@ -116,6 +117,10 @@ var createVHDCommand = cli.Command{
  Name: passwordFlag + ",p",
  Usage: "Optional: custom registry password",
  },
+ cli.BoolFlag{
+ Name: hashDeviceVhdFlag + ",hdv",
+ Usage: "Optional: save hash-device as a VHD",
+ },
  },
  Action: func(ctx *cli.Context) error {
  verbose := ctx.GlobalBool(verboseFlag)
@@ -159,13 +164,31 @@ var createVHDCommand = cli.Command{
  opts := []tar2ext4.Option{
  tar2ext4.ConvertWhiteout,
  tar2ext4.MaximumDiskSize(maxVHDSize),
- tar2ext4.AppendVhdFooter,
- tar2ext4.AppendDMVerity,
+ }
+ if !ctx.Bool(hashDeviceVhdFlag) {
+ opts = append(opts, tar2ext4.AppendDMVerity)
  }
  if err := tar2ext4.Convert(rc, out, opts...); err != nil {
  return errors.Wrap(err, "failed to convert tar to ext4")
  }
- fmt.Fprintf(os.Stdout, "Layer %d: %s\n", layerNumber, vhdPath)
+ if ctx.Bool(hashDeviceVhdFlag) {
+ hashDevPath := filepath.Join(ctx.String(outputDirFlag), diffID.Hex+".hash-dev.vhd")
+ hashDev, err := os.Create(hashDevPath)
+ if err != nil {
+ return errors.Wrap(err, "failed to create hash device VHD file")
+ }
+ if err := dmverity.ComputeAndWriteHashDevice(out, hashDev); err != nil {
+ return err
+ }
+ if err := tar2ext4.ConvertToVhd(hashDev); err != nil {
+ return err
+ }
+ fmt.Fprintf(os.Stdout, "Layer %d: hash device created at %s\n", layerNumber, hashDevPath)
+ }
+ if err := tar2ext4.ConvertToVhd(out); err != nil {
+ return errors.Wrap(err, "failed to append VHD footer")
+ }
+ fmt.Fprintf(os.Stdout, "Layer %d: layer VHD created at %s\n", layerNumber, vhdPath)
  }
  return nil
  },

ext4/dmverity/dmverity.go

@@ -9,6 +9,7 @@ import (
  "fmt"
  "io"
  "os"
+ "unsafe"
 
  "github.com/pkg/errors"
 
@@ -203,3 +204,38 @@ func ReadDMVerityInfo(vhdPath string, offsetInBytes int64) (*VerityInfo, error)
  Version: dmvSB.Version,
  }, nil
 }
+
+// ComputeAndWriteHashDevice builds merkle tree from a given io.ReadSeeker and writes the result
+// hash device (dm-verity super-block combined with merkle tree) to io.WriteSeeker.
+func ComputeAndWriteHashDevice(r io.ReadSeeker, w io.WriteSeeker) error {
+ if _, err := r.Seek(0, io.SeekStart); err != nil {
+ return err
+ }
+ tree, err := MerkleTree(r)
+ if err != nil {
+ return errors.Wrap(err, "failed to build merkle tree")
+ }
+
+ devSize, err := r.Seek(0, io.SeekEnd)
+ if err != nil {
+ return err
+ }
+ dmVeritySB := NewDMVeritySuperblock(uint64(devSize))
+ if _, err := w.Seek(0, io.SeekEnd); err != nil {
+ return err
+ }
+ if err := binary.Write(w, binary.LittleEndian, dmVeritySB); err != nil {
+ return errors.Wrap(err, "failed to write dm-verity super-block")
+ }
+ // write super-block padding
+ sbSize := int(unsafe.Sizeof(*dmVeritySB))
+ padding := bytes.Repeat([]byte{0}, blockSize-(sbSize%blockSize))
+ if _, err = w.Write(padding); err != nil {
+ return err
+ }
+ // write tree
+ if _, err := w.Write(tree); err != nil {
+ return errors.Wrap(err, "failed to write merkle tree")
+ }
+ return nil
+}

ext4/tar2ext4/tar2ext4.go

@@ -3,17 +3,14 @@ package tar2ext4
 import (
  "archive/tar"
  "bufio"
- "bytes"
  "encoding/binary"
  "fmt"
+ "github.com/pkg/errors"
  "io"
  "io/ioutil"
  "os"
  "path"
  "strings"
- "unsafe"
-
- "github.com/pkg/errors"
 
  "github.com/Microsoft/hcsshim/ext4/dmverity"
  "github.com/Microsoft/hcsshim/ext4/internal/compactext4"
@@ -194,50 +191,13 @@ func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
  }
 
  if p.appendDMVerity {
- // Rewind the stream for dm-verity processing
- if _, err := w.Seek(0, io.SeekStart); err != nil {
- return err
- }
-
- merkleTree, err := dmverity.MerkleTree(bufio.NewReaderSize(w, dmverity.MerkleTreeBufioSize))
- if err != nil {
- return errors.Wrap(err, "failed to build merkle tree")
- }
-
- // Write dm-verity super-block and then the merkle tree after the end of the
- // ext4 filesystem
- ext4size, err := w.Seek(0, io.SeekEnd)
- if err != nil {
- return err
- }
-
- superBlock := dmverity.NewDMVeritySuperblock(uint64(ext4size))
- if err = binary.Write(w, binary.LittleEndian, superBlock); err != nil {
- return err
- }
-
- // pad the super-block
- sbsize := int(unsafe.Sizeof(*superBlock))
- padding := bytes.Repeat([]byte{0}, ext4BlockSize-(sbsize%ext4BlockSize))
- if _, err = w.Write(padding); err != nil {
- return err
- }
-
- // write the tree
- if _, err = w.Write(merkleTree); err != nil {
+ if err := dmverity.ComputeAndWriteHashDevice(w, w); err != nil {
  return err
  }
  }
 
  if p.appendVhdFooter {
- size, err := w.Seek(0, io.SeekEnd)
- if err != nil {
- return err
- }
- err = binary.Write(w, binary.BigEndian, makeFixedVHDFooter(size))
- if err != nil {
- return err
- }
+ return ConvertToVhd(w)
  }
  return nil
 }
@@ -312,3 +272,12 @@ func ConvertAndComputeRootDigest(r io.Reader) (string, error) {
  hash := dmverity.RootHash(tree)
  return fmt.Sprintf("%x", hash), nil
 }
+
+// ConvertToVhd converts given io.WriteSeeker to VHD, by appending the VHD footer with a fixed size.
+func ConvertToVhd(w io.WriteSeeker) error {
+ size, err := w.Seek(0, io.SeekEnd)
+ if err != nil {
+ return err
+ }
+ return binary.Write(w, binary.BigEndian, makeFixedVHDFooter(size))
+}