diff --git a/go.mod b/go.mod
index a236098a..e020a704 100644
--- a/go.mod
+++ b/go.mod
@@ -5,11 +5,13 @@ go 1.20
 require (
 	github.com/alecthomas/chroma/v2 v2.12.0
 	github.com/google/go-cmp v0.5.9
-	github.com/lucasb-eyer/go-colorful v1.2.0
 	github.com/sirupsen/logrus v1.8.1
 	golang.org/x/sys v0.1.0
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
 	gotest.tools/v3 v3.3.0
 )
 
-require github.com/dlclark/regexp2 v1.10.0 // indirect
+require (
+	github.com/dlclark/regexp2 v1.10.0 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
+)
diff --git a/go.sum b/go.sum
index e7947952..ee5bbe0c 100644
--- a/go.sum
+++ b/go.sum
@@ -2,6 +2,7 @@ github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2
 github.com/alecthomas/chroma/v2 v2.12.0 h1:Wh8qLEgMMsN7mgyG8/qIpegky2Hvzr4By6gEF7cmWgw=
 github.com/alecthomas/chroma/v2 v2.12.0/go.mod h1:4TQu7gdfuPjSh76j78ietmqh9LiurGF0EpseFXdKMBw=
 github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
@@ -10,15 +11,15 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
-github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
-github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -47,5 +48,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
 gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
diff --git a/twin/colors.go b/twin/colors.go
index 06122917..d46347fa 100644
--- a/twin/colors.go
+++ b/twin/colors.go
@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"math"
 
-	"github.com/lucasb-eyer/go-colorful"
+	"github.com/alecthomas/chroma/v2"
 )
 
 // Create using NewColor16(), NewColor256 or NewColor24Bit(), or use
@@ -163,6 +163,19 @@ func (color Color) String() string {
 	panic(fmt.Errorf("unhandled color type %d", color.colorType()))
 }
 
+func (color Color) to24Bit() Color {
+	if color.colorType() == ColorType24bit {
+		return color
+	}
+
+	if color.colorType() == ColorType8 || color.colorType() == ColorType16 || color.colorType() == ColorType256 {
+		r0, g0, b0 := color256ToRGB(uint8(color.colorValue()))
+		return NewColor24Bit(r0, g0, b0)
+	}
+
+	panic(fmt.Errorf("unhandled color type %d", color.colorType()))
+}
+
 func (color Color) downsampleTo(terminalColorCount ColorType) Color {
 	if color.colorType() == colorTypeDefault || terminalColorCount == colorTypeDefault {
 		panic(fmt.Errorf("downsampling to or from default color not supported, %s -> %#v", color.String(), terminalColorCount))
@@ -173,17 +186,7 @@ func (color Color) downsampleTo(terminalColorCount ColorType) Color {
 		return color
 	}
 
-	// Convert existing color to 24 bit
-	var targetR float64
-	var targetG float64
-	var targetB float64
-	if color.colorType() == ColorType24bit {
-		targetR = float64(color.colorValue()>>16) / 255.0
-		targetG = float64(color.colorValue()>>8&0xff) / 255.0
-		targetB = float64(color.colorValue()&0xff) / 255.0
-	} else {
-		targetR, targetG, targetB = color256ToRGB(uint8(color.colorValue()))
-	}
+	target := color.to24Bit()
 
 	// Find the closest match in the terminal color palette
 	scanRange := 255
@@ -201,20 +204,11 @@ func (color Color) downsampleTo(terminalColorCount ColorType) Color {
 	// Iterate over the scan range and find the best matching index
 	bestMatch := 0
 	bestDistance := math.MaxFloat64
-	target := colorful.Color{
-		R: targetR,
-		G: targetG,
-		B: targetB,
-	}
 	for i := 0; i <= scanRange; i++ {
 		r, g, b := color256ToRGB(uint8(i))
-		candidate := colorful.Color{
-			R: r,
-			G: g,
-			B: b,
-		}
+		candidate := NewColor24Bit(r, g, b)
 
-		distance := target.DistanceLab(candidate)
+		distance := target.Distance(candidate)
 		if distance < bestDistance {
 			bestDistance = distance
 			bestMatch = i
@@ -227,3 +221,31 @@ func (color Color) downsampleTo(terminalColorCount ColorType) Color {
 		return NewColor256(uint8(bestMatch))
 	}
 }
+
+// Wrapper for Chroma's color distance function.
+//
+// That one says it uses this formula: https://www.compuphase.com/cmetric.htm
+//
+// The result from this function has been scaled to 0.0-1.0, where 1.0 is the
+// distance between black and white.
+func (c Color) Distance(other Color) float64 {
+	if c.colorType() != ColorType24bit {
+		panic(fmt.Errorf("contrast only supported for 24 bit colors, got %s vs %s", c.String(), other.String()))
+	}
+
+	baseColor := chroma.NewColour(
+		uint8(c.colorValue()>>16&0xff),
+		uint8(c.colorValue()>>8&0xff),
+		uint8(c.colorValue()&0xff),
+	)
+
+	otherColor := chroma.NewColour(
+		uint8(other.colorValue()>>16&0xff),
+		uint8(other.colorValue()>>8&0xff),
+		uint8(other.colorValue()&0xff),
+	)
+
+	// Magic constant comes from testing
+	maxDistance := 764.8333151739665
+	return baseColor.Distance(otherColor) / maxDistance
+}
diff --git a/twin/colors_test.go b/twin/colors_test.go
index 806d5921..09f3296d 100644
--- a/twin/colors_test.go
+++ b/twin/colors_test.go
@@ -40,3 +40,17 @@ func TestAnsiStringDefault(t *testing.T) {
 		"\x1b[39m",
 	)
 }
+
+func TestDistance(t *testing.T) {
+	// Black -> white
+	assert.Equal(t,
+		NewColor24Bit(0, 0, 0).Distance(NewColor24Bit(255, 255, 255)),
+		1.0,
+	)
+
+	// White -> black
+	assert.Equal(t,
+		NewColor24Bit(255, 255, 255).Distance(NewColor24Bit(0, 0, 0)),
+		1.0,
+	)
+}
diff --git a/twin/palette256.go b/twin/palette256.go
index e2cdeb70..67864f15 100644
--- a/twin/palette256.go
+++ b/twin/palette256.go
@@ -1,17 +1,17 @@
 package twin
 
-func color256ToRGB(color256 uint8) (r, g, b float64) {
+func color256ToRGB(color256 uint8) (r, g, b uint8) {
 	if color256 < 16 {
 		// Standard ANSI colors
-		r := float64(standardAnsiColors[color256][0]) / 255.0
-		g := float64(standardAnsiColors[color256][1]) / 255.0
-		b := float64(standardAnsiColors[color256][2]) / 255.0
+		r := standardAnsiColors[color256][0]
+		g := standardAnsiColors[color256][1]
+		b := standardAnsiColors[color256][2]
 		return r, g, b
 	}
 
 	if color256 >= 232 {
 		// Grayscale. Colors 232-255 map to components 0x08 to 0xee
-		gray := float64((color256-232)*0x0a+0x08) / 255.0
+		gray := (color256-232)*0x0a + 0x08
 		return gray, gray, gray
 	}
 
@@ -19,9 +19,9 @@ func color256ToRGB(color256 uint8) (r, g, b float64) {
 	color0_to_215 := color256 - 16
 
 	components := []uint8{0x00, 0x5f, 0x87, 0xaf, 0xd7, 0xff}
-	r = float64(components[(color0_to_215/36)%6]) / 255.0
-	g = float64(components[(color0_to_215/6)%6]) / 255.0
-	b = float64(components[(color0_to_215/1)%6]) / 255.0
+	r = components[(color0_to_215/36)%6]
+	g = components[(color0_to_215/6)%6]
+	b = components[(color0_to_215/1)%6]
 	return r, g, b
 }
 
diff --git a/twin/palette256_test.go b/twin/palette256_test.go
index 2073c9a0..29ed7ea4 100644
--- a/twin/palette256_test.go
+++ b/twin/palette256_test.go
@@ -9,39 +9,39 @@ import (
 func TestColorRgbFirst16(t *testing.T) {
 	r, g, b := color256ToRGB(5)
 
-	assert.Equal(t, r, float64(0x80)/255.0)
-	assert.Equal(t, g, float64(0x00)/255.0)
-	assert.Equal(t, b, float64(0x80)/255.0)
+	assert.Equal(t, r, uint8(0x80))
+	assert.Equal(t, g, uint8(0x00))
+	assert.Equal(t, b, uint8(0x80))
 }
 
 func TestColorToRgbInTheGrey(t *testing.T) {
 	r, g, b := color256ToRGB(252)
 
-	assert.Equal(t, r, float64(0xd0)/255.0)
-	assert.Equal(t, g, float64(0xd0)/255.0)
-	assert.Equal(t, b, float64(0xd0)/255.0)
+	assert.Equal(t, r, uint8(0xd0))
+	assert.Equal(t, g, uint8(0xd0))
+	assert.Equal(t, b, uint8(0xd0))
 }
 
 func TestColorToRgbInThe6x6Cube(t *testing.T) {
 	r, g, b := color256ToRGB(101)
 
-	assert.Equal(t, r, float64(0x87)/255.0)
-	assert.Equal(t, g, float64(0x87)/255.0)
-	assert.Equal(t, b, float64(0x5f)/255.0)
+	assert.Equal(t, r, uint8(0x87))
+	assert.Equal(t, g, uint8(0x87))
+	assert.Equal(t, b, uint8(0x5f))
 }
 
 func TestColorToRgbStart6x6Cube(t *testing.T) {
 	r, g, b := color256ToRGB(16)
 
-	assert.Equal(t, r, float64(0x00)/255.0)
-	assert.Equal(t, g, float64(0x00)/255.0)
-	assert.Equal(t, b, float64(0x00)/255.0)
+	assert.Equal(t, r, uint8(0x00))
+	assert.Equal(t, g, uint8(0x00))
+	assert.Equal(t, b, uint8(0x00))
 }
 
 func TestColorRgbEnd6x6Cube(t *testing.T) {
 	r, g, b := color256ToRGB(231)
 
-	assert.Equal(t, r, float64(0xff)/255.0)
-	assert.Equal(t, g, float64(0xff)/255.0)
-	assert.Equal(t, b, float64(0xff)/255.0)
+	assert.Equal(t, r, uint8(0xff))
+	assert.Equal(t, g, uint8(0xff))
+	assert.Equal(t, b, uint8(0xff))
 }